diff --git a/backend/docs/prune_db command.md b/backend/docs/prune_db command.md index 760633544..2b3cd5090 100644 --- a/backend/docs/prune_db command.md +++ b/backend/docs/prune_db command.md @@ -15,9 +15,10 @@ Models use `DO_NOTHING` foreign keys, so the command applies manual cascade rule ### Optional Parameters -- `--tables`: Limit pruning to specific tables (comma-separated). Valid options: `checkouts`, `builds`, `tests`. Default: all three. +- `--tables`: Limit pruning to specific tables (comma-separated). Valid options: `checkouts`, `builds`, `tests`, `hardware_daily_builds`, `hardware_daily_tests`. Default: all. - Cascade only drags a child when the child's parent table is also selected. For example, `--tables tests` removes only tests past the cutoff; recent tests under an old build/checkout are kept because those parents are not being pruned. With `--tables builds,tests`, an old build still drags its recent tests, but an old checkout does not drag its recent builds (checkouts are not selected). - Tables not listed are not deleted. For example, `--tables builds` removes old builds but leaves their tests in place. Selecting a parent without its children (e.g. only `checkouts`) can therefore leave orphaned rows. + - `hardware_daily_builds` and `hardware_daily_tests` are independent of the raw cascade: they are pruned by their own `checkout_day` and can be targeted on their own (e.g. `--tables hardware_daily_builds,hardware_daily_tests`) to trim only the aggregates. - `--origins`: Limit age-based pruning to specific origins (comma-separated). If omitted, any origin is considered. - Cascade ignores origin: once a parent row is doomed, its children are removed even if they belong to a different origin. - `--batch-size`: Number of rows deleted per batch (default: `10000`). Must be at least `1`. @@ -51,6 +52,12 @@ python manage.py prune_db --older-than "30 days" --origins maestro,0dayci --dry- python manage.py prune_db --older-than "30 days" --tables tests --yes ``` +### Prune only the hardware daily aggregates + +```bash +python manage.py prune_db --older-than "60 days" --tables hardware_daily_builds,hardware_daily_tests --yes +``` + ### Prune rows linked to issues (override default protection) ```bash @@ -59,7 +66,11 @@ python manage.py prune_db --older-than "30 days" --skip-issue-protection --yes ## What Is Not Deleted -The command only touches `checkouts`, `builds`, and `tests`. Related tables are left as-is, including: +The command touches `checkouts`, `builds`, `tests`, and the hardware daily aggregates +`hardware_daily_builds` / `hardware_daily_tests`. The aggregates are pruned by their +own `checkout_day` (same age window as `--older-than`), because a daily summary is a +coarser fact table retained by date range, not by which raw checkouts survive; use +`--tables` to select any subset. Other related tables are left as-is, including: - `incidents` rows themselves (only used to decide which builds/tests/checkouts to keep) - `hardware_status`, `latest_checkout`, `tree_tests_rollup` (reference checkouts) diff --git a/backend/kernelCI/settings.py b/backend/kernelCI/settings.py index 73d157270..0d9e62f11 100644 --- a/backend/kernelCI/settings.py +++ b/backend/kernelCI/settings.py @@ -252,6 +252,22 @@ def get_json_env_var(name, default): f"--index={HARDWARE_REGISTRY_INDEX_URL}", ], ), + ( + "0 */6 * * *", + "django.core.management.call_command", + [ + "recompute_hardware_daily", + "--days=15", + ], + ), + ( + "50 * * * *", + "django.core.management.call_command", + [ + "recompute_hardware_daily", + "--days=1", + ], + ), ] # Email settings for SMTP backend diff --git a/backend/kernelCI_app/constants/localization.py b/backend/kernelCI_app/constants/localization.py index 190ce2dd2..875c0b7dc 100644 --- a/backend/kernelCI_app/constants/localization.py +++ b/backend/kernelCI_app/constants/localization.py @@ -102,7 +102,10 @@ class DocStrings: "Dictionary mapping tree names to selected commit hashes" ) - HARDWARE_LISTING_ORIGIN_DESCRIPTION = "Origin of the hardware" + HARDWARE_LISTING_ORIGIN_DESCRIPTION = ( + "Origin of the checkout the hardware was tested from. " + "Pass it empty to list every origin." + ) HARDWARE_LISTING_COMMITS_LIST_DESCRIPTION = ( "Optional comma-separated git commit identifiers: full SHA(s) " "and/or tag strings that appear in checkout.git_commit_tags." diff --git a/backend/kernelCI_app/management/commands/prune_db.py b/backend/kernelCI_app/management/commands/prune_db.py index 849b73067..e9bf50738 100644 --- a/backend/kernelCI_app/management/commands/prune_db.py +++ b/backend/kernelCI_app/management/commands/prune_db.py @@ -9,9 +9,8 @@ Rows linked to an incident (an issue) are kept by default, together with their ancestors so nothing is orphaned; pass --skip-issue-protection to prune them too. -Only checkouts, builds and tests are touched. Aggregate and derived tables (e.g. -tree_tests_rollup, hardware_status, latest_checkout) are left untouched and must -be cleaned up separately. +Also prunes hardware_daily_* by checkout_day (same age). Other derived tables +(tree_tests_rollup, hardware_status, latest_checkout) are left untouched. """ from django.core.management.base import BaseCommand, CommandError @@ -19,8 +18,9 @@ from kernelCI_app.management.commands.helpers.intervals import parse_interval -# Strict parent-before-child order: a checkout owns builds, a build owns tests. PRUNABLE_TABLES = ("checkouts", "builds", "tests") +HARDWARE_DAILY_TABLES = ("hardware_daily_builds", "hardware_daily_tests") +VALID_TABLES = PRUNABLE_TABLES + HARDWARE_DAILY_TABLES class Command(BaseCommand): @@ -61,11 +61,12 @@ def add_arguments(self, parser): parser.add_argument( "--tables", type=lambda s: [t.strip() for t in s.split(",")], - default=list(PRUNABLE_TABLES), + default=list(VALID_TABLES), help="Limit pruning to specific tables (comma-separated: " - f"{', '.join(PRUNABLE_TABLES)}). Only the listed tables are deleted; " - "unlisted child tables are left untouched, so selecting a parent without " - "its children (e.g. only 'checkouts') can leave orphans. Default: all.", + f"{', '.join(VALID_TABLES)}). Only listed tables are deleted; a parent " + "without its children (e.g. only 'checkouts') can leave orphans. " + "hardware_daily_* prune by checkout_day and can be targeted alone. " + "Default: all.", ) parser.add_argument( "--skip-issue-protection", @@ -87,13 +88,16 @@ def handle(self, *args, **options): "positive number." ) - unknown_tables = [t for t in options["tables"] if t not in PRUNABLE_TABLES] + unknown_tables = [t for t in options["tables"] if t not in VALID_TABLES] if unknown_tables: raise CommandError( f"Unknown table(s): {', '.join(unknown_tables)}. " - f"Valid options are: {', '.join(PRUNABLE_TABLES)}." + f"Valid options are: {', '.join(VALID_TABLES)}." ) selected_tables = [t for t in PRUNABLE_TABLES if t in options["tables"]] + selected_daily_tables = [ + t for t in HARDWARE_DAILY_TABLES if t in options["tables"] + ] protect_incidents = not options["skip_issue_protection"] dry_run = options["dry_run"] @@ -122,16 +126,28 @@ def handle(self, *args, **options): counts = { t: self._count(cursor, temp_tables[t]) for t in selected_tables } - total = sum(counts.values()) + daily_counts = { + t: self._count_hardware_daily(cursor, t, cutoff.date()) + for t in selected_daily_tables + } + total = sum(counts.values()) + sum(daily_counts.values()) lines = [f"Rows older than {cutoff.isoformat()}:"] lines += [f"* {t}:\t{counts[t]:>8}" for t in selected_tables] + lines += [ + f"* {t}:\t{daily_counts[t]:>8}" for t in selected_daily_tables + ] lines += ["----------------------", f"* total:\t{total:>8}"] - lines.append( - "Note: counts include children cascaded from pruned parents." - ) - if protect_incidents: + if selected_tables: + lines.append( + "Note: counts include children cascaded from pruned parents." + ) + if protect_incidents and selected_tables: lines.append("Note: rows linked to an incident are kept.") + if daily_counts: + lines.append( + "Note: hardware daily rows are pruned by their own checkout_day." + ) self.stdout.write("\n".join(lines)) if total == 0: @@ -156,11 +172,13 @@ def handle(self, *args, **options): self.stdout.write("Aborted.") return - # Delete child-first (reverse of PRUNABLE_TABLES order): each batch - # commits on its own, so a crash mid-run leaves children already gone - # before their parents, never the reverse. Reordering this would risk - # orphans. deleted = 0 + for table in selected_daily_tables: + deleted += self._batch_delete_hardware_daily( + cursor, table, cutoff.date(), options["batch_size"] + ) + + # Child-first: a crash must not leave orphans. for table in reversed(selected_tables): deleted += self._batch_delete( cursor, table, temp_tables[table], options["batch_size"] @@ -254,3 +272,26 @@ def _batch_delete(self, cursor, table, temp_table, batch_size): deleted_total += deleted self.stdout.write(f"Deleted {table}(n={deleted}) total={deleted_total}") return deleted_total + + def _count_hardware_daily(self, cursor, table, cutoff_day): + cursor.execute( + f'SELECT COUNT(*) FROM "{table}" WHERE checkout_day < %(cutoff_day)s', + {"cutoff_day": cutoff_day}, + ) + return cursor.fetchone()[0] + + def _batch_delete_hardware_daily(self, cursor, table, cutoff_day, batch_size): + sql = ( + f'DELETE FROM "{table}" WHERE ctid IN (' + f'SELECT ctid FROM "{table}" WHERE checkout_day < %(cutoff_day)s ' + f"LIMIT %(batch_size)s)" + ) + deleted_total = 0 + while True: + cursor.execute(sql, {"cutoff_day": cutoff_day, "batch_size": batch_size}) + deleted = cursor.rowcount + if deleted == 0: + break + deleted_total += deleted + self.stdout.write(f"Deleted {table}(n={deleted}) total={deleted_total}") + return deleted_total diff --git a/backend/kernelCI_app/management/commands/recompute_hardware_daily.py b/backend/kernelCI_app/management/commands/recompute_hardware_daily.py new file mode 100644 index 000000000..3930cd94e --- /dev/null +++ b/backend/kernelCI_app/management/commands/recompute_hardware_daily.py @@ -0,0 +1,208 @@ +"""Rebuild hardware_daily_* for a UTC checkout day from raw checkouts/builds/tests. + +Idempotent: DELETE + INSERT. If raw is gone but aggregates exist, roll back. +""" + +import hashlib +from datetime import date, timedelta + +from django.core.management.base import BaseCommand +from django.db import connection, transaction +from django.utils import timezone + +from kernelCI_app.constants.general import MAESTRO_DUMMY_BUILD_PREFIX +from kernelCI_app.helpers.logger import out + + +def lock_key(table: str) -> int: + return int.from_bytes(hashlib.sha256(table.encode()).digest()[:4]) % 2**31 + + +# PASS/FAIL are verdicts; NULL, MISS, SKIP, ERROR, DONE all land in inc. +_BUCKET = ( + "CASE WHEN {col} = 'PASS' THEN 'pass'" + " WHEN {col} = 'FAIL' THEN 'failed' ELSE 'inc' END" +) + +# Half-open UTC day so the start_time index is usable. +DAY_TESTS = f""" + SELECT + c.id AS checkout_id, + c.origin AS checkout_origin, + t.origin, + COALESCE(NULLIF(t.misc ->> 'runtime', ''), t.origin) AS lab, + t.environment_misc ->> 'platform' AS platform, + t.environment_compatible AS compatibles, + COALESCE(t.path = 'boot' OR t.path LIKE 'boot.%%', false) AS is_boot, + {_BUCKET.format(col="t.status")} AS result, + b.id AS build_id, + b.origin AS build_origin, + COALESCE( + NULLIF(b.misc ->> 'lab', ''), NULLIF(b.misc ->> 'runtime', ''), b.origin + ) AS build_lab, + {_BUCKET.format(col="b.status")} AS build_result + FROM tests t + JOIN builds b ON b.id = t.build_id + JOIN checkouts c ON c.id = b.checkout_id + WHERE c.start_time >= %(day)s::timestamp AT TIME ZONE 'UTC' + AND c.start_time < (%(day)s::date + 1)::timestamp AT TIME ZONE 'UTC' + AND t.environment_misc ->> 'platform' IS NOT NULL +""" + +# One label per platform: longest compatible chain, so labs cannot list the board twice. +PLATFORM_COMPATIBLES = """ + SELECT DISTINCT ON (checkout_id, platform) checkout_id, platform, compatibles + FROM day_tests + WHERE compatibles IS NOT NULL + ORDER BY checkout_id, platform, cardinality(compatibles) DESC, compatibles +""" + +INSERT_TESTS = """ +INSERT INTO hardware_daily_tests ( + checkout_day, checkout_id, checkout_origin, test_origin, test_lab, platform, + compatibles, boot_pass, boot_failed, boot_inc, test_pass, test_failed, test_inc +) +WITH counted AS ( + SELECT + checkout_id, + checkout_origin, + origin, + lab, + platform, + count(*) FILTER (WHERE is_boot AND result = 'pass') AS boot_pass, + count(*) FILTER (WHERE is_boot AND result = 'failed') AS boot_failed, + count(*) FILTER (WHERE is_boot AND result = 'inc') AS boot_inc, + count(*) FILTER (WHERE NOT is_boot AND result = 'pass') AS test_pass, + count(*) FILTER (WHERE NOT is_boot AND result = 'failed') AS test_failed, + count(*) FILTER (WHERE NOT is_boot AND result = 'inc') AS test_inc + FROM day_tests + GROUP BY checkout_id, checkout_origin, origin, lab, platform +) +SELECT + %(day)s, + c.checkout_id, + c.checkout_origin, + c.origin, + c.lab, + c.platform, + pc.compatibles, + c.boot_pass, c.boot_failed, c.boot_inc, c.test_pass, c.test_failed, c.test_inc +FROM counted c +LEFT JOIN platform_compatibles pc USING (checkout_id, platform) +""" + +# Builds have no platform; attach them to every platform they tested, but DISTINCT so +# a build is not counted once per test (or per test lab). +INSERT_BUILDS = f""" +INSERT INTO hardware_daily_builds ( + checkout_day, checkout_id, checkout_origin, build_origin, build_lab, platform, + compatibles, build_pass, build_failed, build_inc +) +WITH day_builds AS ( + SELECT DISTINCT checkout_id, checkout_origin, build_id, build_origin, build_lab, + platform, build_result + FROM day_tests + WHERE build_id NOT LIKE '{MAESTRO_DUMMY_BUILD_PREFIX}%%' +), +counted AS ( + SELECT + checkout_id, + checkout_origin, + build_origin, + build_lab, + platform, + count(*) FILTER (WHERE build_result = 'pass') AS build_pass, + count(*) FILTER (WHERE build_result = 'failed') AS build_failed, + count(*) FILTER (WHERE build_result = 'inc') AS build_inc + FROM day_builds + GROUP BY checkout_id, checkout_origin, build_origin, build_lab, platform +) +SELECT + %(day)s, + c.checkout_id, + c.checkout_origin, + c.build_origin, + c.build_lab, + c.platform, + pc.compatibles, + c.build_pass, c.build_failed, c.build_inc +FROM counted c +LEFT JOIN platform_compatibles pc USING (checkout_id, platform) +""" + + +class Command(BaseCommand): + help = "Recompute the hardware daily aggregates from checkouts, builds and tests" + + def add_arguments(self, parser): + parser.add_argument( + "--days", + type=int, + default=1, + help="Days to recompute back from today, kept below the prune_db retention", + ) + parser.add_argument( + "--day", + type=date.fromisoformat, + help="Recompute a single day (YYYY-MM-DD) instead of the recent window", + ) + + def handle(self, *args, **options): + today = timezone.now().date() + days = ( + [options["day"]] + if options["day"] + else [ + today - timedelta(days=offset) + for offset in reversed(range(options["days"])) + ] + ) + + for day in days: + self._recompute_day(day) + + def _recompute_day(self, day: date) -> None: + deleted = inserted = 0 + tables = ("hardware_daily_builds", "hardware_daily_tests") + inserts = (INSERT_BUILDS, INSERT_TESTS) + + with transaction.atomic(), connection.cursor() as cursor: + # Cron-only txn: keep the day's sort/hash in memory. + cursor.execute("SET LOCAL work_mem = '128MB'") + + for table in tables: + cursor.execute( + "SELECT pg_try_advisory_xact_lock(%s, %s)", + [lock_key(table), day.toordinal()], + ) + if not cursor.fetchone()[0]: + transaction.set_rollback(True) + out(f"{day}: {table} already being recomputed, skipped") + return + + # Nested pytest transactions never COMMIT, so ON COMMIT DROP never fires. + cursor.execute("DROP TABLE IF EXISTS day_tests") + cursor.execute("DROP TABLE IF EXISTS platform_compatibles") + cursor.execute( + f"CREATE TEMP TABLE day_tests ON COMMIT DROP AS\n{DAY_TESTS}", + {"day": day}, + ) + # Temp tables have no stats; ANALYZE so the planner hashes instead of sorting. + cursor.execute("ANALYZE day_tests") + cursor.execute( + f"CREATE TEMP TABLE platform_compatibles ON COMMIT DROP AS\n" + f"{PLATFORM_COMPATIBLES}" + ) + + for table, insert in zip(tables, inserts, strict=True): + cursor.execute(f"DELETE FROM {table} WHERE checkout_day = %s", [day]) + deleted += cursor.rowcount + cursor.execute(insert, {"day": day}) + inserted += cursor.rowcount + + if inserted == 0 and deleted > 0: + transaction.set_rollback(True) + out(f"{day}: raw data pruned, kept the {deleted} existing rows") + return + + out(f"{day}: {inserted} rows written, {deleted} replaced") diff --git a/backend/kernelCI_app/management/commands/seed_test_data.py b/backend/kernelCI_app/management/commands/seed_test_data.py index 8ab91b7a9..b98a62931 100644 --- a/backend/kernelCI_app/management/commands/seed_test_data.py +++ b/backend/kernelCI_app/management/commands/seed_test_data.py @@ -4,6 +4,7 @@ import sys +from django.core.management import call_command from django.core.management.base import BaseCommand from django.db import transaction @@ -17,6 +18,8 @@ from kernelCI_app.models import ( Builds, Checkouts, + HardwareDailyBuilds, + HardwareDailyTests, HardwareStatus, Incidents, Issues, @@ -88,6 +91,9 @@ def handle(self, *args, **options): rollup_rows = self.create_tests_rollup(tests=tests, incidents=incidents) latest_checkouts = self.create_latest_checkouts(checkouts=checkouts) hardware_rows = self.create_hardware_status(tests=tests) + daily_build_count, daily_test_count = self.create_hardware_daily( + checkouts=checkouts + ) self.stdout.write( self.style.SUCCESS( @@ -100,6 +106,8 @@ def handle(self, *args, **options): f"- {len(rollup_rows)} tree_tests_rollup rows\n" f"- {len(latest_checkouts)} latest_checkout rows\n" f"- {len(hardware_rows)} hardware_status rows\n" + f"- {daily_build_count} hardware_daily_builds rows\n" + f"- {daily_test_count} hardware_daily_tests rows\n" ) ) @@ -128,6 +136,8 @@ def _validate_clear_operation(self, *, skip_confirmation: bool) -> None: def clear_data(self) -> None: """Clear existing test data.""" + HardwareDailyBuilds.objects.all().delete() + HardwareDailyTests.objects.all().delete() HardwareStatus.objects.all().delete() LatestCheckout.objects.all().delete() TreeTestsRollup.objects.all().delete() @@ -441,3 +451,22 @@ def create_hardware_status(self, *, tests: list[Tests]) -> list[HardwareStatus]: hardware_rows = [HardwareStatus(**data) for data in hardware_data.values()] return HardwareStatus.objects.bulk_create(hardware_rows) + + def create_hardware_daily(self, *, checkouts: list[Checkouts]) -> tuple[int, int]: + """Rebuild the daily hardware aggregates from the seeded raw rows.""" + self.stdout.write("Creating hardware daily aggregations...") + + days = sorted( + { + checkout.start_time.date() + for checkout in checkouts + if checkout.start_time + } + ) + for day in days: + call_command("recompute_hardware_daily", day=day, verbosity=0) + + return ( + HardwareDailyBuilds.objects.count(), + HardwareDailyTests.objects.count(), + ) diff --git a/backend/kernelCI_app/management/commands/update_db.py b/backend/kernelCI_app/management/commands/update_db.py index 75b364a25..2479f9271 100644 --- a/backend/kernelCI_app/management/commands/update_db.py +++ b/backend/kernelCI_app/management/commands/update_db.py @@ -12,12 +12,14 @@ from django.core.management.base import BaseCommand, CommandError from django.db import connections, models -from django.utils.dateparse import parse_datetime +from django.utils.dateparse import parse_date, parse_datetime from kernelCI_app.management.commands.helpers.intervals import parse_interval from kernelCI_app.models import ( Builds, Checkouts, + HardwareDailyBuilds, + HardwareDailyTests, HardwareStatus, Incidents, Issues, @@ -143,7 +145,8 @@ def _invalid_table_error(self, table: str) -> str: return ( f"Unknown table '{table}'.\n" "\tValid options are: issues, checkouts, builds, tests, incidents, " - "latest_checkout, hardware_status, tree_listing, tree_tests_rollup." + "latest_checkout, hardware_status, hardware_daily_builds, " + "hardware_daily_tests, tree_listing, tree_tests_rollup." ) def handle(self, *args, command, **options): @@ -174,6 +177,8 @@ def handle_snapshot( end_interval_unsafe_tables = ( None, "latest_checkout", + "hardware_daily_builds", + "hardware_daily_tests", "tree_listing", "tree_tests_rollup", ) @@ -228,6 +233,8 @@ def snapshot(self, table, snapshot_filepath: Path): self.snapshot_incidents() self.snapshot_latest_checkout() self.snapshot_hardware_status() + self.snapshot_hardware_daily_builds() + self.snapshot_hardware_daily_tests() self.snapshot_tree_listing() self.snapshot_tree_tests_rollup() case "issues": @@ -244,6 +251,10 @@ def snapshot(self, table, snapshot_filepath: Path): self.snapshot_latest_checkout() case "hardware_status": self.snapshot_hardware_status() + case "hardware_daily_builds": + self.snapshot_hardware_daily_builds() + case "hardware_daily_tests": + self.snapshot_hardware_daily_tests() case "tree_listing": self.snapshot_tree_listing() case "tree_tests_rollup": @@ -269,6 +280,8 @@ def restore(self, snapshot_filepath: Path): self.restore_incidents() self.restore_latest_checkout() self.restore_hardware_status() + self.restore_hardware_daily_builds() + self.restore_hardware_daily_tests() self.restore_tree_listing() self.restore_tree_tests_rollup() self.stdout.write( @@ -965,6 +978,138 @@ def restore_hardware_status(self) -> None: self.insert_hardware_status_data(records) self.stdout.write("HardwareStatus migration completed") + # HARDWARE DAILY BUILDS ######################################## + def select_hardware_daily_builds_data(self) -> list[tuple]: + origin_condition = ( + f"AND (checkout_origin IN ({','.join(['%s'] * len(self.origins))})" + f" OR build_origin IN ({','.join(['%s'] * len(self.origins))}))" + if self.origins + else "" + ) + query = f""" + SELECT checkout_day, checkout_id, checkout_origin, build_origin, build_lab, + platform, compatibles, build_pass, build_failed, build_inc + FROM hardware_daily_builds + WHERE checkout_day >= (NOW() - INTERVAL %s)::date + AND checkout_day <= (NOW() - INTERVAL %s)::date + {origin_condition} + ORDER BY checkout_day, build_origin, build_lab, platform, checkout_id + """ + query_params = [self.start_interval, self.end_interval] + self.origins * 2 + + with connections["default"].cursor() as kcidb_cursor: + kcidb_cursor.execute(query, query_params) + return kcidb_cursor.fetchall() + + def insert_hardware_daily_builds_data(self, records: list[tuple]) -> int: + rows = [ + HardwareDailyBuilds( + checkout_day=parse_date(record[0]) if record[0] else None, + checkout_id=record[1], + checkout_origin=record[2], + build_origin=record[3], + build_lab=record[4], + platform=record[5], + compatibles=parse_array(record[6]), + build_pass=record[7] or 0, + build_failed=record[8] or 0, + build_inc=record[9] or 0, + ) + for record in records + ] + total_inserted = len( + HardwareDailyBuilds.objects.bulk_create( + rows, ignore_conflicts=True, batch_size=DEFAULT_BATCH_SIZE + ) + ) + self.stdout.write(f"Processed {total_inserted} HardwareDailyBuilds records") + return total_inserted + + def snapshot_hardware_daily_builds(self) -> None: + with SpooledTemporaryFile(mode="w+b", max_size=MAX_MEMORY_BUFFER_BYTES) as file: + self.stdout.write("\nMigrating hardware_daily_builds...") + records = self.select_hardware_daily_builds_data() + self.insert_records(file, "hardware_daily_builds", records) + self.add_file_to_snapshot(file, "hardware_daily_builds") + self.stdout.write("hardware_daily_builds migration completed") + + def restore_hardware_daily_builds(self) -> None: + with TextIOWrapper( + self.snapshot_archive.extractfile("hardware_daily_builds.csv") + ) as file: + self.stdout.write("\nMigrating hardware_daily_builds...") + records = self.read_records(csv.reader(file)) + self.insert_hardware_daily_builds_data(records) + self.stdout.write("hardware_daily_builds migration completed") + + # HARDWARE DAILY TESTS ######################################## + def select_hardware_daily_tests_data(self) -> list[tuple]: + origin_condition = ( + f"AND (checkout_origin IN ({','.join(['%s'] * len(self.origins))})" + f" OR test_origin IN ({','.join(['%s'] * len(self.origins))}))" + if self.origins + else "" + ) + query = f""" + SELECT checkout_day, checkout_id, checkout_origin, test_origin, test_lab, + platform, compatibles, boot_pass, boot_failed, boot_inc, + test_pass, test_failed, test_inc + FROM hardware_daily_tests + WHERE checkout_day >= (NOW() - INTERVAL %s)::date + AND checkout_day <= (NOW() - INTERVAL %s)::date + {origin_condition} + ORDER BY checkout_day, test_origin, test_lab, platform, checkout_id + """ + query_params = [self.start_interval, self.end_interval] + self.origins * 2 + + with connections["default"].cursor() as kcidb_cursor: + kcidb_cursor.execute(query, query_params) + return kcidb_cursor.fetchall() + + def insert_hardware_daily_tests_data(self, records: list[tuple]) -> int: + rows = [ + HardwareDailyTests( + checkout_day=parse_date(record[0]) if record[0] else None, + checkout_id=record[1], + checkout_origin=record[2], + test_origin=record[3], + test_lab=record[4], + platform=record[5], + compatibles=parse_array(record[6]), + boot_pass=record[7] or 0, + boot_failed=record[8] or 0, + boot_inc=record[9] or 0, + test_pass=record[10] or 0, + test_failed=record[11] or 0, + test_inc=record[12] or 0, + ) + for record in records + ] + total_inserted = len( + HardwareDailyTests.objects.bulk_create( + rows, ignore_conflicts=True, batch_size=DEFAULT_BATCH_SIZE + ) + ) + self.stdout.write(f"Processed {total_inserted} HardwareDailyTests records") + return total_inserted + + def snapshot_hardware_daily_tests(self) -> None: + with SpooledTemporaryFile(mode="w+b", max_size=MAX_MEMORY_BUFFER_BYTES) as file: + self.stdout.write("\nMigrating hardware_daily_tests...") + records = self.select_hardware_daily_tests_data() + self.insert_records(file, "hardware_daily_tests", records) + self.add_file_to_snapshot(file, "hardware_daily_tests") + self.stdout.write("hardware_daily_tests migration completed") + + def restore_hardware_daily_tests(self) -> None: + with TextIOWrapper( + self.snapshot_archive.extractfile("hardware_daily_tests.csv") + ) as file: + self.stdout.write("\nMigrating hardware_daily_tests...") + records = self.read_records(csv.reader(file)) + self.insert_hardware_daily_tests_data(records) + self.stdout.write("hardware_daily_tests migration completed") + # TREE LISTING ######################################## def select_tree_listing_data(self) -> list[tuple]: query = f""" diff --git a/backend/kernelCI_app/migrations/0019_hardwaredailybuilds_hardwaredailytests.py b/backend/kernelCI_app/migrations/0019_hardwaredailybuilds_hardwaredailytests.py new file mode 100644 index 000000000..d3afcd7d9 --- /dev/null +++ b/backend/kernelCI_app/migrations/0019_hardwaredailybuilds_hardwaredailytests.py @@ -0,0 +1,116 @@ +# Generated by Django 5.2.11 on 2026-07-31 18:06 + +import django.contrib.postgres.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("kernelCI_app", "0018_hardwareregistryplatformvendor_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="HardwareDailyBuilds", + fields=[ + ( + "pk", + models.CompositePrimaryKey( + "checkout_day", + "build_origin", + "build_lab", + "platform", + "checkout_id", + blank=True, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("checkout_day", models.DateField()), + ("checkout_id", models.TextField()), + ( + "checkout_origin", + models.CharField(blank=True, max_length=100, null=True), + ), + ("build_origin", models.CharField(max_length=100)), + ("build_lab", models.TextField()), + ("platform", models.CharField(max_length=100)), + ( + "compatibles", + django.contrib.postgres.fields.ArrayField( + base_field=models.TextField(), null=True, size=None + ), + ), + ("build_pass", models.IntegerField(default=0)), + ("build_failed", models.IntegerField(default=0)), + ("build_inc", models.IntegerField(default=0)), + ], + options={ + "db_table": "hardware_daily_builds", + "indexes": [ + models.Index( + fields=["build_origin", "checkout_day"], + name="hw_daily_builds_org_day", + ), + models.Index( + fields=["checkout_origin", "checkout_day"], + name="hw_daily_builds_co_day", + ), + ], + }, + ), + migrations.CreateModel( + name="HardwareDailyTests", + fields=[ + ( + "pk", + models.CompositePrimaryKey( + "checkout_day", + "test_origin", + "test_lab", + "platform", + "checkout_id", + blank=True, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("checkout_day", models.DateField()), + ("checkout_id", models.TextField()), + ( + "checkout_origin", + models.CharField(blank=True, max_length=100, null=True), + ), + ("test_origin", models.CharField(max_length=100)), + ("test_lab", models.TextField()), + ("platform", models.CharField(max_length=100)), + ( + "compatibles", + django.contrib.postgres.fields.ArrayField( + base_field=models.TextField(), null=True, size=None + ), + ), + ("boot_pass", models.IntegerField(default=0)), + ("boot_failed", models.IntegerField(default=0)), + ("boot_inc", models.IntegerField(default=0)), + ("test_pass", models.IntegerField(default=0)), + ("test_failed", models.IntegerField(default=0)), + ("test_inc", models.IntegerField(default=0)), + ], + options={ + "db_table": "hardware_daily_tests", + "indexes": [ + models.Index( + fields=["test_origin", "checkout_day"], + name="hw_daily_tests_org_day", + ), + models.Index( + fields=["checkout_origin", "checkout_day"], + name="hw_daily_tests_co_day", + ), + ], + }, + ), + ] diff --git a/backend/kernelCI_app/migrations/0021_merge_20260901_1632.py b/backend/kernelCI_app/migrations/0021_merge_20260901_1632.py new file mode 100644 index 000000000..1a4304e1b --- /dev/null +++ b/backend/kernelCI_app/migrations/0021_merge_20260901_1632.py @@ -0,0 +1,12 @@ +# Generated by Django 5.2.16 on 2026-09-01 16:32 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("kernelCI_app", "0019_hardwaredailybuilds_hardwaredailytests"), + ("kernelCI_app", "0020_add_labs_fk_indexes"), + ] + + operations = [] diff --git a/backend/kernelCI_app/models.py b/backend/kernelCI_app/models.py index f3e259dc9..30c37f393 100644 --- a/backend/kernelCI_app/models.py +++ b/backend/kernelCI_app/models.py @@ -313,6 +313,78 @@ class Meta: ] +class HardwareDailyBuilds(models.Model): + pk = models.CompositePrimaryKey( + "checkout_day", + "build_origin", + "build_lab", + "platform", + "checkout_id", + ) + checkout_day = models.DateField() + checkout_id = models.TextField() + checkout_origin = models.CharField(max_length=100, blank=True, null=True) + build_origin = models.CharField(max_length=100) + build_lab = models.TextField() + platform = models.CharField(max_length=100) + + compatibles = ArrayField(models.TextField(), null=True) + + build_pass = models.IntegerField(default=0) + build_failed = models.IntegerField(default=0) + build_inc = models.IntegerField(default=0) + + class Meta: + db_table = "hardware_daily_builds" + indexes = [ + models.Index( + fields=["build_origin", "checkout_day"], name="hw_daily_builds_org_day" + ), + models.Index( + fields=["checkout_origin", "checkout_day"], + name="hw_daily_builds_co_day", + ), + ] + + +class HardwareDailyTests(models.Model): + pk = models.CompositePrimaryKey( + "checkout_day", + "test_origin", + "test_lab", + "platform", + "checkout_id", + ) + checkout_day = models.DateField() + checkout_id = models.TextField() + checkout_origin = models.CharField(max_length=100, blank=True, null=True) + test_origin = models.CharField(max_length=100) + test_lab = models.TextField() + platform = models.CharField(max_length=100) + + compatibles = ArrayField(models.TextField(), null=True) + + boot_pass = models.IntegerField(default=0) + boot_failed = models.IntegerField(default=0) + boot_inc = models.IntegerField(default=0) + + test_pass = models.IntegerField(default=0) + test_failed = models.IntegerField(default=0) + test_inc = models.IntegerField(default=0) + + class Meta: + db_table = "hardware_daily_tests" + indexes = [ + models.Index( + fields=["test_origin", "checkout_day"], name="hw_daily_tests_org_day" + ), + models.Index( + fields=["checkout_origin", "checkout_day"], + name="hw_daily_tests_co_day", + ), + ] + + class LatestCheckout(models.Model): id = models.AutoField(primary_key=True) checkout_id = models.TextField() diff --git a/backend/kernelCI_app/queries/hardware.py b/backend/kernelCI_app/queries/hardware.py index 422b487e6..6505c5125 100644 --- a/backend/kernelCI_app/queries/hardware.py +++ b/backend/kernelCI_app/queries/hardware.py @@ -103,18 +103,22 @@ def _get_hardware_listing_count_clauses() -> str: return build_count_clause + boot_count_clause + test_count_clause -def get_hardware_selectors(origin: str) -> list[dict]: +def get_hardware_selectors(build_origin: Optional[list[str]]) -> list[dict]: cache_key = "hardwareSelectors" - cache_params = {"origin": origin} + cache_params = {"build_origin": build_origin} rows = get_query_cache(cache_key, cache_params) if rows is not None: return rows - params = {"origin": origin} cross_origin_test_origins = {"aspeed", "ti"} - if origin in cross_origin_test_origins: + if ( + build_origin + and len(build_origin) == 1 + and build_origin[0] in cross_origin_test_origins + ): + params = {"origin": build_origin[0]} from_where = """ FROM tests t INNER JOIN builds b ON b.id = t.build_id @@ -125,12 +129,16 @@ def get_hardware_selectors(origin: str) -> list[dict]: AND t.environment_misc ->> 'platform' IS NOT NULL """ else: - from_where = """ + params = {"build_origin": build_origin} + build_origin_clause = ( + "AND b.origin = ANY(%(build_origin)s)" if build_origin else "" + ) + from_where = f""" FROM checkouts c INNER JOIN builds b ON b.checkout_id = c.id WHERE - b.origin = %(origin)s - AND b.start_time > (NOW() - INTERVAL '30 days') + b.start_time > (NOW() - INTERVAL '30 days') + {build_origin_clause} """ query = f""" @@ -171,59 +179,40 @@ def get_hardware_selectors(origin: str) -> list[dict]: def get_hardware_listing_data_by_revision( *, - origin: str, + checkout_origin: Optional[list[str]], + build_origin: Optional[list[str]], + test_origin: Optional[list[str]], + build_lab: Optional[list[str]], + test_lab: Optional[list[str]], tree_name: str, git_repository_url: str, git_repository_branch: str, git_commit_hash: str, -) -> list[dict]: - count_clauses = _get_hardware_listing_count_clauses() - params = { - "origin": origin, - "tree_name": tree_name, - "git_repository_url": git_repository_url, - "git_repository_branch": git_repository_branch, - "git_commit_hash": git_commit_hash, - } - - query = f""" - WITH relevant_tests AS ( - SELECT - tests.environment_compatible AS hardware, - tests.environment_misc ->> 'platform' AS platform, - tests.status, - tests.path, - tests.id, - b.id AS build_id, - b.status AS build_status - FROM - checkouts c - INNER JOIN builds b ON b.checkout_id = c.id - INNER JOIN tests ON tests.build_id = b.id - WHERE - c.tree_name = %(tree_name)s - AND c.git_repository_url = %(git_repository_url)s - AND c.git_repository_branch = %(git_repository_branch)s - AND c.git_commit_hash = %(git_commit_hash)s - AND tests.origin = %(origin)s - AND tests.environment_misc ->> 'platform' IS NOT NULL - ) - SELECT - relevant_tests.platform, - relevant_tests.hardware, - {count_clauses} - FROM - relevant_tests - GROUP BY - relevant_tests.platform, - relevant_tests.hardware - ORDER BY - relevant_tests.platform ASC +) -> list[tuple]: + checkouts = """ + SELECT id AS checkout_id + FROM checkouts + WHERE tree_name = %(tree_name)s + AND git_repository_url = %(git_repository_url)s + AND git_repository_branch = %(git_repository_branch)s + AND git_commit_hash = %(git_commit_hash)s """ - with connection.cursor() as cursor: - cursor.execute(query, params) - return dict_fetchall(cursor) + return _hardware_daily_counts( + checkouts=checkouts, + day_range="", + params={ + "tree_name": tree_name, + "git_repository_url": git_repository_url, + "git_repository_branch": git_repository_branch, + "git_commit_hash": git_commit_hash, + }, + checkout_origin=checkout_origin, + build_origin=build_origin, + test_origin=test_origin, + build_lab=build_lab, + test_lab=test_lab, + ) def get_hardware_listing_data_bulk( @@ -309,120 +298,280 @@ def get_hardware_listing_data_bulk( return dict_fetchall(cursor) -def get_hardware_listing_data_from_status_table( +def _hardware_filter_clause( + column: str, parameter: str, values: Optional[list[str]] +) -> str: + return f"AND {column} = ANY(%({parameter})s)" if values else "" + + +def _daily_aggregate_rows(table: str, checkouts: Optional[str], filters: str) -> str: + if checkouts is not None: + return f""" + SELECT daily.* + FROM {table} daily + INNER JOIN selected_checkouts selected + ON selected.checkout_id = daily.checkout_id + {filters}""" + + predicates = [ + line.strip().removeprefix("AND ") + for line in filters.splitlines() + if line.strip() + ] + if not predicates: + return f""" + SELECT daily.* + FROM {table} daily""" + + return f""" + SELECT daily.* + FROM {table} daily + WHERE {" AND ".join(predicates)}""" + + +def _platform_counts_cte(*, rows: str, prefix: str, sums: str) -> str: + return f""" + {prefix}_hardware AS ( + SELECT DISTINCT ON (platform) platform, compatibles + FROM {rows} + WHERE compatibles IS NOT NULL + ORDER BY platform, CARDINALITY(compatibles) DESC, compatibles + ), + {prefix}_counts AS ( + SELECT + daily.platform, + hardware.compatibles AS hardware, + {sums} + FROM {rows} daily + LEFT JOIN {prefix}_hardware hardware USING (platform) + GROUP BY daily.platform, hardware.compatibles + )""" + + +def _hardware_daily_counts( + *, + checkouts: Optional[str], + day_range: str, + params: dict, + checkout_origin: Optional[list[str]], + build_origin: Optional[list[str]], + test_origin: Optional[list[str]], + build_lab: Optional[list[str]], + test_lab: Optional[list[str]], +) -> list[tuple]: + """Counts from daily aggregates. Build/test filtered independently; narrowed side picks platforms. + Optional checkouts narrows checkout_id; else checkout_day window only.""" + # checkout_origin is nullable: rows whose checkout has been pruned, and rows + # aggregated before the column existed, have no origin to compare. Narrowing + # to an origin leaves them out rather than matching them against every one. + checkout_origin_clause = _hardware_filter_clause( + "daily.checkout_origin", "checkout_origin", checkout_origin + ) + build_filters = "\n".join( + ( + day_range, + checkout_origin_clause, + _hardware_filter_clause("daily.build_origin", "build_origin", build_origin), + _hardware_filter_clause("daily.build_lab", "build_lab", build_lab), + ) + ) + test_filters = "\n".join( + ( + day_range, + checkout_origin_clause, + _hardware_filter_clause("daily.test_origin", "test_origin", test_origin), + _hardware_filter_clause("daily.test_lab", "test_lab", test_lab), + ) + ) + sides_join = { + (True, True): "INNER JOIN", + (True, False): "LEFT JOIN", + (False, True): "RIGHT JOIN", + (False, False): "FULL OUTER JOIN", + }[bool(build_origin or build_lab), bool(test_origin or test_lab)] + + scope_cte = f"selected_checkouts AS ({checkouts}),\n " if checkouts else "" + + query = f""" + WITH {scope_cte}build_rows AS ( + {_daily_aggregate_rows("hardware_daily_builds", checkouts, build_filters)} + ), + { + _platform_counts_cte( + rows="build_rows", + prefix="build", + sums=''' + SUM(daily.build_pass) AS build_pass, + SUM(daily.build_failed) AS build_failed, + SUM(daily.build_inc) AS build_inc''', + ) + }, + test_rows AS ( + {_daily_aggregate_rows("hardware_daily_tests", checkouts, test_filters)} + ), + { + _platform_counts_cte( + rows="test_rows", + prefix="test", + sums=''' + SUM(daily.boot_pass) AS boot_pass, + SUM(daily.boot_failed) AS boot_failed, + SUM(daily.boot_inc) AS boot_inc, + SUM(daily.test_pass) AS test_pass, + SUM(daily.test_failed) AS test_failed, + SUM(daily.test_inc) AS test_inc''', + ) + } + SELECT + COALESCE(builds.platform, tests.platform) AS platform, + COALESCE(tests.hardware, builds.hardware) AS hardware, + COALESCE(builds.build_pass, 0), + COALESCE(builds.build_failed, 0), + COALESCE(builds.build_inc, 0), + COALESCE(tests.boot_pass, 0), + COALESCE(tests.boot_failed, 0), + COALESCE(tests.boot_inc, 0), + COALESCE(tests.test_pass, 0), + COALESCE(tests.test_failed, 0), + COALESCE(tests.test_inc, 0) + FROM build_counts builds + {sides_join} test_counts tests ON tests.platform = builds.platform + ORDER BY platform + """ + + with connection.cursor() as cursor: + cursor.execute( + query, + { + **params, + "checkout_origin": checkout_origin, + "build_origin": build_origin, + "test_origin": test_origin, + "build_lab": build_lab, + "test_lab": test_lab, + }, + ) + return cursor.fetchall() + + +def get_hardware_listing_data( + *, start_date: datetime, end_date: datetime, - origin: str, + checkout_origin: Optional[list[str]], + build_origin: Optional[list[str]], + test_origin: Optional[list[str]], + build_lab: Optional[list[str]], + test_lab: Optional[list[str]], commits_list: Optional[list[str]] = None, ) -> list[tuple]: - """ - Retrieves hardware listing data from the HardwareStatus denormalized table. - Groups by platform and compatibles, aggregating status counts. - """ params = { "start_date": start_date, "end_date": end_date, - "origin": origin, + "start_day": start_date.date(), + "end_day": end_date.date(), } if commits_list: params["commits_list"] = commits_list - query = """ - SELECT - platform, - compatibles AS hardware, - SUM(build_pass) AS build_pass, - SUM(build_failed) AS build_fail, - SUM(build_inc) AS build_null, - SUM(boot_pass) AS boot_pass, - SUM(boot_failed) AS boot_fail, - SUM(boot_inc) AS boot_null, - SUM(test_pass) AS test_pass, - SUM(test_failed) AS test_fail, - SUM(test_inc) AS test_null - FROM - hardware_status - INNER JOIN - checkouts C - ON - hardware_status.checkout_id = C.id - AND - C.start_time >= %(start_date)s - AND - C.start_time <= %(end_date)s - AND ( - C.git_commit_hash = ANY(%(commits_list)s) - OR ( - C.git_commit_tags IS NOT NULL - AND C.git_commit_tags && %(commits_list)s::text[] - ) - ) - WHERE - hardware_status.test_origin = %(origin)s - GROUP BY - platform, - compatibles - ORDER BY - platform, - compatibles - """ + checkouts = """ + SELECT id AS checkout_id + FROM checkouts + WHERE start_time >= %(start_date)s + AND start_time <= %(end_date)s + AND ( + git_commit_hash = ANY(%(commits_list)s) + OR git_commit_tags && %(commits_list)s::text[] + ) + """ else: - query = """ - WITH latest_per_tree AS ( - SELECT DISTINCT ON ( - HS.platform, - HS.compatibles, - C.tree_name, - C.git_repository_branch, - C.git_repository_url, - C.origin - ) - HS.platform, - HS.compatibles, - HS.build_pass, HS.build_failed, HS.build_inc, - HS.boot_pass, HS.boot_failed, HS.boot_inc, - HS.test_pass, HS.test_failed, HS.test_inc - FROM - hardware_status HS - INNER JOIN checkouts C ON C.id = HS.checkout_id - WHERE - HS.test_origin = %(origin)s - AND HS.start_time >= %(start_date)s - AND HS.start_time <= %(end_date)s - ORDER BY - HS.platform ASC, - HS.compatibles ASC, - C.tree_name ASC, - C.git_repository_branch ASC, - C.git_repository_url ASC, - C.origin ASC, - C.start_time DESC + checkouts = None + + return _hardware_daily_counts( + checkouts=checkouts, + day_range="AND daily.checkout_day BETWEEN %(start_day)s AND %(end_day)s", + params=params, + checkout_origin=checkout_origin, + build_origin=build_origin, + test_origin=test_origin, + build_lab=build_lab, + test_lab=test_lab, + ) + + +def get_hardware_filters( + *, start_date: datetime, end_date: datetime +) -> dict[str, list[str]]: + """Filter option lists for the listing window. Lists are independent (no cascading).""" + cache_key = "hardwareFilters" + cache_params = {"start_date": start_date, "end_date": end_date} + + cached = get_query_cache(cache_key, cache_params) + if cached is not None: + return cached[0] + + query = """ + WITH days AS ( + SELECT %(start_day)s::date AS start_day, %(end_day)s::date AS end_day ) SELECT - platform, - compatibles AS hardware, - SUM(build_pass) AS build_pass, - SUM(build_failed) AS build_fail, - SUM(build_inc) AS build_null, - SUM(boot_pass) AS boot_pass, - SUM(boot_failed) AS boot_fail, - SUM(boot_inc) AS boot_null, - SUM(test_pass) AS test_pass, - SUM(test_failed) AS test_fail, - SUM(test_inc) AS test_null - FROM - latest_per_tree - GROUP BY - platform, - compatibles - ORDER BY - platform, - compatibles + ARRAY( + SELECT DISTINCT origin FROM ( + SELECT checkout_origin AS origin + FROM hardware_daily_builds, days + WHERE checkout_day BETWEEN start_day AND end_day + UNION + SELECT checkout_origin AS origin + FROM hardware_daily_tests, days + WHERE checkout_day BETWEEN start_day AND end_day + ) checkout_origins + WHERE origin IS NOT NULL AND origin <> '' + ORDER BY origin + ), + ARRAY( + SELECT DISTINCT build_origin + FROM hardware_daily_builds, days + WHERE checkout_day BETWEEN start_day AND end_day + AND build_origin <> '' + ORDER BY build_origin + ), + ARRAY( + SELECT DISTINCT test_origin + FROM hardware_daily_tests, days + WHERE checkout_day BETWEEN start_day AND end_day + AND test_origin <> '' + ORDER BY test_origin + ), + ARRAY( + SELECT DISTINCT build_lab + FROM hardware_daily_builds, days + WHERE checkout_day BETWEEN start_day AND end_day + AND build_lab <> '' + ORDER BY build_lab + ), + ARRAY( + SELECT DISTINCT test_lab + FROM hardware_daily_tests, days + WHERE checkout_day BETWEEN start_day AND end_day + AND test_lab <> '' + ORDER BY test_lab + ) """ - with connection.cursor() as cursor: - cursor.execute(query, params) - return cursor.fetchall() + cursor.execute( + query, {"start_day": start_date.date(), "end_day": end_date.date()} + ) + row = cursor.fetchone() + + filters = { + "checkout_origins": row[0], + "build_origins": row[1], + "test_origins": row[2], + "build_labs": row[3], + "test_labs": row[4], + } + set_query_cache(key=cache_key, params=cache_params, rows=[filters]) + return filters def get_hardware_details_data( diff --git a/backend/kernelCI_app/tests/integrationTests/hardware_listing_test.py b/backend/kernelCI_app/tests/integrationTests/hardware_listing_test.py new file mode 100644 index 000000000..013632e94 --- /dev/null +++ b/backend/kernelCI_app/tests/integrationTests/hardware_listing_test.py @@ -0,0 +1,168 @@ +"""Integration tests for hardware listing queries over daily aggregates.""" + +from datetime import timedelta + +import pytest + +from kernelCI_app.models import HardwareDailyBuilds, HardwareDailyTests +from kernelCI_app.queries.hardware import ( + get_hardware_filters, + get_hardware_listing_data, + get_hardware_listing_data_by_revision, +) +from kernelCI_app.tests.factories import BuildFactory +from kernelCI_app.tests.integrationTests.recompute_hardware_daily_test import ( + DAY_START, + _checkout, + _recompute, + _test_on, +) + +WINDOW = (DAY_START - timedelta(hours=1), DAY_START + timedelta(hours=1)) + + +def _independent_filter_fixture(): + checkout = _checkout( + origin="checkout-origin", + tree_name="tree", + git_repository_url="https://example.com/linux.git", + git_repository_branch="main", + git_commit_hash="a" * 40, + ) + selected_build = BuildFactory( + checkout=checkout, + origin="selected-build-origin", + misc={"lab": "selected-build-lab"}, + status="PASS", + ) + other_build = BuildFactory( + checkout=checkout, + origin="other-build-origin", + misc={"lab": "other-build-lab"}, + status="FAIL", + ) + _test_on( + selected_build, + "pA", + origin="other-test-origin", + lab="other-test-lab", + status="FAIL", + ) + _test_on( + other_build, + "pA", + origin="selected-test-origin", + lab="selected-test-lab", + status="PASS", + ) + _recompute() + return checkout + + +@pytest.mark.django_db +def test_listing_filters_builds_and_tests_independently(): + _independent_filter_fixture() + start, end = WINDOW + + rows = get_hardware_listing_data( + start_date=start, + end_date=end, + checkout_origin=["checkout-origin"], + build_origin=["selected-build-origin"], + test_origin=["selected-test-origin"], + build_lab=["selected-build-lab"], + test_lab=["selected-test-lab"], + commits_list=["a" * 40], + ) + + assert len(rows) == 1 + assert rows[0][0] == "pA" + assert rows[0][2:5] == (1, 0, 0) + assert rows[0][8:11] == (1, 0, 0) + + +@pytest.mark.django_db +def test_filters_api_lists_every_option_in_the_window(): + _independent_filter_fixture() + start, end = WINDOW + + assert get_hardware_filters(start_date=start, end_date=end) == { + "checkout_origins": ["checkout-origin"], + "build_origins": ["other-build-origin", "selected-build-origin"], + "test_origins": ["other-test-origin", "selected-test-origin"], + "build_labs": ["other-build-lab", "selected-build-lab"], + "test_labs": ["other-test-lab", "selected-test-lab"], + } + + +@pytest.mark.django_db +def test_by_revision_listing_honours_side_filters(): + _independent_filter_fixture() + + revision_rows = get_hardware_listing_data_by_revision( + checkout_origin=["checkout-origin"], + build_origin=["selected-build-origin"], + test_origin=["selected-test-origin"], + build_lab=["selected-build-lab"], + test_lab=["selected-test-lab"], + tree_name="tree", + git_repository_url="https://example.com/linux.git", + git_repository_branch="main", + git_commit_hash="a" * 40, + ) + assert revision_rows[0][2] == 1 + assert revision_rows[0][8] == 1 + + +@pytest.mark.django_db +def test_a_narrowed_side_decides_which_platforms_are_listed(): + checkout = _checkout(origin="maestro") + build = BuildFactory(checkout=checkout, origin="maestro", status="PASS") + _test_on(build, "tested-here", lab="lava-broonie") + _test_on(build, "tested-elsewhere", lab="lava-collabora") + _recompute() + + def platforms(**filters): + rows = get_hardware_listing_data( + start_date=WINDOW[0], + end_date=WINDOW[1], + **{ + "checkout_origin": None, + "build_origin": None, + "test_origin": None, + "build_lab": None, + "test_lab": None, + **filters, + }, + ) + return [row[0] for row in rows] + + assert platforms() == ["tested-elsewhere", "tested-here"] + assert platforms(test_lab=["lava-broonie"]) == ["tested-here"] + assert platforms(build_origin=["maestro"], test_lab=["lava-broonie"]) == [ + "tested-here" + ] + assert platforms(test_origin=["nobody"]) == [] + + +@pytest.mark.django_db +def test_unknown_checkout_origin_is_listed_until_an_origin_is_chosen(): + checkout = _checkout(origin="maestro") + _test_on(BuildFactory(checkout=checkout, status="PASS"), "pA") + _recompute() + HardwareDailyBuilds.objects.update(checkout_origin=None) + HardwareDailyTests.objects.update(checkout_origin=None) + + def listing(checkout_origin): + return get_hardware_listing_data( + start_date=WINDOW[0], + end_date=WINDOW[1], + checkout_origin=checkout_origin, + build_origin=None, + test_origin=None, + build_lab=None, + test_lab=None, + ) + + assert [row[0] for row in listing(None)] == ["pA"] + assert listing(["maestro"]) == [] diff --git a/backend/kernelCI_app/tests/integrationTests/prune_db_test.py b/backend/kernelCI_app/tests/integrationTests/prune_db_test.py index d9159534a..d2adf6875 100644 --- a/backend/kernelCI_app/tests/integrationTests/prune_db_test.py +++ b/backend/kernelCI_app/tests/integrationTests/prune_db_test.py @@ -7,7 +7,13 @@ from django.core.management.base import CommandError from django.utils import timezone -from kernelCI_app.models import Builds, Checkouts, Tests +from kernelCI_app.models import ( + Builds, + Checkouts, + HardwareDailyBuilds, + HardwareDailyTests, + Tests, +) from kernelCI_app.tests.factories import ( BuildFactory, CheckoutFactory, @@ -41,6 +47,58 @@ def test_old_checkout_cascades(): assert not Tests.objects.filter(id=test.id).exists() +@pytest.mark.django_db +def test_hardware_daily_pruned_by_checkout_day(): + """The aggregate is pruned on its own checkout_day even when the raw checkout + survives: retention follows the summary's date grain, not the surviving facts.""" + checkout_time = _days_ago(30) + checkout = CheckoutFactory(start_time=checkout_time, field_timestamp=_days_ago(1)) + build = BuildFactory(checkout=checkout, status="PASS", field_timestamp=_days_ago(1)) + TestFactory( + build=build, + environment_misc={"platform": "pA"}, + path="boot", + status="PASS", + field_timestamp=_days_ago(1), + ) + call_command("recompute_hardware_daily", day=checkout_time.date()) + + assert HardwareDailyBuilds.objects.filter(checkout_id=checkout.id).exists() + assert HardwareDailyTests.objects.filter(checkout_id=checkout.id).exists() + + _prune(yes=True) + + assert Checkouts.objects.filter(id=checkout.id).exists() + assert not HardwareDailyBuilds.objects.filter(checkout_id=checkout.id).exists() + assert not HardwareDailyTests.objects.filter(checkout_id=checkout.id).exists() + + +@pytest.mark.django_db +def test_tables_target_only_hardware_daily(): + """--tables can prune an aggregate on its own, leaving raw and the other grain.""" + checkout_time = _days_ago(30) + checkout = CheckoutFactory(start_time=checkout_time, field_timestamp=_days_ago(30)) + build = BuildFactory( + checkout=checkout, status="PASS", field_timestamp=_days_ago(30) + ) + test = TestFactory( + build=build, + environment_misc={"platform": "pA"}, + path="boot", + status="PASS", + field_timestamp=_days_ago(30), + ) + call_command("recompute_hardware_daily", day=checkout_time.date()) + + _prune(yes=True, tables=["hardware_daily_builds"]) + + assert not HardwareDailyBuilds.objects.filter(checkout_id=checkout.id).exists() + assert HardwareDailyTests.objects.filter(checkout_id=checkout.id).exists() + assert Checkouts.objects.filter(id=checkout.id).exists() + assert Builds.objects.filter(id=build.id).exists() + assert Tests.objects.filter(id=test.id).exists() + + @pytest.mark.django_db def test_old_build_cascades(): """An old build drags its newer tests; its recent checkout survives.""" @@ -133,10 +191,20 @@ def test_origins_cascade_ignores_child_origin(): def test_tables_tests_only(): """--tables tests deletes only old tests; parents and a recent test under an old build/checkout are kept because those parents are not being pruned.""" - checkout = CheckoutFactory(field_timestamp=_days_ago(30)) - build = BuildFactory(checkout=checkout, field_timestamp=_days_ago(30)) - old_test = TestFactory(build=build, field_timestamp=_days_ago(30)) + checkout_time = _days_ago(30) + checkout = CheckoutFactory(start_time=checkout_time, field_timestamp=_days_ago(30)) + build = BuildFactory( + checkout=checkout, status="PASS", field_timestamp=_days_ago(30) + ) + old_test = TestFactory( + build=build, + environment_misc={"platform": "pA"}, + path="boot", + status="PASS", + field_timestamp=_days_ago(30), + ) recent_test = TestFactory(build=build, field_timestamp=_days_ago(1)) + call_command("recompute_hardware_daily", day=checkout_time.date()) _prune(yes=True, tables=["tests"]) @@ -144,6 +212,8 @@ def test_tables_tests_only(): assert Builds.objects.filter(id=build.id).exists() assert not Tests.objects.filter(id=old_test.id).exists() assert Tests.objects.filter(id=recent_test.id).exists() + assert HardwareDailyBuilds.objects.filter(checkout_id=checkout.id).exists() + assert HardwareDailyTests.objects.filter(checkout_id=checkout.id).exists() @pytest.mark.django_db diff --git a/backend/kernelCI_app/tests/integrationTests/recompute_hardware_daily_test.py b/backend/kernelCI_app/tests/integrationTests/recompute_hardware_daily_test.py new file mode 100644 index 000000000..c1a75b10e --- /dev/null +++ b/backend/kernelCI_app/tests/integrationTests/recompute_hardware_daily_test.py @@ -0,0 +1,185 @@ +"""Integration tests for the recompute_hardware_daily management command.""" + +from datetime import date, datetime, timedelta, timezone + +import pytest +from django.core.management import call_command +from django.db import connections + +from kernelCI_app.constants.general import MAESTRO_DUMMY_BUILD_PREFIX +from kernelCI_app.management.commands.recompute_hardware_daily import lock_key +from kernelCI_app.models import ( + HardwareDailyBuilds, + HardwareDailyTests, + Tests, +) +from kernelCI_app.tests.factories import BuildFactory, CheckoutFactory, TestFactory + +DAY = date(2026, 7, 15) +DAY_START = datetime(2026, 7, 15, 10, tzinfo=timezone.utc) + + +def _checkout(**kwargs): + return CheckoutFactory(start_time=DAY_START, **kwargs) + + +def _test_on( + build, + platform, + *, + path="ltp.x", + status="PASS", + lab="lab-1", + compatibles=None, + **kwargs, +): + TestFactory( + build=build, + path=path, + status=status, + environment_misc={"platform": platform}, + environment_compatible=compatibles, + misc={"runtime": lab} if lab else {}, + **kwargs, + ) + + +def _recompute(): + call_command("recompute_hardware_daily", day=DAY) + + +@pytest.mark.django_db +def test_boot_and_test_counters_are_split(): + """Boot paths, plain tests and anything but PASS/FAIL land in their own counter.""" + build = BuildFactory(checkout=_checkout(), status="PASS") + _test_on(build, "pA", path="boot", status="PASS") + _test_on(build, "pA", path="boot.nfs", status="FAIL") + _test_on(build, "pA", path="boot", status=None) + _test_on(build, "pA", path="ltp.x", status="SKIP") + _test_on(build, "pA", path=None, status="PASS") + other_day = CheckoutFactory(start_time=DAY_START - timedelta(days=1)) + _test_on(BuildFactory(checkout=other_day, status="PASS"), "pA", path="boot") + + _recompute() + + row = HardwareDailyTests.objects.get() + assert row.checkout_day == DAY + assert (row.boot_pass, row.boot_failed, row.boot_inc) == (1, 1, 1) + assert (row.test_pass, row.test_failed, row.test_inc) == (1, 0, 1) + + +@pytest.mark.django_db +def test_dummy_build_is_not_counted_but_its_tests_are(): + checkout = _checkout() + real = BuildFactory(checkout=checkout, status="PASS") + dummy = BuildFactory( + checkout=checkout, id=f"{MAESTRO_DUMMY_BUILD_PREFIX}1", status="PASS" + ) + _test_on(real, "pA", path="boot") + _test_on(dummy, "pA", path="boot") + + _recompute() + + assert HardwareDailyBuilds.objects.get().build_pass == 1 + assert HardwareDailyTests.objects.get().boot_pass == 2 + + +@pytest.mark.django_db +def test_build_is_counted_once_per_platform(): + build = BuildFactory(checkout=_checkout(), status="PASS") + _test_on(build, "pA") + _test_on(build, "pA") + _test_on(build, "pB") + + _recompute() + + counted = HardwareDailyBuilds.objects.order_by("platform") + assert [(row.platform, row.build_pass) for row in counted] == [("pA", 1), ("pB", 1)] + + +@pytest.mark.django_db +def test_lab_comes_from_misc_and_falls_back_to_origin(): + checkout = _checkout(origin="maestro") + from_misc = BuildFactory( + checkout=checkout, status="PASS", misc={"lab": "build-lab"}, origin="maestro" + ) + from_runtime = BuildFactory( + checkout=checkout, status="PASS", misc={"runtime": "run-lab"}, origin="maestro" + ) + _test_on(from_misc, "pA", lab=None) + _test_on(from_runtime, "pB", lab="test-lab") + + _recompute() + + assert HardwareDailyTests.objects.get(platform="pA").test_lab == "maestro" + assert HardwareDailyTests.objects.get(platform="pB").test_lab == "test-lab" + assert HardwareDailyBuilds.objects.get(platform="pA").build_lab == "build-lab" + assert HardwareDailyBuilds.objects.get(platform="pB").build_lab == "run-lab" + + +@pytest.mark.django_db +def test_checkout_origin_is_stored_on_both_tables(): + checkout = _checkout(origin="maestro") + build = BuildFactory(checkout=checkout, status="PASS", origin="maestro") + _test_on(build, "pA", path="boot", origin="linaro") + + _recompute() + + assert HardwareDailyBuilds.objects.get().checkout_origin == "maestro" + test_row = HardwareDailyTests.objects.get() + assert test_row.checkout_origin == "maestro" + assert test_row.test_origin == "linaro" + + +@pytest.mark.django_db +def test_rerun_replaces_the_day_without_duplicating(): + _test_on(BuildFactory(checkout=_checkout(), status="PASS"), "pA", path="boot") + + _recompute() + _recompute() + + assert HardwareDailyTests.objects.get().boot_pass == 1 + assert HardwareDailyBuilds.objects.get().build_pass == 1 + + +@pytest.mark.django_db +def test_compatibles_resolve_to_the_most_specific_chain(): + """Labs disagreeing on a platform get one label, the longest chain reported.""" + build = BuildFactory(checkout=_checkout(), status="PASS") + _test_on(build, "pA", lab="lab-1", compatibles=["rockchip", "rk3399"]) + _test_on(build, "pA", lab="lab-2", compatibles=["rk3399"]) + _test_on(build, "pA", lab="lab-3", compatibles=None) + + _recompute() + + assert HardwareDailyBuilds.objects.get().compatibles == ["rockchip", "rk3399"] + labelled = HardwareDailyTests.objects.values_list("compatibles", flat=True) + assert list(labelled) == [["rockchip", "rk3399"]] * 3 + + +@pytest.mark.django_db +def test_day_locked_by_another_run_is_skipped(): + _test_on(BuildFactory(checkout=_checkout(), status="PASS"), "pA", path="boot") + + other_run = connections.create_connection("default") + with other_run.cursor() as cursor: + cursor.execute( + "SELECT pg_advisory_lock(%s, %s)", + [lock_key("hardware_daily_builds"), DAY.toordinal()], + ) + _recompute() + other_run.close() + + assert not HardwareDailyTests.objects.exists() + + +@pytest.mark.django_db +def test_pruned_raw_data_keeps_the_existing_rows(): + _test_on(BuildFactory(checkout=_checkout(), status="PASS"), "pA", path="boot") + _recompute() + + Tests.objects.all().delete() + _recompute() + + assert HardwareDailyTests.objects.get().boot_pass == 1 + assert HardwareDailyBuilds.objects.get().build_pass == 1 diff --git a/backend/kernelCI_app/tests/unitTests/views/hardwareByRevisionView_test.py b/backend/kernelCI_app/tests/unitTests/views/hardwareByRevisionView_test.py new file mode 100644 index 000000000..80cb477b4 --- /dev/null +++ b/backend/kernelCI_app/tests/unitTests/views/hardwareByRevisionView_test.py @@ -0,0 +1,71 @@ +from http import HTTPStatus +from unittest.mock import patch + +from django.test.testcases import SimpleTestCase +from rest_framework.test import APIRequestFactory + +from kernelCI_app.views.hardwareByRevisionView import HardwareByRevisionView + + +class TestHardwareByRevisionView(SimpleTestCase): + @patch( + "kernelCI_app.views.hardwareByRevisionView." + "get_hardware_listing_data_by_revision" + ) + def test_get_passes_independent_filters(self, mock_get_listing): + mock_get_listing.return_value = [("platform1", ["hardware1"], *range(9))] + request = APIRequestFactory().get( + "/hardware-by-revision/", + { + "checkoutOrigin": "checkout-origin", + "buildOrigin": "build-origin", + "testOrigin": "test-origin", + "buildLab": "build-lab", + "testLab": "test-lab", + "tree_name": "tree", + "git_repository_url": "https://example.com/linux.git", + "git_repository_branch": "main", + "git_commit_hash": "a" * 40, + }, + ) + + response = HardwareByRevisionView().get(request) + + self.assertEqual(response.status_code, HTTPStatus.OK) + mock_get_listing.assert_called_once_with( + checkout_origin=["checkout-origin"], + build_origin=["build-origin"], + test_origin=["test-origin"], + build_lab=["build-lab"], + test_lab=["test-lab"], + tree_name="tree", + git_repository_url="https://example.com/linux.git", + git_repository_branch="main", + git_commit_hash="a" * 40, + ) + + @patch( + "kernelCI_app.views.hardwareByRevisionView." + "get_hardware_listing_data_by_revision" + ) + def test_origin_is_deprecated_checkout_and_build_alias(self, mock_get_listing): + mock_get_listing.return_value = [] + request = APIRequestFactory().get( + "/hardware-by-revision/", + { + "origin": "legacy", + "tree_name": "tree", + "git_repository_url": "https://example.com/linux.git", + "git_repository_branch": "main", + "git_commit_hash": "a" * 40, + }, + ) + + response = HardwareByRevisionView().get(request) + + self.assertEqual(response.status_code, HTTPStatus.OK) + self.assertEqual( + mock_get_listing.call_args.kwargs["checkout_origin"], ["legacy"] + ) + self.assertEqual(mock_get_listing.call_args.kwargs["build_origin"], ["legacy"]) + self.assertIsNone(mock_get_listing.call_args.kwargs["test_origin"]) diff --git a/backend/kernelCI_app/tests/unitTests/views/hardwareFiltersView_test.py b/backend/kernelCI_app/tests/unitTests/views/hardwareFiltersView_test.py new file mode 100644 index 000000000..5995f1370 --- /dev/null +++ b/backend/kernelCI_app/tests/unitTests/views/hardwareFiltersView_test.py @@ -0,0 +1,39 @@ +from http import HTTPStatus +from unittest.mock import ANY, patch + +from django.test.testcases import SimpleTestCase +from rest_framework.test import APIRequestFactory + +from kernelCI_app.views.hardwareFiltersView import HardwareFiltersView + +WINDOW = { + "startTimestampInSeconds": "1741192200", + "endTimestampInSeconds": "1741624200", +} + + +class TestHardwareFiltersView(SimpleTestCase): + def setUp(self): + self.factory = APIRequestFactory() + self.view = HardwareFiltersView() + self.url = "/hardware/filters/" + + @patch("kernelCI_app.views.hardwareFiltersView.get_hardware_filters") + def test_get_passes_the_requested_window(self, mock_get_hardware_filters): + mock_get_hardware_filters.return_value = { + "checkout_origins": [], + "build_origins": [], + "test_origins": [], + "build_labs": [], + "test_labs": [], + } + + response = self.view.get(self.factory.get(self.url, WINDOW)) + + self.assertEqual(response.status_code, HTTPStatus.OK) + mock_get_hardware_filters.assert_called_once_with(start_date=ANY, end_date=ANY) + + def test_get_hardware_filters_without_window_returns_bad_request(self): + response = self.view.get(self.factory.get(self.url)) + + self.assertEqual(response.status_code, HTTPStatus.BAD_REQUEST) diff --git a/backend/kernelCI_app/tests/unitTests/views/hardwareSelectorsView_test.py b/backend/kernelCI_app/tests/unitTests/views/hardwareSelectorsView_test.py new file mode 100644 index 000000000..9ee70d158 --- /dev/null +++ b/backend/kernelCI_app/tests/unitTests/views/hardwareSelectorsView_test.py @@ -0,0 +1,43 @@ +from unittest.mock import patch + +from django.test import SimpleTestCase +from rest_framework.test import APIRequestFactory + +from kernelCI_app.constants.general import DEFAULT_ORIGIN +from kernelCI_app.views.hardwareSelectorsView import HardwareSelectorsView + + +class TestHardwareSelectorsView(SimpleTestCase): + def setUp(self): + self.factory = APIRequestFactory() + + @patch( + "kernelCI_app.views.hardwareSelectorsView.get_hardware_selectors", + return_value=[], + ) + def test_build_origin_is_optional(self, mock_get_selectors): + HardwareSelectorsView().get( + self.factory.get("/hardware/selectors", {"buildOrigin": ""}) + ) + + mock_get_selectors.assert_called_once_with(build_origin=None) + + @patch( + "kernelCI_app.views.hardwareSelectorsView.get_hardware_selectors", + return_value=[], + ) + def test_defaults_to_maestro(self, mock_get_selectors): + HardwareSelectorsView().get(self.factory.get("/hardware/selectors")) + + mock_get_selectors.assert_called_once_with(build_origin=[DEFAULT_ORIGIN]) + + @patch( + "kernelCI_app.views.hardwareSelectorsView.get_hardware_selectors", + return_value=[], + ) + def test_honours_deprecated_origin(self, mock_get_selectors): + HardwareSelectorsView().get( + self.factory.get("/hardware/selectors", {"origin": "legacy"}) + ) + + mock_get_selectors.assert_called_once_with(build_origin=["legacy"]) diff --git a/backend/kernelCI_app/tests/unitTests/views/hardwareView_test.py b/backend/kernelCI_app/tests/unitTests/views/hardwareView_test.py index 066b4c4b4..9f14fd09e 100644 --- a/backend/kernelCI_app/tests/unitTests/views/hardwareView_test.py +++ b/backend/kernelCI_app/tests/unitTests/views/hardwareView_test.py @@ -4,6 +4,7 @@ from django.test.testcases import SimpleTestCase from rest_framework.test import APIRequestFactory +from kernelCI_app.constants.general import DEFAULT_ORIGIN from kernelCI_app.constants.localization import ClientStrings from kernelCI_app.views.hardwareView import HardwareView @@ -14,11 +15,39 @@ def setUp(self): self.view = HardwareView() self.url = "/hardware" - @patch( - "kernelCI_app.views.hardwareView.get_hardware_listing_data_from_status_table" - ) - def test_get_hardware_listing_success(self, mock_get_status_table_data): - mock_get_status_table_data.return_value = [ + @patch("kernelCI_app.views.hardwareView.get_hardware_listing_data") + def test_get_hardware_listing_defaults_to_maestro_builds( + self, mock_get_hardware_listing + ): + mock_get_hardware_listing.return_value = [ + ("platform1", "hardware1", *range(9)), + ] + + query_params = { + "startTimestampInSeconds": "1741192200", + "endTimestampInSeconds": "1741624200", + } + + request = self.factory.get(self.url, query_params) + response = self.view.get(request) + + self.assertEqual(response.status_code, HTTPStatus.OK) + mock_get_hardware_listing.assert_called_once_with( + checkout_origin=[DEFAULT_ORIGIN], + build_origin=[DEFAULT_ORIGIN], + test_origin=None, + build_lab=None, + test_lab=None, + start_date=ANY, + end_date=ANY, + commits_list=None, + ) + + @patch("kernelCI_app.views.hardwareView.get_hardware_listing_data") + def test_get_hardware_listing_honours_deprecated_origin( + self, mock_get_hardware_listing + ): + mock_get_hardware_listing.return_value = [ ("platform1", "hardware1", *range(9)), ] @@ -32,18 +61,20 @@ def test_get_hardware_listing_success(self, mock_get_status_table_data): response = self.view.get(request) self.assertEqual(response.status_code, HTTPStatus.OK) - mock_get_status_table_data.assert_called_once_with( - origin="origin1", + mock_get_hardware_listing.assert_called_once_with( + checkout_origin=["origin1"], + build_origin=["origin1"], + test_origin=None, + build_lab=None, + test_lab=None, start_date=ANY, end_date=ANY, commits_list=None, ) - @patch( - "kernelCI_app.views.hardwareView.get_hardware_listing_data_from_status_table" - ) - def test_get_hardware_listing_passes_commits_list(self, mock_get_status_table_data): - mock_get_status_table_data.return_value = [ + @patch("kernelCI_app.views.hardwareView.get_hardware_listing_data") + def test_get_hardware_listing_passes_commits_list(self, mock_get_hardware_listing): + mock_get_hardware_listing.return_value = [ ("platform1", "hardware1", *range(9)), ] h1 = "a" * 40 @@ -61,13 +92,51 @@ def test_get_hardware_listing_passes_commits_list(self, mock_get_status_table_da response = self.view.get(request) self.assertEqual(response.status_code, HTTPStatus.OK) - mock_get_status_table_data.assert_called_once_with( - origin="origin1", + mock_get_hardware_listing.assert_called_once_with( + checkout_origin=["origin1"], + build_origin=["origin1"], + test_origin=None, + build_lab=None, + test_lab=None, start_date=ANY, end_date=ANY, commits_list=[h1, h2], ) + @patch("kernelCI_app.views.hardwareView.get_hardware_listing_data") + def test_get_hardware_listing_passes_independent_filters( + self, mock_get_hardware_listing + ): + mock_get_hardware_listing.return_value = [ + ("platform1", "hardware1", *range(9)), + ] + + request = self.factory.get( + self.url, + { + "startTimestampInSeconds": "1741192200", + "endTimestampInSeconds": "1741624200", + "checkoutOrigin": "", + "buildOrigin": "build-origin", + "testOrigin": "test-origin", + "buildLab": "build-lab", + "testLab": "test-lab", + }, + ) + response = self.view.get(request) + + self.assertEqual(response.status_code, HTTPStatus.OK) + mock_get_hardware_listing.assert_called_once_with( + checkout_origin=None, + build_origin=["build-origin"], + test_origin=["test-origin"], + build_lab=["build-lab"], + test_lab=["test-lab"], + start_date=ANY, + end_date=ANY, + commits_list=None, + ) + def test_get_hardware_listing_invalid_query_params_returns_bad_request(self): query_params = {"origin": "origin1"} @@ -78,13 +147,11 @@ def test_get_hardware_listing_invalid_query_params_returns_bad_request(self): self.assertIn("start_date", response.data) self.assertIn("end_date", response.data) - @patch( - "kernelCI_app.views.hardwareView.get_hardware_listing_data_from_status_table" - ) + @patch("kernelCI_app.views.hardwareView.get_hardware_listing_data") def test_get_hardware_listing_no_hardware_found_returns_ok_with_error( - self, mock_get_status_table_data + self, mock_get_hardware_listing ): - mock_get_status_table_data.return_value = [] + mock_get_hardware_listing.return_value = [] query_params = { "startTimestampInSeconds": "1741192200", @@ -98,13 +165,11 @@ def test_get_hardware_listing_no_hardware_found_returns_ok_with_error( self.assertEqual(response.status_code, HTTPStatus.OK) self.assertEqual(response.data, {"error": ClientStrings.NO_HARDWARE_FOUND}) - @patch( - "kernelCI_app.views.hardwareView.get_hardware_listing_data_from_status_table" - ) - def test_get_hardware_listing_sanitize_validation_error_returns_internal_server_error( - self, mock_get_status_table_data + @patch("kernelCI_app.views.hardwareView.get_hardware_listing_data") + def test_get_hardware_listing_from_row_validation_error_returns_internal_server_error( + self, mock_get_hardware_listing ): - mock_get_status_table_data.return_value = [ + mock_get_hardware_listing.return_value = [ (None, "hardware1", *range(9)), ] diff --git a/backend/kernelCI_app/tests/utils/client/hardwareClient.py b/backend/kernelCI_app/tests/utils/client/hardwareClient.py index 7a34a5df4..ec4117fed 100644 --- a/backend/kernelCI_app/tests/utils/client/hardwareClient.py +++ b/backend/kernelCI_app/tests/utils/client/hardwareClient.py @@ -13,7 +13,9 @@ def get_hardware_listing( self, *, query: HardwareQueryParamsDocumentationOnly ) -> requests.Response: path = reverse("hardware") - url = self.get_endpoint(path=path, query=query.model_dump()) + url = self.get_endpoint( + path=path, query=query.model_dump(exclude_defaults=True) + ) return requests.get(url) def post_hardware_boots( diff --git a/backend/kernelCI_app/typeModels/hardwareListing.py b/backend/kernelCI_app/typeModels/hardwareListing.py index 972621a97..526e9a9b3 100644 --- a/backend/kernelCI_app/typeModels/hardwareListing.py +++ b/backend/kernelCI_app/typeModels/hardwareListing.py @@ -9,7 +9,7 @@ from kernelCI_app.typeModels.commonListing import ListingStatusCount -def _normalize_commits_list(value: object) -> Optional[list[str]]: +def _normalize_comma_list(value: object) -> Optional[list[str]]: if value is None: return None if isinstance(value, str): @@ -33,26 +33,100 @@ class HardwareListingItem(BaseModel): boot_status_summary: ListingStatusCount build_status_summary: ListingStatusCount + @classmethod + def from_row(cls, row: tuple) -> "HardwareListingItem": + return cls( + platform=row[0], + hardware=row[1], + build_status_summary=ListingStatusCount( + PASS=row[2], FAIL=row[3], INCONCLUSIVE=row[4] + ), + boot_status_summary=ListingStatusCount( + PASS=row[5], FAIL=row[6], INCONCLUSIVE=row[7] + ), + test_status_summary=ListingStatusCount( + PASS=row[8], FAIL=row[9], INCONCLUSIVE=row[10] + ), + ) + class HardwareListingResponse(BaseModel): hardware: list[HardwareListingItem] -class HardwareListingByRevisionResponse(BaseModel): - hardware: list[HardwareItem] +class HardwareFiltersResponse(BaseModel): + checkout_origins: list[str] + build_origins: list[str] + test_origins: list[str] + build_labs: list[str] + test_labs: list[str] + + +class HardwareFiltersQueryParams(BaseModel): + start_date: datetime + end_date: datetime + + +class HardwareFiltersQueryParamsDocumentationOnly(BaseModel): + startTimestampInSeconds: str = Field( # noqa: N815 + description=DocStrings.DEFAULT_START_TS_DESCRIPTION + ) + endTimestampInSeconds: str = Field( # noqa: N815 + description=DocStrings.DEFAULT_END_TS_DESCRIPTION + ) + + +class HardwareFilterParams(BaseModel): + """Side-specific origin/lab filters; empty clears to all. `origin` aliases checkout+build only.""" + + checkout_origin: Optional[list[str]] = Field( + default_factory=lambda: [DEFAULT_ORIGIN] + ) + build_origin: Optional[list[str]] = Field(default_factory=lambda: [DEFAULT_ORIGIN]) + test_origin: Optional[list[str]] = None + build_lab: Optional[list[str]] = None + test_lab: Optional[list[str]] = None + + @classmethod + def from_request(cls, query, **extra): + origin_default = query.get("origin", DEFAULT_ORIGIN) + + def parse(name: str, fallback: Optional[str]) -> Optional[list[str]]: + raw = query.get(name) + if raw is None: + raw = fallback + return _normalize_comma_list(raw) + + return cls( + checkout_origin=parse("checkoutOrigin", origin_default), + build_origin=parse("buildOrigin", origin_default), + test_origin=parse("testOrigin", None), + build_lab=parse("buildLab", None), + test_lab=parse("testLab", None), + **extra, + ) # Since OpenAPI does not support timestamp as datetime we add an extra model just for # documentation purposes. This model is not used in the code. # TODO Remove timestamp from the api and this model class HardwareQueryParamsDocumentationOnly(BaseModel): - origin: Annotated[ - str, + checkoutOrigin: Annotated[ # noqa: N815 + Optional[str], Field( default=DEFAULT_ORIGIN, description=DocStrings.HARDWARE_LISTING_ORIGIN_DESCRIPTION, ), ] + buildOrigin: Optional[str] = DEFAULT_ORIGIN # noqa: N815 + testOrigin: Optional[str] = None # noqa: N815 + buildLab: Optional[str] = None # noqa: N815 + testLab: Optional[str] = None # noqa: N815 + origin: Optional[str] = Field( + default=None, + deprecated=True, + description="Deprecated alias for checkoutOrigin and buildOrigin", + ) startTimestampInSeconds: str = Field( # noqa: N815 description=DocStrings.DEFAULT_START_TS_DESCRIPTION ) @@ -65,15 +139,10 @@ class HardwareQueryParamsDocumentationOnly(BaseModel): ) -class HardwareQueryParams(BaseModel): - origin: Annotated[ - str, - Field(default=DEFAULT_ORIGIN), - BeforeValidator(lambda o: DEFAULT_ORIGIN if o is None else o), - ] +class HardwareQueryParams(HardwareFilterParams): start_date: datetime end_date: datetime commits_list: Annotated[ Optional[list[str]], - BeforeValidator(_normalize_commits_list), + BeforeValidator(_normalize_comma_list), ] = Field(default=None) diff --git a/backend/kernelCI_app/typeModels/hardwareListingByRevision.py b/backend/kernelCI_app/typeModels/hardwareListingByRevision.py index 34cecdd15..28fc765a4 100644 --- a/backend/kernelCI_app/typeModels/hardwareListingByRevision.py +++ b/backend/kernelCI_app/typeModels/hardwareListingByRevision.py @@ -1,19 +1,23 @@ -from typing import Annotated +from typing import Optional -from pydantic import BaseModel, BeforeValidator, Field +from pydantic import BaseModel, Field from kernelCI_app.constants.general import DEFAULT_ORIGIN from kernelCI_app.constants.localization import DocStrings +from kernelCI_app.typeModels.hardwareListing import HardwareFilterParams class HardwareListingByRevisionQueryParamsDocumentationOnly(BaseModel): - origin: Annotated[ - str, - Field( - default=DEFAULT_ORIGIN, - description=DocStrings.HARDWARE_LISTING_ORIGIN_DESCRIPTION, - ), - ] + checkoutOrigin: Optional[str] = DEFAULT_ORIGIN # noqa: N815 + buildOrigin: Optional[str] = DEFAULT_ORIGIN # noqa: N815 + testOrigin: Optional[str] = None # noqa: N815 + buildLab: Optional[str] = None # noqa: N815 + testLab: Optional[str] = None # noqa: N815 + origin: Optional[str] = Field( + default=None, + deprecated=True, + description="Deprecated alias for checkoutOrigin and buildOrigin", + ) tree_name: str = Field(description=DocStrings.TREE_NAME_PATH_DESCRIPTION) git_repository_url: str = Field( description=DocStrings.TREE_QUERY_GIT_URL_DESCRIPTION @@ -24,12 +28,7 @@ class HardwareListingByRevisionQueryParamsDocumentationOnly(BaseModel): git_commit_hash: str = Field(description=DocStrings.COMMIT_HASH_PATH_DESCRIPTION) -class HardwareListingByRevisionQueryParams(BaseModel): - origin: Annotated[ - str, - Field(default=DEFAULT_ORIGIN), - BeforeValidator(lambda o: DEFAULT_ORIGIN if o is None else o), - ] +class HardwareListingByRevisionQueryParams(HardwareFilterParams): tree_name: str git_repository_url: str git_repository_branch: str diff --git a/backend/kernelCI_app/typeModels/hardwareSelectors.py b/backend/kernelCI_app/typeModels/hardwareSelectors.py index e584e5c77..082bcc8c0 100644 --- a/backend/kernelCI_app/typeModels/hardwareSelectors.py +++ b/backend/kernelCI_app/typeModels/hardwareSelectors.py @@ -1,10 +1,10 @@ from datetime import datetime -from typing import Annotated +from typing import Optional -from pydantic import BaseModel, BeforeValidator, Field +from pydantic import BaseModel, Field from kernelCI_app.constants.general import DEFAULT_ORIGIN -from kernelCI_app.constants.localization import DocStrings +from kernelCI_app.typeModels.hardwareListing import _normalize_comma_list class HardwareSelectorRevision(BaseModel): @@ -29,18 +29,21 @@ class HardwareSelectorsResponse(BaseModel): class HardwareSelectorsQueryParamsDocumentationOnly(BaseModel): - origin: Annotated[ - str, - Field( - default=DEFAULT_ORIGIN, - description=DocStrings.HARDWARE_LISTING_ORIGIN_DESCRIPTION, - ), - ] + buildOrigin: Optional[str] = Field( # noqa: N815 + default=DEFAULT_ORIGIN, + description="Optional origin of builds that provide revisions", + ) + origin: Optional[str] = Field( + default=None, + deprecated=True, + description="Deprecated alias for buildOrigin", + ) class HardwareSelectorsQueryParams(BaseModel): - origin: Annotated[ - str, - Field(default=DEFAULT_ORIGIN), - BeforeValidator(lambda o: DEFAULT_ORIGIN if o is None else o), - ] + build_origin: Optional[list[str]] = Field(default_factory=lambda: [DEFAULT_ORIGIN]) + + @classmethod + def from_request(cls, query): + raw = query.get("buildOrigin", query.get("origin", DEFAULT_ORIGIN)) + return cls(build_origin=_normalize_comma_list(raw)) diff --git a/backend/kernelCI_app/urls.py b/backend/kernelCI_app/urls.py index 09584cd25..6e2650efc 100644 --- a/backend/kernelCI_app/urls.py +++ b/backend/kernelCI_app/urls.py @@ -136,6 +136,11 @@ def view_cache(view, timeout: int = settings.CACHE_TIMEOUT): view_cache(views.HardwareSelectorsView), name="hardwareSelectors", ), + path( + "hardware/filters/", + view_cache(views.HardwareFiltersView), + name="hardwareFilters", + ), path( "hardware/", view_cache(views.HardwareDetails), diff --git a/backend/kernelCI_app/views/hardwareByRevisionView.py b/backend/kernelCI_app/views/hardwareByRevisionView.py index 931548b70..15d1652b0 100644 --- a/backend/kernelCI_app/views/hardwareByRevisionView.py +++ b/backend/kernelCI_app/views/hardwareByRevisionView.py @@ -8,8 +8,8 @@ from kernelCI_app.queries.hardware import get_hardware_listing_data_by_revision from kernelCI_app.typeModels.hardwareListing import ( - HardwareItem, - HardwareListingByRevisionResponse, + HardwareListingItem, + HardwareListingResponse, ) from kernelCI_app.typeModels.hardwareListingByRevision import ( HardwareListingByRevisionQueryParams, @@ -18,53 +18,14 @@ class HardwareByRevisionView(APIView): - def _sanitize_records(self, hardwares_raw: list[dict]) -> list[HardwareItem]: - hardwares = [] - for hardware in hardwares_raw: - hardwares.append( - HardwareItem( - platform=hardware["platform"], - hardware=hardware["hardware"], - build_status_summary={ - "PASS": hardware["pass_builds"], - "FAIL": hardware["fail_builds"], - "NULL": hardware["null_builds"], - "ERROR": hardware["error_builds"], - "MISS": hardware["miss_builds"], - "DONE": hardware["done_builds"], - "SKIP": hardware["skip_builds"], - }, - boot_status_summary={ - "PASS": hardware["pass_boots"], - "FAIL": hardware["fail_boots"], - "NULL": hardware["null_boots"], - "ERROR": hardware["error_boots"], - "MISS": hardware["miss_boots"], - "DONE": hardware["done_boots"], - "SKIP": hardware["skip_boots"], - }, - test_status_summary={ - "PASS": hardware["pass_tests"], - "FAIL": hardware["fail_tests"], - "NULL": hardware["null_tests"], - "ERROR": hardware["error_tests"], - "MISS": hardware["miss_tests"], - "DONE": hardware["done_tests"], - "SKIP": hardware["skip_tests"], - }, - ) - ) - - return hardwares - @extend_schema( parameters=[HardwareListingByRevisionQueryParamsDocumentationOnly], - responses=HardwareListingByRevisionResponse, + responses=HardwareListingResponse, ) def get(self, request: Request): try: - query_params = HardwareListingByRevisionQueryParams( - origin=request.GET.get("origin"), + query_params = HardwareListingByRevisionQueryParams.from_request( + request.GET, tree_name=request.GET.get("tree_name"), git_repository_url=request.GET.get("git_repository_url"), git_repository_branch=request.GET.get("git_repository_branch"), @@ -74,7 +35,11 @@ def get(self, request: Request): return Response(data=e.json(), status=HTTPStatus.BAD_REQUEST) hardwares_raw = get_hardware_listing_data_by_revision( - origin=query_params.origin, + checkout_origin=query_params.checkout_origin, + build_origin=query_params.build_origin, + test_origin=query_params.test_origin, + build_lab=query_params.build_lab, + test_lab=query_params.test_lab, tree_name=query_params.tree_name, git_repository_url=query_params.git_repository_url, git_repository_branch=query_params.git_repository_branch, @@ -82,8 +47,9 @@ def get(self, request: Request): ) try: - sanitized_records = self._sanitize_records(hardwares_raw=hardwares_raw) - result = HardwareListingByRevisionResponse(hardware=sanitized_records) + result = HardwareListingResponse( + hardware=[HardwareListingItem.from_row(row) for row in hardwares_raw] + ) except ValidationError as e: return Response(data=e.json(), status=HTTPStatus.INTERNAL_SERVER_ERROR) diff --git a/backend/kernelCI_app/views/hardwareFiltersView.py b/backend/kernelCI_app/views/hardwareFiltersView.py new file mode 100644 index 000000000..ad6b81a5e --- /dev/null +++ b/backend/kernelCI_app/views/hardwareFiltersView.py @@ -0,0 +1,36 @@ +from http import HTTPStatus + +from drf_spectacular.utils import extend_schema +from pydantic import ValidationError +from rest_framework.request import Request +from rest_framework.response import Response +from rest_framework.views import APIView + +from kernelCI_app.queries.hardware import get_hardware_filters +from kernelCI_app.typeModels.hardwareListing import ( + HardwareFiltersQueryParams, + HardwareFiltersQueryParamsDocumentationOnly, + HardwareFiltersResponse, +) + + +class HardwareFiltersView(APIView): + @extend_schema( + parameters=[HardwareFiltersQueryParamsDocumentationOnly], + responses=HardwareFiltersResponse, + ) + def get(self, request: Request): + try: + query_params = HardwareFiltersQueryParams( + start_date=request.GET.get("startTimestampInSeconds"), + end_date=request.GET.get("endTimestampInSeconds"), + ) + except ValidationError as e: + return Response(data=e.json(), status=HTTPStatus.BAD_REQUEST) + + filters = get_hardware_filters( + start_date=query_params.start_date, end_date=query_params.end_date + ) + result = HardwareFiltersResponse(**filters) + + return Response(data=result.model_dump(), status=HTTPStatus.OK) diff --git a/backend/kernelCI_app/views/hardwareSelectorsView.py b/backend/kernelCI_app/views/hardwareSelectorsView.py index cd78f9b61..15805b1e6 100644 --- a/backend/kernelCI_app/views/hardwareSelectorsView.py +++ b/backend/kernelCI_app/views/hardwareSelectorsView.py @@ -96,13 +96,11 @@ def _sanitize_records(self, selectors_raw: list[dict]) -> HardwareSelectorsRespo ) def get(self, request: Request): try: - query_params = HardwareSelectorsQueryParams( - origin=request.GET.get("origin") - ) + query_params = HardwareSelectorsQueryParams.from_request(request.GET) except ValidationError as e: return Response(data=e.json(), status=HTTPStatus.BAD_REQUEST) - selectors_raw = get_hardware_selectors(origin=query_params.origin) + selectors_raw = get_hardware_selectors(build_origin=query_params.build_origin) try: result = self._sanitize_records(selectors_raw=selectors_raw) diff --git a/backend/kernelCI_app/views/hardwareView.py b/backend/kernelCI_app/views/hardwareView.py index f2ab36312..321da7912 100644 --- a/backend/kernelCI_app/views/hardwareView.py +++ b/backend/kernelCI_app/views/hardwareView.py @@ -1,4 +1,3 @@ -from datetime import datetime from http import HTTPStatus from drf_spectacular.utils import extend_schema @@ -9,8 +8,7 @@ from kernelCI_app.constants.localization import ClientStrings from kernelCI_app.helpers.errorHandling import create_api_error_response -from kernelCI_app.queries.hardware import get_hardware_listing_data_from_status_table -from kernelCI_app.typeModels.commonListing import ListingStatusCount +from kernelCI_app.queries.hardware import get_hardware_listing_data from kernelCI_app.typeModels.hardwareListing import ( HardwareListingItem, HardwareListingResponse, @@ -20,64 +18,36 @@ class HardwareView(APIView): - def _sanitize_records( - self, hardwares_raw: list[tuple] - ) -> list[HardwareListingItem]: - hardwares = [] - for hardware in hardwares_raw: - hardwares.append( - HardwareListingItem( - platform=hardware[0], - hardware=hardware[1], - build_status_summary=ListingStatusCount( - PASS=hardware[2], - FAIL=hardware[3], - INCONCLUSIVE=hardware[4], - ), - boot_status_summary=ListingStatusCount( - PASS=hardware[5], - FAIL=hardware[6], - INCONCLUSIVE=hardware[7], - ), - test_status_summary=ListingStatusCount( - PASS=hardware[8], - FAIL=hardware[9], - INCONCLUSIVE=hardware[10], - ), - ) - ) - - return hardwares - @extend_schema( parameters=[HardwareQueryParamsDocumentationOnly], responses=HardwareListingResponse, ) def get(self, request: Request): try: - query_params = HardwareQueryParams( + query_params = HardwareQueryParams.from_request( + request.GET, start_date=request.GET.get("startTimestampInSeconds"), end_date=request.GET.get("endTimestampInSeconds"), - origin=request.GET.get("origin"), commits_list=request.GET.get("commitsList"), ) - - start_date: datetime = query_params.start_date - end_date: datetime = query_params.end_date - origin = query_params.origin except ValidationError as e: return Response(data=e.json(), status=HTTPStatus.BAD_REQUEST) - hardwares_raw = get_hardware_listing_data_from_status_table( - origin=origin, - start_date=start_date, - end_date=end_date, + hardwares_raw = get_hardware_listing_data( + start_date=query_params.start_date, + end_date=query_params.end_date, + checkout_origin=query_params.checkout_origin, + build_origin=query_params.build_origin, + test_origin=query_params.test_origin, + build_lab=query_params.build_lab, + test_lab=query_params.test_lab, commits_list=query_params.commits_list, ) try: - sanitized_records = self._sanitize_records(hardwares_raw=hardwares_raw) - result = HardwareListingResponse(hardware=sanitized_records) + result = HardwareListingResponse( + hardware=[HardwareListingItem.from_row(row) for row in hardwares_raw] + ) if len(result.hardware) < 1: return create_api_error_response( diff --git a/backend/schema.yml b/backend/schema.yml index 6269c5c28..b8fb5cb6b 100644 --- a/backend/schema.yml +++ b/backend/schema.yml @@ -77,6 +77,32 @@ paths: get: operationId: hardware_retrieve parameters: + - in: query + name: buildLab + schema: + anyOf: + - type: string + - type: 'null' + default: null + title: Buildlab + - in: query + name: buildOrigin + schema: + anyOf: + - type: string + - type: 'null' + default: maestro + title: Buildorigin + - in: query + name: checkoutOrigin + schema: + anyOf: + - type: string + - type: 'null' + default: maestro + title: Checkoutorigin + description: Origin of the checkout the hardware was tested from. Pass it + empty to list every origin. - in: query name: commitsList schema: @@ -97,10 +123,13 @@ paths: - in: query name: origin schema: - default: maestro + anyOf: + - type: string + - type: 'null' + default: null + deprecated: true title: Origin - type: string - description: Origin of the hardware + description: Deprecated alias for checkoutOrigin and buildOrigin - in: query name: startTimestampInSeconds schema: @@ -108,6 +137,22 @@ paths: type: string description: Interval start timestamp in seconds for the results required: true + - in: query + name: testLab + schema: + anyOf: + - type: string + - type: 'null' + default: null + title: Testlab + - in: query + name: testOrigin + schema: + anyOf: + - type: string + - type: 'null' + default: null + title: Testorigin tags: - hardware security: @@ -125,6 +170,30 @@ paths: get: operationId: hardware_by_revision_retrieve parameters: + - in: query + name: buildLab + schema: + anyOf: + - type: string + - type: 'null' + default: null + title: Buildlab + - in: query + name: buildOrigin + schema: + anyOf: + - type: string + - type: 'null' + default: maestro + title: Buildorigin + - in: query + name: checkoutOrigin + schema: + anyOf: + - type: string + - type: 'null' + default: maestro + title: Checkoutorigin - in: query name: git_commit_hash schema: @@ -149,10 +218,29 @@ paths: - in: query name: origin schema: - default: maestro + anyOf: + - type: string + - type: 'null' + default: null + deprecated: true title: Origin - type: string - description: Origin of the hardware + description: Deprecated alias for checkoutOrigin and buildOrigin + - in: query + name: testLab + schema: + anyOf: + - type: string + - type: 'null' + default: null + title: Testlab + - in: query + name: testOrigin + schema: + anyOf: + - type: string + - type: 'null' + default: null + title: Testorigin - in: query name: tree_name schema: @@ -171,7 +259,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/HardwareListingByRevisionResponse' + $ref: '#/components/schemas/HardwareListingResponse' description: '' /api/hardware/{hardware_id}: post: @@ -399,17 +487,60 @@ paths: schema: $ref: '#/components/schemas/HardwareDetailsTestsResponse' description: '' + /api/hardware/filters/: + get: + operationId: hardware_filters_retrieve + parameters: + - in: query + name: endTimestampInSeconds + schema: + title: Endtimestampinseconds + type: string + description: Interval end timestamp in seconds for the results + required: true + - in: query + name: startTimestampInSeconds + schema: + title: Starttimestampinseconds + type: string + description: Interval start timestamp in seconds for the results + required: true + tags: + - hardware + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/HardwareFiltersResponse' + description: '' /api/hardware/selectors/: get: operationId: hardware_selectors_retrieve parameters: - in: query - name: origin + name: buildOrigin schema: + anyOf: + - type: string + - type: 'null' default: maestro + title: Buildorigin + description: Optional origin of builds that provide revisions + - in: query + name: origin + schema: + anyOf: + - type: string + - type: 'null' + default: null + deprecated: true title: Origin - type: string - description: Origin of the hardware + description: Deprecated alias for buildOrigin tags: - hardware security: @@ -3236,44 +3367,40 @@ components: - tests title: HardwareDetailsTestsResponse type: object - HardwareItem: - properties: - hardware: - anyOf: - - type: string - - items: - type: string - type: array - uniqueItems: true - - type: 'null' - title: Hardware - platform: - title: Platform - type: string - test_status_summary: - $ref: '#/components/schemas/StatusCount' - boot_status_summary: - $ref: '#/components/schemas/StatusCount' - build_status_summary: - $ref: '#/components/schemas/StatusCount' - required: - - hardware - - platform - - test_status_summary - - boot_status_summary - - build_status_summary - title: HardwareItem - type: object - HardwareListingByRevisionResponse: + HardwareFiltersResponse: properties: - hardware: + checkout_origins: items: - $ref: '#/components/schemas/HardwareItem' - title: Hardware + type: string + title: Checkout Origins + type: array + build_origins: + items: + type: string + title: Build Origins + type: array + test_origins: + items: + type: string + title: Test Origins + type: array + build_labs: + items: + type: string + title: Build Labs + type: array + test_labs: + items: + type: string + title: Test Labs type: array required: - - hardware - title: HardwareListingByRevisionResponse + - checkout_origins + - build_origins + - test_origins + - build_labs + - test_labs + title: HardwareFiltersResponse type: object HardwareListingItem: properties: diff --git a/dashboard/e2e/hardware-listing.spec.ts b/dashboard/e2e/hardware-listing.spec.ts index d6c6746a1..1aa6b8940 100644 --- a/dashboard/e2e/hardware-listing.spec.ts +++ b/dashboard/e2e/hardware-listing.spec.ts @@ -1,6 +1,6 @@ import { test, expect, type Locator, type Page } from '@playwright/test'; -import { HARDWARE_LISTING_SELECTORS } from './e2e-selectors'; +import { COMMON_SELECTORS, HARDWARE_LISTING_SELECTORS } from './e2e-selectors'; const SELECTOR_LOAD_TIMEOUT = 15000; const SELECTION_URL_PARAMS = ['t=', 'gu=', 'gb=', 'ch='] as const; @@ -47,6 +47,17 @@ const expectNoSelectionInUrl = (page: Page): void => { } }; +const openFilterDrawer = async (page: Page): Promise => { + await page.getByRole('button', { name: 'Filters', exact: true }).click(); + await expect(page.getByText('Test lab', { exact: true })).toBeVisible({ + timeout: SELECTOR_LOAD_TIMEOUT, + }); +}; + +const applyFilters = async (page: Page): Promise => { + await page.getByRole('button', { name: 'Filter', exact: true }).click(); +}; + test.describe('Hardware Listing Page Tests', () => { test.beforeEach(async ({ page }) => { await page.goto('/hardware'); @@ -167,6 +178,60 @@ test.describe('Hardware Listing Page Tests', () => { await expect(filterLabel).toBeVisible(); }); + test('origin is filtered in the drawer instead of the top bar', async ({ + page, + }) => { + await expect(page.locator(COMMON_SELECTORS.originDropdown)).toBeHidden(); + await openFilterDrawer(page); + await expect( + page + .locator('h3', { hasText: 'Checkout origin' }) + .locator('..') + .getByRole('checkbox', { name: 'maestro', exact: true }), + ).toBeChecked(); + }); + + test('selecting a test lab puts it in the URL', async ({ page }) => { + await openFilterDrawer(page); + + const testLabSection = page + .locator('h3', { hasText: 'Test lab' }) + .locator('..'); + const labCheckbox = testLabSection.getByRole('checkbox').first(); + test.skip( + (await testLabSection.getByRole('checkbox').count()) === 0, + 'Need at least one test lab option', + ); + + const lab = + (await labCheckbox.locator('xpath=..').textContent())?.trim() ?? ''; + await labCheckbox.click(); + await applyFilters(page); + + await expect(page).toHaveURL(/[?&]df=/); + await expect(page).toHaveURL(new RegExp(encodeURIComponent(lab))); + }); + + test('clearing the checkout origin keeps every origin in the URL', async ({ + page, + }) => { + await openFilterDrawer(page); + await page + .locator('h3', { hasText: 'Checkout origin' }) + .locator('..') + .getByRole('checkbox', { name: 'maestro', exact: true }) + .click(); + await applyFilters(page); + + await expect(page).toHaveURL(/[?&]df=/); + await expect( + page + .locator('h3', { hasText: 'Checkout origin' }) + .locator('..') + .getByRole('checkbox', { name: 'maestro', exact: true }), + ).not.toBeChecked(); + }); + test('loading URL with revision does not show filter label', async ({ page, }) => { diff --git a/dashboard/src/api/hardware.ts b/dashboard/src/api/hardware.ts index bc1961912..e0b72cb06 100644 --- a/dashboard/src/api/hardware.ts +++ b/dashboard/src/api/hardware.ts @@ -1,77 +1,73 @@ import type { UseQueryResult } from '@tanstack/react-query'; import { useQuery } from '@tanstack/react-query'; -import { useSearch } from '@tanstack/react-router'; +import type { TFilter } from '@/types/general'; +import { HARDWARE_LISTING_FILTER_SECTIONS } from '@/utils/constants/hardwareListingFilters'; import type { + HardwareFiltersResponse, HardwareListingResponse, HardwareRevisionSelection, HardwareSelectorsResponse, } from '@/types/hardware'; -import type { StatusCount } from '@/types/general'; -import { statusCountToShortStatusCount } from '@/utils/status'; - -import type { HardwareListingRoutesMap } from '@/utils/constants/hardwareListing'; import { RequestData } from './commonRequest'; -type HardwareListingByRevisionApiItem = { - hardware?: string[]; - platform: string; - build_status_summary: StatusCount; - test_status_summary: StatusCount; - boot_status_summary: StatusCount; -}; +const selectedFilterValues = (section?: Record): string => + Object.entries(section ?? {}) + .filter(([, checked]) => checked) + .map(([key]) => key) + .join(','); + +export const hardwareListingParams = ( + diffFilter: TFilter, +): Record => + Object.fromEntries( + HARDWARE_LISTING_FILTER_SECTIONS.map(({ sectionKey, paramKey }) => [ + paramKey, + selectedFilterValues(diffFilter[sectionKey]), + ]), + ); -type HardwareListingByRevisionApiResponse = { - hardware: HardwareListingByRevisionApiItem[]; -}; +export const buildOriginForSelectors = (diffFilter: TFilter): string => + selectedFilterValues(diffFilter.buildOrigin); const fetchHardwareListing = async ( - origin: string, startTimestampInSeconds: number, endTimestampInSeconds: number, + params: Record, commitsList?: string[], ): Promise => { - const data = await RequestData.get( - '/api/hardware/', - { - params: { - startTimestampInSeconds, - endTimestampInSeconds, - origin, - ...(commitsList?.length ? { commitsList: commitsList.join(',') } : {}), - }, + return await RequestData.get('/api/hardware/', { + params: { + startTimestampInSeconds, + endTimestampInSeconds, + ...params, + ...(commitsList?.length ? { commitsList: commitsList.join(',') } : {}), }, - ); - - return data; + }); }; export const useHardwareListing = ( startTimestampInSeconds: number, endTimestampInSeconds: number, - searchFrom: HardwareListingRoutesMap['search'], + params: Record, commitsList?: string[], enabled = true, ): UseQueryResult => { - const { origin } = useSearch({ from: searchFrom }); - - const queryKey = [ - 'hardwareListing', - startTimestampInSeconds, - endTimestampInSeconds, - origin, - commitsList ?? null, - ]; - return useQuery({ - queryKey, + queryKey: [ + 'hardwareListing', + startTimestampInSeconds, + endTimestampInSeconds, + params, + commitsList ?? null, + ], queryFn: () => fetchHardwareListing( - origin, startTimestampInSeconds, endTimestampInSeconds, + params, commitsList, ), enabled, @@ -79,42 +75,60 @@ export const useHardwareListing = ( }); }; +const fetchHardwareFilters = async ( + startTimestampInSeconds: number, + endTimestampInSeconds: number, +): Promise => { + return await RequestData.get( + '/api/hardware/filters/', + { params: { startTimestampInSeconds, endTimestampInSeconds } }, + ); +}; + +export const useHardwareFilters = ( + startTimestampInSeconds: number, + endTimestampInSeconds: number, +): UseQueryResult => { + return useQuery({ + queryKey: [ + 'hardwareFilters', + startTimestampInSeconds, + endTimestampInSeconds, + ], + queryFn: () => + fetchHardwareFilters(startTimestampInSeconds, endTimestampInSeconds), + refetchOnWindowFocus: false, + }); +}; + const fetchHardwareSelectors = async ( - origin: string, + buildOrigin: string, ): Promise => { - const data = await RequestData.get( + return await RequestData.get( '/api/hardware/selectors/', - { - params: { - origin, - }, - }, + { params: { buildOrigin } }, ); - - return data; }; export const useHardwareSelectors = ( - searchFrom: HardwareListingRoutesMap['search'], + buildOrigin: string, ): UseQueryResult => { - const { origin } = useSearch({ from: searchFrom }); - return useQuery({ - queryKey: ['hardwareSelectors', origin], - queryFn: () => fetchHardwareSelectors(origin), + queryKey: ['hardwareSelectors', buildOrigin], + queryFn: () => fetchHardwareSelectors(buildOrigin), refetchOnWindowFocus: false, }); }; const fetchHardwareListingByRevision = async ( selection: HardwareRevisionSelection, - origin: string, + params: Record, ): Promise => { - const data = await RequestData.get( + return await RequestData.get( '/api/hardware-by-revision/', { params: { - origin, + ...params, tree_name: selection.treeName, git_repository_url: selection.gitRepositoryUrl, git_repository_branch: selection.gitBranch, @@ -122,47 +136,19 @@ const fetchHardwareListingByRevision = async ( }, }, ); - - return { - hardware: data.hardware.map(item => ({ - hardware: item.hardware, - platform: item.platform, - build_status_summary: statusCountToShortStatusCount( - item.build_status_summary, - ), - test_status_summary: statusCountToShortStatusCount( - item.test_status_summary, - ), - boot_status_summary: statusCountToShortStatusCount( - item.boot_status_summary, - ), - })), - }; }; export const useHardwareListingByRevision = ( selection: HardwareRevisionSelection | null, - searchFrom: HardwareListingRoutesMap['search'], + params: Record, ): UseQueryResult => { - const { origin } = useSearch({ from: searchFrom }); - - const queryKey = [ - 'hardwareListingByRevision', - origin, - selection?.treeName, - selection?.gitRepositoryUrl, - selection?.gitBranch, - selection?.gitCommitHash, - selection, - ]; - return useQuery({ - queryKey, + queryKey: ['hardwareListingByRevision', params, selection], queryFn: () => { if (selection === null) { return { hardware: [] }; } - return fetchHardwareListingByRevision(selection, origin); + return fetchHardwareListingByRevision(selection, params); }, enabled: Boolean( selection?.treeName && diff --git a/dashboard/src/components/Footer/KcidevFooter.tsx b/dashboard/src/components/Footer/KcidevFooter.tsx index 5485ef39f..54f629f27 100644 --- a/dashboard/src/components/Footer/KcidevFooter.tsx +++ b/dashboard/src/components/Footer/KcidevFooter.tsx @@ -34,7 +34,7 @@ type HardwareDetailsCmdFlags = { type HardwareListingCmdFlags = { cmdName: 'hardware list'; - origin: string; + origin?: string; json: boolean; }; diff --git a/dashboard/src/components/TopBar/TopBar.tsx b/dashboard/src/components/TopBar/TopBar.tsx index 618092f46..7b75d0ee4 100644 --- a/dashboard/src/components/TopBar/TopBar.tsx +++ b/dashboard/src/components/TopBar/TopBar.tsx @@ -147,12 +147,18 @@ const TopBar = (): JSX.Element => { isLabsListing || cleanFullPath.includes('issues'); + const isHardwarePage = cleanFullPath.includes('hardware'); + return { firstUrlLocation, isTreeListing: isTreeListing, - isHardwarePage: cleanFullPath.includes('hardware'), + isHardwarePage, isLabsPage: isLabsListing, isListingPage: isListingPage, + showOriginSelect: + isTreeListing || + isLabsListing || + (isHardwarePage && !isHardwareListing), }; }, [matches]); @@ -173,9 +179,7 @@ const TopBar = (): JSX.Element => { - {(routeInfo.isTreeListing || - routeInfo.isHardwarePage || - routeInfo.isLabsPage) && ( + {routeInfo.showOriginSelect && ( { + const filters: TFilter = {}; + + for (const { sectionKey, optionsKey } of HARDWARE_LISTING_FILTER_SECTIONS) { + const section: Record = {}; + for (const value of data?.[optionsKey] ?? []) { + section[value] = false; + } + + const current = paramFilter[sectionKey]; + if (current && typeof current === 'object') { + for (const [value, checked] of Object.entries(current)) { + if (!(value in section)) { + section[value] = checked; + } + } + } + + filters[sectionKey] = section; + } + + return filters; +}; + +interface HardwareListingFilterProps { + paramFilter: TFilter; + data?: HardwareFiltersResponse; + navigateFrom: HardwareListingRoutesMap['navigate']; +} + +export const HardwareListingFilter = ({ + paramFilter, + data, + navigateFrom, +}: HardwareListingFilterProps): JSX.Element => { + const navigate = useNavigate({ from: navigateFrom }); + const filter = useMemo( + () => createFilter(data, paramFilter), + [data, paramFilter], + ); + const [diffFilter, setDiffFilter] = useState(paramFilter); + + const onFilter = useCallback(() => { + const cleanedFilter = cleanFalseFilters(diffFilter); + navigate({ + search: previousSearch => ({ + ...previousSearch, + diffFilter: cleanedFilter, + }), + state: s => s, + }); + }, [diffFilter, navigate]); + + const resetDraft = useCallback( + () => setDiffFilter(paramFilter), + [paramFilter], + ); + + return ( + + + + ); +}; diff --git a/dashboard/src/pages/Hardware/HardwareListingPage.tsx b/dashboard/src/pages/Hardware/HardwareListingPage.tsx index f5dd5af79..554b9398d 100644 --- a/dashboard/src/pages/Hardware/HardwareListingPage.tsx +++ b/dashboard/src/pages/Hardware/HardwareListingPage.tsx @@ -9,6 +9,9 @@ import { Toaster } from '@/components/ui/toaster'; import type { HardwareItem, HardwareRevisionSelection } from '@/types/hardware'; import { + buildOriginForSelectors, + hardwareListingParams, + useHardwareFilters, useHardwareListing, useHardwareListingByRevision, useHardwareSelectors, @@ -25,6 +28,7 @@ import { REDUCED_TIME_SEARCH } from '@/utils/constants/general'; import type { HardwareListingRoutesMap } from '@/utils/constants/hardwareListing'; import type { SearchIntent } from '@/lib/intent'; +import { HardwareListingFilter } from './HardwareListingFilter'; import { HardwareTable } from './HardwareTable'; import { decodeBranchValue, @@ -47,12 +51,12 @@ const HardwareListingPage = ({ }: HardwareListingPageProps): JSX.Element => { const navigate = useNavigate({ from: urlFromMap.navigate }); const { - origin, intervalInDays, treeName, gitRepositoryUrl, gitBranch, gitCommitHash, + diffFilter, } = useSearch({ from: urlFromMap.search }); const inputFilter = intent.search; const intentCommits = @@ -79,9 +83,10 @@ const HardwareListingPage = ({ treeName || gitRepositoryUrl || gitBranch || gitCommitHash, ); - const { data: selectorsData, status: selectorsStatus } = useHardwareSelectors( - urlFromMap.search, - ); + const buildOrigin = buildOriginForSelectors(diffFilter); + + const { data: selectorsData, status: selectorsStatus } = + useHardwareSelectors(buildOrigin); const trees = useMemo(() => selectorsData?.trees ?? [], [selectorsData]); @@ -154,17 +159,24 @@ const HardwareListingPage = ({ return getBranchBySelection(selectedTree, gitRepositoryUrl, gitBranch); }, [selectedTree, gitRepositoryUrl, gitBranch]); + const filters = useMemo( + () => hardwareListingParams(diffFilter), + [diffFilter], + ); + + const { data: filterOptions } = useHardwareFilters( + startTimestampInSeconds, + endTimestampInSeconds, + ); + const defaultListing = useHardwareListing( startTimestampInSeconds, endTimestampInSeconds, - urlFromMap.search, + filters, intentCommits, !hasSelection, ); - const revisionListing = useHardwareListingByRevision( - selection, - urlFromMap.search, - ); + const revisionListing = useHardwareListingByRevision(selection, filters); const activeListing = hasSelection ? revisionListing : defaultListing; const listItems: HardwareItem[] = useMemo(() => { @@ -180,15 +192,7 @@ const HardwareListingPage = ({ includesInAnStringOrStringArray(hardware.hardware ?? '', inputFilter) ); }) - .map((hardware): HardwareItem => { - return { - hardware: hardware.hardware, - platform: hardware.platform, - build_status_summary: hardware.build_status_summary, - test_status_summary: hardware.test_status_summary, - boot_status_summary: hardware.boot_status_summary, - }; - }); + .sort((a, b) => a.platform.localeCompare(b.platform)); }, [activeListing.data, activeListing.error, inputFilter]); const selectedRevision = @@ -215,10 +219,14 @@ const HardwareListingPage = ({ () => ( ), - [origin], + [buildOrigin], ); const onTreeChange = ({ @@ -300,12 +308,19 @@ const HardwareListingPage = ({ <>
- - }} +
+ + }} + /> + + - +
), z.record(z.never()), ]) diff --git a/dashboard/src/types/hardware.ts b/dashboard/src/types/hardware.ts index 6da6b0bd7..86fc346dc 100644 --- a/dashboard/src/types/hardware.ts +++ b/dashboard/src/types/hardware.ts @@ -8,16 +8,8 @@ export type HardwareItem = { boot_status_summary: ShortStatusCount; }; -export type HardwareListingApiItem = { - hardware?: string[]; - platform: string; - build_status_summary: ShortStatusCount; - test_status_summary: ShortStatusCount; - boot_status_summary: ShortStatusCount; -}; - export interface HardwareListingResponse { - hardware: HardwareListingApiItem[]; + hardware: HardwareItem[]; } export type HardwareSelectorRevision = { @@ -47,3 +39,11 @@ export type HardwareRevisionSelection = { gitBranch: string; gitCommitHash: string; }; + +export interface HardwareFiltersResponse { + checkout_origins: string[]; + build_origins: string[]; + test_origins: string[]; + build_labs: string[]; + test_labs: string[]; +} diff --git a/dashboard/src/utils/constants/hardwareListingFilters.ts b/dashboard/src/utils/constants/hardwareListingFilters.ts new file mode 100644 index 000000000..a700990be --- /dev/null +++ b/dashboard/src/utils/constants/hardwareListingFilters.ts @@ -0,0 +1,54 @@ +import type { ISectionItem } from '@/components/Filter/CheckboxSection'; +import type { TFilterObjectsKeys } from '@/types/general'; +import type { HardwareFiltersResponse } from '@/types/hardware'; + +export const HARDWARE_LISTING_FILTER_SECTIONS = [ + { + sectionKey: 'testLabs', + optionsKey: 'test_labs', + paramKey: 'testLab', + title: 'hardwareFilter.testLab', + subtitle: 'hardwareFilter.testLabSubtitle', + }, + { + sectionKey: 'checkoutOrigins', + optionsKey: 'checkout_origins', + paramKey: 'checkoutOrigin', + title: 'hardwareFilter.checkoutOrigin', + subtitle: 'hardwareFilter.checkoutOriginSubtitle', + }, + { + sectionKey: 'buildOrigin', + optionsKey: 'build_origins', + paramKey: 'buildOrigin', + title: 'hardwareFilter.buildOrigin', + subtitle: 'hardwareFilter.buildOriginSubtitle', + }, + { + sectionKey: 'buildLabs', + optionsKey: 'build_labs', + paramKey: 'buildLab', + title: 'hardwareFilter.buildLab', + subtitle: 'hardwareFilter.buildLabSubtitle', + }, + { + sectionKey: 'testOrigin', + optionsKey: 'test_origins', + paramKey: 'testOrigin', + title: 'hardwareFilter.testOrigin', + subtitle: 'hardwareFilter.testOriginSubtitle', + }, +] as const satisfies ReadonlyArray< + ISectionItem & { + sectionKey: TFilterObjectsKeys; + optionsKey: keyof HardwareFiltersResponse; + paramKey: string; + } +>; + +export const hardwareListingFilterSections: ISectionItem[] = + HARDWARE_LISTING_FILTER_SECTIONS.map(({ title, subtitle, sectionKey }) => ({ + title, + subtitle, + sectionKey, + })); diff --git a/dashboard/src/utils/search.test.ts b/dashboard/src/utils/search.test.ts index ffc0eb0a3..b39a22412 100644 --- a/dashboard/src/utils/search.test.ts +++ b/dashboard/src/utils/search.test.ts @@ -307,6 +307,20 @@ describe('parseSearch', () => { }); }); +describe('hardware listing filters', () => { + const filters = { + diffFilter: { + checkoutOrigins: { maestro: true }, + testOrigin: { ti: true }, + testLabs: { 'lava-broonie': true }, + }, + }; + + it('keeps selected filters in the URL', () => { + expect(parseSearch(stringifySearch(filters))).toStrictEqual(filters); + }); +}); + describe('stringifySearch', () => { const assertSearchParams = ( result: string, diff --git a/dashboard/src/utils/search.ts b/dashboard/src/utils/search.ts index bd2ae4741..3bf37a6a2 100644 --- a/dashboard/src/utils/search.ts +++ b/dashboard/src/utils/search.ts @@ -201,6 +201,9 @@ const diffFilterMinifiedParams: Record = { buildOrigin: 'buo', bootOrigin: 'boo', testOrigin: 'to', + checkoutOrigins: 'co', + buildLabs: 'bl', + testLabs: 'tl', } as const satisfies Record; type MinifiedParams = Record<