feat/hardware improved filters

backend/docs/prune_db command.md

@@ -15,9 +15,10 @@ Models use `DO_NOTHING` foreign keys, so the command applies manual cascade rule
 
 ### Optional Parameters
 
-- `--tables`: Limit pruning to specific tables (comma-separated). Valid options: `checkouts`, `builds`, `tests`. Default: all three.
+- `--tables`: Limit pruning to specific tables (comma-separated). Valid options: `checkouts`, `builds`, `tests`, `hardware_daily_builds`, `hardware_daily_tests`. Default: all.
   - Cascade only drags a child when the child's parent table is also selected. For example, `--tables tests` removes only tests past the cutoff; recent tests under an old build/checkout are kept because those parents are not being pruned. With `--tables builds,tests`, an old build still drags its recent tests, but an old checkout does not drag its recent builds (checkouts are not selected).
   - Tables not listed are not deleted. For example, `--tables builds` removes old builds but leaves their tests in place. Selecting a parent without its children (e.g. only `checkouts`) can therefore leave orphaned rows.
+  - `hardware_daily_builds` and `hardware_daily_tests` are independent of the raw cascade: they are pruned by their own `checkout_day` and can be targeted on their own (e.g. `--tables hardware_daily_builds,hardware_daily_tests`) to trim only the aggregates.
 - `--origins`: Limit age-based pruning to specific origins (comma-separated). If omitted, any origin is considered.
   - Cascade ignores origin: once a parent row is doomed, its children are removed even if they belong to a different origin.
 - `--batch-size`: Number of rows deleted per batch (default: `10000`). Must be at least `1`.
@@ -51,6 +52,12 @@ python manage.py prune_db --older-than "30 days" --origins maestro,0dayci --dry-
 python manage.py prune_db --older-than "30 days" --tables tests --yes
 ```
 
+### Prune only the hardware daily aggregates
+
+```bash
+python manage.py prune_db --older-than "60 days" --tables hardware_daily_builds,hardware_daily_tests --yes
+```
+
 ### Prune rows linked to issues (override default protection)
 
 ```bash
@@ -59,7 +66,11 @@ python manage.py prune_db --older-than "30 days" --skip-issue-protection --yes
 
 ## What Is Not Deleted
 
-The command only touches `checkouts`, `builds`, and `tests`. Related tables are left as-is, including:
+The command touches `checkouts`, `builds`, `tests`, and the hardware daily aggregates
+`hardware_daily_builds` / `hardware_daily_tests`. The aggregates are pruned by their
+own `checkout_day` (same age window as `--older-than`), because a daily summary is a
+coarser fact table retained by date range, not by which raw checkouts survive; use
+`--tables` to select any subset. Other related tables are left as-is, including:
 
 - `incidents` rows themselves (only used to decide which builds/tests/checkouts to keep)
 - `hardware_status`, `latest_checkout`, `tree_tests_rollup` (reference checkouts)

backend/kernelCI/settings.py

@@ -252,6 +252,22 @@ def get_json_env_var(name, default):
                 f"--index={HARDWARE_REGISTRY_INDEX_URL}",
             ],
         ),
+        (
+            "0 */6 * * *",
+            "django.core.management.call_command",
+            [
+                "recompute_hardware_daily",
+                "--days=15",
+            ],
+        ),
+        (
+            "50 * * * *",
+            "django.core.management.call_command",
+            [
+                "recompute_hardware_daily",
+                "--days=1",
+            ],
+        ),
     ]
 
 # Email settings for SMTP backend

backend/kernelCI_app/constants/localization.py

@@ -101,7 +101,10 @@ class DocStrings:
         "Dictionary mapping tree names to selected commit hashes"
     )
 
-    HARDWARE_LISTING_ORIGIN_DESCRIPTION = "Origin of the hardware"
+    HARDWARE_LISTING_ORIGIN_DESCRIPTION = (
+        "Origin of the checkout the hardware was tested from. "
+        "Pass it empty to list every origin."
+    )
     HARDWARE_LISTING_COMMITS_LIST_DESCRIPTION = (
         "Optional comma-separated git commit identifiers: full SHA(s) "
         "and/or tag strings that appear in checkout.git_commit_tags."

backend/kernelCI_app/management/commands/prune_db.py

@@ -9,18 +9,18 @@
 Rows linked to an incident (an issue) are kept by default, together with their
 ancestors so nothing is orphaned; pass --skip-issue-protection to prune them too.
 
-Only checkouts, builds and tests are touched. Aggregate and derived tables (e.g.
-tree_tests_rollup, hardware_status, latest_checkout) are left untouched and must
-be cleaned up separately.
+Also prunes hardware_daily_* by checkout_day (same age). Other derived tables
+(tree_tests_rollup, hardware_status, latest_checkout) are left untouched.
 """
 
 from django.core.management.base import BaseCommand, CommandError
 from django.db import connections
 
 from kernelCI_app.management.commands.helpers.intervals import parse_interval
 
-# Strict parent-before-child order: a checkout owns builds, a build owns tests.
 PRUNABLE_TABLES = ("checkouts", "builds", "tests")
+HARDWARE_DAILY_TABLES = ("hardware_daily_builds", "hardware_daily_tests")
+VALID_TABLES = PRUNABLE_TABLES + HARDWARE_DAILY_TABLES
 
 
 class Command(BaseCommand):
@@ -61,11 +61,12 @@ def add_arguments(self, parser):
         parser.add_argument(
             "--tables",
             type=lambda s: [t.strip() for t in s.split(",")],
-            default=list(PRUNABLE_TABLES),
+            default=list(VALID_TABLES),
             help="Limit pruning to specific tables (comma-separated: "
-            f"{', '.join(PRUNABLE_TABLES)}). Only the listed tables are deleted; "
-            "unlisted child tables are left untouched, so selecting a parent without "
-            "its children (e.g. only 'checkouts') can leave orphans. Default: all.",
+            f"{', '.join(VALID_TABLES)}). Only listed tables are deleted; a parent "
+            "without its children (e.g. only 'checkouts') can leave orphans. "
+            "hardware_daily_* prune by checkout_day and can be targeted alone. "
+            "Default: all.",
         )
         parser.add_argument(
             "--skip-issue-protection",
@@ -87,13 +88,16 @@ def handle(self, *args, **options):
                 "positive number."
             )
 
-        unknown_tables = [t for t in options["tables"] if t not in PRUNABLE_TABLES]
+        unknown_tables = [t for t in options["tables"] if t not in VALID_TABLES]
         if unknown_tables:
             raise CommandError(
                 f"Unknown table(s): {', '.join(unknown_tables)}. "
-                f"Valid options are: {', '.join(PRUNABLE_TABLES)}."
+                f"Valid options are: {', '.join(VALID_TABLES)}."
             )
         selected_tables = [t for t in PRUNABLE_TABLES if t in options["tables"]]
+        selected_daily_tables = [
+            t for t in HARDWARE_DAILY_TABLES if t in options["tables"]
+        ]
         protect_incidents = not options["skip_issue_protection"]
 
         dry_run = options["dry_run"]
@@ -122,16 +126,28 @@ def handle(self, *args, **options):
                 counts = {
                     t: self._count(cursor, temp_tables[t]) for t in selected_tables
                 }
-                total = sum(counts.values())
+                daily_counts = {
+                    t: self._count_hardware_daily(cursor, t, cutoff.date())
+                    for t in selected_daily_tables
+                }
+                total = sum(counts.values()) + sum(daily_counts.values())
 
                 lines = [f"Rows older than {cutoff.isoformat()}:"]
                 lines += [f"* {t}:\t{counts[t]:>8}" for t in selected_tables]
+                lines += [
+                    f"* {t}:\t{daily_counts[t]:>8}" for t in selected_daily_tables
+                ]
                 lines += ["----------------------", f"* total:\t{total:>8}"]
-                lines.append(
-                    "Note: counts include children cascaded from pruned parents."
-                )
-                if protect_incidents:
+                if selected_tables:
+                    lines.append(
+                        "Note: counts include children cascaded from pruned parents."
+                    )
+                if protect_incidents and selected_tables:
                     lines.append("Note: rows linked to an incident are kept.")
+                if daily_counts:
+                    lines.append(
+                        "Note: hardware daily rows are pruned by their own checkout_day."
+                    )
                 self.stdout.write("\n".join(lines))
 
                 if total == 0:
@@ -156,11 +172,13 @@ def handle(self, *args, **options):
                         self.stdout.write("Aborted.")
                         return
 
-                # Delete child-first (reverse of PRUNABLE_TABLES order): each batch
-                # commits on its own, so a crash mid-run leaves children already gone
-                # before their parents, never the reverse. Reordering this would risk
-                # orphans.
                 deleted = 0
+                for table in selected_daily_tables:
+                    deleted += self._batch_delete_hardware_daily(
+                        cursor, table, cutoff.date(), options["batch_size"]
+                    )
+
+                # Child-first: a crash must not leave orphans.
                 for table in reversed(selected_tables):
                     deleted += self._batch_delete(
                         cursor, table, temp_tables[table], options["batch_size"]
@@ -254,3 +272,26 @@ def _batch_delete(self, cursor, table, temp_table, batch_size):
             deleted_total += deleted
             self.stdout.write(f"Deleted {table}(n={deleted}) total={deleted_total}")
         return deleted_total
+
+    def _count_hardware_daily(self, cursor, table, cutoff_day):
+        cursor.execute(
+            f'SELECT COUNT(*) FROM "{table}" WHERE checkout_day < %(cutoff_day)s',
+            {"cutoff_day": cutoff_day},
+        )
+        return cursor.fetchone()[0]
+
+    def _batch_delete_hardware_daily(self, cursor, table, cutoff_day, batch_size):
+        sql = (
+            f'DELETE FROM "{table}" WHERE ctid IN ('
+            f'SELECT ctid FROM "{table}" WHERE checkout_day < %(cutoff_day)s '
+            f"LIMIT %(batch_size)s)"
+        )
+        deleted_total = 0
+        while True:
+            cursor.execute(sql, {"cutoff_day": cutoff_day, "batch_size": batch_size})
+            deleted = cursor.rowcount
+            if deleted == 0:
+                break
+            deleted_total += deleted
+            self.stdout.write(f"Deleted {table}(n={deleted}) total={deleted_total}")
+        return deleted_total