Skip to content

Commit bba74aa

Browse files
alanpeixinhoalanpeixinho
committed
fix(hardware): prune hardware_status by time window instead of latest_checkout
The listing/details now surface boards tested on any checkout within the window, not just the latest tip. But delete_unused_hardware_status still removed every hardware_status row not present in latest_checkout, so the weekly cron would wipe those older-checkout rows and regress #1983. Prune hardware_status (and ProcessedListingItems) by age older than HARDWARE_STATUS_RETENTION_DAYS instead. Delete hardware rows by their composite key and start_time, not by checkout_id alone, so an old platform row cannot wipe an in-window sibling. Both tables use the same cutoff so the dedup ledger never predeceases its status rows. Integration tests pin the non-tip retention guard, the shared cutoff, and the sibling-row delete behavior. Signed-off-by: Alan Peixinho <alan.peixinho@profusion.mobi>
1 parent 013376e commit bba74aa

3 files changed

Lines changed: 266 additions & 67 deletions

File tree

‎backend/kernelCI/settings.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -428,6 +428,11 @@ def get_json_env_var(name, default):
428428
os.environ.get("DEFAULT_ORIGIN_LISTING_INTERVAL_IN_DAYS", 30)
429429
)
430430

431+
# How long hardware_status and processed_listing_items rows are kept.
432+
HARDWARE_STATUS_RETENTION_DAYS = int(
433+
os.environ.get("HARDWARE_STATUS_RETENTION_DAYS", 30)
434+
)
435+
431436
PROMETHEUS_METRICS_ENABLED = is_boolean_or_string_true(
432437
os.environ.get("PROMETHEUS_METRICS_ENABLED", False)
433438
)

‎backend/kernelCI_app/management/commands/delete_unused_hardware_status.py‎

Lines changed: 66 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,28 @@
11
"""
2-
Management command to delete unused entries from hardware_status table.
3-
4-
Removes HardwareStatus entries that have no corresponding checkout_id in the LatestCheckout table.
2+
Prune HardwareStatus and ProcessedListingItems older than
3+
HARDWARE_STATUS_RETENTION_DAYS. Both use the same cutoff so already-processed
4+
entries stay in sync with status rows and we avoid over/undercounting on
5+
re-ingest.
56
"""
67

7-
import logging
8+
from datetime import timedelta
89

10+
from django.conf import settings
911
from django.core.management.base import BaseCommand
1012
from django.db import transaction
13+
from django.utils import timezone
1114

1215
from kernelCI_app.management.commands.helpers.healthcheck import (
1316
MONITORING_ID_PARAM_HELP_TEXT,
1417
run_with_healthcheck_monitoring,
1518
)
16-
from kernelCI_app.models import HardwareStatus, LatestCheckout, ProcessedListingItems
17-
18-
logger = logging.getLogger(__name__)
19+
from kernelCI_app.models import Checkouts, HardwareStatus, ProcessedListingItems
1920

2021

2122
class Command(BaseCommand):
2223
help = (
23-
"Delete HardwareStatus entries with no corresponding checkout_id "
24-
"in the LatestCheckout table"
24+
"Delete HardwareStatus entries (and their ProcessedListingItems) older "
25+
"than HARDWARE_STATUS_RETENTION_DAYS"
2526
)
2627

2728
def add_arguments(self, parser):
@@ -54,88 +55,86 @@ def _run_action(self, options):
5455
dry_run = options["dry_run"]
5556
batch_size = options["batch_size"]
5657

57-
with transaction.atomic():
58-
valid_checkout_ids = set(
59-
LatestCheckout.objects.values_list("checkout_id", flat=True)
60-
)
61-
62-
orphaned_hardware_entries = HardwareStatus.objects.exclude(
63-
checkout_id__in=valid_checkout_ids
64-
).values_list("checkout_id", flat=True)
65-
orphaned_hardware_count = orphaned_hardware_entries.count()
58+
cutoff = timezone.now() - timedelta(
59+
days=settings.HARDWARE_STATUS_RETENTION_DAYS
60+
)
6661

67-
orphaned_processed_hardware_entries = (
68-
ProcessedListingItems.objects.exclude(
69-
checkout_id__in=valid_checkout_ids
70-
)
71-
).values_list("listing_item_key", flat=True)
62+
stale_hardware = HardwareStatus.objects.filter(start_time__lt=cutoff)
63+
recent_checkout_ids = Checkouts.objects.filter(start_time__gte=cutoff).values(
64+
"id"
65+
)
66+
stale_processed = ProcessedListingItems.objects.exclude(
67+
checkout_id__in=recent_checkout_ids
68+
)
7269

73-
orphaned_processed_hardware_count = (
74-
orphaned_processed_hardware_entries.count()
75-
)
70+
stale_hardware_count = stale_hardware.count()
71+
stale_processed_count = stale_processed.count()
7672

77-
if orphaned_hardware_count == 0 and orphaned_processed_hardware_count == 0:
78-
self.stdout.write(
79-
self.style.SUCCESS(
80-
"No orphaned HardwareStatus/ProcessedListingItems entries found."
81-
)
82-
)
83-
return
84-
85-
if dry_run:
86-
self.stdout.write(
87-
self.style.WARNING(
88-
f"[DRY RUN] Would delete {orphaned_hardware_count} HardwareStatus entries and "
89-
f"{orphaned_processed_hardware_count} ProcessedListingItems entries "
90-
"Run without --dry-run to execute deletion."
91-
)
73+
if stale_hardware_count == 0 and stale_processed_count == 0:
74+
self.stdout.write(
75+
self.style.SUCCESS(
76+
"No orphaned HardwareStatus/ProcessedListingItems entries found."
9277
)
93-
return
78+
)
79+
return
9480

81+
if dry_run:
9582
self.stdout.write(
96-
f"Found {orphaned_hardware_count} HardwareStatus entries "
97-
f"and {orphaned_processed_hardware_count} ProcessedListingItems entries "
98-
"with no corresponding LatestCheckout."
83+
self.style.WARNING(
84+
f"[DRY RUN] Would delete {stale_hardware_count} HardwareStatus entries and "
85+
f"{stale_processed_count} ProcessedListingItems entries "
86+
"Run without --dry-run to execute deletion."
87+
)
9988
)
89+
return
10090

101-
total_hardware_deleted = 0
102-
total_processed_hardware_deleted = 0
91+
self.stdout.write(
92+
f"Found {stale_hardware_count} HardwareStatus entries "
93+
f"and {stale_processed_count} ProcessedListingItems entries "
94+
f"older than {settings.HARDWARE_STATUS_RETENTION_DAYS} days."
95+
)
96+
97+
total_hardware_deleted = 0
98+
total_processed_deleted = 0
99+
with transaction.atomic():
103100
while True:
104-
hardware_batch_ids = list(orphaned_hardware_entries[:batch_size])
105-
processed_hardware_batch_ids = list(
106-
orphaned_processed_hardware_entries[:batch_size]
101+
hardware_batch = list(
102+
stale_hardware.values_list(
103+
"test_origin", "platform", "checkout_id"
104+
)[:batch_size]
105+
)
106+
processed_batch = list(
107+
stale_processed.values_list("listing_item_key", flat=True)[
108+
:batch_size
109+
]
107110
)
108111

109-
if not hardware_batch_ids and not processed_hardware_batch_ids:
112+
if not hardware_batch and not processed_batch:
110113
break
111114

112-
if hardware_batch_ids:
115+
if hardware_batch:
113116
hardware_delete_count = HardwareStatus.objects.filter(
114-
checkout_id__in=hardware_batch_ids
117+
pk__in=hardware_batch
115118
).delete()[0]
119+
total_hardware_deleted += hardware_delete_count
116120
self.stdout.write(
117121
f"Deleted hardware_status(n={hardware_delete_count}) entries "
118-
f"(total: {total_hardware_deleted}/{orphaned_hardware_count})"
119-
)
120-
total_hardware_deleted += hardware_delete_count
121-
122-
if processed_hardware_batch_ids:
123-
processed_hardware_delete_count = (
124-
ProcessedListingItems.objects.filter(
125-
listing_item_key__in=processed_hardware_batch_ids
126-
).delete()[0]
122+
f"(total: {total_hardware_deleted}/{stale_hardware_count})"
127123
)
128124

129-
total_processed_hardware_deleted += processed_hardware_delete_count
130-
125+
if processed_batch:
126+
processed_delete_count = ProcessedListingItems.objects.filter(
127+
listing_item_key__in=processed_batch
128+
).delete()[0]
129+
total_processed_deleted += processed_delete_count
131130
self.stdout.write(
132-
f"Deleted processed_hardware_status(n={processed_hardware_delete_count}) entries "
133-
f"(total: {total_processed_hardware_deleted}/{orphaned_processed_hardware_count})"
131+
f"Deleted processed_listing_items(n={processed_delete_count}) entries "
132+
f"(total: {total_processed_deleted}/{stale_processed_count})"
134133
)
135134

136135
self.stdout.write(
137136
self.style.SUCCESS(
138137
f"Successfully deleted hardware_status(n={total_hardware_deleted}) "
139-
f"and processed_hardware_status(n={total_processed_hardware_deleted})."
138+
f"and processed_listing_items(n={total_processed_deleted})."
140139
)
141140
)

‎backend/kernelCI_app/tests/integrationTests/delete_unused_hardware_status_test.py‎

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
"""Integration tests for delete_unused_hardware_status retention.
2+
3+
Pins the #1983 regression: hardware_status rows for non-tip checkouts inside the
4+
retention window must survive the weekly cron. Also pins that HardwareStatus and
5+
ProcessedListingItems share the same age cutoff.
6+
"""
7+
8+
from io import StringIO
9+
10+
import pytest
11+
from django.core.management import call_command
12+
from django.test import override_settings
13+
from django.utils import timezone
14+
15+
from kernelCI_app.models import (
16+
HardwareStatus,
17+
LatestCheckout,
18+
ProcessedListingItems,
19+
)
20+
from kernelCI_app.tests.factories import CheckoutFactory
21+
22+
RETENTION_DAYS = 7
23+
24+
25+
def _days_ago(days: int):
26+
return timezone.now() - timezone.timedelta(days=days)
27+
28+
29+
def _run_delete(**kwargs) -> str:
30+
out = StringIO()
31+
err = StringIO()
32+
call_command("delete_unused_hardware_status", stdout=out, stderr=err, **kwargs)
33+
return out.getvalue() + err.getvalue()
34+
35+
36+
def _make_hardware_status(
37+
*, checkout, platform: str, start_time, test_origin="maestro"
38+
):
39+
return HardwareStatus.objects.create(
40+
checkout_id=checkout.id,
41+
test_origin=test_origin,
42+
platform=platform,
43+
compatibles=None,
44+
start_time=start_time,
45+
test_pass=1,
46+
)
47+
48+
49+
def _make_processed(*, checkout_id: str, key_byte: int):
50+
return ProcessedListingItems.objects.create(
51+
listing_item_key=bytes([key_byte]) * 32,
52+
checkout_id=checkout_id,
53+
status="P",
54+
)
55+
56+
57+
@pytest.mark.django_db
58+
@override_settings(HARDWARE_STATUS_RETENTION_DAYS=RETENTION_DAYS)
59+
def test_keeps_non_tip_hardware_within_window():
60+
"""#1983: board tested on an older (non-tip) checkout must not be wiped.
61+
62+
Tip checkout is in latest_checkout and has no board. Older checkout is not a
63+
tip but is inside the retention window and has the board. Old tip-based cron
64+
deleted that row; time-window retention must keep it.
65+
"""
66+
tip = CheckoutFactory(start_time=_days_ago(1), id="ret_tip_checkout")
67+
older = CheckoutFactory(start_time=_days_ago(3), id="ret_older_checkout")
68+
69+
LatestCheckout.objects.create(
70+
checkout_id=tip.id,
71+
start_time=tip.start_time,
72+
origin=tip.origin,
73+
tree_name=tip.tree_name,
74+
git_repository_url=tip.git_repository_url,
75+
git_repository_branch=tip.git_repository_branch,
76+
)
77+
78+
board = _make_hardware_status(
79+
checkout=older, platform="exynos", start_time=older.start_time
80+
)
81+
82+
_run_delete()
83+
84+
assert HardwareStatus.objects.filter(
85+
test_origin=board.test_origin,
86+
platform=board.platform,
87+
checkout_id=board.checkout_id,
88+
).exists()
89+
90+
91+
@pytest.mark.django_db
92+
@override_settings(HARDWARE_STATUS_RETENTION_DAYS=RETENTION_DAYS)
93+
def test_deletes_hardware_older_than_retention():
94+
old = CheckoutFactory(start_time=_days_ago(RETENTION_DAYS + 5), id="ret_old_hw")
95+
board = _make_hardware_status(
96+
checkout=old, platform="old-board", start_time=old.start_time
97+
)
98+
99+
_run_delete()
100+
101+
assert not HardwareStatus.objects.filter(
102+
test_origin=board.test_origin,
103+
platform=board.platform,
104+
checkout_id=board.checkout_id,
105+
).exists()
106+
107+
108+
@pytest.mark.django_db
109+
@override_settings(HARDWARE_STATUS_RETENTION_DAYS=RETENTION_DAYS)
110+
def test_processed_items_match_hardware_cutoff():
111+
"""ProcessedListingItems and hardware_status use the same age cutoff."""
112+
recent = CheckoutFactory(start_time=_days_ago(2), id="ret_recent_processed")
113+
old = CheckoutFactory(
114+
start_time=_days_ago(RETENTION_DAYS + 5), id="ret_old_processed"
115+
)
116+
117+
recent_hw = _make_hardware_status(
118+
checkout=recent, platform="recent-board", start_time=recent.start_time
119+
)
120+
old_hw = _make_hardware_status(
121+
checkout=old, platform="old-board", start_time=old.start_time
122+
)
123+
recent_processed = _make_processed(checkout_id=recent.id, key_byte=1)
124+
old_processed = _make_processed(checkout_id=old.id, key_byte=2)
125+
126+
_run_delete()
127+
128+
assert HardwareStatus.objects.filter(
129+
test_origin=recent_hw.test_origin,
130+
platform=recent_hw.platform,
131+
checkout_id=recent_hw.checkout_id,
132+
).exists()
133+
assert not HardwareStatus.objects.filter(
134+
test_origin=old_hw.test_origin,
135+
platform=old_hw.platform,
136+
checkout_id=old_hw.checkout_id,
137+
).exists()
138+
assert ProcessedListingItems.objects.filter(
139+
listing_item_key=recent_processed.listing_item_key
140+
).exists()
141+
assert not ProcessedListingItems.objects.filter(
142+
listing_item_key=old_processed.listing_item_key
143+
).exists()
144+
145+
146+
@pytest.mark.django_db
147+
@override_settings(HARDWARE_STATUS_RETENTION_DAYS=RETENTION_DAYS)
148+
def test_does_not_wipe_recent_sibling_platform_rows():
149+
"""Deleting by checkout_id alone would wipe in-window siblings. Must not."""
150+
checkout = CheckoutFactory(start_time=_days_ago(2), id="ret_sibling_checkout")
151+
stale = _make_hardware_status(
152+
checkout=checkout,
153+
platform="stale-board",
154+
start_time=_days_ago(RETENTION_DAYS + 5),
155+
)
156+
fresh = _make_hardware_status(
157+
checkout=checkout,
158+
platform="fresh-board",
159+
start_time=checkout.start_time,
160+
)
161+
162+
_run_delete()
163+
164+
assert not HardwareStatus.objects.filter(
165+
test_origin=stale.test_origin,
166+
platform=stale.platform,
167+
checkout_id=stale.checkout_id,
168+
).exists()
169+
assert HardwareStatus.objects.filter(
170+
test_origin=fresh.test_origin,
171+
platform=fresh.platform,
172+
checkout_id=fresh.checkout_id,
173+
).exists()
174+
175+
176+
@pytest.mark.django_db
177+
@override_settings(HARDWARE_STATUS_RETENTION_DAYS=RETENTION_DAYS)
178+
def test_dry_run_deletes_nothing():
179+
old = CheckoutFactory(start_time=_days_ago(RETENTION_DAYS + 5), id="ret_dry_run")
180+
board = _make_hardware_status(
181+
checkout=old, platform="dry-board", start_time=old.start_time
182+
)
183+
processed = _make_processed(checkout_id=old.id, key_byte=3)
184+
185+
output = _run_delete(dry_run=True)
186+
187+
assert "DRY RUN" in output
188+
assert HardwareStatus.objects.filter(
189+
test_origin=board.test_origin,
190+
platform=board.platform,
191+
checkout_id=board.checkout_id,
192+
).exists()
193+
assert ProcessedListingItems.objects.filter(
194+
listing_item_key=processed.listing_item_key
195+
).exists()

0 commit comments

Comments
 (0)