From 7831f674d004342daae9c5bf513d72ed76e32624 Mon Sep 17 00:00:00 2001 From: Brigs Date: Sun, 27 Sep 2026 09:49:37 -0400 Subject: [PATCH] Add a Reminders artifact for macOS Reads the Reminders Core Data stores in each home folder, under Library/Reminders/Container_v1/Stores (older macOS) and the group.com.apple.reminders container (newer macOS). One row per reminder with title, notes, list, account, creation, modification, due and completion dates, flag, priority, parent reminder, the last banner presentation time and the ModifiedByDevice value of the reminder's CloudKit record. In the older layout reminders share ZREMCDOBJECT with other entities and some property names get numbered columns (ZTITLE1), so the column for each property is worked out from the model the store caches in Z_MODELCACHE. An all-day due date is reported as a date, and a due time only when the store records its time zone. Co-Authored-By: Claude Opus 5.5 --- admin/test/scripts/test_macos_reminders.py | 317 +++++++++++++++++ scripts/artifacts/macosReminders.py | 392 +++++++++++++++++++++ 2 files changed, 709 insertions(+) create mode 100644 admin/test/scripts/test_macos_reminders.py create mode 100644 scripts/artifacts/macosReminders.py diff --git a/admin/test/scripts/test_macos_reminders.py b/admin/test/scripts/test_macos_reminders.py new file mode 100644 index 0000000..889b174 --- /dev/null +++ b/admin/test/scripts/test_macos_reminders.py @@ -0,0 +1,317 @@ +"""Pin the Reminders artifact in scripts/artifacts/macosReminders.py. + +Every store, cached model and CloudKit record below is built by the test; no row comes from a +real device. The models mirror the two layouts the artifact reads: reminders in their own +ZREMCDREMINDER table, and reminders as rows of ZREMCDOBJECT beside other entities, where a +property name several entities define is stored in numbered columns. Expected values are +literals. +""" +import fnmatch +import os +import pathlib +import plistlib +import shutil +import sqlite3 +import sys +import tempfile +import unittest +import zlib +from datetime import datetime, timezone +from unittest.mock import patch + +REPO_ROOT = pathlib.Path(__file__).resolve().parents[3] +sys.path.insert(0, str(REPO_ROOT)) + +from scripts.artifacts import macosReminders as artifact # pylint: disable=wrong-import-position + +UTC = timezone.utc +NEWER = 'Users/alex/Library/Group Containers/group.com.apple.reminders/Container_v1/Stores/' +OLDER = 'Users/sam/Library/Reminders/Container_v1/Stores/' +ATTRIBUTES = ('creationDate', 'lastModifiedDate', 'title', 'notes', 'allDay', 'dueDate', 'timeZone', + 'completed', 'completionDate', 'flagged', 'priority', 'lastBannerPresentationDate') + + +def model_cache(entities, style): + """A Z_MODELCACHE blob: raw deflate of an NSKeyedArchiver archive of entity descriptions. + + entities is [(name, parent, {property: kind})] with kind 'attr', ('one', destination) or + ('many', destination). A subentity lists its parent's properties as well as its own, as + _NSPropertyDescriptionProxy objects (style 'proxy') or as copies (style 'copy').""" + uid = plistlib.UID + objects, classes = ['$null'], {} + + def add(value): + objects.append(value) + return uid(len(objects) - 1) + + def cls(name): + if name not in classes: + classes[name] = add({'$classname': name, '$classes': [name, 'NSObject']}) + return classes[name] + + def description(prop, kind): + if kind == 'attr': + return add({'$class': cls('NSAttributeDescription'), 'NSPropertyName': add(prop), 'NSAttributeType': 700}) + return add({'$class': cls('NSRelationshipDescription'), 'NSPropertyName': add(prop), + 'NSMaxCount': 1 if kind[0] == 'one' else 0, '_NSDestinationEntityName': add(kind[1])}) + + spec = {name: (parent, props) for name, parent, props in entities} + slots = {name: add({}) for name in spec} + own = {} + + def build(name): + if name in own: + return own[name] + parent, props = spec[name] + listed = {} + if parent: + for prop, (desc, kind) in build(parent).items(): + listed[prop] = ((add({'$class': cls('_NSPropertyDescriptionProxy'), 'NSUnderlyingProperty': desc, + 'NSEntityDescription': slots[name]}) if style == 'proxy' + else description(prop, kind)), kind) + for prop, kind in props.items(): + listed[prop] = (description(prop, kind), kind) + own[name] = listed + return listed + + for name in spec: + listed = build(name) + table = add({'$class': cls('NSDictionary'), 'NS.keys': [add(p) for p in listed], + 'NS.objects': [desc for desc, _ in listed.values()]}) + parent = spec[name][0] + objects[slots[name].data] = {'$class': cls('NSEntityDescription'), 'NSEntityName': add(name), + 'NSSuperentity': slots[parent] if parent else uid(0), 'NSProperties': table} + root = add({'$class': cls('NSManagedObjectModel'), 'NSEntities': [slots[name] for name in spec]}) + archive = plistlib.dumps({'$version': 100000, '$archiver': 'NSKeyedArchiver', '$top': {'root': root}, + '$objects': objects}, fmt=plistlib.PlistFormat.FMT_BINARY) + packer = zlib.compressobj(9, zlib.DEFLATED, -15) + return packer.compress(archive) + packer.flush() + + +def record(device): + """A CloudKit record archive with a ModifiedByDevice value.""" + uid = plistlib.UID + return plistlib.dumps({'$version': 100000, '$archiver': 'NSKeyedArchiver', + '$top': {'ModifiedByDevice': uid(1), 'RecordCtime': uid(2)}, + '$objects': ['$null', device, {'NS.time': 788054402.0, '$class': uid(3)}, + {'$classname': 'NSDate', '$classes': ['NSDate', 'NSObject']}]}, + fmt=plistlib.PlistFormat.FMT_BINARY) + + +REMINDER_PROPS = {**{name: 'attr' for name in ATTRIBUTES}, 'ckServerRecordData': 'attr', + 'markedForDeletion': 'attr', 'list': ('one', 'REMCDList'), + 'account': ('one', 'REMCDAccount'), 'parentReminder': ('one', 'REMCDReminder'), + 'children': ('many', 'REMCDReminder')} +NEWER_MODEL = [ + ('REMCDBaseList', None, {'name': 'attr', 'account': ('one', 'REMCDAccount')}), + ('REMCDList', 'REMCDBaseList', {'reminders': ('many', 'REMCDReminder')}), + ('REMCDObject', None, {'account': ('one', 'REMCDAccount'), 'markedForDeletion': 'attr'}), + ('REMCDAccount', 'REMCDObject', {'name': 'attr', 'reminders': ('many', 'REMCDReminder')}), + ('REMCDAlarmLocationTrigger', 'REMCDObject', {'title': 'attr'}), + ('REMCDReminder', None, REMINDER_PROPS)] +NEWER_NUMBERS = {'REMCDBaseList': 2, 'REMCDList': 3, 'REMCDObject': 14, 'REMCDAccount': 15, + 'REMCDAlarmLocationTrigger': 19, 'REMCDReminder': 39} +OLDER_MODEL = [ + ('REMCDObject', None, {'account': ('one', 'REMCDAccount'), 'markedForDeletion': 'attr', 'ckServerRecordData': 'attr'}), + ('REMCDAccount', 'REMCDObject', {'name': 'attr', 'reminders': ('many', 'REMCDReminder')}), + ('REMCDAlarmLocationTrigger', 'REMCDObject', {'title': 'attr'}), + ('REMCDList', 'REMCDObject', {'name': 'attr', 'reminders': ('many', 'REMCDReminder')}), + ('REMCDReminder', 'REMCDObject', {name: kind for name, kind in REMINDER_PROPS.items() + if name not in ('account', 'markedForDeletion', 'ckServerRecordData')}), + ('REMCDSharee', 'REMCDObject', {'list': ('one', 'REMCDList')}), + ('REMCDSmartListOrder', 'REMCDObject', {'lastModifiedDate': 'attr'})] +OLDER_NUMBERS = {'REMCDObject': 3, 'REMCDAccount': 4, 'REMCDAlarmLocationTrigger': 8, 'REMCDList': 22, + 'REMCDReminder': 24, 'REMCDSharee': 25, 'REMCDSmartListOrder': 27} +REMINDER_COLUMNS = ('ZCREATIONDATE, ZLASTMODIFIEDDATE, ZTITLE, ZNOTES, ZALLDAY, ZDUEDATE, ZTIMEZONE, ZCOMPLETED, ' + 'ZCOMPLETIONDATE, ZFLAGGED, ZPRIORITY, ZLASTBANNERPRESENTATIONDATE, ZCKSERVERRECORDDATA, ' + 'ZMARKEDFORDELETION, ZLIST, ZACCOUNT, ZPARENTREMINDER') + +OLDER_COLUMNS = ('Z_PK, Z_ENT, ZACCOUNT, ZMARKEDFORDELETION, ZCKSERVERRECORDDATA, ZNAME, ZTITLE, ZNAME1, ZTITLE1, ' + 'ZNOTES, ZCREATIONDATE, ZLASTMODIFIEDDATE, ZALLDAY, ZDUEDATE, ZTIMEZONE, ZCOMPLETED, ZCOMPLETIONDATE, ' + 'ZFLAGGED, ZPRIORITY, ZLASTBANNERPRESENTATIONDATE, ZLIST, ZPARENTREMINDER, ZLIST1, ZLASTMODIFIEDDATE1') + + +class Context: + def __init__(self, root, files): + self.root = root + self.files = files + + def get_files_found(self): + return self.files + + def get_relative_path(self, path): + return os.path.relpath(path, self.root).replace(os.sep, '/') + + +def walk(root): + return sorted(os.path.join(folder, name) for folder, _, names in os.walk(root) for name in names) + + +class ArtifactTest(unittest.TestCase): + def setUp(self): + self.root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.root) + self.logged = [] + for module in (artifact, sys.modules['scripts.macos_plists']): + patcher = patch.object(module, 'logfunc', self.logged.append) + patcher.start() + self.addCleanup(patcher.stop) + + def store(self, relative, numbers, cache, statements): + path = os.path.join(self.root, relative) + os.makedirs(os.path.dirname(path), exist_ok=True) + with sqlite3.connect(path) as db: + db.execute('CREATE TABLE Z_PRIMARYKEY (Z_ENT INTEGER PRIMARY KEY, Z_NAME VARCHAR, Z_SUPER INTEGER, Z_MAX INTEGER)') + db.executemany('INSERT INTO Z_PRIMARYKEY VALUES (?, ?, 0, 0)', [(n, name) for name, n in numbers.items()]) + if cache is not None: + db.execute('CREATE TABLE Z_MODELCACHE (Z_CONTENT BLOB)') + db.execute('INSERT INTO Z_MODELCACHE VALUES (?)', (cache,)) + for statement, *rows in statements: + if rows: + db.executemany(statement, rows) + else: + db.execute(statement) + db.close() + return path + + def run_artifact(self, extra=()): + return artifact.macosReminders.__wrapped__(Context(self.root, walk(self.root) + list(extra))) + + def newer_store(self, relative): + return self.store(relative, NEWER_NUMBERS, model_cache(NEWER_MODEL, 'copy'), [ + ('CREATE TABLE ZREMCDBASELIST (Z_PK INTEGER PRIMARY KEY, Z_ENT INTEGER, ZNAME VARCHAR, ZACCOUNT INTEGER)',), + # Row 7 is the abstract base entity, not a list. + ('INSERT INTO ZREMCDBASELIST VALUES (?, ?, ?, ?)', (1, 3, 'Groceries', 1), (2, 3, 'Work', 1), + (7, 2, 'Base', 1)), + ('CREATE TABLE ZREMCDOBJECT (Z_PK INTEGER PRIMARY KEY, Z_ENT INTEGER, ZACCOUNT INTEGER, ' + 'ZMARKEDFORDELETION INTEGER, ZNAME VARCHAR, ZTITLE VARCHAR)',), + ('INSERT INTO ZREMCDOBJECT VALUES (?, ?, ?, ?, ?, ?)', (1, 15, None, 0, 'iCloud', None), + (2, 19, 1, 0, None, 'Home')), + (f'CREATE TABLE ZREMCDREMINDER (Z_PK INTEGER PRIMARY KEY, Z_ENT INTEGER, {REMINDER_COLUMNS})',), + (f'INSERT INTO ZREMCDREMINDER (Z_PK, Z_ENT, {REMINDER_COLUMNS}) VALUES ({", ".join("?" * 19)})', + # All day, due at midnight UTC, with a banner and a CloudKit record. + (1, 39, 788000000.0, 788000000.0, 'Pay rent', 'By transfer', 1, 788054400.0, None, 0, None, 0, 0, + 788104800.5, record('Test Mac'), 0, 1, 1, None), + # A time and a stored time zone, completed, a child of the first. + (2, 39, 790000000.0, 790000001.0, 'Call back', None, 0, 800000000.25, 'America/New_York', 1, + 800100000.0, 1, 5, None, b'not a plist', 0, 2, 1, 1), + # A time and no time zone, and an all-day date that is not midnight UTC: neither is reported. + (3, 39, 790000000.0, 790000000.0, 'No zone', None, 0, 800000000.0, None, 0, None, 0, 0, None, None, 1, + 1, 1, None), + (4, 39, 810000000.0, 810000000.0, 'Odd day', None, 1, 810003600.0, None, 0, None, 0, 0, None, None, 0, + 7, 1, None))]) + + def test_newer_layout(self): + path = self.newer_store(NEWER + 'Data-A.sqlite') + headers, rows, source = self.run_artifact() + self.assertEqual([h if isinstance(h, str) else h[0] for h in headers], + ['Created (UTC)', 'Last Modified (UTC)', 'Title', 'Notes', 'List', 'Account', 'Due (UTC)', + 'Due Date (All Day)', 'Due Time Zone (as stored)', 'Completed (as stored)', + 'Completion Date (UTC)', 'Flagged (as stored)', 'Priority (as stored)', 'Parent Reminder', + 'Marked for Deletion (as stored)', 'Last Banner Presentation (UTC)', + 'CloudKit ModifiedByDevice (as stored)', 'User', 'Source File']) + where = NEWER + 'Data-A.sqlite' + self.assertEqual(rows, [ + (datetime(2025, 12, 21, 8, 53, 20, tzinfo=UTC), datetime(2025, 12, 21, 8, 53, 20, tzinfo=UTC), 'Pay rent', + 'By transfer', 'Groceries', 'iCloud', '', '2025-12-22', '', 0, '', 0, 0, '', 0, + datetime(2025, 12, 22, 14, 0, 0, 500000, tzinfo=UTC), 'Test Mac', 'alex', where), + (datetime(2026, 1, 13, 12, 26, 40, tzinfo=UTC), datetime(2026, 1, 13, 12, 26, 41, tzinfo=UTC), 'Call back', + '', 'Work', 'iCloud', datetime(2026, 5, 9, 6, 13, 20, 250000, tzinfo=UTC), '', 'America/New_York', 1, + datetime(2026, 5, 10, 10, 0, tzinfo=UTC), 1, 5, 'Pay rent', 0, '', '', 'alex', where), + (datetime(2026, 1, 13, 12, 26, 40, tzinfo=UTC), datetime(2026, 1, 13, 12, 26, 40, tzinfo=UTC), 'No zone', + '', 'Groceries', 'iCloud', '', '', '', 0, '', 0, 0, '', 1, '', '', 'alex', where), + # Its list relationship points at a row of another entity. + (datetime(2026, 9, 2, tzinfo=UTC), datetime(2026, 9, 2, tzinfo=UTC), 'Odd day', '', '', 'iCloud', '', '', '', + 0, '', 0, 0, '', 0, '', '', 'alex', where)]) + self.assertEqual(source, path) + self.assertEqual(sorted(self.logged), sorted([ + f'Reminders: 1 due date(s) in {where} marked all day and not at midnight UTC, not reported', + f'Reminders: 1 due date(s) in {where} with a time and no stored time zone, not reported'])) + + def test_older_layout(self): + columns = OLDER_COLUMNS + + def row(**values): + return tuple(values.get(name.strip(), None) for name in columns.split(',')) + + self.store(OLDER + 'Data-B.sqlite', OLDER_NUMBERS, model_cache(OLDER_MODEL, 'proxy'), [ + (f'CREATE TABLE ZREMCDOBJECT ({columns})',), + (f'INSERT INTO ZREMCDOBJECT ({columns}) VALUES ({", ".join("?" * 24)})', + row(Z_PK=1, Z_ENT=4, ZNAME='iCloud'), + row(Z_PK=2, Z_ENT=8, ZACCOUNT=1, ZTITLE='Home'), + row(Z_PK=3, Z_ENT=22, ZACCOUNT=1, ZNAME1='Errands'), + row(Z_PK=4, Z_ENT=24, ZACCOUNT=1, ZTITLE1='Buy milk', ZNOTES='two', ZCREATIONDATE=600000000.0, + ZLASTMODIFIEDDATE=600000100.5, ZALLDAY=0, ZDUEDATE=600086400.0, ZTIMEZONE='Europe/Madrid', + ZCOMPLETED=0, ZFLAGGED=0, ZPRIORITY=0, ZMARKEDFORDELETION=0, ZLIST=3), + row(Z_PK=5, Z_ENT=25, ZACCOUNT=1, ZLIST1=3), + row(Z_PK=6, Z_ENT=27, ZACCOUNT=1, ZLASTMODIFIEDDATE1=700000000.0), + # Created before row 4, so it is reported first. + row(Z_PK=7, Z_ENT=24, ZACCOUNT=1, ZTITLE1='Oat milk', ZCREATIONDATE=590000000.0, + ZLASTMODIFIEDDATE=600000000.0, ZCOMPLETED=0, ZFLAGGED=0, ZPRIORITY=0, ZMARKEDFORDELETION=0, + ZLIST=2, ZPARENTREMINDER=4))]) + _, rows, _ = self.run_artifact() + where = OLDER + 'Data-B.sqlite' + self.assertEqual(rows, [ + # Its list relationship points at the alarm trigger row, which is not a list. + (datetime(2019, 9, 12, 16, 53, 20, tzinfo=UTC), datetime(2020, 1, 6, 10, 40, tzinfo=UTC), 'Oat milk', '', '', + 'iCloud', '', '', '', 0, '', 0, 0, 'Buy milk', 0, '', '', 'sam', where), + (datetime(2020, 1, 6, 10, 40, tzinfo=UTC), datetime(2020, 1, 6, 10, 41, 40, 500000, tzinfo=UTC), 'Buy milk', + 'two', 'Errands', 'iCloud', datetime(2020, 1, 7, 10, 40, tzinfo=UTC), '', 'Europe/Madrid', 0, '', 0, 0, '', 0, + '', '', 'sam', where)]) + self.assertEqual(self.logged, []) + + def test_copies_and_stores_that_cannot_be_read(self): + self.newer_store(NEWER + 'Data-A.sqlite') + # A byte-identical copy under System/Volumes/Data is not read again. + os.makedirs(os.path.join(self.root, 'System/Volumes/Data', NEWER)) + shutil.copy(os.path.join(self.root, NEWER, 'Data-A.sqlite'), os.path.join(self.root, 'System/Volumes/Data', NEWER)) + # A copy that differs holds the same reminders: each is reported once, naming both copies. + other = 'Users/alex/Backup/Library/Group Containers/group.com.apple.reminders/Container_v1/Stores/Data-A.sqlite' + self.newer_store(other) + with sqlite3.connect(os.path.join(self.root, other)) as db: + db.execute('CREATE TABLE extra (a TEXT)') + db.close() + self.store(NEWER + 'Data-none.sqlite', NEWER_NUMBERS, None, []) + self.store(NEWER + 'Data-junk.sqlite', NEWER_NUMBERS, b'not deflate', []) + self.store(NEWER + 'Data-noreminder.sqlite', {'REMCDObject': 1}, + model_cache([('REMCDObject', None, {'name': 'attr'})], 'copy'), []) + # A sibling entity of the older layout that defines notes as a to-many relationship leaves + # the reminder's notes column undecided, so it is reported blank. + ambiguous = OLDER_MODEL + [('REMCDTemplate', 'REMCDObject', {'notes': ('many', 'REMCDReminder')})] + self.store(OLDER + 'Data-C.sqlite', {**OLDER_NUMBERS, 'REMCDTemplate': 30}, model_cache(ambiguous, 'proxy'), [ + (f'CREATE TABLE ZREMCDOBJECT ({OLDER_COLUMNS})',), + ('INSERT INTO ZREMCDOBJECT (Z_PK, Z_ENT, ZTITLE1, ZNOTES, ZCREATIONDATE) VALUES (1, 24, ?, ?, ?)', + ('Kept', 'hidden', 788000000.0))]) + folder = os.path.join(self.root, NEWER, 'Data-dir.sqlite') + os.makedirs(folder) + # A sidecar with no database beside it, and an AppleDouble file, are not stores. + for name in ('Data-orphan.sqlite-shm', '._Data-A.sqlite'): + with open(os.path.join(self.root, NEWER, name), 'wb') as handle: + handle.write(b'\x00\x05\x16\x07 not a database') + _, rows, source = self.run_artifact([folder]) + both = f'{NEWER}Data-A.sqlite\n{other}' + # Stores are read shortest path first, so the older-layout store comes first. + self.assertEqual([(r[2], r[-1]) for r in rows], + [('Kept', OLDER + 'Data-C.sqlite'), ('Pay rent', both), ('Call back', both), ('No zone', both), + ('Odd day', both)]) + self.assertEqual(rows[0][3], '') + self.assertEqual(sorted(source.split('\n')), sorted(os.path.join(self.root, p) for p in + (NEWER + 'Data-A.sqlite', other, OLDER + 'Data-C.sqlite'))) + self.assertEqual(sorted(line for line in self.logged if 'due date' not in line), sorted([ + 'Reminders: 1 byte-identical copy(ies) under System/Volumes/Data not read again', + f'Reminders: no readable Core Data model in {NEWER}Data-none.sqlite', + f'Reminders: no readable Core Data model in {NEWER}Data-junk.sqlite', + f'Reminders: no REMCDReminder entity in the model of {NEWER}Data-noreminder.sqlite', + f'Reminders: no column found for notes in {OLDER}Data-C.sqlite; reported blank'])) + + def test_declared_paths(self): + paths = artifact.__artifacts_v2__['macosReminders']['paths'] + for member in (OLDER + 'Data-B.sqlite', NEWER + 'Data-A.sqlite-wal', + 'System/Volumes/Data/' + NEWER + 'Data-local.sqlite'): + self.assertTrue(any(fnmatch.fnmatch(member, pattern) for pattern in paths), member) + for member in ('Users/alex/Library/Other/Container_v1/Stores/Data-A.sqlite', NEWER + 'Data-A.plist'): + self.assertFalse(any(fnmatch.fnmatch(member, pattern) for pattern in paths), member) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/artifacts/macosReminders.py b/scripts/artifacts/macosReminders.py new file mode 100644 index 0000000..fb93a3e --- /dev/null +++ b/scripts/artifacts/macosReminders.py @@ -0,0 +1,392 @@ +"""Reminders in the macOS Reminders stores, for DLEAPP. + +Author: @AlexisBrignoni, Claude. + +The stores are Core Data SQLite databases. The table and column that hold each reminder +property are worked out from the store's own cached model (Z_MODELCACHE, a raw deflate +stream holding an NSKeyedArchiver of the model), so the older layout, where reminders are +rows of ZREMCDOBJECT beside other entities, and the newer ZREMCDREMINDER table are read the +same way. The artifact notes say how the column names are derived and how that was checked. +""" + +__artifacts_v2__ = { + "macosReminders": { + "name": "Reminders", + "description": "Reminders in each user's Reminders stores, with title, notes, list, account, the stored " + 'creation, modification, due and completion dates, flag, priority, parent reminder and ' + "the ModifiedByDevice value of each reminder's CloudKit record.", + "author": "@AlexisBrignoni, Claude", + "creation_date": "2026-09-27", + "last_update_date": "2026-09-27", + "requirements": "none", + "category": "Reminders (macOS)", + "notes": ( + ( + ( + 'Reads the Core Data stores of the Reminders app in each home folder, the ' + 'Data-.sqlite files, each of which holds one account on the tested images, and ' + 'Data-local.sqlite, under Library/Reminders/Container_v1/Stores, where ' + 'dleapp_macos_bigsur (macOS 11.2.1) keeps them, and under Library/Group ' + 'Containers/group.com.apple.reminders/Container_v1/Stores, where the public MacBook ' + 'Pro logical extraction (macOS 15.4, not a registered corpus key) keeps them. One row ' + 'is reported per row of the REMCDReminder entity. On the MacBook Pro those rows are ' + 'in a ZREMCDREMINDER table. The stores on dleapp_macos_bigsur have no such table: ' + 'their model makes REMCDReminder a subentity of REMCDObject, so a reminder would be a ' + 'row of ZREMCDOBJECT, which also holds the accounts, lists and other entities, and a ' + 'property name that several of those entities define is kept in numbered columns such ' + 'as ZTITLE and ZTITLE1. The column each property is read from is worked out from the ' + 'Core Data model the store caches in Z_MODELCACHE, a raw deflate stream holding an ' + 'NSKeyedArchiver archive of the model: Z followed by the property name in capitals, ' + 'with 1, 2 and so on added when entities of the same table with lower numbers in ' + 'Z_PRIMARYKEY define a property of that name themselves. That rule comes from ' + 'measurement, not from a published source. On the 78 stores of dleapp_macos_bigsur, ' + 'the MacBook Pro and 16 registered iOS images it names a column that exists for all ' + '75,737 attributes and to-one relationships of their entities, and in all 415 checks ' + 'of a name that several entities share, the rows of an entity leave the other ' + "entities' columns of that name empty. On dleapp_macos_bigsur it places a reminder's " + "title in ZTITLE1, the column a guest post on Ciofeca Forensics read a reminder's " + 'title from in the Reminders database of an iPhone in the Cellebrite 2020 CTF ' + "(Reference: Ciofeca Forensics, 'Cellebrite CTF 2020: Ruth Langmore', " + 'https://www.ciofecaforensics.com/2020/11/02/cellebrite-ctf-ruth/). A store whose ' + 'model cannot be read is logged and not read, and a property whose column cannot be ' + 'decided is logged and reported blank. List, Account and Parent Reminder are the name ' + "of the row the reminder's list relationship points to, the name of the row its " + 'account relationship points to, and the title of the reminder its parentReminder ' + 'relationship points to, with the entity each relationship points to, REMCDList, ' + 'REMCDAccount and REMCDReminder, taken from the model. On the MacBook Pro the list of ' + "each reminder belongs to the reminder's own account on all 37 rows, and Parent " + 'Reminder has no value on any row, since no tested reminder has a parent reminder. ' + 'Created (UTC), Last Modified (UTC), Completion Date (UTC) and Last Banner ' + 'Presentation (UTC) are creationDate, lastModifiedDate, completionDate and ' + 'lastBannerPresentationDate read as seconds since 00:00:00 UTC on 1 January 2001, the ' + "reference date Apple documents for NSDate (Reference: Apple, 'NSDate', " + 'https://developer.apple.com/documentation/foundation/nsdate). Read that way the 37 ' + 'reminders on the MacBook Pro were created on 26 November 2025, where the 1970 epoch ' + 'would place them in 1994, and the CloudKit record each row keeps in ' + 'ckServerRecordData, an NSKeyedArchiver archive, holds a RecordCtime between 2 ' + 'seconds and 5 days after Created, never before it. Last Modified equals Created on ' + '33 of the 37 rows. A reminder whose allDay is 1 has its due date in Due Date (All ' + 'Day), written as a date. On the one such reminder, on the MacBook Pro, dueDate is ' + "midnight UTC on that date and the row's displayDateDate is midnight at the offset of " + '-18,000 seconds the row records in displayDateUpdatedForSecondsFromGMT, so the date ' + 'is not reported as a time. For a reminder with a time, Due (UTC) is dueDate read as ' + 'the dates above and Due Time Zone (as stored) is its timeZone. No reminder on the ' + 'MacBook Pro has a time, so Due (UTC) and Due Time Zone (as stored) have no value on ' + 'any row there; the 4 reminders with a time in the registered iOS images ' + 'cookbook_ios1751 and otto_ios17 each store a time zone and a displayDateDate equal ' + 'to dueDate. A due date marked all day that is not at midnight UTC, or one with a ' + 'time and no stored time zone, is not reported and the run log counts it; no tested ' + 'reminder has either. Title and Notes are title and notes as stored; the ' + 'titleDocument and notesDocument data beside them are not read. Completed (as ' + 'stored), Flagged (as stored), Priority (as stored) and Marked for Deletion (as ' + 'stored) are completed, flagged, priority and markedForDeletion as stored, and what ' + 'markedForDeletion records is not established. On the MacBook Pro Completed, Flagged, ' + 'Priority and Marked for Deletion hold 0 on every row and Completion Date has no ' + 'value on any row; in the registered iOS images one reminder is completed with a ' + 'completion date (otto_ios17), one has a priority other than 0 (cookbook_ios1751) and ' + 'one is marked for deletion (fsfull002_ios17). What sets lastBannerPresentationDate ' + 'is not established. On the MacBook Pro one reminder has one, 9 hours after that ' + "reminder's displayDateDate, and the registered iPhone 14 Plus image " + 'iphone14plus_ios18, whose store holds the same 37 reminders by CloudKit identifier, ' + 'with the same Created, Last Modified, Title, Notes, due date and list on all 37, has ' + 'a value 2.2 seconds later on that reminder. CloudKit ModifiedByDevice (as stored) is ' + 'the ModifiedByDevice value of that CloudKit record. A post on Ciofeca Forensics ' + 'reported that this field of an Apple Notes record holds the hostname of a device ' + 'used by the account, and cautioned that there can be false positives (Reference: ' + "Ciofeca Forensics, 'Revisiting Apple Notes (7): Cloudkit Data', " + 'https://www.ciofecaforensics.com/2020/10/20/apple-notes-cloudkit-data/). On the ' + 'MacBook Pro CloudKit ModifiedByDevice holds one value on every row, a name ending in ' + 'MacBook Pro, and the iPhone 14 Plus copies hold the same value; in cookbook_ios1751 ' + 'one record holds the string CKDatabaseRpc. User is the folder after Users in the ' + 'source path. On the MacBook Pro every row comes from one store of one user with one ' + 'account, so Account, User and Source File each hold one value there. When a logical ' + 'extraction holds the same store under Users/ and under System/Volumes/Data/Users/, a ' + 'second copy whose database and -wal file are both byte-identical to the first is not ' + 'read again and is counted in the run log, as the 4 copies on the MacBook Pro are; a ' + 'row that differing copies both hold with the same values is reported once, and ' + 'Source File lists every copy. The five stores on dleapp_macos_bigsur hold 4 accounts ' + 'and 4 lists and no reminder. Alarms and their triggers, attachments, recurrence ' + 'rules, hashtags, assignments and sharees, REMCDAuxiliaryReminderChangeInfo and its ' + "delete and move subentities, templates, saved reminders and Core Data's persistent " + 'history tables (ACHANGE and ATRANSACTION) are not reported.' + ) + ) + ), + "paths": ('*/Library/Reminders/Container_v1/Stores/*.sqlite*', + '*/Library/Group Containers/group.com.apple.reminders/Container_v1/Stores/*.sqlite*'), + "output_types": ["html", "tsv", "timeline", "lava"], + "artifact_icon": "bell", + "sample_data": { + "dleapp_macos_bigsur": "macOS 11.2.1 build 20D74 | 0 rows (five Reminders stores in the older layout, none holding a reminder)", + }, + }, +} + +import os +import plistlib +import zlib +from datetime import timedelta + +from scripts.ilapfuncs import (artifact_processor, does_table_exist_in_db, + get_sqlite_db_records, logfunc) +from scripts.macos_plists import (EPOCH_2001, mac_absolute_utc, resolve_keyed_archive, + unique_sources, user_from_path) +from scripts.macos_powerlog import merge_sources + +_REMINDER = 'REMCDReminder' +# REMCDReminder properties read into each row, with the name each takes in the query. +_PROPERTIES = (('creationDate', 'created'), ('lastModifiedDate', 'modified'), + ('title', 'title'), ('notes', 'notes'), ('allDay', 'all_day'), + ('dueDate', 'due'), ('timeZone', 'time_zone'), ('completed', 'completed'), + ('completionDate', 'completion'), ('flagged', 'flagged'), + ('priority', 'priority'), ('markedForDeletion', 'marked'), + ('lastBannerPresentationDate', 'banner'), ('ckServerRecordData', 'record')) +# To-one relationships of REMCDReminder, the property read from the row they point to, and +# the name that takes in the query. +_JOINS = (('list', 'name', 'list_name'), ('account', 'name', 'account_name'), + ('parentReminder', 'title', 'parent_title')) +_COLUMN_KINDS = ('attribute', 'to-one') +_DAY = 86400 + + +def _model(blob): + """{entity: (superentity or None, {property: (owned, kind, destination)})} from a + Z_MODELCACHE blob, or None when it cannot be read. kind is 'attribute', 'to-one', + 'to-many' or the class name of anything else. owned is False for a property its + superentity also lists: a subentity lists its superentity's properties as well as its own, + and archives differ in how they store an inherited one (a _NSPropertyDescriptionProxy of + the superentity's description, or a copy of it), so the name is what decides.""" + try: + archive = plistlib.loads(zlib.decompress(bytes(blob), -15)) + except (zlib.error, plistlib.InvalidFileException, ValueError, TypeError, OverflowError): + return None + objects = archive.get('$objects') if isinstance(archive, dict) else None + if not isinstance(objects, list): + return None + + def get(value): + if isinstance(value, plistlib.UID): + return objects[value.data] if value.data < len(objects) else None + return value + + def class_name(item): + meta = get(item.get('$class')) if isinstance(item, dict) else None + return meta.get('$classname') if isinstance(meta, dict) else None + + def entity_name(value): + value = get(value) + return get(value.get('NSEntityName')) if isinstance(value, dict) else None + + def describe(item): + kind = class_name(item) + if kind == 'NSAttributeDescription': + return 'attribute', None + if kind == 'NSRelationshipDescription': + destination = get(item.get('_NSDestinationEntityName')) or \ + entity_name(item.get('NSDestinationEntity')) + return ('to-one' if get(item.get('NSMaxCount')) == 1 else 'to-many'), destination + return kind, None + + model = {} + for item in objects: + if class_name(item) != 'NSEntityDescription': + continue + properties = {} + table = get(item.get('NSProperties')) + if isinstance(table, dict): + for key, value in zip(table.get('NS.keys', []), table.get('NS.objects', [])): + prop = get(value) + if class_name(prop) == '_NSPropertyDescriptionProxy': + prop = get(prop.get('NSUnderlyingProperty')) + properties[get(key)] = describe(prop) + name = get(item.get('NSEntityName')) + if isinstance(name, str): + model[name] = (entity_name(item.get('NSSuperentity')), properties) + return {name: (parent, {prop: (not (parent in model and prop in model[parent][1]), *kind) + for prop, kind in props.items()}) + for name, (parent, props) in model.items()} or None + + +def _root(model, entity): + seen = set() + while model.get(entity, (None,))[0] in model and entity not in seen: + seen.add(entity) + entity = model[entity][0] + return entity + + +def _descends(model, entity, ancestor): + seen = set() + while entity in model and entity not in seen: + if entity == ancestor: + return True + seen.add(entity) + entity = model[entity][0] + return False + + +def _owner(model, entity, prop): + """The entity that defines prop for entity: itself, or the ancestor it inherits it from.""" + seen = set() + while entity in model and entity not in seen: + seen.add(entity) + owned = model[entity][1].get(prop, (False,))[0] + if owned: + return entity + entity = model[entity][0] + return None + + +def _column(model, order, entity, prop): + """The column Core Data stores prop of entity in: Z and the property name in capitals, + followed by 1, 2, ... when entities of the same table with lower entity numbers define a + property of that name themselves. None when prop has no column of its own (to-many and + other kinds), when an entity of the table defines that name as anything other than an + attribute or a to-one relationship, or when an entity number is missing.""" + owner = _owner(model, entity, prop) + if owner is None or model[owner][1][prop][1] not in _COLUMN_KINDS: + return None + root = _root(model, entity) + owners = [name for name, (_parent, props) in model.items() + if _root(model, name) == root and props.get(prop, (False,))[0]] + if any(model[name][1][prop][1] not in _COLUMN_KINDS for name in owners): + return None + if any(name not in order for name in owners): + return None + owners.sort(key=lambda name: order[name]) + index = owners.index(owner) + return f'Z{prop.upper()}{index if index else ""}' + + +def _columns(path, table): + return {row['name'] for row in get_sqlite_db_records(path, f'PRAGMA table_info("{table}")')} + + +def _modified_by_device(blob): + """ModifiedByDevice from the CloudKit record archive in ckServerRecordData, or ''.""" + if not isinstance(blob, (bytes, bytearray)): + return '' + try: + archive = plistlib.loads(bytes(blob)) + except (plistlib.InvalidFileException, ValueError, TypeError, OverflowError): + return '' + value = resolve_keyed_archive(archive, 'ModifiedByDevice') + return value if isinstance(value, str) else '' + + +def _blank(value): + return '' if value is None else value + + +def _query(path, relative): + """(query, unresolved property names) for the reminders of one store, or (None, reason).""" + rows = get_sqlite_db_records(path, 'SELECT Z_CONTENT FROM Z_MODELCACHE') \ + if does_table_exist_in_db(path, 'Z_MODELCACHE') else [] + model = _model(rows[0][0]) if rows and rows[0][0] else None + if not model: + return None, f'no readable Core Data model in {relative}' + if _REMINDER not in model: + return None, f'no {_REMINDER} entity in the model of {relative}' + order = {row['Z_NAME']: row['Z_ENT'] for row in get_sqlite_db_records( + path, 'SELECT Z_NAME, Z_ENT FROM Z_PRIMARYKEY')} \ + if does_table_exist_in_db(path, 'Z_PRIMARYKEY') else {} + table = f'Z{_root(model, _REMINDER).upper()}' + if _REMINDER not in order or not does_table_exist_in_db(path, table): + return None, f'no {table} table for {_REMINDER} in {relative}' + columns = _columns(path, table) + select, joins, unresolved = ['r.Z_PK AS pk'], [], [] + for prop, alias in _PROPERTIES: + column = _column(model, order, _REMINDER, prop) + if column in columns: + select.append(f'r."{column}" AS {alias}') + else: + select.append(f'NULL AS {alias}') + unresolved.append(prop) + for index, (relation, target, alias) in enumerate(_JOINS): + key = _column(model, order, _REMINDER, relation) + owner = _owner(model, _REMINDER, relation) + destination = model[owner][1][relation][2] if owner else None + found = None + if key in columns and destination in model: + other = f'Z{_root(model, destination).upper()}' + column = _column(model, order, destination, target) + numbers = sorted(order[name] for name in model + if name in order and _descends(model, name, destination)) + if numbers and does_table_exist_in_db(path, other) and column in _columns(path, other): + found = (other, column, numbers) + if found is None: + select.append(f'NULL AS {alias}') + unresolved.append(relation) + continue + other, column, numbers = found + name = f'j{index}' + joins.append(f'LEFT JOIN "{other}" {name} ON {name}.Z_PK = r."{key}" AND ' + f'{name}.Z_ENT IN ({", ".join(str(number) for number in numbers)})') + select.append(f'{name}."{column}" AS {alias}') + query = (f'SELECT {", ".join(select)} FROM "{table}" r {" ".join(joins)} ' + f'WHERE r.Z_ENT = {int(order[_REMINDER])} ORDER BY created, pk') + return query, unresolved + + +def _due(row, counts): + """(due instant, all-day date, stored time zone) for one reminder row.""" + due, time_zone = row['due'], _blank(row['time_zone']) + if due is None: + return '', '', time_zone + if isinstance(due, bool) or not isinstance(due, (int, float)): + counts['that are not numbers'] += 1 + return '', '', time_zone + if row['all_day'] == 1: + if due % _DAY == 0: + return '', (EPOCH_2001 + timedelta(seconds=due)).date().isoformat(), time_zone + counts['marked all day and not at midnight UTC'] += 1 + return '', '', time_zone + if time_zone: + return mac_absolute_utc(due), '', time_zone + counts['with a time and no stored time zone'] += 1 + return '', '', time_zone + + +@artifact_processor +def macosReminders(context): + data_headers = (('Created (UTC)', 'datetime'), ('Last Modified (UTC)', 'datetime'), 'Title', + 'Notes', 'List', 'Account', ('Due (UTC)', 'datetime'), 'Due Date (All Day)', + 'Due Time Zone (as stored)', 'Completed (as stored)', + ('Completion Date (UTC)', 'datetime'), 'Flagged (as stored)', + 'Priority (as stored)', 'Parent Reminder', 'Marked for Deletion (as stored)', + ('Last Banner Presentation (UTC)', 'datetime'), + 'CloudKit ModifiedByDevice (as stored)', 'User', 'Source File') + stores = [path for path in context.get_files_found() + if str(path).endswith('.sqlite') and not os.path.basename(str(path)).startswith('._')] + paths, _skipped = unique_sources(context, stores, sidecars=('-wal',), label='Reminders') + records, read = [], [] + for path in paths: + relative = context.get_relative_path(path) + query, detail = _query(path, relative) + if query is None: + logfunc(f'Reminders: {detail}') + continue + if detail: + logfunc(f'Reminders: no column found for {", ".join(detail)} in {relative}; ' + 'reported blank') + rows = get_sqlite_db_records(path, query) + if rows: + read.append(path) + counts = {'that are not numbers': 0, 'marked all day and not at midnight UTC': 0, + 'with a time and no stored time zone': 0} + for row in rows: + due, day, time_zone = _due(row, counts) + records.append(((mac_absolute_utc(row['created']), mac_absolute_utc(row['modified']), + _blank(row['title']), _blank(row['notes']), + _blank(row['list_name']), _blank(row['account_name']), due, day, + time_zone, _blank(row['completed']), + mac_absolute_utc(row['completion']), _blank(row['flagged']), + _blank(row['priority']), _blank(row['parent_title']), + _blank(row['marked']), mac_absolute_utc(row['banner']), + _modified_by_device(row['record']), user_from_path(relative)), + relative)) + for reason, count in counts.items(): + if count: + logfunc(f'Reminders: {count} due date(s) in {relative} {reason}, not reported') + data_list = [values + ('\n'.join(sources),) for values, sources in merge_sources(records)] + return data_headers, data_list, '\n'.join(read)