diff --git a/admin/test/results/dleapp_imessage_bigsur.json b/admin/test/results/dleapp_imessage_bigsur.json new file mode 100644 index 0000000..06130fc --- /dev/null +++ b/admin/test/results/dleapp_imessage_bigsur.json @@ -0,0 +1,278 @@ +{ + "artifacts": { + "iMessage Attachments": { + "column_digests": { + "attachment_guid": "bd2765210fac19039443aa7c9746ab458c25534b44caa0790589e2b53574d560", + "created": "e1a95d6e8a8a0813c1fda0dc8d91f9a235dcf91b507915667a10b149b163889a", + "file": "b3b9b08788135dd41aa6d200525327dcda1bfd6deaa77b6ede64389b3cc8018c", + "filename": "51940b637048141982918415f728761c0de3d147ca4c50e4f30586722665edc4", + "is_sticker": "de2d1aa0509a80c5446acd90700fbcd632b3bb5065b5096a7603dc7b205892da", + "message_guid": "1ed65e75cbb37ee31938f4409be2f481defd7079ffa318c8c6fce29bd2bffd5a", + "mime_type": "60a5d57f99c3ab586676727fb6d45df536473cd2395c5ad81e50536cad1564a7", + "size_bytes": "0c9d3834bfec20c896973b8d49e1fc4acec1bc178adaa4c158c72dd1a8fb8308", + "source_file": "ade820f0a6f7d1889ab924991108b8f7c541362e7b403847f3274a6aff5fe4ca", + "start": "c71f928f33661ca32fa44591ac4723cd66a037622879c01f5558cfad10d1ef89", + "transfer_state_raw": "be2342d1c2b4e2eda0e283e560abfe13f428df7f41dc65b7aa5ee596521c0650" + }, + "columns": [ + { + "name": "created", + "type": "datetime" + }, + { + "name": "start", + "type": "datetime" + }, + { + "name": "filename", + "type": "text" + }, + { + "name": "file", + "type": "media" + }, + { + "name": "mime_type", + "type": "text" + }, + { + "name": "size_bytes", + "type": "text" + }, + { + "name": "is_sticker", + "type": "text" + }, + { + "name": "transfer_state_raw", + "type": "text" + }, + { + "name": "attachment_guid", + "type": "text" + }, + { + "name": "message_guid", + "type": "text" + }, + { + "name": "source_file", + "type": "text" + } + ], + "datetime_columns": { + "created": { + "in_plausible_range": true, + "parsed": 5 + }, + "start": { + "in_plausible_range": true, + "parsed": 5 + } + }, + "digest": "28309aaab31fd33857fd91dabe95ad92dbca9c638bec4eabd443971097dadcae", + "non_empty": { + "attachment_guid": 5, + "created": 5, + "file": 5, + "filename": 5, + "is_sticker": 0, + "message_guid": 5, + "mime_type": 5, + "size_bytes": 5, + "source_file": 5, + "start": 5, + "transfer_state_raw": 5 + }, + "rows": 5 + }, + "iMessage Chat Threads": { + "column_digests": { + "archived": "ffe679bb831c95b67dc17819c63c5090d221aac6f4c7bf530f594ab43d21fa1e", + "chat_guid": "8ef9af549667fe664eb380885028186a1e707f9876ab09ae1a89c6b56a92b299", + "chat_identifier": "fbd27f24e5bc50c49f555166411b27af587c7c657fa10a0c902c9bacc5ad3d05", + "display_name": "ffe679bb831c95b67dc17819c63c5090d221aac6f4c7bf530f594ab43d21fa1e", + "participants": "fbd27f24e5bc50c49f555166411b27af587c7c657fa10a0c902c9bacc5ad3d05", + "service": "49926fe651cac62fdf0fc7eec6e18bfff002c8f6c62d5588824f81a3f24a3d36", + "source_file": "b1c141ef236bf57d75ed10093cb413c337666683b1518e5001c70777d0cd81e8", + "style_raw": "a25fa4e52f269490377ef695b0ccb5d35a53a808ce628a551766e85d1b3fb73d" + }, + "columns": [ + { + "name": "chat_identifier", + "type": "text" + }, + { + "name": "display_name", + "type": "text" + }, + { + "name": "service", + "type": "text" + }, + { + "name": "style_raw", + "type": "text" + }, + { + "name": "archived", + "type": "text" + }, + { + "name": "participants", + "type": "text" + }, + { + "name": "chat_guid", + "type": "text" + }, + { + "name": "source_file", + "type": "text" + } + ], + "datetime_columns": {}, + "digest": "ce4269497eb2870c58c663523c41420d7790fe0b97abd4b4cca040ad1906f041", + "non_empty": { + "archived": 0, + "chat_guid": 1, + "chat_identifier": 1, + "display_name": 0, + "participants": 1, + "service": 1, + "source_file": 1, + "style_raw": 1 + }, + "rows": 1 + }, + "iMessage Messages": { + "column_digests": { + "associated_message_guid": "a249c08b5d285dcaf9172d44afde32ac6a66ffa8fa12aa94272cdfb4e70d7fd6", + "body_source": "3bd952f775807298076ac5cb916b976756981905d90b89649d1f73d8f0209c67", + "chat_display_name": "a249c08b5d285dcaf9172d44afde32ac6a66ffa8fa12aa94272cdfb4e70d7fd6", + "chat_identifier": "a14551d2aa4cd88910a14ed11b55f3867237b27736eb92d27763e29bf85cd270", + "date": "a06355976e268c928cabfae47532fffad82b03af5807eeaa0eb6d36ef15f36e4", + "date_delivered": "f4e95d5521c4cfdd7ec04c0f4604822bfe9643ed1e474d4807aa24126c58aecb", + "date_read": "664f6ac8c93c575cf5853de665720e94b55e9abd37b55abb8d61192188530d33", + "direction": "f941194037b84b4bd7d3f397ea6a590f115ef94e7afb4a450c2c813a859974ef", + "group_action_type_raw": "59220831e3d9dbecc14b740a0255c54105d9cb9f23745a9fbd4539018b55e0b5", + "has_attachment": "850d4b5274bec8bc389edb7bbdbe31c17c4cd41cb3f1c49c921b94fd560d355c", + "item_type_raw": "59220831e3d9dbecc14b740a0255c54105d9cb9f23745a9fbd4539018b55e0b5", + "message_guid": "daccaf41edb0ddaf8adc7e031c8b1ff2be2afc81bae394b7dfb407c003b1bd21", + "sender": "6a19a66b9d59ed6d714da41e38d97a9d2eb11e8f68a24fd9e642aa58ff368d13", + "service": "1f1fe4ffeb2f754906d15bb793d527ec20c5f30cd5a8dadb89e764832894ea6c", + "source_file": "1ce748742cbb925609dab435d616def61f30493589560625476a24512d55c6be", + "tapback": "a249c08b5d285dcaf9172d44afde32ac6a66ffa8fa12aa94272cdfb4e70d7fd6", + "text": "67fa5060554388efb7ca08aed73ed94a7d1718abb045c43408c97b1cab4b221a" + }, + "columns": [ + { + "name": "date", + "type": "datetime" + }, + { + "name": "date_read", + "type": "datetime" + }, + { + "name": "date_delivered", + "type": "datetime" + }, + { + "name": "direction", + "type": "text" + }, + { + "name": "sender", + "type": "text" + }, + { + "name": "chat_identifier", + "type": "text" + }, + { + "name": "text", + "type": "text" + }, + { + "name": "chat_display_name", + "type": "text" + }, + { + "name": "body_source", + "type": "text" + }, + { + "name": "service", + "type": "text" + }, + { + "name": "has_attachment", + "type": "text" + }, + { + "name": "tapback", + "type": "text" + }, + { + "name": "associated_message_guid", + "type": "text" + }, + { + "name": "item_type_raw", + "type": "text" + }, + { + "name": "group_action_type_raw", + "type": "text" + }, + { + "name": "message_guid", + "type": "text" + }, + { + "name": "source_file", + "type": "text" + } + ], + "datetime_columns": { + "date": { + "in_plausible_range": true, + "parsed": 25 + }, + "date_delivered": { + "in_plausible_range": true, + "parsed": 19 + }, + "date_read": { + "in_plausible_range": true, + "parsed": 17 + } + }, + "digest": "0e3a1c5be9e95b99080d91eb8b52f37d0168ea6cfeac0af12d735bdb8f2d1a3d", + "non_empty": { + "associated_message_guid": 0, + "body_source": 25, + "chat_display_name": 0, + "chat_identifier": 25, + "date": 25, + "date_delivered": 19, + "date_read": 17, + "direction": 25, + "group_action_type_raw": 25, + "has_attachment": 5, + "item_type_raw": 25, + "message_guid": 25, + "sender": 25, + "service": 25, + "source_file": 25, + "tapback": 0, + "text": 25 + }, + "rows": 25 + } + }, + "corpus": "dleapp_imessage_bigsur", + "note": "Content-free fingerprint. Digests and counts only, no rows, because the corpora are private application profiles.", + "recorded_with_commit": "fae8bd17c03690b8d7cf7294f1f7ac2dedeb178d" +} diff --git a/scripts/artifacts/iMessageMessages.py b/scripts/artifacts/iMessageMessages.py new file mode 100644 index 0000000..6746da6 --- /dev/null +++ b/scripts/artifacts/iMessageMessages.py @@ -0,0 +1,419 @@ +__artifacts_v2__ = { + "imessageMessages": { + "name": "iMessage Messages", + "description": "Each row in chat.db's message table: composed/read/" + "delivered time, direction, sender, the chat it " + "belongs to, body text, service (iMessage vs SMS/" + "RCS), whether it carries an attachment, and tapback/" + "reaction labels where the message is one.", + "author": "@AlexisBrignoni, Claude", + "creation_date": "2026-09-01", + "last_update_date": "2026-09-01", + "requirements": "none", + "category": "iMessage (macOS)", + "notes": "Every chat.db found is parsed, so a Mac with more than one " + "user account reports each account's messages, tagged by " + "Source File. message.date/date_read/date_delivered are Mac " + "Absolute Time; the converter treats values above 1e15 as " + "nanoseconds (macOS 10.13+) and smaller values as seconds " + "(macOS 10.12 and earlier), so both eras decode correctly. " + "attributedBody is not decoded: on the validation image every " + "message carried plain text, so a row with empty text but a " + "non-null attributedBody is flagged in Body Source rather " + "than shown as an empty message. That flag path was present " + "but not exercised by the validation data.", + "paths": ( + "*/Library/Messages/chat.db*", + ), + "output_types": ["standard"], + "artifact_icon": "message-circle", + "sample_data": { + "dleapp_imessage_bigsur": "macOS Big Sur (Josh Hickman public test " + "image, thisisdfir), chat.db + chat.db-wal | 25 messages " + "(24 in the committed database, 1 recoverable only with the " + "write-ahead log applied), 1 chat, 1 handle", + }, + "data_views": { + "conversation": { + "conversationDiscriminatorColumn": "Chat Identifier", + "conversationLabelColumn": "Chat Identifier", + "textColumn": "Text", + "timeColumn": "Date", + "directionColumn": "Direction", + "directionSentValue": "Outgoing", + "senderColumn": "Sender", + } + }, + }, + "imessageAttachments": { + "name": "iMessage Attachments", + "description": "Each row in chat.db's attachment table: filename, " + "MIME type, size and created/start time, linked back " + "to the message it was sent or received on. The file " + "itself is embedded when it is present in the " + "extraction.", + "author": "@AlexisBrignoni, Claude", + "creation_date": "2026-09-01", + "last_update_date": "2026-09-01", + "requirements": "none", + "category": "iMessage (macOS)", + "notes": "created_date/start_date are Mac Absolute Time in seconds, a " + "different unit from the message table's dates. The stored " + "filename is a '~/Library/...' path; it is rewritten to a " + "'*/Library/...' pattern so the staged copy under the " + "extraction is matched and embedded. Embedding therefore " + "depends on the attachment files being present under " + "Library/Messages/Attachments, which this artifact's paths " + "glob picks up.", + "paths": ( + "*/Library/Messages/chat.db*", + "*/Library/Messages/Attachments/*", + ), + "output_types": ["standard"], + "artifact_icon": "paperclip", + "sample_data": { + "dleapp_imessage_bigsur": "macOS Big Sur (Josh Hickman public test " + "image, thisisdfir), chat.db + Attachments folder | 5 " + "attachments, all 5 present and embedded from the extraction", + }, + }, + "imessageChats": { + "name": "iMessage Chat Threads", + "description": "One row per chat.db chat: identifier, display " + "name, service, group/1:1 style, and the " + "participant handles joined via chat_handle_join.", + "author": "@AlexisBrignoni, Claude", + "creation_date": "2026-09-01", + "last_update_date": "2026-09-01", + "requirements": "none", + "category": "iMessage (macOS)", + "notes": "Every chat.db found is parsed, tagged by Source File.", + "paths": ( + "*/Library/Messages/chat.db*", + ), + "output_types": ["standard"], + "artifact_icon": "users", + "sample_data": { + "dleapp_imessage_bigsur": "macOS Big Sur (Josh Hickman public test " + "image, thisisdfir), chat.db | 1 chat thread, 1 participant " + "handle", + }, + }, + "imessageDeletedItems": { + "name": "iMessage Deletion Tombstones", + "description": "GUIDs recorded in chat.db's deleted_messages, " + "sync_deleted_chats, sync_deleted_messages and " + "sync_deleted_attachments tables: tombstones " + "Messages keeps after a chat, message or attachment " + "is deleted, even though the content itself is " + "gone.", + "author": "@AlexisBrignoni, Claude", + "creation_date": "2026-09-01", + "last_update_date": "2026-09-01", + "requirements": "none", + "category": "iMessage (macOS)", + "notes": "All four tables were empty on the validation image (no " + "deletions had occurred), so only the schema, not real " + "tombstone data, was confirmed. Forensically these rows " + "matter precisely when something WAS deleted, so this is " + "included even though the tested sample had none. " + "sync_deleted_chats.timestamp is reported with the same " + "nanosecond Mac Absolute Time handling as the message dates, " + "but that unit was not independently confirmed because the " + "table was empty here.", + "paths": ( + "*/Library/Messages/chat.db*", + ), + "output_types": ["standard"], + "artifact_icon": "trash-2", + "sample_data": { + "dleapp_imessage_bigsur": "macOS Big Sur (Josh Hickman public test " + "image, thisisdfir), chat.db | 0 rows in deleted_messages, " + "sync_deleted_chats, sync_deleted_messages and " + "sync_deleted_attachments (no deletions had occurred on this " + "image; table schemas confirmed present, tombstone data was " + "not)", + }, + }, +} + +import os +from datetime import datetime, timedelta, timezone + +from scripts.ilapfuncs import (artifact_processor, check_in_media, logfunc, + open_sqlite_db_readonly) + +# Seconds between the Unix epoch (1970-01-01) and the Mac/Cocoa epoch +# (2001-01-01). +_MAC_EPOCH = datetime(2001, 1, 1, tzinfo=timezone.utc) + +# message.date and friends are nanoseconds past the Mac epoch on macOS 10.13+ +# and seconds past it on 10.12 and earlier. A 2001-2100 value in seconds is +# ~0 to 3.1e9; the same instant in nanoseconds is ~1e17 to 3.1e18. 1e15 sits +# well above any plausible seconds value and well below any plausible +# nanosecond value, so it separates the two units cleanly. +_NS_THRESHOLD = 1_000_000_000_000_000 + + +def _cocoa_to_utc(value, unit): + """Mac Absolute Time to UTC. unit='auto' detects nanoseconds vs seconds by + magnitude (message table, which changed units across macOS releases); + unit='seconds' forces the seconds reading (attachment table).""" + if not value: + return None + try: + value = int(value) + except (TypeError, ValueError): + return None + if unit == "auto" and abs(value) > _NS_THRESHOLD: + value = value / 1_000_000_000 + try: + return _MAC_EPOCH + timedelta(seconds=value) + except (OverflowError, OSError, ValueError): + return None + + +_TAPBACK_LABELS = { + 2000: "Loved", 2001: "Liked", 2002: "Disliked", 2003: "Laughed", + 2004: "Emphasized", 2005: "Questioned", + 3000: "Removed Loved", 3001: "Removed Liked", 3002: "Removed Disliked", + 3003: "Removed Laughed", 3004: "Removed Emphasized", + 3005: "Removed Questioned", +} + + +def _chat_dbs(files_found): + """Every chat.db in the extraction (one per user account), excluding the + -wal/-shm/-journal sidecars the glob also matches.""" + return [p for p in files_found if os.path.basename(p) == "chat.db"] + + +_MESSAGES_QUERY = """ + SELECT + m.ROWID, m.guid, m.date, m.date_read, m.date_delivered, + m.is_from_me, m.service, m.text, m.attributedBody, + m.cache_has_attachments, m.item_type, m.group_action_type, + m.associated_message_type, m.associated_message_guid, + h.id AS handle_id, + c.chat_identifier, c.display_name + FROM message m + LEFT JOIN handle h ON m.handle_id = h.ROWID + LEFT JOIN chat_message_join cmj ON cmj.message_id = m.ROWID + LEFT JOIN chat c ON c.ROWID = cmj.chat_id + ORDER BY m.date +""" + + +@artifact_processor +def imessageMessages(context): + data_headers = ( + ("Date", "datetime"), ("Date Read", "datetime"), + ("Date Delivered", "datetime"), "Direction", "Sender", + "Chat Identifier", "Text", "Chat Display Name", "Body Source", + "Service", "Has Attachment", "Tapback", "Associated Message GUID", + "Item Type (raw)", "Group Action Type (raw)", "Message GUID", + "Source File", + ) + files_found = [str(f) for f in context.get_files_found()] + sources = _chat_dbs(files_found) + + data_list = [] + read_sources = [] + for source in sources: + database = open_sqlite_db_readonly(source) + if database is None: + continue + relative_source = context.get_relative_path(source) + rows_here = 0 + for row in database.execute(_MESSAGES_QUERY): + (_pk, guid, date, date_read, date_delivered, is_from_me, service, + text, attributed_body, has_attachments, item_type, + group_action_type, assoc_type, assoc_guid, handle_id, + chat_identifier, display_name) = row + + display_text = text or "" + if text: + body_source = "text" + elif attributed_body: + body_source = "attributedBody only (not decoded)" + else: + body_source = "" + + direction = "" if is_from_me is None else ("Outgoing" if is_from_me else "Incoming") + sender = "Local User" if is_from_me else (handle_id or "") + + data_list.append(( + _cocoa_to_utc(date, "auto"), _cocoa_to_utc(date_read, "auto"), + _cocoa_to_utc(date_delivered, "auto"), direction, sender, + chat_identifier or "", display_text, display_name or "", + body_source, service or "", + "Yes" if has_attachments else "", + _TAPBACK_LABELS.get(assoc_type, ""), + assoc_guid or "", + item_type if item_type is not None else "", + group_action_type if group_action_type is not None else "", + guid or "", relative_source, + )) + rows_here += 1 + database.close() + if rows_here: + read_sources.append(relative_source) + + logfunc(f"iMessage Messages: {len(data_list)} message(s) across " + f"{len(read_sources)} chat.db file(s).") + return data_headers, data_list, "\n".join(read_sources) + + +_ATTACHMENTS_QUERY = """ + SELECT + a.ROWID, a.guid, a.filename, a.transfer_name, a.mime_type, + a.total_bytes, a.created_date, a.start_date, a.is_sticker, + a.transfer_state, m.guid AS message_guid + FROM attachment a + LEFT JOIN message_attachment_join maj ON maj.attachment_id = a.ROWID + LEFT JOIN message m ON m.ROWID = maj.message_id + ORDER BY a.created_date +""" + + +@artifact_processor +def imessageAttachments(context): + data_headers = ( + ("Created", "datetime"), ("Start", "datetime"), "Filename", + ("File", "media"), "MIME Type", "Size (bytes)", "Is Sticker", + "Transfer State (raw)", "Attachment GUID", "Message GUID", + "Source File", + ) + files_found = [str(f) for f in context.get_files_found()] + sources = _chat_dbs(files_found) + + data_list = [] + read_sources = [] + embedded = 0 + for source in sources: + database = open_sqlite_db_readonly(source) + if database is None: + continue + relative_source = context.get_relative_path(source) + rows_here = 0 + for row in database.execute(_ATTACHMENTS_QUERY): + (_pk, guid, filename, transfer_name, mime_type, total_bytes, + created_date, start_date, is_sticker, transfer_state, + message_guid) = row + + media_ref = "" + if filename: + # Stored as '~/Library/Messages/Attachments/...'. Rewrite the + # leading '~' to a '*' pattern so the staged copy is matched. + pattern = "*" + filename[1:] if filename.startswith("~") else filename + reference = check_in_media( + pattern, name=transfer_name or os.path.basename(filename)) + if reference: + media_ref = reference + embedded += 1 + + data_list.append(( + _cocoa_to_utc(created_date, "seconds"), + _cocoa_to_utc(start_date, "seconds"), + filename or "", media_ref, mime_type or "", + total_bytes if total_bytes is not None else "", + "Yes" if is_sticker else "", + transfer_state if transfer_state is not None else "", + guid or "", message_guid or "", relative_source, + )) + rows_here += 1 + database.close() + if rows_here: + read_sources.append(relative_source) + + logfunc(f"iMessage Attachments: {len(data_list)} attachment(s); " + f"{embedded} embedded from the extraction.") + return data_headers, data_list, "\n".join(read_sources) + + +_CHATS_QUERY = """ + SELECT c.ROWID, c.guid, c.chat_identifier, c.display_name, + c.service_name, c.style, c.is_archived + FROM chat c + ORDER BY c.ROWID +""" + + +@artifact_processor +def imessageChats(context): + data_headers = ("Chat Identifier", "Display Name", "Service", "Style (raw)", + "Archived", "Participants", "Chat GUID", "Source File") + files_found = [str(f) for f in context.get_files_found()] + sources = _chat_dbs(files_found) + + data_list = [] + read_sources = [] + for source in sources: + database = open_sqlite_db_readonly(source) + if database is None: + continue + relative_source = context.get_relative_path(source) + rows_here = 0 + for row in database.execute(_CHATS_QUERY): + chat_pk, guid, chat_identifier, display_name, service_name, style, is_archived = row + participants = [r[0] for r in database.execute( + "SELECT h.id FROM chat_handle_join chj " + "JOIN handle h ON h.ROWID = chj.handle_id WHERE chj.chat_id = ?", + (chat_pk,))] + data_list.append(( + chat_identifier or "", display_name or "", service_name or "", + style if style is not None else "", "Yes" if is_archived else "", + ", ".join(p for p in participants if p), guid or "", relative_source, + )) + rows_here += 1 + database.close() + if rows_here: + read_sources.append(relative_source) + + logfunc(f"iMessage Chat Threads: {len(data_list)} chat(s) across " + f"{len(read_sources)} chat.db file(s).") + return data_headers, data_list, "\n".join(read_sources) + + +_DELETED_QUERIES = ( + ("deleted_messages", "SELECT guid, NULL, NULL FROM deleted_messages"), + ("sync_deleted_chats", "SELECT guid, recordID, timestamp FROM sync_deleted_chats"), + ("sync_deleted_messages", "SELECT guid, recordID, NULL FROM sync_deleted_messages"), + ("sync_deleted_attachments", "SELECT guid, recordID, NULL FROM sync_deleted_attachments"), +) + + +@artifact_processor +def imessageDeletedItems(context): + data_headers = ("Tombstone Table", "GUID", "CloudKit Record ID", + ("Timestamp", "datetime"), "Source File") + files_found = [str(f) for f in context.get_files_found()] + sources = _chat_dbs(files_found) + + data_list = [] + read_sources = [] + for source in sources: + database = open_sqlite_db_readonly(source) + if database is None: + continue + relative_source = context.get_relative_path(source) + rows_here = 0 + for table_name, query in _DELETED_QUERIES: + try: + rows = database.execute(query).fetchall() + except Exception as ex: # pylint: disable=broad-exception-caught + logfunc(f"iMessage Deletion Tombstones: could not read " + f"'{table_name}' in '{relative_source}': {ex}") + continue + for guid, record_id, timestamp in rows: + data_list.append(( + table_name, guid or "", record_id or "", + _cocoa_to_utc(timestamp, "auto"), relative_source, + )) + rows_here += 1 + database.close() + if rows_here: + read_sources.append(relative_source) + + logfunc(f"iMessage Deletion Tombstones: {len(data_list)} tombstone(s).") + return data_headers, data_list, "\n".join(read_sources)