From 13a979a93ad92ce8de7161b1c821bd44809d0164 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 24 Jun 2026 05:40:09 -0500 Subject: [PATCH 1/3] Memoize UI contract source reads --- Tests/UIAutomationSurfaceContractTests.swift | 418 +++++++++---------- 1 file changed, 195 insertions(+), 223 deletions(-) diff --git a/Tests/UIAutomationSurfaceContractTests.swift b/Tests/UIAutomationSurfaceContractTests.swift index 5c48a2b3..7613c5ea 100644 --- a/Tests/UIAutomationSurfaceContractTests.swift +++ b/Tests/UIAutomationSurfaceContractTests.swift @@ -2,42 +2,55 @@ // It asserts that menubar sources keep their stable AX identifiers and that // the smoke script still references them, so external UI automation stays in sync. // It runs no UI and exercises no runtime behavior; it only greps source text. +// +// Adding a new contract guard is purely additive: call `contractSource("Sources/.../X.swift")` +// inline inside an assertion. Do NOT reintroduce a top-of-suite block of +// `let xSource = readUIAutomationContractFile(...)` declarations — that append-only +// hotspot is what made two concurrent UI PRs collide on a duplicate `let` declaration +// (the same pattern that bit AnalyticsEventPolicy.swift). `contractSource` reads and +// memoizes each file on demand, so repeated reads of the same path are free and two +// PRs can add guards for the same file without redeclaring anything. import Foundation +// On-demand, memoized source reader. Each path is read at most once per run. +private var uiAutomationContractSourceCache: [String: String] = [:] + +private func contractSource(_ relativePath: String) -> String { + if let cached = uiAutomationContractSourceCache[relativePath] { + return cached + } + let contents = readUIAutomationContractFile(relativePath) + uiAutomationContractSourceCache[relativePath] = contents + return contents +} + func testUIAutomationSurfaceContract() { runSuite("UI automation surface contract - menubar controls expose stable identifiers") { - let appSource = readUIAutomationContractFile("Sources/TranscriptedApp.swift") - let actionRowSource = readUIAutomationContractFile("Sources/UI/MenuBar/MenuBarActionRowView.swift") - let menuTokensSource = readUIAutomationContractFile("Sources/UI/MenuBar/MenuTokens.swift") - let primarySource = readUIAutomationContractFile("Sources/UI/MenuBar/MenuBarPrimaryActionsView.swift") - let utilitySource = readUIAutomationContractFile("Sources/UI/MenuBar/MenuBarUtilityActionsView.swift") - let smokeScript = readUIAutomationContractFile("scripts/entrypoints/build.sh") - assertTrue( - appSource.contains("transcripted.status-item.button") - && appSource.contains("setAccessibilityIdentifier(\"transcripted.status-item.button\")"), + contractSource("Sources/TranscriptedApp.swift").contains("transcripted.status-item.button") + && contractSource("Sources/TranscriptedApp.swift").contains("setAccessibilityIdentifier(\"transcripted.status-item.button\")"), "the real menu bar status item should expose a stable AX identifier for external UI automation" ) assertTrue( - actionRowSource.contains("let automationIdentifier: String") - && actionRowSource.contains("setAutomationIdentifier(_ rawValue: String)") - && actionRowSource.contains("setAccessibilityIdentifier(rawValue)") - && actionRowSource.contains("setAccessibilityRole(.button)") - && actionRowSource.contains("setAccessibilityLabel(title)") - && actionRowSource.contains("override func accessibilityPerformPress()") - && actionRowSource.contains("guard isEnabled else { return false }") - && actionRowSource.contains("accessibilityIdentifier()"), + contractSource("Sources/UI/MenuBar/MenuBarActionRowView.swift").contains("let automationIdentifier: String") + && contractSource("Sources/UI/MenuBar/MenuBarActionRowView.swift").contains("setAutomationIdentifier(_ rawValue: String)") + && contractSource("Sources/UI/MenuBar/MenuBarActionRowView.swift").contains("setAccessibilityIdentifier(rawValue)") + && contractSource("Sources/UI/MenuBar/MenuBarActionRowView.swift").contains("setAccessibilityRole(.button)") + && contractSource("Sources/UI/MenuBar/MenuBarActionRowView.swift").contains("setAccessibilityLabel(title)") + && contractSource("Sources/UI/MenuBar/MenuBarActionRowView.swift").contains("override func accessibilityPerformPress()") + && contractSource("Sources/UI/MenuBar/MenuBarActionRowView.swift").contains("guard isEnabled else { return false }") + && contractSource("Sources/UI/MenuBar/MenuBarActionRowView.swift").contains("accessibilityIdentifier()"), "menubar smoke snapshots should carry the same accessibility identifier and AXPress path AppKit automation sees" ) assertTrue( - menuTokensSource.contains("static let minimumHitTargetSize: CGFloat = 40") - && menuTokensSource.contains("static let panelHeight: CGFloat = 480") - && actionRowSource.contains("MenuTokens.minimumHitTargetSize") - && actionRowSource.contains("MenuTokens.utilityActionRowHeight") - && actionRowSource.contains("MenuTokens.compactActionRowHeight"), + contractSource("Sources/UI/MenuBar/MenuTokens.swift").contains("static let minimumHitTargetSize: CGFloat = 40") + && contractSource("Sources/UI/MenuBar/MenuTokens.swift").contains("static let panelHeight: CGFloat = 480") + && contractSource("Sources/UI/MenuBar/MenuBarActionRowView.swift").contains("MenuTokens.minimumHitTargetSize") + && contractSource("Sources/UI/MenuBar/MenuBarActionRowView.swift").contains("MenuTokens.utilityActionRowHeight") + && contractSource("Sources/UI/MenuBar/MenuBarActionRowView.swift").contains("MenuTokens.compactActionRowHeight"), "menubar action rows should keep a real 40pt hit-target floor and a panel height that keeps default rows visible" ) @@ -49,7 +62,8 @@ func testUIAutomationSurfaceContract() { "transcripted.menubar.primary.recent-meetings", ] { assertTrue( - primarySource.contains(identifier) && smokeScript.contains(identifier), + contractSource("Sources/UI/MenuBar/MenuBarPrimaryActionsView.swift").contains(identifier) + && contractSource("scripts/entrypoints/build.sh").contains(identifier), "\(identifier) should be attached in source and enforced by launch smoke" ) } @@ -62,7 +76,8 @@ func testUIAutomationSurfaceContract() { "transcripted.menubar.utility.quit", ] { assertTrue( - utilitySource.contains(identifier) && smokeScript.contains(identifier), + contractSource("Sources/UI/MenuBar/MenuBarUtilityActionsView.swift").contains(identifier) + && contractSource("scripts/entrypoints/build.sh").contains(identifier), "\(identifier) should be attached in source and enforced by launch smoke" ) } @@ -70,19 +85,16 @@ func testUIAutomationSurfaceContract() { } runSuite("UI automation surface contract - menubar controls keep polished hit targets") { - let tokenSource = readUIAutomationContractFile("Sources/UI/MenuBar/MenuTokens.swift") - let actionRowSource = readUIAutomationContractFile("Sources/UI/MenuBar/MenuBarActionRowView.swift") - let iconButtonSource = readUIAutomationContractFile("Sources/UI/MenuBar/MenuIconButton.swift") - let outlineButtonSource = readUIAutomationContractFile("Sources/UI/MenuBar/MenuOutlineButton.swift") - let modelStatusSource = readUIAutomationContractFile("Sources/UI/MenuBar/MenuBarModelStatusView.swift") - assertTrue( - tokenSource.contains("minimumHitTargetSize: CGFloat = 40") - && actionRowSource.contains("MenuTokens.minimumHitTargetSize"), + contractSource("Sources/UI/MenuBar/MenuTokens.swift").contains("minimumHitTargetSize: CGFloat = 40") + && contractSource("Sources/UI/MenuBar/MenuBarActionRowView.swift").contains("MenuTokens.minimumHitTargetSize"), "menubar rows should stay at or above the 40px minimum hit target" ) - for source in [iconButtonSource, outlineButtonSource] { + for source in [ + contractSource("Sources/UI/MenuBar/MenuIconButton.swift"), + contractSource("Sources/UI/MenuBar/MenuOutlineButton.swift"), + ] { assertTrue( source.contains("override func hitTest(_ point: NSPoint)") && source.contains("let localPoint = convert(point, from: superview)") @@ -95,18 +107,16 @@ func testUIAutomationSurfaceContract() { } assertTrue( - modelStatusSource.contains("NSFont.monospacedDigitSystemFont") - && modelStatusSource.contains("setAccessibilityRole(.button)") - && modelStatusSource.contains("setAccessibilityValue(label.stringValue)") - && modelStatusSource.contains("override func resetCursorRects()") - && modelStatusSource.contains("override func accessibilityPerformPress()"), + contractSource("Sources/UI/MenuBar/MenuBarModelStatusView.swift").contains("NSFont.monospacedDigitSystemFont") + && contractSource("Sources/UI/MenuBar/MenuBarModelStatusView.swift").contains("setAccessibilityRole(.button)") + && contractSource("Sources/UI/MenuBar/MenuBarModelStatusView.swift").contains("setAccessibilityValue(label.stringValue)") + && contractSource("Sources/UI/MenuBar/MenuBarModelStatusView.swift").contains("override func resetCursorRects()") + && contractSource("Sources/UI/MenuBar/MenuBarModelStatusView.swift").contains("override func accessibilityPerformPress()"), "menubar model status should keep tabular progress digits and a real AX button contract" ) } runSuite("UI automation surface contract - app commands expose capture shortcuts") { - let commandsSource = readUIAutomationContractFile("Sources/TranscriptedMenuCommands.swift") - for requiredCommandHook in [ "CommandMenu(\"Capture\")", "Button(\"Start Dictation\")", @@ -119,14 +129,11 @@ func testUIAutomationSurfaceContract() { "appDelegate.menuImportAudio()", ".keyboardShortcut(\"o\", modifiers: .command)", ] { - assertTrue(commandsSource.contains(requiredCommandHook), "\(requiredCommandHook) should stay pinned in the app command menu") + assertTrue(contractSource("Sources/TranscriptedMenuCommands.swift").contains(requiredCommandHook), "\(requiredCommandHook) should stay pinned in the app command menu") } } runSuite("UI automation surface contract - app commands expose primary Go shortcuts") { - let commandsSource = readUIAutomationContractFile("Sources/TranscriptedMenuCommands.swift") - let pagesSource = readUIAutomationContractFile("Sources/UI/Settings/TranscriptedSettingsPage.swift") - for requiredCommandHook in [ "CommandMenu(\"Go\")", "Button(\"Home\")", @@ -145,7 +152,7 @@ func testUIAutomationSurfaceContract() { "appDelegate.menuFindSpeaker()", ".keyboardShortcut(\"f\", modifiers: .command)", ] { - assertTrue(commandsSource.contains(requiredCommandHook), "\(requiredCommandHook) should stay pinned in the Go command menu") + assertTrue(contractSource("Sources/TranscriptedMenuCommands.swift").contains(requiredCommandHook), "\(requiredCommandHook) should stay pinned in the Go command menu") } for requiredPageHook in [ @@ -155,13 +162,11 @@ func testUIAutomationSurfaceContract() { "case .connectAgent: return \"4\"", "return \"\\(title) ⌘\\(key)\"", ] { - assertTrue(pagesSource.contains(requiredPageHook), "\(requiredPageHook) should keep sidebar help aligned with Go shortcuts") + assertTrue(contractSource("Sources/UI/Settings/TranscriptedSettingsPage.swift").contains(requiredPageHook), "\(requiredPageHook) should keep sidebar help aligned with Go shortcuts") } } runSuite("UI automation surface contract - app commands route through existing delegate entry points") { - let appSource = readUIAutomationContractFile("Sources/TranscriptedApp.swift") - for requiredAppHook in [ "TranscriptedMenuCommands(appDelegate: appDelegate)", "func menuStartDictation()", @@ -175,13 +180,11 @@ func testUIAutomationSurfaceContract() { "func menuFindSpeaker()", "settingsWindowController.focusSpeakerSearch(source: \"menu_command\")", ] { - assertTrue(appSource.contains(requiredAppHook), "\(requiredAppHook) should keep app commands wired through existing app-delegate actions") + assertTrue(contractSource("Sources/TranscriptedApp.swift").contains(requiredAppHook), "\(requiredAppHook) should keep app commands wired through existing app-delegate actions") } } runSuite("UI automation surface contract - app commands do not remap global trigger preferences") { - let commandsSource = readUIAutomationContractFile("Sources/TranscriptedMenuCommands.swift") - for forbiddenTriggerHook in [ "PhysicalDictationTriggerPreferences", "HotkeyPreferences", @@ -195,27 +198,13 @@ func testUIAutomationSurfaceContract() { "modifiers: [.control", ] { assertFalse( - commandsSource.contains(forbiddenTriggerHook), + contractSource("Sources/TranscriptedMenuCommands.swift").contains(forbiddenTriggerHook), "app-active commands must not remap or shadow global recordable trigger preferences (\(forbiddenTriggerHook))" ) } } runSuite("UI automation surface contract - major settings and Home flows stay mapped") { - let pagesSource = readUIAutomationContractFile("Sources/UI/Settings/TranscriptedSettingsPage.swift") - let settingsSidebarSource = readUIAutomationContractFile("Sources/UI/Settings/TranscriptedSettingsSidebar.swift") - let settingsComponentsSource = readUIAutomationContractFile("Sources/UI/Settings/TranscriptedSettingsComponents.swift") - let generalControlsSource = readUIAutomationContractFile("Sources/UI/Settings/TranscriptedSettingsGeneralControls.swift") - let settingsRowsSource = readUIAutomationContractFile("Sources/UI/Settings/TranscriptedSettingsRows.swift") - let settingsSource = readUIAutomationContractFile("Sources/UI/Settings/TranscriptedSettingsView.swift") - let homeSource = readUIAutomationContractFile("Sources/UI/Settings/HomeView.swift") - let meetingAudioPlaybackSource = readUIAutomationContractFile("Sources/UI/Shared/MeetingAudioPlayback.swift") - let onboardingSource = readUIAutomationContractFile("Sources/UI/Settings/PermissionsOnboardingView.swift") - let speakerReviewSource = readUIAutomationContractFile("Sources/UI/Settings/SpeakerNamingSheet.swift") - let agentSettingsSource = readUIAutomationContractFile("Sources/UI/Settings/AgentConnectionSettingsPage.swift") - let deletePolicySource = readUIAutomationContractFile("Sources/UI/Settings/HomeDeleteConfirmationPolicy.swift") - let failedMeetingRecoverySource = readUIAutomationContractFile("Sources/UI/Settings/FailedMeetingRecoveryPresentation.swift") - for pageCase in [ "case home", "case dictations", @@ -230,29 +219,29 @@ func testUIAutomationSurfaceContract() { "case support", "case about", ] { - assertTrue(pagesSource.contains(pageCase), "\(pageCase) should stay in the settings navigation surface map") + assertTrue(contractSource("Sources/UI/Settings/TranscriptedSettingsPage.swift").contains(pageCase), "\(pageCase) should stay in the settings navigation surface map") } assertTrue( - pagesSource.contains("var automationIdentifier: String") - && settingsSidebarSource.contains(".accessibilityIdentifier(page.automationIdentifier)"), + contractSource("Sources/UI/Settings/TranscriptedSettingsPage.swift").contains("var automationIdentifier: String") + && contractSource("Sources/UI/Settings/TranscriptedSettingsSidebar.swift").contains(".accessibilityIdentifier(page.automationIdentifier)"), "settings sidebar pages should expose stable automation identifiers" ) assertTrue( - generalControlsSource.contains(".frame(width: 40, height: 40)") - && generalControlsSource.contains("accessibilityIdentifier(\"transcripted.settings.general.info.\\(automationSlug(info.title))\")"), + contractSource("Sources/UI/Settings/TranscriptedSettingsGeneralControls.swift").contains(".frame(width: 40, height: 40)") + && contractSource("Sources/UI/Settings/TranscriptedSettingsGeneralControls.swift").contains("accessibilityIdentifier(\"transcripted.settings.general.info.\\(automationSlug(info.title))\")"), "General settings info buttons should keep compact visuals with a 40pt hit target and stable AX identifiers" ) assertTrue( - settingsRowsSource.contains(".frame(width: 40, height: 40)") - && settingsRowsSource.contains(".accessibilityLabel(Text(\"Remove correction\"))"), + contractSource("Sources/UI/Settings/TranscriptedSettingsRows.swift").contains(".frame(width: 40, height: 40)") + && contractSource("Sources/UI/Settings/TranscriptedSettingsRows.swift").contains(".accessibilityLabel(Text(\"Remove correction\"))"), "custom dictionary remove controls should keep a 40pt destructive hit target with a clear AX label" ) assertTrue( - settingsSource.contains("Label(\"Add correction\", systemImage: \"plus\")") - && settingsSource.contains(".frame(minHeight: 40)") - && settingsSource.contains(".contentShape(RoundedRectangle(cornerRadius: 8, style: .continuous))"), + contractSource("Sources/UI/Settings/TranscriptedSettingsView.swift").contains("Label(\"Add correction\", systemImage: \"plus\")") + && contractSource("Sources/UI/Settings/TranscriptedSettingsView.swift").contains(".frame(minHeight: 40)") + && contractSource("Sources/UI/Settings/TranscriptedSettingsView.swift").contains(".contentShape(RoundedRectangle(cornerRadius: 8, style: .continuous))"), "custom dictionary add correction should keep a 40pt tactile action target" ) @@ -264,7 +253,7 @@ func testUIAutomationSurfaceContract() { "trackSettingsToggle(\"local_ai_meeting_summaries\"", "trackSettingsToggle(\"live_meeting_sidecar\"", ] { - assertTrue(settingsSource.contains(requiredSourceHook), "\(requiredSourceHook) should stay source-addressable") + assertTrue(contractSource("Sources/UI/Settings/TranscriptedSettingsView.swift").contains(requiredSourceHook), "\(requiredSourceHook) should stay source-addressable") } for requiredHomeActionHook in [ @@ -274,10 +263,10 @@ func testUIAutomationSurfaceContract() { "HomeDeleteConfirmationPolicy.failedMeeting", "homeDeleteConfirmation = HomeDeleteConfirmation(", ] { - assertTrue(settingsSource.contains(requiredHomeActionHook), "\(requiredHomeActionHook) should keep Home action coverage visible") + assertTrue(contractSource("Sources/UI/Settings/TranscriptedSettingsView.swift").contains(requiredHomeActionHook), "\(requiredHomeActionHook) should keep Home action coverage visible") } assertFalse( - settingsSource.contains("presentFailedMeetingDeleteConfirmation("), + contractSource("Sources/UI/Settings/TranscriptedSettingsView.swift").contains("presentFailedMeetingDeleteConfirmation("), "failed-meeting delete confirmation should use SwiftUI alert state instead of a hand-built NSAlert" ) @@ -291,12 +280,12 @@ func testUIAutomationSurfaceContract() { "ClosureMenuItem(menuItem: item)", "title: \"Copy for agent\"", ] { - assertTrue(homeSource.contains(requiredHomeRendererHook), "\(requiredHomeRendererHook) should keep Home action rendering visible") + assertTrue(contractSource("Sources/UI/Settings/HomeView.swift").contains(requiredHomeRendererHook), "\(requiredHomeRendererHook) should keep Home action rendering visible") } assertTrue( - homeSource.contains("private enum HomeHitTarget") - && homeSource.contains("static let minimum: CGFloat = 40") - && homeSource.contains("HomeHitTarget.minimum"), + contractSource("Sources/UI/Settings/HomeView.swift").contains("private enum HomeHitTarget") + && contractSource("Sources/UI/Settings/HomeView.swift").contains("static let minimum: CGFloat = 40") + && contractSource("Sources/UI/Settings/HomeView.swift").contains("HomeHitTarget.minimum"), "Home icon buttons and compact row actions should keep a shared 40pt hit-target floor" ) for requiredFailedMeetingPolicyHook in [ @@ -305,37 +294,37 @@ func testUIAutomationSurfaceContract() { "clearIsDestructive: hasRetainedAudioFiles", "return \"This meeting does not have enough saved audio to retry.\"", ] { - assertTrue(failedMeetingRecoverySource.contains(requiredFailedMeetingPolicyHook), "\(requiredFailedMeetingPolicyHook) should keep failed-meeting action policy visible") + assertTrue(contractSource("Sources/UI/Settings/FailedMeetingRecoveryPresentation.swift").contains(requiredFailedMeetingPolicyHook), "\(requiredFailedMeetingPolicyHook) should keep failed-meeting action policy visible") } assertTrue( - settingsComponentsSource.contains(".frame(minHeight: 40)") - && settingsComponentsSource.contains(".contentShape(RoundedRectangle(cornerRadius: 8, style: .continuous))"), + contractSource("Sources/UI/Settings/TranscriptedSettingsComponents.swift").contains(".frame(minHeight: 40)") + && contractSource("Sources/UI/Settings/TranscriptedSettingsComponents.swift").contains(".contentShape(RoundedRectangle(cornerRadius: 8, style: .continuous))"), "shared inline settings actions should keep a 40pt hit floor for failed-meeting recovery controls" ) assertTrue( - homeSource.contains(".frame(minHeight: 40, alignment: .leading)") - && homeSource.contains(".accessibilityIdentifier(\"transcripted.home.audio.inline-toggle\")"), + contractSource("Sources/UI/Settings/HomeView.swift").contains(".frame(minHeight: 40, alignment: .leading)") + && contractSource("Sources/UI/Settings/HomeView.swift").contains(".accessibilityIdentifier(\"transcripted.home.audio.inline-toggle\")"), "Home retained-audio play controls should keep a 40pt hit floor" ) assertTrue( - settingsRowsSource.contains(".frame(minHeight: 40, alignment: .leading)") - && settingsRowsSource.contains("struct SettingsRecentMeetingAudioControl"), + contractSource("Sources/UI/Settings/TranscriptedSettingsRows.swift").contains(".frame(minHeight: 40, alignment: .leading)") + && contractSource("Sources/UI/Settings/TranscriptedSettingsRows.swift").contains("struct SettingsRecentMeetingAudioControl"), "Settings retained-audio play controls should keep a 40pt hit floor" ) assertTrue( - meetingAudioPlaybackSource.contains(".frame(minHeight: 40)") - && meetingAudioPlaybackSource.contains("struct MeetingAudioSourceMenu"), + contractSource("Sources/UI/Shared/MeetingAudioPlayback.swift").contains(".frame(minHeight: 40)") + && contractSource("Sources/UI/Shared/MeetingAudioPlayback.swift").contains("struct MeetingAudioSourceMenu"), "retained-audio source menus should keep a 40pt hit floor" ) assertFalse( - homeSource.contains("representedObject = item.id"), + contractSource("Sources/UI/Settings/HomeView.swift").contains("representedObject = item.id"), "Home row menus should not depend on unstable SwiftUI-generated menu item IDs" ) assertTrue( - settingsComponentsSource.contains(".frame(width: 40, height: 40)") - && settingsComponentsSource.contains(".accessibilityIdentifier(\"transcripted.settings.activity-card.dismiss\")") - && settingsComponentsSource.contains(".frame(minHeight: 40)") - && settingsComponentsSource.contains(".contentShape(RoundedRectangle(cornerRadius: 8, style: .continuous))"), + contractSource("Sources/UI/Settings/TranscriptedSettingsComponents.swift").contains(".frame(width: 40, height: 40)") + && contractSource("Sources/UI/Settings/TranscriptedSettingsComponents.swift").contains(".accessibilityIdentifier(\"transcripted.settings.activity-card.dismiss\")") + && contractSource("Sources/UI/Settings/TranscriptedSettingsComponents.swift").contains(".frame(minHeight: 40)") + && contractSource("Sources/UI/Settings/TranscriptedSettingsComponents.swift").contains(".contentShape(RoundedRectangle(cornerRadius: 8, style: .continuous))"), "activity cards should keep 40pt action and dismiss hit targets for Home progress/notice cards" ) @@ -349,21 +338,21 @@ func testUIAutomationSurfaceContract() { // root view; SwiftUI keeps only the last, shadowing the (first) // delete confirmation. The three states must share one presenter. assertTrue( - homeSource.contains("DispatchQueue.main.async { [handler] in handler() }"), + contractSource("Sources/UI/Settings/HomeView.swift").contains("DispatchQueue.main.async { [handler] in handler() }"), "ClosureMenuItem should defer its handler off the NSMenu.popUp tracking loop so menu-triggered SwiftUI alerts/sheets present" ) assertTrue( - settingsSource.contains(".alert(item: rootAlertBinding)") - && settingsSource.contains("enum RootAlert") - && settingsSource.contains("case deleteConfirmation") - && settingsSource.contains("case deleteFailure") - && settingsSource.contains("case audioRetention"), + contractSource("Sources/UI/Settings/TranscriptedSettingsView.swift").contains(".alert(item: rootAlertBinding)") + && contractSource("Sources/UI/Settings/TranscriptedSettingsView.swift").contains("enum RootAlert") + && contractSource("Sources/UI/Settings/TranscriptedSettingsView.swift").contains("case deleteConfirmation") + && contractSource("Sources/UI/Settings/TranscriptedSettingsView.swift").contains("case deleteFailure") + && contractSource("Sources/UI/Settings/TranscriptedSettingsView.swift").contains("case audioRetention"), "the Home delete, delete-failure, and audio-retention alerts should present through one rootAlertBinding so none is shadowed" ) assertFalse( - settingsSource.contains(".alert(item: $homeDeleteConfirmation)") - || settingsSource.contains(".alert(item: $homeDeleteFailure)") - || settingsSource.contains(".alert(item: $pendingAudioRetentionWindow)"), + contractSource("Sources/UI/Settings/TranscriptedSettingsView.swift").contains(".alert(item: $homeDeleteConfirmation)") + || contractSource("Sources/UI/Settings/TranscriptedSettingsView.swift").contains(".alert(item: $homeDeleteFailure)") + || contractSource("Sources/UI/Settings/TranscriptedSettingsView.swift").contains(".alert(item: $pendingAudioRetentionWindow)"), "Home alerts must not be re-stacked as separate `.alert(item:)` modifiers — stacked legacy alerts shadow all but the last" ) // The shared binding must dismiss only the active alert via @@ -372,8 +361,8 @@ func testUIAutomationSurfaceContract() { // everything would wipe it before it presents. (HomeRootAlertPolicyTests // covers the priority/dismissal behavior directly.) assertTrue( - settingsSource.contains("HomeRootAlertPolicy.activeSlot") - && settingsSource.contains("switch activeRootAlert"), + contractSource("Sources/UI/Settings/TranscriptedSettingsView.swift").contains("HomeRootAlertPolicy.activeSlot") + && contractSource("Sources/UI/Settings/TranscriptedSettingsView.swift").contains("switch activeRootAlert"), "the shared alert binding should clear only the dismissed alert through HomeRootAlertPolicy, not reset all three states" ) @@ -384,23 +373,20 @@ func testUIAutomationSurfaceContract() { // an error instead of a silent dead click. Behavior is covered by // OwnFileResolverTests; these guard the wiring so a regression that re-adds a // raw stale-path call (the #1126/#1131/#1134 whack-a-mole) fails CI. - let ownFileResolverSource = readUIAutomationContractFile("Sources/UI/Shared/OwnFileResolver.swift") - let playbackSource = readUIAutomationContractFile("Sources/UI/Shared/MeetingAudioPlayback.swift") - assertTrue( - ownFileResolverSource.contains("static func resolveForReveal(") - && ownFileResolverSource.contains("static func resolveExistingFile(") - && ownFileResolverSource.contains("static func resolveExistingFiles(") - && ownFileResolverSource.contains("case reveal([URL])") - && ownFileResolverSource.contains("case unavailable"), + contractSource("Sources/UI/Shared/OwnFileResolver.swift").contains("static func resolveForReveal(") + && contractSource("Sources/UI/Shared/OwnFileResolver.swift").contains("static func resolveExistingFile(") + && contractSource("Sources/UI/Shared/OwnFileResolver.swift").contains("static func resolveExistingFiles(") + && contractSource("Sources/UI/Shared/OwnFileResolver.swift").contains("case reveal([URL])") + && contractSource("Sources/UI/Shared/OwnFileResolver.swift").contains("case unavailable"), "OwnFileResolver must keep both reveal (with enclosing-folder fallback) and open/play (regular-file-only) resolution modes" ) assertTrue( - settingsSource.contains("private func revealOwnFile(") - && settingsSource.contains("OwnFileResolver.resolveForReveal(candidateURLs:") - && settingsSource.contains("private func openOwnFile(") - && settingsSource.contains("OwnFileResolver.resolveExistingFile(candidateURLs:"), + contractSource("Sources/UI/Settings/TranscriptedSettingsView.swift").contains("private func revealOwnFile(") + && contractSource("Sources/UI/Settings/TranscriptedSettingsView.swift").contains("OwnFileResolver.resolveForReveal(candidateURLs:") + && contractSource("Sources/UI/Settings/TranscriptedSettingsView.swift").contains("private func openOwnFile(") + && contractSource("Sources/UI/Settings/TranscriptedSettingsView.swift").contains("OwnFileResolver.resolveExistingFile(candidateURLs:"), "Home reveal/open should route through OwnFileResolver helpers, surfacing presentHomeActionFailure on .unavailable instead of a dead click" ) @@ -417,7 +403,7 @@ func testUIAutomationSurfaceContract() { "NSWorkspace.shared.open(transcriptURL)", ] { assertFalse( - settingsSource.contains(staleRawCall), + contractSource("Sources/UI/Settings/TranscriptedSettingsView.swift").contains(staleRawCall), "Home own-file action must not call NSWorkspace on a raw scan-time URL (\(staleRawCall)) — route it through OwnFileResolver" ) } @@ -425,19 +411,19 @@ func testUIAutomationSurfaceContract() { // Copy/export and re-transcribe must surface a failure, not a silent beep, // when the source file cannot be resolved. assertFalse( - settingsSource.contains("NSWorkspace.shared.activateFileViewerSelecting(audioRevealURLs)"), + contractSource("Sources/UI/Settings/TranscriptedSettingsView.swift").contains("NSWorkspace.shared.activateFileViewerSelecting(audioRevealURLs)"), "failed-meeting reveal audio must route through OwnFileResolver, not beep-or-reveal on raw URLs" ) assertTrue( - settingsSource.contains("Could not copy meeting") - && settingsSource.contains("Could not re-transcribe meeting"), + contractSource("Sources/UI/Settings/TranscriptedSettingsView.swift").contains("Could not copy meeting") + && contractSource("Sources/UI/Settings/TranscriptedSettingsView.swift").contains("Could not re-transcribe meeting"), "copy-for-agent and re-transcribe should surface a failure alert when the own file is missing, instead of NSSound.beep()" ) // Retained-audio playback follows recompressed/moved files instead of going // silently Unavailable on a stale path. assertTrue( - playbackSource.contains("OwnFileResolver.resolveExistingFile(candidateURLs:"), + contractSource("Sources/UI/Shared/MeetingAudioPlayback.swift").contains("OwnFileResolver.resolveExistingFile(candidateURLs:"), "meeting audio playback should resolve each source URL through OwnFileResolver so WAV→M4A recompression still plays" ) @@ -445,9 +431,9 @@ func testUIAutomationSurfaceContract() { // still on disk (stale path), instead of letting the row reappear // unexplained. Deletion intentionally does not use the lenient resolver. assertTrue( - settingsSource.contains("result.removedTranscriptURLs.isEmpty") - && settingsSource.contains("FileManager.default.fileExists(atPath: item.transcriptURL.path)") - && settingsSource.contains("presentHomeActionFailure("), + contractSource("Sources/UI/Settings/TranscriptedSettingsView.swift").contains("result.removedTranscriptURLs.isEmpty") + && contractSource("Sources/UI/Settings/TranscriptedSettingsView.swift").contains("FileManager.default.fileExists(atPath: item.transcriptURL.path)") + && contractSource("Sources/UI/Settings/TranscriptedSettingsView.swift").contains("presentHomeActionFailure("), "deleteMeeting should detect a no-op delete (stale path) and surface a failure rather than silently re-showing the row" ) @@ -457,11 +443,11 @@ func testUIAutomationSurfaceContract() { // full-width hit shape (its idle background is Color.clear) and the // canvas-header action button must not draw a focus ring. assertTrue( - homeSource.contains(".focusEffectDisabled()"), + contractSource("Sources/UI/Settings/HomeView.swift").contains(".focusEffectDisabled()"), "the Home canvas-header action button should keep .focusEffectDisabled() so it draws no focus ring" ) assertTrue( - homeSource.contains("across the full row.\n .contentShape(Rectangle())"), + contractSource("Sources/UI/Settings/HomeView.swift").contains("across the full row.\n .contentShape(Rectangle())"), "recent-capture rows should keep their full-width .contentShape(Rectangle()) so hover reveals row actions everywhere, not only over the title text" ) @@ -470,7 +456,7 @@ func testUIAutomationSurfaceContract() { "Allow calendar access", "Skip for now", ] { - assertTrue(onboardingSource.contains(requiredOnboardingHook), "\(requiredOnboardingHook) should stay in onboarding automation scope") + assertTrue(contractSource("Sources/UI/Settings/PermissionsOnboardingView.swift").contains(requiredOnboardingHook), "\(requiredOnboardingHook) should stay in onboarding automation scope") } for identifier in [ @@ -483,21 +469,21 @@ func testUIAutomationSurfaceContract() { "transcripted.speaker-review.row.discard-voice", ] { assertTrue( - speakerReviewSource.contains(identifier), + contractSource("Sources/UI/Settings/SpeakerNamingSheet.swift").contains(identifier), "\(identifier) should keep speaker review scriptable without using speaker names" ) } assertTrue( - speakerReviewSource.contains("static let minimum: CGFloat = 40") - && speakerReviewSource.contains("let btnH = SpeakerNamingHitTargets.minimum") - && speakerReviewSource.contains("let fieldH = SpeakerNamingHitTargets.minimum") - && speakerReviewSource.contains("let hitTarget = SpeakerNamingHitTargets.minimum"), + contractSource("Sources/UI/Settings/SpeakerNamingSheet.swift").contains("static let minimum: CGFloat = 40") + && contractSource("Sources/UI/Settings/SpeakerNamingSheet.swift").contains("let btnH = SpeakerNamingHitTargets.minimum") + && contractSource("Sources/UI/Settings/SpeakerNamingSheet.swift").contains("let fieldH = SpeakerNamingHitTargets.minimum") + && contractSource("Sources/UI/Settings/SpeakerNamingSheet.swift").contains("let hitTarget = SpeakerNamingHitTargets.minimum"), "speaker review save/cancel/name/play/confirm/discard controls should keep a 40pt hit floor" ) assertTrue( - speakerReviewSource.contains("static let sectionHeaderHeight: CGFloat = 40") - && speakerReviewSource.contains("let headerHeight = SpeakerNamingHitTargets.sectionHeaderHeight") - && speakerReviewSource.contains("keepAsYouButton.frame = NSRect("), + contractSource("Sources/UI/Settings/SpeakerNamingSheet.swift").contains("static let sectionHeaderHeight: CGFloat = 40") + && contractSource("Sources/UI/Settings/SpeakerNamingSheet.swift").contains("let headerHeight = SpeakerNamingHitTargets.sectionHeaderHeight") + && contractSource("Sources/UI/Settings/SpeakerNamingSheet.swift").contains("keepAsYouButton.frame = NSRect("), "speaker review Keep Local Mic as You should keep a 40pt section-header hit floor" ) @@ -510,77 +496,67 @@ func testUIAutomationSurfaceContract() { "Copy Paths", "Live meetings", ] { - assertTrue(agentSettingsSource.contains(requiredAgentHook), "\(requiredAgentHook) should stay in agent/connect automation scope") + assertTrue(contractSource("Sources/UI/Settings/AgentConnectionSettingsPage.swift").contains(requiredAgentHook), "\(requiredAgentHook) should stay in agent/connect automation scope") } - let meetingOverlaySource = readUIAutomationContractFile("Sources/UI/Overlay/MeetingOverlayController.swift") - let liveViewPolicySource = readUIAutomationContractFile("Sources/UI/Overlay/MeetingLiveViewAffordancePolicy.swift") assertTrue( - liveViewPolicySource.contains("transcripted.meeting-overlay.live-view") - && meetingOverlaySource.contains("setAccessibilityIdentifier(MeetingLiveViewAffordancePolicy.automationIdentifier)"), + contractSource("Sources/UI/Overlay/MeetingLiveViewAffordancePolicy.swift").contains("transcripted.meeting-overlay.live-view") + && contractSource("Sources/UI/Overlay/MeetingOverlayController.swift").contains("setAccessibilityIdentifier(MeetingLiveViewAffordancePolicy.automationIdentifier)"), "the recording pill body should keep a stable automation identifier for the transcript toggle" ) - let transcriptDrawerSource = readUIAutomationContractFile("Sources/UI/Overlay/MeetingLiveTranscriptDrawerView.swift") assertTrue( - liveViewPolicySource.contains("transcripted.meeting-overlay.live-view.copy") - && transcriptDrawerSource.contains("MeetingLiveViewAffordancePolicy.copyAutomationIdentifier"), + contractSource("Sources/UI/Overlay/MeetingLiveViewAffordancePolicy.swift").contains("transcripted.meeting-overlay.live-view.copy") + && contractSource("Sources/UI/Overlay/MeetingLiveTranscriptDrawerView.swift").contains("MeetingLiveViewAffordancePolicy.copyAutomationIdentifier"), "the transcript drawer's copy action should keep a stable automation identifier" ) assertTrue( - liveViewPolicySource.contains("transcripted.meeting-overlay.live-view.more") - && transcriptDrawerSource.contains("MeetingLiveViewAffordancePolicy.moreAutomationIdentifier"), + contractSource("Sources/UI/Overlay/MeetingLiveViewAffordancePolicy.swift").contains("transcripted.meeting-overlay.live-view.more") + && contractSource("Sources/UI/Overlay/MeetingLiveTranscriptDrawerView.swift").contains("MeetingLiveViewAffordancePolicy.moreAutomationIdentifier"), "the transcript drawer's overflow menu should keep a stable automation identifier" ) assertTrue( - meetingOverlaySource.contains("MeetingLiveViewAffordancePolicy.discardRecordingMenuTitle") - && meetingOverlaySource.contains("MeetingLiveViewAffordancePolicy.keepControlsVisibleMenuTitle") - && transcriptDrawerSource.contains("MeetingLiveViewAffordancePolicy.openInBrowserMenuTitle"), + contractSource("Sources/UI/Overlay/MeetingOverlayController.swift").contains("MeetingLiveViewAffordancePolicy.discardRecordingMenuTitle") + && contractSource("Sources/UI/Overlay/MeetingOverlayController.swift").contains("MeetingLiveViewAffordancePolicy.keepControlsVisibleMenuTitle") + && contractSource("Sources/UI/Overlay/MeetingLiveTranscriptDrawerView.swift").contains("MeetingLiveViewAffordancePolicy.openInBrowserMenuTitle"), "pill context-menu and drawer overflow actions should keep policy-pinned titles for automation" ) assertTrue( - transcriptDrawerSource.contains("transientStatusText ?? statusText") - && transcriptDrawerSource.contains("openInBrowserFailedStatus") - && transcriptDrawerSource.contains(".announcement: MeetingLiveViewAffordancePolicy.openInBrowserFailedStatus"), + contractSource("Sources/UI/Overlay/MeetingLiveTranscriptDrawerView.swift").contains("transientStatusText ?? statusText") + && contractSource("Sources/UI/Overlay/MeetingLiveTranscriptDrawerView.swift").contains("openInBrowserFailedStatus") + && contractSource("Sources/UI/Overlay/MeetingLiveTranscriptDrawerView.swift").contains(".announcement: MeetingLiveViewAffordancePolicy.openInBrowserFailedStatus"), "browser-open failures should remain visible and announced even while transcript updates continue" ) assertTrue( - meetingOverlaySource.contains("showLiveViewBrowserOpenFailure") - && meetingOverlaySource.contains("isTranscriptExpanded = true") - && meetingOverlaySource.contains("rootView?.flashTranscriptBrowserOpenFailure()"), + contractSource("Sources/UI/Overlay/MeetingOverlayController.swift").contains("showLiveViewBrowserOpenFailure") + && contractSource("Sources/UI/Overlay/MeetingOverlayController.swift").contains("isTranscriptExpanded = true") + && contractSource("Sources/UI/Overlay/MeetingOverlayController.swift").contains("rootView?.flashTranscriptBrowserOpenFailure()"), "browser-open failures from the collapsed pill menu should reveal the drawer before showing feedback" ) assertTrue( - deletePolicySource.contains("Delete this meeting?") - && deletePolicySource.contains("Delete Meeting") - && deletePolicySource.contains("Delete this failed meeting?") - && deletePolicySource.contains("Delete Failed Meeting"), + contractSource("Sources/UI/Settings/HomeDeleteConfirmationPolicy.swift").contains("Delete this meeting?") + && contractSource("Sources/UI/Settings/HomeDeleteConfirmationPolicy.swift").contains("Delete Meeting") + && contractSource("Sources/UI/Settings/HomeDeleteConfirmationPolicy.swift").contains("Delete this failed meeting?") + && contractSource("Sources/UI/Settings/HomeDeleteConfirmationPolicy.swift").contains("Delete Failed Meeting"), "delete confirmation copy should stay pinned for destructive-flow automation" ) assertTrue( - settingsComponentsSource.contains("settingsAutomationIdentifier") - && settingsComponentsSource.contains("transcripted.settings.permissions.\\(kind.rawValue).action"), + contractSource("Sources/UI/Settings/TranscriptedSettingsComponents.swift").contains("settingsAutomationIdentifier") + && contractSource("Sources/UI/Settings/TranscriptedSettingsComponents.swift").contains("transcripted.settings.permissions.\\(kind.rawValue).action"), "settings shared controls should support stable automation IDs" ) assertTrue( - generalControlsSource.contains("generalAutomationIdentifier") - && generalControlsSource.contains("transcripted.settings.general.dictation-window.options") - && generalControlsSource.contains("transcripted.settings.general.dictation-window.\\(mode.rawValue)") - && generalControlsSource.contains("transcripted.settings.general.info.\\(automationSlug(info.title))"), + contractSource("Sources/UI/Settings/TranscriptedSettingsGeneralControls.swift").contains("generalAutomationIdentifier") + && contractSource("Sources/UI/Settings/TranscriptedSettingsGeneralControls.swift").contains("transcripted.settings.general.dictation-window.options") + && contractSource("Sources/UI/Settings/TranscriptedSettingsGeneralControls.swift").contains("transcripted.settings.general.dictation-window.\\(mode.rawValue)") + && contractSource("Sources/UI/Settings/TranscriptedSettingsGeneralControls.swift").contains("transcripted.settings.general.info.\\(automationSlug(info.title))"), "general settings controls should keep scriptable row and choice IDs" ) } runSuite("UI automation surface contract - deterministic click-flow identifiers stay mapped") { - let settingsSource = readUIAutomationContractFile("Sources/UI/Settings/TranscriptedSettingsView.swift") - let homeSource = readUIAutomationContractFile("Sources/UI/Settings/HomeView.swift") - let speakerPeopleSource = readUIAutomationContractFile("Sources/UI/Settings/SpeakerPeopleSettingsSection.swift") - let settingsComponentsSource = readUIAutomationContractFile("Sources/UI/Settings/TranscriptedSettingsComponents.swift") - let onboardingSource = readUIAutomationContractFile("Sources/UI/Settings/PermissionsOnboardingView.swift") - let firstRunSource = readUIAutomationContractFile("Sources/UI/Shared/FirstRunExperience.swift") - for identifier in [ "transcripted.home.stats.view", "transcripted.home.stats.done", @@ -607,7 +583,7 @@ func testUIAutomationSurfaceContract() { "transcripted.home.load-more", "transcripted.home.needs-attention.review.", ] { - assertTrue(homeSource.contains(identifier), "\(identifier) should stay attached to Home click-flow controls") + assertTrue(contractSource("Sources/UI/Settings/HomeView.swift").contains(identifier), "\(identifier) should stay attached to Home click-flow controls") } for identifier in [ @@ -630,27 +606,27 @@ func testUIAutomationSurfaceContract() { "transcripted.settings.beta.local-summary.install-uv", "transcripted.settings.beta.open-agent-setup", ] { - assertTrue(settingsSource.contains(identifier), "\(identifier) should stay attached to Settings click-flow controls") + assertTrue(contractSource("Sources/UI/Settings/TranscriptedSettingsView.swift").contains(identifier), "\(identifier) should stay attached to Settings click-flow controls") } assertTrue( - speakerPeopleSource.contains("transcripted.speakers.inbox") - && speakerPeopleSource.contains(".id(ScrollTarget.reviewQueue)") - && settingsSource.contains("proxy.scrollTo(SpeakerPeopleSettingsSection.ScrollTarget.reviewQueue"), + contractSource("Sources/UI/Settings/SpeakerPeopleSettingsSection.swift").contains("transcripted.speakers.inbox") + && contractSource("Sources/UI/Settings/SpeakerPeopleSettingsSection.swift").contains(".id(ScrollTarget.reviewQueue)") + && contractSource("Sources/UI/Settings/TranscriptedSettingsView.swift").contains("proxy.scrollTo(SpeakerPeopleSettingsSection.ScrollTarget.reviewQueue"), "The voices-to-name section should keep a stable automation anchor that review deep-links can scroll to" ) assertTrue( - speakerPeopleSource.contains("enum SpeakerPeopleSettingsPolishContract") - && speakerPeopleSource.contains("struct SpeakerCompactIconLabel") - && speakerPeopleSource.contains("static let minimumHitTarget: CGFloat = 40") - && speakerPeopleSource.contains("static let playButtonVisibleDiameter: CGFloat = 36") - && speakerPeopleSource.contains("static let compactIconVisibleDiameter: CGFloat = 28") - && speakerPeopleSource.contains(".contentShape(Rectangle())"), + contractSource("Sources/UI/Settings/SpeakerPeopleSettingsSection.swift").contains("enum SpeakerPeopleSettingsPolishContract") + && contractSource("Sources/UI/Settings/SpeakerPeopleSettingsSection.swift").contains("struct SpeakerCompactIconLabel") + && contractSource("Sources/UI/Settings/SpeakerPeopleSettingsSection.swift").contains("static let minimumHitTarget: CGFloat = 40") + && contractSource("Sources/UI/Settings/SpeakerPeopleSettingsSection.swift").contains("static let playButtonVisibleDiameter: CGFloat = 36") + && contractSource("Sources/UI/Settings/SpeakerPeopleSettingsSection.swift").contains("static let compactIconVisibleDiameter: CGFloat = 28") + && contractSource("Sources/UI/Settings/SpeakerPeopleSettingsSection.swift").contains(".contentShape(Rectangle())"), "speaker settings should pin compact icon chrome separately from the 40pt hit shape" ) - let speakerCompactIconLabelApplications = speakerPeopleSource + let speakerCompactIconLabelApplications = contractSource("Sources/UI/Settings/SpeakerPeopleSettingsSection.swift") .components(separatedBy: "SpeakerCompactIconLabel(") .count - 1 assertTrue( @@ -666,23 +642,23 @@ func testUIAutomationSurfaceContract() { "transcripted.speakers.person.menu", ] { assertTrue( - speakerPeopleSource.contains(identifier), + contractSource("Sources/UI/Settings/SpeakerPeopleSettingsSection.swift").contains(identifier), "\(identifier) should keep the speakers surface's icon-only controls scriptable without using speaker names" ) } assertTrue( - homeSource.contains("HomeAttentionPillsRow") - && homeSource.contains(".accessibilityHint(issue.detail)") - && homeSource.contains("transcripted.home.needs-attention.review."), + contractSource("Sources/UI/Settings/HomeView.swift").contains("HomeAttentionPillsRow") + && contractSource("Sources/UI/Settings/HomeView.swift").contains(".accessibilityHint(issue.detail)") + && contractSource("Sources/UI/Settings/HomeView.swift").contains("transcripted.home.needs-attention.review."), "Home attention pills should stay labeled and scriptable" ) assertTrue( - settingsSource.contains("HomeRowMenuItem(title: \"Review speakers\"") - && settingsSource.contains("let audioRevealURLs = HomeMeetingRowActionTargets.audioRevealURLs(for: item)") - && settingsSource.contains("if !audioRevealURLs.isEmpty") - && settingsSource.contains("title: \"Re-transcribe with speaker ID\""), + contractSource("Sources/UI/Settings/TranscriptedSettingsView.swift").contains("HomeRowMenuItem(title: \"Review speakers\"") + && contractSource("Sources/UI/Settings/TranscriptedSettingsView.swift").contains("let audioRevealURLs = HomeMeetingRowActionTargets.audioRevealURLs(for: item)") + && contractSource("Sources/UI/Settings/TranscriptedSettingsView.swift").contains("if !audioRevealURLs.isEmpty") + && contractSource("Sources/UI/Settings/TranscriptedSettingsView.swift").contains("title: \"Re-transcribe with speaker ID\""), "meeting speaker review and re-transcribe actions should stay reachable from the row menu when retained audio has a Finder target" ) @@ -704,46 +680,42 @@ func testUIAutomationSurfaceContract() { "transcripted.onboarding.agent.connect-claude-desktop", "transcripted.onboarding.agent.copy-local-agent-prompt", ] { - assertTrue(onboardingSource.contains(identifier), "\(identifier) should stay attached to onboarding click-flow controls") + assertTrue(contractSource("Sources/UI/Settings/PermissionsOnboardingView.swift").contains(identifier), "\(identifier) should stay attached to onboarding click-flow controls") } assertTrue( - onboardingSource.contains("UseCaseChoiceCard(") - && onboardingSource.contains("transcripted.onboarding.use-case.meetings") - && onboardingSource.contains("transcripted.onboarding.use-case.dictation") - && onboardingSource.contains("selectedStateStrokeWidth") - && onboardingSource.contains(".contentShape(RoundedRectangle(cornerRadius: 12, style: .continuous))"), + contractSource("Sources/UI/Settings/PermissionsOnboardingView.swift").contains("UseCaseChoiceCard(") + && contractSource("Sources/UI/Settings/PermissionsOnboardingView.swift").contains("transcripted.onboarding.use-case.meetings") + && contractSource("Sources/UI/Settings/PermissionsOnboardingView.swift").contains("transcripted.onboarding.use-case.dictation") + && contractSource("Sources/UI/Settings/PermissionsOnboardingView.swift").contains("selectedStateStrokeWidth") + && contractSource("Sources/UI/Settings/PermissionsOnboardingView.swift").contains(".contentShape(RoundedRectangle(cornerRadius: 12, style: .continuous))"), "onboarding use-case cards should keep their scriptable card-button and selected-state hooks" ) assertTrue( - firstRunSource.contains("static let minimumHitTarget: Double = 44") - && firstRunSource.contains("static let modelProgressLabelMinimumWidth: Double = 104") - && firstRunSource.contains("static let selectedStateStrokeWidth: Double = 2") - && onboardingSource.contains("transcripted.onboarding.nav.skip") - && onboardingSource.contains("transcripted.onboarding.nav.primary") - && onboardingSource.contains("FirstRunOnboardingPolishContract.minimumHitTarget") - && onboardingSource.contains(".contentShape(Rectangle())") - && onboardingSource.contains(".contentShape(RoundedRectangle(cornerRadius: 10, style: .continuous))"), + contractSource("Sources/UI/Shared/FirstRunExperience.swift").contains("static let minimumHitTarget: Double = 44") + && contractSource("Sources/UI/Shared/FirstRunExperience.swift").contains("static let modelProgressLabelMinimumWidth: Double = 104") + && contractSource("Sources/UI/Shared/FirstRunExperience.swift").contains("static let selectedStateStrokeWidth: Double = 2") + && contractSource("Sources/UI/Settings/PermissionsOnboardingView.swift").contains("transcripted.onboarding.nav.skip") + && contractSource("Sources/UI/Settings/PermissionsOnboardingView.swift").contains("transcripted.onboarding.nav.primary") + && contractSource("Sources/UI/Settings/PermissionsOnboardingView.swift").contains("FirstRunOnboardingPolishContract.minimumHitTarget") + && contractSource("Sources/UI/Settings/PermissionsOnboardingView.swift").contains(".contentShape(Rectangle())") + && contractSource("Sources/UI/Settings/PermissionsOnboardingView.swift").contains(".contentShape(RoundedRectangle(cornerRadius: 10, style: .continuous))"), "onboarding nav and compact controls should keep pinned polish constants and hit-shape hooks" ) assertTrue( - settingsComponentsSource.contains(".monospacedDigit()") - && settingsComponentsSource.contains("modelProgressLabelMinimumWidth") - && settingsComponentsSource.contains(".accessibilityLabel(Text(status))"), + contractSource("Sources/UI/Settings/TranscriptedSettingsComponents.swift").contains(".monospacedDigit()") + && contractSource("Sources/UI/Settings/TranscriptedSettingsComponents.swift").contains("modelProgressLabelMinimumWidth") + && contractSource("Sources/UI/Settings/TranscriptedSettingsComponents.swift").contains(".accessibilityLabel(Text(status))"), "onboarding/local-model progress labels should stay stable, tabular, and accessible" ) } runSuite("UI automation surface contract - QA CLI exposes a real AX smoke") { - let qaEntrySource = readUIAutomationContractFile("Tools/TranscriptedQA/Sources/TranscriptedQA/TranscriptedQA.swift") - let uiSmokeSource = readUIAutomationContractFile("Tools/TranscriptedQA/Sources/TranscriptedQA/Commands/UISmoke.swift") - let qaBenchSource = readUIAutomationContractFile("scripts/ops/transcripted-qa-bench.sh") - assertTrue( - qaEntrySource.contains("UISmoke.self") - && uiSmokeSource.contains("commandName: \"ui-smoke\""), + contractSource("Tools/TranscriptedQA/Sources/TranscriptedQA/TranscriptedQA.swift").contains("UISmoke.self") + && contractSource("Tools/TranscriptedQA/Sources/TranscriptedQA/Commands/UISmoke.swift").contains("commandName: \"ui-smoke\""), "TranscriptedQA should expose a ui-smoke command for repo-owned UI automation" ) @@ -773,14 +745,14 @@ func testUIAutomationSurfaceContract() { "transcripted.onboarding.use-case.dictation", "transcripted.onboarding.permissions.system-audio", ] { - assertTrue(uiSmokeSource.contains(requiredHarnessHook), "\(requiredHarnessHook) should stay pinned in the UI smoke harness") + assertTrue(contractSource("Tools/TranscriptedQA/Sources/TranscriptedQA/Commands/UISmoke.swift").contains(requiredHarnessHook), "\(requiredHarnessHook) should stay pinned in the UI smoke harness") } assertTrue( - qaBenchSource.contains("quick|deep|full|ui|packaged|artifact") - && qaBenchSource.contains("run_ui_tail") - && qaBenchSource.contains("transcripted-qa ui-smoke") - && qaBenchSource.contains("ui-automation-smoke.json"), + contractSource("scripts/ops/transcripted-qa-bench.sh").contains("quick|deep|full|ui|packaged|artifact") + && contractSource("scripts/ops/transcripted-qa-bench.sh").contains("run_ui_tail") + && contractSource("scripts/ops/transcripted-qa-bench.sh").contains("transcripted-qa ui-smoke") + && contractSource("scripts/ops/transcripted-qa-bench.sh").contains("ui-automation-smoke.json"), "QA bench should keep a callable ui mode with local JSON evidence" ) } From 253180f62d7921067aeba85050ae06d6bf1d9a40 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 24 Jun 2026 06:16:51 -0500 Subject: [PATCH 2/3] Add local Sparkle update UI smoke --- .../SparkleUpdaterController.swift | 39 +- Sources/UI/MenuBar/MenuBarContentView.swift | 2 + .../UI/MenuBar/MenuBarPanelController.swift | 8 + Tests/RepoCommandContractTests.swift | 2 +- Tests/UIAutomationSurfaceContractTests.swift | 2 +- Tools/TranscriptedQA/CLAUDE.md | 10 +- .../Commands/SparkleUpdateSmoke.swift | 446 ++++++++++++++++++ .../TranscriptedQA/TranscriptedQA.swift | 1 + .../SparkleUpdateSmokeTests.swift | 96 ++++ docs/qa-test-bench.md | 20 + docs/sparkle-updates.md | 22 + scripts/ops/release-gate-report.py | 4 +- scripts/ops/transcripted-qa-bench.sh | 19 +- 13 files changed, 661 insertions(+), 10 deletions(-) create mode 100644 Tools/TranscriptedQA/Sources/TranscriptedQA/Commands/SparkleUpdateSmoke.swift create mode 100644 Tools/TranscriptedQA/Tests/TranscriptedQATests/SparkleUpdateSmokeTests.swift diff --git a/Sources/Observability/SparkleUpdaterController.swift b/Sources/Observability/SparkleUpdaterController.swift index 1d1baf26..91925595 100644 --- a/Sources/Observability/SparkleUpdaterController.swift +++ b/Sources/Observability/SparkleUpdaterController.swift @@ -82,12 +82,49 @@ final class SparkleUpdaterController: NSObject, ObservableObject { override init() { super.init() - guard !Self.isLaunchUISmoke else { return } + guard !Self.isLaunchUISmoke else { + applyLaunchUISmokeUpdateStateIfPresent() + return + } trackInstalledUpdateIfNeeded() observeUpdaterReadiness() observeUpdaterSettings() } + private func applyLaunchUISmokeUpdateStateIfPresent() { + guard let state = Self.launchUISmokeUpdateState() else { return } + updateStatus = UpdateStatus(state: state, canCheckForUpdates: true) + automaticUpdateSettings = AutomaticUpdateSettings( + automaticChecksEnabled: true, + automaticDownloadsAllowed: true, + automaticDownloadsEnabled: false + ) + } + + nonisolated private static func launchUISmokeUpdateState() -> UpdateStatus.State? { + let environment = ProcessInfo.processInfo.environment + guard let rawState = environment["TRANSCRIPTED_LAUNCH_UI_SMOKE_UPDATE_STATE"]? + .trimmingCharacters(in: .whitespacesAndNewlines) + .lowercased(), + !rawState.isEmpty else { + return nil + } + let version = environment["TRANSCRIPTED_LAUNCH_UI_SMOKE_UPDATE_VERSION"]? + .trimmingCharacters(in: .whitespacesAndNewlines) + let displayVersion = version?.isEmpty == false ? version! : "9.9.9" + + switch rawState { + case "available", "update-available": + return .updateAvailable(version: displayVersion) + case "downloading", "download-progress": + return .downloading(version: displayVersion) + case "ready", "ready-to-install": + return .readyToInstall(version: displayVersion) + default: + return nil + } + } + func performStartupUpdateCheckIfNeeded() { guard !Self.isLaunchUISmoke else { return } guard !hasPerformedStartupCheck else { return } diff --git a/Sources/UI/MenuBar/MenuBarContentView.swift b/Sources/UI/MenuBar/MenuBarContentView.swift index 6e79165e..96a7c344 100644 --- a/Sources/UI/MenuBar/MenuBarContentView.swift +++ b/Sources/UI/MenuBar/MenuBarContentView.swift @@ -5,6 +5,7 @@ import AppKit struct MenuBarContentSmokeSnapshot: Codable, Equatable { let header: MenuBarHeaderSmokeSnapshot + let updateCallout: MenuBarActionRowSmokeSnapshot let primaryActions: [String: MenuBarActionRowSmokeSnapshot] let utilityActions: [String: MenuBarActionRowSmokeSnapshot] } @@ -175,6 +176,7 @@ final class MenuBarContentView: NSView { var smokeSnapshot: MenuBarContentSmokeSnapshot { MenuBarContentSmokeSnapshot( header: headerView.smokeSnapshot, + updateCallout: updateCalloutRow.smokeSnapshot, primaryActions: primaryActionsView.smokeSnapshot, utilityActions: utilityActionsView.smokeSnapshot ) diff --git a/Sources/UI/MenuBar/MenuBarPanelController.swift b/Sources/UI/MenuBar/MenuBarPanelController.swift index 3d0d424c..a4197a35 100644 --- a/Sources/UI/MenuBar/MenuBarPanelController.swift +++ b/Sources/UI/MenuBar/MenuBarPanelController.swift @@ -163,6 +163,14 @@ final class MenuBarPanelController: NSViewController { warningText: "", isReady: false ), + updateCallout: MenuBarActionRowSmokeSnapshot( + title: "", + detail: "", + trailingText: "", + automationIdentifier: "", + isVisible: false, + isEnabled: false + ), primaryActions: [:], utilityActions: [:] ) diff --git a/Tests/RepoCommandContractTests.swift b/Tests/RepoCommandContractTests.swift index 1c663f7d..08bd3a21 100644 --- a/Tests/RepoCommandContractTests.swift +++ b/Tests/RepoCommandContractTests.swift @@ -361,7 +361,7 @@ func testRepoCommandContract() { let matrix = readRepoTextFile(".agents/test-matrix.yml") assertTrue( - qaBench.contains("quick|deep|full|ui|packaged|artifact|audio-synthetic|pasteback-synthetic|corpus|corpus-compare|scorecard|live") + qaBench.contains("quick|deep|full|ui|sparkle-update|packaged|artifact|audio-synthetic|pasteback-synthetic|corpus|corpus-compare|scorecard|live") && qaBench.contains("run_full_tail") && qaBench.contains("60-release-health") && qaBench.contains("61-gemma-summary-plan") diff --git a/Tests/UIAutomationSurfaceContractTests.swift b/Tests/UIAutomationSurfaceContractTests.swift index 5c48a2b3..bf90728b 100644 --- a/Tests/UIAutomationSurfaceContractTests.swift +++ b/Tests/UIAutomationSurfaceContractTests.swift @@ -777,7 +777,7 @@ func testUIAutomationSurfaceContract() { } assertTrue( - qaBenchSource.contains("quick|deep|full|ui|packaged|artifact") + qaBenchSource.contains("quick|deep|full|ui|sparkle-update|packaged|artifact") && qaBenchSource.contains("run_ui_tail") && qaBenchSource.contains("transcripted-qa ui-smoke") && qaBenchSource.contains("ui-automation-smoke.json"), diff --git a/Tools/TranscriptedQA/CLAUDE.md b/Tools/TranscriptedQA/CLAUDE.md index c71f5988..fcf46f92 100644 --- a/Tools/TranscriptedQA/CLAUDE.md +++ b/Tools/TranscriptedQA/CLAUDE.md @@ -1,6 +1,6 @@ # TranscriptedQA - QA Testing CLI Tool -QA testing suite for Transcripted. `Package.swift`, 27 files under `Sources/TranscriptedQA/`, and 5 test files under `Tests/TranscriptedQATests/`. +QA testing suite for Transcripted. `Package.swift`, files under `Sources/TranscriptedQA/`, and test files under `Tests/TranscriptedQATests/`. The current package is intentionally small: @@ -23,11 +23,12 @@ The current package is intentionally small: | `CheckHealth.swift` | Quick health check: DB integrity, model presence, disk space | | `GenerateFixtures.swift` | Generate valid test data (transcripts, legacy JSON artifacts, DB records) for CI or manual verification | | `ImportedAudioSmoke.swift` | Deterministic imported-audio artifact smoke: synthetic WAV, imported meeting Markdown, retained single-file audio, parser and validator proof | +| `UISmoke.swift` | Launch a built app and validate onboarding, menu bar, Home, Settings, and General navigation through macOS Accessibility | | `PermissionState.swift` | No-prompt macOS permission-state probe for Codex computer-use and live QA blockers | | `PackagedAppSmoke.swift` | Pre-publish packaged app smoke for app bundle metadata, Sparkle config, signing, dSYM, DMG, optional UI, and privacy-safe local logs | | `RoundTrip.swift` | Generate test data, validate, corrupt, re-validate, and confirm validators catch real defects | | `StressTest.swift` | Generate large datasets and validate performance + correctness | -| `UISmoke.swift` | Launch a built app and validate onboarding, menu bar, Home, Settings, and General navigation through macOS Accessibility | +| `SparkleUpdateSmoke.swift` | No-publish fake-state Sparkle update UI smoke for update-available and downloading menu surfaces | | `ValidateAll.swift` | Run all validators: transcripts, dictations, DB, index, logs, artifacts | | `ValidateArtifacts.swift` | Check optional legacy JSON artifacts, YAML frontmatter, speaker clips | | `ValidateDatabase.swift` | SpeakerDB and StatsDB integrity, schema validation, corruption check | @@ -67,13 +68,14 @@ The current package is intentionally small: |------|---------| | `ValidationResult.swift` | shared `ValidationResult`, `ValidationReport`, and PASS/WARN/FAIL status types used for structured text or JSON validator output | -### Tests/ (4 files) +### Tests/ | File | Purpose | |------|---------| | `PackagedAppSmokeTests.swift` | package-level coverage for packaged app metadata, Sparkle config, dSYM UUIDs, DMG, and log privacy checks | | `PermissionStateProbeTests.swift` | package-level coverage for permission-state probe modes and blocker classification | | `PermissionStateRuntimeGateTests.swift` | package-level coverage for duplicate/wrong-running-app runtime gate warnings | +| `SparkleUpdateSmokeTests.swift` | package-level coverage for fake-state Sparkle update UI smoke evaluation | | `ValidatorTests.swift` | package-level coverage for YAML parsing, legacy index validation, JSON sidecar validation, and `ValidationReport` exit-code behavior | ## Usage @@ -94,6 +96,7 @@ swift run transcripted-qa validate-logs swift run transcripted-qa check-health swift run transcripted-qa permission-state --mode computer-use swift run transcripted-qa imported-audio-smoke --output /tmp/transcripted-imported-audio-smoke +swift run transcripted-qa sparkle-update-smoke --app ../../build/Transcripted.app --output /tmp/transcripted-sparkle-update-smoke swift run transcripted-qa packaged-app-smoke --app ../../build/Transcripted.app --dsym ../../build/Transcripted.app.dSYM --run-ui-smoke # UI automation smoke, local Accessibility permission required @@ -134,6 +137,7 @@ For agent and automation use, the JSON form also includes: - **Round-trip testing**: `round-trip` validates that validators correctly catch injected corruption - **Stress testing**: `stress-test` generates large datasets to surface performance and correctness issues - **Imported audio smoke**: `imported-audio-smoke` proves deterministic imported meeting artifact shape, `system_audio` metadata, retained single-file audio, parser discovery, and transcript validation. It is not native file-picker or real ML transcription proof. +- **Sparkle update smoke**: `sparkle-update-smoke` launches the built app through the launch-smoke harness with fake update-available and downloading states, then checks the real menu snapshot. It is local UI proof only, not live appcast/download/install proof. - **UI smoke**: `ui-smoke` checks stable AX identifiers across first-run onboarding, menu bar, Home, and Settings, and exits `3` for Accessibility/TCC blockers - **Packaged app smoke**: `packaged-app-smoke` validates a no-publish `build-beta.sh` artifact, including version/config parity, Sparkle keys, signing, dSYM UUIDs, DMG readability, optional menu bar UI, and local log privacy - **Permission state**: `permission-state` prints the expected manual grant state, checks Codex host Accessibility/Event Posting/Input Monitoring/Screen Recording/Automation, verifies the Transcripted app bundle id, and warns on duplicate or wrong running Transcripted app instances diff --git a/Tools/TranscriptedQA/Sources/TranscriptedQA/Commands/SparkleUpdateSmoke.swift b/Tools/TranscriptedQA/Sources/TranscriptedQA/Commands/SparkleUpdateSmoke.swift new file mode 100644 index 00000000..2002881a --- /dev/null +++ b/Tools/TranscriptedQA/Sources/TranscriptedQA/Commands/SparkleUpdateSmoke.swift @@ -0,0 +1,446 @@ +import ArgumentParser +import Foundation + +struct SparkleUpdateSmoke: ParsableCommand { + static let configuration = CommandConfiguration( + commandName: "sparkle-update-smoke", + abstract: "Run a no-publish Sparkle update UI smoke with fake update-available and downloading states." + ) + + @Option(name: .long, help: "Path to the built Transcripted.app bundle.") + var app: String = "build/Transcripted.app" + + @Option(name: .long, help: "Output directory for JSON evidence and the fake appcast fixture.") + var output: String = "/tmp/transcripted-sparkle-update-smoke" + + @Option(name: .long, help: "Fake display version shown in the update UI.") + var version: String = "9.9.9" + + @Option(name: .long, help: "Seconds to wait for each launch-smoke report.") + var timeout: Double = 12 + + func run() throws { + let runner = SparkleUpdateSmokeRunner( + appBundlePath: app, + outputDirectory: output, + version: version, + timeout: timeout + ) + let report = runner.run() + try report.write(to: URL(fileURLWithPath: output, isDirectory: true)) + report.printText() + if report.exitCode != 0 { + throw ExitCode(report.exitCode) + } + } +} + +struct SparkleUpdateSmokeRunner { + let appBundlePath: String + let outputDirectory: String + let version: String + let timeout: TimeInterval + var fileManager: FileManager = .default + + func run() -> SparkleUpdateSmokeReport { + let runID = UUID().uuidString + let outputURL = URL(fileURLWithPath: outputDirectory, isDirectory: true).standardizedFileURL + var scenarios: [SparkleUpdateSmokeScenarioReport] = [] + let appURL = URL(fileURLWithPath: appBundlePath).standardizedFileURL + let executableURL = appURL.appendingPathComponent("Contents/MacOS/Transcripted", isDirectory: false) + + do { + try fileManager.createDirectory(at: outputURL, withIntermediateDirectories: true) + try writeFakeAppcast(to: outputURL.appendingPathComponent("fake-appcast.xml", isDirectory: false)) + } catch { + return SparkleUpdateSmokeReport( + runID: runID, + status: .fail, + exitCode: 1, + generatedAt: ISO8601DateFormatter().string(from: Date()), + appBundlePath: appURL.path, + outputDirectory: outputURL.path, + fakeAppcastPath: outputURL.appendingPathComponent("fake-appcast.xml", isDirectory: false).path, + scenarios: [ + SparkleUpdateSmokeScenarioReport( + state: "setup", + status: .fail, + reportPath: nil, + appLogPath: nil, + checks: [ + SparkleUpdateSmokeCheck( + id: "output-directory", + status: .fail, + detail: "Could not prepare output directory: \(error.localizedDescription)" + ), + ] + ), + ], + limitations: Self.limitations + ) + } + + guard fileManager.fileExists(atPath: appURL.path) else { + scenarios.append(.singleFailure( + state: "setup", + id: "app-bundle", + detail: "Run bash build.sh --no-open first, or pass --app." + )) + return buildReport(runID: runID, appURL: appURL, outputURL: outputURL, scenarios: scenarios) + } + + guard fileManager.isExecutableFile(atPath: executableURL.path) else { + scenarios.append(.singleFailure( + state: "setup", + id: "app-executable", + detail: "Transcripted.app is missing Contents/MacOS/Transcripted." + )) + return buildReport(runID: runID, appURL: appURL, outputURL: outputURL, scenarios: scenarios) + } + + scenarios.append(runScenario( + state: "available", + appExecutableURL: executableURL, + outputURL: outputURL + )) + scenarios.append(runScenario( + state: "downloading", + appExecutableURL: executableURL, + outputURL: outputURL + )) + + return buildReport(runID: runID, appURL: appURL, outputURL: outputURL, scenarios: scenarios) + } + + private func runScenario( + state: String, + appExecutableURL: URL, + outputURL: URL + ) -> SparkleUpdateSmokeScenarioReport { + let scenarioDirectory = outputURL.appendingPathComponent(state, isDirectory: true) + let reportURL = scenarioDirectory.appendingPathComponent("launch-ui-smoke.json", isDirectory: false) + let logURL = scenarioDirectory.appendingPathComponent("app.log", isDirectory: false) + do { + try fileManager.createDirectory(at: scenarioDirectory, withIntermediateDirectories: true) + } catch { + return .singleFailure( + state: state, + id: "scenario-directory", + detail: "Could not create scenario directory: \(error.localizedDescription)" + ) + } + + let process = Process() + process.executableURL = appExecutableURL + process.arguments = [ + "-permissionsOnboardingCompleted", "YES", + "-forcePermissionsOnboarding", "NO", + "-observability-anonymous-analytics-enabled", "NO", + "-observability-crash-reporting-enabled", "NO", + ] + + var environment = ProcessInfo.processInfo.environment + environment.removeValue(forKey: "__CFBundleIdentifier") + environment["HOME"] = scenarioDirectory.path + environment["CFFIXED_USER_HOME"] = scenarioDirectory.path + environment["TRANSCRIPTED_DISABLE_FILE_LOGGER"] = "1" + environment["TRANSCRIPTED_DISABLE_RUNTIME_DIAGNOSTICS"] = "1" + environment["TRANSCRIPTED_DISABLE_SINGLE_INSTANCE_GUARD"] = "1" + environment["TRANSCRIPTED_LAUNCH_UI_SMOKE_REPORT"] = reportURL.path + environment["TRANSCRIPTED_LAUNCH_UI_SMOKE_TERMINATE_AFTER_REPORT"] = "1" + environment["TRANSCRIPTED_LAUNCH_UI_SMOKE_TERMINATE_DELAY_SECONDS"] = "0.1" + environment["TRANSCRIPTED_LAUNCH_UI_SMOKE_UPDATE_STATE"] = state + environment["TRANSCRIPTED_LAUNCH_UI_SMOKE_UPDATE_VERSION"] = version + process.environment = environment + + fileManager.createFile(atPath: logURL.path, contents: nil) + let logHandle = try? FileHandle(forWritingTo: logURL) + process.standardOutput = logHandle + process.standardError = logHandle + + do { + try process.run() + } catch { + try? logHandle?.close() + return .singleFailure( + state: state, + id: "launch-app", + detail: "Could not launch app: \(error.localizedDescription)", + reportPath: reportURL.path, + appLogPath: logURL.path + ) + } + + let deadline = Date().addingTimeInterval(max(2, timeout)) + while process.isRunning && Date() < deadline { + Thread.sleep(forTimeInterval: 0.1) + } + if process.isRunning { + process.terminate() + Thread.sleep(forTimeInterval: 0.2) + } + if process.isRunning { + Darwin.kill(process.processIdentifier, SIGKILL) + } + try? logHandle?.close() + + guard fileManager.fileExists(atPath: reportURL.path) else { + return .singleFailure( + state: state, + id: "launch-report", + detail: "App launched, but did not write the launch UI smoke report.", + reportPath: reportURL.path, + appLogPath: logURL.path + ) + } + + do { + let data = try Data(contentsOf: reportURL) + let launchReport = try JSONDecoder().decode(SparkleLaunchSmokeReport.self, from: data) + return SparkleUpdateSmokeEvaluator.evaluate( + state: state, + version: version, + launchReport: launchReport, + reportPath: reportURL.path, + appLogPath: logURL.path + ) + } catch { + return .singleFailure( + state: state, + id: "decode-launch-report", + detail: "Could not decode launch report: \(error.localizedDescription)", + reportPath: reportURL.path, + appLogPath: logURL.path + ) + } + } + + private func writeFakeAppcast(to url: URL) throws { + try """ + + + + Transcripted Local Sparkle UI Smoke + + Version \(version) + \(version) + \(version) + + + + + """.write(to: url, atomically: true, encoding: .utf8) + } + + private func buildReport( + runID: String, + appURL: URL, + outputURL: URL, + scenarios: [SparkleUpdateSmokeScenarioReport] + ) -> SparkleUpdateSmokeReport { + let status: SparkleUpdateSmokeStatus = scenarios.contains { $0.status == .fail } ? .fail : .pass + return SparkleUpdateSmokeReport( + runID: runID, + status: status, + exitCode: status == .pass ? 0 : 1, + generatedAt: ISO8601DateFormatter().string(from: Date()), + appBundlePath: appURL.path, + outputDirectory: outputURL.path, + fakeAppcastPath: outputURL.appendingPathComponent("fake-appcast.xml", isDirectory: false).path, + scenarios: scenarios, + limitations: Self.limitations + ) + } + + static let limitations = [ + "Uses a launch-smoke-only fake Sparkle state; it does not contact the live appcast.", + "Does not download, verify, install, relaunch, notarize, publish, update Homebrew, or prove an existing installed app can upgrade.", + ] +} + +enum SparkleUpdateSmokeEvaluator { + static func evaluate( + state: String, + version: String, + launchReport: SparkleLaunchSmokeReport, + reportPath: String?, + appLogPath: String? + ) -> SparkleUpdateSmokeScenarioReport { + var checks: [SparkleUpdateSmokeCheck] = [ + .pass("launch-report", "App wrote launch UI smoke report."), + ] + + switch state { + case "available": + checks.append(checkEqual( + id: "available-callout-title", + actual: launchReport.content.updateCallout.title, + expected: "Update available: \(version)" + )) + checks.append(checkEqual( + id: "available-callout-detail", + actual: launchReport.content.updateCallout.detail, + expected: "A new version is ready to install" + )) + checks.append(checkEqual( + id: "available-callout-trailing", + actual: launchReport.content.updateCallout.trailingText, + expected: "Install" + )) + checks.append(checkTrue( + id: "available-callout-visible", + condition: launchReport.content.updateCallout.isVisible, + detail: "Expected the prominent menu update callout to be visible." + )) + checks.append(checkTrue( + id: "available-utility-hidden", + condition: !launchReport.content.utilityActions.checkUpdates.isVisible, + detail: "Expected the utility update row to be hidden while the prominent available-update callout is shown." + )) + case "downloading": + checks.append(checkEqual( + id: "downloading-utility-title", + actual: launchReport.content.utilityActions.checkUpdates.title, + expected: "Preparing Update" + )) + checks.append(checkEqual( + id: "downloading-utility-detail", + actual: launchReport.content.utilityActions.checkUpdates.detail, + expected: "Transcripted will ask you to restart when \(version) is ready" + )) + checks.append(checkTrue( + id: "downloading-utility-visible", + condition: launchReport.content.utilityActions.checkUpdates.isVisible, + detail: "Expected the utility update-progress row to be visible." + )) + checks.append(checkTrue( + id: "downloading-utility-disabled", + condition: !launchReport.content.utilityActions.checkUpdates.isEnabled, + detail: "Expected the update-progress row to be disabled while Sparkle is downloading." + )) + checks.append(checkTrue( + id: "downloading-callout-hidden", + condition: !launchReport.content.updateCallout.isVisible, + detail: "Expected no prominent install callout while the update is still downloading." + )) + default: + checks.append(.fail("unknown-state", "Unknown fake Sparkle state: \(state).")) + } + + let status: SparkleUpdateSmokeStatus = checks.contains { $0.status == .fail } ? .fail : .pass + return SparkleUpdateSmokeScenarioReport( + state: state, + status: status, + reportPath: reportPath, + appLogPath: appLogPath, + checks: checks + ) + } + + private static func checkEqual(id: String, actual: String, expected: String) -> SparkleUpdateSmokeCheck { + actual == expected + ? .pass(id, "Observed \(expected).") + : .fail(id, "Expected \(expected), got \(actual).") + } + + private static func checkTrue(id: String, condition: Bool, detail: String) -> SparkleUpdateSmokeCheck { + condition ? .pass(id, detail) : .fail(id, detail) + } +} + +enum SparkleUpdateSmokeStatus: String, Codable, Equatable { + case pass = "PASS" + case fail = "FAIL" +} + +struct SparkleUpdateSmokeCheck: Codable, Equatable { + let id: String + let status: SparkleUpdateSmokeStatus + let detail: String + + static func pass(_ id: String, _ detail: String) -> SparkleUpdateSmokeCheck { + SparkleUpdateSmokeCheck(id: id, status: .pass, detail: detail) + } + + static func fail(_ id: String, _ detail: String) -> SparkleUpdateSmokeCheck { + SparkleUpdateSmokeCheck(id: id, status: .fail, detail: detail) + } +} + +struct SparkleUpdateSmokeScenarioReport: Codable, Equatable { + let state: String + let status: SparkleUpdateSmokeStatus + let reportPath: String? + let appLogPath: String? + let checks: [SparkleUpdateSmokeCheck] + + static func singleFailure( + state: String, + id: String, + detail: String, + reportPath: String? = nil, + appLogPath: String? = nil + ) -> SparkleUpdateSmokeScenarioReport { + SparkleUpdateSmokeScenarioReport( + state: state, + status: .fail, + reportPath: reportPath, + appLogPath: appLogPath, + checks: [.fail(id, detail)] + ) + } +} + +struct SparkleUpdateSmokeReport: Codable, Equatable { + let runID: String + let status: SparkleUpdateSmokeStatus + let exitCode: Int32 + let generatedAt: String + let appBundlePath: String + let outputDirectory: String + let fakeAppcastPath: String + let scenarios: [SparkleUpdateSmokeScenarioReport] + let limitations: [String] + + func printText() { + print("\(status.rawValue): Sparkle update UI smoke") + print("Report: \(outputDirectory)/sparkle-update-smoke.json") + print("Fake appcast fixture: \(fakeAppcastPath)") + for scenario in scenarios { + print("- \(scenario.status.rawValue): \(scenario.state)") + for check in scenario.checks where check.status == .fail { + print(" - \(check.id): \(check.detail)") + } + } + print("Limitations: \(limitations.joined(separator: " "))") + } + + func write(to outputURL: URL) throws { + try FileManager.default.createDirectory(at: outputURL, withIntermediateDirectories: true) + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + let data = try encoder.encode(self) + try data.write(to: outputURL.appendingPathComponent("sparkle-update-smoke.json", isDirectory: false), options: .atomic) + } +} + +struct SparkleLaunchSmokeReport: Codable, Equatable { + let content: SparkleLaunchSmokeContent +} + +struct SparkleLaunchSmokeContent: Codable, Equatable { + let updateCallout: SparkleLaunchSmokeRow + let utilityActions: SparkleLaunchSmokeUtilityActions +} + +struct SparkleLaunchSmokeUtilityActions: Codable, Equatable { + let checkUpdates: SparkleLaunchSmokeRow +} + +struct SparkleLaunchSmokeRow: Codable, Equatable { + let title: String + let detail: String + let trailingText: String + let isVisible: Bool + let isEnabled: Bool +} diff --git a/Tools/TranscriptedQA/Sources/TranscriptedQA/TranscriptedQA.swift b/Tools/TranscriptedQA/Sources/TranscriptedQA/TranscriptedQA.swift index f3f15f66..af2d93e4 100644 --- a/Tools/TranscriptedQA/Sources/TranscriptedQA/TranscriptedQA.swift +++ b/Tools/TranscriptedQA/Sources/TranscriptedQA/TranscriptedQA.swift @@ -180,6 +180,7 @@ struct TranscriptedQA: ParsableCommand { StressTest.self, ImportedAudioSmoke.self, UISmoke.self, + SparkleUpdateSmoke.self, PackagedAppSmoke.self, ], defaultSubcommand: ValidateAll.self diff --git a/Tools/TranscriptedQA/Tests/TranscriptedQATests/SparkleUpdateSmokeTests.swift b/Tools/TranscriptedQA/Tests/TranscriptedQATests/SparkleUpdateSmokeTests.swift new file mode 100644 index 00000000..e54fc169 --- /dev/null +++ b/Tools/TranscriptedQA/Tests/TranscriptedQATests/SparkleUpdateSmokeTests.swift @@ -0,0 +1,96 @@ +import XCTest +@testable import transcripted_qa + +final class SparkleUpdateSmokeTests: XCTestCase { + func testAvailableScenarioRequiresProminentInstallCallout() { + let report = SparkleUpdateSmokeEvaluator.evaluate( + state: "available", + version: "9.9.9", + launchReport: launchReport( + updateCallout: row( + title: "Update available: 9.9.9", + detail: "A new version is ready to install", + trailingText: "Install", + isVisible: true, + isEnabled: true + ), + checkUpdates: row(title: "Check for Updates", isVisible: false) + ), + reportPath: nil, + appLogPath: nil + ) + + XCTAssertEqual(report.status, .pass) + XCTAssertTrue(report.checks.contains { $0.id == "available-callout-title" && $0.status == .pass }) + } + + func testDownloadingScenarioRequiresProgressRowNotInstallCallout() { + let report = SparkleUpdateSmokeEvaluator.evaluate( + state: "downloading", + version: "9.9.9", + launchReport: launchReport( + updateCallout: row(isVisible: false), + checkUpdates: row( + title: "Preparing Update", + detail: "Transcripted will ask you to restart when 9.9.9 is ready", + isVisible: true, + isEnabled: false + ) + ), + reportPath: nil, + appLogPath: nil + ) + + XCTAssertEqual(report.status, .pass) + XCTAssertTrue(report.checks.contains { $0.id == "downloading-utility-disabled" && $0.status == .pass }) + } + + func testAvailableScenarioFailsStaleCopy() { + let report = SparkleUpdateSmokeEvaluator.evaluate( + state: "available", + version: "9.9.9", + launchReport: launchReport( + updateCallout: row( + title: "Update ready", + detail: "A new version is ready to install", + trailingText: "Install", + isVisible: true + ), + checkUpdates: row(isVisible: false) + ), + reportPath: nil, + appLogPath: nil + ) + + XCTAssertEqual(report.status, .fail) + XCTAssertTrue(report.checks.contains { $0.id == "available-callout-title" && $0.status == .fail }) + } + + private func launchReport( + updateCallout: SparkleLaunchSmokeRow, + checkUpdates: SparkleLaunchSmokeRow + ) -> SparkleLaunchSmokeReport { + SparkleLaunchSmokeReport( + content: SparkleLaunchSmokeContent( + updateCallout: updateCallout, + utilityActions: SparkleLaunchSmokeUtilityActions(checkUpdates: checkUpdates) + ) + ) + } + + private func row( + title: String = "", + detail: String = "", + trailingText: String = "", + isVisible: Bool = true, + isEnabled: Bool = true + ) -> SparkleLaunchSmokeRow { + SparkleLaunchSmokeRow( + title: title, + detail: detail, + trailingText: trailingText, + isVisible: isVisible, + isEnabled: isEnabled + ) + } +} diff --git a/docs/qa-test-bench.md b/docs/qa-test-bench.md index 03903e57..d2b7d025 100644 --- a/docs/qa-test-bench.md +++ b/docs/qa-test-bench.md @@ -75,6 +75,26 @@ This requires Accessibility permission for the terminal or Codex runner. If macOS blocks AX observation/control, the result is `INCOMPLETE` with exit code `3`. Do not treat that as product proof. +## Sparkle Update UI Run + +```bash +bash scripts/ops/transcripted-qa-bench.sh --mode sparkle-update +``` + +This builds the app, then runs: + +```bash +swift run --package-path Tools/TranscriptedQA transcripted-qa sparkle-update-smoke --app build/Transcripted.app +``` + +The smoke uses the app's launch-smoke harness with a fake Sparkle update +available state and a fake downloading state. It checks the real menu +presentation snapshot for `Update available: `, `Install`, and the +disabled `Preparing Update` progress row. It writes local JSON evidence and a +fake appcast fixture. This is no-publish UI proof only; it does not prove the +live appcast, download, signature, install, relaunch, existing-install upgrade, +or Homebrew path. + ## Deep Run ```bash diff --git a/docs/sparkle-updates.md b/docs/sparkle-updates.md index 80061101..4ea35d08 100644 --- a/docs/sparkle-updates.md +++ b/docs/sparkle-updates.md @@ -70,6 +70,28 @@ change so the inventory stays complete. - `deps-tools/sparkle/bin/sign_update` - `deps-tools/sparkle/bin/generate_keys` +For a no-publish UI smoke of the native Transcripted update surfaces, build the +app and run: + +```bash +bash build.sh --no-open +swift run --package-path Tools/TranscriptedQA transcripted-qa sparkle-update-smoke --app build/Transcripted.app --output /tmp/transcripted-sparkle-update-smoke +``` + +Or through the QA bench: + +```bash +bash scripts/ops/transcripted-qa-bench.sh --mode sparkle-update +``` + +This launches the built app in the existing launch-smoke harness with a fake +Sparkle `updateAvailable` state and a fake `downloading` state. It verifies the +menu update-available callout and the download-progress row from the app's own +menu snapshot, and writes local JSON evidence plus a fake appcast fixture under +the output directory. It does not contact the live feed, download an update, +verify a signature, install, relaunch, publish, update Homebrew, or prove an +existing installed app can upgrade. + ## Release flow 1. Build a signed/notarized Transcripted archive, typically with `build-beta.sh`. diff --git a/scripts/ops/release-gate-report.py b/scripts/ops/release-gate-report.py index e7011056..7fe16677 100644 --- a/scripts/ops/release-gate-report.py +++ b/scripts/ops/release-gate-report.py @@ -778,8 +778,8 @@ def manual_items(args: argparse.Namespace, root: Path, out_dir: Path) -> list[Re ReportItem( "Existing-install update path", "yellow", - "UNKNOWN: on a real installed app, verify Sparkle check/install behavior and Homebrew install/upgrade when publishing.", - evidence_list([root / "docs/sparkle-updates.md", root / "docs/release-packaging.md"]), + "UNKNOWN: local fake-state smoke can verify available/downloading UI copy, but a real installed app still needs Sparkle check/install behavior and Homebrew install/upgrade proof when publishing.", + evidence_list([root / "docs/sparkle-updates.md", root / "docs/release-packaging.md", root / "docs/qa-test-bench.md"]), ), ] if args.qa_mode != "live": diff --git a/scripts/ops/transcripted-qa-bench.sh b/scripts/ops/transcripted-qa-bench.sh index 743360da..8f4a38e8 100755 --- a/scripts/ops/transcripted-qa-bench.sh +++ b/scripts/ops/transcripted-qa-bench.sh @@ -31,7 +31,7 @@ PACKAGED_USER_NAME="${TRANSCRIPTED_QA_PACKAGED_USER_NAME:-${USER:-codex}}" usage() { cat <<'USAGE' -Usage: bash scripts/ops/transcripted-qa-bench.sh [--mode quick|deep|full|ui|packaged|artifact|audio-synthetic|pasteback-synthetic|corpus|corpus-compare|scorecard|live] [options] +Usage: bash scripts/ops/transcripted-qa-bench.sh [--mode quick|deep|full|ui|sparkle-update|packaged|artifact|audio-synthetic|pasteback-synthetic|corpus|corpus-compare|scorecard|live] [options] Runs a local Transcripted QA bench and writes: /tmp/transcripted-qa-bench//qa-report.md @@ -41,6 +41,7 @@ Modes: deep quick + integration, Core tests, QA CLI, synthetic audio full deep + release-health and local Gemma summary dry-run gates ui build + Accessibility-driven menu bar/Home/Settings smoke + sparkle-update build + no-publish fake Sparkle available/downloading UI smoke packaged no-publish build-beta smoke + packaged app/version/Sparkle/dSYM/log checks artifact validate current saved Transcripted artifacts strictly audio-synthetic run only the deterministic audio failure-shape matrix @@ -185,7 +186,7 @@ while [[ $# -gt 0 ]]; do done case "$MODE" in - quick|deep|full|ui|packaged|artifact|audio-synthetic|pasteback-synthetic|corpus|corpus-compare|scorecard|live) ;; + quick|deep|full|ui|sparkle-update|packaged|artifact|audio-synthetic|pasteback-synthetic|corpus|corpus-compare|scorecard|live) ;; *) echo "Unknown mode: $MODE" >&2 usage >&2 @@ -832,6 +833,11 @@ run_ui_tail() { "TRANSCRIPTED_DISABLE_FILE_LOGGER=1 swift run --package-path Tools/TranscriptedQA transcripted-qa ui-smoke --app build/Transcripted.app --report $(shell_quote "${RAW_DIR}/ui-automation-smoke.json")" } +run_sparkle_update_tail() { + run_step "05-sparkle-update-smoke" "No-publish Sparkle update UI smoke" "yes" \ + "TRANSCRIPTED_DISABLE_FILE_LOGGER=1 swift run --package-path Tools/TranscriptedQA transcripted-qa sparkle-update-smoke --app build/Transcripted.app --output $(shell_quote "${RAW_DIR}/sparkle-update-smoke")" +} + run_packaged_tail() { if [[ "${SKIP_BUILD}" -eq 1 ]]; then skip_step "69-build-beta-smoke" "No-publish build-beta package smoke" @@ -952,6 +958,15 @@ case "${MODE}" in fi run_ui_tail ;; + sparkle-update) + run_step "00-preflight" "Agent preflight" "no" "bash scripts/dev/agent-preflight.sh" + if [[ "${SKIP_BUILD}" -eq 1 ]]; then + skip_step "01-build" "Build app" + else + run_step "01-build" "Build app" "yes" "bash build.sh --no-open" + fi + run_sparkle_update_tail + ;; packaged) run_step "00-preflight" "Agent preflight" "no" "bash scripts/dev/agent-preflight.sh" run_packaged_tail From 63bff8988386e2b51be5b52779f12bb0ba3fd1cb Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 25 Jun 2026 18:56:52 -0500 Subject: [PATCH 3/3] Document Sparkle update smoke lane --- .agents/qa-gates.yml | 5 ++++- Tests/README.md | 22 +++++++++++++++++++--- docs/agent-onboarding.md | 3 +++ docs/repo-layout.md | 6 ++++-- docs/test-automation-strategy.md | 9 ++++++--- 5 files changed, 36 insertions(+), 9 deletions(-) diff --git a/.agents/qa-gates.yml b/.agents/qa-gates.yml index 720166d5..29c35fe2 100644 --- a/.agents/qa-gates.yml +++ b/.agents/qa-gates.yml @@ -53,6 +53,7 @@ test_layers: deep: "bash scripts/ops/transcripted-qa-bench.sh --mode deep" full: "bash scripts/ops/transcripted-qa-bench.sh --mode full" ui: "bash scripts/ops/transcripted-qa-bench.sh --mode ui" + sparkle_update: "bash scripts/ops/transcripted-qa-bench.sh --mode sparkle-update" packaged: "bash scripts/ops/transcripted-qa-bench.sh --mode packaged" artifact: "bash scripts/ops/transcripted-qa-bench.sh --mode artifact" audio_synthetic: "bash scripts/ops/transcripted-qa-bench.sh --mode audio-synthetic" @@ -60,7 +61,7 @@ test_layers: corpus_compare: "bash scripts/ops/transcripted-qa-bench.sh --mode corpus-compare" live: "bash scripts/ops/transcripted-qa-bench.sh --mode live" proves: - - "one local QA report can combine build, fast tests, E2E, integration, QA CLI, synthetic audio, release-health, packaged-app, optional Gemma planning, corpus, and live capture proof" + - "one local QA report can combine build, fast tests, E2E, integration, QA CLI, Sparkle update UI smoke, synthetic audio, release-health, packaged-app, optional Gemma planning, corpus, and live capture proof" agent_notes: - "Reports live under /tmp/transcripted-qa-bench//qa-report.md." - "Corpus and live modes depend on private local data, permissions, and hardware. Permission blockers are INCOMPLETE, not green." @@ -107,6 +108,7 @@ test_layers: release_candidate_report: "python3 scripts/ops/release-gate-report.py --release-candidate" packaging_smoke: "SKIP_NOTARIZATION=1 bash build-beta.sh '' " packaged_app_smoke: "bash scripts/ops/transcripted-qa-bench.sh --mode packaged" + sparkle_update_smoke: "bash scripts/ops/transcripted-qa-bench.sh --mode sparkle-update" sparkle_verify: "bash scripts/release/verify-sparkle-release.sh " sentry_register: "SENTRY_REQUIRE_DEBUG_FILES=1 bash scripts/release/register-sentry-release.sh " cask_update: "bash scripts/release/update-cask.sh " @@ -171,6 +173,7 @@ recommended_gates: - "swift test" - "SKIP_NOTARIZATION=1 bash build-beta.sh '' " - "bash scripts/ops/transcripted-qa-bench.sh --mode packaged" + - "bash scripts/ops/transcripted-qa-bench.sh --mode sparkle-update" add_when_shipping_to_users: - "bash scripts/ops/transcripted-qa-bench.sh --mode deep --strict-artifacts when fixture or private artifact roots are available" - "full notarized build-beta.sh" diff --git a/Tests/README.md b/Tests/README.md index aadd81fe..48d7fd9b 100644 --- a/Tests/README.md +++ b/Tests/README.md @@ -2,7 +2,7 @@ ## Test Surfaces -This repo has ten distinct verification layers: +This repo has eleven distinct verification layers: 1. `bash run-tests.sh` Curated fast test runner built with raw `swiftc` @@ -20,9 +20,11 @@ This repo has ten distinct verification layers: Local hardware/TCC smoke for app launch plus production mic + system-audio capture 8. `bash scripts/ops/transcripted-qa-bench.sh --mode ui` Accessibility-driven UI smoke for first-run onboarding, menu bar, Home, Settings, buttons, and basic navigation -9. `bash scripts/ops/transcripted-qa-bench.sh --mode packaged` +9. `bash scripts/ops/transcripted-qa-bench.sh --mode sparkle-update` + No-publish fake-state Sparkle update UI smoke for update-available and downloading menu surfaces +10. `bash scripts/ops/transcripted-qa-bench.sh --mode packaged` No-publish `build-beta.sh` package smoke plus built app version, Sparkle, signing, dSYM, DMG, optional menu bar, and local log privacy checks -10. `bash scripts/ops/transcripted-qa-bench.sh --mode full` +11. `bash scripts/ops/transcripted-qa-bench.sh --mode full` Deep QA plus release-health fixture proof and local Gemma summary planning when eligible transcripts exist There is also an orchestrated QA bench for human-style passes: @@ -32,6 +34,7 @@ bash scripts/ops/transcripted-qa-bench.sh --mode quick bash scripts/ops/transcripted-qa-bench.sh --mode deep bash scripts/ops/transcripted-qa-bench.sh --mode full bash scripts/ops/transcripted-qa-bench.sh --mode ui +bash scripts/ops/transcripted-qa-bench.sh --mode sparkle-update bash scripts/ops/transcripted-qa-bench.sh --mode packaged bash scripts/ops/transcripted-qa-bench.sh --mode pasteback-synthetic bash scripts/ops/transcripted-qa-bench.sh --mode corpus @@ -189,6 +192,19 @@ Accessibility permission for the terminal or Codex runner so it can inspect AX identifiers and press controls. Missing permission exits `3` and is reported as `INCOMPLETE`, not green. +## Sparkle Update UI Smoke + +`bash scripts/ops/transcripted-qa-bench.sh --mode sparkle-update` builds the +app, then runs `transcripted-qa sparkle-update-smoke` against +`build/Transcripted.app`. It launches the app through the launch-smoke harness +with fake update-available and downloading states, then validates the menu +snapshot copy and visibility for the prominent install callout and disabled +download-progress row. + +This is no-publish local UI proof only. It does not contact the live appcast, +download or verify an update, install, relaunch, notarize, publish, update +Homebrew, or prove an existing installed app can upgrade. + ## Packaged App Smoke `bash scripts/ops/transcripted-qa-bench.sh --mode packaged` runs a no-publish diff --git a/docs/agent-onboarding.md b/docs/agent-onboarding.md index 9d9c4543..c14087b8 100644 --- a/docs/agent-onboarding.md +++ b/docs/agent-onboarding.md @@ -115,6 +115,9 @@ full current map: - `bash scripts/ops/transcripted-qa-bench.sh --mode ui` Accessibility-driven smoke for menu bar, Home, Settings, buttons, and basic navigation. TCC blockers are `INCOMPLETE`, not green. +- `bash scripts/ops/transcripted-qa-bench.sh --mode sparkle-update` + No-publish fake-state Sparkle update UI smoke for update-available and + downloading menu surfaces. Live feed/download/install proof is still separate. Rule of thumb: diff --git a/docs/repo-layout.md b/docs/repo-layout.md index 67b7797b..760cb337 100644 --- a/docs/repo-layout.md +++ b/docs/repo-layout.md @@ -36,6 +36,8 @@ python3 scripts/ops/release-gate-report.py bash scripts/ops/transcripted-qa-bench.sh --mode quick bash scripts/ops/transcripted-qa-bench.sh --mode full bash scripts/ops/transcripted-qa-bench.sh --mode ui +bash scripts/ops/transcripted-qa-bench.sh --mode sparkle-update +bash scripts/ops/transcripted-qa-bench.sh --mode packaged bash scripts/ops/transcripted-qa-bench.sh --mode corpus bash scripts/ops/transcripted-qa-bench.sh --mode corpus-compare swift test @@ -54,7 +56,7 @@ Command ownership: - `run-live-capture-smoke.sh` — thin root wrapper for local hardware/TCC capture smoke - `run-daily-audio-reliability.sh` — thin root wrapper for the interactive and synthetic daily audio reliability check - `scripts/ops/release-gate-report.py` — single pre-merge/release report covering QA bench, telemetry, release surfaces, and local log warnings -- `scripts/ops/transcripted-qa-bench.sh` — orchestrated QA tester pass with local report output, including `--mode ui` for the Accessibility-driven onboarding/menu bar/Home/Settings smoke +- `scripts/ops/transcripted-qa-bench.sh` — orchestrated QA tester pass with local report output, including `--mode ui` for the Accessibility-driven onboarding/menu bar/Home/Settings smoke, `--mode sparkle-update` for fake-state Sparkle update UI proof, and `--mode packaged` for no-publish package smoke - `scripts/ops/validate-meeting-corpus.py` — local-only meeting corpus validator for Downloads fixtures - `scripts/ops/compare-meeting-corpus.py` — local-only Transcripted-vs-Zoom corpus comparator for Downloads fixtures - `swift test` — `TranscriptedCore` package seam tests @@ -117,7 +119,7 @@ Use these docs for these jobs: - `docs/retention-cohort-analytics.md` — privacy-safe PostHog habit and retention report for first/second artifact, next-day and 7-day return, repeat use, 3-days-this-week, and health-skill output - `docs/storage-paths.md` — canonical storage and fallback path map - `docs/audio-reliability-daily-check.md` — daily manual audio reliability loop and evidence contract -- `docs/qa-test-bench.md` — orchestrated QA tester bench for quick, deep, corpus, corpus-compare, live, artifact, and synthetic audio passes +- `docs/qa-test-bench.md` — orchestrated QA tester bench for quick, deep, UI, Sparkle update, packaged, corpus, corpus-compare, live, artifact, and synthetic audio passes - `docs/test-automation-strategy.md` — agent-first QA coverage map, gate strategy, and automation roadmap - `docs/qa-issue-500-meeting-audio.md` — manual WebRTC / meeting-volume QA matrix for issue #500 - `docs/release-packaging.md` — release packaging flow diff --git a/docs/test-automation-strategy.md b/docs/test-automation-strategy.md index ffc8d9ae..f0e1471b 100644 --- a/docs/test-automation-strategy.md +++ b/docs/test-automation-strategy.md @@ -34,9 +34,9 @@ As of 2026-06-06, the repo has these automated layers: Markdown, retained audio, parser discovery, and TranscriptedQA validation. It is not native picker or real media transcription proof. - `bash scripts/ops/transcripted-qa-bench.sh --mode ...`: orchestrated QA - reports for `quick`, `deep`, `full`, `ui`, `packaged`, `artifact`, - `audio-synthetic`, `pasteback-synthetic`, `corpus`, `corpus-compare`, - `scorecard`, and `live`. + reports for `quick`, `deep`, `full`, `ui`, `sparkle-update`, `packaged`, + `artifact`, `audio-synthetic`, `pasteback-synthetic`, `corpus`, + `corpus-compare`, `scorecard`, and `live`. - `.github/workflows/repo-hygiene.yml`: PR/workflow-dispatch hygiene that runs preflight plus shell, Ruby, and Python syntax checks. - BET-88 GitHub workflows: historical label-gated fixtures for the closed QA @@ -62,6 +62,9 @@ As of 2026-06-06, the repo has these automated layers: - Release: release docs and scripts exist, but agents need one release-health report that compares source truth with GitHub release, appcast, cask, live download, crawler text, and Sentry metadata. +- Updates: `--mode sparkle-update` covers local fake-state update-available and + downloading menu UI, but live appcast, signature/download/install, relaunch, + and existing-install upgrade proof remain separate release-path checks. - Privacy: sanitizer tests exist, but QA bench reports, generated PR text, local logs, and release notes need a single leakage sweep. - Summaries: summary preferences and local summarizer behavior have tests, but