diff --git a/src/programs/__tests__/api-key-login.test.ts b/src/programs/__tests__/api-key-login.test.ts new file mode 100644 index 000000000..d33cb0b9f --- /dev/null +++ b/src/programs/__tests__/api-key-login.test.ts @@ -0,0 +1,111 @@ +import { apiKeyCredentials, resolveApiKeyLogin } from '../api-key-login'; +import { detectRegion } from '@utils/urls'; +import { fetchProjectData, fetchUserData } from '@shared/api'; + +vi.mock('@utils/urls', () => ({ + detectRegion: vi.fn(), + getHost: (r: string) => `https://${r}.posthog.com`, + getCloudUrl: (r: string) => `https://${r}.posthog.com`, + getUiHostFromHost: (host: string) => host, + resolveBaseUrl: (baseUrl?: string) => baseUrl, +})); +vi.mock('@shared/api', () => ({ + fetchProjectData: vi.fn(), + fetchUserData: vi.fn(), +})); +vi.mock('@utils/analytics', () => ({ + analytics: { + identifyUser: vi.fn(), + captureException: vi.fn(), + setTag: vi.fn(), + }, +})); + +const mockedDetect = detectRegion as unknown as ReturnType; +const mockedFetchProject = fetchProjectData as unknown as ReturnType< + typeof vi.fn +>; +const mockedFetchUser = fetchUserData as unknown as ReturnType; + +const project = { + id: 123, + uuid: '00000000-0000-0000-0000-000000000000', + organization: '11111111-1111-1111-1111-111111111111', + api_token: 'phc_test', + name: 'Test Project', +}; + +describe('resolveApiKeyLogin CI region', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockedFetchProject.mockResolvedValue(project); + mockedFetchUser.mockResolvedValue({ + distinct_id: 'user-1', + role_at_organization: null, + }); + }); + + it('uses the provided region and never probes @me for it', async () => { + const result = await resolveApiKeyLogin('phx_test', { + projectId: 123, + region: 'eu', + }); + + // The flaky region probe must not run when the region was handed in. + expect(mockedDetect).not.toHaveBeenCalled(); + // And the project is fetched from the given region's cloud. + expect(mockedFetchProject).toHaveBeenCalledWith( + 'phx_test', + 123, + 'https://eu.posthog.com', + ); + expect(result.posthog.host.region).toBe('eu'); + }); + + it('reports the CI login line once', async () => { + const onInfo = vi.fn(); + await resolveApiKeyLogin('phx_test', { + projectId: 123, + region: 'eu', + onInfo, + }); + expect(onInfo).toHaveBeenCalledOnce(); + }); + + it('falls back to detection only when no region is provided', async () => { + mockedDetect.mockResolvedValue('us'); + + await resolveApiKeyLogin('phx_test', { projectId: 123 }); + + expect(mockedDetect).toHaveBeenCalledTimes(1); + }); +}); + +describe('apiKeyCredentials', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('logs in once, and logs in again after a failed login', async () => { + mockedFetchUser.mockResolvedValue({ + distinct_id: 'user-1', + role_at_organization: null, + }); + mockedFetchProject + .mockRejectedValueOnce(new Error('503 transient')) + .mockResolvedValue(project); + const provider = apiKeyCredentials('phx_test', { + projectId: 123, + region: 'eu', + }); + const context = { signal: new AbortController().signal }; + + await expect(provider.resolve('metrics', context)).rejects.toThrow( + '503 transient', + ); + const login = await provider.resolve('metrics', context); + expect(login.posthog.projectId).toBe(123); + await provider.resolve('metrics', context); + expect(mockedFetchProject).toHaveBeenCalledTimes(2); + }); +}); diff --git a/src/programs/__tests__/login.test.ts b/src/programs/__tests__/login.test.ts new file mode 100644 index 000000000..35e4b25ff --- /dev/null +++ b/src/programs/__tests__/login.test.ts @@ -0,0 +1,71 @@ +vi.mock(import('@utils/analytics'), () => ({ + analytics: { + wizardCapture: vi.fn(), + setTag: vi.fn(), + identifyUser: vi.fn(), + setGroups: vi.fn(), + } as never, + groupsFromUser: vi.fn(() => ({})), +})); + +import type { ApiUser } from '@shared/api'; +import { HostResolution } from '@shared/host-resolution'; +import { analytics } from '@utils/analytics'; +import { logIn } from '../login'; +import { SessionStore } from '../session/session-store'; +import { buildSession } from '../session/wizard-session'; + +const apiUser = { role_at_organization: 'engineering' } as ApiUser; +const login = { + posthog: { + accessToken: 'phx_test', + projectApiKey: 'phc_test', + projectId: 7, + host: HostResolution.fromRegion('us'), + }, + project: null, + apiUser, +}; +const never = new AbortController().signal; + +describe('logIn', () => { + beforeEach(() => vi.clearAllMocks()); + + it("records a provider's login once, with no auth-complete event of its own", async () => { + const store = new SessionStore(buildSession({ installDir: '/project' })); + const resolve = vi.fn(() => Promise.resolve(login)); + await logIn('metrics', store, { provider: { resolve }, signal: never }); + expect(store.session.credentials).toEqual(login.posthog); + expect(store.session.apiUser).toBe(apiUser); + expect(store.session.roleAtOrganization).toBe('engineering'); + // The OAuth login reports `auth complete`; a key login is not a user signing in. + expect(analytics.wizardCapture).not.toHaveBeenCalled(); + + // A second call in the same store reuses the login. + await logIn('metrics', store, { provider: { resolve }, signal: never }); + expect(resolve).toHaveBeenCalledOnce(); + }); + + it("puts a CI run's pre-issued gateway token on the login", async () => { + const store = new SessionStore(buildSession({ installDir: '/project' })); + const gateway = { token: 'gw_test', url: 'http://localhost:8080' }; + store.update({ ciGateway: gateway }); + const result = await logIn('metrics', store, { + credentials: login, + signal: never, + }); + expect(result.posthog.gateway).toEqual(gateway); + expect(store.session.credentials?.gateway).toEqual(gateway); + }); + + it("keeps the store's role when the gateway token rewrites a held login", async () => { + const store = new SessionStore(buildSession({ installDir: '/project' })); + store.setLogin(login); + store.setRoleAtOrganization('product'); + store.update({ + ciGateway: { token: 'gw_test', url: 'http://localhost:8080' }, + }); + await logIn('metrics', store, { signal: never }); + expect(store.session.roleAtOrganization).toBe('product'); + }); +}); diff --git a/src/programs/__tests__/posthog-cli-preinstall.test.ts b/src/programs/__tests__/posthog-cli-preinstall.test.ts index d91421bc4..3bb7b96a6 100644 --- a/src/programs/__tests__/posthog-cli-preinstall.test.ts +++ b/src/programs/__tests__/posthog-cli-preinstall.test.ts @@ -5,15 +5,18 @@ import { resetPostHogCliPreinstallForTests, } from '@programs/shared/posthog-cli-preinstall'; import { installOrUpdatePostHogCli } from '@shared/install-cli-steering'; +import { Integration } from '@shared/constants'; import { analytics } from '@utils/analytics'; +import { SYMBOL_UPLOAD_CLI_FRAMEWORKS } from '@programs/error-tracking'; +import { VARIANTS_REQUIRING_POSTHOG_CLI } from '@programs/error-tracking-upload-source-maps'; -vi.mock('@shared/install-cli-steering', () => ({ +vi.mock(import('@shared/install-cli-steering'), () => ({ installOrUpdatePostHogCli: vi.fn(), })); vi.mock('@utils/analytics', () => ({ analytics: { wizardCapture: vi.fn(), captureException: vi.fn() }, })); -const log = { warn: vi.fn() }; +const log = { info: vi.fn(), warn: vi.fn() }; describe('preinstallPostHogCliOnce', () => { beforeEach(() => { @@ -71,3 +74,15 @@ describe('preinstallPostHogCliOnce', () => { expect(log.warn).not.toHaveBeenCalled(); }); }); + +describe('the two programs that pre-install posthog-cli', () => { + test('error tracking matches the source-maps program set, keyed by Integration', () => { + // Both programs pre-install the CLI for the same platforms. The source-maps + // program keys them by uploader variant, and only `ios` is spelled + // differently (`swift` in Integration). + const expected = [...VARIANTS_REQUIRING_POSTHOG_CLI] + .map((variant) => (variant === 'ios' ? Integration.swift : variant)) + .sort(); + expect([...SYMBOL_UPLOAD_CLI_FRAMEWORKS].sort()).toEqual(expected); + }); +}); diff --git a/src/programs/__tests__/program-registry.test.ts b/src/programs/__tests__/program-registry.test.ts index 891fe4022..9d5dbb98d 100644 --- a/src/programs/__tests__/program-registry.test.ts +++ b/src/programs/__tests__/program-registry.test.ts @@ -1,22 +1,28 @@ import { PROGRAM_REGISTRY, - agentSkillConfig, getCommandPath, - getLaunchablePrograms, getProgramConfig, getSubcommandPrograms, } from '../program-registry'; -import type { WizardSession } from '@programs/session/wizard-session'; -import { testRunnerContext } from '../../../test/runner-context'; +import { DEFAULT_BINDING } from '@agent'; +import { + DEFAULT_AGENT_MODEL, + GPT5_6_SOL_MODEL, + GPT5_6_TERRA_MODEL, + Harness, + Sequence, +} from '@shared/constants'; +import { config as agentSkill } from '@programs/agent-skill'; +import { testRunnerContext } from '../shared/__tests__/runner-context.no-jest'; +import type { WizardSession } from '../session/wizard-session'; describe('PROGRAM_REGISTRY', () => { - it('every entry has unique id, description, and non-empty steps', () => { + it('every entry has a unique id and a description', () => { const ids = PROGRAM_REGISTRY.map((c) => c.id); expect(new Set(ids).size).toBe(ids.length); for (const config of PROGRAM_REGISTRY) { expect(config.description).toBeTruthy(); - expect(config.steps.length).toBeGreaterThan(0); } }); }); @@ -32,6 +38,54 @@ describe('getProgramConfig', () => { }); }); +// A binding sets a program's sequence, harness, model and effort, so an edit +// changes the cost and output of every run of it. +describe('program bindings', () => { + const linearOnSol = { + sequence: Sequence.linear, + harness: Harness.pi, + model: GPT5_6_SOL_MODEL, + thinkingLevel: 'medium', + }; + const orchestratorOnPi = { + sequence: Sequence.orchestrator, + harness: Harness.pi, + model: DEFAULT_AGENT_MODEL, + }; + + it('routes every program, and the default for one with no binding', () => { + const routes = Object.fromEntries( + PROGRAM_REGISTRY.map((c) => [c.id, c.binding ?? DEFAULT_BINDING]), + ); + expect(routes).toEqual({ + 'posthog-integration': linearOnSol, + 'mcp-analytics': linearOnSol, + 'replay-vision': { + sequence: Sequence.orchestrator, + harness: Harness.anthropic, + model: DEFAULT_AGENT_MODEL, + }, + 'ai-observability': { + sequence: Sequence.linear, + harness: Harness.pi, + model: GPT5_6_TERRA_MODEL, + thinkingLevel: 'high', + }, + metrics: orchestratorOnPi, + audit: linearOnSol, + 'events-audit': linearOnSol, + 'web-analytics-doctor': linearOnSol, + migration: linearOnSol, + 'revenue-analytics-setup': linearOnSol, + 'warehouse-source': linearOnSol, + 'self-driving': linearOnSol, + 'error-tracking-upload-source-maps': linearOnSol, + 'error-tracking': orchestratorOnPi, + 'agent-skill': linearOnSol, + }); + }); +}); + describe('getSubcommandPrograms', () => { it('returns only programs that have a CLI command', () => { const subcommands = getSubcommandPrograms(); @@ -60,36 +114,9 @@ describe('getCommandPath', () => { 'revenue-analytics', ); }); -}); - -describe('getLaunchablePrograms', () => { - // The list is curated, so an id that stops matching drops its row in silence. - it("offers the intro's programs, in order, all resolving", () => { - expect(getLaunchablePrograms().map((config) => config.id)).toEqual([ - 'self-driving', - 'error-tracking-upload-source-maps', - 'warehouse-source', - 'audit', - 'posthog-doctor', - 'mcp-analytics', - 'replay-vision', - 'ai-observability', - 'metrics', - 'revenue-analytics-setup', - ]); - }); - - // A row wider than the terminal stops the whole block from centering. - it('keeps every row inside an 80-column terminal', () => { - const COMMAND_COLUMN = 21; - const MARKER_PREFIX = 2; - const BUDGET = 80 - COMMAND_COLUMN - MARKER_PREFIX; - const tooLong = getLaunchablePrograms() - .filter((config) => config.description.length > BUDGET) - .map((config) => `${config.id} (${config.description.length})`); - - expect(tooLong).toEqual([]); + it('keeps `metrics` a flat command', () => { + expect(getCommandPath(subcommand('metrics'))).toBe('metrics'); }); }); @@ -121,30 +148,21 @@ describe('parentCommand nesting', () => { }); }); -describe('agentSkillConfig run recipe', () => { - // Regression guard: `agentSkillConfig` backs `wizard skill ` and the - // narrow `audit` leaves. The runner skips the agent entirely when a config - // has no `run` (run-wizard.ts `skipAgent`), so a missing recipe means those - // commands silently no-op instead of running the skill. - it('defines a run recipe so the agent is not skipped', () => { - expect(agentSkillConfig.run).toBeDefined(); - }); - +describe('agent-skill run recipe', () => { + // Regression guard: the agent-skill config backs `wizard skill ` and the + // narrow `audit` leaves. runProgram fails with "has no run configuration" + // when a config has no `run`, so a missing recipe means those commands fail + // instead of running the skill. it('derives run metadata from the dispatched skillId', async () => { - expect(typeof agentSkillConfig.run).toBe('function'); + expect(typeof agentSkill.run).toBe('function'); const session = { skillId: 'audit-events' } as unknown as WizardSession; const run = - typeof agentSkillConfig.run === 'function' - ? await agentSkillConfig.run(session, testRunnerContext()) - : agentSkillConfig.run!; + typeof agentSkill.run === 'function' + ? await agentSkill.run(session, testRunnerContext()) + : agentSkill.run!; expect(run.skillId).toBe('audit-events'); expect(run.integrationLabel).toBe('audit-events'); expect(run.reportFile).toContain('audit-events'); - // Fields the runner relies on to render the run + outro. - expect(run.spinnerMessage).toBeTruthy(); - expect(run.successMessage).toBeTruthy(); - expect(run.docsUrl).toBeTruthy(); - expect(run.estimatedDurationMinutes).toBeGreaterThan(0); }); }); diff --git a/src/programs/__tests__/program-scopes.test.ts b/src/programs/__tests__/program-scopes.test.ts index 1200b444b..4de5ce10b 100644 --- a/src/programs/__tests__/program-scopes.test.ts +++ b/src/programs/__tests__/program-scopes.test.ts @@ -6,7 +6,7 @@ import { getOAuthScopesForProgram, getProvisioningScopesForProgram, -} from '@programs/oauth/program-scopes'; +} from '../program-registry'; describe('posthog-integration scopes', () => { it('includes the warehouse pair for the orchestrator warehouse task', () => { @@ -73,7 +73,7 @@ describe('provisioning scopes', () => { expect(getProvisioningScopesForProgram(null)).not.toContain( 'replay_scanner:write', ); - expect(getProvisioningScopesForProgram('mcp-tutorial')).not.toContain( + expect(getProvisioningScopesForProgram('metrics')).not.toContain( 'replay_scanner:write', ); }); diff --git a/src/programs/__tests__/refresh-access-token-if-needed.test.ts b/src/programs/__tests__/refresh-access-token-if-needed.test.ts index cfe5988ca..160a5cfeb 100644 --- a/src/programs/__tests__/refresh-access-token-if-needed.test.ts +++ b/src/programs/__tests__/refresh-access-token-if-needed.test.ts @@ -1,5 +1,5 @@ import { rotateCredentials } from '../credentials'; -import { refreshAccessToken } from '@tui/auth/oauth'; +import { refreshAccessToken } from '../oauth/tokens'; import { OAuthError } from '@utils/oauth-errors'; import { isGrantRevoked, @@ -12,8 +12,8 @@ import { } from '@shared/oauth-session'; import type { Credentials } from '@shared/api'; -vi.mock('@tui/auth/oauth', async (original) => ({ - ...(await original()), +vi.mock(import('../oauth/tokens'), async (original) => ({ + ...(await original()), refreshAccessToken: vi.fn(), })); vi.mock('@utils/debug', () => ({ logToFile: vi.fn() })); @@ -21,8 +21,6 @@ vi.mock('@utils/analytics', () => ({ analytics: { wizardCapture: vi.fn() }, groupsFromUser: vi.fn(), })); -// The real @utils/oauth loads the UI module. -vi.mock('@ui', () => ({ getUI: vi.fn() })); const mockedRefresh = refreshAccessToken as Mock; diff --git a/src/programs/__tests__/run-program.test.ts b/src/programs/__tests__/run-program.test.ts index fba58bb6f..fe21108aa 100644 --- a/src/programs/__tests__/run-program.test.ts +++ b/src/programs/__tests__/run-program.test.ts @@ -1,16 +1,60 @@ -import { runAgent, RunOutcome } from '@agent'; -import { Harness, Sequence } from '@shared/constants'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import { DEFAULT_BINDING, runAgent, RunOutcome } from '@agent'; +import { Harness, Integration, Sequence } from '@shared/constants'; import { HostResolution } from '@shared/host-resolution'; import type { ApiUser } from '@shared/api'; -import type { RunResult } from '@agent/types'; +import type { AgentProgress, RunResult } from '@agent/types'; import { ErrorCodes } from '@shared/errors'; -import { DiscoveredFeature } from '@programs/session/wizard-session'; +import { DiscoveredFeature } from '@shared/discovered-feature'; +import { OutroKind } from '@shared/outro'; +import { RunPhase, ScanConsent } from '@shared/run-state'; +import { + checkAllSettingsConflicts, + backupAndFixClaudeSettings, + restoreClaudeSettings, + type SettingsConflict, +} from '@shared/claude-settings'; +import { + evaluateWizardReadiness, + WizardReadiness, + type WizardReadinessResult, +} from '@shared/health-checks/readiness'; +import { ServiceHealthStatus } from '@shared/health-checks/types'; import { analytics } from '@utils/analytics'; -import { refreshAccessToken } from '@tui/auth/oauth'; -import { oauthCredentials, resetOAuthSession } from '@shared/oauth-session'; +import { registerCleanup } from '@utils/cleanup'; +import { logToFile } from '@utils/debug'; +import { refreshAccessToken } from '../oauth/tokens'; +import { + configureOAuthSession, + oauthCredentials, + resetOAuthSession, +} from '@shared/oauth-session'; +import { preinstallPostHogCliOnce } from '@programs/shared/posthog-cli-preinstall'; import type { ResolvedProgramCredentials } from '../credentials'; -import type { ProgramInput, ProgramOptions } from '../run-program'; -import { runProgram } from '@programs'; +import type { + ProgramInput, + ProgramOptions, + ProgramProgress, + ProgramStep, +} from '../program-input'; +import type { ProgramSession } from '../program-session'; +import type { ProgramReadyContext } from '../program-step'; +import type { ProgramRun } from '../program-run'; +import type { CiRunnerContext, RunnerContext } from '../runner-context'; +import type { WizardSession } from '../session/wizard-session'; +import { AUDIT_CHECKS_KEY } from '@programs/audit'; +import { FRAMEWORK_REGISTRY } from '../frameworks/registry'; +import { detectErrorCode } from '../detect-map'; +import { config as metrics } from '@programs/metrics'; +import { + buildSession, + ProgramAbort, + runProgram, + SessionStore, + TASK_OUTCOMES_KEY, +} from '@programs'; vi.mock('@agent', async (importOriginal) => ({ ...(await importOriginal()), @@ -29,11 +73,27 @@ vi.mock('@utils/analytics', async (importOriginal) => ({ groupIdentify: vi.fn(), }, })); -vi.mock('@tui/auth/oauth', () => ({ +vi.mock(import('../oauth/tokens'), () => ({ refreshAccessToken: vi.fn(), missingOAuthScopes: vi.fn(() => []), })); vi.mock('@utils/debug'); +vi.mock(import('@utils/cleanup'), () => ({ + registerCleanup: vi.fn(() => () => undefined), +})); +vi.mock(import('@shared/health-checks/readiness'), async (importOriginal) => ({ + ...(await importOriginal()), + evaluateWizardReadiness: vi.fn(), +})); +vi.mock(import('@programs/shared/posthog-cli-preinstall'), () => ({ + preinstallPostHogCliOnce: vi.fn(), +})); +vi.mock(import('@shared/claude-settings'), async (importOriginal) => ({ + ...(await importOriginal()), + checkAllSettingsConflicts: vi.fn(), + backupAndFixClaudeSettings: vi.fn(), + restoreClaudeSettings: vi.fn(), +})); const run = { integrationLabel: 'metrics', @@ -88,13 +148,56 @@ const login = (apiUser: ApiUser | null = credentials.apiUser) => ({ resolve: () => Promise.resolve({ ...credentials, apiUser }), }); -/** A program with one post-auth gate, like the source-maps project picker. */ -const gated: ProgramInput = { - installDir: '/project', - run, - program: { postAuthGates: ['detect'] }, +/** A caller's session store, as a host builds it from launch values. */ +const store = (fields: Partial = {}) => { + const s = new SessionStore(buildSession({ installDir: '/project' })); + s.update(fields); + return s; }; +/** The metrics program with the test's run definition laid over it. */ +const input = (over: Partial = {}): ProgramInput => ({ + store: store(), + config: { run }, + ...over, +}); + +/** A host's workflow: answers every step with `answer(step)`, in the order asked. */ +const workflow = (answer: (step: ProgramStep) => boolean = () => true) => { + const steps: ProgramStep[] = []; + return { + steps, + confirmStep: vi.fn((step: ProgramStep) => { + steps.push(step); + return Promise.resolve(answer(step)); + }), + finishStep: vi.fn(), + }; +}; + +const readiness = (decision: WizardReadiness): WizardReadinessResult => ({ + decision, + health: { + skillsOrigin: { + status: + decision === WizardReadiness.No + ? ServiceHealthStatus.Down + : ServiceHealthStatus.Degraded, + }, + }, + reasons: [], +}); + +const managedConflict: SettingsConflict = { + source: 'managed', + path: '/etc/claude/managed-settings.json', + keys: ['ANTHROPIC_BASE_URL'], + writable: false, +}; + +const agentConfig = (call = 0) => vi.mocked(runAgent).mock.calls[call][0]; +const agentInput = (call = 0) => vi.mocked(runAgent).mock.calls[call][1]; + describe('runProgram', () => { beforeEach(() => { vi.clearAllMocks(); @@ -103,71 +206,638 @@ describe('runProgram', () => { outcome: RunOutcome.Success, snapshot, }); + vi.mocked(evaluateWizardReadiness).mockResolvedValue( + readiness(WizardReadiness.Yes), + ); + vi.mocked(checkAllSettingsConflicts).mockReturnValue([]); + vi.mocked(backupAndFixClaudeSettings).mockReturnValue(true); }); - it("runs the caller's run definition, settings and route, and returns its final results", async () => { + it("runs the registered program with the caller's config laid over it, routed by its binding, and records the run in the store", async () => { const excludedTaskTypes = () => ['logs']; const { signal } = new AbortController(); + const interaction = { ask: vi.fn() }; + const resolved = { + sequence: Sequence.linear, + harness: Harness.anthropic, + model: 'm', + }; + vi.mocked(runAgent).mockImplementationOnce((_config, _input, options) => { + options?.onProgress?.({ kind: 'binding', binding: resolved }); + options?.onProgress?.({ kind: 'lifecycle', phase: 'started' }); + options?.onProgress?.({ kind: 'status', message: 'Installing' }); + options?.onProgress?.({ + kind: 'lifecycle', + phase: 'completed', + message: 'Metrics configured', + }); + return Promise.resolve({ outcome: RunOutcome.Success, snapshot }); + }); + const s = store({ harness: Harness.anthropic, sequence: Sequence.linear }); + const seen: ProgramProgress[] = []; const outcome = await runProgram( 'metrics', { - installDir: '/project', + store: s, runId: 'run-1', - run, - program: { + config: { + run, agentFlow: 'metrics-flow', allowedTools: ['Agent'], disallowedTools: ['wizard_ask'], excludedTaskTypes, }, credentials, - overrides: { harness: Harness.anthropic, sequence: Sequence.linear }, }, - { signal }, + { + signal, + interaction, + onProgress: (progress) => void seen.push(progress), + }, ); - const [config, input, agentOptions] = vi.mocked(runAgent).mock.calls[0]; + const [config, runInput, agentOptions] = vi.mocked(runAgent).mock.calls[0]; expect(config).toMatchObject({ programId: 'metrics', run, agentFlow: 'metrics-flow', allowedTools: ['Agent'], disallowedTools: ['wizard_ask'], - binding: { sequence: Sequence.linear, harness: Harness.anthropic }, - switchboard: { - program: 'metrics', - cliHarness: Harness.anthropic, - cliSequence: Sequence.linear, - }, - wizardMetadata: { - program_id: 'metrics', - integration: 'metrics', - run_id: 'analytics-run-id', - build: 'test', - call_type: 'agent', - SEQUENCE: Sequence.linear, - HARNESS: Harness.anthropic, + // The agent resolves the launch overrides and flags over the program's own binding. + routing: { + binding: metrics.binding, + overrides: { harness: Harness.anthropic, sequence: Sequence.linear }, }, }); expect(config.excludedTaskTypes).toBe(excludedTaskTypes); - expect(input.credentials).toBe(credentials.posthog); + expect(runInput.credentials).toBe(credentials.posthog); expect(agentOptions?.signal).toBe(signal); - expect(analytics.setTag).toHaveBeenCalledWith('harness', Harness.anthropic); - expect(analytics.wizardCapture).toHaveBeenCalledWith( - 'switchboard resolved', - expect.objectContaining({ program: 'metrics', cli_harness: 'anthropic' }), - ); + // The agent asks and notices through the caller's answerer. + expect(agentOptions?.interaction).toBe(interaction); expect(outcome).toMatchObject({ programId: 'metrics', outcome: RunOutcome.Success, - data: { credentials: { projectId: 42 }, binding: config.binding }, - settledRuns: [ - { runId: 'run-1', result: { outcome: RunOutcome.Success } }, - ], + runResults: [{ outcome: RunOutcome.Success }], diagnostics: [], artifacts: { reportFile: '/project/posthog-metrics-report.md' }, }); expect(outcome.failure).toBeUndefined(); + // The route the agent reported reaches the observer, labelled with its run. + expect(seen[0]).toEqual({ + runId: 'run-1', + event: { kind: 'binding', binding: resolved }, + }); + // The run's state is in the caller's store. + expect(s.session).toMatchObject({ + credentials: { projectId: 42 }, + skillId: 'metrics', + runPhase: RunPhase.Completed, + outroData: { kind: OutroKind.Success, message: 'Metrics configured' }, + }); + expect(s.statusMessages).toContain('Installing'); + }); + + it('a composed run ends completed and leaves the outro to its caller', async () => { + vi.mocked(runAgent).mockImplementationOnce((_config, _input, options) => { + options?.onProgress?.({ kind: 'lifecycle', phase: 'started' }); + return Promise.resolve({ outcome: RunOutcome.Success, snapshot }); + }); + const s = store(); + + const result = await runProgram( + 'metrics', + input({ store: s, credentials, composed: true }), + ); + + expect(result.outcome).toBe(RunOutcome.Success); + expect(s.session.runPhase).toBe(RunPhase.Completed); + expect(s.session.outroData).toBeNull(); + }); + + it('routes a program that declares no binding with the default one', async () => { + await runProgram('not-registered', input({ credentials })); + expect(agentConfig().routing.binding).toEqual(DEFAULT_BINDING); + expect(agentConfig().programId).toBe('not-registered'); + }); + + it('fails a program with no run configuration before anything starts', async () => { + const s = store(); + const result = await runProgram('not-registered', { + store: s, + credentials, + }); + expect(result).toMatchObject({ + outcome: RunOutcome.Failed, + failure: { + message: 'Program "not-registered" has no run configuration.', + }, + }); + expect(evaluateWizardReadiness).not.toHaveBeenCalled(); + expect(runAgent).not.toHaveBeenCalled(); + // A settled failure is in the store, so a host's last push carries it. + expect(s.session.runPhase).toBe(RunPhase.Error); + expect(s.session.outroData).toMatchObject({ + kind: OutroKind.Error, + errorCode: ErrorCodes.InternalUnhandled, + }); + }); + + describe('detection', () => { + it("runs a CI session's ciPreRun on a copy of the session and keeps what it wrote", async () => { + const s = store({ ci: true }); + const ciPreRun = vi.fn((session: ProgramSession) => { + session.installDir = '/project/apps/web'; + session.frameworkContext.scanned = true; + return Promise.resolve(); + }); + await runProgram( + 'metrics', + { store: s, config: { run, ciPreRun }, credentials }, + {}, + ); + expect(ciPreRun).toHaveBeenCalledOnce(); + expect(s.session.detectionComplete).toBe(true); + expect(s.session.frameworkContext.scanned).toBe(true); + expect(agentInput().installDir).toBe('/project/apps/web'); + }); + + it("gives a CI ciPreRun's copy of the session the login it asked for", async () => { + let seen: WizardSession['credentials'] = null; + const ciPreRun = async ( + session: ProgramSession, + runner: CiRunnerContext, + ) => { + await runner.authenticate('metrics'); + seen = session.credentials; + }; + const s = store({ ci: true }); + await runProgram( + 'metrics', + { store: s, config: { run, ciPreRun }, credentials }, + {}, + ); + expect(seen).toBe(credentials.posthog); + expect(s.session.credentials).toBe(credentials.posthog); + }); + + it("ends a ProgramAbort from a CI ciPreRun as an aborted run, as main's decided stop ended", async () => { + const s = store({ ci: true }); + const result = await runProgram( + 'metrics', + { + store: s, + config: { + run, + ciPreRun: () => + Promise.reject( + new ProgramAbort({ + code: ErrorCodes.DetectNoFramework, + message: 'Could not auto-detect your framework.', + }), + ), + }, + credentials, + }, + {}, + ); + expect(result).toMatchObject({ + outcome: RunOutcome.Aborted, + failure: { code: ErrorCodes.DetectNoFramework }, + }); + expect(s.session.outroData?.errorCode).toBe(ErrorCodes.DetectNoFramework); + expect(runAgent).not.toHaveBeenCalled(); + }); + + it("reports a CI detection's log lines as the program's own progress", async () => { + const seen: ProgramProgress[] = []; + const ciPreRun = (_session: ProgramSession, runner: CiRunnerContext) => { + runner.log.info('Scanning the repo'); + runner.log.warn('Scan failed'); + return Promise.resolve(); + }; + await runProgram( + 'metrics', + { store: store({ ci: true }), config: { run, ciPreRun }, credentials }, + { onProgress: (progress) => void seen.push(progress) }, + ); + expect(seen.map((p) => p.event).slice(0, 2)).toEqual([ + { kind: 'log', level: 'info', message: 'Scanning the repo' }, + { kind: 'log', level: 'warn', message: 'Scan failed' }, + ]); + }); + + it('skips the detection a host already ran', async () => { + const onReady = vi.fn(); + await runProgram( + 'metrics', + input({ + store: store({ detectionComplete: true }), + config: { run, onReady }, + credentials, + }), + ); + expect(onReady).not.toHaveBeenCalled(); + }); + + it('crashes on a CI ciPreRun that throws, keeping the error for the host', async () => { + const s = store({ ci: true }); + const crash = new Error('detector crashed'); + const result = await runProgram( + 'metrics', + { + store: s, + config: { + run, + ciPreRun: () => Promise.reject(crash), + }, + credentials, + }, + {}, + ); + expect(result).toMatchObject({ + outcome: RunOutcome.Crashed, + failure: { + code: ErrorCodes.InternalUnhandled, + message: 'detector crashed', + error: crash, + }, + }); + expect(s.session.runPhase).toBe(RunPhase.Error); + expect(runAgent).not.toHaveBeenCalled(); + }); + + it('fails an unmet prerequisite with its code and detail before any login', async () => { + const resolve = vi.fn(); + const onReady = vi.fn((ctx: ProgramReadyContext) => + ctx.setFrameworkContext('detectError', { kind: 'not-a-git-repo' }), + ); + const s = store(); + const result = await runProgram( + 'metrics', + { store: s, config: { run, onReady } }, + { credentials: { resolve } }, + ); + expect(result).toMatchObject({ + outcome: RunOutcome.Failed, + failure: { + code: detectErrorCode('not-a-git-repo'), + detail: { kind: 'not-a-git-repo' }, + message: expect.stringContaining('Prerequisites not met'), + }, + }); + expect(s.session.outroData?.errorCode).toBe( + detectErrorCode('not-a-git-repo'), + ); + expect(resolve).not.toHaveBeenCalled(); + expect(runAgent).not.toHaveBeenCalled(); + }); + }); + + describe('the run definition', () => { + it('builds the run from config.run on a copy of the session, keeps what it wrote, and labels the skill', async () => { + const s = store(); + const build = vi.fn((session: ProgramSession, runner: RunnerContext) => { + runner.setFrameworkContext('picked', 'ios'); + session.typescript = true; + return Promise.resolve({ ...run, skillId: 'skill-x' } as ProgramRun); + }); + + await runProgram('metrics', { + store: s, + config: { run: build }, + credentials, + }); + + expect(build).toHaveBeenCalledWith( + expect.objectContaining({ installDir: '/project' }), + expect.any(Object), + ); + expect(s.session.frameworkContext.picked).toBe('ios'); + expect(s.session.typescript).toBe(true); + expect(s.session.skillId).toBe('skill-x'); + expect(agentConfig().run).toMatchObject({ skillId: 'skill-x' }); + expect(agentInput().skillId).toBe('skill-x'); + }); + + it("reports config.run's log lines and spinner as the program's own progress", async () => { + const seen: ProgramProgress[] = []; + await runProgram( + 'metrics', + input({ + runId: 'run-1', + config: { + run: (_session: ProgramSession, runner: RunnerContext) => { + runner.log.warn('Installing the CLI'); + runner.spinner().start('Working'); + return Promise.resolve(run as ProgramRun); + }, + }, + credentials, + }), + { onProgress: (progress) => void seen.push(progress) }, + ); + expect(seen.slice(0, 2)).toEqual([ + { + runId: 'run-1', + event: { kind: 'log', level: 'warn', message: 'Installing the CLI' }, + }, + { + runId: 'run-1', + event: { kind: 'spinner', action: 'start', message: 'Working' }, + }, + ]); + }); + + it('turns a ProgramAbort from config.run into a failed outcome with its code, before any preflight or login', async () => { + const resolve = vi.fn(); + const s = store(); + const result = await runProgram( + 'metrics', + { + store: s, + config: { + run: () => + Promise.reject( + new ProgramAbort({ + code: ErrorCodes.DetectUnsupportedPlatform, + message: 'Not supported yet', + }), + ), + }, + }, + { credentials: { resolve } }, + ); + + expect(result).toMatchObject({ + outcome: RunOutcome.Failed, + failure: { + code: ErrorCodes.DetectUnsupportedPlatform, + message: 'Not supported yet', + }, + }); + expect(s.session.outroData?.errorCode).toBe( + ErrorCodes.DetectUnsupportedPlatform, + ); + expect(evaluateWizardReadiness).not.toHaveBeenCalled(); + expect(resolve).not.toHaveBeenCalled(); + expect(runAgent).not.toHaveBeenCalled(); + }); + + it('rethrows any other error config.run throws, after recording it in the store', async () => { + const error = new Error('detector crashed'); + const s = store(); + await expect( + runProgram( + 'metrics', + input({ + store: s, + config: { run: () => Promise.reject(error) }, + credentials, + }), + ), + ).rejects.toBe(error); + expect(runAgent).not.toHaveBeenCalled(); + expect(s.session.runPhase).toBe(RunPhase.Error); + expect(s.session.outroData).toMatchObject({ + kind: OutroKind.Error, + message: 'detector crashed', + errorCode: ErrorCodes.InternalUnhandled, + }); + }); + + it("binds the run's hooks and seed tasks to the store's session", async () => { + const s = store(); + const postRun = vi.fn(() => Promise.resolve()); + const buildOutroData = vi.fn(() => null); + const nextSteps = { heading: 'Next', items: ['next'] }; + const buildOutroNextSteps = vi.fn(() => nextSteps); + const seedTasks = vi.fn(() => []); + + await runProgram('metrics', { + store: s, + config: { + run: { ...run, postRun, buildOutroData, buildOutroNextSteps }, + seedTasks, + }, + credentials, + }); + + const { hooks, seedTasks: boundSeed } = agentConfig(); + const creds = credentials.posthog; + await hooks?.postRun?.(creds); + expect(postRun).toHaveBeenCalledWith(s.session, creds); + // A null outro becomes "none", so the agent builds its default. + expect(hooks?.buildOutroData?.(creds)).toBeUndefined(); + expect(buildOutroData).toHaveBeenCalledWith(s.session, creds); + expect(hooks?.buildOutroNextSteps?.(creds, ['install'])).toBe(nextSteps); + expect(buildOutroNextSteps).toHaveBeenCalledWith(s.session, creds, [ + 'install', + ]); + hooks?.recordTaskOutcomes?.([]); + expect(s.session.frameworkContext[TASK_OUTCOMES_KEY]).toEqual([]); + boundSeed?.(); + expect(seedTasks).toHaveBeenCalledWith(s.session); + }); + + it("derives the run input from the store's launch values", async () => { + await runProgram('metrics', { + store: store({ + ci: true, + debug: true, + yaraReport: true, + e2eAsk: true, + projectId: 7, + apiKey: 'phx_session', + region: 'eu', + integration: Integration.nextjs, + }), + config: { run }, + credentials, + }); + + expect(agentInput()).toMatchObject({ + installDir: '/project', + skillId: 'metrics', + integration: Integration.nextjs, + frameworkDocsUrl: + FRAMEWORK_REGISTRY[Integration.nextjs].metadata.docsUrl, + flags: { + ci: true, + signup: false, + debug: true, + yaraReport: true, + e2eAsk: true, + localMcp: false, + }, + host: { projectId: 7, apiKey: 'phx_session', region: 'eu' }, + }); + }); + }); + + describe('the preflight', () => { + it.each<[string, ReturnType | undefined]>([ + ['no workflow', undefined], + ['a workflow that continues', workflow()], + ])('a blocking outage with %s runs anyway', async (_case, host) => { + vi.mocked(evaluateWizardReadiness).mockResolvedValueOnce( + readiness(WizardReadiness.No), + ); + const s = store(); + const warnings: string[] = []; + const result = await runProgram( + 'metrics', + input({ store: s, credentials }), + { + workflow: host, + onProgress: ({ event }) => { + if (event.kind === 'log') warnings.push(event.message); + }, + }, + ); + expect(result.outcome).toBe(RunOutcome.Success); + expect(runAgent).toHaveBeenCalledOnce(); + expect(s.session.readinessResult?.decision).toBe(WizardReadiness.No); + // With nobody to ask, the outage is reported as the run goes on. + if (!host) { + expect(warnings).toContain('Service health issues detected.'); + expect(warnings.join('\n')).toContain('✖ Skills download: down'); + } + }); + + it('a blocking outage the host declines fails the run before any login', async () => { + const outage = readiness(WizardReadiness.No); + vi.mocked(evaluateWizardReadiness).mockResolvedValueOnce(outage); + const host = workflow((step) => step.kind !== 'service-outage'); + const resolve = vi.fn(); + + const result = await runProgram('metrics', input(), { + workflow: host, + credentials: { resolve }, + }); + + expect(host.steps).toEqual([ + expect.objectContaining({ kind: 'service-outage', readiness: outage }), + ]); + expect(result).toMatchObject({ + outcome: RunOutcome.Failed, + failure: { + code: ErrorCodes.EnvServiceOutage, + message: expect.stringContaining('Skills download (down)'), + }, + }); + expect(resolve).not.toHaveBeenCalled(); + expect(runAgent).not.toHaveBeenCalled(); + }); + + it('reports degraded services that do not block, and runs', async () => { + const degraded = readiness(WizardReadiness.YesWithWarnings); + vi.mocked(evaluateWizardReadiness).mockResolvedValueOnce(degraded); + const s = store(); + const seen: AgentProgress[] = []; + + const result = await runProgram( + 'metrics', + input({ store: s, credentials }), + { onProgress: ({ event }) => void seen.push(event) }, + ); + + expect(seen[0]).toEqual({ + kind: 'log', + level: 'warn', + message: 'Service health warnings detected.', + }); + expect(s.session.readinessResult).toEqual(degraded); + expect(result.outcome).toBe(RunOutcome.Success); + }); + + it.each<[string, () => Partial]>([ + [ + 'the host already ran it', + () => ({ + store: store({ readinessResult: readiness(WizardReadiness.No) }), + }), + ], + ['the program opts out', () => ({ config: { run, healthCheck: false } })], + ])('skips the health check when %s', async (_case, over) => { + const host = workflow(); + const result = await runProgram( + 'metrics', + input({ credentials, ...over() }), + { workflow: host }, + ); + expect(evaluateWizardReadiness).not.toHaveBeenCalled(); + expect(host.steps.map((step) => step.kind)).not.toContain( + 'service-outage', + ); + expect(result.outcome).toBe(RunOutcome.Success); + }); + + it('fails closed on a settings conflict it cannot neutralize when there is no host to ask', async () => { + vi.mocked(checkAllSettingsConflicts).mockReturnValueOnce([ + managedConflict, + ]); + + const result = await runProgram('metrics', input({ credentials })); + + expect(result).toMatchObject({ + outcome: RunOutcome.Failed, + failure: { + code: ErrorCodes.SettingsUnfixableConflict, + message: expect.stringContaining('ANTHROPIC_BASE_URL'), + }, + }); + expect(runAgent).not.toHaveBeenCalled(); + }); + + it('hands an unfixable conflict to the host with a fix it can apply, then runs and puts the settings back', async () => { + vi.mocked(checkAllSettingsConflicts).mockReturnValueOnce([ + managedConflict, + ]); + const host = workflow((step) => { + if (step.kind === 'settings-conflict') step.fix(); + return true; + }); + + const result = await runProgram('metrics', input({ credentials }), { + workflow: host, + }); + + expect(host.steps).toContainEqual( + expect.objectContaining({ + kind: 'settings-conflict', + conflicts: [managedConflict], + }), + ); + expect(backupAndFixClaudeSettings).toHaveBeenCalledWith('/project'); + expect(result.outcome).toBe(RunOutcome.Success); + expect(restoreClaudeSettings).toHaveBeenCalledWith('/project'); + }); + + it.each([ + [true, RunOutcome.Success], + [false, RunOutcome.Failed], + ])( + 'backs up a writable project settings conflict without asking (backed up: %s → %s)', + async (backedUp, outcome) => { + vi.mocked(checkAllSettingsConflicts).mockReturnValueOnce([ + { ...managedConflict, source: 'project', writable: true }, + ]); + vi.mocked(backupAndFixClaudeSettings).mockReturnValueOnce(backedUp); + + const result = await runProgram('metrics', input({ credentials })); + + expect(backupAndFixClaudeSettings).toHaveBeenCalledWith('/project'); + expect(result.outcome).toBe(outcome); + if (backedUp) { + // The run neutralized them for itself; they're back once it settles. + expect(restoreClaudeSettings).toHaveBeenCalledWith('/project'); + } else { + expect(result.failure?.code).toBe( + ErrorCodes.SettingsUnfixableConflict, + ); + } + }, + ); }); const closed = () => Promise.reject(new Error('caller closed')); @@ -185,16 +855,16 @@ describe('runProgram', () => { 'caller closed', ], [ - 'no org AI approval and no caller approval capability', + 'no org AI approval and no host to ask', () => ({ credentials: login(null) }), RunOutcome.Failed, 'AI processing approval is required before this program can run.', ], [ - 'a declined caller AI approval', + 'an AI approval the host declines', () => ({ credentials: login(null), - awaitAiApproval: () => Promise.resolve(false), + workflow: workflow((step) => step.kind !== 'ai-approval'), }), RunOutcome.Aborted, 'AI processing approval declined.', @@ -202,30 +872,27 @@ describe('runProgram', () => { ])( '%s is a decided result before the agent it guards', async (_case, options, outcome, message) => { - const result = await runProgram( - 'metrics', - { installDir: '/project', run }, - options(), - ); + const result = await runProgram('metrics', input(), options()); expect(result).toMatchObject({ outcome, failure: { message } }); - expect(result.settledRuns).toHaveLength(0); + expect(result.runResults).toHaveLength(0); expect(runAgent).not.toHaveBeenCalled(); }, ); - it('a program that needs no AI runs without an approval', async () => { - const awaitAiApproval = vi.fn(); - - const result = await runProgram( - 'metrics', - { installDir: '/project', run, program: { requiresAi: false } }, - { credentials: login(null), awaitAiApproval }, - ); + it.each([{ ci: true }, { signup: true }])( + 'skips the AI approval for a %o session', + async (fields) => { + const result = await runProgram( + 'metrics', + input({ store: store(fields) }), + { credentials: login(null) }, + ); - expect(result.outcome).toBe(RunOutcome.Success); - expect(awaitAiApproval).not.toHaveBeenCalled(); - }); + expect(result.outcome).toBe(RunOutcome.Success); + expect(runAgent).toHaveBeenCalledTimes(1); + }, + ); it.each<[RunOutcome, RunResult['failure']]>([ [ @@ -245,28 +912,29 @@ describe('runProgram', () => { }, ], ])( - 'a %s agent run settles with its failure and its run', + 'a %s agent run settles with its failure and its run, recorded in the store', async (outcome, failure) => { const result = { outcome, failure, snapshot } as RunResult; vi.mocked(runAgent).mockResolvedValueOnce(result); + const s = store(); - const settled = await runProgram('metrics', { - installDir: '/project', - runId: 'run-1', - run, - credentials, - }); + const settled = await runProgram( + 'metrics', + input({ store: s, runId: 'run-1', credentials }), + ); expect(settled).toMatchObject({ outcome, failure }); - expect(settled.settledRuns).toEqual([{ runId: 'run-1', result }]); + expect(settled.runResults).toEqual([result]); + expect(s.session.runPhase).toBe(RunPhase.Error); + expect(s.session.outroData).toMatchObject({ + kind: OutroKind.Error, + message: failure?.message, + errorCode: failure?.code, + }); }, ); - it.each([ - 'credential resolution', - 'AI approval', - 'a post-auth gate', - ] as const)( + it.each(['credential resolution', 'AI approval', 'the run step'] as const)( 'a caller abort during %s reaches the capability, returns Aborted and starts nothing else', async (gate) => { const controller = new AbortController(); @@ -280,13 +948,23 @@ describe('runProgram', () => { ); }), ); + const parkOn = + (kind: ProgramStep['kind']) => + (step: ProgramStep, context: { signal: AbortSignal }) => + step.kind === kind ? park(step, context) : Promise.resolve(true); const options: ProgramOptions = { 'credential resolution': { credentials: { resolve: park } }, - 'AI approval': { credentials: login(null), awaitAiApproval: park }, - 'a post-auth gate': { credentials: login(), awaitPostAuthGates: park }, + 'AI approval': { + credentials: login(null), + workflow: { confirmStep: parkOn('ai-approval') }, + }, + 'the run step': { + credentials: login(), + workflow: { confirmStep: parkOn('run') }, + }, }[gate]; - const pending = runProgram('gated', gated, { + const pending = runProgram('metrics', input(), { ...options, signal: controller.signal, }); @@ -310,14 +988,11 @@ describe('runProgram', () => { controller.abort(); return Promise.resolve(refreshedToken); }); + const s = store(); const result = await runProgram( 'metrics', - { - installDir: '/project', - run, - credentials: { ...credentials, posthog: aging() }, - }, + input({ store: s, credentials: { ...credentials, posthog: aging() } }), { signal: controller.signal }, ); @@ -326,24 +1001,28 @@ describe('runProgram', () => { accessToken: 'pha_refreshed', refreshToken: 'phr_rotated', }; - expect(result.data.credentials).toMatchObject(rotated); + expect(s.session.credentials).toMatchObject(rotated); expect(await oauthCredentials()).toMatchObject(rotated); expect(runAgent).not.toHaveBeenCalled(); }); - it('runs in order: agent started, credentials, approval, post-auth, flags, refresh, route, agent', async () => { + it('runs in order: readiness, agent started, settings, credentials, approval, flags, the run step, refresh, agent', async () => { const order: string[] = []; const answer = (name: string, value: T) => vi.fn(() => { order.push(name); return Promise.resolve(value); }); + vi.mocked(evaluateWizardReadiness).mockImplementationOnce( + answer('readiness', readiness(WizardReadiness.Yes)), + ); + vi.mocked(checkAllSettingsConflicts).mockImplementationOnce(() => { + order.push('settings'); + return []; + }); vi.mocked(analytics.wizardCapture).mockImplementationOnce((event) => { order.push(event); }); - vi.mocked(analytics.setTag).mockImplementationOnce(() => { - order.push('route'); - }); vi.mocked(refreshAccessToken).mockImplementationOnce( answer('refresh', refreshedToken), ); @@ -354,14 +1033,18 @@ describe('runProgram', () => { flags: { 'wizard-test-flag': 'on' }, payloads: { 'wizard-test-flag': { variant: 'b' } }, }; - const awaitPostAuthGates = answer('post-auth', undefined); + const host = workflow((step) => { + order.push(step.kind === 'run' ? `run step ${step.stepId}` : step.kind); + return true; + }); const result = await runProgram( - 'gated', - { - ...gated, - run: { ...run, integrationLabel: 'custom-label', skillId: 'skill-x' }, - }, + 'metrics', + input({ + config: { + run: { ...run, integrationLabel: 'custom-label', skillId: 'skill-x' }, + }, + }), { credentials: { resolve: answer('credentials', { @@ -370,55 +1053,71 @@ describe('runProgram', () => { apiUser: null, }), }, - awaitAiApproval: answer('approval', true), - awaitPostAuthGates, + workflow: host, featureFlags: answer('flags', flags), }, ); expect(result.outcome).toBe(RunOutcome.Success); expect(order).toEqual([ + 'readiness', 'agent started', + 'settings', 'credentials', - 'approval', - 'post-auth', + 'ai-approval', 'flags', + 'run step run', 'refresh', - 'route', 'runAgent', ]); expect(analytics.wizardCapture).toHaveBeenCalledWith('agent started', { integration: 'custom-label', - program_id: 'gated', + program_id: 'metrics', skill_id: 'skill-x', }); - expect(awaitPostAuthGates).toHaveBeenCalledWith({ - programId: 'gated', - gates: ['detect'], - signal: expect.objectContaining({ aborted: false }), - }); - expect(vi.mocked(runAgent).mock.calls[0][0]).toMatchObject({ + expect(agentConfig()).toMatchObject({ wizardFlags: flags.flags, wizardFlagPayloads: flags.payloads, }); }); - it('a caller mutation after the call does not reach the run', async () => { - const flags = { ci: false }; - const host: NonNullable = { region: 'us' }; + it("each agent run uses this run's login even when another is held", async () => { + let parked = false; + let release: (go: boolean) => void = () => undefined; + const host = { + confirmStep: (step: ProgramStep) => + step.kind === 'run' + ? new Promise((resolve) => { + parked = true; + release = resolve; + }) + : Promise.resolve(true), + }; - const pending = runProgram( - 'metrics', - { installDir: '/project', run, flags, host }, - { credentials: login() }, + const pending = runProgram('metrics', input({ credentials }), { + workflow: host, + }); + await vi.waitFor(() => expect(parked).toBe(true)); + configureOAuthSession( + { ...credentials.posthog, accessToken: 'phx_other', projectId: 99 }, + { rotate: (held) => Promise.resolve(held) }, ); - flags.ci = true; - host.region = 'eu'; + release(true); + await pending; + + expect(agentInput().credentials.projectId).toBe(42); + }); + + it('a caller mutation after the call does not reach the run', async () => { + const wizardFlags = { 'wizard-test-flag': 'on' }; + + const pending = runProgram('metrics', input({ wizardFlags }), { + credentials: login(), + }); + wizardFlags['wizard-test-flag'] = 'off'; await pending; - const [, runInput] = vi.mocked(runAgent).mock.calls[0]; - expect(runInput.flags.ci).toBe(false); - expect(runInput.host.region).toBe('us'); + expect(agentConfig().wizardFlags).toEqual({ 'wizard-test-flag': 'on' }); }); it('a provider is resolved once, then identified and stamped, and refreshed before the agent starts', async () => { @@ -430,18 +1129,15 @@ describe('runProgram', () => { const resolve = vi .fn() .mockResolvedValue({ posthog: aging(), project: null, apiUser }); + const s = store({ + scanConsent: ScanConsent.Granted, + discoveredFeatures: [DiscoveredFeature.LLM], + baseUrl: 'https://posthog.example', + }); - const result = await runProgram( - 'metrics', - { - installDir: '/project', - run, - host: { baseUrl: 'https://posthog.example' }, - mayReportScanResults: true, - discoveredFeatures: [DiscoveredFeature.LLM], - }, - { credentials: { resolve } }, - ); + await runProgram('metrics', input({ store: s }), { + credentials: { resolve }, + }); expect(resolve).toHaveBeenCalledOnce(); expect(analytics.identifyUser).toHaveBeenCalledExactlyOnceWith(apiUser); @@ -455,12 +1151,261 @@ describe('runProgram', () => { 'https://posthog.example', undefined, ); - expect(vi.mocked(runAgent).mock.calls[0][1].credentials.accessToken).toBe( - 'pha_refreshed', - ); - expect(result.data).toMatchObject({ + expect(agentInput().credentials.accessToken).toBe('pha_refreshed'); + expect(s.session).toMatchObject({ credentials: { refreshToken: 'phr_rotated' }, aiSdkStampReported: true, }); }); + + it("reuses the store's login and records a refreshed token on it, keeping its host", async () => { + vi.mocked(refreshAccessToken).mockResolvedValueOnce(refreshedToken); + const resolve = vi.fn(); + const s = store({ credentials: aging(), apiUser: credentials.apiUser }); + + await runProgram('metrics', input({ store: s }), { + credentials: { resolve }, + }); + + expect(resolve).not.toHaveBeenCalled(); + expect(agentInput().credentials.accessToken).toBe('pha_refreshed'); + expect(s.session.credentials?.accessToken).toBe('pha_refreshed'); + expect(s.session.credentials?.refreshToken).toBe('phr_rotated'); + expect(s.session.credentials?.host).toBeInstanceOf(HostResolution); + expect(s.session.aiSdkStampReported).toBe(true); + }); + + it('keeps a throwing observer as a diagnostic, not a failure, and ignores a late event', async () => { + let late: ((event: AgentProgress) => void) | undefined; + vi.mocked(runAgent).mockImplementationOnce((_config, _input, options) => { + late = options?.onProgress; + options?.onProgress?.({ kind: 'status', message: 'Installing' }); + return Promise.resolve({ outcome: RunOutcome.Success, snapshot }); + }); + const s = store(); + const result = await runProgram( + 'metrics', + input({ store: s, runId: 'run-1', credentials }), + { + onProgress: () => { + throw new Error('observer broke'); + }, + }, + ); + late?.({ kind: 'status', message: 'After' }); + + expect(result.outcome).toBe(RunOutcome.Success); + expect(result.diagnostics).toEqual([ + { runId: 'run-1', eventKind: 'status', message: 'observer broke' }, + ]); + // The store still has the event the observer threw on, and not the late one. + expect(s.statusMessages).toEqual(['Installing']); + }); + + it("pre-installs error-tracking's posthog-cli for the project the host picks before its run step", async () => { + const s = store({ detectionComplete: true }); + const host = workflow((step) => { + // The TUI's project pick lands before it confirms the run step. + if (step.kind === 'run') s.update({ integration: Integration.swift }); + return true; + }); + + await runProgram( + 'error-tracking', + { store: s, credentials }, + { workflow: host }, + ); + + expect(preinstallPostHogCliOnce).toHaveBeenCalledWith( + 'error tracking posthog-cli preinstall failed', + { integration: Integration.swift }, + expect.anything(), + ); + expect(runAgent).toHaveBeenCalledOnce(); + }); + + describe('composed runs', () => { + const composed = (prep = vi.fn()) => ({ + run, + runSteps: { + 'integrate-run': { + runProgramId: 'metrics', + targetDir: () => '/project/apps/web', + onRunPrep: prep, + }, + }, + }); + + it('with a workflow, runs each run step it confirms, then its own run, each reported', async () => { + const prep = vi.fn((session: ProgramSession) => { + session.frameworkContext.picked = 'web'; + return Promise.resolve(); + }); + const host = workflow(); + // The host ran self-driving's detection before the run. + const s = store({ detectionComplete: true }); + + const result = await runProgram( + 'self-driving', + { store: s, config: composed(prep), credentials }, + { workflow: host }, + ); + + expect(result.outcome).toBe(RunOutcome.Success); + expect(result.runResults).toHaveLength(2); + expect(agentConfig(0)).toMatchObject({ + programId: 'metrics', + composed: true, + }); + expect(agentInput(0).installDir).toBe('/project/apps/web'); + expect(agentConfig(1)).toMatchObject({ + programId: 'self-driving', + composed: false, + }); + expect(agentInput(1).installDir).toBe('/project'); + expect(host.steps.filter((step) => step.kind === 'run')).toEqual([ + expect.objectContaining({ + stepId: 'integrate-run', + programId: 'metrics', + }), + expect.objectContaining({ stepId: 'run', programId: 'self-driving' }), + ]); + expect(host.finishStep).toHaveBeenCalledTimes(2); + // A scoped run's prep writes stay in its own copy of the session. + expect(prep).toHaveBeenCalledOnce(); + expect(s.session.frameworkContext.picked).toBeUndefined(); + }); + + it('copies a framework a run step detects back to the store', async () => { + const prep = vi.fn((session: ProgramSession) => { + session.detectedFrameworkLabel = 'Django'; + return Promise.resolve(); + }); + const s = store({ detectionComplete: true }); + + await runProgram( + 'self-driving', + { store: s, config: composed(prep), credentials }, + { workflow: workflow() }, + ); + + expect(s.session.detectedFrameworkLabel).toBe('Django'); + }); + + it('skips a run step the host declines', async () => { + const host = workflow( + (step) => step.kind !== 'run' || step.stepId !== 'integrate-run', + ); + + await runProgram( + 'self-driving', + input({ + store: store({ detectionComplete: true }), + config: composed(), + credentials, + }), + { workflow: host }, + ); + + expect(runAgent).toHaveBeenCalledOnce(); + expect(agentConfig().programId).toBe('self-driving'); + }); + + it('with no workflow, runs only its own run', async () => { + await runProgram( + 'self-driving', + input({ + store: store({ detectionComplete: true }), + config: composed(), + credentials, + }), + ); + + expect(runAgent).toHaveBeenCalledOnce(); + expect(agentConfig().programId).toBe('self-driving'); + }); + }); + + describe('the audit ledger', () => { + let installDir: string; + const ledgerFile = '.posthog-audit-checks.json'; + const ledgerPath = () => path.join(installDir, ledgerFile); + const audit = (s = new SessionStore(buildSession({ installDir }))) => ({ + store: s, + config: { run, auditLedgerFile: ledgerFile }, + credentials, + }); + /** The agent seeds the ledger and, like a real run, never runs the `rm`. */ + const seedThen = + (finish: typeof runAgent): typeof runAgent => + (...args) => { + fs.writeFileSync(ledgerPath(), '[]'); + return finish(...args); + }; + const succeed: typeof runAgent = () => + Promise.resolve({ outcome: RunOutcome.Success, snapshot }); + + beforeEach(() => { + installDir = fs.mkdtempSync(path.join(os.tmpdir(), 'audit-ledger-')); + }); + afterEach(() => fs.rmSync(installDir, { recursive: true, force: true })); + + it('is removed from the project once the run settles', async () => { + vi.mocked(runAgent).mockImplementation(seedThen(succeed)); + await runProgram('audit', audit()); + expect(fs.existsSync(ledgerPath())).toBe(false); + }); + + it('is removed when the run throws', async () => { + const error = new Error('agent crashed'); + vi.mocked(runAgent).mockImplementation( + seedThen(() => Promise.reject(error)), + ); + await expect(runProgram('audit', audit())).rejects.toBe(error); + expect(fs.existsSync(ledgerPath())).toBe(false); + }); + + it('is removed by the abort cleanup', async () => { + const onAbort: Array<() => void> = []; + vi.mocked(registerCleanup).mockImplementation((fn) => { + onAbort.push(fn); + return () => undefined; + }); + let leftAfterAbort = true; + vi.mocked(runAgent).mockImplementation( + seedThen((...args) => { + onAbort.forEach((fn) => fn()); + leftAfterAbort = fs.existsSync(ledgerPath()); + return succeed(...args); + }), + ); + await runProgram('audit', audit()); + expect(leftAfterAbort).toBe(false); + }); + + it('keeps a finished run a success when the ledger cannot be removed', async () => { + vi.mocked(runAgent).mockImplementation((...args) => { + fs.mkdirSync(ledgerPath()); + return succeed(...args); + }); + const result = await runProgram('audit', audit()); + expect(result.outcome).toBe(RunOutcome.Success); + expect(logToFile).toHaveBeenCalledWith( + expect.stringContaining('[audit-ledger] could not remove'), + ); + }); + + it('mirrors a last write the watcher has not read yet into the store', async () => { + const checks = [ + { id: 'sdk', area: 'SDK', label: 'Install the SDK', status: 'pass' }, + ]; + vi.mocked(runAgent).mockImplementation((...args) => { + fs.writeFileSync(ledgerPath(), JSON.stringify(checks)); + return succeed(...args); + }); + const s = new SessionStore(buildSession({ installDir })); + await runProgram('audit', audit(s)); + expect(s.session.frameworkContext[AUDIT_CHECKS_KEY]).toEqual(checks); + }); + }); }); diff --git a/src/programs/agent-skill/__tests__/agent-skill.test.ts b/src/programs/agent-skill/__tests__/agent-skill.test.ts index c5dd79654..fdfbd15e8 100644 --- a/src/programs/agent-skill/__tests__/agent-skill.test.ts +++ b/src/programs/agent-skill/__tests__/agent-skill.test.ts @@ -1,11 +1,8 @@ -import { AGENT_SKILL_STEPS } from '@programs/agent-skill/index'; import { createSkillProgram, type SkillProgramOptions, } from '@programs/shared/skill-program'; import type { ProgramRun } from '@programs/program-run'; -import { buildSession, RunPhase } from '@programs/session/wizard-session'; -import { HostResolution } from '@shared/host-resolution'; const baseOpts: SkillProgramOptions = { skillId: 'error-tracking-setup', @@ -26,7 +23,6 @@ describe('createSkillProgram', () => { expect(config.command).toBe('errors'); expect(config.id).toBe('error-tracking'); - expect(config.steps).toBe(AGENT_SKILL_STEPS); // run must be a static object — skill programs don't need dynamic resolution const run = config.run as ProgramRun; @@ -48,46 +44,3 @@ describe('createSkillProgram', () => { expect((without.run as ProgramRun).customPrompt).toBeUndefined(); }); }); - -describe('AGENT_SKILL_STEPS', () => { - it('is intro → health-check → auth → run → outro → skills, all with screens and working predicates', () => { - expect(AGENT_SKILL_STEPS.map((s) => s.id)).toEqual([ - 'intro', - 'health-check', - 'auth', - 'run', - 'outro', - 'skills', - ]); - - const session = buildSession({}); - const [intro, , auth, run, outro] = AGENT_SKILL_STEPS; - - // Intro gate starts closed - expect(intro.gate!(session)).toBe(false); - - // All incomplete initially - expect(auth.isComplete!(session)).toBe(false); - expect(run.isComplete!(session)).toBe(false); - expect(outro.isComplete!(session)).toBe(false); - - // Intro gate opens after setup confirmed - session.setupConfirmed = true; - expect(intro.gate!(session)).toBe(true); - - // Completing each - session.credentials = { - accessToken: 't', - projectApiKey: 'k', - host: HostResolution.fromApiHost('h'), - projectId: 1, - }; - expect(auth.isComplete!(session)).toBe(true); - - session.runPhase = RunPhase.Completed; - expect(run.isComplete!(session)).toBe(true); - - session.outroDismissed = true; - expect(outro.isComplete!(session)).toBe(true); - }); -}); diff --git a/src/programs/audit/__tests__/seed.test.ts b/src/programs/audit/__tests__/seed.test.ts index ee2a303cb..419f83516 100644 --- a/src/programs/audit/__tests__/seed.test.ts +++ b/src/programs/audit/__tests__/seed.test.ts @@ -4,7 +4,6 @@ import * as path from 'path'; import { AUDIT_SEED_CHECKS, seedAuditLedger } from '@programs/audit/seed'; import { AUDIT_CHECKS_FILE, type AuditCheck } from '@programs/audit/types'; -import { COL_AREA_WIDTH } from '@tui/programs/audit/screens/AuditChecksViewer/layout'; const ids = (checks: AuditCheck[]) => checks.map((c) => c.id); @@ -34,17 +33,6 @@ describe('AUDIT_SEED_CHECKS', () => { expect.arrayContaining(['init-correct', 'init-not-duplicated']), ); }); - - it('fits every area in the checks viewer column', () => { - // Area is the one hard constraint: computeLayout pins it to a fixed - // COL_AREA_WIDTH that never flexes, so a longer area name is truncated at - // every terminal size. Labels get the flexed column and are allowed to run - // past COL_LABEL_MIN — several seeded ones already do, and only clip on a - // narrow terminal. - for (const check of AUDIT_SEED_CHECKS) { - expect(check.area.length).toBeLessThanOrEqual(COL_AREA_WIDTH); - } - }); }); describe('seedAuditLedger', () => { diff --git a/src/programs/detection/__tests__/agentic-progress.test.ts b/src/programs/detection/__tests__/agentic-progress.test.ts index 49f9f891f..10e582152 100644 --- a/src/programs/detection/__tests__/agentic-progress.test.ts +++ b/src/programs/detection/__tests__/agentic-progress.test.ts @@ -1,49 +1,22 @@ -import { Harness } from '@shared/constants'; +/** Detection progress and failed-scan results over a stubbed runAgent. */ import { detectProjectsWithAgent } from '../agentic'; -import { - AgentErrorType, - initializeAgent, - runAgent as executeAgent, -} from '@agent/agent-interface'; -import { analytics } from '@utils/analytics'; +import { runAgent, RunOutcome } from '@agent'; +import type { AgentProgress } from '@agent/types'; import { buildSession } from '@programs/session/wizard-session'; +import { analytics } from '@utils/analytics'; import { HostResolution } from '@shared/host-resolution'; import { ErrorCodes } from '@shared/errors'; -import { getUI } from '@ui'; +import { snapshot } from './helpers/run-snapshot.no-jest'; vi.mock('@utils/debug'); -// Detection runs the real runAgent pipeline: no analytics or gateway mint may leave the process. vi.mock('@utils/analytics'); -vi.mock('@agent/gateway-session', async (original) => ({ - ...(await original()), - gatewayAuth: vi.fn(() => - Promise.resolve({ - gatewayUrl: 'https://gateway.test', - token: 'phe_test', - refreshAtMs: Infinity, - }), - ), -})); -vi.mock('@ui', () => ({ getUI: () => ui })); -const ui = vi.hoisted(() => ({ - addTokenUsage: vi.fn(), - setStage: vi.fn(), - pushStatus: vi.fn(), - showAuthError: vi.fn(), - startRun: vi.fn(), - log: { error: vi.fn(), warn: vi.fn(), info: vi.fn() }, -})); -vi.mock('@agent/agent-interface', async (original) => ({ - ...(await original()), - initializeAgent: vi.fn(), +vi.mock(import('@agent'), async (importOriginal) => ({ + ...(await importOriginal()), runAgent: vi.fn(), })); function detectionSession() { - const session = buildSession({ - installDir: '/tmp/detection-test', - harness: Harness.anthropic, - }); + const session = buildSession({ installDir: '/tmp/detection-test' }); session.credentials = { accessToken: 'test', projectApiKey: 'phc_test', @@ -53,14 +26,29 @@ function detectionSession() { return session; } +const scan = () => + detectProjectsWithAgent(detectionSession(), { + programId: 'posthog-integration', + targets: [{ id: 'node', name: 'Node.js' }], + }); + beforeEach(() => { - vi.clearAllMocks(); + vi.mocked(runAgent).mockReset(); vi.mocked(analytics.getAllFlagsForWizard).mockResolvedValue({}); vi.mocked(analytics.getWizardFlagPayloads).mockReturnValue({}); - vi.mocked(initializeAgent).mockReset(); - vi.mocked(executeAgent).mockReset(); }); +/** The scan's run emits `events`, then succeeds with `transcriptTail`. */ +function emitting(events: AgentProgress[], transcriptTail: string) { + vi.mocked(runAgent).mockImplementation((_config, _input, options) => { + for (const event of events) options?.onProgress?.(event); + return Promise.resolve({ + outcome: RunOutcome.Success, + snapshot: snapshot(transcriptTail), + }); + }); +} + it('keeps initialization and execution progress visible during detection', async () => { const delta = { inputTokens: 5, @@ -70,160 +58,99 @@ it('keeps initialization and execution progress visible during detection', async cacheCreation5m: 0, cacheCreation1h: 0, }; - vi.mocked(initializeAgent).mockImplementation((config) => { - config.emit?.({ - kind: 'log', - level: 'error', - message: 'Initialization diagnostic', - }); - return Promise.resolve({ emit: config.emit } as Awaited< - ReturnType - >); - }); - vi.mocked(executeAgent).mockImplementation( - (config, _prompt, _options, _spinner, _messages, middleware) => { - config.emit?.({ kind: 'usage', delta }); - config.emit?.({ kind: 'stage', stage: 'Scanning' }); - config.emit?.({ kind: 'status', message: 'Found a project' }); - config.emit?.({ - kind: 'log', - level: 'error', - message: 'Execution diagnostic', - }); - middleware?.onMessage({ - type: 'result', - result: - '{"projects":[{"path":".","targetId":"node","framework":"Node.js"}]}', - }); - return Promise.resolve({ kind: 'success' }); - }, + const visible: AgentProgress[] = [ + { kind: 'log', level: 'error', message: 'Initialization diagnostic' }, + { kind: 'usage', delta }, + { kind: 'stage', stage: 'Scanning' }, + { kind: 'status', message: 'Found a project' }, + { kind: 'log', level: 'error', message: 'Execution diagnostic' }, + ]; + emitting( + visible, + '{"projects":[{"path":".","targetId":"node","framework":"Node.js"}]}', ); + const seen: AgentProgress[] = []; + const report = await detectProjectsWithAgent(detectionSession(), { programId: 'posthog-integration', targets: [{ id: 'node', name: 'Node.js' }], + onProgress: (event) => seen.push(event), }); expect(report.projects[0].targetId).toBe('node'); - expect(getUI().addTokenUsage).toHaveBeenCalledWith(delta); - expect(ui.setStage).toHaveBeenCalledWith('Scanning'); - expect(ui.pushStatus).toHaveBeenCalledWith('Found a project'); - expect(ui.log.error.mock.calls).toEqual([ - ['Initialization diagnostic'], - ['Execution diagnostic'], - ]); + expect(seen).toEqual(visible); }); it('sends each agent step to onEvent and the UI only the progress it saw before runAgent', async () => { - vi.mocked(initializeAgent).mockImplementation((config) => - Promise.resolve({ emit: config.emit } as Awaited< - ReturnType - >), - ); - vi.mocked(executeAgent).mockImplementation( - (config, _prompt, _options, _spinner, _messages, middleware) => { - config.emit?.({ kind: 'status', message: 'Scanning' }); - config.emit?.({ kind: 'log', level: 'info', message: 'Info line' }); - config.emit?.({ kind: 'log', level: 'warn', message: 'Warn line' }); - middleware?.onMessage({ - type: 'assistant', - message: { - content: [ - { type: 'text', text: 'Reading the root manifest.' }, - { - type: 'tool_use', - name: 'Read', - input: { file_path: 'package.json' }, - }, - ], - }, - }); - middleware?.onMessage({ - type: 'result', - result: '{"path":".","targetId":"node","framework":"Node.js"}', - }); - return Promise.resolve({ kind: 'success' }); - }, + emitting( + [ + { kind: 'lifecycle', phase: 'started' }, + { kind: 'log', level: 'step', message: 'Initializing Claude agent...' }, + { kind: 'status', message: 'Scanning' }, + { kind: 'log', level: 'info', message: 'Info line' }, + { kind: 'log', level: 'warn', message: 'Warn line' }, + { kind: 'activity', line: 'Reading the root manifest.' }, + { kind: 'activity', line: 'Read package.json' }, + { kind: 'lifecycle', phase: 'completed', message: 'Detection complete' }, + ], + '{"path":".","targetId":"node","framework":"Node.js"}', ); const lines: string[] = []; + const seen: AgentProgress[] = []; const report = await detectProjectsWithAgent(detectionSession(), { programId: 'posthog-integration', targets: [{ id: 'node', name: 'Node.js' }], onEvent: (line) => lines.push(line), + onProgress: (event) => seen.push(event), }); expect(report.projects[0].targetId).toBe('node'); expect(lines).toEqual(['Reading the root manifest.', 'Read package.json']); - expect(ui.pushStatus).toHaveBeenCalledWith('Scanning'); - expect(ui.log.warn).toHaveBeenCalledWith('Warn line'); // The scan's run lifecycle and setup logs never reach the program's UI. - expect(ui.log.info).not.toHaveBeenCalled(); - expect(ui.startRun).not.toHaveBeenCalled(); + expect(seen).toEqual([ + { kind: 'status', message: 'Scanning' }, + { kind: 'log', level: 'warn', message: 'Warn line' }, + { kind: 'activity', line: 'Reading the root manifest.' }, + { kind: 'activity', line: 'Read package.json' }, + ]); }); it('stops optional detection on a data-only 401 before parsing partial JSON', async () => { - vi.mocked(initializeAgent).mockResolvedValue( - {} as Awaited>, - ); - vi.mocked(executeAgent).mockImplementation( - (_config, _prompt, _options, _spinner, _messages, middleware) => { - middleware?.onMessage({ - type: 'result', - result: '{"projects":[{"path":".","targetId":"node"}]}', - }); - return Promise.resolve({ - kind: 'decided_failure', - failure: { - code: ErrorCodes.AuthInvalidOrExpired, - message: 'Authentication failed (401)', - }, - }); + vi.mocked(runAgent).mockResolvedValue({ + outcome: RunOutcome.Failed, + failure: { + code: ErrorCodes.AuthInvalidOrExpired, + message: 'Authentication failed (401)', }, - ); - await expect( - detectProjectsWithAgent(detectionSession(), { - programId: 'posthog-integration', - targets: [{ id: 'node', name: 'Node.js' }], - }), - ).rejects.toThrow('Authentication failed (401)'); - expect(ui.showAuthError).not.toHaveBeenCalled(); + snapshot: snapshot('{"projects":[{"path":".","targetId":"node"}]}'), + }); + await expect(scan()).rejects.toThrow('Authentication failed (401)'); }); it('preserves the original error from a decided failure', async () => { const original = new Error('Gateway bearer rejected'); - vi.mocked(initializeAgent).mockResolvedValue( - {} as Awaited>, - ); - vi.mocked(executeAgent).mockResolvedValue({ - kind: 'decided_failure', + vi.mocked(runAgent).mockResolvedValue({ + outcome: RunOutcome.Failed, failure: { code: ErrorCodes.GatewayMintRefused, message: original.message, error: original, }, + snapshot: snapshot(), }); - await expect( - detectProjectsWithAgent(detectionSession(), { - programId: 'posthog-integration', - targets: [{ id: 'node', name: 'Node.js' }], - }), - ).rejects.toBe(original); + await expect(scan()).rejects.toBe(original); }); it('rejects classified agent failures', async () => { - vi.mocked(initializeAgent).mockResolvedValue( - {} as Awaited>, - ); - vi.mocked(executeAgent).mockResolvedValue({ - kind: 'failure', - classification: AgentErrorType.API_ERROR, - message: 'Agent API unavailable', + vi.mocked(runAgent).mockResolvedValue({ + outcome: RunOutcome.Failed, + failure: { + code: ErrorCodes.AgentApiError, + message: 'API Error\n\nAgent API unavailable', + }, + snapshot: snapshot(), }); - await expect( - detectProjectsWithAgent(detectionSession(), { - programId: 'posthog-integration', - targets: [{ id: 'node', name: 'Node.js' }], - }), - ).rejects.toThrow('Agent API unavailable'); + await expect(scan()).rejects.toThrow('Agent API unavailable'); }); diff --git a/src/programs/detection/__tests__/agentic-retry.test.ts b/src/programs/detection/__tests__/agentic-retry.test.ts index f0fb747f4..5c4214c1b 100644 --- a/src/programs/detection/__tests__/agentic-retry.test.ts +++ b/src/programs/detection/__tests__/agentic-retry.test.ts @@ -1,52 +1,22 @@ +/** Detection's scan and its one retry, over a stubbed runAgent. */ import { AgenticDetectionTimeoutError, detectProjectsWithAgent, } from '@programs/detection/agentic'; -import * as agentEntry from '@agent'; -import { piBackend } from '@agent/runner/harness/pi'; -import { triageModelFor } from '@agent/runner/switchboard/models'; -import { - AgentErrorType, - initializeAgent, - runAgent, -} from '@agent/agent-interface'; -import { analytics } from '@utils/analytics'; +import { DEFAULT_BINDING, runAgent, RunOutcome } from '@agent'; import { buildSession } from '@programs/session/wizard-session'; +import { analytics } from '@utils/analytics'; +import { ErrorCodes, type ErrorCode } from '@shared/errors'; import { HostResolution } from '@shared/host-resolution'; -import { flushScanReport } from '@agent/yara-hooks'; -import { Harness, HAIKU_MODEL, Sequence } from '@shared/constants'; +import { Harness } from '@shared/constants'; +import { snapshot } from './helpers/run-snapshot.no-jest'; vi.mock('@utils/analytics'); -vi.mock('@agent/runner/harness/pi', () => ({ - piBackend: { name: 'pi', run: vi.fn() }, -})); -vi.mock('@agent/agent-interface', async (importOriginal) => ({ - ...(await importOriginal()), - initializeAgent: vi.fn(), +vi.mock('@agent', async (importOriginal) => ({ + ...(await importOriginal()), runAgent: vi.fn(), })); -// The entry's runAgent is the real one, spied so each attempt is visible. -vi.mock('@agent', async (importOriginal) => { - const actual = await importOriginal(); - return { ...actual, runAgent: vi.fn(actual.runAgent) }; -}); -vi.mock('@agent/yara-hooks', async (importOriginal) => ({ - ...(await importOriginal()), - flushScanReport: vi.fn(), -})); -// The runner mints before each attempt; no mint may leave the process. -vi.mock('@agent/gateway-session', async (importOriginal) => ({ - ...(await importOriginal()), - gatewayAuth: vi.fn(() => - Promise.resolve({ - gatewayUrl: 'https://gateway.test', - token: 'phe_test', - refreshAtMs: Infinity, - }), - ), -})); -const init = vi.mocked(initializeAgent); const execute = vi.mocked(runAgent); const options = { programId: 'posthog-integration', @@ -56,10 +26,7 @@ const verdict = '{"path":".","framework":"Next.js","targetId":"nextjs","hasPostHog":false}'; function session() { - const value = buildSession({ - installDir: '/repo', - harness: Harness.anthropic, - }); + const value = buildSession({ installDir: '/repo', harness: Harness.pi }); value.credentials = { accessToken: 'token', projectApiKey: 'key', @@ -69,18 +36,43 @@ function session() { return value; } +/** The run succeeds with `text` as its collected transcript. */ function emitResult(text: string) { - return execute.mockImplementationOnce((...args) => { - args[5]?.onMessage({ type: 'result', result: text }); - return Promise.resolve({ kind: 'success' }); - }); + return execute.mockImplementationOnce(() => + Promise.resolve({ + outcome: RunOutcome.Success, + snapshot: snapshot(text), + }), + ); } -/** The attempt's deadline fires while its SDK run is active. */ +/** The run ends on its own deadline. */ function timeOut() { return execute.mockResolvedValueOnce({ - kind: 'failure', - classification: AgentErrorType.AGENTIC_DETECTION_TIMEOUT, + outcome: RunOutcome.Failed, + failure: { + code: ErrorCodes.AgenticDetectionTimeout, + message: 'Project scan timed out', + }, + snapshot: snapshot(''), + }); +} + +/** The run fails with `code`, and `error` when it has one. */ +function fail(code: ErrorCode, error?: Error) { + return execute.mockResolvedValueOnce({ + outcome: RunOutcome.Failed, + failure: { code, message: error?.message ?? code, ...(error && { error }) }, + snapshot: snapshot(''), + }); +} + +/** The run succeeds with a typed report. */ +function succeedWith(structuredOutput: unknown) { + return execute.mockResolvedValueOnce({ + outcome: RunOutcome.Success, + structuredOutput, + snapshot: snapshot(''), }); } @@ -89,72 +81,55 @@ describe('agentic detection retry', () => { vi.resetAllMocks(); vi.mocked(analytics.getAllFlagsForWizard).mockResolvedValue({}); vi.mocked(analytics.getWizardFlagPayloads).mockReturnValue({}); - init.mockImplementation(() => - Promise.resolve({ id: init.mock.calls.length } as unknown as Awaited< - ReturnType - >), - ); }); afterEach(() => vi.restoreAllMocks()); - it('runs both attempts through runAgent on linear Haiku with a schema, read-only tools, its own prompt and a deferred scan report', async () => { + it('runs both attempts through runAgent, read-only, with a schema and deadline, its own prompt, no remark and a deferred scan report', async () => { + vi.mocked(analytics.getAllFlagsForWizard).mockResolvedValue({ + 'a-flag': 'variant', + }); + vi.mocked(analytics.getWizardFlagPayloads).mockReturnValue({ + 'a-flag': { route: 'pi' }, + }); timeOut(); emitResult(verdict); await detectProjectsWithAgent(session(), options); - const calls = vi.mocked(agentEntry.runAgent).mock.calls; + const calls = execute.mock.calls; expect(calls).toHaveLength(2); + // The launch's harness applies to the first attempt and the retry runs the + // SDK; the scan keeps out of the run's routing analytics. + expect(calls.map(([config]) => config.routing)).toEqual( + (['first', 'retry'] as const).map((scan) => ({ + binding: DEFAULT_BINDING, + overrides: { harness: Harness.pi }, + record: false, + scan, + })), + ); + expect(calls.map(([config]) => config.run.structured?.timeoutMs)).toEqual([ + 60_000, 90_000, + ]); for (const [config] of calls) { - expect(config.binding).toEqual({ - sequence: Sequence.linear, - harness: Harness.anthropic, - model: HAIKU_MODEL, + expect(config.wizardFlags).toEqual({ 'a-flag': 'variant' }); + expect(config.wizardFlagPayloads).toEqual({ 'a-flag': { route: 'pi' } }); + expect(config.run.structured?.schema).toMatchObject({ + required: ['repoType', 'projects'], }); expect(config.run.readOnly).toBe(true); + // The program run's report counts the scan's scans. expect(config.scanReport).toBe('defer'); expect(config.run).toMatchObject({ collectTranscript: true, requestRemark: false, }); + // The run definition's prompt replaces the assembled program prompt. + expect(config.run.prompt?.({} as never)).toContain( + 'You are scanning a code repository', + ); } - // The run definition's prompt replaces the assembled program prompt. - expect(execute.mock.calls[0][1]).toContain( - 'You are scanning a code repository', - ); - expect(execute.mock.calls[0][4]).toMatchObject({ requestRemark: false }); - // The program run's report counts the scan's scans. - expect(flushScanReport).not.toHaveBeenCalled(); - }); - - it('falls back from the bound Pi scan to SDK triage after a timeout', async () => { - const value = session(); - value.harness = Harness.pi; - vi.mocked(piBackend.run).mockResolvedValueOnce({ - kind: 'failure', - classification: AgentErrorType.AGENTIC_DETECTION_TIMEOUT, - }); - emitResult(verdict); - expect( - (await detectProjectsWithAgent(value, options)).projects[0].targetId, - ).toBe('nextjs'); - expect( - vi - .mocked(agentEntry.runAgent) - .mock.calls.map(([config]) => config.binding), - ).toEqual([ - { - harness: Harness.pi, - sequence: Sequence.linear, - model: triageModelFor(Harness.pi), - }, - { - harness: Harness.anthropic, - sequence: Sequence.linear, - model: triageModelFor(Harness.anthropic), - }, - ]); }); it('uses the typed result when the model emits a pretty-printed final report', async () => { @@ -171,37 +146,26 @@ describe('agentic detection retry', () => { }, ], }; - execute.mockImplementationOnce((...args) => { - args[5]?.onMessage({ - type: 'result', - result: JSON.stringify(report, null, 2), - }); - return Promise.resolve({ kind: 'success', structuredOutput: report }); + execute.mockResolvedValueOnce({ + outcome: RunOutcome.Success, + structuredOutput: report, + snapshot: snapshot(JSON.stringify(report, null, 2)), }); + expect( (await detectProjectsWithAgent(session(), options)).projects[0].targetId, ).toBe('nextjs'); expect(execute).toHaveBeenCalledTimes(1); - expect(init.mock.calls[0][0].outputFormat?.schema).toMatchObject({ - required: ['repoType', 'projects'], - }); }); it('retries invalid structured output once but propagates ordinary API failures', async () => { - execute.mockResolvedValueOnce({ - kind: 'failure', - classification: AgentErrorType.INVALID_STRUCTURED_OUTPUT, - }); + fail(ErrorCodes.AgentInvalidStructuredOutput); emitResult(verdict); expect( (await detectProjectsWithAgent(session(), options)).projects[0].targetId, ).toBe('nextjs'); const failure = new Error('API unavailable'); - execute.mockResolvedValueOnce({ - kind: 'failure', - classification: AgentErrorType.API_ERROR, - error: failure, - }); + fail(ErrorCodes.AgentApiError, failure); await expect(detectProjectsWithAgent(session(), options)).rejects.toThrow( failure, ); @@ -216,19 +180,13 @@ describe('agentic detection retry', () => { targetId: 'nextjs', evidence: 'next in dependencies', }; - execute.mockResolvedValueOnce({ - kind: 'success', - structuredOutput: { - repoType: 'single', - projects: [{ ...project, hasPostHog: 'yes' }], - }, + succeedWith({ + repoType: 'single', + projects: [{ ...project, hasPostHog: 'yes' }], }); - execute.mockResolvedValueOnce({ - kind: 'success', - structuredOutput: { - repoType: 'single', - projects: [{ ...project, hasPostHog: true }], - }, + succeedWith({ + repoType: 'single', + projects: [{ ...project, hasPostHog: true }], }); const report = await detectProjectsWithAgent(session(), options); @@ -247,10 +205,7 @@ describe('agentic detection retry', () => { evidence: 'next in dependencies', recommended: true, }; - execute.mockResolvedValueOnce({ - kind: 'success', - structuredOutput: { repoType: 'single', projects: [project] }, - }); + succeedWith({ repoType: 'single', projects: [project] }); const report = await detectProjectsWithAgent(session(), { ...options, @@ -258,7 +213,7 @@ describe('agentic detection retry', () => { }); expect(report.projects[0].recommended).toBe(true); - expect(init.mock.calls[0][0].outputFormat?.schema).toMatchObject({ + expect(execute.mock.calls[0][0].run.structured?.schema).toMatchObject({ properties: { projects: { items: { required: expect.arrayContaining(['recommended']) }, @@ -268,10 +223,7 @@ describe('agentic detection retry', () => { }); it('accepts an empty report without retrying, typed or recovered', async () => { - execute.mockResolvedValueOnce({ - kind: 'success', - structuredOutput: { repoType: 'single', projects: [] }, - }); + succeedWith({ repoType: 'single', projects: [] }); emitResult('{"repoType":"single","projects":[]}'); for (let run = 0; run < 2; run++) { await expect( @@ -295,10 +247,10 @@ describe('agentic detection retry', () => { hasPostHog: false, }, ]); - expect(init).toHaveBeenCalledTimes(2); expect(execute).toHaveBeenCalledTimes(2); - expect(execute.mock.calls.map((call) => call[4]?.timeoutMs)).toEqual([ - 60_000, 90_000, + expect(execute.mock.calls.map(([config]) => config.routing.scan)).toEqual([ + 'first', + 'retry', ]); }); @@ -308,11 +260,10 @@ describe('agentic detection retry', () => { const report = await detectProjectsWithAgent(session(), options); expect(report.projects).toHaveLength(1); - expect(init).toHaveBeenCalledTimes(1); expect(execute).toHaveBeenCalledTimes(1); }); - it('retries a timed-out first run with a fresh Haiku session', async () => { + it('retries a timed-out first run on the SDK', async () => { const events: string[] = []; timeOut(); emitResult(verdict); @@ -324,9 +275,9 @@ describe('agentic detection retry', () => { expect(report.projects).toHaveLength(1); expect(events).toContain('Project scan timed out; retrying...'); - expect(execute.mock.calls[0][0]).not.toBe(execute.mock.calls[1][0]); - expect(execute.mock.calls.map((call) => call[4]?.timeoutMs)).toEqual([ - 60_000, 90_000, + expect(execute.mock.calls.map(([config]) => config.routing.scan)).toEqual([ + 'first', + 'retry', ]); }); @@ -346,14 +297,8 @@ describe('agentic detection retry', () => { it('accepts a streamed verdict after a no-JSON result', async () => { const events: string[] = []; emitResult('Found a project, but no JSON report.'); - execute.mockImplementationOnce((...args) => { - args[5]?.onMessage({ - type: 'assistant', - message: { content: [{ type: 'text', text: verdict }] }, - }); - args[5]?.onMessage({ type: 'result', result: 'Done.' }); - return Promise.resolve({ kind: 'success' }); - }); + // The transcript holds the streamed text before the final message. + emitResult(`${verdict}\nDone.`); const report = await detectProjectsWithAgent(session(), { ...options, @@ -369,8 +314,7 @@ describe('agentic detection retry', () => { }, ]); expect(events).toContain('Retrying project scan...'); - expect(init).toHaveBeenCalledTimes(2); - expect(execute.mock.calls[0][0]).not.toBe(execute.mock.calls[1][0]); + expect(execute).toHaveBeenCalledTimes(2); expect(execute.mock.calls[0][1]).toBe(execute.mock.calls[1][1]); }); diff --git a/src/programs/detection/__tests__/context.test.ts b/src/programs/detection/__tests__/context.test.ts index acb64478d..bc0a96a74 100644 --- a/src/programs/detection/__tests__/context.test.ts +++ b/src/programs/detection/__tests__/context.test.ts @@ -2,7 +2,7 @@ import { gatherFrameworkContext, checkFrameworkVersion, } from '@programs/detection/context'; -import type { FrameworkConfig } from '@programs/types'; +import type { FrameworkConfig } from '@programs/framework-config'; import type { WizardRunOptions } from '@utils/types'; const baseOptions: WizardRunOptions = { diff --git a/src/programs/detection/__tests__/detected-framework.test.ts b/src/programs/detection/__tests__/detected-framework.test.ts new file mode 100644 index 000000000..e4868c60c --- /dev/null +++ b/src/programs/detection/__tests__/detected-framework.test.ts @@ -0,0 +1,50 @@ +import { analytics } from '@utils/analytics'; +import type { FrameworkConfig } from '../../framework-config'; +import type { ProgramSession } from '../../program-session'; +import { noteDetectedFramework } from '../detected-framework'; + +vi.mock('@utils/analytics', () => ({ analytics: { setTag: vi.fn() } })); + +const django = { + metadata: { + getDetectedFrameworkLabel: (context: Record) => + context.wagtail ? 'Django with Wagtail CMS' : undefined, + }, +} as unknown as FrameworkConfig; + +describe('noteDetectedFramework', () => { + beforeEach(() => vi.clearAllMocks()); + + it('prints a variant label for a CI run and leaves its session alone', () => { + const session = { ci: true, detectedFrameworkLabel: null }; + const log = { info: vi.fn() }; + noteDetectedFramework( + session as unknown as ProgramSession, + django, + { wagtail: true }, + log, + ); + expect(log.info).toHaveBeenCalledExactlyOnceWith( + 'Framework: Django with Wagtail CMS', + ); + expect(session.detectedFrameworkLabel).toBeNull(); + expect(analytics.setTag).not.toHaveBeenCalled(); + }); + + it("keeps a variant label on a TUI run's session and analytics tag", () => { + const session = { ci: false, detectedFrameworkLabel: 'Django' }; + const log = { info: vi.fn() }; + noteDetectedFramework( + session as unknown as ProgramSession, + django, + { wagtail: true }, + log, + ); + expect(session.detectedFrameworkLabel).toBe('Django with Wagtail CMS'); + expect(analytics.setTag).toHaveBeenCalledWith( + 'detected_framework', + 'Django with Wagtail CMS', + ); + expect(log.info).not.toHaveBeenCalled(); + }); +}); diff --git a/src/programs/detection/__tests__/features.test.ts b/src/programs/detection/__tests__/features.test.ts index 394161e1c..5eddbd531 100644 --- a/src/programs/detection/__tests__/features.test.ts +++ b/src/programs/detection/__tests__/features.test.ts @@ -2,7 +2,7 @@ import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; import { discoverFeatures } from '@programs/detection/features'; -import { DiscoveredFeature } from '@programs/session/wizard-session'; +import { DiscoveredFeature } from '@shared/discovered-feature'; function makeTmpDir(): string { return fs.mkdtempSync(path.join(os.tmpdir(), 'features-detect-')); diff --git a/src/programs/detection/__tests__/helpers/run-snapshot.no-jest.ts b/src/programs/detection/__tests__/helpers/run-snapshot.no-jest.ts new file mode 100644 index 000000000..861ccf37a --- /dev/null +++ b/src/programs/detection/__tests__/helpers/run-snapshot.no-jest.ts @@ -0,0 +1,14 @@ +import type { RunResult } from '@agent/types'; + +/** The snapshot of a detection scan's run: nothing done, with `transcriptTail` as its transcript. */ +export const snapshot = (transcriptTail?: string): RunResult['snapshot'] => ({ + tasks: [], + statusMessages: [], + usage: { + inputTokens: 0, + outputTokens: 0, + cacheReadTokens: 0, + cacheCreationTokens: 0, + }, + transcriptTail, +}); diff --git a/src/programs/detection/__tests__/integration.test.ts b/src/programs/detection/__tests__/integration.test.ts index e55803de6..230c9e489 100644 --- a/src/programs/detection/__tests__/integration.test.ts +++ b/src/programs/detection/__tests__/integration.test.ts @@ -28,23 +28,22 @@ vi.mock('@programs/warehouse-sources/detect', async (importOriginal) => { }); import { analytics } from '@utils/analytics'; -import { detectWarehouseSources } from '@programs/warehouse-sources/detect'; +import { + detectWarehouseSources, + getDetectedWarehouseSources, +} from '@programs/warehouse-sources/detect'; import { detectPostHogIntegration, - maybeStampAiSdkDetected, reportWarehouseSourcesDetected, } from '@programs/detection/integration'; -import { posthogIntegrationConfig } from '@programs/posthog-integration/index'; -import { DETECTED_WAREHOUSE_SOURCES_KEY } from '@programs/warehouse-source/detect'; -import type { ProgramReadyContext } from '@programs/types'; -import { - buildSession, - DiscoveredFeature, - ScanConsent, - type WizardSession, -} from '@programs/session/wizard-session'; +import { DETECTED_WAREHOUSE_SOURCES_KEY } from '@programs/warehouse-sources/detect'; +import type { ProgramReadyContext } from '@programs/program-step'; +import type { WizardSession } from '@programs/session/wizard-session'; +import { buildSession } from '@programs/session/wizard-session'; +import { DiscoveredFeature } from '@shared/discovered-feature'; +import { mayReportScanResults, ScanConsent } from '@shared/run-state'; +import { stampAiSdkDetected } from '@programs/detection/ai-sdk-stamp'; import type { DetectedSource } from '@programs/warehouse-sources/types'; -import { testRunnerContext } from '../../../../test/runner-context'; function makeTmpDir(): string { return fs.mkdtempSync(path.join(os.tmpdir(), 'warehouse-reporting-')); @@ -284,30 +283,6 @@ describe('reportWarehouseSourcesDetected', () => { expect(analytics.setTag).not.toHaveBeenCalled(); }); - it('the standalone warehouse command does not report through this path', async () => { - // `wizard warehouse` writes the same frameworkContext key from its own - // detect, and sets its own tags. Without a scan-state marker it would also - // emit this event, which six saved insights read as "the integration flow - // scanned". - const session = buildSession({ installDir: tmpDir, ci: true }); - const { detectWarehousePrerequisites } = await import( - '@programs/warehouse-source/detect' - ); - detectWarehousePrerequisites(session, (key, value) => { - session.frameworkContext[key] = value; - }); - expect( - session.frameworkContext[DETECTED_WAREHOUSE_SOURCES_KEY], - ).toBeDefined(); - - reportWarehouseSourcesDetected(session); - - expect(analytics.wizardCapture).not.toHaveBeenCalledWith( - 'warehouse sources detected', - expect.anything(), - ); - }); - it('is idempotent: a second call, from either consent path, does nothing', async () => { const session = await scannedSession(ScanConsent.Granted); @@ -323,84 +298,15 @@ describe('reportWarehouseSourcesDetected', () => { }); }); -describe('the full decline contract, end to end', () => { - const FRAMEWORK_CONFIG = { - metadata: { name: 'Next.js', docsUrl: 'https://posthog.com/docs' }, - environment: { getEnvVars: () => ({ POSTHOG_KEY: 'phc_test' }) }, - ui: { getOutroChanges: () => ['Added PostHog provider'] }, - detection: { - usesPackageJson: false, - getVersion: () => '15.0.0', - packageName: 'next', - packageDisplayName: 'Next.js', - }, - analytics: { getTags: () => ({}) }, - prompts: { projectTypeDetection: 'app router' }, - }; - - const CREDENTIALS = { - accessToken: 'tok', - projectApiKey: 'phc_test', - projectId: '1', - host: { - apiHost: 'https://us.i.posthog.com', - appHost: 'https://us.posthog.com', - }, - }; - - let tmpDir: string; - - beforeEach(() => { - vi.clearAllMocks(); - tmpDir = makeTmpDir(); - fs.writeFileSync( - path.join(tmpDir, 'package.json'), - JSON.stringify({ dependencies: { stripe: '^14.0.0' } }), - ); - }); - - afterEach(() => cleanup(tmpDir)); - - it('sets the key, keeps the outro suggestion, and reports nothing, for a declined run', async () => { - const session = buildSession({ installDir: tmpDir }); - session.scanConsent = ScanConsent.Declined; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - session.frameworkConfig = FRAMEWORK_CONFIG as any; - - await detectPostHogIntegration(makeCtx(session)); - reportWarehouseSourcesDetected(session); - - const sources = session.frameworkContext[ - DETECTED_WAREHOUSE_SOURCES_KEY - ] as DetectedSource[]; - expect(sources.map((s) => s.kind)).toContain('Stripe'); - - const { run } = posthogIntegrationConfig; - if (typeof run !== 'function') throw new Error('expected a run function'); - const runDef = await run(session, testRunnerContext(session)); - const outro = runDef.buildOutroData!( - session, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - CREDENTIALS as any, - ); - if (!outro) throw new Error('expected outro data'); - expect(outro.nextSteps).toBeDefined(); - expect(outro.nextSteps!.items.join(' ')).toContain('Stripe'); - - expect(analytics.wizardCapture).not.toHaveBeenCalledWith( - 'warehouse sources detected', - expect.anything(), - ); - expect(analytics.setTag).not.toHaveBeenCalledWith( - 'warehouse_source_kinds', - expect.anything(), - ); - expect(analytics.setTag).not.toHaveBeenCalledWith( - 'warehouse_source_count', - expect.anything(), - ); +/** The org stamp, read from the session after login. */ +function stampAfterLogin(session: WizardSession): void { + stampAiSdkDetected({ + apiUser: session.apiUser, + discoveredFeatures: session.discoveredFeatures, + warehouseSources: getDetectedWarehouseSources(session), + mayReportScanResults: mayReportScanResults(session), }); -}); +} describe('wizard_ai_sdk_detected group stamp', () => { let tmpDir: string; @@ -431,7 +337,7 @@ describe('wizard_ai_sdk_detected group stamp', () => { withOrgUser(session); await detectPostHogIntegration(makeCtx(session)); - maybeStampAiSdkDetected(session); + stampAfterLogin(session); expect(analytics.groupIdentify).toHaveBeenCalledWith( 'organization', @@ -450,7 +356,7 @@ describe('wizard_ai_sdk_detected group stamp', () => { withOrgUser(session); await detectPostHogIntegration(makeCtx(session)); - maybeStampAiSdkDetected(session); + stampAfterLogin(session); expect(analytics.groupIdentify).toHaveBeenCalledWith( 'organization', @@ -468,7 +374,7 @@ describe('wizard_ai_sdk_detected group stamp', () => { withOrgUser(session); await detectPostHogIntegration(makeCtx(session)); - maybeStampAiSdkDetected(session); + stampAfterLogin(session); expect(analytics.groupIdentify).not.toHaveBeenCalled(); }); @@ -479,7 +385,7 @@ describe('wizard_ai_sdk_detected group stamp', () => { withOrgUser(session); await detectPostHogIntegration(makeCtx(session)); - maybeStampAiSdkDetected(session); + stampAfterLogin(session); expect(analytics.groupIdentify).not.toHaveBeenCalled(); }); @@ -491,7 +397,7 @@ describe('wizard_ai_sdk_detected group stamp', () => { withOrgUser(session); await detectPostHogIntegration(makeCtx(session)); - maybeStampAiSdkDetected(session); + stampAfterLogin(session); expect(analytics.groupIdentify).not.toHaveBeenCalled(); }); @@ -502,17 +408,13 @@ describe('wizard_ai_sdk_detected group stamp', () => { session.scanConsent = ScanConsent.Granted; await detectPostHogIntegration(makeCtx(session)); - maybeStampAiSdkDetected(session); + stampAfterLogin(session); expect(analytics.groupIdentify).not.toHaveBeenCalled(); }); - // Reproduces the original dead-code bug: the intro screen resolves consent - // before the user has authenticated, so a stamp fired from that path would - // always see a null apiUser and silently no-op forever (the ordering - // `reportWarehouseSourcesDetected` alone cannot fix, since it only knows - // about consent, not login state). - it('stamps once authenticate() completes, not when consent resolves first', async () => { + // Consent resolves before login, so only a stamp after login sees an apiUser. + it('stamps only once an apiUser exists and consent is granted', async () => { withDeps({ openai: '^4.0.0' }); const session = buildSession({ installDir: tmpDir }); await detectPostHogIntegration(makeCtx(session)); @@ -523,9 +425,9 @@ describe('wizard_ai_sdk_detected group stamp', () => { reportWarehouseSourcesDetected(session); expect(analytics.groupIdentify).not.toHaveBeenCalled(); - // authenticate() sets apiUser; the post-auth hook runs right after it. + // The login sets apiUser, then the stamp runs. withOrgUser(session); - maybeStampAiSdkDetected(session); + stampAfterLogin(session); expect(analytics.groupIdentify).toHaveBeenCalledTimes(1); expect(analytics.groupIdentify).toHaveBeenCalledWith( diff --git a/src/programs/detection/__tests__/package-manager.test.ts b/src/programs/detection/__tests__/package-manager.test.ts index ae102520a..65fd3c4f2 100644 --- a/src/programs/detection/__tests__/package-manager.test.ts +++ b/src/programs/detection/__tests__/package-manager.test.ts @@ -15,9 +15,6 @@ import { } from '@programs/detection/package-manager'; vi.mock('@utils/debug'); -vi.mock('../../../telemetry', () => ({ - withProgress: (_name: string, fn: () => unknown) => fn(), -})); vi.mock('@utils/analytics', () => ({ analytics: { setTag: vi.fn() }, })); diff --git a/src/programs/detection/__tests__/project-scope.test.ts b/src/programs/detection/__tests__/project-scope.test.ts index 76adb670b..7ad94477b 100644 --- a/src/programs/detection/__tests__/project-scope.test.ts +++ b/src/programs/detection/__tests__/project-scope.test.ts @@ -8,15 +8,11 @@ import { scopeInstallDirToProject, } from '@programs/detection/project-scope'; import { WIZARD_BASIC_INTEGRATION_AGENTIC_DETECTION_FLAG_KEY } from '@shared/constants'; -import { authenticate } from '@programs/authenticate'; import type { CiRunnerContext } from '@programs/runner-context'; import { buildSession } from '@programs/session/wizard-session'; import { analytics } from '@utils/analytics'; // Mock only the two network edges of scopeInstallDirToProject; everything else runs real. -vi.mock('@programs/authenticate', () => ({ - authenticate: vi.fn().mockResolvedValue(undefined), -})); vi.mock('@programs/detection/agentic', async (importOriginal) => ({ ...(await importOriginal()), detectProjectsWithAgent: vi.fn(), @@ -73,7 +69,11 @@ describe('chooseIntegrationProject', () => { }); describe('scopeInstallDirToProject', () => { - const runner: CiRunnerContext = { log: { info: vi.fn(), warn: vi.fn() } }; + const authenticate = vi.fn().mockResolvedValue(undefined); + const runner: CiRunnerContext = { + log: { info: vi.fn(), warn: vi.fn() }, + authenticate, + }; const scan = vi.mocked(detectProjectsWithAgent); const FLAG_ON = { [WIZARD_BASIC_INTEGRATION_AGENTIC_DETECTION_FLAG_KEY]: 'true', @@ -119,7 +119,7 @@ describe('scopeInstallDirToProject', () => { const session = buildSession({ installDir: '/repo' }); await scopeInstallDirToProject(session, runner); - expect(vi.mocked(authenticate)).toHaveBeenCalledTimes(1); + expect(authenticate).toHaveBeenCalledTimes(1); expect(session.installDir).toBe('/repo'); expect(outcomeEvent()).toMatchObject({ outcome: 'flag-off' }); expect(scan).not.toHaveBeenCalled(); diff --git a/src/programs/error-tracking-upload-source-maps/__tests__/detect.test.ts b/src/programs/error-tracking-upload-source-maps/__tests__/detect.test.ts index f44a32239..723af0ab7 100644 --- a/src/programs/error-tracking-upload-source-maps/__tests__/detect.test.ts +++ b/src/programs/error-tracking-upload-source-maps/__tests__/detect.test.ts @@ -4,7 +4,7 @@ import * as os from 'os'; import { detectSourceMapsPrerequisites, SOURCE_MAPS_CONTEXT_KEYS, -} from '@programs/error-tracking-upload-source-maps/index'; +} from '@programs/error-tracking-upload-source-maps'; import { buildSession } from '@programs/session/wizard-session'; function makeTmpDir(): string { diff --git a/src/programs/error-tracking/__tests__/error-tracking.test.ts b/src/programs/error-tracking/__tests__/error-tracking.test.ts index d71ecb8c6..03e8be59c 100644 --- a/src/programs/error-tracking/__tests__/error-tracking.test.ts +++ b/src/programs/error-tracking/__tests__/error-tracking.test.ts @@ -3,31 +3,25 @@ import { beforeEach, describe, expect, test, vi } from 'vitest'; import type { ProgramRun } from '@programs/program-run'; import { Integration } from '@shared/constants'; import type { AgenticDetectionReport } from '@programs/detection/agentic'; -import { detectFramework } from '@programs/detection/index'; +import { detectFramework } from '@programs/detection/framework'; import { ErrorCodes } from '@shared/errors'; -import { ERROR_TRACKING_TIPS } from '@tui/programs/error-tracking/deck/tips'; import { ERROR_TRACKING_PROJECT_PATH_KEY, toErrorTrackingReport, } from '@programs/error-tracking/detect-agentic'; -import { - errorTrackingConfig, - SYMBOL_UPLOAD_CLI_FRAMEWORKS, -} from '@programs/error-tracking/index'; -import { VARIANTS_REQUIRING_POSTHOG_CLI } from '@programs/error-tracking-upload-source-maps/detect'; +import { config as errorTracking } from '@programs/error-tracking'; import { preinstallPostHogCliOnce } from '@programs/shared/posthog-cli-preinstall'; -import type { WizardSession } from '@programs/session/wizard-session'; -import type { RunnerContext } from '@programs/runner-context'; -import { scopeInstallDirToProject } from '@programs/detection/project-scope'; import { testCiRunnerContext, testRunnerContext, -} from '../../../../test/runner-context'; +} from '@programs/shared/__tests__/runner-context.no-jest'; +import type { WizardSession } from '@programs/session/wizard-session'; +import { scopeInstallDirToProject } from '@programs/detection/project-scope'; import { analytics } from '@utils/analytics'; -import { wizardAbort } from '@host/wizard-abort'; +import { ProgramAbort } from '@programs/program-abort'; -vi.mock('@programs/detection/index', async (importOriginal) => ({ - ...(await importOriginal()), +vi.mock(import('@programs/detection/framework'), async (importOriginal) => ({ + ...(await importOriginal()), detectFramework: vi.fn(), })); vi.mock('@programs/detection/project-scope', async (importOriginal) => ({ @@ -40,17 +34,6 @@ vi.mock('@programs/detection/project-scope', async (importOriginal) => ({ vi.mock('@programs/shared/posthog-cli-preinstall', () => ({ preinstallPostHogCliOnce: vi.fn(), })); -vi.mock('@host/wizard-abort', async (importOriginal) => ({ - ...(await importOriginal()), - wizardAbort: vi.fn(), -})); - -const resolveRun = errorTrackingConfig.run as ( - session: WizardSession, - runner: RunnerContext, -) => Promise; - -const step = (id: string) => errorTrackingConfig.steps.find((s) => s.id === id); beforeEach(() => { vi.clearAllMocks(); @@ -58,38 +41,15 @@ beforeEach(() => { }); describe('error-tracking program', () => { - test('runs the error-tracking agent flow', () => { - expect(errorTrackingConfig.agentFlow).toBe('error-tracking'); - }); - - test('declares ci prerequisite work for headless runs', () => { - expect(errorTrackingConfig.ciPreRun).toBeDefined(); - }); - - test('shows the program-specific intro screen', () => { - expect(step('intro')?.screenId).toBe('error-tracking-intro'); - }); - - test('pre-installs no skill — the flow resolves variants per framework', async () => { + test('pre-installs no skill — the flow resolves variants per framework', () => { // There is no bare `error-tracking` menu entry; a seeded skillId would // send the linear path to a skill-not-found abort and mislead the intro. - expect(errorTrackingConfig.skillId).toBeUndefined(); - const run = await resolveRun( - { integration: null } as WizardSession, - testRunnerContext(), - ); - expect(run.skillId).toBeUndefined(); - }); - - test('picks the project after login and before the run', () => { - const ids = errorTrackingConfig.steps.map((s) => s.id); - expect(ids.indexOf('auth')).toBeLessThan(ids.indexOf('detect')); - expect(ids.indexOf('detect')).toBeLessThan(ids.indexOf('run')); - expect(step('detect')?.screenId).toBe('error-tracking-detect'); + expect(errorTracking.skillId).toBeUndefined(); + expect((errorTracking.run as ProgramRun).skillId).toBeUndefined(); }); test('runs the agent in the picked project, else the repo root', () => { - const targetDir = step('run')?.targetDir; + const targetDir = errorTracking.runSteps?.run?.targetDir; const picked = { installDir: '/repo', frameworkContext: { [ERROR_TRACKING_PROJECT_PATH_KEY]: 'apps/web' }, @@ -182,24 +142,30 @@ describe('error-tracking ciPreRun', () => { } as unknown as WizardSession; const runner = testCiRunnerContext(); - await errorTrackingConfig.ciPreRun?.(session, runner); + const stopped = errorTracking.ciPreRun?.(session, runner); + await expect(stopped).rejects.toBeInstanceOf(ProgramAbort); + await expect(stopped).rejects.toMatchObject({ + code: ErrorCodes.DetectUnsupportedPlatform, + }); expect(scopeInstallDirToProject).toHaveBeenCalledWith(session, runner); - - expect(wizardAbort).toHaveBeenCalledWith( - expect.objectContaining({ code: ErrorCodes.DetectUnsupportedPlatform }), - ); expect(session.integration).toBeUndefined(); }); }); -describe('error-tracking run config', () => { - test('pre-installs posthog-cli when run resolves, after the project pick', async () => { - const runner = { ...testRunnerContext(), log: { warn: vi.fn() } }; - await resolveRun( - { integration: Integration.swift } as WizardSession, - runner, - ); +describe('error-tracking posthog-cli pre-install', () => { + test('headless, pre-installs once ciPreRun detects the framework', async () => { + vi.mocked(detectFramework).mockResolvedValue(Integration.swift); + const runner = { + ...testCiRunnerContext(), + log: { info: vi.fn(), warn: vi.fn() }, + }; + const session = { + installDir: '/tmp/error-tracking-ci', + frameworkContext: {}, + } as unknown as WizardSession; + + await errorTracking.ciPreRun?.(session, runner); expect(preinstallPostHogCliOnce).toHaveBeenCalledWith( 'error tracking posthog-cli preinstall failed', @@ -212,43 +178,14 @@ describe('error-tracking run config', () => { }); test('skips the pre-install for platforms without symbol upload', async () => { - await resolveRun( - { integration: Integration.nextjs } as WizardSession, - testRunnerContext(), + await errorTracking.runSteps?.run?.onRunPrep?.( + { + integration: Integration.nextjs, + frameworkContext: {}, + } as unknown as WizardSession, + testRunnerContext().log, ); expect(preinstallPostHogCliOnce).not.toHaveBeenCalled(); }); }); - -describe('error-tracking posthog-cli pre-install set', () => { - test('contains only real Integration values', () => { - for (const integration of SYMBOL_UPLOAD_CLI_FRAMEWORKS) { - expect(Object.values(Integration)).toContain(integration); - } - }); - - test('matches the source-maps program set, keyed by Integration', () => { - // Both programs pre-install the CLI for the same platforms. The source-maps - // program keys them by uploader variant, and only `ios` is spelled - // differently (`swift` in Integration). - const expected = [...VARIANTS_REQUIRING_POSTHOG_CLI] - .map((variant) => (variant === 'ios' ? Integration.swift : variant)) - .sort(); - expect([...SYMBOL_UPLOAD_CLI_FRAMEWORKS].sort()).toEqual(expected); - }); -}); - -describe('error-tracking tips', () => { - const replayTip = ERROR_TRACKING_TIPS.find((t) => t.id === 'session-replay'); - const storeFor = (integration: Integration | null) => - ({ session: { integration } } as never); - - test('shows the replay tip only where session replay records', () => { - expect(replayTip?.visible?.(storeFor(Integration.nextjs))).toBe(true); - expect(replayTip?.visible?.(storeFor(Integration.javascriptNode))).toBe( - false, - ); - expect(replayTip?.visible?.(storeFor(null))).toBe(false); - }); -}); diff --git a/src/programs/mcp-analytics/__tests__/mcp-analytics.test.ts b/src/programs/mcp-analytics/__tests__/mcp-analytics.test.ts index 726515d6e..d597778bb 100644 --- a/src/programs/mcp-analytics/__tests__/mcp-analytics.test.ts +++ b/src/programs/mcp-analytics/__tests__/mcp-analytics.test.ts @@ -1,7 +1,7 @@ import { MCP_ANALYTICS_ABORT_CASES, - mcpAnalyticsConfig, -} from '@programs/mcp-analytics/index'; + config as mcpAnalytics, +} from '@programs/mcp-analytics'; describe('MCP_ANALYTICS_ABORT_CASES', () => { // These are the exact `[ABORT] ` strings the mcp-analytics skill @@ -37,11 +37,11 @@ describe('MCP_ANALYTICS_ABORT_CASES', () => { }); }); -describe('mcpAnalyticsConfig', () => { +describe('mcp-analytics config', () => { it('wires the mcp-analytics abort cases into the run config', () => { // `run` is statically a defined object for this program (createSkillProgram // always sets it, and never uses the session-derived function form). - const run = mcpAnalyticsConfig.run; + const run = mcpAnalytics.run; if (!run || typeof run === 'function') { throw new Error('expected a static run object'); } diff --git a/src/programs/__tests__/metrics-program.test.ts b/src/programs/metrics/__tests__/metrics.test.ts similarity index 53% rename from src/programs/__tests__/metrics-program.test.ts rename to src/programs/metrics/__tests__/metrics.test.ts index 0cb62a358..92fbeff2c 100644 --- a/src/programs/__tests__/metrics-program.test.ts +++ b/src/programs/metrics/__tests__/metrics.test.ts @@ -1,10 +1,6 @@ -import { AGENT_SKILL_STEPS } from '@programs/agent-skill/index'; -import { getProgramConfig, Program } from '@programs'; -import { metricsConfig } from '@programs/metrics/index'; +import { config as metricsConfig } from '../index'; import type { ProgramRun } from '@programs/program-run'; -import { metricsCommand } from '../../cli/commands/metrics'; - function staticRun(config: typeof metricsConfig): ProgramRun { if (typeof config.run === 'function') { throw new Error('expected a static ProgramRun, got a function'); @@ -14,22 +10,7 @@ function staticRun(config: typeof metricsConfig): ProgramRun { } describe('metrics program', () => { - it('is registered as a flat top-level `metrics` command', () => { - const config = getProgramConfig('metrics'); - expect(config).toBe(metricsConfig); - expect(config.command).toBe('metrics'); - expect(config.parentCommand).toBeUndefined(); - expect(Program.Metrics).toBe('metrics'); - }); - - it('uses the agent-skill steps with a metrics-specific intro', () => { - const [intro, ...rest] = metricsConfig.steps; - expect(intro.id).toBe('intro'); - expect(intro.screenId).toBe('metrics-intro'); - expect(rest).toEqual(AGENT_SKILL_STEPS.slice(1)); - }); - - it('runs the metrics agent flow on the orchestrator', () => { + it('runs the metrics agent flow', () => { expect(metricsConfig.agentFlow).toBe('metrics'); }); @@ -58,10 +39,4 @@ describe('metrics program', () => { expect(run.reportFile).toBe('posthog-metrics-report.md'); expect(metricsConfig.reportFile).toBe(run.reportFile); }); - - it('is exposed as a yargs command via nativeCommandFactory', () => { - expect(metricsCommand.name).toBe('metrics'); - expect(metricsCommand.description).toBe(metricsConfig.description); - expect(typeof metricsCommand.handler).toBe('function'); - }); }); diff --git a/src/programs/oauth/__tests__/refresh.test.ts b/src/programs/oauth/__tests__/refresh.test.ts index c45584f18..9232da116 100644 --- a/src/programs/oauth/__tests__/refresh.test.ts +++ b/src/programs/oauth/__tests__/refresh.test.ts @@ -1,17 +1,14 @@ import axios from 'axios'; -import { refreshAccessToken } from '@tui/auth/oauth'; +import { refreshAccessToken } from '../tokens'; import { POSTHOG_PROXY_CLIENT_ID } from '@shared/constants'; vi.mock('axios'); // No base-URL override resolves to prod routing (kills IS_DEV's implicit localhost). -vi.mock('../../../shared/utils/urls', async (importOriginal) => ({ - ...(await importOriginal()), +vi.mock(import('@utils/urls'), async (importOriginal) => ({ + ...(await importOriginal()), resolveBaseUrl: (baseUrl?: string) => baseUrl, })); -vi.mock('../../../shared/utils/debug', () => ({ - logToFile: vi.fn(), - setDebugSink: vi.fn(), -})); +vi.mock(import('@utils/debug'), () => ({ logToFile: vi.fn() })); const mockedAxios = axios as Mocked; diff --git a/src/programs/oauth/__tests__/tokens.test.ts b/src/programs/oauth/__tests__/tokens.test.ts new file mode 100644 index 000000000..8b3af7834 --- /dev/null +++ b/src/programs/oauth/__tests__/tokens.test.ts @@ -0,0 +1,71 @@ +import { missingOAuthScopes, OAuthTokenResponseSchema } from '../tokens'; + +// A grant can be narrower than the request with no error: the consent screen +// lets users deselect non-required scopes, and out-of-ceiling scopes are +// silently clamped server-side. The diff is how the wizard notices at login +// instead of via a permission failure minutes into the run. +describe('missingOAuthScopes', () => { + it('returns an empty list when the grant matches the request', () => { + expect( + missingOAuthScopes( + ['user:read', 'project:read'], + 'user:read project:read', + ), + ).toEqual([]); + }); + + it('names the scopes a deselecting user unticked at consent', () => { + expect( + missingOAuthScopes( + ['user:read', 'notebook:write', 'external_data_source:read'], + 'user:read', + ), + ).toEqual(['notebook:write', 'external_data_source:read']); + }); + + it('ignores extra granted scopes the wizard never asked for', () => { + expect( + missingOAuthScopes(['user:read'], 'user:read feature_flag:read'), + ).toEqual([]); + }); + + it('treats an empty grant as everything missing', () => { + expect(missingOAuthScopes(['user:read', 'query:read'], '')).toEqual([ + 'user:read', + 'query:read', + ]); + }); +}); + +describe('OAuthTokenResponseSchema posthog_region', () => { + const base = { + access_token: 'pha_test', + expires_in: 3600, + token_type: 'Bearer', + scope: 'event_definition:write', + }; + + it('passes a recognized region through', () => { + const token = OAuthTokenResponseSchema.parse({ + ...base, + posthog_region: 'eu', + posthog_base_url: 'https://eu.posthog.com', + }); + expect(token.posthog_region).toBe('eu'); + expect(token.posthog_base_url).toBe('https://eu.posthog.com'); + }); + + it('degrades an unrecognized region to undefined instead of failing login', () => { + const token = OAuthTokenResponseSchema.parse({ + ...base, + posthog_region: 'apac', + }); + expect(token.access_token).toBe('pha_test'); + expect(token.posthog_region).toBeUndefined(); + }); + + it('parses responses without region fields (self-hosted)', () => { + const token = OAuthTokenResponseSchema.parse(base); + expect(token.posthog_region).toBeUndefined(); + }); +}); diff --git a/src/programs/posthog-integration/__tests__/helpers/integration-prompt.no-jest.ts b/src/programs/posthog-integration/__tests__/helpers/integration-prompt.no-jest.ts index 9f77bcaf1..da3296ca3 100644 --- a/src/programs/posthog-integration/__tests__/helpers/integration-prompt.no-jest.ts +++ b/src/programs/posthog-integration/__tests__/helpers/integration-prompt.no-jest.ts @@ -1,17 +1,11 @@ -/** - * Shared fixtures for tests that build the default integration's run - * definition and prompt (`warehouse-suggestion.test.ts`, - * `posthog-integration-prompt.test.ts`). - */ +/** Fixtures for the tests that build the default integration's run definition and prompt. */ -import { posthogIntegrationConfig } from '@programs/posthog-integration/index'; -import { DETECTED_WAREHOUSE_SOURCES_KEY } from '@programs/warehouse-source/detect'; -import { - buildSession, - type WizardSession, -} from '@programs/session/wizard-session'; +import { DETECTED_WAREHOUSE_SOURCES_KEY } from '@programs/warehouse-sources/detect'; +import { testRunnerContext } from '@programs/shared/__tests__/runner-context.no-jest'; +import { buildSession } from '@programs/session/wizard-session'; +import type { WizardSession } from '@programs/session/wizard-session'; import type { DetectedSource } from '@programs/warehouse-sources/types'; -import { testRunnerContext } from '../../../../../test/runner-context'; +import { config as posthogIntegration } from '../../index'; export const CREDENTIALS = { accessToken: 'tok', @@ -23,7 +17,7 @@ export const CREDENTIALS = { }, }; -const FRAMEWORK_CONFIG = { +export const FRAMEWORK_CONFIG = { metadata: { name: 'Next.js', docsUrl: 'https://posthog.com/docs' }, environment: { getEnvVars: () => ({ POSTHOG_KEY: 'phc_test' }) }, ui: { getOutroChanges: () => ['Added PostHog provider'] }, @@ -48,7 +42,7 @@ export function sessionWith(sources: DetectedSource[]): WizardSession { } export async function resolveRun(session: WizardSession) { - const { run } = posthogIntegrationConfig; + const { run } = posthogIntegration; if (typeof run !== 'function') throw new Error('expected a run function'); return run(session, testRunnerContext(session)); } diff --git a/src/programs/posthog-integration/__tests__/index.test.ts b/src/programs/posthog-integration/__tests__/index.test.ts index 2742d5a4c..6b36bef33 100644 --- a/src/programs/posthog-integration/__tests__/index.test.ts +++ b/src/programs/posthog-integration/__tests__/index.test.ts @@ -6,14 +6,12 @@ * state, so the value rides on every later capture either way. */ -import { posthogIntegrationConfig } from '@programs/posthog-integration/index'; -import { - buildSession, - type WizardSession, -} from '@programs/session/wizard-session'; +import { config as posthogIntegration } from '@programs/posthog-integration'; +import { buildSession } from '@programs/session/wizard-session'; +import type { WizardSession } from '@programs/session/wizard-session'; import { analytics } from '@utils/analytics'; import { isUsingTypeScript } from '@utils/setup-utils'; -import { testRunnerContext } from '../../../../test/runner-context'; +import { testRunnerContext } from '@programs/shared/__tests__/runner-context.no-jest'; vi.mock('@utils/analytics', () => ({ analytics: { @@ -52,7 +50,7 @@ function sessionWithFramework(): WizardSession { } async function resolveRun(session: WizardSession) { - const { run } = posthogIntegrationConfig; + const { run } = posthogIntegration; if (typeof run !== 'function') throw new Error('expected a run function'); return run(session, testRunnerContext(session)); } diff --git a/src/programs/posthog-integration/__tests__/prompt.test.ts b/src/programs/posthog-integration/__tests__/prompt.test.ts index df6fb83da..17bb0f393 100644 --- a/src/programs/posthog-integration/__tests__/prompt.test.ts +++ b/src/programs/posthog-integration/__tests__/prompt.test.ts @@ -8,7 +8,7 @@ */ import { WIZARD_DEFAULT_AIO_LOGS_FLAG_KEY } from '@shared/constants'; -import { posthogIntegrationConfig } from '@programs/posthog-integration/index'; +import { config as posthogIntegration } from '@programs/posthog-integration'; import { analytics } from '@utils/analytics'; import { promptFor } from './helpers/integration-prompt.no-jest'; @@ -57,7 +57,7 @@ describe('linear-run flag gate', () => { describe('default observability flag gating', () => { const excluded = (flags: Record) => - posthogIntegrationConfig.excludedTaskTypes!(flags); + posthogIntegration.excludedTaskTypes!(flags); it("excludes AIO and Logs only on an explicit 'false'", () => { expect(excluded({ [WIZARD_DEFAULT_AIO_LOGS_FLAG_KEY]: 'false' })).toEqual([ diff --git a/src/programs/posthog-integration/__tests__/warehouse-decline.test.ts b/src/programs/posthog-integration/__tests__/warehouse-decline.test.ts new file mode 100644 index 000000000..fc0f1cba5 --- /dev/null +++ b/src/programs/posthog-integration/__tests__/warehouse-decline.test.ts @@ -0,0 +1,98 @@ +/** The full decline contract: detection sets the key and the outro keeps its suggestion, but nothing is reported. */ + +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; + +vi.mock(import('@utils/analytics'), () => ({ + analytics: { + wizardCapture: vi.fn(), + setTag: vi.fn(), + capture: vi.fn(), + captureException: vi.fn(), + groupIdentify: vi.fn(), + // Empty map = flags unreadable = the shipped default (AIO + Logs on). + getAllFlagsForWizard: vi.fn().mockResolvedValue({}), + } as never, +})); + +import { analytics } from '@utils/analytics'; +import { + detectPostHogIntegration, + reportWarehouseSourcesDetected, +} from '@programs/detection/integration'; +import { DETECTED_WAREHOUSE_SOURCES_KEY } from '@programs/warehouse-sources/detect'; +import type { DetectedSource } from '@programs/warehouse-sources/types'; +import { SessionStore } from '@programs/session/session-store'; +import { buildSession } from '@programs/session/wizard-session'; +import { testRunnerContext } from '@programs/shared/__tests__/runner-context.no-jest'; +import { ScanConsent } from '@shared/run-state'; +import { config as posthogIntegration } from '../index'; +import { + CREDENTIALS, + FRAMEWORK_CONFIG, +} from './helpers/integration-prompt.no-jest'; + +function makeTmpDir(): string { + return fs.mkdtempSync(path.join(os.tmpdir(), 'warehouse-reporting-')); +} + +function cleanup(dir: string): void { + fs.rmSync(dir, { recursive: true, force: true }); +} + +describe('the full decline contract, end to end', () => { + let tmpDir: string; + + beforeEach(() => { + vi.clearAllMocks(); + tmpDir = makeTmpDir(); + fs.writeFileSync( + path.join(tmpDir, 'package.json'), + JSON.stringify({ dependencies: { stripe: '^14.0.0' } }), + ); + }); + + afterEach(() => cleanup(tmpDir)); + + it('sets the key, keeps the outro suggestion, and reports nothing, for a declined run', async () => { + const session = buildSession({ installDir: tmpDir }); + session.scanConsent = ScanConsent.Declined; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + session.frameworkConfig = FRAMEWORK_CONFIG as any; + + const store = new SessionStore(session); + await detectPostHogIntegration(store.readyContext()); + reportWarehouseSourcesDetected(store.session); + + const sources = store.session.frameworkContext[ + DETECTED_WAREHOUSE_SOURCES_KEY + ] as DetectedSource[]; + expect(sources.map((s) => s.kind)).toContain('Stripe'); + + const { run } = posthogIntegration; + if (typeof run !== 'function') throw new Error('expected a run function'); + const runDef = await run(store.session, testRunnerContext(store.session)); + const outro = runDef.buildOutroData!( + store.session, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + CREDENTIALS as any, + ); + if (!outro) throw new Error('expected outro data'); + expect(outro.nextSteps).toBeDefined(); + expect(outro.nextSteps!.items.join(' ')).toContain('Stripe'); + + expect(analytics.wizardCapture).not.toHaveBeenCalledWith( + 'warehouse sources detected', + expect.anything(), + ); + expect(analytics.setTag).not.toHaveBeenCalledWith( + 'warehouse_source_kinds', + expect.anything(), + ); + expect(analytics.setTag).not.toHaveBeenCalledWith( + 'warehouse_source_count', + expect.anything(), + ); + }); +}); diff --git a/src/programs/posthog-integration/__tests__/warehouse-seed-task.test.ts b/src/programs/posthog-integration/__tests__/warehouse-seed-task.test.ts index c9b091a28..0d0bdab87 100644 --- a/src/programs/posthog-integration/__tests__/warehouse-seed-task.test.ts +++ b/src/programs/posthog-integration/__tests__/warehouse-seed-task.test.ts @@ -15,8 +15,8 @@ vi.mock('@utils/analytics', () => ({ }, })); -import { posthogIntegrationConfig } from '@programs/posthog-integration/index'; -import { DETECTED_WAREHOUSE_SOURCES_KEY } from '@programs/warehouse-source/detect'; +import { config as posthogIntegration } from '@programs/posthog-integration'; +import { DETECTED_WAREHOUSE_SOURCES_KEY } from '@programs/warehouse-sources/detect'; const POSTGRES: DetectedSource = { kind: 'Postgres', @@ -34,7 +34,7 @@ function session(over: Partial = {}): WizardSession { } function seed(sess: WizardSession) { - return posthogIntegrationConfig.seedTasks?.(sess) ?? []; + return posthogIntegration.seedTasks?.(sess) ?? []; } function sources(n: number): DetectedSource[] { diff --git a/src/programs/posthog-integration/__tests__/warehouse-suggestion.test.ts b/src/programs/posthog-integration/__tests__/warehouse-suggestion.test.ts index ffcc8dcd7..914a399f5 100644 --- a/src/programs/posthog-integration/__tests__/warehouse-suggestion.test.ts +++ b/src/programs/posthog-integration/__tests__/warehouse-suggestion.test.ts @@ -13,8 +13,8 @@ * say the run already connected the sources. */ -import { POSTHOG_INTEGRATION_PROGRAM } from '@tui/programs/posthog-integration/flow'; import type { WizardSession } from '@programs/session/wizard-session'; +import { config as posthogIntegration } from '@programs/posthog-integration'; import type { DetectedSource } from '@programs/warehouse-sources/types'; import { analytics } from '@utils/analytics'; @@ -153,27 +153,11 @@ describe('env tool instruction', () => { }); }); -describe('flow shape', () => { - it('adds no steps — the suggestion never becomes an inline run', () => { - const ids = POSTHOG_INTEGRATION_PROGRAM.map((s) => s.id); - expect(ids).toEqual([ - 'detect', - 'intro', - 'health-check', - 'setup', - 'auth', - 'run', - 'outro', - 'mcp', - 'slack-connect', - 'keep-skills', - ]); - }); - +describe('run shape', () => { it('keeps the program single-run, so the outro stays terminal', () => { - // A step carrying its own `run` would flip run-wizard into the composed - // walk, where a second agent run could abort before the outro is pushed. - expect(POSTHOG_INTEGRATION_PROGRAM.some((s) => s.run)).toBe(false); + // A run step naming another program would flip run-wizard into the + // composed walk, where a second agent run could abort before the outro. + expect(posthogIntegration.runSteps).toBeUndefined(); }); }); diff --git a/src/programs/posthog-integration/upload-environment-variables/providers/__tests__/vercel.test.ts b/src/programs/posthog-integration/upload-environment-variables/providers/__tests__/vercel.test.ts index 49e989634..48b11b3bc 100644 --- a/src/programs/posthog-integration/upload-environment-variables/providers/__tests__/vercel.test.ts +++ b/src/programs/posthog-integration/upload-environment-variables/providers/__tests__/vercel.test.ts @@ -5,7 +5,13 @@ import * as child_process from 'child_process'; vi.mock('fs'); vi.mock('child_process'); -const mockOptions = { installDir: '/tmp/project' }; +const mockOptions = { + installDir: '/tmp/project', + runner: { + log: { info: vi.fn(), warn: vi.fn() }, + spinner: () => ({ start: vi.fn(), stop: vi.fn() }), + }, +}; describe('VercelEnvironmentProvider', () => { let provider: VercelEnvironmentProvider; diff --git a/src/programs/replay-vision/__tests__/replay-vision.test.ts b/src/programs/replay-vision/__tests__/replay-vision.test.ts index ec97a39fd..b4efb8e83 100644 --- a/src/programs/replay-vision/__tests__/replay-vision.test.ts +++ b/src/programs/replay-vision/__tests__/replay-vision.test.ts @@ -1,36 +1,8 @@ import { describe, expect, test } from 'vitest'; -import { Integration } from '@shared/constants'; -import { - replayVisionConfig, - REPLAY_VISION_SUPPORTED, -} from '@programs/replay-vision/index'; - -describe('replay-vision program', () => { - test('runs the replay-vision agent flow', () => { - expect(replayVisionConfig.agentFlow).toBe('replay-vision'); - }); - - test('detects the framework before the agent-skill steps', () => { - expect(replayVisionConfig.steps[0]?.id).toBe('detect'); - expect(replayVisionConfig.steps[0]?.onReady).toBeDefined(); - }); - - test('declares ci prerequisite work for headless runs', () => { - expect(replayVisionConfig.ciPreRun).toBeDefined(); - }); -}); +import { Integration, REPLAY_VISION_SUPPORTED } from '@shared/constants'; describe('replay-vision platform support', () => { - test('covers every Integration with an explicit verdict', () => { - // The gate is an allow-list: a new Integration enum entry is unsupported - // until someone decides otherwise. This test only pins that the set - // contains real Integration values. - for (const integration of REPLAY_VISION_SUPPORTED) { - expect(Object.values(Integration)).toContain(integration); - } - }); - test('supports web and replay-capable mobile platforms', () => { expect(REPLAY_VISION_SUPPORTED.has(Integration.nextjs)).toBe(true); expect(REPLAY_VISION_SUPPORTED.has(Integration.javascript_web)).toBe(true); diff --git a/src/programs/revenue-analytics/__tests__/detect.test.ts b/src/programs/revenue-analytics/__tests__/detect.test.ts index 4340949d6..bece0abdf 100644 --- a/src/programs/revenue-analytics/__tests__/detect.test.ts +++ b/src/programs/revenue-analytics/__tests__/detect.test.ts @@ -1,7 +1,7 @@ import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; -import { detectRevenuePrerequisites } from '@programs/revenue-analytics/index'; +import { detectRevenuePrerequisites } from '@programs/revenue-analytics'; import { buildSession } from '@programs/session/wizard-session'; function makeTmpDir(): string { diff --git a/src/programs/self-driving/__tests__/detect.test.ts b/src/programs/self-driving/__tests__/detect.test.ts index 148a67f91..ea5093ee4 100644 --- a/src/programs/self-driving/__tests__/detect.test.ts +++ b/src/programs/self-driving/__tests__/detect.test.ts @@ -3,9 +3,9 @@ import * as path from 'path'; import * as os from 'os'; import { detectSelfDrivingPrerequisites, - selfDrivingConfig, + config as selfDriving, SELF_DRIVING_ABORT_CASES, -} from '@programs/self-driving/index'; +} from '@programs/self-driving'; import { detectPostHogPresent, POSTHOG_MANIFESTS, @@ -13,8 +13,8 @@ import { SELF_DRIVING_TOOL_KINDS, getSelfDrivingDetectedTools, } from '@programs/self-driving/detect'; -import { getDetectedWarehouseSources } from '@programs/warehouse-source/detect'; -import { WizardStore } from '@tui/store'; +import { getDetectedWarehouseSources } from '@programs/warehouse-sources/detect'; +import { SessionStore } from '@programs/session/session-store'; import { SOURCE_DETECTORS } from '@programs/warehouse-sources/registry'; import type { DetectedSource } from '@programs/warehouse-sources/types'; import { toIntegrationReport } from '@programs/self-driving/detect-agentic'; @@ -23,9 +23,9 @@ import { type AgenticDetectionReport, } from '@programs/detection/agentic'; import { Integration } from '@shared/constants'; -import { WIZARD_TOOL_NAMES } from '@agent/tools'; +import { WIZARD_TOOL_NAMES } from '@agent'; import { buildSession } from '@programs/session/wizard-session'; -import { testRunnerContext } from '../../../../test/runner-context'; +import { testRunnerContext } from '@programs/shared/__tests__/runner-context.no-jest'; import type { Mock } from 'vitest'; function makeTmpDir(): string { @@ -132,9 +132,8 @@ describe('the detect step does not leak into the composed integration run', () = afterEach(() => cleanup(tmpDir)); it('stashes under its own key and leaves the warehouse key untouched', async () => { - const store = new WizardStore('self-driving'); - store.session = buildSession({ installDir: tmpDir }); - await store.runReadyHooks(); + const store = new SessionStore(buildSession({ installDir: tmpDir })); + await selfDriving.onReady?.(store.readyContext()); // Self-driving sees its tools... expect( @@ -168,21 +167,16 @@ describe('SELF_DRIVING_ABORT_CASES', () => { }); }); -describe('selfDrivingConfig', () => { +describe('self-driving config', () => { it('keeps wizard_ask enabled — the flow is interview-driven', () => { - expect(selfDrivingConfig.disallowedTools ?? []).not.toContain( + expect(selfDriving.disallowedTools ?? []).not.toContain( WIZARD_TOOL_NAMES.wizardAsk, ); }); - it('ships its own Learn deck', () => { - const blocks = selfDrivingConfig.getContentBlocks?.() ?? []; - expect(blocks.length).toBeGreaterThan(0); - }); - it('gives wizard_ask a 30-min timeout for the browser-handoff steps', async () => { // `run` is resolved per-session so the prompt can carry the integrate flag. - const { run } = selfDrivingConfig; + const { run } = selfDriving; const resolved = typeof run === 'function' ? await run(buildSession({}), testRunnerContext()) @@ -191,28 +185,10 @@ describe('selfDrivingConfig', () => { }); it('wires the self-driving-setup skill and CLI command', () => { - expect(selfDrivingConfig.command).toBe('self-driving'); - expect(selfDrivingConfig.skillId).toBe('self-driving-setup'); - expect(selfDrivingConfig.id).toBe('self-driving'); - expect(selfDrivingConfig.requires).toContain('posthog-integration'); - }); - - it('has no keep-skills step — the setup skill is removed in postRun', () => { - const stepIds = selfDrivingConfig.steps.map((s) => s.id); - expect(stepIds).not.toContain('skills'); - expect(stepIds).toEqual([ - 'detect', - 'intro', - 'integration-check', - 'health-check', - 'auth', - 'integrate-detect', - 'integrate-run', - 'self-driving-handoff', - 'self-driving-github', - 'run', - 'outro', - ]); + expect(selfDriving.command).toBe('self-driving'); + expect(selfDriving.skillId).toBe('self-driving-setup'); + expect(selfDriving.id).toBe('self-driving'); + expect(selfDriving.requires).toContain('posthog-integration'); }); }); @@ -521,32 +497,6 @@ describe('detectPostHogPresent', () => { }); }); -describe('integrate-detect step', () => { - const step = selfDrivingConfig.steps.find((s) => s.id === 'integrate-detect'); - - it('is incomplete while integrating and no project picked yet', () => { - const session = buildSession({}); - session.integrate = true; - session.integration = null; - expect(step?.isComplete?.(session)).toBe(false); - }); - - it('is complete once a project is picked to integrate', () => { - const session = buildSession({}); - session.integrate = true; - session.integration = Integration.nextjs; - expect(step?.isComplete?.(session)).toBe(true); - }); - - it('is complete once the user continues with an existing install', () => { - // integrate=false must complete the step or the orchestrator hangs. - const session = buildSession({}); - session.integrate = false; - session.integration = null; - expect(step?.isComplete?.(session)).toBe(true); - }); -}); - describe('toIntegrationReport', () => { const build = ( p: Partial, @@ -621,9 +571,7 @@ describe('manifest list sync', () => { }); describe('integrate-run targetDir', () => { - const targetDir = selfDrivingConfig.steps.find( - (s) => s.id === 'integrate-run', - )?.targetDir; + const targetDir = selfDriving.runSteps?.['integrate-run']?.targetDir; const dirFor = (picked: string): string | undefined => { const session = buildSession({ installDir: '/repo' }); diff --git a/src/programs/self-driving/__tests__/prompt.test.ts b/src/programs/self-driving/__tests__/prompt.test.ts index 8d8cacf8f..b80752e9c 100644 --- a/src/programs/self-driving/__tests__/prompt.test.ts +++ b/src/programs/self-driving/__tests__/prompt.test.ts @@ -1,5 +1,5 @@ import { buildSelfDrivingPrompt } from '@programs/self-driving/prompt'; -import type { PromptContext } from '@agent/agent-runner'; +import type { PromptContext } from '@agent/types'; import { HostResolution } from '@shared/host-resolution'; import type { DetectedSource } from '@programs/warehouse-sources/types'; diff --git a/src/programs/session/__tests__/ask-policy.test.ts b/src/programs/session/__tests__/ask-policy.test.ts new file mode 100644 index 000000000..dbd323e45 --- /dev/null +++ b/src/programs/session/__tests__/ask-policy.test.ts @@ -0,0 +1,40 @@ +/** The ask gate on a built session: e2eAsk defaults off and the env bag cannot enable it. */ +import { shouldDisableAsk } from '@shared/ask-policy'; +import { readEnvironment } from '@utils/environment'; +import { buildSession } from '../wizard-session'; + +describe('the ask gate on a built session', () => { + afterEach(() => { + delete process.env.POSTHOG_WIZARD_e2e_ask; + }); + + it('leaves a plain --ci session disabled — buildSession defaults e2eAsk to false', () => { + const session = buildSession({ + installDir: '/tmp/ask-policy', + ci: true, + }); + expect(session.e2eAsk).toBe(false); + expect(shouldDisableAsk(session)).toBe(true); + }); + + it('re-enables the bridge when the harness asks for it', () => { + const session = buildSession({ + installDir: '/tmp/ask-policy', + ci: true, + e2eAsk: true, + }); + expect(shouldDisableAsk(session)).toBe(false); + }); + + // The CI runner spreads the env bag into buildSession. + it('cannot re-enable wizard_ask in a --ci run', () => { + process.env.POSTHOG_WIZARD_e2e_ask = 'true'; + const session = buildSession({ + installDir: '/tmp/env-bag', + ci: true, + ...readEnvironment(), + }); + expect(session.e2eAsk).toBe(false); + expect(shouldDisableAsk(session)).toBe(true); + }); +}); diff --git a/src/programs/session/__tests__/interaction.test.ts b/src/programs/session/__tests__/interaction.test.ts new file mode 100644 index 000000000..9bfba8ef0 --- /dev/null +++ b/src/programs/session/__tests__/interaction.test.ts @@ -0,0 +1,128 @@ +vi.mock(import('@utils/debug')); +vi.mock(import('@utils/analytics'), () => ({ + analytics: { wizardCapture: vi.fn() } as never, +})); + +import { logToFile } from '@utils/debug'; +import { storeInteraction } from '../interaction'; +import { SessionStore } from '../session-store'; +import { buildSession } from '../wizard-session'; + +beforeEach(() => { + vi.mocked(logToFile).mockClear(); +}); + +const question = { id: 'q', source: 'test', questions: [] }; +const notice = { + title: 'Optional', + body: [], + items: [], + prompt: 'Continue?', + confirmLabel: 'Yes', + cancelLabel: 'No', +}; + +it('forwards answers and notices, leaving the host alone once they settle', async () => { + const store = new SessionStore(buildSession({ installDir: '/project' })); + const ask = vi + .spyOn(store, 'requestQuestion') + .mockResolvedValue({ q: 'yes' }); + const cancelAsk = vi.spyOn(store, 'cancelPendingQuestion'); + const show = vi.spyOn(store, 'showTaskNotice').mockResolvedValue(true); + const cancelNotice = vi.spyOn(store, 'resolveTaskNotice'); + const interaction = storeInteraction(store); + const asked = new AbortController(); + const noticed = new AbortController(); + const onAnswer = vi.fn(); + await expect( + interaction.ask?.(question, { signal: asked.signal, onAnswer }), + ).resolves.toEqual({ q: 'yes' }); + await expect( + interaction.taskNotice?.(notice, { signal: noticed.signal }), + ).resolves.toBe(true); + // A late abort must not dismiss whatever the host shows next. + asked.abort(); + noticed.abort(); + // The store gets the bridge's timeout heartbeat alongside the question. + expect(ask).toHaveBeenCalledWith(question, onAnswer); + expect(show).toHaveBeenCalledWith(notice); + expect(cancelAsk).not.toHaveBeenCalled(); + expect(cancelNotice).not.toHaveBeenCalled(); +}); + +it('dismisses an open question or notice when its signal aborts', () => { + const store = new SessionStore(buildSession({ installDir: '/project' })); + vi.spyOn(store, 'requestQuestion').mockReturnValue( + new Promise(() => undefined), + ); + const cancelAsk = vi.spyOn(store, 'cancelPendingQuestion'); + vi.spyOn(store, 'showTaskNotice').mockReturnValue( + new Promise(() => undefined), + ); + const cancelNotice = vi.spyOn(store, 'resolveTaskNotice'); + const interaction = storeInteraction(store); + const asked = new AbortController(); + const noticed = new AbortController(); + void interaction.ask?.(question, { signal: asked.signal }); + void interaction.taskNotice?.(notice, { signal: noticed.signal }); + + asked.abort(); + expect(cancelAsk).toHaveBeenCalledOnce(); + expect(cancelNotice).not.toHaveBeenCalled(); + noticed.abort(); + expect(cancelNotice).toHaveBeenCalledOnce(); +}); + +it('dismisses at once when the request signal aborted before it opened', () => { + const store = new SessionStore(buildSession({ installDir: '/project' })); + vi.spyOn(store, 'requestQuestion').mockReturnValue( + new Promise(() => undefined), + ); + const cancelAsk = vi.spyOn(store, 'cancelPendingQuestion'); + vi.spyOn(store, 'showTaskNotice').mockReturnValue( + new Promise(() => undefined), + ); + const cancelNotice = vi.spyOn(store, 'resolveTaskNotice'); + const interaction = storeInteraction(store); + // An abort listener added to an aborted signal never fires. + void interaction.ask?.(question, { signal: AbortSignal.abort() }); + void interaction.taskNotice?.(notice, { signal: AbortSignal.abort() }); + expect(cancelAsk).toHaveBeenCalledOnce(); + expect(cancelNotice).toHaveBeenCalledOnce(); +}); + +// The ask bridge aborts the signal on its timeout and settles on its own +// (wizard-ask-bridge.test.ts); the answerer's part is to not throw from the abort. +it('logs a throwing question dismissal instead of throwing from the abort', () => { + const store = new SessionStore(buildSession({ installDir: '/project' })); + vi.spyOn(store, 'requestQuestion').mockReturnValue( + new Promise(() => undefined), + ); + const broken = new Error('overlay broken'); + vi.spyOn(store, 'cancelPendingQuestion').mockImplementation(() => { + throw broken; + }); + const asked = new AbortController(); + void storeInteraction(store).ask?.(question, { signal: asked.signal }); + + // Node rethrows an abort listener's error as an uncaught exception the + // bridge cannot catch, so the answerer logs it instead. + asked.abort(); + expect(logToFile).toHaveBeenCalledWith(expect.any(String), broken); +}); + +it('logs a throwing notice dismissal instead of throwing from the abort', () => { + const store = new SessionStore(buildSession({ installDir: '/project' })); + vi.spyOn(store, 'showTaskNotice').mockReturnValue( + new Promise(() => undefined), + ); + const broken = new Error('overlay broken'); + vi.spyOn(store, 'resolveTaskNotice').mockImplementation(() => { + throw broken; + }); + const noticed = new AbortController(); + void storeInteraction(store).taskNotice?.(notice, { signal: noticed.signal }); + + noticed.abort(); + expect(logToFile).toHaveBeenCalledWith(expect.any(String), broken); +}); diff --git a/src/programs/session/__tests__/session-properties.test.ts b/src/programs/session/__tests__/session-properties.test.ts new file mode 100644 index 000000000..b948e14a2 --- /dev/null +++ b/src/programs/session/__tests__/session-properties.test.ts @@ -0,0 +1,118 @@ +/** What `sessionProperties` reports for a session `buildSession` made: scan results wait on consent. */ +import { sessionProperties } from '@utils/analytics'; +import { DiscoveredFeature } from '@shared/discovered-feature'; +import { ScanConsent } from '@shared/run-state'; +import { buildSession } from '../wizard-session'; + +describe('sessionProperties: the SDK verdict', () => { + it('includes the posthog_sdk_detected verdict once sharing is granted', () => { + const session = buildSession({}); + session.scanConsent = ScanConsent.Granted; + expect(sessionProperties(session).posthog_sdk_detected).toBe(false); + + session.posthogSdkDetected = true; + expect(sessionProperties(session).posthog_sdk_detected).toBe(true); + }); + + // It is a package.json scan result, so it waits on the same consent. + it('omits the verdict while consent is undecided or declined', () => { + const session = buildSession({}); + session.posthogSdkDetected = true; + + expect(session.scanConsent).toBe(ScanConsent.Undecided); + expect(sessionProperties(session)).not.toHaveProperty( + 'posthog_sdk_detected', + ); + + session.scanConsent = ScanConsent.Declined; + expect(sessionProperties(session)).not.toHaveProperty( + 'posthog_sdk_detected', + ); + }); +}); + +describe('sessionProperties: discovered features', () => { + it('includes discovered_features once consent is granted', () => { + const session = buildSession({ installDir: '/tmp/app' }); + session.discoveredFeatures = [DiscoveredFeature.Stripe]; + session.scanConsent = ScanConsent.Granted; + + const properties = sessionProperties(session); + + expect(properties.discovered_features).toEqual([DiscoveredFeature.Stripe]); + }); + + it('omits discovered_features entirely when the user declined sharing', () => { + const session = buildSession({ installDir: '/tmp/app' }); + session.discoveredFeatures = [DiscoveredFeature.Stripe]; + session.scanConsent = ScanConsent.Declined; + + const properties = sessionProperties(session); + + expect(properties).not.toHaveProperty('discovered_features'); + }); + + it('omits discovered_features on a --signup run before the user answers', () => { + // --signup renders the full TUI, so these events fire while the intro + // screen is still on screen. Granting on the flag would put scan results + // on every one of them, including for a user who then declines. + const session = buildSession({ installDir: '/tmp/app', signup: true }); + session.discoveredFeatures = [DiscoveredFeature.Stripe]; + + const properties = sessionProperties(session); + + expect(properties).not.toHaveProperty('discovered_features'); + }); + + it('omits discovered_features while consent is still undecided', () => { + const session = buildSession({ installDir: '/tmp/app' }); + session.discoveredFeatures = [DiscoveredFeature.Stripe]; + session.scanConsent = ScanConsent.Undecided; + + const properties = sessionProperties(session); + + // Undecided reads the same as declined: a path that reports before the + // user has been asked must send nothing, not everything. + expect(properties).not.toHaveProperty('discovered_features'); + }); + + it('sends scan_consent in every state, so an absent list is explainable', () => { + for (const consent of [ + ScanConsent.Undecided, + ScanConsent.Granted, + ScanConsent.Declined, + ]) { + const session = buildSession({ installDir: '/tmp/app' }); + session.scanConsent = consent; + + expect(sessionProperties(session).scan_consent).toBe(consent); + } + }); + + it('never sends an empty array in place of the omitted key', () => { + const session = buildSession({ installDir: '/tmp/app' }); + session.discoveredFeatures = []; + session.scanConsent = ScanConsent.Declined; + + const properties = sessionProperties(session); + + // Absent, not []. An empty array would misread as "we looked and found + // nothing" instead of "we didn't report what we found". + expect('discovered_features' in properties).toBe(false); + }); + + it('leaves every other property untouched by a decline', () => { + const session = buildSession({ installDir: '/tmp/app' }); + session.scanConsent = ScanConsent.Declined; + session.integration = null; + + const properties = sessionProperties(session); + + expect(properties).toMatchObject({ + integration: null, + detected_framework: null, + typescript: false, + run_phase: session.runPhase, + }); + }); +}); diff --git a/src/programs/session/task-stream/__tests__/audit-areas.test.ts b/src/programs/session/task-stream/__tests__/audit-areas.test.ts index a6da0e9e0..233e7b317 100644 --- a/src/programs/session/task-stream/__tests__/audit-areas.test.ts +++ b/src/programs/session/task-stream/__tests__/audit-areas.test.ts @@ -1,9 +1,6 @@ -import { - rollUpAuditAreas, - MAX_AUDIT_AREAS, -} from '@programs/session/task-stream/audit-areas'; -import { StreamTaskStatus } from '@programs/session/task-stream/types'; -import type { AuditCheck } from '@programs/audit/types'; +import { rollUpAuditAreas, MAX_AUDIT_AREAS } from '../audit-areas'; +import { StreamTaskStatus } from '../types'; +import type { AuditCheck } from '@shared/audit-ledger'; const check = (over: Partial): AuditCheck => ({ id: 'id', diff --git a/src/programs/session/task-stream/__tests__/event-plan-watcher.test.ts b/src/programs/session/task-stream/__tests__/event-plan-watcher.test.ts index 156377743..236f945d4 100644 --- a/src/programs/session/task-stream/__tests__/event-plan-watcher.test.ts +++ b/src/programs/session/task-stream/__tests__/event-plan-watcher.test.ts @@ -7,12 +7,10 @@ import { } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { - EventPlanWatcher, - normalizeEventPlan, -} from '@programs/session/task-stream/event-plan-watcher'; -import { EVENT_PLAN_FILE } from '@programs/posthog-integration/constants'; -import type { PlannedEvent, WizardStore } from '@tui/store'; +import { EventPlanWatcher, normalizeEventPlan } from '../event-plan-watcher'; +import { EVENT_PLAN_FILE } from '@shared/constants'; +import type { SessionStore } from '../../session-store'; +import type { PlannedEvent } from '@programs/session/session-store'; const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); @@ -26,7 +24,7 @@ function createStore(installDir: string) { setEventPlan(events: PlannedEvent[]) { eventPlan = events; }, - } as WizardStore; + } as SessionStore; } describe('EventPlanWatcher', () => { diff --git a/src/programs/session/task-stream/__tests__/file-destination.test.ts b/src/programs/session/task-stream/__tests__/file-destination.test.ts index 18357f98f..db0d9287e 100644 --- a/src/programs/session/task-stream/__tests__/file-destination.test.ts +++ b/src/programs/session/task-stream/__tests__/file-destination.test.ts @@ -1,16 +1,10 @@ import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { - FileDestination, - createFileDestination, -} from '@programs/session/task-stream/destinations/file'; -import { - StreamEvent, - type TaskStreamUpdate, -} from '@programs/session/task-stream/types'; +import { FileDestination, createFileDestination } from '../destinations/file'; +import { StreamEvent, type TaskStreamUpdate } from '../types'; import { WIZARD_TASK_STREAM_FILE } from '@utils/paths'; -import { RunPhase } from '@programs/session/wizard-session'; +import { RunPhase } from '@shared/run-state'; const payload = (over: Partial = {}): TaskStreamUpdate => ({ session_id: 'audit-audit-2026-01-01T00:00:00Z', diff --git a/src/programs/session/task-stream/__tests__/posthog-destination.test.ts b/src/programs/session/task-stream/__tests__/posthog-destination.test.ts index 48035d7f6..5243d81f2 100644 --- a/src/programs/session/task-stream/__tests__/posthog-destination.test.ts +++ b/src/programs/session/task-stream/__tests__/posthog-destination.test.ts @@ -1,6 +1,7 @@ import { PostHogDestination } from '../destinations/posthog'; import { StreamEvent, type TaskStreamUpdate } from '../types'; -import { RunPhase, type Credentials } from '../../wizard-session'; +import { RunPhase } from '@shared/run-state'; +import { type Credentials } from '@shared/api'; import { HostResolution } from '@shared/host-resolution'; const SAMPLE_CREDS: Credentials = { diff --git a/src/programs/session/task-stream/__tests__/task-stream-push.test.ts b/src/programs/session/task-stream/__tests__/task-stream-push.test.ts index 759437d4b..7c1be711a 100644 --- a/src/programs/session/task-stream/__tests__/task-stream-push.test.ts +++ b/src/programs/session/task-stream/__tests__/task-stream-push.test.ts @@ -1,22 +1,15 @@ -import { TaskStreamPush } from '@programs/session/task-stream/task-stream-push'; -import { - StreamEvent, - StreamTaskStatus, -} from '@programs/session/task-stream/types'; -import type { - TaskStreamDestination, - TaskStreamUpdate, -} from '@programs/session/task-stream/types'; -import type { WizardStore, TaskItem } from '@tui/store'; -import { TaskStatus } from '@ui/wizard-ui'; -import { - RunPhase, - type PendingQuestion, -} from '@programs/session/wizard-session'; +import { TaskStreamPush } from '../task-stream-push'; +import { StreamEvent, StreamTaskStatus } from '../types'; +import type { TaskStreamDestination, TaskStreamUpdate } from '../types'; +import type { SessionStore } from '../../session-store'; +import type { TaskItem } from '@programs/session/session-store'; +import { TaskStatus } from '@shared/task-status'; +import { RunPhase } from '@shared/run-state'; +import { type PendingQuestion } from '@agent/types'; import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { EVENT_PLAN_FILE } from '@programs/posthog-integration/constants'; +import { EVENT_PLAN_FILE } from '@shared/constants'; type Listener = () => void; @@ -97,7 +90,7 @@ function createMockStore(overrides: Partial = {}) { }, }; - return store as typeof store & WizardStore; + return store as typeof store & SessionStore; } function createMockDestination(name = 'test'): TaskStreamDestination & { diff --git a/src/programs/session/task-stream/__tests__/wizard-run-sync.test.ts b/src/programs/session/task-stream/__tests__/wizard-run-sync.test.ts index ad6538b3c..2993c2b60 100644 --- a/src/programs/session/task-stream/__tests__/wizard-run-sync.test.ts +++ b/src/programs/session/task-stream/__tests__/wizard-run-sync.test.ts @@ -3,10 +3,11 @@ import { RunTaskNames, createWizardRunSync, } from '../wizard-run-sync'; -import { RunPhase, buildSession } from '@programs/session/wizard-session'; +import { RunPhase } from '@shared/run-state'; +import { buildSession } from '../../wizard-session'; import { HostResolution } from '@shared/host-resolution'; -import { TaskStatus } from '@ui/wizard-ui'; -import type { TaskItem } from '@tui/store'; +import { TaskStatus } from '@shared/task-status'; +import type { TaskItem } from '@programs/session/session-store'; import { VERSION } from '@shared/version'; import { currentCredentials } from '@shared/oauth-session'; @@ -420,7 +421,7 @@ it('uses a fresh creation key for each independent execution', async () => { it.each(['local', 'cloud'] as const)( 'selects exactly one remote transport for %s executions and keeps file output', async (mode) => { - const { WizardStore } = await import('@tui/store'); + const { SessionStore } = await import('../../session-store'); const { TaskStreamPush } = await import('../task-stream-push'); for (const variant of [ 'wizard-run', @@ -430,8 +431,7 @@ it.each(['local', 'cloud'] as const)( undefined, ]) { const { session, fetchImpl, options, writes } = setup(mode); - const store = new WizardStore(); - store.session = session; + const store = new SessionStore(session); let flags: Record = variant ? { 'wizard-run-sync': variant } : {}; @@ -480,11 +480,10 @@ it.each(['local', 'cloud'] as const)( ); it('waits for authenticated flags, then keeps run failures on the selected transport', async () => { - const { WizardStore } = await import('@tui/store'); + const { SessionStore } = await import('../../session-store'); const { TaskStreamPush } = await import('../task-stream-push'); const { session, options, fetchImpl } = setup('cloud'); - const store = new WizardStore(); - store.session = session; + const store = new SessionStore(session); let flags: Record | null = null; const legacy = { name: 'posthog', @@ -512,11 +511,10 @@ it('waits for authenticated flags, then keeps run failures on the selected trans }); it('sends the first WizardSession snapshot as Create after flags load', async () => { - const { WizardStore } = await import('@tui/store'); + const { SessionStore } = await import('../../session-store'); const { TaskStreamPush } = await import('../task-stream-push'); const { session, options } = setup('cloud'); - const store = new WizardStore(); - store.session = session; + const store = new SessionStore(session); let flags: Record | null = null; const legacy = { name: 'posthog', diff --git a/test/runner-context.ts b/src/programs/shared/__tests__/runner-context.no-jest.ts similarity index 56% rename from test/runner-context.ts rename to src/programs/shared/__tests__/runner-context.no-jest.ts index 92bf9cc83..d88e08cc5 100644 --- a/test/runner-context.ts +++ b/src/programs/shared/__tests__/runner-context.no-jest.ts @@ -1,4 +1,4 @@ -import type { CiRunnerContext, RunnerContext } from '@programs/types'; +import type { CiRunnerContext, RunnerContext } from '@programs/runner-context'; export function testRunnerContext(session?: { frameworkContext: Record; @@ -9,15 +9,22 @@ export function testRunnerContext(session?: { setFrameworkContext: (key, value) => { context[key] = value; }, - log: { warn: () => undefined }, + log: { info: () => undefined, warn: () => undefined }, + spinner: () => ({ + start: () => undefined, + stop: () => undefined, + message: () => undefined, + }), }; } +/** A headless runner that is already logged in. */ export function testCiRunnerContext(): CiRunnerContext { return { log: { info: () => undefined, warn: () => undefined, }, + authenticate: () => Promise.resolve(), }; } diff --git a/src/programs/warehouse-source/__tests__/ask-timeout.test.ts b/src/programs/warehouse-source/__tests__/ask-timeout.test.ts index 35ae56a0e..992ebaa8b 100644 --- a/src/programs/warehouse-source/__tests__/ask-timeout.test.ts +++ b/src/programs/warehouse-source/__tests__/ask-timeout.test.ts @@ -7,8 +7,8 @@ * The command was on the 5-minute default, so the fallback route gave the * user a quarter of the time the in-run prompt does for identical questions. */ +import { testRunnerContext } from '@programs/shared/__tests__/runner-context.no-jest'; import type { WizardSession } from '@programs/session/wizard-session'; -import { testRunnerContext } from '../../../../test/runner-context'; vi.mock('@utils/analytics', () => ({ analytics: { @@ -19,11 +19,11 @@ vi.mock('@utils/analytics', () => ({ }, })); -import { warehouseSourceConfig } from '@programs/warehouse-source/index'; +import { config as warehouseSource } from '@programs/warehouse-source'; import { - LONGER_ASK_TIMEOUT_MS, DEFAULT_ASK_TIMEOUT_MS, -} from '@agent/wizard-ask-bridge'; + LONGER_ASK_TIMEOUT_MS, +} from '@shared/ask-policy'; function session(): WizardSession { return { installDir: '/tmp/app', frameworkContext: {} } as WizardSession; @@ -31,7 +31,7 @@ function session(): WizardSession { describe('warehouse command ask timeout', () => { it('gives credential questions the shared allowance, not the default', async () => { - const { run } = warehouseSourceConfig; + const { run } = warehouseSource; const resolved = typeof run === 'function' ? await run(session(), testRunnerContext()) diff --git a/src/programs/warehouse-source/__tests__/reporting.test.ts b/src/programs/warehouse-source/__tests__/reporting.test.ts new file mode 100644 index 000000000..173a744a8 --- /dev/null +++ b/src/programs/warehouse-source/__tests__/reporting.test.ts @@ -0,0 +1,56 @@ +/** The standalone `wizard warehouse` command's detection, and the integration flow's scan report. */ + +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; + +vi.mock(import('@utils/analytics'), () => ({ + analytics: { + wizardCapture: vi.fn(), + setTag: vi.fn(), + capture: vi.fn(), + captureException: vi.fn(), + } as never, +})); + +import { analytics } from '@utils/analytics'; +import { reportWarehouseSourcesDetected } from '@programs/detection/integration'; +import { DETECTED_WAREHOUSE_SOURCES_KEY } from '@programs/warehouse-sources/detect'; +import { buildSession } from '@programs/session/wizard-session'; +import { detectWarehousePrerequisites } from '../detect'; + +describe('reportWarehouseSourcesDetected', () => { + let tmpDir: string; + + beforeEach(() => { + vi.clearAllMocks(); + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'warehouse-reporting-')); + fs.writeFileSync( + path.join(tmpDir, 'package.json'), + JSON.stringify({ dependencies: { stripe: '^14.0.0' } }), + ); + }); + + afterEach(() => fs.rmSync(tmpDir, { recursive: true, force: true })); + + it('the standalone warehouse command does not report through this path', () => { + // `wizard warehouse` writes the same frameworkContext key from its own + // detect, and sets its own tags. Without a scan-state marker it would also + // emit this event, which six saved insights read as "the integration flow + // scanned". + const session = buildSession({ installDir: tmpDir, ci: true }); + detectWarehousePrerequisites(session, (key, value) => { + session.frameworkContext[key] = value; + }); + expect( + session.frameworkContext[DETECTED_WAREHOUSE_SOURCES_KEY], + ).toBeDefined(); + + reportWarehouseSourcesDetected(session); + + expect(analytics.wizardCapture).not.toHaveBeenCalledWith( + 'warehouse sources detected', + expect.anything(), + ); + }); +}); diff --git a/src/programs/web-analytics-doctor/__tests__/detect.test.ts b/src/programs/web-analytics-doctor/__tests__/detect.test.ts index 393b53012..a56f68c92 100644 --- a/src/programs/web-analytics-doctor/__tests__/detect.test.ts +++ b/src/programs/web-analytics-doctor/__tests__/detect.test.ts @@ -3,10 +3,10 @@ import * as path from 'path'; import * as os from 'os'; import { detectWebAnalyticsPrerequisites, - webAnalyticsDoctorConfig, + config as webAnalyticsDoctor, WEB_ANALYTICS_ABORT_CASES, -} from '@programs/web-analytics-doctor/index'; -import { WIZARD_TOOL_NAMES } from '@agent/tools'; +} from '@programs/web-analytics-doctor'; +import { WIZARD_TOOL_NAMES } from '@agent'; import { buildSession } from '@programs/session/wizard-session'; function makeTmpDir(): string { @@ -111,16 +111,16 @@ describe('WEB_ANALYTICS_ABORT_CASES', () => { }); }); -describe('webAnalyticsDoctorConfig', () => { +describe('web-analytics-doctor config', () => { it('keeps wizard_ask enabled so the user can pick which fixes to apply', () => { - expect(webAnalyticsDoctorConfig.disallowedTools ?? []).not.toContain( + expect(webAnalyticsDoctor.disallowedTools ?? []).not.toContain( WIZARD_TOOL_NAMES.wizardAsk, ); }); it('wires the web-analytics-doctor skill and CLI command', () => { - expect(webAnalyticsDoctorConfig.command).toBe('web-analytics'); - expect(webAnalyticsDoctorConfig.skillId).toBe('web-analytics-doctor'); - expect(webAnalyticsDoctorConfig.id).toBe('web-analytics-doctor'); + expect(webAnalyticsDoctor.command).toBe('web-analytics'); + expect(webAnalyticsDoctor.skillId).toBe('web-analytics-doctor'); + expect(webAnalyticsDoctor.id).toBe('web-analytics-doctor'); }); });