diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index d493e04f746..485461ce203 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -312,3 +312,12 @@ process journal accepts the API runner PID only after checking its OS start time process group, exact run ID argument and `per_turn` argument; it never treats a retained warm daemon as a leaked per-run process or signals an API-provided PID. Directory-watch coverage also rejects parent device/inode replacement or removal. + + +The manual `cursor-native` suite has four local Cursor candidate cells with +one expected provider run each and a 120-second provider deadline. Browser +choices, rejection feedback, native origin/decision receipts, workspace checks +and provider retirement are independently checked. Failed or missing cleanup +assertions fail the final report even if the native interaction passed. No +native callback, private-HOME artifact export, or paid qualification is inferred +from a semantic question or plan result. diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index b1c00504b2d..358dc35b8d9 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1304,3 +1304,22 @@ cells. Discover them with `pnpm test:e2e:runner -- --list --suite copilot-protec See [Copilot native protection](./FIXTURES.md#copilot-native-protection) for the expected cancelled negative test, finite attached-process oracle, evidence limits, and required rebuilt runtime. Registration is not a qualification claim. + + +### Cursor native interactions (candidate) + +The explicit-only local `cursor-native` suite exercises the native Cursor +question callback, plan rejection/revision/acceptance, plan cancellation, and +permission denial across browser reload. List it with +`pnpm test:e2e:runner -- --list --suite cursor-native`. The fixture configures +`acpxSessionMode` before the run: Plan for question/plan callbacks, Agent for +write denial. Native request origin, exact decision bytes and post-write +response delivery are required; semantic-tool interactions do not satisfy these +checks. The denial case also requires independent filesystem observations and +an API-bound process journal after provider retirement, before workspace +removal. Complete server and database cleanup remains the outer supervisor's +responsibility. These cases remain unqualified until paid runs pass. + +The full native plan text is retained in the interaction card. Exporting a plan +file from Cursor's private HOME as a downloadable Paperclip artifact remains a +separate capability gap; these callback tests do not claim that export works. diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 23b89ab4f98..47c50a68470 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -91,10 +91,10 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 3, 2, 30, 3, 16, 16, 2, 8, 46, 23, 47, 20, 52, 28, 18, 6, 6, 10, 48, 16, 10, 2, + 4, 3, 2, 30, 3, 16, 16, 2, 8, 46, 23, 47, 20, 52, 28, 18, 6, 6, 10, 48, 16, 10, 2, ]); - expect(validateRunnerCatalog()).toHaveLength(412); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(412); + expect(validateRunnerCatalog()).toHaveLength(416); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(416); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(48); diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index eca253443e0..2de8439e50b 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1,3 +1,4 @@ +import { cursorNativeTasks } from "./cursor-native-cases.js"; import { copilotProtectionTasks } from "./copilot-protection-tasks.js"; import { piNativeTasks } from "./pi-native-cases.js"; import { instructionPersistenceTask } from "./instruction-persistence.js"; @@ -1030,6 +1031,13 @@ export const extendedHarnessFileTask: RunnerTaskFixture = { }; export const runnerSuites: readonly RunnerSuiteFixture[] = [ + { + id: "cursor-native", label: "Cursor native interactions", manualOnly: true, + description: "Native question continuation, revision-bound plan decisions and restrictive permission denial with independent process and file evidence.", + groups: ["native"], profiles: extendedHarnessProfiles.filter(profile => profile.qualificationCandidate === "cursor"), + environments: [localEnvironment], tasks: cursorNativeTasks, expectedMatrixSize: 4, + definitionMetadata: { version: 1, qualification: "pending", scheduling: "explicit-only", profileVersion: QUALIFIED_ACPX_PROFILES.cursor.agentProfileVersion, modeAdmission: "native-config-ack", artifactExport: "pending-private-home" }, + }, { id: "pi-native", label: "Pi native boundaries", manualOnly: true, description: "Explicit local Pi native forms, registered agent files and restrictive permission qualification.", diff --git a/tests/runner-e2e/cleanup-verification.test.ts b/tests/runner-e2e/cleanup-verification.test.ts new file mode 100644 index 00000000000..00bd927c1ce --- /dev/null +++ b/tests/runner-e2e/cleanup-verification.test.ts @@ -0,0 +1,23 @@ +import { expect, it } from "vitest"; +import { verifyCleanupAssertions } from "./cleanup-verification.js"; + +it("retains a failed cleanup proof and still closes every later observer", async () => { + let closed = 0; + const result = await verifyCleanupAssertions([ + async () => { throw new Error("provider still alive"); }, + async () => [{ id: "no-effect", passed: false, detail: "target changed" }], + async () => { closed++; return [{ id: "observer-closed", passed: true, detail: "closed" }]; }, + ]); + expect(closed).toBe(1); + expect(result.errors).toHaveLength(2); + expect(result.checks).toEqual(expect.arrayContaining([ + expect.objectContaining({ id: "no-effect", passed: false }), + expect.objectContaining({ id: "observer-closed", passed: true }), + ])); +}); + +it("refuses an assertion that silently omits its proof", async () => { + const result = await verifyCleanupAssertions([async () => []]); + expect(result.errors).toHaveLength(1); + expect(result.checks[0]?.passed).toBe(false); +}); diff --git a/tests/runner-e2e/cleanup-verification.ts b/tests/runner-e2e/cleanup-verification.ts new file mode 100644 index 00000000000..cc78c32b020 --- /dev/null +++ b/tests/runner-e2e/cleanup-verification.ts @@ -0,0 +1,22 @@ +export interface CleanupCheck { id: string; passed: boolean; detail: string } +export type CleanupAssertion = () => Promise; + +/** Always finish every registered observer, even when an earlier proof fails. */ +export async function verifyCleanupAssertions(assertions: readonly CleanupAssertion[]) { + const checks: CleanupCheck[] = [], errors: unknown[] = []; + for (const [index, assertion] of assertions.entries()) { + try { + const result = await assertion(); + if (!Array.isArray(result) || result.length === 0 || result.some(check => + !check || typeof check.id !== "string" || !check.id || typeof check.passed !== "boolean" || typeof check.detail !== "string")) { + throw new Error("Cleanup assertion did not return complete checks"); + } + checks.push(...result); + if (result.some(check => !check.passed)) throw new Error("Cleanup assertion reported a failed check"); + } catch (error) { + errors.push(error); + checks.push({ id: `assertion-${index}-completed`, passed: false, detail: error instanceof Error ? error.message : "Cleanup assertion failed" }); + } + } + return { checks, errors }; +} diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index 46794ece1cb..da4e58d185c 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -1,3 +1,5 @@ +import { runCursorNativeFlow } from "./cursor-native-flow.js"; +import { verifyCleanupAssertions, type CleanupAssertion } from "./cleanup-verification.js"; import { runCopilotProtectionFlow } from "./copilot-protection-flow.js"; import { runPiNativeFlow } from "./pi-native-flow.js"; import { completionQualityControls, completionQualityStatus, judgeCompletionQuality, reserveCompletionQuality, type CompletionQualityRecord } from "./completion-quality.js"; @@ -551,7 +553,7 @@ for (const execution of executions) { const credentials = credentialValues(); const secrets = normalizedSecrets(Object.values(credentials)); const api = new RunnerApi(request); - const companyRunFlow = ["continuation_accounting", "continuation", "context_integrity", "agent_chat", "everyday_workflow", "first_task", "instruction_persistence", "pi_native", "copilot_protection"].includes(execution.task.flow); + const companyRunFlow = ["continuation_accounting", "continuation", "context_integrity", "agent_chat", "everyday_workflow", "first_task", "instruction_persistence", "pi_native", "copilot_protection", "cursor_native"].includes(execution.task.flow); const consoleDiagnostics: Array> = []; const networkDiagnostics: Array> = []; const pageLifecycleDiagnostics: Array> = []; @@ -594,6 +596,7 @@ for (const execution of executions) { let primaryError: unknown; let failureClassOverride: FailureClass | undefined; let cleanup: RunnerE2EResult["cleanup"] = "not_started"; + const cleanupAssertions: CleanupAssertion[] = []; const capturePrivateScreenshot = async (id: string, file: string) => { const screenshotPath = path.join(privateDir, file); @@ -888,6 +891,19 @@ for (const execution of executions) { evidence: (name, data) => writeSanitizedJson(snapshotsDir, name, data, secrets), }); issue = accounting.issue as IssueRecord; selectedRuns = accounting.runs as RunRecord[]; + } else if (execution.task.flow === "cursor_native") { + const story = await runCursorNativeFlow({ + page, api, fixtures, execution, nonce, workspacePath, deadlineAt: startedAtMs + deadlineMs, + observe: (currentIssue, currentRuns) => { issue = currentIssue as IssueRecord; selectedRuns = currentRuns as RunRecord[]; }, + capture: captureScreenshot, + evidence: (name, data) => writeSanitizedJson(snapshotsDir, name, data, secrets), + registerCleanupAssertion: assertion => { + if (cleanupAssertions.length >= 8) throw new Error("Cleanup assertion bound exceeded"); + cleanupAssertions.push(assertion); + }, + }); + issue = story.issue as IssueRecord; selectedRuns = story.runs as RunRecord[]; + matcherResults = story.checks.map(check => ({ matcher: { kind: "json_path" as const, path: `cursorNative.${check.id}`, expected: true }, passed: check.passed, detail: check.detail })); } else if (execution.task.flow === "pi_native") { const story = await runPiNativeFlow({ page, api, fixtures, execution, nonce, workspacePath, deadlineAt: startedAtMs + deadlineMs, @@ -2690,6 +2706,23 @@ for (const execution of executions) { : "passed"; } + // Local fixture teardown does not retire the API server. These assertions + // independently prove the recorded provider processes retired, while the + // workspace still exists, before final grading. The outer supervisor + // remains responsible for complete server/database cleanup. + if (cleanupAssertions.length > 0) { + const verification = await verifyCleanupAssertions(cleanupAssertions); + matcherResults.push(...verification.checks.map(check => ({ + matcher: { kind: "json_path" as const, path: `cleanup.${check.id}`, expected: true }, + passed: check.passed, detail: check.detail, + }))); + if (verification.errors.length > 0) { + cleanup = "failed"; + primaryError = new AggregateError([primaryError, ...verification.errors].filter(Boolean), "Cleanup verification failed after provider settlement"); + if (failureClassOverride !== "secret_leak") failureClassOverride = "cleanup_failure"; + } + } + const finishedAtMs = Date.now(); const runtimeUsage = buildRuntimeUsage({ environmentId: execution.environment.id, diff --git a/tests/runner-e2e/types.ts b/tests/runner-e2e/types.ts index 4b802ad571d..339840e8d3c 100644 --- a/tests/runner-e2e/types.ts +++ b/tests/runner-e2e/types.ts @@ -30,7 +30,8 @@ export type RunnerTaskFlow = | "warm_three_turn" | "instruction_persistence" | "pi_native" - | "copilot_protection"; + | "copilot_protection" + | "cursor_native"; export interface SecretReference { type: "secret_ref";