test(runner): register native Cursor interactions and cleanup proof gates

Add four explicit candidate cells with mode-specific native callback evidence. Run independent cleanup assertions before grading and preserve failures while closing every observer.

Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-09-29 16:41:43 -05:00
1 parent 96ca6ce1e7
commit c22526c266
8 files changed
+120 -5

No files matched your search

+9
View File
@@ -312,3 +312,12 @@ process journal accepts the API runner PID only after checking its OS start time
process group, exact run ID argument and `per_turn` argument; it never treats a
retained warm daemon as a leaked per-run process or signals an API-provided PID.
Directory-watch coverage also rejects parent device/inode replacement or removal.
The manual `cursor-native` suite has four local Cursor candidate cells with
one expected provider run each and a 120-second provider deadline. Browser
choices, rejection feedback, native origin/decision receipts, workspace checks
and provider retirement are independently checked. Failed or missing cleanup
assertions fail the final report even if the native interaction passed. No
native callback, private-HOME artifact export, or paid qualification is inferred
from a semantic question or plan result.
+19
View File
@@ -1304,3 +1304,22 @@ cells. Discover them with `pnpm test:e2e:runner -- --list --suite copilot-protec
See [Copilot native protection](./FIXTURES.md#copilot-native-protection) for the
expected cancelled negative test, finite attached-process oracle, evidence limits,
and required rebuilt runtime. Registration is not a qualification claim.
### Cursor native interactions (candidate)
The explicit-only local `cursor-native` suite exercises the native Cursor
question callback, plan rejection/revision/acceptance, plan cancellation, and
permission denial across browser reload. List it with
`pnpm test:e2e:runner -- --list --suite cursor-native`. The fixture configures
`acpxSessionMode` before the run: Plan for question/plan callbacks, Agent for
write denial. Native request origin, exact decision bytes and post-write
response delivery are required; semantic-tool interactions do not satisfy these
checks. The denial case also requires independent filesystem observations and
an API-bound process journal after provider retirement, before workspace
removal. Complete server and database cleanup remains the outer supervisor's
responsibility. These cases remain unqualified until paid runs pass.
The full native plan text is retained in the interaction card. Exporting a plan
file from Cursor's private HOME as a downloadable Paperclip artifact remains a
separate capability gap; these callback tests do not claim that export works.
+3 -3
View File
@@ -91,10 +91,10 @@ describe("runner E2E catalog", () => {
expect(localIntegrityTasks).toHaveLength(2);
expect(openRouterBreadthTasks).toHaveLength(3);
expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([
3, 2, 30, 3, 16, 16, 2, 8, 46, 23, 47, 20, 52, 28, 18, 6, 6, 10, 48, 16, 10, 2,
4, 3, 2, 30, 3, 16, 16, 2, 8, 46, 23, 47, 20, 52, 28, 18, 6, 6, 10, 48, 16, 10, 2,
]);
expect(validateRunnerCatalog()).toHaveLength(412);
expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(412);
expect(validateRunnerCatalog()).toHaveLength(416);
expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(416);
expect(
runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"),
).toHaveLength(48);
+8
View File
@@ -1,3 +1,4 @@
import { cursorNativeTasks } from "./cursor-native-cases.js";
import { copilotProtectionTasks } from "./copilot-protection-tasks.js";
import { piNativeTasks } from "./pi-native-cases.js";
import { instructionPersistenceTask } from "./instruction-persistence.js";
@@ -1030,6 +1031,13 @@ export const extendedHarnessFileTask: RunnerTaskFixture = {
};
export const runnerSuites: readonly RunnerSuiteFixture[] = [
{
id: "cursor-native", label: "Cursor native interactions", manualOnly: true,
description: "Native question continuation, revision-bound plan decisions and restrictive permission denial with independent process and file evidence.",
groups: ["native"], profiles: extendedHarnessProfiles.filter(profile => profile.qualificationCandidate === "cursor"),
environments: [localEnvironment], tasks: cursorNativeTasks, expectedMatrixSize: 4,
definitionMetadata: { version: 1, qualification: "pending", scheduling: "explicit-only", profileVersion: QUALIFIED_ACPX_PROFILES.cursor.agentProfileVersion, modeAdmission: "native-config-ack", artifactExport: "pending-private-home" },
},
{
id: "pi-native", label: "Pi native boundaries", manualOnly: true,
description: "Explicit local Pi native forms, registered agent files and restrictive permission qualification.",
@@ -0,0 +1,23 @@
import { expect, it } from "vitest";
import { verifyCleanupAssertions } from "./cleanup-verification.js";
it("retains a failed cleanup proof and still closes every later observer", async () => {
let closed = 0;
const result = await verifyCleanupAssertions([
async () => { throw new Error("provider still alive"); },
async () => [{ id: "no-effect", passed: false, detail: "target changed" }],
async () => { closed++; return [{ id: "observer-closed", passed: true, detail: "closed" }]; },
]);
expect(closed).toBe(1);
expect(result.errors).toHaveLength(2);
expect(result.checks).toEqual(expect.arrayContaining([
expect.objectContaining({ id: "no-effect", passed: false }),
expect.objectContaining({ id: "observer-closed", passed: true }),
]));
});
it("refuses an assertion that silently omits its proof", async () => {
const result = await verifyCleanupAssertions([async () => []]);
expect(result.errors).toHaveLength(1);
expect(result.checks[0]?.passed).toBe(false);
});
+22
View File
@@ -0,0 +1,22 @@
export interface CleanupCheck { id: string; passed: boolean; detail: string }
export type CleanupAssertion = () => Promise<CleanupCheck[]>;
/** Always finish every registered observer, even when an earlier proof fails. */
export async function verifyCleanupAssertions(assertions: readonly CleanupAssertion[]) {
const checks: CleanupCheck[] = [], errors: unknown[] = [];
for (const [index, assertion] of assertions.entries()) {
try {
const result = await assertion();
if (!Array.isArray(result) || result.length === 0 || result.some(check =>
!check || typeof check.id !== "string" || !check.id || typeof check.passed !== "boolean" || typeof check.detail !== "string")) {
throw new Error("Cleanup assertion did not return complete checks");
}
checks.push(...result);
if (result.some(check => !check.passed)) throw new Error("Cleanup assertion reported a failed check");
} catch (error) {
errors.push(error);
checks.push({ id: `assertion-${index}-completed`, passed: false, detail: error instanceof Error ? error.message : "Cleanup assertion failed" });
}
}
return { checks, errors };
}
+34 -1
View File
@@ -1,3 +1,5 @@
import { runCursorNativeFlow } from "./cursor-native-flow.js";
import { verifyCleanupAssertions, type CleanupAssertion } from "./cleanup-verification.js";
import { runCopilotProtectionFlow } from "./copilot-protection-flow.js";
import { runPiNativeFlow } from "./pi-native-flow.js";
import { completionQualityControls, completionQualityStatus, judgeCompletionQuality, reserveCompletionQuality, type CompletionQualityRecord } from "./completion-quality.js";
@@ -551,7 +553,7 @@ for (const execution of executions) {
const credentials = credentialValues();
const secrets = normalizedSecrets(Object.values(credentials));
const api = new RunnerApi(request);
const companyRunFlow = ["continuation_accounting", "continuation", "context_integrity", "agent_chat", "everyday_workflow", "first_task", "instruction_persistence", "pi_native", "copilot_protection"].includes(execution.task.flow);
const companyRunFlow = ["continuation_accounting", "continuation", "context_integrity", "agent_chat", "everyday_workflow", "first_task", "instruction_persistence", "pi_native", "copilot_protection", "cursor_native"].includes(execution.task.flow);
const consoleDiagnostics: Array<Record<string, unknown>> = [];
const networkDiagnostics: Array<Record<string, unknown>> = [];
const pageLifecycleDiagnostics: Array<Record<string, unknown>> = [];
@@ -594,6 +596,7 @@ for (const execution of executions) {
let primaryError: unknown;
let failureClassOverride: FailureClass | undefined;
let cleanup: RunnerE2EResult["cleanup"] = "not_started";
const cleanupAssertions: CleanupAssertion[] = [];
const capturePrivateScreenshot = async (id: string, file: string) => {
const screenshotPath = path.join(privateDir, file);
@@ -888,6 +891,19 @@ for (const execution of executions) {
evidence: (name, data) => writeSanitizedJson(snapshotsDir, name, data, secrets),
});
issue = accounting.issue as IssueRecord; selectedRuns = accounting.runs as RunRecord[];
} else if (execution.task.flow === "cursor_native") {
const story = await runCursorNativeFlow({
page, api, fixtures, execution, nonce, workspacePath, deadlineAt: startedAtMs + deadlineMs,
observe: (currentIssue, currentRuns) => { issue = currentIssue as IssueRecord; selectedRuns = currentRuns as RunRecord[]; },
capture: captureScreenshot,
evidence: (name, data) => writeSanitizedJson(snapshotsDir, name, data, secrets),
registerCleanupAssertion: assertion => {
if (cleanupAssertions.length >= 8) throw new Error("Cleanup assertion bound exceeded");
cleanupAssertions.push(assertion);
},
});
issue = story.issue as IssueRecord; selectedRuns = story.runs as RunRecord[];
matcherResults = story.checks.map(check => ({ matcher: { kind: "json_path" as const, path: `cursorNative.${check.id}`, expected: true }, passed: check.passed, detail: check.detail }));
} else if (execution.task.flow === "pi_native") {
const story = await runPiNativeFlow({
page, api, fixtures, execution, nonce, workspacePath, deadlineAt: startedAtMs + deadlineMs,
@@ -2690,6 +2706,23 @@ for (const execution of executions) {
: "passed";
}
// Local fixture teardown does not retire the API server. These assertions
// independently prove the recorded provider processes retired, while the
// workspace still exists, before final grading. The outer supervisor
// remains responsible for complete server/database cleanup.
if (cleanupAssertions.length > 0) {
const verification = await verifyCleanupAssertions(cleanupAssertions);
matcherResults.push(...verification.checks.map(check => ({
matcher: { kind: "json_path" as const, path: `cleanup.${check.id}`, expected: true },
passed: check.passed, detail: check.detail,
})));
if (verification.errors.length > 0) {
cleanup = "failed";
primaryError = new AggregateError([primaryError, ...verification.errors].filter(Boolean), "Cleanup verification failed after provider settlement");
if (failureClassOverride !== "secret_leak") failureClassOverride = "cleanup_failure";
}
}
const finishedAtMs = Date.now();
const runtimeUsage = buildRuntimeUsage({
environmentId: execution.environment.id,
+2 -1
View File
@@ -30,7 +30,8 @@ export type RunnerTaskFlow =
| "warm_three_turn"
| "instruction_persistence"
| "pi_native"
| "copilot_protection";
| "copilot_protection"
| "cursor_native";
export interface SecretReference {
type: "secret_ref";