mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-22 11:54:35 +01:00
test(runner): register native Cursor interactions and cleanup proof gates
Add four explicit candidate cells with mode-specific native callback evidence. Run independent cleanup assertions before grading and preserve failures while closing every observer. Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
96ca6ce1e7
commit
c22526c266
8 files changed
+120
-5
No files matched your search
@@ -312,3 +312,12 @@ process journal accepts the API runner PID only after checking its OS start time
|
||||
process group, exact run ID argument and `per_turn` argument; it never treats a
|
||||
retained warm daemon as a leaked per-run process or signals an API-provided PID.
|
||||
Directory-watch coverage also rejects parent device/inode replacement or removal.
|
||||
|
||||
|
||||
The manual `cursor-native` suite has four local Cursor candidate cells with
|
||||
one expected provider run each and a 120-second provider deadline. Browser
|
||||
choices, rejection feedback, native origin/decision receipts, workspace checks
|
||||
and provider retirement are independently checked. Failed or missing cleanup
|
||||
assertions fail the final report even if the native interaction passed. No
|
||||
native callback, private-HOME artifact export, or paid qualification is inferred
|
||||
from a semantic question or plan result.
|
||||
@@ -1304,3 +1304,22 @@ cells. Discover them with `pnpm test:e2e:runner -- --list --suite copilot-protec
|
||||
See [Copilot native protection](./FIXTURES.md#copilot-native-protection) for the
|
||||
expected cancelled negative test, finite attached-process oracle, evidence limits,
|
||||
and required rebuilt runtime. Registration is not a qualification claim.
|
||||
|
||||
|
||||
### Cursor native interactions (candidate)
|
||||
|
||||
The explicit-only local `cursor-native` suite exercises the native Cursor
|
||||
question callback, plan rejection/revision/acceptance, plan cancellation, and
|
||||
permission denial across browser reload. List it with
|
||||
`pnpm test:e2e:runner -- --list --suite cursor-native`. The fixture configures
|
||||
`acpxSessionMode` before the run: Plan for question/plan callbacks, Agent for
|
||||
write denial. Native request origin, exact decision bytes and post-write
|
||||
response delivery are required; semantic-tool interactions do not satisfy these
|
||||
checks. The denial case also requires independent filesystem observations and
|
||||
an API-bound process journal after provider retirement, before workspace
|
||||
removal. Complete server and database cleanup remains the outer supervisor's
|
||||
responsibility. These cases remain unqualified until paid runs pass.
|
||||
|
||||
The full native plan text is retained in the interaction card. Exporting a plan
|
||||
file from Cursor's private HOME as a downloadable Paperclip artifact remains a
|
||||
separate capability gap; these callback tests do not claim that export works.
|
||||
@@ -91,10 +91,10 @@ describe("runner E2E catalog", () => {
|
||||
expect(localIntegrityTasks).toHaveLength(2);
|
||||
expect(openRouterBreadthTasks).toHaveLength(3);
|
||||
expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([
|
||||
3, 2, 30, 3, 16, 16, 2, 8, 46, 23, 47, 20, 52, 28, 18, 6, 6, 10, 48, 16, 10, 2,
|
||||
4, 3, 2, 30, 3, 16, 16, 2, 8, 46, 23, 47, 20, 52, 28, 18, 6, 6, 10, 48, 16, 10, 2,
|
||||
]);
|
||||
expect(validateRunnerCatalog()).toHaveLength(412);
|
||||
expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(412);
|
||||
expect(validateRunnerCatalog()).toHaveLength(416);
|
||||
expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(416);
|
||||
expect(
|
||||
runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"),
|
||||
).toHaveLength(48);
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { cursorNativeTasks } from "./cursor-native-cases.js";
|
||||
import { copilotProtectionTasks } from "./copilot-protection-tasks.js";
|
||||
import { piNativeTasks } from "./pi-native-cases.js";
|
||||
import { instructionPersistenceTask } from "./instruction-persistence.js";
|
||||
@@ -1030,6 +1031,13 @@ export const extendedHarnessFileTask: RunnerTaskFixture = {
|
||||
};
|
||||
|
||||
export const runnerSuites: readonly RunnerSuiteFixture[] = [
|
||||
{
|
||||
id: "cursor-native", label: "Cursor native interactions", manualOnly: true,
|
||||
description: "Native question continuation, revision-bound plan decisions and restrictive permission denial with independent process and file evidence.",
|
||||
groups: ["native"], profiles: extendedHarnessProfiles.filter(profile => profile.qualificationCandidate === "cursor"),
|
||||
environments: [localEnvironment], tasks: cursorNativeTasks, expectedMatrixSize: 4,
|
||||
definitionMetadata: { version: 1, qualification: "pending", scheduling: "explicit-only", profileVersion: QUALIFIED_ACPX_PROFILES.cursor.agentProfileVersion, modeAdmission: "native-config-ack", artifactExport: "pending-private-home" },
|
||||
},
|
||||
{
|
||||
id: "pi-native", label: "Pi native boundaries", manualOnly: true,
|
||||
description: "Explicit local Pi native forms, registered agent files and restrictive permission qualification.",
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
import { expect, it } from "vitest";
|
||||
import { verifyCleanupAssertions } from "./cleanup-verification.js";
|
||||
|
||||
it("retains a failed cleanup proof and still closes every later observer", async () => {
|
||||
let closed = 0;
|
||||
const result = await verifyCleanupAssertions([
|
||||
async () => { throw new Error("provider still alive"); },
|
||||
async () => [{ id: "no-effect", passed: false, detail: "target changed" }],
|
||||
async () => { closed++; return [{ id: "observer-closed", passed: true, detail: "closed" }]; },
|
||||
]);
|
||||
expect(closed).toBe(1);
|
||||
expect(result.errors).toHaveLength(2);
|
||||
expect(result.checks).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ id: "no-effect", passed: false }),
|
||||
expect.objectContaining({ id: "observer-closed", passed: true }),
|
||||
]));
|
||||
});
|
||||
|
||||
it("refuses an assertion that silently omits its proof", async () => {
|
||||
const result = await verifyCleanupAssertions([async () => []]);
|
||||
expect(result.errors).toHaveLength(1);
|
||||
expect(result.checks[0]?.passed).toBe(false);
|
||||
});
|
||||
@@ -0,0 +1,22 @@
|
||||
export interface CleanupCheck { id: string; passed: boolean; detail: string }
|
||||
export type CleanupAssertion = () => Promise<CleanupCheck[]>;
|
||||
|
||||
/** Always finish every registered observer, even when an earlier proof fails. */
|
||||
export async function verifyCleanupAssertions(assertions: readonly CleanupAssertion[]) {
|
||||
const checks: CleanupCheck[] = [], errors: unknown[] = [];
|
||||
for (const [index, assertion] of assertions.entries()) {
|
||||
try {
|
||||
const result = await assertion();
|
||||
if (!Array.isArray(result) || result.length === 0 || result.some(check =>
|
||||
!check || typeof check.id !== "string" || !check.id || typeof check.passed !== "boolean" || typeof check.detail !== "string")) {
|
||||
throw new Error("Cleanup assertion did not return complete checks");
|
||||
}
|
||||
checks.push(...result);
|
||||
if (result.some(check => !check.passed)) throw new Error("Cleanup assertion reported a failed check");
|
||||
} catch (error) {
|
||||
errors.push(error);
|
||||
checks.push({ id: `assertion-${index}-completed`, passed: false, detail: error instanceof Error ? error.message : "Cleanup assertion failed" });
|
||||
}
|
||||
}
|
||||
return { checks, errors };
|
||||
}
|
||||
@@ -1,3 +1,5 @@
|
||||
import { runCursorNativeFlow } from "./cursor-native-flow.js";
|
||||
import { verifyCleanupAssertions, type CleanupAssertion } from "./cleanup-verification.js";
|
||||
import { runCopilotProtectionFlow } from "./copilot-protection-flow.js";
|
||||
import { runPiNativeFlow } from "./pi-native-flow.js";
|
||||
import { completionQualityControls, completionQualityStatus, judgeCompletionQuality, reserveCompletionQuality, type CompletionQualityRecord } from "./completion-quality.js";
|
||||
@@ -551,7 +553,7 @@ for (const execution of executions) {
|
||||
const credentials = credentialValues();
|
||||
const secrets = normalizedSecrets(Object.values(credentials));
|
||||
const api = new RunnerApi(request);
|
||||
const companyRunFlow = ["continuation_accounting", "continuation", "context_integrity", "agent_chat", "everyday_workflow", "first_task", "instruction_persistence", "pi_native", "copilot_protection"].includes(execution.task.flow);
|
||||
const companyRunFlow = ["continuation_accounting", "continuation", "context_integrity", "agent_chat", "everyday_workflow", "first_task", "instruction_persistence", "pi_native", "copilot_protection", "cursor_native"].includes(execution.task.flow);
|
||||
const consoleDiagnostics: Array<Record<string, unknown>> = [];
|
||||
const networkDiagnostics: Array<Record<string, unknown>> = [];
|
||||
const pageLifecycleDiagnostics: Array<Record<string, unknown>> = [];
|
||||
@@ -594,6 +596,7 @@ for (const execution of executions) {
|
||||
let primaryError: unknown;
|
||||
let failureClassOverride: FailureClass | undefined;
|
||||
let cleanup: RunnerE2EResult["cleanup"] = "not_started";
|
||||
const cleanupAssertions: CleanupAssertion[] = [];
|
||||
|
||||
const capturePrivateScreenshot = async (id: string, file: string) => {
|
||||
const screenshotPath = path.join(privateDir, file);
|
||||
@@ -888,6 +891,19 @@ for (const execution of executions) {
|
||||
evidence: (name, data) => writeSanitizedJson(snapshotsDir, name, data, secrets),
|
||||
});
|
||||
issue = accounting.issue as IssueRecord; selectedRuns = accounting.runs as RunRecord[];
|
||||
} else if (execution.task.flow === "cursor_native") {
|
||||
const story = await runCursorNativeFlow({
|
||||
page, api, fixtures, execution, nonce, workspacePath, deadlineAt: startedAtMs + deadlineMs,
|
||||
observe: (currentIssue, currentRuns) => { issue = currentIssue as IssueRecord; selectedRuns = currentRuns as RunRecord[]; },
|
||||
capture: captureScreenshot,
|
||||
evidence: (name, data) => writeSanitizedJson(snapshotsDir, name, data, secrets),
|
||||
registerCleanupAssertion: assertion => {
|
||||
if (cleanupAssertions.length >= 8) throw new Error("Cleanup assertion bound exceeded");
|
||||
cleanupAssertions.push(assertion);
|
||||
},
|
||||
});
|
||||
issue = story.issue as IssueRecord; selectedRuns = story.runs as RunRecord[];
|
||||
matcherResults = story.checks.map(check => ({ matcher: { kind: "json_path" as const, path: `cursorNative.${check.id}`, expected: true }, passed: check.passed, detail: check.detail }));
|
||||
} else if (execution.task.flow === "pi_native") {
|
||||
const story = await runPiNativeFlow({
|
||||
page, api, fixtures, execution, nonce, workspacePath, deadlineAt: startedAtMs + deadlineMs,
|
||||
@@ -2690,6 +2706,23 @@ for (const execution of executions) {
|
||||
: "passed";
|
||||
}
|
||||
|
||||
// Local fixture teardown does not retire the API server. These assertions
|
||||
// independently prove the recorded provider processes retired, while the
|
||||
// workspace still exists, before final grading. The outer supervisor
|
||||
// remains responsible for complete server/database cleanup.
|
||||
if (cleanupAssertions.length > 0) {
|
||||
const verification = await verifyCleanupAssertions(cleanupAssertions);
|
||||
matcherResults.push(...verification.checks.map(check => ({
|
||||
matcher: { kind: "json_path" as const, path: `cleanup.${check.id}`, expected: true },
|
||||
passed: check.passed, detail: check.detail,
|
||||
})));
|
||||
if (verification.errors.length > 0) {
|
||||
cleanup = "failed";
|
||||
primaryError = new AggregateError([primaryError, ...verification.errors].filter(Boolean), "Cleanup verification failed after provider settlement");
|
||||
if (failureClassOverride !== "secret_leak") failureClassOverride = "cleanup_failure";
|
||||
}
|
||||
}
|
||||
|
||||
const finishedAtMs = Date.now();
|
||||
const runtimeUsage = buildRuntimeUsage({
|
||||
environmentId: execution.environment.id,
|
||||
|
||||
@@ -30,7 +30,8 @@ export type RunnerTaskFlow =
|
||||
| "warm_three_turn"
|
||||
| "instruction_persistence"
|
||||
| "pi_native"
|
||||
| "copilot_protection";
|
||||
| "copilot_protection"
|
||||
| "cursor_native";
|
||||
|
||||
export interface SecretReference {
|
||||
type: "secret_ref";
|
||||
|
||||
Reference in new issue
Block a user