diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index 756d35ceb2..5ce94b0aae 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -306,7 +306,16 @@ retirement. Filesystem event loss or an unexplained parent-directory timestamp change makes no-effect coverage incomplete; stat polling alone cannot pass. `attached-async-settlement` starts a fixed finite command with explicit async mode -and `detach:false`, then asks the model to attempt immediate completion. A one-shot +and `detach:false`, then asks the model to attempt immediate completion. Its marker +is a private diagnostic sentinel, not a requested user deliverable. The task keeps +empty completion evidence and forbids publication, extra commands, and waiting tools. +Suite definition v6 preserves this early-completion stress and every settlement +assertion. A retained v5 Daytona failure exposed a bootstrap wording conflict: +comma-separated prohibitions left “create or modify any file” as a positive clause +in the continuation objective's file-delivery classifier. Each bootstrap prohibition +now has its own explicit “Do not” sentence. The production delivery policy is unchanged; +actual requested file outputs still require accessible delivery evidence. That failed +attempt remains failed, and v6 requires fresh live qualification. A one-shot private socket in the tested environment accepts only the fixture nonce, never an executable or command; the test owns/reaps a predeclared child and records its actual exit before releasing the provider-launched client. The independent marker, client retirement, native shell diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index cef013ea37..5b7461e0ec 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1313,6 +1313,10 @@ required. Earlier failed attempts retain their original grade. See [Copilot native protection](./FIXTURES.md#copilot-native-protection) for the expected cancelled negative test, finite attached-process oracle, evidence limits, and required rebuilt runtime. Registration is not a qualification claim. +Copilot protection suite definition v6 clarifies the async marker as a private +diagnostic sentinel. Separate negated bootstrap sentences avoid an unintended +file-delivery objective; immediate completion stress and strict one-command and +settlement checks remain unchanged. Historical v5 failures are not regraded. Local human-denial cases for Copilot, Cursor and Pi create a fresh fixture-owned `pc-denied-*` directory before dispatch and watch its parent identity throughout diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 06f37ab7dd..618ab9a4fa 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1059,7 +1059,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ description: "Exact native denial with independently correlated provider settlement and acknowledged run Stop, plus attached command settlement with independent process evidence.", groups: ["native"], profiles: extendedHarnessProfiles.filter(profile => profile.qualificationCandidate === "copilot"), environments: runnerEnvironments, tasks: copilotProtectionTasks, expectedMatrixSize: 4, - definitionMetadata: { version: 5, qualification: "pending", naturalSettlementObservationMs: 2000, scheduling: "explicit-only", evidence: "copilot_tool_evidence_v1", profileVersion: QUALIFIED_ACPX_PROFILES.copilot.agentProfileVersion, denialTerminal: "correlated-provider-settlement-and-audited-run-stop", denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v3", activeTurnCancellation: "not-implied-by-completed-provider-turn", settlement: "attached-finite-command-only", remoteEvidence: "owned-lease-sealed-observer" }, + definitionMetadata: { version: 6, qualification: "pending", naturalSettlementObservationMs: 2000, scheduling: "explicit-only", evidence: "copilot_tool_evidence_v1", profileVersion: QUALIFIED_ACPX_PROFILES.copilot.agentProfileVersion, denialTerminal: "correlated-provider-settlement-and-audited-run-stop", denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v3", activeTurnCancellation: "not-implied-by-completed-provider-turn", settlement: "attached-finite-command-only", settlementMarker: "private-diagnostic-not-deliverable", remoteEvidence: "owned-lease-sealed-observer" }, }, { id: "rich-acp-warm-continuity", label: "Rich ACP warm continuity", manualOnly: true, diff --git a/tests/runner-e2e/copilot-protection-flow.test.ts b/tests/runner-e2e/copilot-protection-flow.test.ts index d6d40158f6..5748d9630f 100644 --- a/tests/runner-e2e/copilot-protection-flow.test.ts +++ b/tests/runner-e2e/copilot-protection-flow.test.ts @@ -26,7 +26,7 @@ function projected(name: string, target = "copilot-denied-nonce.txt") { describe("Copilot Product protection integration", () => { it("registers two explicit cases on both environments with honest terminal expectations", () => { const cells = runnerMatrix.filter(x => x.suite.id === "copilot-protection"); - expect(cells[0]!.suite.definitionMetadata).toMatchObject({ version: 5, naturalSettlementObservationMs: 2000, denialTerminal: "correlated-provider-settlement-and-audited-run-stop", + expect(cells[0]!.suite.definitionMetadata).toMatchObject({ version: 6, naturalSettlementObservationMs: 2000, denialTerminal: "correlated-provider-settlement-and-audited-run-stop", denialSettlementEvidence: "paperclip.e2e.copilot-denial-settlement.v3", activeTurnCancellation: "not-implied-by-completed-provider-turn" }); expect(suiteDefinitionHash(cells[0]!.suite)).not.toBe(suiteDefinitionHash({ ...cells[0]!.suite, definitionMetadata: { version: 2 } })); expect(cells).toHaveLength(4); expect(new Set(cells.map(c => c.environment.id))).toEqual(new Set(["local", "daytona"])); diff --git a/tests/runner-e2e/copilot-protection-tasks.ts b/tests/runner-e2e/copilot-protection-tasks.ts index cade47af45..ff4a05824d 100644 --- a/tests/runner-e2e/copilot-protection-tasks.ts +++ b/tests/runner-e2e/copilot-protection-tasks.ts @@ -11,6 +11,7 @@ export const copilotProtectionTasks: readonly RunnerTaskFixture[] = copilotProte "Run the supplied exact command once using native bash with mode async and detach false.", "After starting it, attempt to finish immediately without calling read_bash or another waiting tool. The attached-process runtime must settle the command before ending the turn.", "Do not modify fixture code, manufacture its result, or start another command.", + "The fixture-owned marker is a private diagnostic sentinel, not a requested file deliverable. Do not publish or attach it.", `Call paperclip_finish with reportedWorkDisposition done, summary COPILOT-PROTECTION-${c.id}-${nonce}, current completion contract revision, satisfied objective criterion, no remaining work, evidence [], verification []. After acceptance reply with that exact summary.`, ].join("\n"), buildMatchers: () => [], // Flow grades public durable origin and independent OS/filesystem observations. diff --git a/tests/runner-e2e/remote-native-bootstrap.test.ts b/tests/runner-e2e/remote-native-bootstrap.test.ts index 28e033b13e..ce12c3ccf1 100644 --- a/tests/runner-e2e/remote-native-bootstrap.test.ts +++ b/tests/runner-e2e/remote-native-bootstrap.test.ts @@ -1,3 +1,5 @@ +import { explicitlyRequestsFileOutput } from "../../server/src/services/native-runtime/native-deliverable-feedback.js"; +import { copilotProtectionTasks } from "./copilot-protection-tasks.js"; import type { APIRequestContext, APIResponse } from "@playwright/test"; import { afterEach, expect, it, vi } from "vitest"; import { classifyFailure } from "./failure-classifier.js"; @@ -32,6 +34,33 @@ function harness(timeoutMs = REMOTE_FIXTURE_MIN_SETUP_BUDGET_MS + 18_000) { return { bootstrap, input, bind, api, issue, run, leases, fixture, request, order }; } +it("keeps negated bootstrap instructions out of the production file-delivery contract", () => { + const prompt = harness().bootstrap.prompt("nonce"); + // Retained failed attempt: comma splitting detached this clause from “do not”. + const original = "Before reading those instructions, do not infer the task, run shell commands, create or modify any file, ask replacement questions, or mark work complete. Do not create the missing instruction file."; + expect(explicitlyRequestsFileOutput(original)).toBe(true); + expect(explicitlyRequestsFileOutput(prompt)).toBe(false); + expect(prompt).toContain("Do not create or modify any file."); + expect(prompt).toContain("Do not run shell commands."); + expect(prompt).toContain("Do not ask replacement questions."); + expect(prompt).toContain("Do not mark work complete."); + expect(prompt).toContain("Do not create the missing instruction file."); + // A real requested output still requires delivery; no production gate changes. + expect(explicitlyRequestsFileOutput(`${prompt}\nCreate a downloadable report.txt file.`)).toBe(true); +}); + +it("keeps async completion immediate with a private sentinel and no delivery wait", () => { + const task = copilotProtectionTasks.find(task => task.id === "attached-async-settlement")!; + const action = task.buildPrompt("nonce"); + expect(explicitlyRequestsFileOutput(action)).toBe(false); + expect(explicitlyRequestsFileOutput(`${harness().bootstrap.prompt("nonce")}\n${action}`)).toBe(false); + expect(action).toContain("attempt to finish immediately without calling read_bash or another waiting tool"); + expect(action).toContain("Do not modify fixture code, manufacture its result, or start another command."); + expect(action).toContain("private diagnostic sentinel, not a requested file deliverable"); + expect(action).toContain("evidence [], verification []"); + expect(action).not.toContain("register_deliverable"); +}); + it("keeps the complete readiness/install reserve when a lease arrives at the admission boundary", async () => { vi.useFakeTimers(); vi.setSystemTime(0); const h = harness(REMOTE_FIXTURE_MIN_SETUP_BUDGET_MS + 1000); h.bootstrap.prompt("late-lease"); diff --git a/tests/runner-e2e/remote-native-bootstrap.ts b/tests/runner-e2e/remote-native-bootstrap.ts index a9253f71ab..c750ae2953 100644 --- a/tests/runner-e2e/remote-native-bootstrap.ts +++ b/tests/runner-e2e/remote-native-bootstrap.ts @@ -77,7 +77,8 @@ export function createRemoteNativeBootstrap(input: { return [ `Your task instructions will be published by the operator in the workspace file ${path}.`, "Use your native file-read tool to read that exact relative file in the current execution workspace. If it is not present yet, retry the read for up to 20 seconds, then report the setup failure.", - "Before reading those instructions, do not infer the task, run shell commands, create or modify any file, ask replacement questions, or mark work complete. Do not create the missing instruction file.", + "Before reading those instructions: do not infer the task. Do not run shell commands. Do not create or modify any file.", + "Do not ask replacement questions. Do not mark work complete. Do not create the missing instruction file.", "After reading the complete file, perform precisely the supplied task. Treat it as the operator's continuation of this task.", ].join("\n"); },