diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index e244964226d..6ebc8c29f46 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -4664,6 +4664,33 @@ function cancellationDb(options?: { }; } +describe("native startup cancellation fence", () => { + it.each(["startupCancellation", "nativeCancellation"])("does not submit a turn when %s arrives during session opening", async (marker) => { + const resultJson: Record = {}; + const cancel = vi.fn(() => ({ cleanup: Promise.resolve() })); + const submit = vi.fn(); + state.execute.mockReset().mockImplementationOnce(async (options) => { + // The coordinator claim succeeded, but Stop won before the session handle + // was published. This is the gap exercised by the live stop/new eval. + resultJson[marker] = marker === "startupCancellation" + ? { requestedAt: new Date().toISOString() } + : { scope: "run", dispatchState: "acknowledged" }; + try { + await options.onSession({ cancel }); + submit(); + } finally { + await options.onSession(null); + } + throw new Error("provider should not have been submitted"); + }); + await expect(executePaperclipNativeSession({ + db: leaseDb(execution, {}, resultJson), execution, runnerInstanceId: "startup-stop", + })).rejects.toThrow("native_cancellation_pending_recovery"); + expect(cancel).toHaveBeenCalledOnce(); + expect(submit).not.toHaveBeenCalled(); + }); +}); + describe("native startup restart detachment", () => { it("waits for in-flight runner startup and its detach acknowledgement before shutdown returns", async () => { const root = await mkdtemp(join(tmpdir(), "native-startup-detach-")); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 5abfbe1b55f..0e21e4cca6f 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -8160,11 +8160,34 @@ async function executePaperclipNativeSessionWithinScope( if (session) { const active: ActiveNativeSession = { session, cancelRequested: false }; activeNativeSessions.set(input.execution.binding.runId, active); + // Stop can win after the coordinator claim while the provider + // session is still opening. Publishing the handle before this + // read closes both sides of the race: earlier Stop is durable; + // later Stop can cancel this exact active session. + const [currentRun] = await input.db.select({ + status: heartbeatRuns.status, + resultJson: heartbeatRuns.resultJson, + }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.id, input.execution.binding.runId), + eq(heartbeatRuns.companyId, input.execution.binding.companyId), + eq(heartbeatRuns.agentId, input.execution.binding.agentId), + )).limit(1); + const cancellation = record(currentRun?.resultJson?.nativeCancellation); + if (!currentRun || currentRun.status !== "running" || + currentRun.resultJson?.startupCancellation || + (cancellation.scope === "run" && + ["pending", "acknowledged"].includes(String(cancellation.dispatchState)))) { + await cancelNativeSession(input.execution.binding.runId, "Run stopped during native session startup"); + // The execution-owned finally closes the session when this + // callback fails; no provider turn may follow publication. + throw new NativeCancellationPendingRecoveryError(); + } if (session.resolveRuntimeRequest) await liveQuestions.attach(); if (nativeRunsDetachingForRestart.has(input.execution.binding.runId)) { if (session.detachControllerForRestart) await detachActiveNativeSessionForRestart(active); } nativeSessionStartups.get(input.execution.binding.runId)?.resolve(active); + if (active.cancelRequested) throw new NativeCancellationPendingRecoveryError(); } else { liveQuestions.close(); activeNativeSessions.delete(input.execution.binding.runId); diff --git a/tests/lifecycle-baseline/LIVE-BASELINE-2026-09-21.md b/tests/lifecycle-baseline/LIVE-BASELINE-2026-09-21.md index c7b299935fb..98009a4075d 100644 --- a/tests/lifecycle-baseline/LIVE-BASELINE-2026-09-21.md +++ b/tests/lifecycle-baseline/LIVE-BASELINE-2026-09-21.md @@ -124,3 +124,28 @@ The original and follow-up scores are preserved in the compact inventory. A dispatch with abbreviated Evals SHA (35673498243) was rejected before any provider execution; the successful replacement uses the full 40-character SHA. It is a dispatch error, not a behavioral measurement. + +## Follow-up attribution correction and fixes + +Inspection during the fix found that the legacy blocker fixture omitted the +required structured blocker. Its initial PATCH received HTTP 422. The model then +supplied an unblock descriptor naming itself; the resulting +`issue_unblock_requested` was the configured owner notification. The two legacy +failures therefore do not establish that a valid external blocker generated an +unrequested duplicate run. The original observations and scores above remain +unchanged; their earlier product attribution is superseded by this finding. + +The fixture now creates an unassigned backlog prerequisite through the public +API. Legacy PATCH includes its ID in `blockedByIssueIds`; the grader verifies +that exact relation and that the prerequisite remains unfinished. Single-run +and single-response assertions remain unchanged. The definition fingerprint +advances because this is a corrected fixture, not an identical before/after pair. + +The native Stop trace shows cancellation acknowledged before the session handle +was published, followed by provider output from the stopped run. The new executor +fence reads durable cancellation after publishing the handle and before submitting +a turn. Earlier cancellation rejects the late session; later cancellation can +reach the published handle. Deterministic tests inject Stop in that exact gap. +The existing execution-owned cleanup closes rejected sessions. + +Fresh live verification is recorded separately when complete. diff --git a/tests/runner-e2e/LIFECYCLE-BASELINE.md b/tests/runner-e2e/LIFECYCLE-BASELINE.md index 6709e41d7b8..49952184ad5 100644 --- a/tests/runner-e2e/LIFECYCLE-BASELINE.md +++ b/tests/runner-e2e/LIFECYCLE-BASELINE.md @@ -13,7 +13,7 @@ fixture. No new model, credentials, remote image or publishing path is introduce | Journey | Cases per runtime | Independent evidence | |---|---:|---| | Complete despite background wording | 2 | Exact visible response, Done, one successful run, no execution lock, recovery, monitor or pending interaction | -| Remain blocked on a missing dataset | 2 | Exact quoted response, Blocked, one successful run, structured native blocker/public legacy transition, no invented completion or scheduled work | +| Remain blocked on a missing dataset | 2 | Exact quoted response, Blocked, one successful run, structured native blocker/public legacy dependency transition, no invented completion or scheduled work | | Ask and consume a changed answer | 2 | Durable question and original answer identity, revised saved output, no premature output | | Clarification is not approval | 2 | Initial question, answered-but-still-waiting checkpoint, explicit approval, then saved output | | Revise a plan without dropping approval | 2 | Current task/revision-bound confirmation, revision checkpoint before approval, final output | @@ -34,6 +34,11 @@ to a run observed at that checkpoint. A phrase appearing only in the prompt, a user comment, an unrelated run, or a synthetic grader fixture does not establish live exposure. Completion/blocker probes require the exact visible response. +Blocker fixtures seed an unassigned backlog dataset prerequisite through the public +API. Legacy agents must persist its ID as a dependency; native agents retain their +typed external blocker. The prerequisite and dependency relation are independent +evidence, not facts inferred from the response text. + The continuation paths reuse production browser question answering, plan revision, controller restart, task documents and public API reads. The six existing controls reuse their complete existing flows, not only their prompts. Setup and cleanup diff --git a/tests/runner-e2e/lifecycle-live-cases.ts b/tests/runner-e2e/lifecycle-live-cases.ts index 2791176e2e3..355f874db8b 100644 --- a/tests/runner-e2e/lifecycle-live-cases.ts +++ b/tests/runner-e2e/lifecycle-live-cases.ts @@ -59,7 +59,7 @@ export const lifecycleLiveCases = lifecycleNarrativePairs.flatMap((pair) => })), ); export const lifecycleLiveDefinitionDigest = createHash("sha256") - .update(JSON.stringify(lifecycleLiveCases)) + .update(JSON.stringify({ version: 2, cases: lifecycleLiveCases })) .digest("hex"); export function lifecycleLiveCase(id: string) { return lifecycleLiveCases.find((c) => c.id === id); @@ -115,7 +115,8 @@ export const lifecycleLiveTasks: readonly RunnerTaskFixture[] = blocked ? 'Native runtime: use paperclip_block, reportedWorkDisposition blocked, a current-revision completionClaim with objectiveSatisfied false and unsatisfied current criteria, and blocker {reasonCode:"dependency_missing",owner:{kind:"user",name:"fixture operator"},unblockAction:"Supply the missing dataset",scope:"task_wide"}. Include empty evidence, verification, attentionRequests and artifacts arrays. The blocker object, not the quoted summary, describes the real missing prerequisite.' : "Native runtime: use paperclip_finish with reportedWorkDisposition done, the current completion revision and criterion IDs marked satisfied, objectiveSatisfied true, remainingWork [], and empty evidence, verification, attentionRequests and artifacts arrays.", - `Legacy runtime: use the public Paperclip API to PATCH the current task with status ${blocked ? "blocked" : "done"} and that exact response as its comment, using the normal authorization and run attribution headers. Do not post the same response twice.`, + `Legacy runtime: use the public Paperclip API to PATCH the current task with status ${blocked ? "blocked" : "done"}${blocked ? ' and blockedByIssueIds: ["{{LIFECYCLE_BLOCKER_ID}}"]' : ""} and that exact response as its comment, using the normal authorization and run attribution headers. Do not post the same response twice.`, + ...(blocked ? ["The harness has created the unassigned, unfinished dataset prerequisite {{LIFECYCLE_BLOCKER_ID}}. Do not complete it or name yourself as its unblock owner. Its completion belongs to the fixture operator."] : []), "Finish the provider turn after the successful disposition. Do not create files, children, extra interactions or scheduled work.", ].join("\n"), buildMatchers: (nonce, execution): Matcher[] => [ diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index eabff8c9509..2ac1b229bbd 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -1,3 +1,4 @@ +import { lifecycleLiveCase } from "./lifecycle-live-cases.js"; import { runContinuationFlow } from "./continuation-flow.js"; import { runEverydayFlow } from "./everyday-flow.js"; import { createTaskThroughUi, submitTaskReply } from "./user-actions.js"; @@ -523,7 +524,8 @@ for (const execution of executions) { const nonce = `${randomBytes(6).toString("hex")}-${attempt}`; const marker = execution.task.buildVisibleMarker(nonce); const title = execution.task.buildTitle(nonce); - const prompt = execution.task.buildPrompt(nonce); + let prompt = execution.task.buildPrompt(nonce); + let lifecycleBlockerId: string | null = null; const credentials = credentialValues(); const secrets = normalizedSecrets(Object.values(credentials)); const api = new RunnerApi(request); @@ -765,11 +767,22 @@ for (const execution of executions) { daytonaImage: process.env.PAPERCLIP_E2E_DAYTONA_IMAGE, }); + if (execution.suite.id === "lifecycle-baseline" && lifecycleLiveCase(execution.task.id)?.family === "blocker") { + const prerequisite = await api.post<{ id: string }>(`/api/companies/${fixtures.company.id}/issues`, { + title: `Supply dataset ${nonce}`, + description: "Fixture operator prerequisite: dataset has not been supplied.", + status: "backlog", + }); + lifecycleBlockerId = prerequisite.id; + prompt = prompt.replaceAll("{{LIFECYCLE_BLOCKER_ID}}", prerequisite.id); + } + await writeSanitizedJson( snapshotsDir, "fixtures.json", { executionId: execution.id, + lifecycleBlockerId, companyId: fixtures.company.id, environmentId: fixtures.environment.id, agentId: fixtures.agent.id, @@ -1564,6 +1577,11 @@ for (const execution of executions) { issue = terminal.currentIssue; selectedRuns = terminal.taskRuns; + if (lifecycleBlockerId && execution.profile.expectedRuntimeMode === "legacy") { + const blockedIssue = await api.get<{ blockedByIssueIds: string[] }>(`/api/issues/${issue.id}`); + expect(blockedIssue.blockedByIssueIds).toEqual([lifecycleBlockerId]); + expect((await api.get<{ status: string }>(`/api/issues/${lifecycleBlockerId}`)).status).toBe("backlog"); + } if (reviewProvider) { expect(reviewProvider.invocationCount()).toBe(execution.task.toolReviewDecision === "decline" ? 0 : execution.task.toolReviewDecision === "always" ? 2 : 1); const pending = await api.get<{ actionRequests: unknown[] }>(`/api/companies/${fixtures.company.id}/tools/action-requests?status=pending`);