diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs index a093f564c43..67cada57c07 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs @@ -2127,7 +2127,9 @@ mod tests { assert_eq!(ready["warmAttachReady"], true); assert_eq!(ready["warmAttachBlockers"], json!([])); let blocked_states: [(&str, fn(&mut AcpxDurableState)); 4] = [ - ("durable_closed", |state| state.lifecycle = "closed".to_owned()), + ("durable_closed", |state| { + state.lifecycle = "closed".to_owned() + }), ("provider_exit_unconfirmed", |state| { state.lifecycle = "prepared".to_owned(); state.provider_exit_unconfirmed = true; diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index 527e8cd3180..063a57e1aa5 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -177,6 +177,9 @@ failed and cancelled runs must not disappear from campaign totals. the ordinary public APIs to seed source documents and blockers. Keep the answer out of the user's status/review request. Grade the exact source values, latest blocker, preserved task identities, worker-authored output, and real executions. +The status request asks for JSON so the grader can distinguish the current +blocker from a historical mention and compare active-run count separately from +task status. The request must not reveal those expected values. The lost-acknowledgement probe may interrupt only the fixture browser's own comment request after the real server has committed it. Retain its request ID and replay that same request through the public API after restarting the server. diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index fd05581c17f..29f5694a7bb 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -172,6 +172,10 @@ Restart continuity requires the agent to recall a phrase after the server restarts. The final prompt does not reveal that phrase. A generic successful reply after restart cannot pass this check. +The blocker query requests a JSON status snapshot. It must name the current +recorded blocker and report zero active runs independently of the task's blocked +status. Mentioning the right blocker only as resolved history cannot pass. + The committed-send case drops the browser's acknowledgement after the server saves its comment. It waits for the agent to save one backlog task, restarts Paperclip, and replays the exact public request with the original client request ID. It diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index b459d3a5af1..1033c2ff3a4 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -959,7 +959,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ environments: [localEnvironment, daytonaWarmEnvironment], tasks: chatHardeningTasks, expectedMatrixSize: 18, excludedExecutionIds: ["runner-codex", "runner-acpx-claude"].flatMap(profile => ["stop-startup-new-resume", "hire-delegate-reuse", "blocked-status-review"].map(task => `agent-chat-hardening.${profile}.daytona.${task}`)), - definitionMetadata: { version: 2, permissions: "production-defaults", instructions: "production", grading: "durable-state-and-source-evidence", scheduling: "explicit-only", restartMemory: "required-after-restart" }, + definitionMetadata: { version: 3, permissions: "production-defaults", instructions: "production", grading: "durable-state-and-source-evidence", scheduling: "explicit-only", restartMemory: "required-after-restart", statusEvidence: "structured-current-blocker-and-active-run-count" }, }, ...(process.env.PAPERCLIP_RUNNER_E2E_CONNECTION_REVIEWS === "1" ? [connectionReviewSuite] : []), { diff --git a/tests/runner-e2e/chat-hardening.test.ts b/tests/runner-e2e/chat-hardening.test.ts index 790fb9c4cb3..95101d05e8c 100644 --- a/tests/runner-e2e/chat-hardening.test.ts +++ b/tests/runner-e2e/chat-hardening.test.ts @@ -31,12 +31,19 @@ describe("agent chat hardening oracles", () => { it("requires the current recorded blocker without silently starting or changing work", () => { const before = { ...task, status: "blocked" }; - const valid = { reply: "RUN-2 is blocked on VENUE123.", expectedIssueIdentifier: "RUN-2", blocker: "VENUE123", staleBlocker: "BUDGET123", + const status = { issueIdentifier: "RUN-2", status: "blocked", currentBlockerLabel: "VENUE123", activeRunCount: 0 }; + const valid = { reply: JSON.stringify(status), expectedIssueIdentifier: "RUN-2", blocker: "VENUE123", before, after: before, taskIdsBefore: ["task"], taskIdsAfter: ["task"], taskRuns: [] }; expect(() => assertGroundedChatStatus(valid)).not.toThrow(); - expect(() => assertGroundedChatStatus({ ...valid, reply: "RUN-2 is blocked on VENUE123. BUDGET123 was resolved." })).not.toThrow(); + expect(() => assertGroundedChatStatus({ ...valid, reply: JSON.stringify({ ...status, explanation: "BUDGET123 was resolved." }) })).not.toThrow(); + expect(() => assertGroundedChatStatus({ ...valid, reply: JSON.stringify({ ...status, currentBlockerLabel: "BUDGET123", explanation: "VENUE123 was resolved." }) })).toThrow(); + expect(() => assertGroundedChatStatus({ ...valid, reply: JSON.stringify({ ...status, activeRunCount: 1 }) })).toThrow(); + expect(() => assertGroundedChatStatus({ ...valid, reply: JSON.stringify({ ...status, status: "in_progress", explanation: "Previously blocked." }) })).toThrow(); + expect(() => assertGroundedChatStatus({ ...valid, reply: JSON.stringify({ ...status, activeRunCount: undefined }) })).toThrow(); expect(() => assertGroundedChatStatus({ ...valid, reply: "RUN-2 is blocked on BUDGET123." })).toThrow(); expect(() => assertGroundedChatStatus({ ...valid, after: task })).toThrow(); + expect(() => assertGroundedChatStatus({ ...valid, after: { ...before, title: "Changed" } })).toThrow(); + expect(() => assertGroundedChatStatus({ ...valid, after: { ...before, projectId: "different" } })).toThrow(); expect(() => assertGroundedChatStatus({ ...valid, taskRuns: [run] })).toThrow(); expect(() => assertGroundedChatStatus({ ...valid, taskIdsAfter: ["task", "replacement"] })).toThrow(); }); diff --git a/tests/runner-e2e/chat-hardening.ts b/tests/runner-e2e/chat-hardening.ts index 19f64e0156c..7370041efa7 100644 --- a/tests/runner-e2e/chat-hardening.ts +++ b/tests/runner-e2e/chat-hardening.ts @@ -37,15 +37,19 @@ export function assertChatHire(input: { } export function assertGroundedChatStatus(input: { - reply: string; expectedIssueIdentifier: string; blocker: string; staleBlocker: string; + reply: string; expectedIssueIdentifier: string; blocker: string; before: ChatIssue; after: ChatIssue; taskIdsBefore: string[]; taskIdsAfter: string[]; taskRuns: ChatRun[]; }) { - expect(input.reply).toContain(input.expectedIssueIdentifier); - expect(input.reply).toMatch(/blocked/i); - expect(input.reply).toContain(input.blocker); - // A correct answer may explain that the old blocker was resolved. Require - // the current label rather than rejecting that useful historical context. - expect(input.after).toMatchObject({ id: input.before.id, status: "blocked", assigneeAgentId: input.before.assigneeAgentId }); + // Mentioning a correct label is insufficient: the answer could falsely call + // it resolved, or confuse a blocked task with an actively running provider. + const json = input.reply.trim().replace(/^```(?:json)?\s*/, "").replace(/\s*```$/, ""); + expect(JSON.parse(json)).toMatchObject({ + issueIdentifier: input.expectedIssueIdentifier, status: "blocked", + currentBlockerLabel: input.blocker, activeRunCount: 0, + }); + for (const field of ["id", "title", "status", "assigneeAgentId", "parentId", "projectId"] as const) { + expect(input.after[field], `Status reporting must preserve ${field}`).toEqual(input.before[field]); + } expect([...input.taskIdsAfter].sort()).toEqual([...input.taskIdsBefore].sort()); expect(input.taskRuns).toHaveLength(0); } @@ -148,8 +152,8 @@ export async function runChatHardeningFlow(context: { const sourceBrief = await api.get(`/api/issues/${source.id}/documents/brief`); await api.post(`/api/issues/${source.id}/comments`, { body: `Earlier blocker: ${staleBlocker}. Waiting for the budget.` }); await api.post(`/api/issues/${source.id}/comments`, { body: `Budget is resolved. Current blocker: ${blocker}. Waiting for the venue confirmation. Keep this task blocked; no execution is active.` }); - await turn(`What is the actual current blocker on ${source.identifier}? Read the latest recorded explanation, include its exact blocker label, and distinguish task status from active execution. Just report; do not change it or create work.`, 1); - assertGroundedChatStatus({ reply: await latestReply(), expectedIssueIdentifier: source.identifier!, blocker, staleBlocker, + await turn(`What is the actual current status of ${source.identifier}? Read its latest recorded blocker and execution state. Reply with only JSON containing issueIdentifier, status, currentBlockerLabel, and activeRunCount. You may include an explanation field. Just report; do not change it or create work.`, 1); + assertGroundedChatStatus({ reply: await latestReply(), expectedIssueIdentifier: source.identifier!, blocker, before: source, after: await api.get(`/api/issues/${source.id}`), taskIdsBefore: [source.id], taskIdsAfter: (await tasks()).map(task => task.id), taskRuns: (await allRuns()).filter(run => run.contextSnapshot?.issueId === source.id) }); const config = execution.profile.buildAgent({ environmentId: f.environment.id, environmentFixtureId: execution.environment.id,