From 551e1389227a3df022e428e3bd2ae12345143510 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 21 Sep 2026 10:01:17 -0500 Subject: [PATCH] test: detect all mutable issue changes during status reporting Derive read-only assertions from the public issue update contract and preserve relation and dedicated-endpoint state. Save the fully seeded pre-turn source snapshot and calibrate unintended mutations against legitimate inbound citations. Co-Authored-By: Paperclip --- tests/runner-e2e/FIXTURES.md | 3 +++ tests/runner-e2e/catalog.ts | 2 +- tests/runner-e2e/chat-hardening.test.ts | 10 ++++++++++ tests/runner-e2e/chat-hardening.ts | 23 ++++++++++++++++++----- 4 files changed, 32 insertions(+), 6 deletions(-) diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index 063a57e1aa5..f03bfda838e 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -180,6 +180,9 @@ blocker, preserved task identities, worker-authored output, and real executions. The status request asks for JSON so the grader can distinguish the current blocker from a historical mention and compare active-run count separately from task status. The request must not reveal those expected values. +Capture the source after seeding and compare every field in the public issue +update contract, plus labels, dependencies, and dedicated-endpoint settings. +Derived inbound references may change when the chat legitimately cites a task. The lost-acknowledgement probe may interrupt only the fixture browser's own comment request after the real server has committed it. Retain its request ID and replay that same request through the public API after restarting the server. diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 1033c2ff3a4..73b6cd99272 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -959,7 +959,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ environments: [localEnvironment, daytonaWarmEnvironment], tasks: chatHardeningTasks, expectedMatrixSize: 18, excludedExecutionIds: ["runner-codex", "runner-acpx-claude"].flatMap(profile => ["stop-startup-new-resume", "hire-delegate-reuse", "blocked-status-review"].map(task => `agent-chat-hardening.${profile}.daytona.${task}`)), - definitionMetadata: { version: 3, permissions: "production-defaults", instructions: "production", grading: "durable-state-and-source-evidence", scheduling: "explicit-only", restartMemory: "required-after-restart", statusEvidence: "structured-current-blocker-and-active-run-count" }, + definitionMetadata: { version: 4, permissions: "production-defaults", instructions: "production", grading: "durable-state-and-source-evidence", scheduling: "explicit-only", restartMemory: "required-after-restart", statusEvidence: "structured-current-blocker-and-active-run-count", readOnlyState: "public-mutation-contract-and-relations" }, }, ...(process.env.PAPERCLIP_RUNNER_E2E_CONNECTION_REVIEWS === "1" ? [connectionReviewSuite] : []), { diff --git a/tests/runner-e2e/chat-hardening.test.ts b/tests/runner-e2e/chat-hardening.test.ts index 95101d05e8c..5ebcadc89a3 100644 --- a/tests/runner-e2e/chat-hardening.test.ts +++ b/tests/runner-e2e/chat-hardening.test.ts @@ -35,6 +35,8 @@ describe("agent chat hardening oracles", () => { const valid = { reply: JSON.stringify(status), expectedIssueIdentifier: "RUN-2", blocker: "VENUE123", before, after: before, taskIdsBefore: ["task"], taskIdsAfter: ["task"], taskRuns: [] }; expect(() => assertGroundedChatStatus(valid)).not.toThrow(); + const cited = { ...before, relatedWork: { inbound: [{ sourceIssueId: "chat" }] } }; + expect(() => assertGroundedChatStatus({ ...valid, after: cited })).not.toThrow(); expect(() => assertGroundedChatStatus({ ...valid, reply: JSON.stringify({ ...status, explanation: "BUDGET123 was resolved." }) })).not.toThrow(); expect(() => assertGroundedChatStatus({ ...valid, reply: JSON.stringify({ ...status, currentBlockerLabel: "BUDGET123", explanation: "VENUE123 was resolved." }) })).toThrow(); expect(() => assertGroundedChatStatus({ ...valid, reply: JSON.stringify({ ...status, activeRunCount: 1 }) })).toThrow(); @@ -44,6 +46,14 @@ describe("agent chat hardening oracles", () => { expect(() => assertGroundedChatStatus({ ...valid, after: task })).toThrow(); expect(() => assertGroundedChatStatus({ ...valid, after: { ...before, title: "Changed" } })).toThrow(); expect(() => assertGroundedChatStatus({ ...valid, after: { ...before, projectId: "different" } })).toThrow(); + for (const mutation of [ + { description: "Changed the brief" }, { priority: "high" }, + { labelIds: ["new-label"] }, { labels: [{ id: "label", name: "Changed" }] }, + { blockedBy: [{ id: "new-blocker" }] }, { blocks: [{ id: "new-dependent" }] }, + { unblockDescriptor: { owner: "board", action: "Different action" } }, + { executionWorkspaceSettings: { mode: "isolated" } }, + { reviewPolicy: { kind: "none" } }, { watchdog: { agentId: "different" } }, + ]) expect(() => assertGroundedChatStatus({ ...valid, after: { ...before, ...mutation } })).toThrow(); expect(() => assertGroundedChatStatus({ ...valid, taskRuns: [run] })).toThrow(); expect(() => assertGroundedChatStatus({ ...valid, taskIdsAfter: ["task", "replacement"] })).toThrow(); }); diff --git a/tests/runner-e2e/chat-hardening.ts b/tests/runner-e2e/chat-hardening.ts index 7370041efa7..3c0d547c29c 100644 --- a/tests/runner-e2e/chat-hardening.ts +++ b/tests/runner-e2e/chat-hardening.ts @@ -1,4 +1,5 @@ import { expect } from "@playwright/test"; +import { updateIssueSchema } from "../../packages/shared/src/validators/issue.js"; import type { RunnerApi } from "./api.js"; import { readChatOutputDocument, sendChatMessage, type ChatFlowInput, type ChatIssue, type ChatRun } from "./chat-flow.js"; import { dropChatSendAcknowledgement } from "./lost-send.js"; @@ -10,6 +11,16 @@ type Agent = { type Comment = { id: string; body: string; authorAgentId?: string; clientRequestId?: string }; type Document = { id: string; body: string; latestRevisionId: string; createdByAgentId?: string }; +function mutableIssueSnapshot(issue: ChatIssue) { + // Follow the public mutation contract as it grows. Include relationships and + // fields governed by dedicated endpoints; omit derived read projections such + // as inbound references, which a legitimate status reply can add to the chat. + const keys = new Set([...Object.keys(updateIssueSchema.shape), + "id", "companyId", "responsibleUserId", "labels", "blockedBy", "blocks", "watchdog", "sourceTrust"]); + const record = issue as unknown as Record; + return Object.fromEntries([...keys].map(key => [key, record[key]])); +} + export function assertChatHire(input: { agents: Agent[]; leadId: string; hireName: string; hiredId?: string; connectionId: string; binding: unknown; taskIds: string[]; tasks: ChatIssue[]; runs: ChatRun[]; @@ -47,9 +58,8 @@ export function assertGroundedChatStatus(input: { issueIdentifier: input.expectedIssueIdentifier, status: "blocked", currentBlockerLabel: input.blocker, activeRunCount: 0, }); - for (const field of ["id", "title", "status", "assigneeAgentId", "parentId", "projectId"] as const) { - expect(input.after[field], `Status reporting must preserve ${field}`).toEqual(input.before[field]); - } + expect(mutableIssueSnapshot(input.after), "Status reporting must preserve mutable issue state") + .toStrictEqual(mutableIssueSnapshot(input.before)); expect([...input.taskIdsAfter].sort()).toEqual([...input.taskIdsBefore].sort()); expect(input.taskRuns).toHaveLength(0); } @@ -152,9 +162,12 @@ export async function runChatHardeningFlow(context: { const sourceBrief = await api.get(`/api/issues/${source.id}/documents/brief`); await api.post(`/api/issues/${source.id}/comments`, { body: `Earlier blocker: ${staleBlocker}. Waiting for the budget.` }); await api.post(`/api/issues/${source.id}/comments`, { body: `Budget is resolved. Current blocker: ${blocker}. Waiting for the venue confirmation. Keep this task blocked; no execution is active.` }); + const sourceBeforeStatus = await api.get(`/api/issues/${source.id}`); + await input.evidence("chat-status-review.json", { source: sourceBeforeStatus, sourcePlan, sourceBrief }); await turn(`What is the actual current status of ${source.identifier}? Read its latest recorded blocker and execution state. Reply with only JSON containing issueIdentifier, status, currentBlockerLabel, and activeRunCount. You may include an explanation field. Just report; do not change it or create work.`, 1); + const sourceAfterStatus = await api.get(`/api/issues/${source.id}`); assertGroundedChatStatus({ reply: await latestReply(), expectedIssueIdentifier: source.identifier!, blocker, - before: source, after: await api.get(`/api/issues/${source.id}`), taskIdsBefore: [source.id], + before: sourceBeforeStatus, after: sourceAfterStatus, taskIdsBefore: [source.id], taskIdsAfter: (await tasks()).map(task => task.id), taskRuns: (await allRuns()).filter(run => run.contextSnapshot?.issueId === source.id) }); const config = execution.profile.buildAgent({ environmentId: f.environment.id, environmentFixtureId: execution.environment.id, workspacePath: input.workspacePath, secretRefs: f.secretRefs, executionId: nonce }); @@ -175,7 +188,7 @@ export async function runChatHardeningFlow(context: { expect(await api.get(`/api/issues/${source.id}/documents/plan`)).toEqual(sourcePlan); expect(await api.get(`/api/issues/${source.id}/documents/brief`)).toEqual(sourceBrief); expect(await api.get(`/api/issues/${source.id}`)).toMatchObject({ status: "blocked" }); - await input.evidence("chat-status-review.json", { source, sourcePlan, sourceBrief, review, document, reviewRuns, reply }); + await input.evidence("chat-status-review.json", { source: sourceBeforeStatus, sourceAfterStatus, sourcePlan, sourceBrief, review, document, reviewRuns, reply }); } else if (execution.task.id === "committed-send-retry") { const interception = await dropChatSendAcknowledgement(page, marker); try {