test: detect all mutable issue changes during status reporting

Derive read-only assertions from the public issue update contract and preserve relation and dedicated-endpoint state. Save the fully seeded pre-turn source snapshot and calibrate unintended mutations against legitimate inbound citations.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-09-21 10:01:17 -05:00
1 parent 50448c228a
commit 551e138922
4 files changed
+32 -6

No files matched your search

+3
View File
@@ -180,6 +180,9 @@ blocker, preserved task identities, worker-authored output, and real executions.
The status request asks for JSON so the grader can distinguish the current
blocker from a historical mention and compare active-run count separately from
task status. The request must not reveal those expected values.
Capture the source after seeding and compare every field in the public issue
update contract, plus labels, dependencies, and dedicated-endpoint settings.
Derived inbound references may change when the chat legitimately cites a task.
The lost-acknowledgement probe may interrupt only the fixture browser's own
comment request after the real server has committed it. Retain its request ID
and replay that same request through the public API after restarting the server.
+1 -1
View File
@@ -959,7 +959,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
environments: [localEnvironment, daytonaWarmEnvironment], tasks: chatHardeningTasks, expectedMatrixSize: 18,
excludedExecutionIds: ["runner-codex", "runner-acpx-claude"].flatMap(profile =>
["stop-startup-new-resume", "hire-delegate-reuse", "blocked-status-review"].map(task => `agent-chat-hardening.${profile}.daytona.${task}`)),
definitionMetadata: { version: 3, permissions: "production-defaults", instructions: "production", grading: "durable-state-and-source-evidence", scheduling: "explicit-only", restartMemory: "required-after-restart", statusEvidence: "structured-current-blocker-and-active-run-count" },
definitionMetadata: { version: 4, permissions: "production-defaults", instructions: "production", grading: "durable-state-and-source-evidence", scheduling: "explicit-only", restartMemory: "required-after-restart", statusEvidence: "structured-current-blocker-and-active-run-count", readOnlyState: "public-mutation-contract-and-relations" },
},
...(process.env.PAPERCLIP_RUNNER_E2E_CONNECTION_REVIEWS === "1" ? [connectionReviewSuite] : []),
{
+10
View File
@@ -35,6 +35,8 @@ describe("agent chat hardening oracles", () => {
const valid = { reply: JSON.stringify(status), expectedIssueIdentifier: "RUN-2", blocker: "VENUE123",
before, after: before, taskIdsBefore: ["task"], taskIdsAfter: ["task"], taskRuns: [] };
expect(() => assertGroundedChatStatus(valid)).not.toThrow();
const cited = { ...before, relatedWork: { inbound: [{ sourceIssueId: "chat" }] } };
expect(() => assertGroundedChatStatus({ ...valid, after: cited })).not.toThrow();
expect(() => assertGroundedChatStatus({ ...valid, reply: JSON.stringify({ ...status, explanation: "BUDGET123 was resolved." }) })).not.toThrow();
expect(() => assertGroundedChatStatus({ ...valid, reply: JSON.stringify({ ...status, currentBlockerLabel: "BUDGET123", explanation: "VENUE123 was resolved." }) })).toThrow();
expect(() => assertGroundedChatStatus({ ...valid, reply: JSON.stringify({ ...status, activeRunCount: 1 }) })).toThrow();
@@ -44,6 +46,14 @@ describe("agent chat hardening oracles", () => {
expect(() => assertGroundedChatStatus({ ...valid, after: task })).toThrow();
expect(() => assertGroundedChatStatus({ ...valid, after: { ...before, title: "Changed" } })).toThrow();
expect(() => assertGroundedChatStatus({ ...valid, after: { ...before, projectId: "different" } })).toThrow();
for (const mutation of [
{ description: "Changed the brief" }, { priority: "high" },
{ labelIds: ["new-label"] }, { labels: [{ id: "label", name: "Changed" }] },
{ blockedBy: [{ id: "new-blocker" }] }, { blocks: [{ id: "new-dependent" }] },
{ unblockDescriptor: { owner: "board", action: "Different action" } },
{ executionWorkspaceSettings: { mode: "isolated" } },
{ reviewPolicy: { kind: "none" } }, { watchdog: { agentId: "different" } },
]) expect(() => assertGroundedChatStatus({ ...valid, after: { ...before, ...mutation } })).toThrow();
expect(() => assertGroundedChatStatus({ ...valid, taskRuns: [run] })).toThrow();
expect(() => assertGroundedChatStatus({ ...valid, taskIdsAfter: ["task", "replacement"] })).toThrow();
});
+18 -5
View File
@@ -1,4 +1,5 @@
import { expect } from "@playwright/test";
import { updateIssueSchema } from "../../packages/shared/src/validators/issue.js";
import type { RunnerApi } from "./api.js";
import { readChatOutputDocument, sendChatMessage, type ChatFlowInput, type ChatIssue, type ChatRun } from "./chat-flow.js";
import { dropChatSendAcknowledgement } from "./lost-send.js";
@@ -10,6 +11,16 @@ type Agent = {
type Comment = { id: string; body: string; authorAgentId?: string; clientRequestId?: string };
type Document = { id: string; body: string; latestRevisionId: string; createdByAgentId?: string };
function mutableIssueSnapshot(issue: ChatIssue) {
// Follow the public mutation contract as it grows. Include relationships and
// fields governed by dedicated endpoints; omit derived read projections such
// as inbound references, which a legitimate status reply can add to the chat.
const keys = new Set([...Object.keys(updateIssueSchema.shape),
"id", "companyId", "responsibleUserId", "labels", "blockedBy", "blocks", "watchdog", "sourceTrust"]);
const record = issue as unknown as Record<string, unknown>;
return Object.fromEntries([...keys].map(key => [key, record[key]]));
}
export function assertChatHire(input: {
agents: Agent[]; leadId: string; hireName: string; hiredId?: string;
connectionId: string; binding: unknown; taskIds: string[]; tasks: ChatIssue[]; runs: ChatRun[];
@@ -47,9 +58,8 @@ export function assertGroundedChatStatus(input: {
issueIdentifier: input.expectedIssueIdentifier, status: "blocked",
currentBlockerLabel: input.blocker, activeRunCount: 0,
});
for (const field of ["id", "title", "status", "assigneeAgentId", "parentId", "projectId"] as const) {
expect(input.after[field], `Status reporting must preserve ${field}`).toEqual(input.before[field]);
}
expect(mutableIssueSnapshot(input.after), "Status reporting must preserve mutable issue state")
.toStrictEqual(mutableIssueSnapshot(input.before));
expect([...input.taskIdsAfter].sort()).toEqual([...input.taskIdsBefore].sort());
expect(input.taskRuns).toHaveLength(0);
}
@@ -152,9 +162,12 @@ export async function runChatHardeningFlow(context: {
const sourceBrief = await api.get<Document>(`/api/issues/${source.id}/documents/brief`);
await api.post(`/api/issues/${source.id}/comments`, { body: `Earlier blocker: ${staleBlocker}. Waiting for the budget.` });
await api.post(`/api/issues/${source.id}/comments`, { body: `Budget is resolved. Current blocker: ${blocker}. Waiting for the venue confirmation. Keep this task blocked; no execution is active.` });
const sourceBeforeStatus = await api.get<ChatIssue>(`/api/issues/${source.id}`);
await input.evidence("chat-status-review.json", { source: sourceBeforeStatus, sourcePlan, sourceBrief });
await turn(`What is the actual current status of ${source.identifier}? Read its latest recorded blocker and execution state. Reply with only JSON containing issueIdentifier, status, currentBlockerLabel, and activeRunCount. You may include an explanation field. Just report; do not change it or create work.`, 1);
const sourceAfterStatus = await api.get<ChatIssue>(`/api/issues/${source.id}`);
assertGroundedChatStatus({ reply: await latestReply(), expectedIssueIdentifier: source.identifier!, blocker,
before: source, after: await api.get(`/api/issues/${source.id}`), taskIdsBefore: [source.id],
before: sourceBeforeStatus, after: sourceAfterStatus, taskIdsBefore: [source.id],
taskIdsAfter: (await tasks()).map(task => task.id), taskRuns: (await allRuns()).filter(run => run.contextSnapshot?.issueId === source.id) });
const config = execution.profile.buildAgent({ environmentId: f.environment.id, environmentFixtureId: execution.environment.id,
workspacePath: input.workspacePath, secretRefs: f.secretRefs, executionId: nonce });
@@ -175,7 +188,7 @@ export async function runChatHardeningFlow(context: {
expect(await api.get(`/api/issues/${source.id}/documents/plan`)).toEqual(sourcePlan);
expect(await api.get(`/api/issues/${source.id}/documents/brief`)).toEqual(sourceBrief);
expect(await api.get(`/api/issues/${source.id}`)).toMatchObject({ status: "blocked" });
await input.evidence("chat-status-review.json", { source, sourcePlan, sourceBrief, review, document, reviewRuns, reply });
await input.evidence("chat-status-review.json", { source: sourceBeforeStatus, sourceAfterStatus, sourcePlan, sourceBrief, review, document, reviewRuns, reply });
} else if (execution.task.id === "committed-send-retry") {
const interception = await dropChatSendAcknowledgement(page, marker);
try {