mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-25 02:26:12 +02:00
test: detect all mutable issue changes during status reporting
Derive read-only assertions from the public issue update contract and preserve relation and dedicated-endpoint state. Save the fully seeded pre-turn source snapshot and calibrate unintended mutations against legitimate inbound citations. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
50448c228a
commit
551e138922
4 files changed
+32
-6
No files matched your search
@@ -180,6 +180,9 @@ blocker, preserved task identities, worker-authored output, and real executions.
|
||||
The status request asks for JSON so the grader can distinguish the current
|
||||
blocker from a historical mention and compare active-run count separately from
|
||||
task status. The request must not reveal those expected values.
|
||||
Capture the source after seeding and compare every field in the public issue
|
||||
update contract, plus labels, dependencies, and dedicated-endpoint settings.
|
||||
Derived inbound references may change when the chat legitimately cites a task.
|
||||
The lost-acknowledgement probe may interrupt only the fixture browser's own
|
||||
comment request after the real server has committed it. Retain its request ID
|
||||
and replay that same request through the public API after restarting the server.
|
||||
|
||||
@@ -959,7 +959,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
|
||||
environments: [localEnvironment, daytonaWarmEnvironment], tasks: chatHardeningTasks, expectedMatrixSize: 18,
|
||||
excludedExecutionIds: ["runner-codex", "runner-acpx-claude"].flatMap(profile =>
|
||||
["stop-startup-new-resume", "hire-delegate-reuse", "blocked-status-review"].map(task => `agent-chat-hardening.${profile}.daytona.${task}`)),
|
||||
definitionMetadata: { version: 3, permissions: "production-defaults", instructions: "production", grading: "durable-state-and-source-evidence", scheduling: "explicit-only", restartMemory: "required-after-restart", statusEvidence: "structured-current-blocker-and-active-run-count" },
|
||||
definitionMetadata: { version: 4, permissions: "production-defaults", instructions: "production", grading: "durable-state-and-source-evidence", scheduling: "explicit-only", restartMemory: "required-after-restart", statusEvidence: "structured-current-blocker-and-active-run-count", readOnlyState: "public-mutation-contract-and-relations" },
|
||||
},
|
||||
...(process.env.PAPERCLIP_RUNNER_E2E_CONNECTION_REVIEWS === "1" ? [connectionReviewSuite] : []),
|
||||
{
|
||||
|
||||
@@ -35,6 +35,8 @@ describe("agent chat hardening oracles", () => {
|
||||
const valid = { reply: JSON.stringify(status), expectedIssueIdentifier: "RUN-2", blocker: "VENUE123",
|
||||
before, after: before, taskIdsBefore: ["task"], taskIdsAfter: ["task"], taskRuns: [] };
|
||||
expect(() => assertGroundedChatStatus(valid)).not.toThrow();
|
||||
const cited = { ...before, relatedWork: { inbound: [{ sourceIssueId: "chat" }] } };
|
||||
expect(() => assertGroundedChatStatus({ ...valid, after: cited })).not.toThrow();
|
||||
expect(() => assertGroundedChatStatus({ ...valid, reply: JSON.stringify({ ...status, explanation: "BUDGET123 was resolved." }) })).not.toThrow();
|
||||
expect(() => assertGroundedChatStatus({ ...valid, reply: JSON.stringify({ ...status, currentBlockerLabel: "BUDGET123", explanation: "VENUE123 was resolved." }) })).toThrow();
|
||||
expect(() => assertGroundedChatStatus({ ...valid, reply: JSON.stringify({ ...status, activeRunCount: 1 }) })).toThrow();
|
||||
@@ -44,6 +46,14 @@ describe("agent chat hardening oracles", () => {
|
||||
expect(() => assertGroundedChatStatus({ ...valid, after: task })).toThrow();
|
||||
expect(() => assertGroundedChatStatus({ ...valid, after: { ...before, title: "Changed" } })).toThrow();
|
||||
expect(() => assertGroundedChatStatus({ ...valid, after: { ...before, projectId: "different" } })).toThrow();
|
||||
for (const mutation of [
|
||||
{ description: "Changed the brief" }, { priority: "high" },
|
||||
{ labelIds: ["new-label"] }, { labels: [{ id: "label", name: "Changed" }] },
|
||||
{ blockedBy: [{ id: "new-blocker" }] }, { blocks: [{ id: "new-dependent" }] },
|
||||
{ unblockDescriptor: { owner: "board", action: "Different action" } },
|
||||
{ executionWorkspaceSettings: { mode: "isolated" } },
|
||||
{ reviewPolicy: { kind: "none" } }, { watchdog: { agentId: "different" } },
|
||||
]) expect(() => assertGroundedChatStatus({ ...valid, after: { ...before, ...mutation } })).toThrow();
|
||||
expect(() => assertGroundedChatStatus({ ...valid, taskRuns: [run] })).toThrow();
|
||||
expect(() => assertGroundedChatStatus({ ...valid, taskIdsAfter: ["task", "replacement"] })).toThrow();
|
||||
});
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { expect } from "@playwright/test";
|
||||
import { updateIssueSchema } from "../../packages/shared/src/validators/issue.js";
|
||||
import type { RunnerApi } from "./api.js";
|
||||
import { readChatOutputDocument, sendChatMessage, type ChatFlowInput, type ChatIssue, type ChatRun } from "./chat-flow.js";
|
||||
import { dropChatSendAcknowledgement } from "./lost-send.js";
|
||||
@@ -10,6 +11,16 @@ type Agent = {
|
||||
type Comment = { id: string; body: string; authorAgentId?: string; clientRequestId?: string };
|
||||
type Document = { id: string; body: string; latestRevisionId: string; createdByAgentId?: string };
|
||||
|
||||
function mutableIssueSnapshot(issue: ChatIssue) {
|
||||
// Follow the public mutation contract as it grows. Include relationships and
|
||||
// fields governed by dedicated endpoints; omit derived read projections such
|
||||
// as inbound references, which a legitimate status reply can add to the chat.
|
||||
const keys = new Set([...Object.keys(updateIssueSchema.shape),
|
||||
"id", "companyId", "responsibleUserId", "labels", "blockedBy", "blocks", "watchdog", "sourceTrust"]);
|
||||
const record = issue as unknown as Record<string, unknown>;
|
||||
return Object.fromEntries([...keys].map(key => [key, record[key]]));
|
||||
}
|
||||
|
||||
export function assertChatHire(input: {
|
||||
agents: Agent[]; leadId: string; hireName: string; hiredId?: string;
|
||||
connectionId: string; binding: unknown; taskIds: string[]; tasks: ChatIssue[]; runs: ChatRun[];
|
||||
@@ -47,9 +58,8 @@ export function assertGroundedChatStatus(input: {
|
||||
issueIdentifier: input.expectedIssueIdentifier, status: "blocked",
|
||||
currentBlockerLabel: input.blocker, activeRunCount: 0,
|
||||
});
|
||||
for (const field of ["id", "title", "status", "assigneeAgentId", "parentId", "projectId"] as const) {
|
||||
expect(input.after[field], `Status reporting must preserve ${field}`).toEqual(input.before[field]);
|
||||
}
|
||||
expect(mutableIssueSnapshot(input.after), "Status reporting must preserve mutable issue state")
|
||||
.toStrictEqual(mutableIssueSnapshot(input.before));
|
||||
expect([...input.taskIdsAfter].sort()).toEqual([...input.taskIdsBefore].sort());
|
||||
expect(input.taskRuns).toHaveLength(0);
|
||||
}
|
||||
@@ -152,9 +162,12 @@ export async function runChatHardeningFlow(context: {
|
||||
const sourceBrief = await api.get<Document>(`/api/issues/${source.id}/documents/brief`);
|
||||
await api.post(`/api/issues/${source.id}/comments`, { body: `Earlier blocker: ${staleBlocker}. Waiting for the budget.` });
|
||||
await api.post(`/api/issues/${source.id}/comments`, { body: `Budget is resolved. Current blocker: ${blocker}. Waiting for the venue confirmation. Keep this task blocked; no execution is active.` });
|
||||
const sourceBeforeStatus = await api.get<ChatIssue>(`/api/issues/${source.id}`);
|
||||
await input.evidence("chat-status-review.json", { source: sourceBeforeStatus, sourcePlan, sourceBrief });
|
||||
await turn(`What is the actual current status of ${source.identifier}? Read its latest recorded blocker and execution state. Reply with only JSON containing issueIdentifier, status, currentBlockerLabel, and activeRunCount. You may include an explanation field. Just report; do not change it or create work.`, 1);
|
||||
const sourceAfterStatus = await api.get<ChatIssue>(`/api/issues/${source.id}`);
|
||||
assertGroundedChatStatus({ reply: await latestReply(), expectedIssueIdentifier: source.identifier!, blocker,
|
||||
before: source, after: await api.get(`/api/issues/${source.id}`), taskIdsBefore: [source.id],
|
||||
before: sourceBeforeStatus, after: sourceAfterStatus, taskIdsBefore: [source.id],
|
||||
taskIdsAfter: (await tasks()).map(task => task.id), taskRuns: (await allRuns()).filter(run => run.contextSnapshot?.issueId === source.id) });
|
||||
const config = execution.profile.buildAgent({ environmentId: f.environment.id, environmentFixtureId: execution.environment.id,
|
||||
workspacePath: input.workspacePath, secretRefs: f.secretRefs, executionId: nonce });
|
||||
@@ -175,7 +188,7 @@ export async function runChatHardeningFlow(context: {
|
||||
expect(await api.get(`/api/issues/${source.id}/documents/plan`)).toEqual(sourcePlan);
|
||||
expect(await api.get(`/api/issues/${source.id}/documents/brief`)).toEqual(sourceBrief);
|
||||
expect(await api.get(`/api/issues/${source.id}`)).toMatchObject({ status: "blocked" });
|
||||
await input.evidence("chat-status-review.json", { source, sourcePlan, sourceBrief, review, document, reviewRuns, reply });
|
||||
await input.evidence("chat-status-review.json", { source: sourceBeforeStatus, sourceAfterStatus, sourcePlan, sourceBrief, review, document, reviewRuns, reply });
|
||||
} else if (execution.task.id === "committed-send-retry") {
|
||||
const interception = await dropChatSendAcknowledgement(page, marker);
|
||||
try {
|
||||
|
||||
Reference in new issue
Block a user