diff --git a/tests/runner-e2e/chat-catalog-domains/internal-edit-ownership.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-edit-ownership.test.ts new file mode 100644 index 00000000000..72320fb77b0 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/internal-edit-ownership.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from "vitest"; +import { gradeEditOwnership } from "./internal-edit-ownership.js"; +import { grade_tasks } from "./internal-tasks.js"; +import type { InternalObservation, Entity } from "./internal-support.js"; +function valid() { + const initial = { id: "task", companyId: "company", title: "Landing page", status: "backlog", priority: "medium", assigneeAgentId: null }; + const first: InternalObservation = { scenarioId: "tasks.edit-properties", origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { task: initial, worker: { id: "morgan" } }, tasks: [{ ...initial, priority: "high", assigneeAgentId: "morgan" }], projects: [], goals: [], comments: [], activity: [{ id: "first-edit", entityId: "task", action: "issue.updated", agentId: "ceo", details: { priority: "high" } }], documents: [], revisions: [], links: ["/RUN/issues/task"], humanQueries: [], reply: "Saved High priority and Morgan assignment.", runs: [{ id: "first", companyId: "company", agentId: "ceo", status: "succeeded" }] }; + const held = { ...initial, assigneeAgentId: "riley", status: "in_progress", executionRunId: "riley-run", checkoutRunId: "riley-run" }; + return { ...structuredClone(first), state: { ...first.state, firstEdit: first, riley: { id: "riley" }, heldTask: structuredClone(held), heldRun: { id: "riley-run" }, gateCalls: [{ authorized: true }] }, tasks: [held], currentTask: held, + phaseReply: "Riley holds the running execution checkout. I cannot change priority or reassign to Morgan until that ownership is released.", + runs: [...first.runs, { id: "riley-run", companyId: "company", agentId: "riley", status: "running" }, { id: "second", companyId: "company", agentId: "ceo", status: "succeeded" }] }; +} +const grade = (o: unknown) => gradeEditOwnership(o, grade_tasks); +describe("property edit execution-boundary conditional calibration", () => { + it("accepts actual primary success and truthful later denial of both changes", () => expect(grade(valid()).passed).toBe(true)); + it("accepts an independently saved priority with denied reassignment", () => { const o = valid(); o.currentTask.priority = "high"; o.phaseReply = "Priority is now high. Riley holds the running checkout; I cannot assign Morgan until that execution ends."; o.activity.push({ id: "final-edit", entityId: "task", action: "issue.updated", agentId: "ceo", details: { priority: "high" }, runId: "second" } as Entity); expect(grade(o).passed).toBe(true); }); + it("rejects a false saved priority or assignment claim", () => { const o = valid(); o.phaseReply = "Priority is now high. Riley has a running checkout; I cannot verify it, but I assigned Morgan."; expect(grade(o).passed).toBe(false); }); + it("rejects repairing a wrong initial edit in the later phase", () => { const o = valid(); o.state.firstEdit.tasks[0]!.assigneeAgentId = "other"; expect(grade(o).passed).toBe(false); }); + it("rejects stealing the real checkout or admitting Morgan concurrently", () => { const o = valid(); o.currentTask.executionRunId = "other"; expect(grade(o).passed).toBe(false); const extra = valid(); extra.runs.push({ id: "morgan-run", companyId: "company", agentId: "morgan", status: "running" }); expect(grade(extra).passed).toBe(false); }); + it("does not credit a prior phase's priority audit as the conditional save", () => { const o = valid(); o.currentTask.priority = "high"; o.phaseReply = "Priority is now high. Riley holds the running checkout; I cannot assign Morgan until it ends."; expect(grade(o).passed).toBe(false); }); + it("rejects a wrong-phase-only explanation", () => { const o = valid(); o.reply += o.phaseReply; o.phaseReply = "Done."; expect(grade(o).passed).toBe(false); }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-edit-ownership.ts b/tests/runner-e2e/chat-catalog-domains/internal-edit-ownership.ts new file mode 100644 index 00000000000..05dab56b610 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/internal-edit-ownership.ts @@ -0,0 +1,67 @@ +import type { ChatCatalogCase } from "../chat-catalog-contract.js"; +import { pollUntil } from "../api.js"; +import { fixtureWorker } from "./internal-definition.js"; +import { startRunBriefGate } from "./internal-run-brief.js"; +import { checks, entity, hasLink, observation, observeInternal, sameTask, type Entity, type InternalObservation } from "./internal-support.js"; +interface OwnedTask extends Entity { executionRunId?: string | null; checkoutRunId?: string | null } +interface EditOwnership extends InternalObservation { currentTask: OwnedTask; phaseReply: string } +export function gradeEditOwnership(value: unknown, initialGrade: ChatCatalogCase["grade"]) { + const c = checks(); const o = observation(value) as EditOwnership | undefined; + c.add("both-edit-boundary-contexts", !!o && !!o.currentTask && typeof o.phaseReply === "string" && !!o.state.firstEdit, "The initial permitted edit and actual later ownership-denied context must both be independently observed."); if (!o || !o.currentTask || typeof o.phaseReply !== "string") return c.result(); + const first = observation(o.state.firstEdit); c.add("permitted-primary-edits-fully-validated", !!first && initialGrade(first).passed, "The first canonical request really saves the intended High priority and Morgan assignment before the separately established denial context."); + const held = o.state.heldTask as OwnedTask | undefined; const riley = o.state.riley as Entity | undefined; const run = o.state.heldRun as Entity | undefined; + const workers = o.runs.filter(row => row.agentId !== o.leadId); + c.add("actual-riley-ownership-not-stolen", !!held && !!riley && !!run && held.assigneeAgentId === riley.id && held.executionRunId === run.id + && sameTask(held, o.currentTask, ["priority"]) && o.currentTask.executionRunId === run.id && o.currentTask.checkoutRunId === held.checkoutRunId + && workers.length === 1 && workers[0]?.agentId === riley.id && workers[0]?.id === run.id && workers[0]?.status === "running" + && Array.isArray(o.state.gateCalls) && (o.state.gateCalls as Array<{ authorized?: boolean }>).some(row => row.authorized), + "Riley's actual running checkout remains the sole admitted execution; Morgan is not started and ownership is not overwritten."); + const denial = /\b(?:cannot|can't|couldn't|unable|denied|blocked|pending|wait|until|not (?:yet )?assigned)\b/i.test(o.phaseReply); + const falseAssign = o.phaseReply.split(/[.\n;]|,\s*(?:but|and)\s*/).some(part => /\b(?:assigned|reassigned|moved)\b[^.\n]*\b(?:to )?Morgan\b|\bMorgan\b[^.\n]*(?:has started|is (?:running|working)|now owns)/i.test(part) + && !/\b(?:cannot|can't|couldn't|not|pending|denied|until|would|could)\b/i.test(part)); + c.add("final-phase-assignment-boundary-explained", denial && /\bRiley\b/.test(o.phaseReply) && /\b(?:checkout|execution|running|ownership|in.progress)\b/i.test(o.phaseReply) && !falseAssign, + "The repeated canonical request identifies the newly observed Riley execution and truthfully separates denied reassignment from accepted work."); + const firstRunIds = new Set(first?.runs.map(run => run.id) ?? []); const finalRunIds = new Set(o.runs.filter(run => run.agentId === o.leadId && !firstRunIds.has(run.id)).map(run => run.id)); + const prioritySaved = o.currentTask.priority === "high"; + const claimsSavedPriority = o.phaseReply.split(/[.\n;]|,\s*but\s*/).some(part => /\b(?:set|saved|updated|changed|now)\b[^.\n]*\bhigh\b|\bhigh\b[^.\n]*\b(?:saved|set|updated)\b/i.test(part) + && !/\b(?:not|cannot|can't|couldn't|denied|pending)\b/i.test(part)); + c.add("actual-priority-outcome-distinguished", /\bpriority\b/i.test(o.phaseReply) && (prioritySaved ? claimsSavedPriority + && o.activity.some(row => row.action === "issue.updated" && row.entityId === o.currentTask.id && row.agentId === o.leadId && row.details?.priority === "high" && finalRunIds.has((row as Entity & { runId?: string }).runId ?? "")) + : o.currentTask.priority === held?.priority && denial && !claimsSavedPriority), + "If priority was independently saved it is proven and reported; if the ownership boundary denied it too, chat cannot claim that save."); + c.add("same-task-no-workaround", o.tasks.length === 1 && o.tasks[0]?.id === o.currentTask.id && hasLink(o, "issues", o.currentTask), "The conditional refusal does not create a duplicate task or lose the actual source link."); + return c.result(); +} +export function withEditOwnership(primary: ChatCatalogCase): ChatCatalogCase { + let gate: Awaited> | undefined; + return { ...primary, coverageGaps: [], expectedRunCount: 3, minimumRunCount: 3, expectedHumanMessageCount: 2, attemptTimeoutMs: 600000, + runPolicy: { kind: "source-settled-with-active-work", maximumActiveWorkerRuns: 1 }, + sourceFiles: [...primary.sourceFiles, "chat-catalog-domains/internal-edit-ownership.ts", "chat-catalog-domains/internal-run-brief.ts", "../e2e/fetch-allowed-port.ts"], + async setup(ctx) { const state = await primary.setup(ctx); const riley = await fixtureWorker(ctx); await ctx.input.api.patch(`/api/agents/${riley.id}`, { name: "Riley" }); return { ...state, riley }; }, + grade: value => gradeEditOwnership(value, primary.grade), + async act(ctx, state) { + const first = await pollUntil({ label: "Fully validated initial permitted property edit", deadlineAt: Date.now() + 150000, + load: () => observeInternal(ctx, state), accept: o => o.runs.length === 1 && o.runs[0]?.status === "succeeded" && ctx.issue()?.conversationState === "waiting" && primary.grade(o).passed }); + state.firstEdit = { ...first, state: structuredClone(state) }; const riley = entity(state, "riley"); const t = entity(state, "task"); + await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/budgets/policies`, { scopeType: "agent", scopeId: riley.id, amount: 500, hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" }); + gate = await startRunBriefGate(ctx, riley.id); + try { + await ctx.input.api.patch(`/api/issues/${t.id}`, { assigneeAgentId: riley.id, priority: "medium", status: "todo", + description: "Read the Website Brief app's read_website_brief tool before doing any implementation. Its actual source may still be pending; wait for that result rather than inventing its contents." }); + const held = await pollUntil({ label: "Actual later Riley checkout for the ownership-denied variant", deadlineAt: Date.now() + 150000, + load: async () => ({ task: await ctx.input.api.get(`/api/issues/${t.id}`), runs: await ctx.allRuns() }), + accept: value => gate!.calls.length > 0 && value.runs.length === 2 && value.runs.some(run => run.agentId === riley.id && run.status === "running" && run.id === value.task.executionRunId), + reject: value => value.runs.some(run => run.agentId === riley.id && ["failed", "cancelled", "succeeded", "timed_out"].includes(run.status)) ? "The conditional real checkout ended before the denial request" : undefined }); + state.heldTask = held.task; state.heldRun = held.runs.find(run => run.agentId === riley.id); state.gateCalls = structuredClone(gate.calls); + await ctx.input.evidence("chat-catalog-edit-properties-boundary-variant.json", { firstEdit: state.firstEdit, heldTask: state.heldTask, heldRun: state.heldRun, gateCalls: state.gateCalls }); + const { sendChatMessage } = await import("../chat-flow.js"); await sendChatMessage(ctx.input.page, ctx.scenario.query); + } catch (error) { await gate.close(); gate = undefined; throw error; } + }, + async observe(ctx, state): Promise { + const o = await observeInternal(ctx, state); const first = observation(state.firstEdit); const ids = new Set(first?.runs.map(run => run.id) ?? []); + const rows = ctx.issue() ? await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`) : []; + return { ...o, currentTask: await ctx.input.api.get(`/api/issues/${entity(state, "task").id}`), phaseReply: rows.filter(row => row.authorAgentId === o.leadId && !ids.has(row.createdByRunId ?? "") && o.runs.some(run => run.id === row.createdByRunId)).map(row => row.body ?? "").join("\n") }; + }, + async cleanup(ctx, state) { try { await primary.cleanup?.(ctx, state); } finally { await gate?.close(); gate = undefined; } }, + }; +} diff --git a/tests/runner-e2e/chat-catalog-domains/internal-tasks.ts b/tests/runner-e2e/chat-catalog-domains/internal-tasks.ts index bec9b246a31..e625b449e63 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-tasks.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-tasks.ts @@ -1,3 +1,4 @@ +import { withEditOwnership } from "./internal-edit-ownership.js"; import { delegatedTaskCases } from "./internal-task-creation.js"; import { taskPrivacyCases } from "./internal-privacy.js"; import { taskInventoryCases } from "./internal-task-inventory.js"; @@ -67,11 +68,11 @@ export const tasksCases: readonly ChatCatalogCase[] = [ ...taskCompletionCases, ...privateDelegationCases, ambiguousTask(), - define("tasks.edit-properties", async ctx => { + withEditOwnership(define("tasks.edit-properties", async ctx => { const p = await project(ctx); const worker = await fixtureWorker(ctx); const t = await task(ctx, "Landing page", p.id, { description: "Build the Launch landing page; this work is saved for later." }); return { project: p, worker, task: t }; - }), + })), define("tasks.comment-with-reference", async ctx => { const p = await project(ctx); const design = await task(ctx, "Approved design", p.id, { status: "done", description: "Approved design: semantic HTML, clear contrast, one primary signup action." }); const t = await task(ctx, "Landing page", p.id, { description: "Build the launch website." }); diff --git a/tests/runner-e2e/chat-catalog-domains/internal.test.ts b/tests/runner-e2e/chat-catalog-domains/internal.test.ts index c38834cbce0..f74a514461f 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal.test.ts @@ -224,7 +224,13 @@ function valid(scenario: string): InternalObservation { } break; } - case "tasks.edit-properties": o.state.worker = { id: "morgan" }; o.tasks = [{ ...o.tasks[0]!, priority: "high", assigneeAgentId: "morgan" }]; o.activity = [event("issue.updated", "task")]; break; + case "tasks.edit-properties": { + o.state.worker = { id: "morgan" }; o.tasks = [{ ...o.tasks[0]!, priority: "high", assigneeAgentId: "morgan" }]; o.activity = [event("issue.updated", "task")]; o.state.firstEdit = structuredClone(o); + const held = { ...o.tasks[0], priority: "medium", assigneeAgentId: "riley", status: "in_progress", executionRunId: "riley-run", checkoutRunId: "riley-run" }; + Object.assign(o.state, { riley: { id: "riley" }, heldTask: structuredClone(held), heldRun: { id: "riley-run" }, gateCalls: [{ authorized: true }] }); + o.tasks[0] = held; Object.assign(o, { currentTask: held, phaseReply: "Riley holds the running execution checkout. I cannot change priority or reassign to Morgan until that ownership is released." }); + o.runs.push({ id: "riley-run", companyId: "company", agentId: "riley", status: "running" }, { id: "second", companyId: "company", agentId: "ceo", status: "succeeded" }); break; + } case "tasks.ambiguous-task": { const other = { ...o.tasks[0]!, id: "other", projectId: "other-project", title: "Landing page" }; o.tasks[0] = { ...o.tasks[0]!, title: "Landing page" }; o.state.task = { ...o.tasks[0] }; o.state.otherTask = other;