diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json
index cbd93dc7ca6..dd586de7b22 100644
--- a/tests/runner-e2e/chat-catalog-descriptors.json
+++ b/tests/runner-e2e/chat-catalog-descriptors.json
@@ -320,7 +320,7 @@
"registered-artifact.ts",
"chat-scenarios/agent-chat.yaml"
],
- "oracleVersion": "paperclip.chat-catalog.runtime.v2",
+ "oracleVersion": "paperclip.chat-catalog.runtime-multiple-handoffs.v3",
"coverageGaps": [],
"expectedRunCount": 5,
"setupHumanMessageCount": 1,
diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.test.ts
index 1a357a740ac..3a3dd7ebbf8 100644
--- a/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.test.ts
+++ b/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.test.ts
@@ -1,21 +1,32 @@
import { expect, it } from "vitest";
import { gradeMultipleHandoffs } from "./runtime-multiple-handoffs.js";
function valid() {
- const logo = { id: "logo", title: "Logo", companyId: "company", status: "done", assigneeAgentId: "riley" };
+ const logo = { id: "logo", identifier: "RUN-1", title: "Logo", companyId: "company", status: "done", assigneeAgentId: "riley" };
const artifact = { contentVerified: true, attachmentId: "logo-file", sha256: "same", content: "" };
return { scenarioId: "agent-chat.multiple-handoffs", companyId: "company", leadId: "ceo", agents: [], links: [], activity: [], humanQueries: [], reply: "",
- tasks: [logo, { id: "website", assigneeAgentId: "morgan", status: "done" }],
+ tasks: [logo, { id: "8ba8b5a2-066a-4bb2-b591-e396942ecdf5", assigneeAgentId: "morgan", status: "done" }],
runs: [{ id: "logo-run", agentId: "riley", status: "succeeded" }, { id: "prior", agentId: "morgan", status: "succeeded" },
{ id: "setup", agentId: "ceo", status: "succeeded" }, { id: "canonical", agentId: "ceo", status: "succeeded" }, { id: "consumer", agentId: "morgan", status: "succeeded" }],
- state: { task: { id: "website" }, worker: { id: "morgan" }, logoWorker: { id: "riley" }, workerRunId: "prior", canonicalRunId: "canonical",
- queueAtReply: { issueId: "website", targetRunId: "prior", state: "queued", entries: [{ comment: { id: "headline", authorAgentId: "ceo", createdByRunId: "canonical", body: "Change the headline to Launch sooner." } }] },
+ state: { task: { id: "8ba8b5a2-066a-4bb2-b591-e396942ecdf5", identifier: "RUN-2" }, worker: { id: "morgan" }, logoWorker: { id: "riley" }, workerRunId: "prior", canonicalRunId: "canonical",
+ queueAtReply: { issueId: "8ba8b5a2-066a-4bb2-b591-e396942ecdf5", targetRunId: "prior", state: "queued", entries: [{ comment: { id: "headline", authorAgentId: "ceo", createdByRunId: "canonical", body: "Change the headline to Launch sooner." } }] },
sourceHistoryBound: true, workerRunningAtReply: true, nativeDeliveryBound: true, queueEmptyAfterDelivery: true, logoBefore: logo, logoAfter: structuredClone(logo),
logoArtifactBefore: artifact, logoArtifactAfter: { ...artifact }, logoMarker: "real-marker", logoRevisionsUnchanged: true,
- queryReply: "I saved the headline feedback in the existing Website queue while Morgan is busy. The logo remains unchanged.",
+ queryReply: "I saved the headline feedback on [RUN-2](/RUN/issues/RUN-2) while Morgan is busy. The logo [RUN-1](/RUN/issues/RUN-1) remains unchanged.",
finalHeadlineRevision: true, originalWebsiteRevisionRetained: true } };
}
it("accepts one Website delivery with real completed Logo preservation", () => expect(gradeMultipleHandoffs(valid()).passed).toBe(true));
-it("accepts natural pending-delivery wording for the exact affected work", () => { const o = valid(); o.state.queryReply = "Website's new headline requirement will wait for Morgan's next turn; I left Riley's completed logo alone."; expect(gradeMultipleHandoffs(o).passed).toBe(true); });
+it("accepts reworded replies that name the affected task by identifier or link", () => {
+ for (const reply of ["RUN-2's new headline requirement will wait for Morgan's next turn; I left Riley's completed logo alone.",
+ "Morgan will pick up \"Launch sooner\" when the current turn ends: [Website task](/RUN/issues/RUN-2#comment-headline).",
+ "Queued for Morgan on RUN-2. Nothing changed on RUN-1."]) {
+ const o = valid(); o.state.queryReply = reply; expect(gradeMultipleHandoffs(o).passed).toBe(true);
+ }
+});
+it("rejects a reply that does not name the affected Website task", () => {
+ for (const reply of ["I saved the headline change for the website.", "Saved on RUN-20 for Morgan.", "The logo RUN-1 is unchanged."]) {
+ const o = valid(); o.state.queryReply = reply; expect(gradeMultipleHandoffs(o).passed).toBe(false);
+ }
+});
it("rejects missing independent two-task evidence", () => expect(gradeMultipleHandoffs(undefined).passed).toBe(false));
it("rejects routing headline feedback to Logo instead", () => { const o = valid(); o.state.queueAtReply.issueId = "logo"; expect(gradeMultipleHandoffs(o).passed).toBe(false); });
it("rejects duplicate saved headline requirements", () => { const o = valid(); o.state.queueAtReply.entries.push({ ...o.state.queueAtReply.entries[0]! }); expect(gradeMultipleHandoffs(o).passed).toBe(false); });
@@ -24,6 +35,6 @@ it("rejects a consuming run without the exact saved comment", () => { const o =
it("rejects changing the actual delivered Logo bytes", () => { const o = valid(); o.state.logoArtifactAfter.content = ""; expect(gradeMultipleHandoffs(o).passed).toBe(false); });
it("rejects retaining a Logo filename but replacing its artifact", () => { const o = valid(); o.state.logoArtifactAfter.attachmentId = "replacement"; expect(gradeMultipleHandoffs(o).passed).toBe(false); });
it("rejects erasing an earlier Logo revision", () => { const o = valid(); o.state.logoRevisionsUnchanged = false; expect(gradeMultipleHandoffs(o).passed).toBe(false); });
-it("rejects claiming implementation before active-run consumption", () => { const o = valid(); o.state.queryReply = "The Website headline has been changed; the queued request is complete."; expect(gradeMultipleHandoffs(o).passed).toBe(false); });
+it("rejects a reply written after the active worker stopped holding the saved instruction", () => { const o = valid(); o.state.workerRunningAtReply = false; expect(gradeMultipleHandoffs(o).passed).toBe(false); });
it("rejects redirection to a different worker", () => { const o = valid(); o.tasks[1]!.assigneeAgentId = "riley"; expect(gradeMultipleHandoffs(o).passed).toBe(false); });
it("rejects losing the original Website work", () => { const o = valid(); o.state.originalWebsiteRevisionRetained = false; expect(gradeMultipleHandoffs(o).passed).toBe(false); });
diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.ts b/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.ts
index 1fa123d6f94..87013476c2d 100644
--- a/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.ts
+++ b/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.ts
@@ -10,6 +10,11 @@ import { busyRuntimeWorker, releaseRuntimeWorker } from "./runtime-busy-worker.j
import { runtimeQueueCases } from "./runtime-queue.js";
import { runtimeDefinition, runtimeObservation, observeRuntime, runtimeWorker } from "./runtime-support.js";
type Artifact = Awaited>[number];
+/** A task is named by its setup-assigned identifier (which chat links) or by a link to its page. */
+function replyNamesTask(reply: string, task: Entity) {
+ const keys = [task.identifier, task.id].filter((key): key is string => !!key);
+ return keys.some(key => new RegExp(`(? row.id === (s.task as Entity).id && row.status === "done" && row.assigneeAgentId === (s.worker as Entity).id)
&& o.runs.length === 5 && o.runs.every(row => row.status === "succeeded") && o.runs.filter(row => row.agentId === (s.worker as Entity).id).length === 2
@@ -48,7 +53,7 @@ export const runtimeMultipleHandoffCases = [runtimeDefinition("agent-chat.multip
const documents = await ctx.input.api.get(`/api/issues/${logo.id}/documents`), revisions: Entity[] = [];
for (const doc of documents) revisions.push(...await ctx.input.api.get(`/api/issues/${logo.id}/documents/${encodeURIComponent(doc.key!)}/revisions`));
const busy = await busyRuntimeWorker(ctx); await openStandingChat(ctx);
- const message = `Keep these two existing work references in this conversation: Website ${busy.task.identifier ?? busy.task.id} is assigned to Morgan and actively waiting for the brief; Logo ${logo.identifier ?? logo.id} was completed by Riley and its actual delivered logo is ${new URL(`/api/attachments/${artifact.attachmentId}/content?download=1`, ctx.input.api.baseURL).href}. Acknowledge these separate references, without creating, reassigning, editing or starting any task. Keep the Logo deliverable unchanged.`;
+ const message = `Keep these two existing work references in this conversation: Website ${busy.task.identifier ?? busy.task.id} is assigned to Morgan, who is working on it now; Logo ${logo.identifier ?? logo.id} was completed by Riley and its actual delivered logo is ${new URL(`/api/attachments/${artifact.attachmentId}/content?download=1`, ctx.input.api.baseURL).href}. Acknowledge these separate references, without creating, reassigning, editing or starting any task. Keep the Logo deliverable unchanged.`;
await sendChatMessage(ctx.input.page, message);
const saved = await pollUntil({ label: "Real source history retains separate active Website and completed Logo", deadlineAt: Date.now() + 150_000,
load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; },
@@ -58,7 +63,7 @@ export const runtimeMultipleHandoffCases = [runtimeDefinition("agent-chat.multip
return { ...busy, logoWorker, logoBefore: complete.task, logoRunId: complete.runs[0]!.id, logoMarker, logoArtifactBefore: artifact, logoDocumentsBefore: documents, logoRevisionsBefore: revisions,
sourceHistoryBound: comments.filter(row => !row.authorAgentId && row.body === message).length === 1, setupRunIds: saved.runs.map(row => row.id) };
}, gradeMultipleHandoffs, {
- setupHumanMessageCount: 1, expectedHumanMessageCount: 2, expectedRunCount: 5, attemptTimeoutMs: 1_020_000,
+ setupHumanMessageCount: 1, expectedHumanMessageCount: 2, expectedRunCount: 5, attemptTimeoutMs: 1_020_000, oracleVersion: "paperclip.chat-catalog.runtime-multiple-handoffs.v3",
sourceFiles: ["chat-catalog-domains/runtime-multiple-handoffs.ts", "chat-catalog-domains/runtime-queue.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "registered-artifact.ts", "chat-scenarios/agent-chat.yaml"],
setupPolicy: runtimeQueueCases[0]!.setupPolicy, verifySetup: runtimeQueueCases[0]!.verifySetup,
async act(ctx, state) {