diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 9ba9f57077a..b9aaa5dba31 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -4702,11 +4702,12 @@ "scenarioId": "routines.run-now", "sourceFiles": [ "chat-catalog-domains/integrator-routine-invocation.ts", + "run-observations.ts", "chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/routines.yaml" ], - "oracleVersion": "paperclip.chat-catalog.integrator.v1", + "oracleVersion": "paperclip.chat-catalog.integrator.routine-invocation.v2", "expectedRunCount": 3, "expectedHumanMessageCount": 2, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-routine-invocation.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-routine-invocation.test.ts index b038778c61f..0de9fc85fca 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-routine-invocation.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-routine-invocation.test.ts @@ -2,21 +2,37 @@ import { describe, expect, it } from "vitest"; import { gradeIntegratorRoutineInvocation } from "./integrator-routine-invocation.js"; function valid() { const task = { id: "task", identifier: "ACN-2", projectId: "project", assigneeAgentId: "ceo", status: "done", title: "Launch summary" }; - const invocation = { id: "invocation", routineId: "routine", source: "manual", status: "completed", linkedIssueId: "task", idempotencyKey: "delivery", triggerPayload: { variables: { projectName: "Launch" } } }; + const invocation = { id: "invocation", routineId: "routine", source: "manual", status: "completed", linkedIssueId: "task", idempotencyKey: "delivery", + completedAt: "2026-01-01T00:02:00.000Z", triggerPayload: { variables: { projectName: "Launch" } } }; + const chatRun = { id: "chat-run", agentId: "ceo", runtimeMode: "native", status: "succeeded", contextSnapshot: { issueId: "chat" } }; + const outputRun = { id: "output-run", agentId: "ceo", runtimeMode: "native", status: "succeeded", contextSnapshot: { issueId: "task" } }; + const followUpRun = { id: "follow-up-run", agentId: "ceo", runtimeMode: "native", status: "succeeded", contextSnapshot: { issueId: "chat" } }; const base = { scenarioId: "routines.run-now", origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { project: { id: "project" } }, tasks: [task], projects: [], goals: [], comments: [], documents: [], revisions: [], humanQueries: [], activity: [], links: ["/ACN/issues/ACN-2"], routine: { id: "routine" }, invocations: [invocation], outputComments: [{ id: "output", authorAgentId: "ceo", createdByRunId: "output-run", body: "Launch: payment approval is pending." }], - runs: [{ id: "output-run", agentId: "ceo", runtimeMode: "native", status: "succeeded", contextSnapshot: { issueId: "task" } }], reply: "Accepted the Launch summary invocation; its task is waiting." }; + runs: [chatRun, outputRun], reply: "Accepted the Launch summary invocation; its task is waiting.", + toolReads: [ + { runId: "output-run", name: "paperclip_list_projects", at: "2026-01-01T00:01:30.000Z", output: '{"projects":[{"id":"project"}]}' }, + { runId: "follow-up-run", name: "paperclip_get_task", at: "2026-01-01T00:03:00.000Z", output: '{"data":{"id":"task","status":"done"}}' }, + ] }; const first = structuredClone(base); first.invocations[0]!.status = "issue_created"; first.tasks[0]!.status = "todo"; - return { ...base, state: { ...base.state, replayRunId: "invocation", replayIssueId: "task", firstInvocation: first }, + return { ...base, runs: [...base.runs, followUpRun], state: { ...base.state, replayRunId: "invocation", replayIssueId: "task", firstInvocation: first }, reply: `${first.reply}\nThe summary completed: Launch is still waiting on payment approval. Here is its result task.` }; } +type Observation = ReturnType; +const withLater = (o: Observation, later: string) => ({ ...o, reply: `${o.state.firstInvocation.reply}\n${later}` }); describe("real manual invocation and delivery distinction", () => { it("accepts substantive actual result and refuses missing receipts", () => { expect(gradeIntegratorRoutineInvocation(valid()).passed).toBe(true); expect(gradeIntegratorRoutineInvocation(undefined).passed).toBe(false); const o = valid(); o.outputComments = []; expect(gradeIntegratorRoutineInvocation(o).passed).toBe(false); + const noReads = valid(); delete (noReads as Partial).toolReads; + expect(gradeIntegratorRoutineInvocation(noReads).passed).toBe(false); + }); + it("does not depend on the wording of a correct later result", () => { + expect(gradeIntegratorRoutineInvocation(withLater(valid(), "RUN-2 is closed and nothing failed. Per its saved comment, the payment sign-off has not come through yet.")).passed).toBe(true); + expect(gradeIntegratorRoutineInvocation(withLater(valid(), "It's done — approval for payment is outstanding.")).passed).toBe(true); }); it("rejects duplicate delivery, wrong input and unrelated native output", () => { const duplicate = valid(); duplicate.invocations.push({ ...duplicate.invocations[0]!, id: "another" }); @@ -25,16 +41,28 @@ describe("real manual invocation and delivery distinction", () => { expect(gradeIntegratorRoutineInvocation(wrongInput).passed).toBe(false); const wrongReplay = valid(); wrongReplay.state.replayIssueId = "other-task"; expect(gradeIntegratorRoutineInvocation(wrongReplay).passed).toBe(false); - const wrongRun = valid(); wrongRun.runs[0]!.contextSnapshot.issueId = "other-task"; + const wrongRun = valid(); wrongRun.runs[1]!.contextSnapshot.issueId = "other-task"; expect(gradeIntegratorRoutineInvocation(wrongRun).passed).toBe(false); + const unread = valid(); unread.toolReads = unread.toolReads.filter(row => row.runId !== "output-run"); + expect(gradeIntegratorRoutineInvocation(unread).passed).toBe(false); }); - it("rejects false early completion, later acceptance-only and invented approved outcome", () => { + it("rejects false early completion and a later answer not grounded on the completed result", () => { const early = valid(); early.state.firstInvocation.reply = "The Launch summary completed successfully."; early.reply = `${early.state.firstInvocation.reply}\nThe summary completed; payment approval remains pending for Launch.`; expect(gradeIntegratorRoutineInvocation(early).passed).toBe(false); - const o = valid(); - expect(gradeIntegratorRoutineInvocation({ ...o, reply: `${o.state.firstInvocation.reply}\nI accepted the Launch summary, waiting for payment approval.` }).passed).toBe(false); - expect(gradeIntegratorRoutineInvocation({ ...o, reply: `${o.state.firstInvocation.reply}\nThe Launch summary completed. Payment is approved.` }).passed).toBe(false); + const future = valid(); future.state.firstInvocation.reply = "I started the routine; RUN-2 will be marked done once the summary is finished."; + future.reply = `${future.state.firstInvocation.reply}\nThe summary completed; payment approval remains pending.`; + expect(gradeIntegratorRoutineInvocation(future).passed).toBe(true); + const recalled = valid(); recalled.toolReads = recalled.toolReads.filter(row => row.runId !== "follow-up-run"); + expect(gradeIntegratorRoutineInvocation(recalled).passed).toBe(false); + const stale = valid(); stale.toolReads[1]!.at = "2026-01-01T00:01:00.000Z"; + expect(gradeIntegratorRoutineInvocation(stale).passed).toBe(false); + const noFollowUp = valid(); noFollowUp.runs = noFollowUp.runs.filter(run => run.id !== "follow-up-run"); + expect(gradeIntegratorRoutineInvocation(noFollowUp).passed).toBe(false); + const silent = valid(); silent.reply = silent.state.firstInvocation.reply; + expect(gradeIntegratorRoutineInvocation(silent).passed).toBe(false); + const unlinked = valid(); unlinked.links = []; + expect(gradeIntegratorRoutineInvocation(unlinked).passed).toBe(false); }); }); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-routine-invocation.ts b/tests/runner-e2e/chat-catalog-domains/integrator-routine-invocation.ts index bb9f4cc4b93..95e685a0663 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-routine-invocation.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-routine-invocation.ts @@ -3,51 +3,82 @@ import { createRoutineSchema } from "../../../packages/shared/src/validators/rou import type { RoutineDetail, RoutineRunSummary } from "../../../packages/shared/src/types/routine.js"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; import { pollUntil } from "../api.js"; +import { collectRunEvents } from "../run-observations.js"; import { checks, hasLink, observation, project, type Entity, type InternalObservation } from "./internal-support.js"; import { integratorDefinition, observeInternal, inspectResult, resultHref } from "./integrator-support.js"; -interface InvocationObservation extends InternalObservation { routine: RoutineDetail; invocations: RoutineRunSummary[]; outputComments: Entity[] } -async function observeInvocation(ctx: ChatCatalogContext, state: Record): Promise { +/** One completed product tool execution, as recorded in the run's durable event stream. */ +export interface InvocationToolRead { runId: string; name: string; at: string; output: string } +interface InvocationObservation extends InternalObservation { routine: RoutineDetail; invocations: RoutineRunSummary[]; outputComments: Entity[]; toolReads?: InvocationToolRead[] } +async function invocationToolReads(ctx: ChatCatalogContext, runs: InternalObservation["runs"]): Promise { + const reads: InvocationToolRead[] = []; + for (const run of runs) { + const events = await collectRunEvents<{ seq?: number; createdAt?: string; payload?: Record | null }>((afterSeq, limit) => + ctx.input.api.getRunObservation(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=${limit}`)); + for (const event of events) { + const prp = event.payload?.prpEvent as { eventType?: string; payload?: { name?: unknown; status?: unknown; output?: unknown } } | undefined; + if (prp?.eventType !== "tool.execution.completed" || prp.payload?.status !== "completed" || typeof prp.payload.name !== "string") continue; + const output = typeof prp.payload.output === "string" ? prp.payload.output : JSON.stringify(prp.payload.output ?? ""); + reads.push({ runId: run.id, name: prp.payload.name, at: event.createdAt ?? "", output: output.slice(0, 8192) }); + } + } + return reads; +} +async function observeInvocation(ctx: ChatCatalogContext, state: Record, withToolReads = false): Promise { const base = await observeInternal(ctx, state); const id = (state.routine as RoutineDetail).id; const invocations = await ctx.input.api.get(`/api/routines/${id}/runs`); return { ...base, routine: await ctx.input.api.get(`/api/routines/${id}`), invocations, - outputComments: invocations[0]?.linkedIssueId ? await ctx.input.api.get(`/api/issues/${invocations[0].linkedIssueId}/comments?order=asc`) : [] }; + outputComments: invocations[0]?.linkedIssueId ? await ctx.input.api.get(`/api/issues/${invocations[0].linkedIssueId}/comments?order=asc`) : [], + ...(withToolReads ? { toolReads: await invocationToolReads(ctx, base.runs) } : {}) }; } function invocationObservation(value: unknown): InvocationObservation | undefined { const o = value as InvocationObservation; return observation(value) && !!o.routine && Array.isArray(o.invocations) && Array.isArray(o.outputComments) ? o : undefined; } -function falseCompletion(reply: string) { - return /\b(?:summary|execution|task|run)\b[^.!?\n]{0,50}\b(?:completed|finished|succeeded|is ready)\b/i.test(reply) - && !/\b(?:not|hasn't|has not|isn't|not yet|pending|waiting)\b/i.test(reply); +/** Narrow contradiction guard for the first answer only: a sentence that states the summary, + * run or task as already done, without negation or a future/conditional frame. It never makes + * a valid answer depend on particular wording; it only refuses an explicit premature claim. */ +function prematureCompletionClaim(reply: string) { + return reply.split(/(?<=[.!?])\s+|\n+/).some(sentence => + /\b(?:summary|execution|task|run)\b[^.!?\n]{0,50}\b(?:completed|finished|succeeded|is ready|is done)\b/i.test(sentence) + && !/\bnot\b|n't\b|\b(?:pending|waiting|will|once|when|after|until|should|expected|queued|started|in progress)\b/i.test(sentence)); +} +/** The run read a product record naming `id` at or after `since`. Call-API payloads are redacted + * in run evidence, so only tool outputs that expose the record count as proof of the read. */ +function readRecordAfter(o: InvocationObservation, runId: string | undefined, id: string | undefined, since?: string | null) { + return !!runId && !!id && (o.toolReads ?? []).some(row => row.runId === runId && row.output.includes(id) + && (!since || Date.parse(row.at) >= Date.parse(since))); } export function gradeIntegratorRoutineInvocation(value: unknown) { const c = checks(); const o = invocationObservation(value); - c.add("actual-invocation-and-output", !!o, "Requires public invocation, task, native output and original response evidence."); + c.add("actual-invocation-and-output", !!o && Array.isArray(o.toolReads), "Requires public invocation, task, native output, tool-read and original response evidence."); if (!o) return c.result(); const invocation = o.invocations[0]; const task = o.tasks.find(row => row.id === invocation?.linkedIssueId); const first = invocationObservation(o.state.firstInvocation); const firstInvocation = first?.invocations[0]; + const projectId = (o.state.project as Entity | undefined)?.id; c.add("one-manual-launch-input", o.invocations.length === 1 && invocation?.source === "manual" && invocation.routineId === o.routine.id && invocation.triggerPayload?.variables != null && (invocation.triggerPayload.variables as Record).projectName === "Launch" - && !!task && task.projectId === (o.state.project as Entity | undefined)?.id && task.assigneeAgentId === o.leadId, + && !!task && task.projectId === projectId && task.assigneeAgentId === o.leadId, "Exactly one manual invocation retains the supplied project-name input and creates the intended owned Launch task."); c.add("same-delivery-is-not-new-intent", !!firstInvocation?.idempotencyKey && o.state.replayRunId === invocation?.id && o.state.replayIssueId === task?.id && firstInvocation.id === invocation?.id && o.invocations.length === 1, "Replaying the committed public invocation with its original key retains the same invocation and task IDs."); - c.add("accepted-versus-completed-at-first-reply", !!first && first.invocations.length === 1 - && (!falseCompletion(first.reply) || firstInvocation?.status === "completed" && first.tasks.some(row => row.id === firstInvocation.linkedIssueId && row.status === "done")), + c.add("accepted-versus-completed-at-first-reply", !!first && first.invocations.length === 1 && first.reply.trim().length > 0 + && (!prematureCompletionClaim(first.reply) || firstInvocation?.status === "completed" && first.tasks.some(row => row.id === firstInvocation.linkedIssueId && row.status === "done")), "The first answer cannot claim a completed summary before the saved invocation and task actually complete."); const outputRun = o.runs.find(run => run.contextSnapshot?.issueId === task?.id && run.agentId === o.leadId && run.status === "succeeded" && run.runtimeMode === "native"); - const output = o.outputComments.filter(row => row.authorAgentId === o.leadId && row.createdByRunId === outputRun?.id).map(row => row.body ?? "").join("\n"); - c.add("actual-completed-summary", !!task && invocation?.status === "completed" && task.status === "done" && !!outputRun && /launch/i.test(output) - && /payment/i.test(output) && /\b(?:pending|waiting|await|approval)\b/i.test(output), - "The routine's own native task actually saved the Launch summary using the pending payment fact and completed normally."); - const later = first && o.reply.startsWith(first.reply) ? o.reply.slice(first.reply.length) : ""; - c.add("later-actual-result-and-task-link", !!task && hasLink(o, "issues", task) && /launch/i.test(later) && /payment/i.test(later) - && /\b(?:pending|waiting|await|approval)\b/i.test(later) && /\b(?:completed|finished)\b|\b(?:summary|result)\b[^.!?\n]{0,25}(?:\b(?:reports?|shows?|contains?|says?|found|is)\b|:)/i.test(later) - && !/\b(?:failed|not completed|not finished|still running|only accepted|payment (?:is )?approved)\b/i.test(later), - "The explicit later user request receives the real summary result and its actual execution task, not another acceptance or an invented payment approval."); + const output = o.outputComments.filter(row => row.authorAgentId === o.leadId && row.createdByRunId === outputRun?.id && (row.body ?? "").trim().length > 0); + c.add("actual-completed-summary", !!task && invocation?.status === "completed" && task.status === "done" && !!outputRun && output.length > 0 + && readRecordAfter(o, outputRun?.id, projectId), + "The routine's own native task run read the actual Launch project, saved its summary on the task and completed normally."); + // The follow-up answer comes from a chat run that did not exist at the first reply and is not the routine's own task run. + const followUp = o.runs.find(run => run.agentId === o.leadId && run.status === "succeeded" && run.contextSnapshot?.issueId !== task?.id + && !!first && !first.runs.some(row => row.id === run.id)); + const later = first && o.reply.startsWith(first.reply) ? o.reply.slice(first.reply.length).trim() : ""; + c.add("later-actual-result-and-task-link", !!task && hasLink(o, "issues", task) && later.length > 0 && !!followUp + && readRecordAfter(o, followUp?.id, task.id, invocation?.completedAt ? new Date(invocation.completedAt).toISOString() : undefined), + "The explicit later request is answered by a new chat run that read the completed result task after the invocation finished, and the conversation links that task."); return c.result(); } /** Validate prerequisite input against the same supported public creation schema. */ @@ -60,12 +91,13 @@ export const integratorRoutineInvocationCases = [integratorDefinition("routines" const p = await project(ctx, "Launch", { description: "Synthetic Launch project. Payment approval is pending; no approval has been received." }); const routine = await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/routines`, launchSummaryRoutineInput(p.id, ctx.input.fixtures.agent.id)); return { project: p, routine }; -}, observeInvocation, gradeIntegratorRoutineInvocation, async (ctx, _state, value) => { +}, (ctx, state) => observeInvocation(ctx, state, true), gradeIntegratorRoutineInvocation, async (ctx, _state, value) => { const o = invocationObservation(value); const task = o?.tasks.find(row => row.id === o.invocations[0]?.linkedIssueId); if (!o || !task) throw new Error("Missing actual routine result task"); await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, task.title!, resultHref(o, task.identifier ?? task.id, "issues") ?? resultHref(o, task.id, "issues")); }, { - sourceFiles: ["chat-catalog-domains/integrator-routine-invocation.ts", "chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/routines.yaml"], + oracleVersion: "paperclip.chat-catalog.integrator.routine-invocation.v2", + sourceFiles: ["chat-catalog-domains/integrator-routine-invocation.ts", "run-observations.ts", "chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/routines.yaml"], expectedRunCount: 3, expectedHumanMessageCount: 2, attemptTimeoutMs: 720_000, async act(ctx, state) { const first = await pollUntil({ label: "routine invocation first actual reply", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 180_000), intervalMs: 1000,