import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely, chatCatalogNativePhaseRemainingBudget } from "../chat-catalog-contract.js"; import { createRoutineSchema } from "../../../packages/shared/src/validators/routine.js"; import type { RoutineDetail, RoutineRunSummary } from "../../../packages/shared/src/types/routine.js"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; import { pollUntil } from "../api.js"; import { collectRunEvents } from "../run-observations.js"; import { apiReadDigests, recordParams, type ApiRead } from "./api-read-receipts.js"; import { checks, hasLink, observation, project, type Entity, type InternalObservation } from "./internal-support.js"; import { integratorDefinition, observeInternal, inspectResult, resultHref } from "./integrator-support.js"; /** * One completed product tool execution, as recorded in the run's durable event stream. A call_api * result's body is redacted before saving, so it is kept as its input digest, operation and status. */ export interface InvocationToolRead { runId: string; name: string; at: string; output: string; apiRead?: { inputSha256: string; operationId: string; ok: boolean; status: number } } interface InvocationObservation extends InternalObservation { routine: RoutineDetail; invocations: RoutineRunSummary[]; outputComments: Entity[]; toolReads?: InvocationToolRead[] } async function invocationToolReads(ctx: ChatCatalogContext, runs: InternalObservation["runs"]): Promise { const reads: InvocationToolRead[] = []; for (const run of runs) { const events = await collectRunEvents<{ seq?: number; createdAt?: string; payload?: Record | null }>((afterSeq, limit) => ctx.input.api.getRunObservation(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=${limit}`)); const uses = new Map(); for (const event of events) { const prp = event.payload?.prpEvent as { eventType?: string; payload?: { name?: unknown; status?: unknown; output?: unknown; item?: Record } } | undefined; const item = prp?.payload?.item; if (item?.type === "tool_use" && item.name === "call_api" && item.input?.apiPayloadRedacted === true && typeof item.input.operationId === "string") uses.set(String(item.id), { inputSha256: String(item.input.apiPayloadSha256), operationId: item.input.operationId }); const use = item?.type === "tool_result" ? uses.get(String(item.tool_use_id)) : undefined; if (use && item?.result?.apiOperationId === use.operationId && typeof item.result.status === "number") reads.push({ runId: run.id, name: "call_api", at: event.createdAt ?? "", output: "", apiRead: { ...use, ok: item.result.ok === true, status: item.result.status } }); if (prp?.eventType !== "tool.execution.completed" || prp.payload?.status !== "completed" || typeof prp.payload.name !== "string") continue; const output = typeof prp.payload.output === "string" ? prp.payload.output : JSON.stringify(prp.payload.output ?? ""); reads.push({ runId: run.id, name: prp.payload.name, at: event.createdAt ?? "", output: output.slice(0, 8192) }); } } return reads; } async function observeInvocation(ctx: ChatCatalogContext, state: Record, withToolReads = false): Promise { const base = await observeInternal(ctx, state); const id = (state.routine as RoutineDetail).id; const invocations = await ctx.input.api.get(`/api/routines/${id}/runs`); return { ...base, routine: await ctx.input.api.get(`/api/routines/${id}`), invocations, outputComments: invocations[0]?.linkedIssueId ? await ctx.input.api.get(`/api/issues/${invocations[0].linkedIssueId}/comments?order=asc`) : [], ...(withToolReads ? { toolReads: await invocationToolReads(ctx, base.runs) } : {}) }; } function invocationObservation(value: unknown): InvocationObservation | undefined { const o = value as InvocationObservation; return observation(value) && !!o.routine && Array.isArray(o.invocations) && Array.isArray(o.outputComments) ? o : undefined; } /** Narrow contradiction guard for the first answer only: a sentence that states the summary, * run or task as already done, without negation or a future/conditional frame. It never makes * a valid answer depend on particular wording; it only refuses an explicit premature claim. */ function prematureCompletionClaim(reply: string) { return reply.split(/(?<=[.!?])\s+|\n+/).some(sentence => /\b(?:summary|execution|task|run)\b[^.!?\n]{0,50}\b(?:completed|finished|succeeded|is ready|is done)\b/i.test(sentence) && !/\bnot\b|n't\b|\b(?:pending|waiting|will|once|when|after|until|should|expected|queued|started|in progress)\b/i.test(sentence)); } /** The run read a product record naming `id` at or after `since`: an unredacted tool output exposing the * record, or a successful call_api read whose saved input digest names it (the body itself is redacted). */ function readRecordAfter(o: InvocationObservation, runId: string | undefined, id: string | undefined, since: string | null | undefined, reads: readonly ApiRead[]) { const digests = apiReadDigests(reads); return !!runId && !!id && (o.toolReads ?? []).some(row => row.runId === runId && (row.output.includes(id) || !!row.apiRead && row.apiRead.ok && row.apiRead.status >= 200 && row.apiRead.status < 300 && digests.has(row.apiRead.inputSha256)) && (!since || Date.parse(row.at) >= Date.parse(since))); } export function gradeIntegratorRoutineInvocation(value: unknown) { const c = checks(); const o = invocationObservation(value); c.add("actual-invocation-and-output", !!o && Array.isArray(o.toolReads), "Requires public invocation, task, native output, tool-read and original response evidence."); if (!o) return c.result(); const invocation = o.invocations[0]; const task = o.tasks.find(row => row.id === invocation?.linkedIssueId); const first = invocationObservation(o.state.firstInvocation); const firstInvocation = first?.invocations[0]; const projectId = (o.state.project as Entity | undefined)?.id; c.add("one-manual-launch-input", o.invocations.length === 1 && invocation?.source === "manual" && invocation.routineId === o.routine.id && invocation.triggerPayload?.variables != null && (invocation.triggerPayload.variables as Record).projectName === "Launch" && !!task && task.projectId === projectId && task.assigneeAgentId === o.leadId, "Exactly one manual invocation retains the supplied project-name input and creates the intended owned Launch task."); c.add("same-delivery-is-not-new-intent", !!firstInvocation?.idempotencyKey && o.state.replayRunId === invocation?.id && o.state.replayIssueId === task?.id && firstInvocation.id === invocation?.id && o.invocations.length === 1, "Replaying the committed public invocation with its original key retains the same invocation and task IDs."); c.add("accepted-versus-completed-at-first-reply", !!first && first.invocations.length === 1 && first.reply.trim().length > 0 && (!prematureCompletionClaim(first.reply) || firstInvocation?.status === "completed" && first.tasks.some(row => row.id === firstInvocation.linkedIssueId && row.status === "done")), "The first answer cannot claim a completed summary before the saved invocation and task actually complete."); const outputRun = o.runs.find(run => run.contextSnapshot?.issueId === task?.id && run.agentId === o.leadId && run.status === "succeeded" && run.runtimeMode === "native"); const output = o.outputComments.filter(row => row.authorAgentId === o.leadId && row.createdByRunId === outputRun?.id && (row.body ?? "").trim().length > 0); c.add("actual-completed-summary", !!task && invocation?.status === "completed" && task.status === "done" && !!outputRun && output.length > 0 && readRecordAfter(o, outputRun?.id, projectId, undefined, [{ operationId: "GET /api/projects/{id}", pathParams: recordParams("id", projectId) }]), "The routine's own native task run read the actual Launch project, saved its summary on the task and completed normally."); // The follow-up answer comes from a chat run that did not exist at the first reply and is not the routine's own task run. const followUp = o.runs.find(run => run.agentId === o.leadId && run.status === "succeeded" && run.contextSnapshot?.issueId !== task?.id && !!first && !first.runs.some(row => row.id === run.id)); const later = first && o.reply.startsWith(first.reply) ? o.reply.slice(first.reply.length).trim() : ""; c.add("later-actual-result-and-task-link", !!task && hasLink(o, "issues", task) && later.length > 0 && !!followUp && readRecordAfter(o, followUp?.id, task.id, invocation?.completedAt ? new Date(invocation.completedAt).toISOString() : undefined, ["GET /api/issues/{id}", "GET /api/issues/{id}/comments"].map(operationId => ({ operationId, pathParams: recordParams("id", task.id, task.identifier), paged: true, queries: [{ order: "asc" }, { order: "desc" }] }))), "The explicit later request is answered by a new chat run that read the completed result task after the invocation finished, and the conversation links that task."); return c.result(); } /** Validate prerequisite input against the same supported public creation schema. */ export function launchSummaryRoutineInput(projectId: string, assigneeAgentId: string) { return createRoutineSchema.parse({ title: "Launch summary", projectId, assigneeAgentId, description: "Summarize the actual {{projectName}} project in one saved task comment, including its actual payment approval state. Link the project and mark this summary task done. Do not change project records or create follow-up tasks.", variables: [{ name: "projectName", type: "text", required: true }] }); } export const integratorRoutineInvocationCases = [integratorDefinition("routines", "routines.run-now", async ctx => { const p = await project(ctx, "Launch", { description: "Synthetic Launch project. Payment approval is pending; no approval has been received." }); const routine = await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/routines`, launchSummaryRoutineInput(p.id, ctx.input.fixtures.agent.id)); return { project: p, routine }; }, (ctx, state) => observeInvocation(ctx, state, true), gradeIntegratorRoutineInvocation, async (ctx, _state, value) => { const o = invocationObservation(value); const task = o?.tasks.find(row => row.id === o.invocations[0]?.linkedIssueId); if (!o || !task) throw new Error("Missing actual routine result task"); await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, task.title!, resultHref(o, task.identifier ?? task.id, "issues") ?? resultHref(o, task.id, "issues")); }, { oracleVersion: "paperclip.chat-catalog.integrator.routine-invocation.v4", sourceFiles: ["chat-catalog-domains/integrator-routine-invocation.ts", "chat-catalog-domains/api-read-receipts.ts", "run-observations.ts", "chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/routines.yaml"], expectedRunCount: 3, expectedHumanMessageCount: 2, attemptTimeoutMs: 720_000, async act(ctx, state) { const first = await pollUntil({ label: "routine invocation first actual reply", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 180_000), intervalMs: 1000, load: () => observeInvocation(ctx, state), accept: o => chatCatalogNativePhaseIsTimely(ctx) && (o.invocations.length === 1 && o.runs.some(row => row.agentId === o.leadId && row.contextSnapshot?.issueId === ctx.issue()?.id && row.status === "succeeded") && ctx.issue()?.conversationState === "waiting"), reject: o => o.invocations.length > 1 ? "Initial request created duplicate routine invocations" : undefined }); state.firstInvocation = { ...first, state: structuredClone(state) }; const invocation = first.invocations[0]!; if (!invocation.idempotencyKey || !invocation.linkedIssueId) throw new Error("Committed routine invocation has no replayable public identity"); // Independent transport-delivery probe of the existing committed request; // it must not supply another user intent or admit another execution. const replay = await ctx.input.api.post(`/api/routines/${invocation.routineId}/run`, { source: invocation.source, triggerId: invocation.triggerId, idempotencyKey: invocation.idempotencyKey, payload: invocation.triggerPayload, variables: invocation.triggerPayload?.variables, }); state.replayRunId = replay.id; state.replayIssueId = replay.linkedIssueId; await ctx.input.evidence("chat-routine-invocation-first-response.json", state.firstInvocation); await pollUntil({ label: "real routine execution completed", deadlineAt: Date.now() + 240_000, intervalMs: 1000, load: () => observeInvocation(ctx, state), accept: o => o.invocations.length === 1 && o.invocations[0]?.status === "completed" && o.runs.length === 2 && o.runs.every(run => run.status === "succeeded"), reject: o => o.invocations.length > 1 || o.runs.length > 2 ? "Delivery replay admitted unexpected additional work" : undefined }); const { sendChatMessage } = await import("../chat-flow.js"); await sendChatMessage(ctx.input.page, "What was the actual result of that summary run?", chatCatalogNativePhaseRemainingBudget(ctx)); }, })];