From f14e6a2a62bd49106249f05a9d45cbed68cf1664 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 4 Nov 2026 09:24:44 -0600 Subject: [PATCH] fix(heartbeat): select native conversation finals by authority Co-Authored-By: Paperclip --- doc/execution-semantics.md | 10 ++- .../__tests__/heartbeat-run-summary.test.ts | 80 ++++++++++++------- server/src/services/heartbeat-run-summary.ts | 46 +++++------ .../services/heartbeat/run-completion.test.ts | 35 +++++++- .../src/services/heartbeat/run-completion.ts | 1 + .../native-runtime/native-run-finalizer.ts | 1 + 6 files changed, 114 insertions(+), 59 deletions(-) diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index bb46e117633..2ddd05e6a19 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1338,10 +1338,12 @@ failed or cancelled even when the agent already reported completed work. The final assistant message is the visible task response. Response selection runs after preceding event persistence completes; the completion summary cannot replace an available final answer. Existing fallback and explicit-comment precedence still -apply. A progress-only promise to investigate does not replace an available final -response backed by an accepted completed semantic result. Substantive explicit -outcomes keep precedence. A yielded result or missing final response does not -cause a second comment. The presentation decision records this selection. +apply for ordinary tasks. In native standing conversations, a completed final +response or accepted non-yielded semantic response takes precedence over earlier +tool comments. Those comments stay in history. Identical text reuses its existing +comment. With no qualified final, the explicit comment remains the response. +Governed waits and interaction cards keep ownership. The presentation decision +records the conversation-specific choice. Stream closure without a turn terminal is not proof of success. Event replay uses the existing source receipts and never repeats provider work merely to recover recorded output. diff --git a/server/src/__tests__/heartbeat-run-summary.test.ts b/server/src/__tests__/heartbeat-run-summary.test.ts index 6462241e590..d322df433c2 100644 --- a/server/src/__tests__/heartbeat-run-summary.test.ts +++ b/server/src/__tests__/heartbeat-run-summary.test.ts @@ -363,37 +363,61 @@ describe("resolveHeartbeatRunResponse", () => { }, }; - it("delivers the completed handoff after an explicit progress promise", () => { - const progress = "Handling the request to set your personal publishing credential. I will verify which piece is missing (definition vs. value) and direct you to the secure entry surface; personal secret values cannot be entered or read through chat."; - const summary = "Open Settings → Secrets → My secrets. Select the publishing definition and Set my value. Keep the value out of chat."; - const response = resolveHeartbeatRunResponse({ - resultJson: { nativeResult: { schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", summary } }, - existingComment: { id: "progress", body: progress }, - }); - expect(response).toMatchObject({ text: summary, decision: { chosenSource: "semantic_result_summary", commentAction: "create", - reasonCodes: ["accepted_semantic_result_summary", "completed_response_after_progress_promise"] } }); - }); - it.each([ - "The credential is configured and the worker check passed.", - "The worker check failed. Keep the credential unchanged.", - "I will check again later. The current worker check failed.", - "I will verify it later, but the deployment failed with HTTP 403.", - ])("retains a substantive explicit outcome: %s", body => { - expect(resolveHeartbeatRunResponse({ resultJson, existingComment: { id: "explicit", body } })) - .toMatchObject({ text: body, decision: { commentAction: "reuse", commentId: "explicit" } }); + "Handling the request to set your personal publishing credential. I will verify which piece is missing (definition vs. value) and direct you to the secure entry surface; personal secret values cannot be entered or read through chat.", + "Checking the configuration.", "Looking at the request.", "Fetching the current state.", + "I will check again tomorrow; the release is ready at /releases/latest.", + ])("delivers the native conversation final without classifying the earlier comment: %s", body => { + const summary = "Open Settings → Secrets → My secrets. Select the publishing definition and Set my value. Keep the value out of chat."; + expect(resolveHeartbeatRunResponse({ + resultJson: { nativeResult: { schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", summary } }, + nativeConversationFinalResponse: true, existingComment: { id: "earlier", body }, + })).toMatchObject({ text: summary, decision: { chosenSource: "semantic_result_summary", commentAction: "create", + reasonCodes: ["accepted_semantic_result_summary", "native_conversation_final_precedence"] } }); + expect(resolveHeartbeatRunResponse({ resultJson, existingComment: { id: "earlier", body } })) + .toMatchObject({ text: body, decision: { commentAction: "reuse", commentId: "earlier" } }); }); - it("retains a progress promise when no completed response is available or the run yields", () => { - const existingComment = { id: "progress", body: "I will verify the configuration." }; - for (const resultJson of [ - {}, { summary: "Unqualified legacy summary" }, - { nativeResult: { schema: "paperclip.run_result.v1", reportedWorkDisposition: "yielded", summary: "Waiting for an answer" } }, - { nativeResult: { schema: "paperclip.run_result.v1", reportedWorkDisposition: "failed", summary: "The provider failed" } }, - { nativeResult: { schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", summary: existingComment.body } }, - ]) { - expect(resolveHeartbeatRunResponse({ resultJson, existingComment })) - .toMatchObject({ text: existingComment.body, decision: { commentAction: "reuse" } }); + it.each(["done", "needs_review", "blocked", "failed"])("presents an accepted %s conversation response and reuses identical text", reportedWorkDisposition => { + const summary = "The requested work needs your attention."; + const resultJson = { nativeResult: { schema: "paperclip.run_result.v1", reportedWorkDisposition, summary } }; + expect(resolveHeartbeatRunResponse({ resultJson, nativeConversationFinalResponse: true, + existingComment: { id: "earlier", body: "I will check." } })).toMatchObject({ text: summary, decision: { commentAction: "create" } }); + expect(resolveHeartbeatRunResponse({ resultJson, nativeConversationFinalResponse: true, + existingComment: { id: "same", body: summary } })).toMatchObject({ text: summary, decision: { commentAction: "reuse", commentId: "same" } }); + }); + + it("prefers the completed provider final over the conversation semantic summary", () => { + expect(resolveHeartbeatRunResponse({ resultJson, nativeConversationFinalResponse: true, + existingComment: { id: "earlier", body: "Earlier comment" }, + finalAgentMessage: { text: "The complete answer", sourceEventId: "final", channel: "final" }, + })).toMatchObject({ text: "The complete answer", decision: { chosenSource: "final_agent_message", commentAction: "create" } }); + }); + + it("does not let compatibility narration outrank an accepted conversation result", () => { + const resultJson = { nativeResult: { schema: "paperclip.run_result.v1", reportedWorkDisposition: "blocked", summary: "The accepted handoff" }, + finalResponse: { final: true, text: "Adapter compatibility prose" }, summary: "Legacy compatibility prose" }; + expect(resolveHeartbeatRunResponse({ resultJson, nativeConversationFinalResponse: true, + existingComment: { id: "earlier", body: "Earlier comment" }, + finalAgentMessage: { text: "Unknown-channel narration", sourceEventId: "unknown", channel: "unknown" }, + })).toMatchObject({ text: "The accepted handoff", decision: { chosenSource: "semantic_result_summary" } }); + }); + + it("keeps the explicit conversation comment when no qualified final exists", () => { + for (const resultJson of [{}, { summary: "Unqualified legacy summary" }]) { + expect(resolveHeartbeatRunResponse({ resultJson, nativeConversationFinalResponse: true, + existingComment: { id: "explicit", body: "The release is ready at /releases/latest." }, + })).toMatchObject({ text: "The release is ready at /releases/latest.", decision: { commentAction: "reuse" } }); + } + }); + + it("keeps a governed interaction in control of the conversation response", () => { + for (const input of [{ resultJson, interactionOwnsResponse: true }, + { resultJson: { nativeResult: { schema: "paperclip.run_result.v1", reportedWorkDisposition: "yielded", summary: "Waiting for an answer" } } }]) { + expect(resolveHeartbeatRunResponse({ ...input, nativeConversationFinalResponse: true, + existingComment: { id: "earlier", body: "I will check." }, + finalAgentMessage: { text: "Do not publish this", sourceEventId: "final", channel: "final" }, + })).toMatchObject({ text: null, decision: { commentAction: "none" } }); } }); diff --git a/server/src/services/heartbeat-run-summary.ts b/server/src/services/heartbeat-run-summary.ts index 1f7b40c481c..e3f8e6067e4 100644 --- a/server/src/services/heartbeat-run-summary.ts +++ b/server/src/services/heartbeat-run-summary.ts @@ -127,17 +127,6 @@ export function summarizeHeartbeatRunResultJson( const NARRATION_OPENERS = /^(let me\b|i['’]ll\b|i['’]m going\b|i need to\b|i can see\b|now i['’]ll\b|next,? i['’]ll\b|looking at\b|fetching\b|checking\b|first,)/i; -// An acknowledgement followed only by a promised investigation is progress, -// even if it was posted through the ordinary comment tool. Keep multiple -// substantive sentences out of this narrow fallback. -function isProgressPromise(text: string): boolean { - const sentences = text.trim().split(/(?<=[.!?])\s+(?=[A-Z])/); - if (/^Handling (?:the|your) request\b/i.test(sentences[0] ?? "")) sentences.shift(); - return sentences.length === 1 - && /^I(?: will|['’]ll)\s+(?:check|verify|inspect|investigate|review|read|fetch)\b/i.test(sentences[0]!) - && !/\b(?:but|however|failed|succeeded|completed|saved)\b/i.test(text); -} - export const LEGACY_WITHHELD_RUN_COMMENT = "Run completed. Agent did not post a summary comment this run (transcript withheld — see run log)."; @@ -418,6 +407,7 @@ export function resolveHeartbeatRunResponse(input: { existingComment?: { id: string; body?: string | null } | null; interactionOwnsResponse?: boolean; preferFinalResponseOverExistingComment?: boolean; + nativeConversationFinalResponse?: boolean; externalChatResponseWakeSummaryAuthorized?: boolean; /** Server-only proof of the exact accepted response after durable status * finalization. Never read this capability from provider/context JSON. */ @@ -453,7 +443,7 @@ export function resolveHeartbeatRunResponse(input: { const explicitProviderFinal = input.finalAgentMessage?.channel === "final"; const compatibleTerminalAssistant = input.finalAgentMessage?.channel === "unknown"; - const resolveCompletedUpstreamResponse = () => { + const resolveCompletedUpstreamResponse = (allowCompatibility = true) => { if ( explicitProviderFinal && finalAgentText && @@ -473,7 +463,7 @@ export function resolveHeartbeatRunResponse(input: { } const adapterFinal = readMarkedAdapterFinalResponse(resultJson); - if (adapterFinal) { + if (allowCompatibility && adapterFinal) { return { text: adapterFinal, decision: decision("adapter_final_response", { @@ -484,7 +474,7 @@ export function resolveHeartbeatRunResponse(input: { } if ( - compatibleTerminalAssistant && + allowCompatibility && compatibleTerminalAssistant && finalAgentText && !isStructuredSemanticResultText(finalAgentText) ) { @@ -516,7 +506,7 @@ export function resolveHeartbeatRunResponse(input: { readCommentText(resultJson.summary) ?? readCommentText(resultJson.result) ?? readCommentText(resultJson.message); - if (legacyText && !NARRATION_OPENERS.test(legacyText.trimStart())) { + if (allowCompatibility && legacyText && !NARRATION_OPENERS.test(legacyText.trimStart())) { return { text: legacyText, decision: decision("adapter_final_response", { @@ -581,18 +571,22 @@ export function resolveHeartbeatRunResponse(input: { } const existingText = readCommentText(input.existingComment?.body); - if (input.existingComment && existingText) { - if (isProgressPromise(existingText) && semanticResultCandidates(resultJson).some(candidate => - candidate.schema === "paperclip.run_result.v1" && candidate.reportedWorkDisposition === "done" - && readCommentText(candidate.summary)) && !hasYieldedSemanticResult(resultJson)) { - const completed = resolveCompletedUpstreamResponse(); - if (completed && completed.text !== existingText) return { - ...completed, - decision: { ...completed.decision, reasonCodes: [ - ...completed.decision.reasonCodes, "completed_response_after_progress_promise", - ] }, - }; + if (input.nativeConversationFinalResponse) { + if (input.interactionOwnsResponse || hasYieldedSemanticResult(resultJson)) return { + text: null, + decision: decision("none", { commentAction: "none", reasonCodes: ["interaction_owns_conversation_response"] }), + }; + const final = resolveCompletedUpstreamResponse(false); + if (final) { + const reuse = !!input.existingComment && final.text === existingText; + return { ...final, decision: { ...final.decision, + commentAction: reuse ? "reuse" : "create", + commentId: reuse ? input.existingComment!.id : null, + reasonCodes: [...final.decision.reasonCodes, "native_conversation_final_precedence"], + } }; } + } + if (input.existingComment && existingText) { return { text: existingText, decision: decision("existing_issue_comment", { diff --git a/server/src/services/heartbeat/run-completion.test.ts b/server/src/services/heartbeat/run-completion.test.ts index 9b8a9810b2a..09e7aea56d8 100644 --- a/server/src/services/heartbeat/run-completion.test.ts +++ b/server/src/services/heartbeat/run-completion.test.ts @@ -2,7 +2,7 @@ import { createNativeUsageDelivery } from "../native-runtime/native-usage-delive import type { PrpEvent } from "../../vendor/paperclip-runner/index.js"; import { randomUUID } from "node:crypto"; import type { Db } from "@paperclipai/db"; -import { agents, agentWakeupRequests, caseHandoffs, cases, companies, costEvents, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, nativeRunFinalizations } from "@paperclipai/db"; +import { agents, agentWakeupRequests, caseHandoffs, cases, companies, costEvents, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issueComments, issues, nativeRunFinalizations } from "@paperclipai/db"; import { eq, sql } from "drizzle-orm"; import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; @@ -342,6 +342,39 @@ describe.skipIf(!support.supported && !process.env.PAPERCLIP_TEST_DATABASE_URL)( expect(f.input.onProviderTraceFinalized).not.toHaveBeenCalled(); }); + it.each([ + { runtimeMode: "native" as const, conversation: true, createsFinal: true }, + { runtimeMode: "native" as const, conversation: false, createsFinal: false }, + { runtimeMode: "legacy" as const, conversation: true, createsFinal: false }, + ])("scopes final-response precedence to native standing conversations: %j", async ({ runtimeMode, conversation, createsFinal }) => { + const f = await fixture({ runtimeMode }); + const [issue] = await db.insert(issues).values({ companyId: f.run.companyId, title: "Conversation reply", + ...(conversation ? { assigneeAgentId: f.agent.id, conversationAgentId: f.agent.id, + conversationUserId: "board-person", conversationState: "active" } : {}), + }).returning(); + const [earlier] = await db.insert(issueComments).values({ companyId: f.run.companyId, issueId: issue.id, + authorAgentId: f.agent.id, createdByRunId: f.run.id, body: "Checking the request.", + }).returning(); + f.input.issueId = issue.id; + f.input.issueRef = issue; + f.input.issueContext = await createHeartbeatRunPreparation(db).getIssueExecutionContext(f.run.companyId, issue.id); + f.input.adapterResult.resultJson = { nativeResult: { schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", summary: "The complete answer" } }; + f.deps.findRunIssueComment.mockResolvedValue(earlier); + f.deps.issuesSvc.addComment.mockImplementation(async (issueId, body) => { + const [comment] = await db.insert(issueComments).values({ companyId: f.run.companyId, issueId, + authorAgentId: f.agent.id, authorType: "agent", createdByRunId: f.run.id, body }).returning(); + return { ...comment, authorType: "agent" as const }; + }); + await f.completion.completeRun(f.input); + const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, issue.id)); + expect(comments.some(comment => comment.id === earlier.id && comment.body === earlier.body)).toBe(true); + expect(comments.filter(comment => comment.body === "The complete answer")).toHaveLength(createsFinal ? 1 : 0); + expect((await read(f.run.id)).resultJson).toMatchObject({ presentationDecision: { + chosenSource: createsFinal ? "semantic_result_summary" : "existing_issue_comment", + commentAction: createsFinal ? "create" : "reuse", + } }); + }); + it("keeps completion and session persistence moving after a response comment fails", async () => { const f = await fixture(); f.input.adapterResult.resultJson = { finalResponseDisposition: "final", finalResponse: "Completed the work." }; diff --git a/server/src/services/heartbeat/run-completion.ts b/server/src/services/heartbeat/run-completion.ts index 9a0696a1d11..22e33f90957 100644 --- a/server/src/services/heartbeat/run-completion.ts +++ b/server/src/services/heartbeat/run-completion.ts @@ -833,6 +833,7 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC await hasChatRunOwnedProviderInteraction(db, { companyId: livenessRun.companyId, issueId, runId: livenessRun.id })), conversationTurnFinished: isConversation(issueContext) && livenessRun.status === "succeeded" && persistedResultJson?.finalizationReasonCode === "conversation_turn_finished", + nativeConversationFinalResponse: livenessRun.runtimeMode === "native" && isConversation(issueContext), existingComment: existingRunComment, finalAgentMessage, preferFinalResponseOverExistingComment: diff --git a/server/src/services/native-runtime/native-run-finalizer.ts b/server/src/services/native-runtime/native-run-finalizer.ts index d39a3b682bb..ec4606dd7e0 100644 --- a/server/src/services/native-runtime/native-run-finalizer.ts +++ b/server/src/services/native-runtime/native-run-finalizer.ts @@ -959,6 +959,7 @@ async function recoverCommittedNativeChatResponseInScope( )).orderBy(desc(issueComments.createdAt), desc(issueComments.id)); const resolved = resolveHeartbeatRunResponse({ resultJson: { ...record(run.resultJson), nativeResult: result }, + nativeConversationFinalResponse: conversation, existingComment: findHeartbeatRunCompletionComment(comments, run.resultJson), finalAgentMessage, });