From 5eb8cdab9524e59ae0faa3a6d9897ff42db0dd25 Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 12 Nov 2026 12:44:33 -0600 Subject: [PATCH] fix(runtime): post the agent's blocker reply when a retried finalization blocks the task Co-Authored-By: Claude Opus 5.5 --- .../native-blocked-reply-recovery.test.ts | 124 ++++++++++++++++++ .../native-response-presentation-query.ts | 14 +- .../native-runtime/native-run-finalizer.ts | 22 +++- 3 files changed, 155 insertions(+), 5 deletions(-) create mode 100644 server/src/services/native-runtime/native-blocked-reply-recovery.test.ts diff --git a/server/src/services/native-runtime/native-blocked-reply-recovery.test.ts b/server/src/services/native-runtime/native-blocked-reply-recovery.test.ts new file mode 100644 index 00000000000..1480d6af9c1 --- /dev/null +++ b/server/src/services/native-runtime/native-blocked-reply-recovery.test.ts @@ -0,0 +1,124 @@ +import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { agents, companies, completionContracts, createDb, heartbeatRunEvents, heartbeatRuns, issueComments, issues, + nativeRunFinalizations, statusDecisions, workAssessments, type Db } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js"; +import { NativeRunCoordinatorStore } from "./native-run-coordinator-store.js"; +import { finalizeNativeRun, recordNativeFinalizationFailure } from "./native-run-finalizer.js"; +import { nativeResponsePresentationWork } from "./native-response-presentation-work.js"; + +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_NATIVE_BLOCKED_REPLY_TEST_DATABASE_URL"); +const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); +describe.skipIf(!support.supported)("blocked task reply after a retried workspace copy-back", () => { + let database: Awaited> | undefined; + let db: Db; + beforeAll(async () => { + if (!external) database = await startEmbeddedPostgresTestDatabase("native-blocked-reply-"); + db = createDb(external ?? database!.connectionString); + }, 60_000); + afterAll(async () => { await db?.$client.end(); await database?.cleanup(); }); + + const summary = "Blocked: the deploy key for the staging bucket was revoked. Please re-issue it so I can finish the upload."; + const finalReply = "I could not finish the upload because the staging deploy key was revoked. Please re-issue it."; + + async function blockedRun(options: { finalReply?: boolean } = {}) { + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID(); + const sessionId = randomUUID(), runnerId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Blocked reply fixture", issuePrefix: companyId.slice(0, 8) }); + await db.insert(agents).values({ id: agentId, companyId, name: "Uploader", adapterType: "paperclip_runner" }); + await db.insert(issues).values({ id: issueId, companyId, title: "Upload the build", status: "in_progress", assigneeAgentId: agentId }); + const [contract] = await db.insert(completionContracts).values({ companyId, issueId, revision: 1, + schemaVersion: "paperclip.completion-contract.v1", policyVersion: "test", risk: "standard", completionAuthority: "server_arbiter", + incompleteCriteriaPolicy: "preserve_non_terminal", contractJson: { revision: "test", objective: "Upload the build", criteria: [] }, + canonicalSha256: `blocked-${runId}`, createdByActorType: "system", createdByActorId: "test" }).returning(); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: issueId, nativeSessionId: sessionId, + runnerInstanceId: runnerId, completionContractId: contract.id, completionContractSha256: contract.canonicalSha256, + status: "running", runtimeMode: "native", contextSnapshot: { issueId, source: "assignment" } }); + await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId)); + const port = new NativeRunCoordinatorStore(db, { companyId, issueId, runId, agentId, normalizedSessionId: sessionId, + runnerSourceInstanceId: runnerId, completionContractId: contract.id, completionContractSha256: contract.canonicalSha256, + completionContractRevision: "test", completionContractCriterionIds: [] }); + await port.completeRun({ result: { schema: "paperclip.run_result.v1", reportedWorkDisposition: "blocked", summary, + blocker: { reasonCode: "access", owner: { kind: "user", name: "Board operator" }, unblockAction: "Re-issue the deploy key", + scope: "task_wide" }, + completionClaim: { contractRevision: "test", objectiveSatisfied: false, criteria: [], remainingWork: [] }, + evidence: [], verification: [], attentionRequests: [], artifacts: [] }, + terminal: { schema: "paperclip.prp.terminal.v1", runTerminalState: "succeeded", turnTerminalState: "completed", + reportedWorkDisposition: "blocked" }, + turnId: "upload-turn" }); + if (options.finalReply) await db.insert(heartbeatRunEvents).values({ companyId, runId, agentId, seq: 90, eventType: "item.completed", + payload: { prpEvent: { runId, normalizedSessionId: sessionId, turnId: "upload-turn", sourceEventId: "upload-final", + payload: { kind: "agentMessage", channel: "final", providerPhase: "final", text: finalReply } } } }); + await recordNativeFinalizationFailure({ db, runId, error: new Error("native_workspace_sync_out_failed"), + failureScope: "workspace", projectRunStatus: true }); + return { companyId, agentId, issueId, runId }; + } + const retryCopyBack = (runId: string, options: { skipNativeResponsePresentation?: boolean } = {}) => + finalizeNativeRun({ db, runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true, preserveProviderAttempt: true, + ...options }); + const comments = (issueId: string) => db.select().from(issueComments).where(eq(issueComments.issueId, issueId)); + const run = async (runId: string) => (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)))[0]; + const issue = async (issueId: string) => (await db.select().from(issues).where(eq(issues.id, issueId)))[0]; + + it("posts the blocker summary once when the retried copy-back commits the blocked status", async () => { + const f = await blockedRun(); + expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)))[0].phase) + .toBe("retryable_failure"); + await retryCopyBack(f.runId); + expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)))[0].phase) + .toBe("committed"); + expect((await issue(f.issueId)).status).toBe("blocked"); + expect((await run(f.runId)).status).toBe("succeeded"); + + expect(await comments(f.issueId)).toEqual([expect.objectContaining({ body: summary, authorAgentId: f.agentId, + createdByRunId: f.runId })]); + expect(((await run(f.runId)).resultJson as Record).presentationDecision).toMatchObject({ + commentAction: "create", chosenSource: "semantic_result_summary", commentId: expect.any(String) }); + + expect(await nativeResponsePresentationWork(db).load({ companyId: f.companyId, issueId: f.issueId })).toEqual([]); + await retryCopyBack(f.runId); + expect(await comments(f.issueId)).toHaveLength(1); + }); + + it("presents the agent's final reply from the background worker exactly once", async () => { + const f = await blockedRun({ finalReply: true }); + await retryCopyBack(f.runId, { skipNativeResponsePresentation: true }); + expect((await issue(f.issueId)).status).toBe("blocked"); + expect(await comments(f.issueId)).toEqual([]); + + const work = nativeResponsePresentationWork(db); + const [entry] = await work.load({ companyId: f.companyId, issueId: f.issueId }); + expect(entry).toMatchObject({ id: f.runId }); + await expect(work.check(entry)).resolves.toBeNull(); + expect(await comments(f.issueId)).toEqual([expect.objectContaining({ body: finalReply, createdByRunId: f.runId })]); + expect(await work.load({ companyId: f.companyId, issueId: f.issueId })).toEqual([]); + await expect(work.check(entry)).resolves.toBeNull(); + expect(await comments(f.issueId)).toHaveLength(1); + }); + + it("does not present a stale blocker after a later status decision or once the task moved on", async () => { + const f = await blockedRun(); + await retryCopyBack(f.runId, { skipNativeResponsePresentation: true }); + const work = nativeResponsePresentationWork(db); + expect(await work.load({ companyId: f.companyId, issueId: f.issueId })).toHaveLength(1); + + const [decision] = await db.select().from(statusDecisions).where(eq(statusDecisions.runId, f.runId)); + const [assessment] = await db.select().from(workAssessments).where(eq(workAssessments.id, decision.assessmentId)); + const [laterAssessment] = await db.insert(workAssessments).values({ ...assessment, id: randomUUID(), + inputDigest: `${assessment.inputDigest}-later` }).returning(); + const [later] = await db.insert(statusDecisions).values({ ...decision, id: randomUUID(), assessmentId: laterAssessment.id, + decisionVersion: decision.decisionVersion + 1, decisionDigest: `${decision.decisionDigest}-later` }).returning(); + expect(await work.load({ companyId: f.companyId, issueId: f.issueId })).toEqual([]); + await retryCopyBack(f.runId); + expect(await comments(f.issueId)).toEqual([]); + + await db.delete(statusDecisions).where(eq(statusDecisions.id, later.id)); + await db.delete(workAssessments).where(eq(workAssessments.id, laterAssessment.id)); + await db.update(issues).set({ status: "todo" }).where(eq(issues.id, f.issueId)); + expect(await work.load({ companyId: f.companyId, issueId: f.issueId })).toEqual([]); + await retryCopyBack(f.runId); + expect(await comments(f.issueId)).toEqual([]); + }); +}); diff --git a/server/src/services/native-runtime/native-response-presentation-query.ts b/server/src/services/native-runtime/native-response-presentation-query.ts index 5ea7b2a3e2c..a86931c1c35 100644 --- a/server/src/services/native-runtime/native-response-presentation-query.ts +++ b/server/src/services/native-runtime/native-response-presentation-query.ts @@ -1,8 +1,10 @@ -import { QueryBuilder } from "drizzle-orm/pg-core"; +import { alias, QueryBuilder } from "drizzle-orm/pg-core"; import { and, asc, eq, exists, gt, inArray, isNotNull, isNull, ne, notExists, or, sql } from "drizzle-orm"; import { chatActions, heartbeatRunEvents, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults, statusDecisions, statusDecisionEffects } from "@paperclipai/db"; import type { NativeResponsePresentationScope } from "../native-response-presentation-notifications.js"; +const laterDecisions = alias(statusDecisions, "later_status_decisions"); + export function nativeResponsePresentationCandidates(scope?: NativeResponsePresentationScope, after?: string, runId?: string) { const db = new QueryBuilder(); return db.select({ id: heartbeatRuns.id, companyId: heartbeatRuns.companyId, @@ -11,7 +13,7 @@ export function nativeResponsePresentationCandidates(scope?: NativeResponsePrese // Durable publication retry schedule for an ordinary final task reply. publicationNextAttemptAt: sql`${heartbeatRuns.resultJson} #>> '{responsePublication,nextAttemptAt}'`.as("publication_next_attempt_at"), // Agent Chat conversations keep their completion outbox as the bounded owner. - taskReply: sql`coalesce(${nativeRunResults.resultJson} #>> '{result,reportedWorkDisposition}' in ('done', 'needs_review') + taskReply: sql`coalesce(${nativeRunResults.resultJson} #>> '{result,reportedWorkDisposition}' in ('done', 'needs_review', 'blocked') and (coalesce(${issues.conversationAgentId}::text, '') = '' or coalesce(${issues.conversationUserId}, '') = ''), false)`.as("task_reply"), }).from(nativeRunFinalizations) .innerJoin(heartbeatRuns, and(eq(heartbeatRuns.id, nativeRunFinalizations.runId), @@ -67,6 +69,14 @@ export function nativeResponsePresentationCandidates(scope?: NativeResponsePrese and(sql`(coalesce(${issues.conversationAgentId}::text, '') = '' or coalesce(${issues.conversationUserId}, '') = '')`, sql`${nativeRunResults.resultJson} #>> '{result,reportedWorkDisposition}' = 'needs_review'`, sql`${nativeRunResults.resultJson} #>> '{terminal,reportedWorkDisposition}' = 'needs_review'`), + and(sql`(coalesce(${issues.conversationAgentId}::text, '') = '' or coalesce(${issues.conversationUserId}, '') = '')`, + sql`${nativeRunResults.resultJson} #>> '{result,reportedWorkDisposition}' = 'blocked'`, + sql`${nativeRunResults.resultJson} #>> '{terminal,reportedWorkDisposition}' = 'blocked'`, + eq(issues.status, "blocked"), + notExists(db.select({ id: laterDecisions.id }).from(laterDecisions).where(and( + eq(laterDecisions.companyId, statusDecisions.companyId), eq(laterDecisions.issueId, statusDecisions.issueId), + eq(laterDecisions.applicationState, "applied"), + gt(laterDecisions.decisionVersion, statusDecisions.decisionVersion))))), and(sql`(coalesce(${issues.conversationAgentId}::text, '') <> '' and coalesce(${issues.conversationUserId}, '') <> '')`, sql`${nativeRunResults.resultJson} #>> '{result,reportedWorkDisposition}' = 'yielded'`, sql`${nativeRunResults.resultJson} #>> '{terminal,reportedWorkDisposition}' = 'yielded'`), diff --git a/server/src/services/native-runtime/native-run-finalizer.ts b/server/src/services/native-runtime/native-run-finalizer.ts index 338786cbec2..048ae3adf19 100644 --- a/server/src/services/native-runtime/native-run-finalizer.ts +++ b/server/src/services/native-runtime/native-run-finalizer.ts @@ -41,7 +41,7 @@ import { issueTreeControlService } from "../issue-tree-control.js"; import { COMPLETION_REVIEW_STAGE_REASON, completionReviewStageEntry } from "../issue-execution-policy.js"; import { randomUUID } from "node:crypto"; import { preserveNativeWorkspaceExportLease } from "./native-workspace-export-resume.js"; -import { and, desc, eq, inArray, isNotNull, isNull, or, sql } from "drizzle-orm"; +import { and, desc, eq, gt, inArray, isNotNull, isNull, or, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { agentWakeupRequests, @@ -973,6 +973,18 @@ async function acceptedResponseDigestMatches( ); } +async function hasLaterAppliedStatusDecision(db: Db, input: { + companyId: string; issueId: string; decision: { decisionVersion: number }; +}) { + const [later] = await db.select({ id: statusDecisions.id }).from(statusDecisions).where(and( + eq(statusDecisions.companyId, input.companyId), + eq(statusDecisions.issueId, input.issueId), + eq(statusDecisions.applicationState, "applied"), + gt(statusDecisions.decisionVersion, input.decision.decisionVersion), + )).limit(1); + return Boolean(later); +} + /** Recover only the already accepted answer. Neither issue disposition nor * provider state is repaired here; an unsafe native session stays quarantined. */ export async function repairCommittedNativeChatResponse( @@ -1171,6 +1183,9 @@ async function recoverCommittedNativeChatResponseInScope( return true; } const conversation = isConversation(issue); + const blockedTaskReply = !conversation && result.reportedWorkDisposition === "blocked" + && terminal.reportedWorkDisposition === "blocked" && issue.status === "blocked" + && !(await hasLaterAppliedStatusDecision(tx, { ...input, decision })); // The turn is still the conversation's unsettled current turn. A held // turn keeps whatever status its gate committed; a finished turn // preserves in_progress. @@ -1195,7 +1210,8 @@ async function recoverCommittedNativeChatResponseInScope( (!conversation && result.reportedWorkDisposition === "needs_review" && terminal.reportedWorkDisposition === "needs_review") || // A conversation turn can yield while an approval, an execution // stage or a blocker holds its task; it still answered the person. - (currentConversation && result.reportedWorkDisposition === "yielded" && terminal.reportedWorkDisposition === "yielded")) && + (currentConversation && result.reportedWorkDisposition === "yielded" && terminal.reportedWorkDisposition === "yielded") || + blockedTaskReply) && (await resolveChatRunPresentationAuthorizationReason(tx, input)) === "internal_agent_write" ) { const rows = await tx.select({ seq: heartbeatRunEvents.seq, payload: heartbeatRunEvents.payload }) @@ -1224,7 +1240,7 @@ async function recoverCommittedNativeChatResponseInScope( return candidate ? [candidate] : []; }), }); - if (!finalAgentMessage) return false; + if (!finalAgentMessage && !blockedTaskReply) return false; const comments = await tx.select({ id: issueComments.id, body: issueComments.body, presentation: issueComments.presentation }) .from(issueComments).where(and( eq(issueComments.companyId, input.companyId),