feat: return completed handoffs to Agent Chat (#14408)

Return completed Agent Chat handoffs through a durable outbox and scope each generated update to its supplied tasks. Add recovery, browser delivery, result access, and calibrated quality coverage.

Validated with two consecutive ten-case Claude/Codex campaigns, all CI checks, and a 5/5 review.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip authored and GitHub committed 2026-09-29 10:25:52 -05:00
1 parent 3b4b270650
commit 83076d7e7c
59 files changed
+2696 -83

No files matched your search

+1 -1
View File
@@ -91,7 +91,7 @@ For review or follow-up tasks, include the existing source materials the assigne
For status updates, distinguish recorded task status from active execution and verified progress. Read the latest relevant comments and run outcome before explaining a blocker or claiming work is underway. Use the advertised API tools for another task's details when needed. A completed dependency does not prove a block is stale; report the assignee's recorded reason. Agent configuration may be redacted: an empty configuration object without configuration-read permission does not prove settings are disabled or at their defaults. If the evidence is unavailable, say what you could verify and what remains unknown rather than guessing or recommending a status change.
Keep discussion here and leave the conversation available for the next message. Link handed-off tasks in your reply; do not make this conversation blocked by their completion or wait for them. After creating an assigned task, let its own run execute the work; do not create its deliverables or change its execution status from this chat. Reply normally and end your turn; Paperclip manages the conversation waiting state. Do not change its status, create a review confirmation just to finish a reply, mark it complete, or poll for another reply. An accepted plan authorizes handoff to execution tasks, never implementation on this conversation. Honor normal approvals. Ask mode is non-mutating. Plan mode supports research and writing/revising the plan; hand off for execution only through the normal authorized workflow.`;
Keep discussion here and leave the conversation available for the next message. Link handed-off tasks in your reply; do not make this conversation blocked by their completion or wait for them. Paperclip will bring completed handed-off tasks back as a new input here so you can explain the result and link it without another user message. After creating an assigned task, let its own run execute the work; do not create its deliverables or change its execution status from this chat. Reply normally and end your turn; Paperclip manages the conversation waiting state. Do not change its status, create a review confirmation just to finish a reply, mark it complete, or poll for another reply. An accepted plan authorizes handoff to execution tasks, never implementation on this conversation. Honor normal approvals. Ask mode is non-mutating. Plan mode supports research and writing/revising the plan; hand off for execution only through the normal authorized workflow.`;
/** A reset keeps history visible, but parked input from a stopped session cannot become a new turn. */
export function currentConversationCommentCondition() {
@@ -0,0 +1,247 @@
import { and, asc, eq, inArray, lte, sql } from "drizzle-orm";
import { agents, agentWakeupRequests, chatCompletionDeliveries as deliveries, chatTaskHandoffs as handoffs,
heartbeatRuns, issueComments, issueDocuments, issues, type Db } from "@paperclipai/db";
import { instanceSettingsService } from "./instance-settings.js";
export const CHAT_COMPLETION_WAKE_REASON = "chat_task_completed";
const MAX_ATTEMPTS = 5;
const LEASE_MS = 60_000;
type Issue = typeof issues.$inferSelect;
type Run = typeof heartbeatRuns.$inferSelect;
type Tx = Parameters<Parameters<Db["transaction"]>[0]>[0];
type Connection = Db | Tx;
const pending = ["pending", "queued"] as const;
const activeRuns = ["queued", "scheduled_retry", "running"];
function ids(run: Run): string[] {
const value = run.contextSnapshot?.chatCompletionDeliveryIds;
return Array.isArray(value) ? value.filter((v): v is string => typeof v === "string") : [];
}
export async function recordChatHandoff(tx: Connection, task: Issue, actorRunId: string | null | undefined) {
if (!actorRunId || task.conversationAgentId || !task.createdByAgentId) return;
const [run] = await tx.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, actorRunId),
eq(heartbeatRuns.companyId, task.companyId), eq(heartbeatRuns.agentId, task.createdByAgentId)));
const sourceId = run?.nativeIssueId ?? run?.contextSnapshot?.issueId;
if (typeof sourceId !== "string" || sourceId === task.id) return;
const [source] = await tx.select().from(issues).where(and(eq(issues.id, sourceId), eq(issues.companyId, task.companyId)));
if (!source?.conversationAgentId || source.conversationAgentId !== run.agentId ||
source.conversationSessionGeneration !== run.contextSnapshot?.conversationSessionGeneration) return;
await tx.insert(handoffs).values({ taskId: task.id, companyId: task.companyId,
conversationId: source.id, agentId: source.conversationAgentId, sessionGeneration: source.conversationSessionGeneration }).onConflictDoNothing();
}
/** Must run on the same transaction as status projection (including native arbitration). */
export async function recordChatCompletion(tx: Connection, before: Issue, after: Issue) {
if (before.status === after.status) return;
await tx.update(deliveries).set({ status: "superseded" }).where(and(eq(deliveries.taskId, after.id),
eq(deliveries.companyId, after.companyId), inArray(deliveries.status, [...pending])));
if (after.status !== "done") return;
const [handoff] = await tx.select().from(handoffs).where(and(eq(handoffs.taskId, after.id), eq(handoffs.companyId, after.companyId)));
if (handoff) await tx.insert(deliveries).values({ companyId: after.companyId, taskId: after.id, statusVersion: after.statusVersion }).onConflictDoNothing();
}
async function loadAudience(tx: Connection, deliveryId: string) {
const [row] = await tx.select({ delivery: deliveries, handoff: handoffs, task: issues }).from(deliveries)
.innerJoin(handoffs, and(eq(handoffs.taskId, deliveries.taskId), eq(handoffs.companyId, deliveries.companyId)))
.innerJoin(issues, and(eq(issues.id, deliveries.taskId), eq(issues.companyId, deliveries.companyId)))
.where(eq(deliveries.id, deliveryId));
if (!row) return null;
const [source] = await tx.select().from(issues).where(and(eq(issues.id, row.handoff.conversationId), eq(issues.companyId, row.handoff.companyId)));
return { ...row, source };
}
function current(row: NonNullable<Awaited<ReturnType<typeof loadAudience>>>) {
return row.task.status === "done" && !["superseded", "exhausted"].includes(row.delivery.status) &&
row.source?.conversationAgentId === row.handoff.agentId &&
row.source.conversationSessionGeneration === row.handoff.sessionGeneration;
}
async function taskResult(tx: Connection, task: Issue) {
// Never copy worker-authored titles, comments or document bodies into the
// source agent's instructions. These links and lifecycle facts are generated
// by the server; the saved work remains on its normal access-controlled task.
const documents = await tx.select({ id: issueDocuments.documentId }).from(issueDocuments)
.where(and(eq(issueDocuments.companyId, task.companyId), eq(issueDocuments.issueId, task.id))).limit(1);
return { id: task.id, identifier: task.identifier, status: task.status,
completedAt: task.completedAt, url: `/issues/${task.identifier ?? task.id}`,
hasSavedDocuments: documents.length > 0 };
}
/** A completed onboarding parent still owes the result of its own child handoff.
* This permits a reporting turn without reopening Done or reviving cancellation. */
export async function isCompletedOnboardingHandoffWake(db: Connection, input: {
companyId: string; issueId: string; agentId: string; reason: string | null;
contextSnapshot: Record<string, unknown>;
}) {
if (input.reason !== "issue_children_completed" || typeof input.contextSnapshot.completedChildIssueId !== "string") return false;
const [source] = await db.select().from(issues).where(and(eq(issues.id, input.issueId), eq(issues.companyId, input.companyId)));
if (source?.originKind !== "onboarding_first_task" || source.status !== "done" || source.assigneeAgentId !== input.agentId) return false;
const children = await db.select({ id: issues.id, status: issues.status }).from(issues)
.where(and(eq(issues.companyId, input.companyId), eq(issues.parentId, source.id)));
return children.some(child => child.id === input.contextSnapshot.completedChildIssueId && child.status === "done") &&
children.every(child => ["done", "cancelled"].includes(child.status));
}
/** Freeze the input at turn start. New completions cannot be consumed by an already-running turn. */
export async function prepareChatCompletionTurn(db: Db, run: Run): Promise<Run> {
const issueId = run.contextSnapshot?.issueId;
if (typeof issueId !== "string") return run;
return db.transaction(async tx => {
const [source] = await tx.select().from(issues).where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId))).for("update");
if (!source) return run;
if (source.originKind === "onboarding_first_task" && run.contextSnapshot?.wakeReason === "issue_children_completed") {
const children = await tx.select().from(issues).where(and(eq(issues.companyId, run.companyId), eq(issues.parentId, source.id), eq(issues.status, "done"))).orderBy(issues.createdAt).limit(20);
const contextSnapshot = { ...run.contextSnapshot, onboardingCompletion: true,
chatCompletionUpdates: await Promise.all(children.map(child => taskResult(tx, child))) };
await tx.update(heartbeatRuns).set({ contextSnapshot }).where(eq(heartbeatRuns.id, run.id));
return { ...run, contextSnapshot };
}
const initialIds = ids(run);
// Only completion wakes absorb pending events; ordinary user turns retain their existing input.
if (initialIds.length === 0 && run.contextSnapshot?.wakeReason !== CHAT_COMPLETION_WAKE_REASON) return run;
if (run.contextSnapshot?.conversationSessionGeneration !== source.conversationSessionGeneration) throw new Error("chat_completion_superseded");
const rows = await tx.select({ delivery: deliveries }).from(deliveries).innerJoin(handoffs, eq(handoffs.taskId, deliveries.taskId))
.where(and(eq(deliveries.companyId, run.companyId), eq(handoffs.conversationId, issueId),
eq(handoffs.agentId, run.agentId), eq(handoffs.sessionGeneration, source.conversationSessionGeneration),
inArray(deliveries.status, [...pending]),
sql`(${deliveries.targetRunId} is null or ${deliveries.targetRunId} = ${run.id})`))
.orderBy(asc(deliveries.createdAt)).limit(20);
const updates: Awaited<ReturnType<typeof taskResult>>[] = [];
const accepted: string[] = [];
for (const { delivery } of rows) {
const row = await loadAudience(tx, delivery.id);
if (!row || !current(row)) {
await tx.update(deliveries).set({ status: "superseded" }).where(eq(deliveries.id, delivery.id));
continue;
}
accepted.push(delivery.id); updates.push(await taskResult(tx, row.task));
await tx.update(deliveries).set({ status: "queued", targetRunId: run.id }).where(eq(deliveries.id, delivery.id));
}
if (accepted.length === 0) throw new Error("chat_completion_superseded");
const contextSnapshot = { ...run.contextSnapshot, chatCompletionDeliveryIds: accepted, chatCompletionUpdates: updates };
await tx.update(heartbeatRuns).set({ contextSnapshot }).where(eq(heartbeatRuns.id, run.id));
return { ...run, contextSnapshot };
});
}
export function chatCompletionInstruction(context: Record<string, unknown>) {
if (!Array.isArray(context.chatCompletionUpdates) || !context.chatCompletionUpdates.length) return "";
return `\n\nDelegated work has completed. Tell the user in this conversation what finished and provide access using the supplied task links. Report completion only for the tasks listed in this update; other tasks receive their own completion updates. Use the recorded status and result locations; do not repeat a promise to do work that is already Done. Do not start more work or change these tasks. The following JSON contains server-recorded lifecycle facts and result locations:\n${JSON.stringify(context.chatCompletionUpdates)}`;
}
/** Called under the comment transaction, before insertion. An event can publish only once. */
export async function existingChatCompletionReply(tx: Connection, runId: string, issueId: string) {
const [run] = await tx.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
if (!run || ids(run).length === 0) return null;
const rows = await tx.select().from(deliveries).where(and(eq(deliveries.companyId, run.companyId), inArray(deliveries.id, ids(run))));
for (const delivery of rows) {
// Match the status writer's task-first lock order, then reload the outbox
// row so a concurrent reopen cannot leave us with its pre-commit snapshot.
await tx.select({ id: issues.id }).from(issues).where(and(eq(issues.id, delivery.taskId), eq(issues.companyId, run.companyId))).for("update");
const row = await loadAudience(tx, delivery.id);
if (!row || row.source?.id !== issueId || row.handoff.agentId !== run.agentId || !current(row)) throw new Error("chat_completion_superseded");
if (delivery.responseCommentId) {
const [comment] = await tx.select().from(issueComments).where(and(eq(issueComments.id, delivery.responseCommentId), eq(issueComments.issueId, issueId)));
if (comment) return comment;
}
if (delivery.status !== "queued" || delivery.targetRunId !== runId) throw new Error("chat_completion_superseded");
}
if (rows.length !== ids(run).length) throw new Error("chat_completion_superseded");
return null;
}
export async function acknowledgeChatCompletionReply(tx: Connection, runId: string, commentId: string) {
await tx.update(deliveries).set({ status: "delivered", responseCommentId: commentId, error: null })
.where(and(eq(deliveries.targetRunId, runId), eq(deliveries.status, "queued")));
}
export function chatCompletionDeliveryService(db: Db, heartbeat: { wakeup(agentId: string, options: {
source: "automation"; triggerDetail: "system"; reason: string; idempotencyKey: string; allowRunCoalescing: boolean;
requestedByActorType: "system"; requestedByActorId: string; payload: Record<string, unknown>; contextSnapshot: Record<string, unknown>;
}): Promise<{ id: string } | null> }) {
async function deliver(id: string): Promise<void> {
// Lease outbox work before leaving the transaction; recovery reuses the same wake key.
const claimed = await db.update(deliveries).set({ nextAttemptAt: new Date(Date.now() + LEASE_MS) })
.where(and(eq(deliveries.id, id), inArray(deliveries.status, [...pending]), lte(deliveries.nextAttemptAt, new Date())))
.returning().then(rows => rows[0]);
if (!claimed) return;
try {
const row = await loadAudience(db, id);
if (!row || !current(row)) {
await db.update(deliveries).set({ status: "superseded" }).where(eq(deliveries.id, id)); return;
}
if (!(await instanceSettingsService(db).getExperimental()).enableAgentChat) return;
const [agent] = await db.select().from(agents).where(and(eq(agents.id, row.handoff.agentId), eq(agents.companyId, claimed.companyId)));
if (!agent || ["paused", "terminated"].includes(agent.status)) return;
const key = `chat-completion:${id}:${claimed.attempts}`;
const wakes = await db.select().from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, claimed.companyId), eq(agentWakeupRequests.idempotencyKey, key)));
const runId = claimed.targetRunId ?? wakes.find(w => w.runId)?.runId;
if (runId) {
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
if (run && activeRuns.includes(run.status)) return;
// A published reply is acknowledged in its own transaction, independent of run outcome.
const [latest] = await db.select().from(deliveries).where(eq(deliveries.id, id));
if (!latest || !pending.includes(latest.status as typeof pending[number])) return;
await db.update(deliveries).set({ status: claimed.attempts + 1 >= MAX_ATTEMPTS ? "exhausted" : "pending",
attempts: claimed.attempts + 1, targetRunId: null, nextAttemptAt: new Date(),
error: run?.error ?? "Completion turn ended without a reply" })
.where(and(eq(deliveries.id, id), inArray(deliveries.status, [...pending])));
// The failed run is already terminal. Reclaim the advanced attempt now;
// a second lease interval would only delay restart recovery. The atomic
// claim and fresh wake key still serialize competing sweepers.
if (claimed.attempts + 1 < MAX_ATTEMPTS) await deliver(id);
return;
}
if (wakes.some(w => ["queued", "claimed", "deferred_issue_execution", "coalesced"].includes(w.status))) return;
if (wakes.length) {
// Terminal receipts must advance the key; reusing them collides forever.
const dailyCap = wakes.some(w => w.reason?.startsWith("heartbeat.daily_"));
const next = new Date(Date.now() + LEASE_MS);
if (dailyCap) next.setUTCHours(24, 0, 0, 0);
await db.update(deliveries).set({ attempts: claimed.attempts + 1, targetRunId: null,
status: !dailyCap && claimed.attempts + 1 >= MAX_ATTEMPTS ? "exhausted" : "pending",
nextAttemptAt: next, error: wakes[0].reason ?? "Completion wake did not start" })
.where(and(eq(deliveries.id, id), inArray(deliveries.status, [...pending])));
return;
}
// One undispatched head per conversation. Its turn absorbs the remaining
// events at admission; events arriving after admission need a later turn.
const siblings = await db.select({ delivery: deliveries, run: heartbeatRuns }).from(deliveries)
.innerJoin(handoffs, eq(handoffs.taskId, deliveries.taskId))
.leftJoin(heartbeatRuns, eq(heartbeatRuns.id, deliveries.targetRunId))
.where(and(eq(deliveries.companyId, claimed.companyId), eq(handoffs.conversationId, row.handoff.conversationId),
eq(handoffs.sessionGeneration, row.handoff.sessionGeneration), inArray(deliveries.status, [...pending])))
.orderBy(asc(deliveries.createdAt), asc(deliveries.id));
if (siblings.some(s => s.run && activeRuns.includes(s.run.status) &&
!Array.isArray(s.run.contextSnapshot?.chatCompletionUpdates))) return;
if (siblings.find(s => !s.delivery.targetRunId)?.delivery.id !== id) return;
const run = await heartbeat.wakeup(row.handoff.agentId, {
source: "automation", triggerDetail: "system", reason: CHAT_COMPLETION_WAKE_REASON, idempotencyKey: key, allowRunCoalescing: false,
requestedByActorType: "system", requestedByActorId: "chat_completion_delivery",
payload: { issueId: row.handoff.conversationId, chatCompletionDeliveryIds: [id] },
contextSnapshot: { issueId: row.handoff.conversationId, taskId: row.handoff.conversationId,
wakeReason: CHAT_COMPLETION_WAKE_REASON, chatCompletionDeliveryIds: [id], conversationSessionGeneration: row.handoff.sessionGeneration },
});
if (!run) {
const receipts = await db.select({ id: agentWakeupRequests.id }).from(agentWakeupRequests)
.where(and(eq(agentWakeupRequests.companyId, claimed.companyId), eq(agentWakeupRequests.idempotencyKey, key)));
if (!receipts.length) throw new Error("Completion wake returned no run or durable receipt");
}
if (run) await db.update(deliveries).set({ targetRunId: run.id, status: "queued" })
.where(and(eq(deliveries.id, id), inArray(deliveries.status, [...pending]), sql`${deliveries.targetRunId} is null`));
} catch (error) {
const receipts = await db.select({ id: agentWakeupRequests.id }).from(agentWakeupRequests)
.where(and(eq(agentWakeupRequests.companyId, claimed.companyId), eq(agentWakeupRequests.idempotencyKey, `chat-completion:${id}:${claimed.attempts}`)));
await db.update(deliveries).set({ error: error instanceof Error ? error.message : String(error),
...(receipts.length ? {} : { attempts: claimed.attempts + 1,
status: claimed.attempts + 1 >= MAX_ATTEMPTS ? "exhausted" as const : "pending" as const }) })
.where(and(eq(deliveries.id, id), inArray(deliveries.status, [...pending])));
}
}
async function sweepPending(scope?: { companyId: string; taskId: string }) {
const due = await db.select({ id: deliveries.id }).from(deliveries)
.where(and(inArray(deliveries.status, [...pending]), lte(deliveries.nextAttemptAt, new Date()),
...(scope ? [eq(deliveries.companyId, scope.companyId), eq(deliveries.taskId, scope.taskId)] : [])))
.orderBy(asc(deliveries.nextAttemptAt)).limit(100);
for (const row of due) await deliver(row.id);
}
return { deliver, sweepPending };
}
@@ -390,6 +390,26 @@ const support = await getEmbeddedPostgresTestSupport();
await db.update(issues).set({ status: "in_progress" }).where(eq(issues.id, issueId));
}
});
it.each(["completed", "cancelled", "ordinary", "unfinished-child"])("admits only verified onboarding result reporting: %s", async kind => {
const [before] = await db.select().from(issues).where(eq(issues.id, issueId));
const childId = randomUUID();
await db.update(issues).set({ status: kind === "cancelled" ? "cancelled" : "done",
originKind: kind === "ordinary" ? "manual" : "onboarding_first_task" }).where(eq(issues.id, issueId));
await db.insert(issues).values({ id: childId, companyId, parentId: issueId, title: "Saved result",
status: kind === "unfinished-child" ? "in_progress" : "done", assigneeAgentId: agentId });
const report = () => buildExecutionContinuation({ db, companyId, issueId, agentId,
context: { wakeReason: "issue_children_completed", completedChildIssueId: childId },
summary: null, exposeLowTrustRaw: false });
try {
if (kind === "completed") await expect(report()).resolves.toMatchObject({ companyId, issueId });
else await expectStaleContinuation(report, "continuation_task_ownership_changed");
expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0].status)
.toBe(kind === "cancelled" ? "cancelled" : "done");
} finally {
await db.delete(issues).where(eq(issues.id, childId));
await db.update(issues).set({ status: before.status, originKind: before.originKind }).where(eq(issues.id, issueId));
}
});
it("rejects another company and an invalidated task owner", async () => {
await expectStaleContinuation(
() =>
@@ -15,6 +15,7 @@ import { sanitizeQuarantinedCommentForHigherTrust } from "./source-trust.js";
import { hasConversationContinuationPolicy } from "./conversation-continuation.js";
import { queuedCommentIdsFromWakePayload } from "./issue-queued-comment-queue.js";
import { childReviewOutcomes } from "./native-runtime/child-review-outcomes.js";
import { isCompletedOnboardingHandoffWake } from "./chat-completion-delivery.js";
export class StaleExecutionContinuationError extends Error {
constructor(readonly code: "continuation_task_ownership_changed") {
@@ -160,7 +161,11 @@ export async function buildExecutionContinuation(input: {
if (
!issue ||
issue.assigneeAgentId !== input.agentId ||
["done", "cancelled"].includes(issue.status)
issue.status === "cancelled" ||
(issue.status === "done" && !await isCompletedOnboardingHandoffWake(db, {
companyId, issueId, agentId: input.agentId,
reason: string(input.context.wakeReason), contextSnapshot: input.context,
}))
)
throw new StaleExecutionContinuationError("continuation_task_ownership_changed");
const rows = await db
@@ -1,6 +1,6 @@
import { hasWorkspaceRestoreFailure } from "@paperclipai/shared";
import { randomUUID } from "node:crypto";
import { conversationRecoveryActionPredicate, getConversationOwnershipBlocker } from "./conversation-continuation.js";
import { claimedAdapterType, conversationRecoveryActionPredicate, getConversationOwnershipBlocker } from "./conversation-continuation.js";
import { persistActivity } from "./activity-log.js";
import { appendHeartbeatRunEvent } from "./heartbeat-run-events.js";
import { logger } from "../middleware/logger.js";
@@ -481,6 +481,7 @@ export async function settleUnrecoverableExecutions(
: "Automatic recovery stopped. Recorded work is preserved; actions with unverified outcomes will not be repeated."
: "Recovery closed because the task's owner, execution, or status changed. No work was replayed.";
let nativeFailureBlock = action.evidence.nativeFailureBlock;
let nativeBootstrapFailureBlock = action.evidence.nativeBootstrapFailureBlock;
if (current) {
const [projected] = await tx
.update(issues)
@@ -496,6 +497,10 @@ export async function settleUnrecoverableExecutions(
if (task.status !== "blocked" && run.runtimeMode === "native") {
nativeFailureBlock = { runId: run.id, statusVersion: projected!.statusVersion };
}
if (task.status !== "blocked" && run.runtimeMode === "legacy" && !run.runtimeModeResolvedAt &&
run.errorCode === "server_shutdown_interrupted" && claimedAdapterType(run) === "paperclip_runner") {
nativeBootstrapFailureBlock = { runId: run.id, statusVersion: projected!.statusVersion, previousStatus: task.status };
}
}
await tx
.update(issueRecoveryActions)
@@ -511,6 +516,7 @@ export async function settleUnrecoverableExecutions(
evidence: {
...action.evidence,
...(nativeFailureBlock ? { nativeFailureBlock } : {}),
...(nativeBootstrapFailureBlock ? { nativeBootstrapFailureBlock } : {}),
automaticRecovery: {
policy: "preserve_without_replay_v1",
runId: run.id,
+36 -7
View File
@@ -1,3 +1,4 @@
import { CHAT_COMPLETION_WAKE_REASON, prepareChatCompletionTurn, chatCompletionInstruction, isCompletedOnboardingHandoffWake } from "./chat-completion-delivery.js";
import { isAgentDirectoryCopy } from "./agent-directory-working-copies.js";
import type { PaperclipTurnContext } from "@paperclipai/adapter-utils/server-utils";
@@ -36,6 +37,7 @@ import { claimQueuedNativeReviewRun } from "./native-runtime/native-review-dispa
import { buildNativeReviewRequest } from "./native-runtime/native-review-prompt.js";
import {
legacyExecutionNeedsReconciliation,
settleInterruptedNativeBootstrap,
terminalizeLegacyExecution,
} from "./legacy-execution-recovery.js";
import {
@@ -1244,6 +1246,7 @@ function mergeAdapterRecoveryMetadata(input: {
};
}
const RUNNING_ISSUE_WAKE_REASONS_REQUIRING_FOLLOWUP = new Set([
CHAT_COMPLETION_WAKE_REASON,
"approval_approved",
ISSUE_BLOCKERS_RESOLVED_WAKE_REASON,
"issue_recovery_action_restored",
@@ -7027,6 +7030,7 @@ export function shouldQueueFollowupForRunningIssueWake(input: {
return true;
}
const wakeReason = readNonEmptyString(input.contextSnapshot?.wakeReason);
if (wakeReason === "issue_children_completed" && input.contextSnapshot?.onboardingCompletion === true) return true;
return Boolean(
wakeReason && RUNNING_ISSUE_WAKE_REASONS_REQUIRING_FOLLOWUP.has(wakeReason),
);
@@ -8100,6 +8104,7 @@ export async function buildPaperclipWakePayload(input: {
const payload = {
reason: readNonEmptyString(input.contextSnapshot.wakeReason),
executionContinuation: input.contextSnapshot.executionContinuation ?? null,
chatCompletionUpdates: input.contextSnapshot.chatCompletionUpdates ?? null,
attachmentOmissions,
externalChatProvider,
recovery:
@@ -14410,6 +14415,10 @@ export function heartbeatService(
agent: typeof agents.$inferSelect,
now: Date,
) {
// Completion deliveries own their durable bounded retry and reply identity.
// A second process-loss retry would compete for the same outbox input.
if (run.contextSnapshot?.wakeReason === CHAT_COMPLETION_WAKE_REASON &&
Array.isArray(run.contextSnapshot?.chatCompletionDeliveryIds)) return null;
// Native sessions have their own fenced same-run controller. Legacy
// bootstrap recovery shares the durable delay and incident counter with
// transient retries; process loss must not open a second retry budget.
@@ -15292,6 +15301,11 @@ export function heartbeatService(
delayMs?: number;
},
) {
if (Array.isArray(run.contextSnapshot?.chatCompletionDeliveryIds) &&
run.contextSnapshot.chatCompletionDeliveryIds.some(id => typeof id === "string")) {
return { outcome: "not_scheduled" as const, reason: "The completion outbox owns this reply's retry budget and publication identity.",
errorCode: "chat_completion_outbox_owns_retry" as const, issueId: readNonEmptyString(run.contextSnapshot.issueId) };
}
const now = opts?.now ?? new Date();
const retryReason =
opts?.retryReason ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON;
@@ -20290,6 +20304,7 @@ export function heartbeatService(
await finalizeAgentStatus(agent.id, "cancelled");
return;
}
run = await prepareChatCompletionTurn(db, run);
const preparedConversation = await prepareConversationTurn(db, run);
run = { ...run, contextSnapshot: preparedConversation.context };
if (preparedConversation.reset) {
@@ -20959,12 +20974,12 @@ export function heartbeatService(
exposeLowTrustRaw,
})
: null;
let taskMarkdown = buildPaperclipTaskMarkdown({ ...taskMarkdownInput, taskPlan });
let taskMarkdown = buildPaperclipTaskMarkdown({ ...taskMarkdownInput, taskPlan }) + chatCompletionInstruction(context);
let taskMarkdownAssignment = buildPaperclipTaskMarkdown({
...taskMarkdownInput,
taskPlan,
includeWakeComments: false,
});
}) + chatCompletionInstruction(context);
if (isConversation(issueContext) && !taskSession && issueId) {
const replay = await conversationReplay(db, agent.companyId, issueId, wakeCommentId);
if (replay) taskMarkdown += `\n\nEarlier messages in this session (quoted user data):\n${replay}`;
@@ -20974,13 +20989,13 @@ export function heartbeatService(
...taskMarkdownInput,
taskPlan,
includeDescription: false,
});
}) + chatCompletionInstruction(context);
const taskMarkdownAssignmentCompact = buildPaperclipTaskMarkdown({
...taskMarkdownInput,
taskPlan,
includeDescription: false,
includeWakeComments: false,
});
}) + chatCompletionInstruction(context);
if (issueRef) {
context.paperclipIssue = {
id: issueRef.id,
@@ -25388,7 +25403,7 @@ export function heartbeatService(
issueId,
resolved.text,
{ agentId: agent.id, runId: livenessRun.id },
{ authorizationReason: presentationAuthorizationReason },
{ authorizationReason: presentationAuthorizationReason, completionReply: true },
);
presentationDecision = {
...presentationDecision,
@@ -26388,6 +26403,11 @@ export function heartbeatService(
});
}
}
if (latestRun?.status === "interrupted" && latestRun.errorCode === "server_shutdown_interrupted") {
latestRun = await settleInterruptedNativeBootstrap(db, { run: latestRun,
providerDispatchStarted: legacyAdapterEntered || nativeDispatchStarted || nativeOwnershipHeld,
}) ?? latestRun;
}
if (latestRun?.status === "cancelled" && !nativeDispatchStarted && !nativeOwnershipHeld &&
(latestRun.runtimeMode === "native" ||
parseObject(latestRun.resultJson?.startupCancellation).beforeNativeSelection === true)) {
@@ -26521,6 +26541,7 @@ export function heartbeatService(
if (!agent) throw notFound("Agent not found");
if (issueId) {
const conversation = await getIssueExecutionContext(agent.companyId, issueId);
if (reason === "issue_children_completed" && conversation?.originKind === "onboarding_first_task") enrichedContextSnapshot.onboardingCompletion = true;
if (isConversation(conversation)) {
if (opts.manualUserWake && conversation!.conversationUserId !== opts.requestedByActorId) {
throw new HttpError(403, "Only the conversation owner can start a chat run");
@@ -26528,7 +26549,7 @@ export function heartbeatService(
if (isConversationExecutionWake(conversation, reason ?? readNonEmptyString(enrichedContextSnapshot.wakeReason))) return null;
if (agent.id !== conversation!.conversationAgentId) return null;
if (!(await instanceSettings.getExperimental()).enableAgentChat) return null;
if (!wakeCommentId && isWaitingConversation(conversation) && !hasInteractionContinuationWakeContext(enrichedContextSnapshot)) return null;
if (!wakeCommentId && isWaitingConversation(conversation) && !hasInteractionContinuationWakeContext(enrichedContextSnapshot) && reason !== CHAT_COMPLETION_WAKE_REASON) return null;
}
}
if (agent.adapterType === "paperclip_runner") {
@@ -28801,10 +28822,12 @@ export function heartbeatService(
continue;
}
let completedOnboardingGuard: WakeupOptions["issueStateGuard"];
if (issueId) {
const targetIssue = await db
.select({
status: issues.status,
statusVersion: issues.statusVersion,
assigneeAgentId: issues.assigneeAgentId,
})
.from(issues)
@@ -28822,9 +28845,14 @@ export function heartbeatService(
contextSnapshot: wakeContext,
})
: null;
const onboardingResultReport = targetIssue?.status === "done" && await isCompletedOnboardingHandoffWake(db, {
companyId: candidate.companyId, issueId, agentId: candidate.agentId,
reason: candidate.reason, contextSnapshot: wakeContext,
});
if (onboardingResultReport) completedOnboardingGuard = { assigneeAgentId: candidate.agentId, statuses: ["done"], statusVersion: targetIssue!.statusVersion };
if (
!targetIssue ||
["done", "cancelled"].includes(targetIssue.status) ||
(["done", "cancelled"].includes(targetIssue.status) && !onboardingResultReport) ||
(targetIssue.assigneeAgentId !== candidate.agentId && !nativeReview) ||
(candidate.reason === "native_completion_review" && !nativeReview)
) {
@@ -28856,6 +28884,7 @@ export function heartbeatService(
idempotencyKey: candidate.idempotencyKey,
requestedByActorType: "system",
requestedByActorId: dispatchActorId,
...(completedOnboardingGuard ? { issueStateGuard: completedOnboardingGuard } : {}),
contextSnapshot: {
...wakeContext,
...(issueId ? { issueId, taskId: issueId } : {}),
+16 -2
View File
@@ -1,3 +1,4 @@
import { recordChatHandoff, recordChatCompletion, existingChatCompletionReply, acknowledgeChatCompletionReply } from "./chat-completion-delivery.js";
import { mirrorSlackBoardComment, slackBoardReplyBindings } from "./slack-board-messages.js";
import { assertAgentRunWriteAllowed } from "../agent-run-cancellation.js";
import { externalConversationStateSql, nonIdleSlackIssueCondition, resumeSlackConversation } from "./slack-conversation-state.js";
@@ -9126,6 +9127,7 @@ export function issueService(db: Db) {
.select({
id: issues.id,
conversationAgentId: issues.conversationAgentId,
originKind: issues.originKind,
assigneeAgentId: issues.assigneeAgentId,
status: issues.status,
companyId: issues.companyId,
@@ -9133,7 +9135,8 @@ export function issueService(db: Db) {
.from(issues)
.where(eq(issues.id, parentIssueId))
.then((rows) => rows[0] ?? null);
if (!parent || parent.conversationAgentId || !parent.assigneeAgentId || ["backlog", "done", "cancelled"].includes(parent.status)) {
if (!parent || parent.conversationAgentId || !parent.assigneeAgentId || ["backlog", "cancelled"].includes(parent.status) ||
(parent.status === "done" && parent.originKind !== "onboarding_first_task")) {
return null;
}
@@ -9205,6 +9208,7 @@ export function issueService(db: Db) {
}));
return {
onboardingCompletion: parent.originKind === "onboarding_first_task",
id: parent.id,
assigneeAgentId: parent.assigneeAgentId,
childIssueIds: children.map((child) => child.id),
@@ -10159,6 +10163,7 @@ export function issueService(db: Db) {
);
const [issue] = await tx.insert(issues).values(values).returning();
await recordChatHandoff(tx, issue, actorRunId);
if (idempotencyKey) {
await tx.insert(issueCreateIdempotencyKeys).values({
companyId,
@@ -10945,6 +10950,7 @@ export function issueService(db: Db) {
.returning()
.then((rows: Array<typeof issues.$inferSelect>) => rows[0] ?? null);
if (!updated) return null;
await recordChatCompletion(tx, receiptExisting, updated);
// An operator explicitly choosing a disposition owns that decision,
// including choosing In Review while the conversation is Idle.
if (actorUserId && issueData.status !== undefined) {
@@ -12114,6 +12120,8 @@ export function issueService(db: Db) {
metadata?: IssueCommentMetadata | null;
attachmentIds?: string[];
authorizationReason?: string | null;
/** Server-only final assistant response, never a tool/progress comment. */
completionReply?: boolean;
sourceTrust?: typeof issueComments.$inferInsert.sourceTrust;
createdAt?: Date | string | null;
clientRequestId?: string;
@@ -12149,7 +12157,7 @@ export function issueService(db: Db) {
.where(eq(issues.id, issueId));
// Caller-owned transactions (including chat and review comments) must
// serialize with question creation before inserting the human comment.
const issue = await (actor.userId ? issueQuery.for("update") : issueQuery)
const issue = await (actor.userId || (actor.runId && dbOrTx !== db) ? issueQuery.for("update") : issueQuery)
.then((rows: Array<{ companyId: string; conversationAgentId: string | null }>) => rows[0] ?? null);
if (!issue) throw notFound("Issue not found");
@@ -12181,6 +12189,10 @@ export function issueService(db: Db) {
throw conflict("Conversation session changed; this reply belongs to an earlier session");
}
}
if (options?.completionReply && actor.agentId && actor.runId) {
const delivered = await existingChatCompletionReply(dbOrTx, actor.runId, issueId);
if (delivered) return redactIssueComment(delivered, currentUserRedactionOptions.enabled);
}
const authorType = issueCommentAuthorTypeSchema.parse(
options?.authorType ??
(actor.agentId ? "agent" : actor.userId ? "user" : "system"),
@@ -12355,6 +12367,7 @@ export function issueService(db: Db) {
!shouldUpgradeAttachmentAuthorization &&
!shouldBindAttachments
) {
if (options?.completionReply && actor.agentId && createdByRunId) await acknowledgeChatCompletionReply(dbOrTx, createdByRunId, existing.id);
return redactIssueComment(
existing,
currentUserRedactionOptions.enabled,
@@ -12413,6 +12426,7 @@ export function issueService(db: Db) {
.returning();
}
if (!comment) throw new Error("Failed to create issue comment");
if (options?.completionReply && actor.agentId && createdByRunId) await acknowledgeChatCompletionReply(dbOrTx, createdByRunId, comment.id);
const boundAttachments: Array<{
id: string;
@@ -1,13 +1,15 @@
import { hasWorkspaceRestoreFailure } from "@paperclipai/shared";
import { normalizeMaxTurnStopReason } from "./heartbeat-stop-metadata.js";
import { hasConversationContinuationPolicy } from "./conversation-continuation.js";
import { claimedAdapterType, hasConversationContinuationPolicy } from "./conversation-continuation.js";
import { randomUUID } from "node:crypto";
import { and, eq, inArray, or, sql } from "drizzle-orm";
import { heartbeatRuns, issueRecoveryActions, issues, type Db } from "@paperclipai/db";
import { and, eq, inArray, isNull, or, sql } from "drizzle-orm";
import { environmentLeases, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, type Db } from "@paperclipai/db";
import { issueRecoveryActionService } from "./issue-recovery-actions.js";
import { parseIssueExecutionState } from "./issue-execution-policy.js";
import { executionFailureRetryCount } from "./execution-recovery-attempt.js";
import { logActivity } from "./activity-log.js";
import { isSupersededConversationRun } from "./agent-conversations.js";
import { issueService } from "./issues.js";
type Run = typeof heartbeatRuns.$inferSelect;
export const LEGACY_RECOVERY_CAUSE = "legacy_execution_requires_reconciliation";
@@ -163,3 +165,68 @@ export async function terminalizeLegacyExecution(input: {
return updated;
});
}
/** Called only by the exact executor's finally block after preparation/cleanup
* settles. A shutdown can win the terminal-status race before the setup catch
* records that no provider started. Retire only that mistaken recovery hold. */
export async function settleInterruptedNativeBootstrap(
db: Db, input: { run: Run; providerDispatchStarted: boolean },
): Promise<Run | null> {
if (input.providerDispatchStarted) return null;
const issueId = input.run.contextSnapshot?.issueId;
if (typeof issueId !== "string") return null;
return db.transaction(async tx => {
const [task] = await tx.select().from(issues).where(and(eq(issues.id, issueId), eq(issues.companyId, input.run.companyId))).for("update");
const [run] = await tx.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, input.run.id), eq(heartbeatRuns.companyId, input.run.companyId))).for("update");
if (!task || !run || run.runtimeMode !== "legacy" || run.runtimeModeResolvedAt ||
run.contextSnapshot?.issueId !== task.id ||
run.status !== "interrupted" || run.errorCode !== "server_shutdown_interrupted" ||
claimedAdapterType(run) !== "paperclip_runner" || run.processPid || run.processGroupId ||
hasWorkspaceRestoreFailure(run.resultJson) || executionFailureRetryCount(run) >= 2) return null;
const [native] = await tx.select({ id: nativeRunFinalizations.runId }).from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id)).limit(1);
const [lease] = await tx.select({ id: environmentLeases.id }).from(environmentLeases).where(and(
eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id),
or(isNull(environmentLeases.releasedAt), eq(environmentLeases.status, "pending_cleanup"), eq(environmentLeases.cleanupStatus, "failed")),
)).limit(1);
if (native || lease) return null;
const holds = await tx.select().from(issueRecoveryActions).where(and(
eq(issueRecoveryActions.companyId, run.companyId), eq(issueRecoveryActions.sourceIssueId, task.id),
eq(issueRecoveryActions.cause, LEGACY_RECOVERY_CAUSE), eq(issueRecoveryActions.fingerprint, `legacy-execution:${run.id}`),
sql`${issueRecoveryActions.evidence}->>'runId' = ${run.id}`,
sql`not (${issueRecoveryActions.evidence} ? 'executionReconciliation')`,
sql`coalesce(${issueRecoveryActions.evidence}->>'workspaceRestoreFailure', '') <> 'restore_unsafe_archive'`,
or(inArray(issueRecoveryActions.status, ["active", "escalated"]), and(
eq(issueRecoveryActions.status, "resolved"), eq(issueRecoveryActions.outcome, "blocked"),
sql`${issueRecoveryActions.evidence}->'automaticRecovery'->>'policy' = 'preserve_without_replay_v1'`,
sql`${issueRecoveryActions.evidence}->'automaticRecovery'->>'replay' = 'blocked'`,
)),
)).for("update");
const ownedBlock = holds.map(hold => hold.evidence.nativeBootstrapFailureBlock as Record<string, unknown> | undefined)
.find(receipt => receipt?.runId === run.id && receipt.statusVersion === task.statusVersion &&
["todo", "in_progress", "in_review"].includes(String(receipt.previousStatus)));
if (task.status === "blocked" && ownedBlock && task.assigneeAgentId === run.agentId &&
!isSupersededConversationRun(task, run) && !task.executionRunId && !task.checkoutRunId) {
// Restore only the unchanged projection made by this exact incident.
// The issue service still enforces dependency/assignee readiness. Human
// reblocks, reassignment and conversation resets invalidate this receipt.
await issueService(tx as unknown as Db).update(task.id, { status: String(ownedBlock.previousStatus) }, tx);
}
const now = new Date();
const [settled] = await tx.update(heartbeatRuns).set({ resultJson: { ...run.resultJson,
executionRecovery: { kind: "bootstrap", providerWorkStarted: false, preparationSettledAt: now.toISOString() },
}, updatedAt: now }).where(eq(heartbeatRuns.id, run.id)).returning();
const resolved = holds.length ? await tx.update(issueRecoveryActions).set({ status: "resolved", outcome: "false_positive",
resolutionNote: "The interrupted executor finished preparation and cleanup without starting a provider.",
resolvedAt: now, updatedAt: now,
// An automatic disposition may already have resolved the incident but
// left replay blocked. Preserve its history while correcting that verdict.
evidence: sql`jsonb_set(${issueRecoveryActions.evidence}, '{automaticRecovery,replay}', '"allowed"'::jsonb, false) ||
${JSON.stringify({ bootstrapPreparationSettledAt: now.toISOString() })}::jsonb`,
}).where(inArray(issueRecoveryActions.id, holds.map(hold => hold.id))).returning({ id: issueRecoveryActions.id }) : [];
if (resolved.length) await logActivity(tx as unknown as Db, { companyId: run.companyId, actorType: "system", actorId: "system",
action: "issue.recovery_action_resolved", entityType: "issue", entityId: task.id, agentId: run.agentId, runId: run.id,
details: { reason: "native_bootstrap_stopped_before_provider", recoveryActionIds: resolved.map(r => r.id) },
});
return settled;
});
}
@@ -44,7 +44,7 @@ export function buildNativeContinuationPrompt(input: {
});
// Other provider, attachment, question, approval, and recovery paths keep
// their specialized framing. A matching prior run still gates every delta.
if ((wake.externalChatProvider && !externalChat) ||
if (rawWake.chatCompletionUpdates || (wake.externalChatProvider && !externalChat) ||
(!['issue_commented', 'issue_children_completed'].includes(wake.reason ?? '') &&
!(externalChat && wake.reason === "External chat message received")) ||
wake.fallbackFetchNeeded || wake.truncated || wake.recovery || continuation?.interruptedRunId ||
@@ -969,7 +969,7 @@ export async function repairCommittedNativeChatResponse(
input.issueId,
resolved.text,
{ agentId: run.agentId, runId: run.id },
{ authorizationReason: CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON },
{ authorizationReason: CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON, completionReply: true },
tx,
);
const presentationDecision = {
@@ -1944,6 +1944,7 @@ export async function commitNativeStatusDecision(input: {
completedChildIssueId: input.issueId,
childIssueIds: parent.childIssueIds,
childIssueSummaries: parent.childIssueSummaries,
onboardingCompletion: parent.onboardingCompletion,
childIssueSummaryTruncated: parent.childIssueSummaryTruncated,
}
: null;
@@ -1988,12 +1989,14 @@ export async function commitNativeStatusDecision(input: {
completedChildIssueId: input.issueId,
childIssueIds: parent.childIssueIds,
childIssueSummaries: parent.childIssueSummaries,
onboardingCompletion: parent.onboardingCompletion,
childIssueSummaryTruncated: parent.childIssueSummaryTruncated,
},
contextSnapshot: {
completedChildIssueId: input.issueId,
childIssueIds: parent.childIssueIds,
childIssueSummaries: parent.childIssueSummaries,
onboardingCompletion: parent.onboardingCompletion,
childIssueSummaryTruncated: parent.childIssueSummaryTruncated,
},
});
@@ -2005,6 +2008,7 @@ export async function commitNativeStatusDecision(input: {
parentIssueId: parent.id,
completedChildIssueId: input.issueId,
childIssueSummaries: parent.childIssueSummaries,
onboardingCompletion: parent.onboardingCompletion,
childIssueSummaryTruncated: parent.childIssueSummaryTruncated,
},
});