fix: renew explicit continuation authorization for bounded retries (#14987)

Preserve exact user-authorized continuation receipts across bounded retries and retain the task claim through owned retry admission. Stop, reassignment, superseding input, and active cleanup continue to block unsafe continuation.

Validation: exact-head Greptile 5/5, passing CI, no unresolved review threads, and a clean merge. Detailed verification and limitations are recorded in the pull request.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Devin FoleyandPaperclip authored and GitHub committed 2026-10-05 10:57:57 -07:00
1 parent 7498705642
commit 2e67ea8dc7
11 files changed
+639 -16

No files matched your search

@@ -397,6 +397,15 @@ export async function buildExecutionContinuation(input: {
priorRuns.some(run => run.id === wake.runId && run.retryOfRunId === failedRunId))
: rows.some(comment => comment.id === value.commentId &&
comment.authorType === "user" &&
(!("commentUpdatedAt" in value || "commentBodyHash" in value || value.automaticRetry) || (
value.commentUpdatedAt === comment.updatedAt.toISOString() &&
value.commentBodyHash === createHash("sha256").update(comment.body).digest("hex")
)) &&
(!value.automaticRetry || (
object(value.automaticRetry).sourceRunId === explicitUserSource &&
comment.body.trim().length > 0 && issue.executionRunId === input.runId &&
priorRuns.some(run => run.id === value.runId && run.retryOfRunId === explicitUserSource)
)) &&
(value.queuedCommentInterruptId
? interruptQueues.some(queue => queue.id === value.queuedCommentInterruptId &&
queue.runId === value.runId &&
@@ -0,0 +1,14 @@
import type { heartbeatRuns, issues } from "@paperclipai/db";
/**
* This claim must be settled by queue-first recovery, not generic terminal-lock
* cleanup. It is pending intent only; admission still validates its authority.
*/
export function isExplicitContinuationRetryClaim(
issue: Pick<typeof issues.$inferSelect, "id" | "companyId" | "executionRunId">,
run: Pick<typeof heartbeatRuns.$inferSelect, "id" | "companyId" | "runtimeMode" | "status" | "contextSnapshot"> | null | undefined,
) {
return Boolean(run && issue.executionRunId === run.id && run.companyId === issue.companyId &&
run.runtimeMode === "legacy" && ["failed", "timed_out"].includes(run.status) &&
run.contextSnapshot?.issueId === issue.id && run.contextSnapshot.explicitUserContinuation);
}
@@ -10,7 +10,7 @@ import { createHash, randomUUID } from "node:crypto";
import { mkdtemp, mkdir, writeFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { and, eq } from "drizzle-orm";
import { and, eq, sql } from "drizzle-orm";
import { beforeAll, afterAll, describe, it, expect, vi } from "vitest";
import {
approvals, issueApprovals, issueThreadInteractions, chatConversations, chatEndpoints, toolApplications, toolConnections,
@@ -19,6 +19,8 @@ import {
} from "@paperclipai/db";
import { startEmbeddedPostgresTestDatabase, getEmbeddedPostgresTestSupport } from "../__tests__/helpers/embedded-postgres.js";
import { admitExplicitNativeContinuation } from "./explicit-native-continuation.js";
import { adapterExecutionControls, createAdapterExecutionControl } from "./adapter-execution-control.js";
import { CONVERSATION_CONTINUATION_POLICY } from "./conversation-continuation.js";
import { buildExecutionContinuation } from "./execution-continuation.js";
import { heartbeatService, persistHeartbeatRunProcessMetadata, type HeartbeatEnvironmentRuntime } from "./heartbeat.js";
import { getExecutionBlocker } from "./execution-blocker.js";
@@ -386,6 +388,197 @@ const support = await getEmbeddedPostgresTestSupport();
return result;
});
async function seedTimedOutExplicitTurn() {
const f = await seed();
const explicitUserContinuation = await admit(f);
expect(explicitUserContinuation).not.toBeNull();
const [receipt] = await db.select().from(issueRecoveryActions)
.where(eq(issueRecoveryActions.sourceIssueId, f.issueId));
await db.update(agents).set({ adapterType: "claude_local" }).where(eq(agents.id, f.agentId));
const now = new Date();
const [parent] = await db.update(heartbeatRuns).set({ runtimeMode: "legacy", status: "timed_out",
errorCode: "timeout", startedAt: now, finishedAt: now,
resultJson: { conversationContinuation: CONVERSATION_CONTINUATION_POLICY },
contextSnapshot: { issueId: f.issueId, wakeReason: "issue_commented", wakeCommentId: f.commentId,
previousRunId: f.sourceRunId, explicitUserContinuation },
}).where(eq(heartbeatRuns.id, f.successorRunId)).returning();
await db.update(issues).set({ status: "in_progress", executionRunId: parent.id })
.where(eq(issues.id, f.issueId));
return { ...f, parent, receipt, now };
}
const dispatchExplicitRetry = (f: Fixture, run: typeof heartbeatRuns.$inferSelect) => buildExecutionContinuation({
db, companyId: f.companyId, issueId: f.issueId, agentId: f.agentId, runId: run.id,
context: run.contextSnapshot!, summary: null, exposeLowTrustRaw: false,
});
it("re-admits an explicit user turn's timeout retry with its own durable authorization", async () => {
const f = await seedTimedOutExplicitTurn();
await expect(dispatchExplicitRetry(f, f.parent)).resolves.toMatchObject({ interruptedRunId: f.sourceRunId });
const heartbeat = heartbeatService(db);
const [scheduled, competing] = await Promise.all([
heartbeat.scheduleBoundedRetry(f.parent.id, { now: f.now, delayMs: 1000 }),
heartbeat.scheduleBoundedRetry(f.parent.id, { now: f.now, delayMs: 1000 }),
]);
expect(scheduled.outcome).toBe("scheduled");
if (scheduled.outcome !== "scheduled") return;
expect(competing).toMatchObject({ outcome: "scheduled", run: { id: scheduled.run.id } });
expect(scheduled.run.contextSnapshot?.explicitUserContinuation).toEqual({
previousRunId: f.parent.id, commentId: f.commentId,
});
// Dispatch validates the new run, not a borrowed parent receipt.
await expect(dispatchExplicitRetry(f, scheduled.run)).resolves.toMatchObject({ interruptedRunId: f.parent.id });
const receipts = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId));
expect(receipts).toHaveLength(2);
expect(receipts.find(row => row.id === f.receipt.id)?.evidence).toEqual(f.receipt.evidence);
expect(receipts.find(row => row.id !== f.receipt.id)?.evidence.explicitUserContinuation).toMatchObject({
runId: scheduled.run.id, previousRunId: f.parent.id, actorId: "board", commentId: f.commentId,
automaticRetry: { sourceRunId: f.parent.id, sourceAuthorizationId: f.receipt.id },
});
expect(await heartbeat.scheduleBoundedRetry(f.parent.id, { now: f.now })).toMatchObject({
outcome: "scheduled", run: { id: scheduled.run.id },
});
expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))).toHaveLength(2);
// Removing the new receipt reproduces the original dispatch failure.
await db.delete(issueRecoveryActions).where(eq(issueRecoveryActions.id, receipts.find(row => row.id !== f.receipt.id)!.id));
await expect(dispatchExplicitRetry(f, scheduled.run)).rejects.toThrow("continuation_user_authorization_missing");
});
it("waits for the parent adapter controller even when no environment lease remains", async () => {
const f = await seedTimedOutExplicitTurn();
const heartbeat = heartbeatService(db);
const control = createAdapterExecutionControl();
adapterExecutionControls.set(f.parent.id, control);
try {
expect(await heartbeat.scheduleBoundedRetry(f.parent.id)).toMatchObject({ outcome: "not_scheduled" });
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, f.parent.id))).toHaveLength(0);
expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))).toHaveLength(1);
} finally {
adapterExecutionControls.delete(f.parent.id);
control.finish();
}
expect(await heartbeat.scheduleBoundedRetry(f.parent.id)).toMatchObject({ outcome: "scheduled" });
});
it("binds each bounded explicit retry to its immediate parent without replacing prior receipts", async () => {
const f = await seedTimedOutExplicitTurn();
const heartbeat = heartbeatService(db);
const first = await heartbeat.scheduleBoundedRetry(f.parent.id, { now: f.now });
expect(first.outcome).toBe("scheduled");
if (first.outcome !== "scheduled") return;
const nextNow = new Date(f.now.getTime() + 2000);
await db.update(heartbeatRuns).set({ status: "timed_out", errorCode: "timeout", startedAt: f.now,
finishedAt: nextNow }).where(eq(heartbeatRuns.id, first.run.id));
const second = await heartbeat.scheduleBoundedRetry(first.run.id, { now: nextNow });
expect(second.outcome).toBe("scheduled");
if (second.outcome !== "scheduled") return;
expect(second.run).toMatchObject({ retryOfRunId: first.run.id, scheduledRetryAttempt: 2 });
await expect(dispatchExplicitRetry(f, second.run)).resolves.toMatchObject({ interruptedRunId: first.run.id });
const receipts = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId));
expect(receipts).toHaveLength(3);
expect(receipts.find(row => row.id === f.receipt.id)?.evidence).toEqual(f.receipt.evidence);
await db.update(heartbeatRuns).set({ status: "timed_out", errorCode: "timeout", startedAt: nextNow,
finishedAt: nextNow }).where(eq(heartbeatRuns.id, second.run.id));
expect(await heartbeat.scheduleBoundedRetry(second.run.id, { now: nextNow })).toMatchObject({ outcome: "retry_exhausted" });
expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))).toHaveLength(3);
});
it("keeps sub-millisecond source ordering when authorizing a timeout retry", async () => {
const f = await seedTimedOutExplicitTurn();
await db.execute(sql`update heartbeat_runs set created_at = ${f.now.toISOString()}::timestamptz +
case when id = ${f.sourceRunId} then interval '100 microseconds' else interval '500 microseconds' end
where company_id = ${f.companyId} and id in (${f.sourceRunId}, ${f.parent.id})`);
const result = await heartbeatService(db).scheduleBoundedRetry(f.parent.id, { now: f.now });
expect(result.outcome).toBe("scheduled");
});
it.each(["max_turns_continuation", "interaction_continuation_infra_retry", "workspace_busy"])(
"does not lend explicit user authority to the separate %s retry contract", async retryReason => {
const f = await seedTimedOutExplicitTurn();
const result = await heartbeatService(db).scheduleBoundedRetry(f.parent.id, { now: f.now, retryReason });
expect(result.outcome).not.toBe("scheduled");
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, f.parent.id))).toHaveLength(0);
},
);
it.each(["deleted", "edited", "same_millisecond_edit", "empty", "actor", "run_authored", "legacy_receipt", "receipt_run", "receipt_company", "source", "task", "reassigned", "done", "superseded", "native_superseding", "interrupted_intent"])(
"refuses to re-admit an automatic explicit retry after %s changes", async kind => {
const f = await seedTimedOutExplicitTurn();
if (["deleted", "edited", "same_millisecond_edit", "empty", "actor", "run_authored"].includes(kind)) await db.update(issueComments).set({
...(kind === "deleted" ? { deletedAt: new Date() } : {}),
...(kind === "edited" ? { body: "Different instructions", updatedAt: new Date(f.now.getTime() + 1000) } : {}),
...(kind === "same_millisecond_edit" ? { body: "Different instructions within the recorded millisecond" } : {}),
...(kind === "empty" ? { body: " " } : {}),
...(kind === "actor" ? { authorUserId: "another-user" } : {}),
...(kind === "run_authored" ? { createdByRunId: f.parent.id } : {}),
}).where(eq(issueComments.id, f.commentId));
if (["receipt_run", "interrupted_intent"].includes(kind)) await db.update(issueRecoveryActions).set({ evidence: {
...f.receipt.evidence, explicitUserContinuation: {
...(f.receipt.evidence.explicitUserContinuation as Record<string, unknown>),
...(kind === "receipt_run" ? { runId: randomUUID() } : { queuedCommentInterruptId: randomUUID() }),
},
} }).where(eq(issueRecoveryActions.id, f.receipt.id));
if (kind === "receipt_company") await db.update(issueRecoveryActions).set({ companyId: (await seed()).companyId })
.where(eq(issueRecoveryActions.id, f.receipt.id));
if (kind === "legacy_receipt") {
const { commentBodyHash: _bodyHash, commentUpdatedAt: _updatedAt, ...legacy } =
f.receipt.evidence.explicitUserContinuation as Record<string, unknown>;
await db.update(issueRecoveryActions).set({ evidence: { ...f.receipt.evidence,
explicitUserContinuation: legacy } }).where(eq(issueRecoveryActions.id, f.receipt.id));
}
if (kind === "source" || kind === "task") await db.update(heartbeatRuns).set({ contextSnapshot: {
...f.parent.contextSnapshot,
...(kind === "task" ? { issueId: randomUUID() } : { explicitUserContinuation: { previousRunId: randomUUID(), commentId: f.commentId } }),
} }).where(eq(heartbeatRuns.id, f.parent.id));
if (kind === "reassigned" || kind === "done") await db.update(issues).set(kind === "done"
? { status: "done" } : { assigneeAgentId: null }).where(eq(issues.id, f.issueId));
if (kind === "superseded") await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId,
status: "succeeded", contextSnapshot: { issueId: f.issueId }, createdAt: new Date(f.now.getTime() + 1000) });
if (kind === "native_superseding") await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId,
status: "running", runtimeMode: "native", nativeIssueId: f.issueId, contextSnapshot: {} });
const result = await heartbeatService(db).scheduleBoundedRetry(f.parent.id, { now: f.now });
expect(result.outcome).not.toBe("scheduled");
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, f.parent.id))).toHaveLength(0);
expect(await db.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.sourceIssueId, f.issueId),
eq(issueRecoveryActions.cause, "explicit_user_continuation_retry")))).toHaveLength(0);
},
);
it("binds the initial user-message admission before any automatic retry", async () => {
const f = await seedTimedOutExplicitTurn();
await expect(dispatchExplicitRetry(f, f.parent)).resolves.toMatchObject({ interruptedRunId: f.sourceRunId });
await db.update(issueComments).set({ body: "Changed after the original admission" })
.where(eq(issueComments.id, f.commentId));
await expect(dispatchExplicitRetry(f, f.parent)).rejects.toThrow("continuation_user_authorization_missing");
});
it.each(["deleted", "edited", "same_millisecond_edit", "actor", "receipt_run", "receipt_source", "retry_parent", "execution_owner"])(
"revalidates the successor's exact authorization at dispatch: %s", async kind => {
const f = await seedTimedOutExplicitTurn();
const scheduled = await heartbeatService(db).scheduleBoundedRetry(f.parent.id, { now: f.now });
expect(scheduled.outcome).toBe("scheduled");
if (scheduled.outcome !== "scheduled") return;
if (["deleted", "edited", "same_millisecond_edit", "actor"].includes(kind)) await db.update(issueComments).set({
...(kind === "deleted" ? { deletedAt: new Date() } : {}),
...(kind === "edited" ? { body: "Changed after scheduling", updatedAt: new Date(f.now.getTime() + 1000) } : {}),
...(kind === "same_millisecond_edit" ? { body: "Changed within the recorded millisecond" } : {}),
...(kind === "actor" ? { authorUserId: "another-user" } : {}),
}).where(eq(issueComments.id, f.commentId));
if (kind === "receipt_run" || kind === "receipt_source") {
const [receipt] = await db.select().from(issueRecoveryActions).where(and(
eq(issueRecoveryActions.sourceIssueId, f.issueId), eq(issueRecoveryActions.cause, "explicit_user_continuation_retry")));
await db.update(issueRecoveryActions).set({ evidence: { ...receipt.evidence, explicitUserContinuation: {
...(receipt.evidence.explicitUserContinuation as Record<string, unknown>),
...(kind === "receipt_run" ? { runId: f.parent.id } : { previousRunId: f.sourceRunId }),
} } }).where(eq(issueRecoveryActions.id, receipt.id));
}
if (kind === "retry_parent") await db.update(heartbeatRuns).set({ retryOfRunId: f.sourceRunId })
.where(eq(heartbeatRuns.id, scheduled.run.id));
if (kind === "execution_owner") await db.update(issues).set({ executionRunId: f.parent.id })
.where(eq(issues.id, f.issueId));
await expect(dispatchExplicitRetry(f, scheduled.run)).rejects.toThrow("continuation_user_authorization_missing");
},
);
it.each(["active", "removed", "unavailable", "paused", "disabled"])("shows usable recovery guidance for external chat cancellations: %s", async kind => {
const f = await seed();
await db.delete(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.sourceRunId));
@@ -1,3 +1,4 @@
import { createHash } from "node:crypto";
import { appendHeartbeatRunEvent } from "./heartbeat-run-events.js";
import { canContinueCancelledRun } from "./run-cancellation.js";
import { readQueuedInteractionResponse } from "./queued-interaction-response.js";
@@ -292,6 +293,8 @@ export async function admitExplicitNativeContinuation(input: {
});
}
const authorization = { actorId, commentId, ...(response ? { interactionId: response.source.interactionId } : {}), ...(retry ? { failedRunId: input.failedRunId } : {}),
...(comment ? { commentUpdatedAt: comment.updatedAt.toISOString(),
commentBodyHash: createHash("sha256").update(comment.body).digest("hex") } : {}),
...(queuedInterrupt ? { queuedCommentInterruptId: input.queuedCommentInterruptId } : {}),
...(queuedRequest ? { queuedCommentRequestId: input.queuedCommentRequestId } : {}), runId: input.successorRunId,
previousRunId: previous.id, recordedAt: new Date().toISOString() };
@@ -329,3 +332,91 @@ export async function admitExplicitNativeContinuation(input: {
});
return { previousRunId: previous.id, commentId, ...(retry ? { failedRunId: input.failedRunId! } : {}) };
}
/** Re-admit a bounded automatic retry without lending it its parent's receipt.
* The caller holds the task lock and creates the successor in this transaction.
* Only unchanged user messages are renewable; Retry/Interrupt intents keep their
* separate one-run contracts. The scheduler still owns retry and cleanup policy.
*/
export async function admitExplicitContinuationRetry(input: {
db: Db; companyId: string; issueId: string; agentId: string;
parentRunId: string; successorRunId: string; now: Date;
}): Promise<{ previousRunId: string; commentId: string } | null> {
const { db, companyId, issueId, agentId } = input;
const [task] = await db.select().from(issues).where(and(
eq(issues.companyId, companyId), eq(issues.id, issueId),
)).for("update");
const [parent] = await db.select().from(heartbeatRuns).where(and(
eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, input.parentRunId),
eq(heartbeatRuns.agentId, agentId),
)).for("update");
if (!task || task.assigneeAgentId !== agentId || task.executionRunId !== input.parentRunId ||
["done", "cancelled"].includes(task.status) || !parent ||
!["failed", "timed_out"].includes(parent.status) || !parent.finishedAt ||
parent.runtimeMode !== "legacy" || parent.contextSnapshot?.issueId !== issueId ||
(parent.nativeIssueId !== null && parent.nativeIssueId !== issueId) ||
adapterExecutionControls.has(parent.id) ||
await getExecutionBlocker(db, companyId, issueId)) return null;
const context = parent.contextSnapshot;
const explicit = context.explicitUserContinuation as Record<string, unknown> | undefined;
if (!explicit || !z.string().guid().safeParse(explicit.previousRunId).success ||
!z.string().guid().safeParse(explicit.commentId).success || explicit.failedRunId) return null;
const commentId = explicit.commentId as string;
const [comment] = await db.select().from(issueComments).where(and(
eq(issueComments.companyId, companyId), eq(issueComments.issueId, issueId),
eq(issueComments.id, commentId), eq(issueComments.authorType, "user"),
isNull(issueComments.createdByRunId), isNull(issueComments.deletedAt),
));
if (!comment?.authorUserId || !comment.body.trim()) return null;
const receipts = await db.select().from(issueRecoveryActions).where(and(
eq(issueRecoveryActions.companyId, companyId), eq(issueRecoveryActions.sourceIssueId, issueId),
eq(issueRecoveryActions.status, "resolved"),
sql`${issueRecoveryActions.evidence}->'explicitUserContinuation'->>'runId' = ${parent.id}`,
));
const receipt = receipts.find(row => {
const auth = row.evidence.explicitUserContinuation as Record<string, unknown> | undefined;
return auth && auth.previousRunId === explicit.previousRunId && auth.commentId === commentId &&
auth.actorId === comment.authorUserId && !auth.failedRunId && !auth.queuedCommentInterruptId &&
auth.commentUpdatedAt === comment.updatedAt.toISOString() &&
auth.commentBodyHash === createHash("sha256").update(comment.body).digest("hex");
});
if (!receipt) return null;
const [superseding] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(
eq(heartbeatRuns.companyId, companyId),
or(eq(heartbeatRuns.nativeIssueId, issueId), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`),
ne(heartbeatRuns.id, parent.id), ne(heartbeatRuns.id, input.successorRunId),
or(eq(heartbeatRuns.retryOfRunId, parent.id),
// Keep PostgreSQL's timestamp precision: Date would round the parent
// down and mistake an older run in the same millisecond for a successor.
sql`${heartbeatRuns.createdAt} > (select source.created_at from heartbeat_runs source
where source.company_id = ${companyId} and source.id = ${parent.id})`,
inArray(heartbeatRuns.status, ["queued", "running", "scheduled_retry"])),
)).limit(1);
if (superseding) return null;
// Reuse dispatch's complete source/actor/receipt validation on the parent.
// It proves only the parent; the new receipt below authorizes the successor.
try {
await buildExecutionContinuation({ db, companyId, issueId, agentId, runId: parent.id,
context, summary: null, exposeLowTrustRaw: false });
} catch (error) {
if (error instanceof Error && ["continuation_user_authorization_missing",
"continuation_source_context_missing", "continuation_task_ownership_changed"].includes(error.message)) return null;
throw error;
}
const continuation = { previousRunId: parent.id, commentId };
await db.insert(issueRecoveryActions).values({
companyId, sourceIssueId: issueId, kind: "active_run_watchdog",
cause: "explicit_user_continuation_retry", fingerprint: input.successorRunId,
status: "resolved", outcome: "retry_authorized", resolvedAt: input.now,
nextAction: "The bounded retry has its own authorization for the unchanged user message.",
evidence: { explicitUserContinuation: {
...continuation, actorId: comment.authorUserId, runId: input.successorRunId,
commentUpdatedAt: comment.updatedAt.toISOString(),
commentBodyHash: createHash("sha256").update(comment.body).digest("hex"),
recordedAt: input.now.toISOString(), automaticRetry: {
sourceRunId: parent.id, sourceAuthorizationId: receipt.id,
},
} },
});
return continuation;
}
+42 -3
View File
@@ -32,7 +32,7 @@ import { legacyControllerBootId, legacyControllerClaim, renewLegacyControllerLea
import { completeTerminatedRemoteNativeSessionCleanup } from "../vendor/paperclip-runner/index.js";
import { hasRemoteTerminationReceipt, remoteExecutionHasStopped, remoteTerminationReceipt, stoppedRemoteCleanupScopes } from "./remote-execution-termination.js";
import { applyConnectorSkills, prepareConnectorSkillDelivery, resolveConnectorAssignments } from "./connector-runtime.js";
import { admitExplicitNativeContinuation, undeliveredLegacyUserCommentIds } from "./explicit-native-continuation.js";
import { admitExplicitNativeContinuation, admitExplicitContinuationRetry, undeliveredLegacyUserCommentIds } from "./explicit-native-continuation.js";
import { canRetryStoppedRun, isCancelledNativeStartup } from "./cancelled-native-startup.js";
import { connectionIntentService } from "./connection-intents.js";
import { managedAiSessionFingerprintConfig, prepareManagedAiRuntime, assertManagedAiProjectAuth, stripAiAuthBindings, isAiConnectionBusy, AI_AUTH_ENV_KEYS } from "./ai-connection-runtime.js";
@@ -9649,6 +9649,7 @@ export function heartbeatService(
const recovery = recoveryService(db, {
enqueueWakeup,
liveRunExecutions,
settleExplicitContinuationRetry: releaseIssueExecutionAndPromote,
scheduleRecoveryRetry: async (runId) => {
const [run] = await db
.select()
@@ -9866,10 +9867,19 @@ export function heartbeatService(
));
const agent = source ? await getAgent(source.agentId) : null;
if (source && agent && agent.companyId === source.companyId) {
await scheduleBoundedRetryForRun(source, agent, effect.reviewParticipant ? {
const retry = await scheduleBoundedRetryForRun(source, agent, effect.reviewParticipant ? {
retryReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON,
wakeReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON,
} : undefined);
const issueId = readNonEmptyString(source.contextSnapshot?.issueId);
if (retry.outcome !== "scheduled" && source.contextSnapshot?.explicitUserContinuation && issueId &&
!adapterExecutionControls.has(source.id) && !(await getExecutionBlocker(db, source.companyId, issueId))) {
// Cleanup has settled, so exhaustion or revoked/missing authority
// must not retain a terminal claim. Do not request another retry.
const settled = await wakeQueue.releaseIssueExecution({ companyId: source.companyId,
runId: source.id, now: new Date(), suppressImmediateRecovery: true });
await applyWakeQueuePostCommitEffects(settled.postCommitEffects);
}
}
} else if (effect.kind === "run_queued") {
publishLiveEvent({
@@ -14347,7 +14357,9 @@ export function heartbeatService(
presentationDecision?: RunPresentationDecision | null,
) {
const contextSnapshot = parseObject(run.contextSnapshot);
if (readNonEmptyString(contextSnapshot.goalControlRequestId)) {
// The explicit receipt admitted one turn. A prose-only follow-up cannot
// reuse it or renew it under the separate transient-failure retry policy.
if (readNonEmptyString(contextSnapshot.goalControlRequestId) || contextSnapshot.explicitUserContinuation) {
if (run.issueCommentStatus !== "not_applicable") {
await patchRunIssueCommentStatus(run.id, {
issueCommentStatus: "not_applicable",
@@ -15702,6 +15714,7 @@ export function heartbeatService(
| "issue_cancelled"
| "issue_terminal_status"
| "issue_not_in_progress"
| "continuation_user_authorization_missing"
| "issue_execution_lock_changed";
issueId: string | null;
details: Record<string, unknown>;
@@ -15979,6 +15992,22 @@ export function heartbeatService(
}
}
const scheduledRunId = randomUUID();
if (contextSnapshot.explicitUserContinuation) {
const continuation = issueId && retryReason === "transient_failure" ? await admitExplicitContinuationRetry({
db: tx as unknown as Db, companyId: run.companyId, issueId, agentId: run.agentId,
parentRunId: run.id, successorRunId: scheduledRunId, now,
}) : null;
if (!continuation) return {
outcome: "not_scheduled", issueId,
errorCode: "continuation_user_authorization_missing",
reason: "The automatic retry could not revalidate the original user continuation.",
details: {},
};
retryContextSnapshot.explicitUserContinuation = continuation;
retryContextSnapshot.previousRunId = continuation.previousRunId;
}
const wakeupRequest = await tx
.insert(agentWakeupRequests)
.values({
@@ -16029,6 +16058,7 @@ export function heartbeatService(
const scheduledRun = await tx
.insert(heartbeatRuns)
.values({
id: scheduledRunId,
companyId: run.companyId,
agentId: run.agentId,
invocationSource: "automation",
@@ -26767,6 +26797,15 @@ export function heartbeatService(
}
// Terminalization precedes lease and adapter cleanup. Only now is the
// owner gone; retry pending input for ordinary completions as well as Stop.
if (latestRun?.runtimeMode === "legacy" && ["failed", "timed_out"].includes(latestRun.status) &&
latestRun.contextSnapshot?.explicitUserContinuation) {
// Re-run the same queue-first recovery decision after cleanup. Its
// earlier retry request could not authorize work while this executor
// still held its controller or environment lease.
await releaseIssueExecutionAndPromote(latestRun).catch(err => {
logger.error({ err, runId: run.id }, "failed to settle explicit continuation after cleanup");
});
}
if (latestRun?.runtimeMode === "legacy" && isHeartbeatRunTerminalStatus(latestRun.status)) {
const [pending] = await db.select({ id: agentWakeupRequests.id, payload: agentWakeupRequests.payload }).from(agentWakeupRequests).where(and(
eq(agentWakeupRequests.companyId, run.companyId), eq(agentWakeupRequests.agentId, run.agentId),
+28 -11
View File
@@ -9,6 +9,7 @@ import { executionProjectionsForRuns } from "./execution-projection.js";
import type { ExecutionProjection } from "@paperclipai/shared";
import { Buffer } from "node:buffer";
import { createHash, randomUUID } from "node:crypto";
import { isExplicitContinuationRetryClaim } from "./explicit-continuation-retry-claim.js";
import { markdownToPlainText, parseMarkdown } from "chat";
import {
and,
@@ -7497,6 +7498,7 @@ export function issueService(db: Db) {
const lockedIssue = await tx
.select({
id: issues.id,
companyId: issues.companyId,
status: issues.status,
assigneeAgentId: issues.assigneeAgentId,
checkoutRunId: issues.checkoutRunId,
@@ -7528,7 +7530,7 @@ export function issueService(db: Db) {
]);
const [existingRun, actorRun] = await Promise.all([
tx
.select({ status: heartbeatRuns.status })
.select()
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, input.expectedCheckoutRunId))
.then((rows) => rows[0] ?? null),
@@ -7540,6 +7542,17 @@ export function issueService(db: Db) {
]);
const stale =
!existingRun || TERMINAL_HEARTBEAT_RUN_STATUSES.has(existingRun.status);
if (isExplicitContinuationRetryClaim(lockedIssue, existingRun)) {
return { adopted: null, latest: lockedIssue };
}
if (lockedIssue.executionRunId && lockedIssue.executionRunId !== input.expectedCheckoutRunId) {
const executionRun = await tx.select().from(heartbeatRuns)
.where(eq(heartbeatRuns.id, lockedIssue.executionRunId)).for("update")
.then(rows => rows[0] ?? null);
if (isExplicitContinuationRetryClaim(lockedIssue, executionRun)) {
return { adopted: null, latest: lockedIssue };
}
}
const actorLive =
actorRun && !TERMINAL_HEARTBEAT_RUN_STATUSES.has(actorRun.status);
if (!stale || !actorLive) {
@@ -7649,7 +7662,7 @@ export function issueService(db: Db) {
sql`select ${issues.id} from ${issues} where ${issues.id} = ${issueId} for update`,
);
const issue = await tx
.select({ executionRunId: issues.executionRunId })
.select({ id: issues.id, companyId: issues.companyId, executionRunId: issues.executionRunId })
.from(issues)
.where(eq(issues.id, issueId))
.then((rows) => rows[0] ?? null);
@@ -7659,11 +7672,12 @@ export function issueService(db: Db) {
sql`select ${heartbeatRuns.id} from ${heartbeatRuns} where ${heartbeatRuns.id} = ${issue.executionRunId} for update`,
);
const run = await tx
.select({ status: heartbeatRuns.status })
.select()
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, issue.executionRunId))
.then((rows) => rows[0] ?? null);
if (run && !TERMINAL_HEARTBEAT_RUN_STATUSES.has(run.status)) return false;
if (isExplicitContinuationRetryClaim(issue, run)) return false;
const updated = await tx
.update(issues)
@@ -7689,8 +7703,7 @@ export function issueService(db: Db) {
// Symmetric to clearExecutionRunIfTerminal. Clears checkoutRunId (and the
// bundled execution lock cols) when the row's checkoutRunId points at a
// heartbeat run that is terminal or no longer exists. No assignee/status
// precondition: a terminal run holds no real claim regardless of who is
// assigned or what status the issue is currently in.
// precondition. Explicit retry claims remain owned by queue-first settlement.
async function clearCheckoutRunIfTerminal(issueId: string): Promise<boolean> {
return db.transaction(async (tx) => {
await tx.execute(
@@ -7698,6 +7711,8 @@ export function issueService(db: Db) {
);
const issue = await tx
.select({
id: issues.id,
companyId: issues.companyId,
checkoutRunId: issues.checkoutRunId,
executionRunId: issues.executionRunId,
})
@@ -7710,11 +7725,12 @@ export function issueService(db: Db) {
sql`select ${heartbeatRuns.id} from ${heartbeatRuns} where ${heartbeatRuns.id} = ${issue.checkoutRunId} for update`,
);
const run = await tx
.select({ status: heartbeatRuns.status })
.select()
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, issue.checkoutRunId))
.then((rows) => rows[0] ?? null);
if (run && !TERMINAL_HEARTBEAT_RUN_STATUSES.has(run.status)) return false;
if (isExplicitContinuationRetryClaim(issue, run)) return false;
if (
issue.executionRunId &&
@@ -7724,7 +7740,7 @@ export function issueService(db: Db) {
sql`select ${heartbeatRuns.id} from ${heartbeatRuns} where ${heartbeatRuns.id} = ${issue.executionRunId} for update`,
);
const executionRun = await tx
.select({ status: heartbeatRuns.status })
.select()
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, issue.executionRunId))
.then((rows) => rows[0] ?? null);
@@ -7733,6 +7749,7 @@ export function issueService(db: Db) {
!TERMINAL_HEARTBEAT_RUN_STATUSES.has(executionRun.status)
)
return false;
if (isExplicitContinuationRetryClaim(issue, executionRun)) return false;
}
const updated = await tx
@@ -11600,10 +11617,10 @@ export function issueService(db: Db) {
current.executionRunId !== checkoutRunId &&
(current.assigneeAgentId === agentId || current.assigneeAgentId == null)
) {
const stale = await isTerminalOrMissingHeartbeatRun(
current.executionRunId,
);
if (stale) {
const executionRun = await db.select().from(heartbeatRuns)
.where(eq(heartbeatRuns.id, current.executionRunId)).then(rows => rows[0] ?? null);
const stale = !executionRun || TERMINAL_HEARTBEAT_RUN_STATUSES.has(executionRun.status);
if (stale && !isExplicitContinuationRetryClaim({ ...current, companyId: issueCompany.companyId }, executionRun)) {
const now = new Date();
const adoptionSet: Record<string, unknown> = {
assigneeAgentId: agentId,
+16
View File
@@ -3,6 +3,7 @@ import { isNativeWorkspaceExportRepairCause } from "@paperclipai/shared";
import { settleSlackConversation } from "../slack-conversation-lifecycle.js";
import { externalConversationStateSql } from "../slack-conversation-state.js";
import { executionRetryAccounting } from "../execution-recovery-attempt.js";
import { isExplicitContinuationRetryClaim } from "../explicit-continuation-retry-claim.js";
import {
decideLegacyContinuation, legacyDispositionEpisode, legacyDispositionFingerprint,
LEGACY_DISPOSITION_REPAIR_INSTRUCTION, type LegacyDispositionEpisode,
@@ -912,6 +913,8 @@ export function recoveryService(
scheduleRecoveryRetry?: (
runId: string,
) => Promise<typeof heartbeatRuns.$inferSelect | null>;
/** Settle retained explicit retry claims through the queue-first release policy. */
settleExplicitContinuationRetry?: (run: typeof heartbeatRuns.$inferSelect) => Promise<void>;
/**
* Whether a failed or interrupted run has consumed every bounded
* transient retry, so `scheduleRecoveryRetry` can no longer produce a
@@ -6098,6 +6101,7 @@ export function recoveryService(
: [];
const runStatusById = new Map<string, string>();
for (const row of runRows) runStatusById.set(row.id, row.status);
const runById = new Map(runRows.map(row => [row.id, row]));
// Collect the runs that a non-terminal issue still references. Such a run is
// the live run of an active issue. A different, terminal issue can also hold
@@ -6159,6 +6163,18 @@ export function recoveryService(
};
for (const issue of candidates) {
const originalOwner = issue.executionRunId ? runById.get(issue.executionRunId) : undefined;
// The pre-pass can lose its terminal write to the executor and observe a
// newer terminal status. Do not test claim ownership using the old status.
const owner = originalOwner ? { ...originalOwner,
status: runStatusById.get(originalOwner.id) ?? originalOwner.status } : undefined;
if (owner && isExplicitContinuationRetryClaim(issue, owner)) {
// A terminal row can still own cleanup and a pending bounded retry.
// Re-enter the same policy rather than erasing its exact-owner proof.
// That policy handles pending cleanup, newer input, and final denial.
await deps.settleExplicitContinuationRetry?.(owner);
continue;
}
if (
!isCleanable(issue.checkoutRunId) ||
!isCleanable(issue.executionRunId)