fix: renew explicit continuation authorization for bounded retries (#14987)

Preserve exact user-authorized continuation receipts across bounded retries and retain the task claim through owned retry admission. Stop, reassignment, superseding input, and active cleanup continue to block unsafe continuation.

Validation: exact-head Greptile 5/5, passing CI, no unresolved review threads, and a clean merge. Detailed verification and limitations are recorded in the pull request.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Devin FoleyandPaperclip authored and GitHub committed 2026-10-05 10:57:57 -07:00
1 parent 7498705642
commit 2e67ea8dc7
11 files changed
+639 -16

No files matched your search

@@ -1,10 +1,12 @@
import * as executionContinuation from "../services/execution-continuation.js";
import * as environmentOrchestrator from "../services/environment-run-orchestrator.js";
import { remoteTerminationReceipt } from "../services/remote-execution-termination.js";
import { legacyDispositionFingerprint, LEGACY_DISPOSITION_REPAIR_INSTRUCTION } from "../services/recovery/legacy-continuation.js";
import * as controllerLeases from "../services/legacy-controller-lease.js";
import * as instructionWorkingCopies from "../services/agent-instruction-working-copies.js";
import * as runEvents from "../services/heartbeat-run-events.js";
import { instanceSettingsService } from "../services/instance-settings.js";
import { randomUUID } from "node:crypto";
import { createHash, randomUUID } from "node:crypto";
import { terminalizeLegacyExecution } from "../services/legacy-execution-recovery.js";
import { issueService } from "../services/issues.js";
import { getExecutionBlocker } from "../services/execution-blocker.js";
@@ -1489,6 +1491,207 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
return { companyId, agentId, runId, wakeupRequestId, issueId };
}
it.each(["timeout", "upstream", "cleanup_pending", "cleanup_pending_edited", "cleanup_pending_exhausted", "edited", "exhausted", "new_message", "no_claim", "reassigned", "superseded", "stopped"])(
"settles explicit retry admission through real executor cleanup: %s", async scenario => {
const { companyId, agentId, runId, issueId } = await seedQueuedIssueRunFixture();
const sourceRunId = randomUUID(), commentId = randomUUID(), nextCommentId = randomUUID();
await db.insert(heartbeatRuns).values({ id: sourceRunId, companyId, agentId, status: "failed",
createdAt: new Date("2020-01-01T00:00:00Z"), finishedAt: new Date("2020-01-01T00:00:01Z"),
contextSnapshot: { issueId } });
const [comment] = await db.insert(issueComments).values({ id: commentId, companyId, issueId,
authorType: "user", authorUserId: "responsible-user", body: "Continue the task." }).returning();
await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: issueId,
kind: "active_run_watchdog", cause: "legacy_execution_requires_reconciliation", fingerprint: sourceRunId,
status: "resolved", outcome: "cancelled", nextAction: "User continued.", evidence: { explicitUserContinuation: {
runId, previousRunId: sourceRunId, commentId, actorId: "responsible-user", recordedAt: new Date().toISOString(),
commentUpdatedAt: comment!.updatedAt.toISOString(), commentBodyHash: createHash("sha256").update(comment!.body).digest("hex"),
} } });
await db.update(heartbeatRuns).set({ createdAt: new Date(), contextSnapshot: {
issueId, taskId: issueId, wakeReason: "issue_commented", wakeCommentId: commentId,
previousRunId: sourceRunId, explicitUserContinuation: { previousRunId: sourceRunId, commentId },
} }).where(eq(heartbeatRuns.id, runId));
const identity = { id: randomUUID(), companyId, heartbeatRunId: runId, provider: "daytona", providerLeaseId: "fixture-explicit-sandbox" };
const supersedingRunId = randomUUID();
let heartbeat: ReturnType<typeof heartbeatService>;
let cancellation: ReturnType<typeof heartbeat.cancelRun> | undefined;
let cleanupObserved = false;
let cleanupPending = scenario.startsWith("cleanup_pending");
const originalFactory = environmentOrchestrator.environmentRunOrchestrator;
const release = vi.fn(async () => {
const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, identity.id));
if (!lease) return { released: [], errors: [] };
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, runId))).toHaveLength(0);
if (!cleanupObserved) {
cleanupObserved = true;
const denied = await db.select().from(heartbeatRunEvents).where(and(eq(heartbeatRunEvents.runId, runId),
eq(heartbeatRunEvents.message, "The automatic retry could not revalidate the original user continuation.")));
// Upstream failure first tries the direct scheduler, then the release
// tail. Both must defer admission until the executor releases ownership.
if (scenario === "upstream") expect(denied.length).toBeGreaterThanOrEqual(2);
if (scenario === "timeout") expect(denied.length).toBeGreaterThanOrEqual(1);
if (scenario === "new_message") {
await db.insert(issueComments).values({ id: nextCommentId, companyId, issueId, authorType: "user",
authorUserId: "responsible-user", body: "Use these new instructions." });
await heartbeat.wakeup(agentId, { source: "automation", reason: "issue_commented",
requestedByActorType: "user", requestedByActorId: "responsible-user",
payload: { issueId, commentId: nextCommentId },
contextSnapshot: { issueId, commentId: nextCommentId, wakeReason: "issue_commented" } });
}
if (scenario === "reassigned") {
const [other] = await db.insert(agents).values({ companyId, name: "New assignee", role: "engineer",
adapterType: "codex_local" }).returning();
await db.update(issues).set({ assigneeAgentId: other!.id }).where(eq(issues.id, issueId));
}
if (scenario === "superseded") {
await db.insert(heartbeatRuns).values({ id: supersedingRunId, companyId, agentId, status: "running",
contextSnapshot: { issueId } });
await db.update(issues).set({ executionRunId: supersedingRunId }).where(eq(issues.id, issueId));
}
}
await db.update(environmentLeases).set({ status: "released", cleanupStatus: "success", releasedAt: new Date() })
.where(and(eq(environmentLeases.companyId, companyId), eq(environmentLeases.heartbeatRunId, runId),
eq(environmentLeases.provider, "local")));
await db.update(environmentLeases).set(cleanupPending
? { status: "pending_cleanup", cleanupStatus: "failed", releasedAt: new Date() }
: { status: "released", cleanupStatus: "success", releasedAt: new Date(), metadata: {
remoteExecutionTermination: remoteTerminationReceipt(identity, { providerLeaseId: identity.providerLeaseId, state: "destroyed" }),
} }).where(eq(environmentLeases.id, identity.id));
return { released: [], errors: [] };
});
const factory = vi.spyOn(environmentOrchestrator, "environmentRunOrchestrator").mockImplementation((...args) => ({
...originalFactory(...args), releaseForRun: release,
}));
heartbeat = heartbeatService(db);
mockAdapterExecute.mockImplementationOnce(async (input?: unknown) => {
const invocation = input as { onCancellationReady: () => Promise<void>; onDispatch: () => void };
await invocation.onCancellationReady();
invocation.onDispatch();
expect(adapterExecutionControls.has(runId)).toBe(true);
await db.insert(environmentLeases).values({ ...identity, status: "active", leasePolicy: "ephemeral" });
if (scenario === "edited") await db.update(issueComments).set({ body: "Changed after dispatch" }).where(eq(issueComments.id, commentId));
if (scenario === "no_claim") await db.update(issues).set({ executionRunId: null }).where(eq(issues.id, issueId));
if (scenario === "exhausted") await db.update(heartbeatRuns).set({ scheduledRetryAttempt: 2,
scheduledRetryReason: "transient_failure" }).where(eq(heartbeatRuns.id, runId));
if (scenario === "stopped") {
cancellation = heartbeat.cancelRun(runId, "Operator Stop");
void cancellation.catch(() => undefined);
await waitForValue(async () => adapterExecutionControls.get(runId)?.controller.signal.aborted);
}
return { exitCode: 1, signal: scenario === "stopped" ? "SIGTERM" : null,
timedOut: scenario !== "upstream" && scenario !== "stopped",
errorMessage: scenario === "upstream" ? "Upstream temporarily unavailable" : "Execution timed out",
errorCode: scenario === "upstream" ? "upstream_unavailable" : "timeout",
resultJson: scenario === "upstream" ? { errorFamily: "transient_upstream" }
: scenario === "stopped" ? { executionCancellation: { state: "acknowledged" } } : {},
summary: "Stopped turn", provider: "test", model: "test-model" };
});
mockAdapterExecute.mockImplementation(async () => {
await db.update(issues).set({ status: "done" }).where(eq(issues.id, issueId));
return { exitCode: 0, signal: null, timedOut: false, errorMessage: null,
summary: "New message handled.", provider: "test", model: "test-model" };
});
try {
await heartbeat.resumeQueuedRuns();
await heartbeat.drainActiveRunExecutions();
if (cancellation) expect((await cancellation)?.status).toBe("cancelled");
expect(release).toHaveBeenCalled();
expect(adapterExecutionControls.has(runId)).toBe(false);
const children = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, runId));
const receipts = await db.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.sourceIssueId, issueId),
eq(issueRecoveryActions.cause, "explicit_user_continuation_retry")));
const [task] = await db.select().from(issues).where(eq(issues.id, issueId));
if (scenario === "timeout" || scenario === "upstream") {
expect(children).toHaveLength(1);
expect(receipts).toHaveLength(1);
expect(task!.executionRunId).toBe(children[0]!.id);
await expect(executionContinuation.buildExecutionContinuation({ db, companyId, issueId, agentId,
runId: children[0]!.id, context: children[0]!.contextSnapshot!, summary: null, exposeLowTrustRaw: false }))
.resolves.toMatchObject({ interruptedRunId: runId });
expect(await heartbeat.scheduleBoundedRetry(runId)).toMatchObject({ outcome: "scheduled", run: { id: children[0]!.id } });
} else {
expect(children).toHaveLength(0);
expect(receipts).toHaveLength(0);
expect(task!.executionRunId).toBe(cleanupPending ? runId : scenario === "superseded" ? supersedingRunId : null);
}
if (scenario === "new_message") expect(await db.select().from(heartbeatRuns).where(and(
eq(heartbeatRuns.companyId, companyId), sql`${heartbeatRuns.contextSnapshot}->>'wakeCommentId' = ${nextCommentId}`)))
.toHaveLength(1);
if (cleanupPending) {
if (scenario === "cleanup_pending") {
const svc = issueService(db);
const [other] = await db.insert(agents).values({ companyId, name: "Other worker", role: "engineer",
adapterType: "codex_local" }).returning();
const actorRunId = randomUUID();
await db.insert(heartbeatRuns).values({ id: actorRunId, companyId, agentId, status: "running", contextSnapshot: {} });
// Neither a rejected checkout nor stale checkout adoption may erase
// the exact pending retry claim, including mixed checkout owners.
for (const checkoutRunId of [null, runId, sourceRunId]) {
await db.update(issues).set({ checkoutRunId }).where(eq(issues.id, issueId));
expect(await svc.clearExecutionRunIfTerminal(issueId)).toBe(false);
expect(await svc.clearCheckoutRunIfTerminal(issueId)).toBe(false);
await expect(svc.checkout(issueId, other!.id, ["in_progress"], actorRunId)).rejects.toMatchObject({ status: 409 });
await expect(svc.assertCheckoutOwner(issueId, other!.id, actorRunId)).rejects.toMatchObject({ status: 409 });
await expect(svc.checkout(issueId, agentId, ["in_progress"], actorRunId)).rejects.toMatchObject({ status: 409 });
await expect(svc.assertCheckoutOwner(issueId, agentId, actorRunId)).rejects.toMatchObject({ status: 409 });
expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0]!.executionRunId).toBe(runId);
}
await db.update(issues).set({ checkoutRunId: null }).where(eq(issues.id, issueId));
await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, actorRunId));
}
await heartbeat.sweepStaleIssueLocks();
expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0]!.executionRunId).toBe(runId);
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, runId))).toHaveLength(0);
expect(await db.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.sourceIssueId, issueId),
eq(issueRecoveryActions.cause, "explicit_user_continuation_retry")))).toHaveLength(0);
if (scenario === "cleanup_pending_edited") await db.update(issueComments).set({ body: "Changed while cleanup waited" })
.where(eq(issueComments.id, commentId));
if (scenario === "cleanup_pending_exhausted") await db.update(heartbeatRuns).set({ scheduledRetryAttempt: 2,
scheduledRetryReason: "transient_failure" }).where(eq(heartbeatRuns.id, runId));
cleanupPending = false;
await heartbeat.releaseEnvironmentLeasesForRun({ runId, companyId, agentId, status: "timed_out" });
await heartbeat.sweepStaleIssueLocks();
await heartbeat.sweepStaleIssueLocks();
const afterCleanup = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, runId));
expect(afterCleanup).toHaveLength(scenario === "cleanup_pending" ? 1 : 0);
expect(await db.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.sourceIssueId, issueId),
eq(issueRecoveryActions.cause, "explicit_user_continuation_retry")))).toHaveLength(afterCleanup.length);
expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0]!.executionRunId).toBe(afterCleanup[0]?.id ?? null);
}
} finally { factory.mockRestore(); }
},
);
it("does not reuse a completed explicit turn's receipt for a missing-comment follow-up", async () => {
const { companyId, agentId, runId, issueId } = await seedQueuedIssueRunFixture();
const sourceRunId = randomUUID(), commentId = randomUUID();
await db.insert(heartbeatRuns).values({ id: sourceRunId, companyId, agentId, status: "failed",
finishedAt: new Date(), contextSnapshot: { issueId } });
await db.insert(issueComments).values({ id: commentId, companyId, issueId,
authorType: "user", authorUserId: "responsible-user", body: "Continue the task." });
await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: issueId,
kind: "active_run_watchdog", cause: "legacy_execution_requires_reconciliation", fingerprint: sourceRunId,
status: "resolved", outcome: "cancelled", nextAction: "User continued.",
evidence: { explicitUserContinuation: { runId, previousRunId: sourceRunId, commentId,
actorId: "responsible-user", recordedAt: new Date().toISOString() } } });
await db.update(heartbeatRuns).set({ contextSnapshot: {
issueId, taskId: issueId, wakeReason: "issue_assigned",
explicitUserContinuation: { previousRunId: sourceRunId, commentId },
} }).where(eq(heartbeatRuns.id, runId));
mockAdapterExecute.mockImplementationOnce(async () => {
await db.update(issues).set({ status: "done" }).where(eq(issues.id, issueId));
return { exitCode: 0, signal: null, timedOut: false, errorMessage: null,
summary: "Task completed.", provider: "test", model: "test-model" };
});
const heartbeat = heartbeatService(db);
await heartbeat.resumeQueuedRuns();
await waitForRunToSettle(heartbeat, runId);
await heartbeat.waitForRunExecutionDrain(runId);
expect(await heartbeat.getRun(runId)).toMatchObject({ status: "succeeded", issueCommentStatus: "not_applicable" });
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, runId))).toHaveLength(0);
expect(await db.select().from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, companyId),
eq(agentWakeupRequests.reason, "missing_issue_comment")))).toHaveLength(0);
});
it("persists the normalized failure without permanently blocking the conversation", async () => {
mockAdapterExecute.mockResolvedValueOnce({
exitCode: 1,
@@ -110,6 +110,31 @@ describeEmbeddedPostgres("recovery sweepStaleIssueLocks", () => {
return { companyId, agentId, failedRunId, runningRunId };
}
it.each([true, false])("preserves explicit retry ownership when finalization wins the sweep write (settler: %s)", async withSettler => {
const { companyId, agentId, runningRunId, failedRunId } = await seed();
const issueId = randomUUID();
await db.insert(issues).values({ id: issueId, companyId, title: "Retry finalization race",
status: "in_progress", assigneeAgentId: agentId, executionRunId: runningRunId });
await db.update(heartbeatRuns).set({ processPid: 2_000_000_000,
contextSnapshot: { issueId, explicitUserContinuation: { previousRunId: failedRunId, commentId: randomUUID() } },
}).where(eq(heartbeatRuns.id, runningRunId));
const beforeOrphanedRunTerminalWrite = vi.fn(async () => {
await db.update(heartbeatRuns).set({ status: "timed_out", finishedAt: new Date() })
.where(eq(heartbeatRuns.id, runningRunId));
});
const settleExplicitContinuationRetry = vi.fn(async () => {});
const result = await recoveryService(db, { enqueueWakeup: vi.fn(), beforeOrphanedRunTerminalWrite,
...(withSettler ? { settleExplicitContinuationRetry } : {}),
}).sweepStaleIssueLocks();
expect(beforeOrphanedRunTerminalWrite).toHaveBeenCalledOnce();
expect(result).toEqual({ cleared: 0, issueIds: [], terminalizedRunIds: [] });
expect(settleExplicitContinuationRetry).toHaveBeenCalledTimes(withSettler ? 1 : 0);
if (withSettler) expect(settleExplicitContinuationRetry).toHaveBeenCalledWith(expect.objectContaining({
id: runningRunId, status: "timed_out",
}));
expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0]!.executionRunId).toBe(runningRunId);
});
it("clears lock columns when checkoutRunId points at a terminal heartbeat run", async () => {
const { companyId, agentId, failedRunId } = await seed();
const issueId = randomUUID();
@@ -1196,6 +1196,18 @@ export function createPostgresWakeQueueAdapter(db: Db, deps: WakeQueuePostgresAd
const locked: LockedIssueExecution = { primaryIssue: toIssueSnapshot(issueRow), run: runSnapshot, recoveryOnly };
const result = await fn(locked, { host: buildHost(tx, deps), transaction: buildTransaction(tx, deps, db, run) });
// Explicit retry admission requires the exact source's task claim.
// Preserve it only when the existing queue-first recovery policy
// requests that retry. The clear/restore stays inside this task lock.
if (run.runtimeMode === "legacy" && ["failed", "timed_out"].includes(run.status) &&
run.contextSnapshot?.explicitUserContinuation && issueRow.executionRunId === run.id &&
result.postCommitEffects.some(effect => effect.kind === "conversation_retry_requested" && effect.runId === run.id)) {
await tx.update(issues).set({ executionRunId: run.id,
executionAgentNameKey: issueRow.executionAgentNameKey,
executionLockedAt: issueRow.executionLockedAt,
}).where(and(eq(issues.companyId, input.companyId), eq(issues.id, issueRow.id),
sql`${issues.executionRunId} is null`));
}
return { ...result, run: runSnapshot };
});
},
@@ -397,6 +397,15 @@ export async function buildExecutionContinuation(input: {
priorRuns.some(run => run.id === wake.runId && run.retryOfRunId === failedRunId))
: rows.some(comment => comment.id === value.commentId &&
comment.authorType === "user" &&
(!("commentUpdatedAt" in value || "commentBodyHash" in value || value.automaticRetry) || (
value.commentUpdatedAt === comment.updatedAt.toISOString() &&
value.commentBodyHash === createHash("sha256").update(comment.body).digest("hex")
)) &&
(!value.automaticRetry || (
object(value.automaticRetry).sourceRunId === explicitUserSource &&
comment.body.trim().length > 0 && issue.executionRunId === input.runId &&
priorRuns.some(run => run.id === value.runId && run.retryOfRunId === explicitUserSource)
)) &&
(value.queuedCommentInterruptId
? interruptQueues.some(queue => queue.id === value.queuedCommentInterruptId &&
queue.runId === value.runId &&
@@ -0,0 +1,14 @@
import type { heartbeatRuns, issues } from "@paperclipai/db";
/**
* This claim must be settled by queue-first recovery, not generic terminal-lock
* cleanup. It is pending intent only; admission still validates its authority.
*/
export function isExplicitContinuationRetryClaim(
issue: Pick<typeof issues.$inferSelect, "id" | "companyId" | "executionRunId">,
run: Pick<typeof heartbeatRuns.$inferSelect, "id" | "companyId" | "runtimeMode" | "status" | "contextSnapshot"> | null | undefined,
) {
return Boolean(run && issue.executionRunId === run.id && run.companyId === issue.companyId &&
run.runtimeMode === "legacy" && ["failed", "timed_out"].includes(run.status) &&
run.contextSnapshot?.issueId === issue.id && run.contextSnapshot.explicitUserContinuation);
}
@@ -10,7 +10,7 @@ import { createHash, randomUUID } from "node:crypto";
import { mkdtemp, mkdir, writeFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { and, eq } from "drizzle-orm";
import { and, eq, sql } from "drizzle-orm";
import { beforeAll, afterAll, describe, it, expect, vi } from "vitest";
import {
approvals, issueApprovals, issueThreadInteractions, chatConversations, chatEndpoints, toolApplications, toolConnections,
@@ -19,6 +19,8 @@ import {
} from "@paperclipai/db";
import { startEmbeddedPostgresTestDatabase, getEmbeddedPostgresTestSupport } from "../__tests__/helpers/embedded-postgres.js";
import { admitExplicitNativeContinuation } from "./explicit-native-continuation.js";
import { adapterExecutionControls, createAdapterExecutionControl } from "./adapter-execution-control.js";
import { CONVERSATION_CONTINUATION_POLICY } from "./conversation-continuation.js";
import { buildExecutionContinuation } from "./execution-continuation.js";
import { heartbeatService, persistHeartbeatRunProcessMetadata, type HeartbeatEnvironmentRuntime } from "./heartbeat.js";
import { getExecutionBlocker } from "./execution-blocker.js";
@@ -386,6 +388,197 @@ const support = await getEmbeddedPostgresTestSupport();
return result;
});
async function seedTimedOutExplicitTurn() {
const f = await seed();
const explicitUserContinuation = await admit(f);
expect(explicitUserContinuation).not.toBeNull();
const [receipt] = await db.select().from(issueRecoveryActions)
.where(eq(issueRecoveryActions.sourceIssueId, f.issueId));
await db.update(agents).set({ adapterType: "claude_local" }).where(eq(agents.id, f.agentId));
const now = new Date();
const [parent] = await db.update(heartbeatRuns).set({ runtimeMode: "legacy", status: "timed_out",
errorCode: "timeout", startedAt: now, finishedAt: now,
resultJson: { conversationContinuation: CONVERSATION_CONTINUATION_POLICY },
contextSnapshot: { issueId: f.issueId, wakeReason: "issue_commented", wakeCommentId: f.commentId,
previousRunId: f.sourceRunId, explicitUserContinuation },
}).where(eq(heartbeatRuns.id, f.successorRunId)).returning();
await db.update(issues).set({ status: "in_progress", executionRunId: parent.id })
.where(eq(issues.id, f.issueId));
return { ...f, parent, receipt, now };
}
const dispatchExplicitRetry = (f: Fixture, run: typeof heartbeatRuns.$inferSelect) => buildExecutionContinuation({
db, companyId: f.companyId, issueId: f.issueId, agentId: f.agentId, runId: run.id,
context: run.contextSnapshot!, summary: null, exposeLowTrustRaw: false,
});
it("re-admits an explicit user turn's timeout retry with its own durable authorization", async () => {
const f = await seedTimedOutExplicitTurn();
await expect(dispatchExplicitRetry(f, f.parent)).resolves.toMatchObject({ interruptedRunId: f.sourceRunId });
const heartbeat = heartbeatService(db);
const [scheduled, competing] = await Promise.all([
heartbeat.scheduleBoundedRetry(f.parent.id, { now: f.now, delayMs: 1000 }),
heartbeat.scheduleBoundedRetry(f.parent.id, { now: f.now, delayMs: 1000 }),
]);
expect(scheduled.outcome).toBe("scheduled");
if (scheduled.outcome !== "scheduled") return;
expect(competing).toMatchObject({ outcome: "scheduled", run: { id: scheduled.run.id } });
expect(scheduled.run.contextSnapshot?.explicitUserContinuation).toEqual({
previousRunId: f.parent.id, commentId: f.commentId,
});
// Dispatch validates the new run, not a borrowed parent receipt.
await expect(dispatchExplicitRetry(f, scheduled.run)).resolves.toMatchObject({ interruptedRunId: f.parent.id });
const receipts = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId));
expect(receipts).toHaveLength(2);
expect(receipts.find(row => row.id === f.receipt.id)?.evidence).toEqual(f.receipt.evidence);
expect(receipts.find(row => row.id !== f.receipt.id)?.evidence.explicitUserContinuation).toMatchObject({
runId: scheduled.run.id, previousRunId: f.parent.id, actorId: "board", commentId: f.commentId,
automaticRetry: { sourceRunId: f.parent.id, sourceAuthorizationId: f.receipt.id },
});
expect(await heartbeat.scheduleBoundedRetry(f.parent.id, { now: f.now })).toMatchObject({
outcome: "scheduled", run: { id: scheduled.run.id },
});
expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))).toHaveLength(2);
// Removing the new receipt reproduces the original dispatch failure.
await db.delete(issueRecoveryActions).where(eq(issueRecoveryActions.id, receipts.find(row => row.id !== f.receipt.id)!.id));
await expect(dispatchExplicitRetry(f, scheduled.run)).rejects.toThrow("continuation_user_authorization_missing");
});
it("waits for the parent adapter controller even when no environment lease remains", async () => {
const f = await seedTimedOutExplicitTurn();
const heartbeat = heartbeatService(db);
const control = createAdapterExecutionControl();
adapterExecutionControls.set(f.parent.id, control);
try {
expect(await heartbeat.scheduleBoundedRetry(f.parent.id)).toMatchObject({ outcome: "not_scheduled" });
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, f.parent.id))).toHaveLength(0);
expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))).toHaveLength(1);
} finally {
adapterExecutionControls.delete(f.parent.id);
control.finish();
}
expect(await heartbeat.scheduleBoundedRetry(f.parent.id)).toMatchObject({ outcome: "scheduled" });
});
it("binds each bounded explicit retry to its immediate parent without replacing prior receipts", async () => {
const f = await seedTimedOutExplicitTurn();
const heartbeat = heartbeatService(db);
const first = await heartbeat.scheduleBoundedRetry(f.parent.id, { now: f.now });
expect(first.outcome).toBe("scheduled");
if (first.outcome !== "scheduled") return;
const nextNow = new Date(f.now.getTime() + 2000);
await db.update(heartbeatRuns).set({ status: "timed_out", errorCode: "timeout", startedAt: f.now,
finishedAt: nextNow }).where(eq(heartbeatRuns.id, first.run.id));
const second = await heartbeat.scheduleBoundedRetry(first.run.id, { now: nextNow });
expect(second.outcome).toBe("scheduled");
if (second.outcome !== "scheduled") return;
expect(second.run).toMatchObject({ retryOfRunId: first.run.id, scheduledRetryAttempt: 2 });
await expect(dispatchExplicitRetry(f, second.run)).resolves.toMatchObject({ interruptedRunId: first.run.id });
const receipts = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId));
expect(receipts).toHaveLength(3);
expect(receipts.find(row => row.id === f.receipt.id)?.evidence).toEqual(f.receipt.evidence);
await db.update(heartbeatRuns).set({ status: "timed_out", errorCode: "timeout", startedAt: nextNow,
finishedAt: nextNow }).where(eq(heartbeatRuns.id, second.run.id));
expect(await heartbeat.scheduleBoundedRetry(second.run.id, { now: nextNow })).toMatchObject({ outcome: "retry_exhausted" });
expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))).toHaveLength(3);
});
it("keeps sub-millisecond source ordering when authorizing a timeout retry", async () => {
const f = await seedTimedOutExplicitTurn();
await db.execute(sql`update heartbeat_runs set created_at = ${f.now.toISOString()}::timestamptz +
case when id = ${f.sourceRunId} then interval '100 microseconds' else interval '500 microseconds' end
where company_id = ${f.companyId} and id in (${f.sourceRunId}, ${f.parent.id})`);
const result = await heartbeatService(db).scheduleBoundedRetry(f.parent.id, { now: f.now });
expect(result.outcome).toBe("scheduled");
});
it.each(["max_turns_continuation", "interaction_continuation_infra_retry", "workspace_busy"])(
"does not lend explicit user authority to the separate %s retry contract", async retryReason => {
const f = await seedTimedOutExplicitTurn();
const result = await heartbeatService(db).scheduleBoundedRetry(f.parent.id, { now: f.now, retryReason });
expect(result.outcome).not.toBe("scheduled");
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, f.parent.id))).toHaveLength(0);
},
);
it.each(["deleted", "edited", "same_millisecond_edit", "empty", "actor", "run_authored", "legacy_receipt", "receipt_run", "receipt_company", "source", "task", "reassigned", "done", "superseded", "native_superseding", "interrupted_intent"])(
"refuses to re-admit an automatic explicit retry after %s changes", async kind => {
const f = await seedTimedOutExplicitTurn();
if (["deleted", "edited", "same_millisecond_edit", "empty", "actor", "run_authored"].includes(kind)) await db.update(issueComments).set({
...(kind === "deleted" ? { deletedAt: new Date() } : {}),
...(kind === "edited" ? { body: "Different instructions", updatedAt: new Date(f.now.getTime() + 1000) } : {}),
...(kind === "same_millisecond_edit" ? { body: "Different instructions within the recorded millisecond" } : {}),
...(kind === "empty" ? { body: " " } : {}),
...(kind === "actor" ? { authorUserId: "another-user" } : {}),
...(kind === "run_authored" ? { createdByRunId: f.parent.id } : {}),
}).where(eq(issueComments.id, f.commentId));
if (["receipt_run", "interrupted_intent"].includes(kind)) await db.update(issueRecoveryActions).set({ evidence: {
...f.receipt.evidence, explicitUserContinuation: {
...(f.receipt.evidence.explicitUserContinuation as Record<string, unknown>),
...(kind === "receipt_run" ? { runId: randomUUID() } : { queuedCommentInterruptId: randomUUID() }),
},
} }).where(eq(issueRecoveryActions.id, f.receipt.id));
if (kind === "receipt_company") await db.update(issueRecoveryActions).set({ companyId: (await seed()).companyId })
.where(eq(issueRecoveryActions.id, f.receipt.id));
if (kind === "legacy_receipt") {
const { commentBodyHash: _bodyHash, commentUpdatedAt: _updatedAt, ...legacy } =
f.receipt.evidence.explicitUserContinuation as Record<string, unknown>;
await db.update(issueRecoveryActions).set({ evidence: { ...f.receipt.evidence,
explicitUserContinuation: legacy } }).where(eq(issueRecoveryActions.id, f.receipt.id));
}
if (kind === "source" || kind === "task") await db.update(heartbeatRuns).set({ contextSnapshot: {
...f.parent.contextSnapshot,
...(kind === "task" ? { issueId: randomUUID() } : { explicitUserContinuation: { previousRunId: randomUUID(), commentId: f.commentId } }),
} }).where(eq(heartbeatRuns.id, f.parent.id));
if (kind === "reassigned" || kind === "done") await db.update(issues).set(kind === "done"
? { status: "done" } : { assigneeAgentId: null }).where(eq(issues.id, f.issueId));
if (kind === "superseded") await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId,
status: "succeeded", contextSnapshot: { issueId: f.issueId }, createdAt: new Date(f.now.getTime() + 1000) });
if (kind === "native_superseding") await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId,
status: "running", runtimeMode: "native", nativeIssueId: f.issueId, contextSnapshot: {} });
const result = await heartbeatService(db).scheduleBoundedRetry(f.parent.id, { now: f.now });
expect(result.outcome).not.toBe("scheduled");
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, f.parent.id))).toHaveLength(0);
expect(await db.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.sourceIssueId, f.issueId),
eq(issueRecoveryActions.cause, "explicit_user_continuation_retry")))).toHaveLength(0);
},
);
it("binds the initial user-message admission before any automatic retry", async () => {
const f = await seedTimedOutExplicitTurn();
await expect(dispatchExplicitRetry(f, f.parent)).resolves.toMatchObject({ interruptedRunId: f.sourceRunId });
await db.update(issueComments).set({ body: "Changed after the original admission" })
.where(eq(issueComments.id, f.commentId));
await expect(dispatchExplicitRetry(f, f.parent)).rejects.toThrow("continuation_user_authorization_missing");
});
it.each(["deleted", "edited", "same_millisecond_edit", "actor", "receipt_run", "receipt_source", "retry_parent", "execution_owner"])(
"revalidates the successor's exact authorization at dispatch: %s", async kind => {
const f = await seedTimedOutExplicitTurn();
const scheduled = await heartbeatService(db).scheduleBoundedRetry(f.parent.id, { now: f.now });
expect(scheduled.outcome).toBe("scheduled");
if (scheduled.outcome !== "scheduled") return;
if (["deleted", "edited", "same_millisecond_edit", "actor"].includes(kind)) await db.update(issueComments).set({
...(kind === "deleted" ? { deletedAt: new Date() } : {}),
...(kind === "edited" ? { body: "Changed after scheduling", updatedAt: new Date(f.now.getTime() + 1000) } : {}),
...(kind === "same_millisecond_edit" ? { body: "Changed within the recorded millisecond" } : {}),
...(kind === "actor" ? { authorUserId: "another-user" } : {}),
}).where(eq(issueComments.id, f.commentId));
if (kind === "receipt_run" || kind === "receipt_source") {
const [receipt] = await db.select().from(issueRecoveryActions).where(and(
eq(issueRecoveryActions.sourceIssueId, f.issueId), eq(issueRecoveryActions.cause, "explicit_user_continuation_retry")));
await db.update(issueRecoveryActions).set({ evidence: { ...receipt.evidence, explicitUserContinuation: {
...(receipt.evidence.explicitUserContinuation as Record<string, unknown>),
...(kind === "receipt_run" ? { runId: f.parent.id } : { previousRunId: f.sourceRunId }),
} } }).where(eq(issueRecoveryActions.id, receipt.id));
}
if (kind === "retry_parent") await db.update(heartbeatRuns).set({ retryOfRunId: f.sourceRunId })
.where(eq(heartbeatRuns.id, scheduled.run.id));
if (kind === "execution_owner") await db.update(issues).set({ executionRunId: f.parent.id })
.where(eq(issues.id, f.issueId));
await expect(dispatchExplicitRetry(f, scheduled.run)).rejects.toThrow("continuation_user_authorization_missing");
},
);
it.each(["active", "removed", "unavailable", "paused", "disabled"])("shows usable recovery guidance for external chat cancellations: %s", async kind => {
const f = await seed();
await db.delete(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.sourceRunId));
@@ -1,3 +1,4 @@
import { createHash } from "node:crypto";
import { appendHeartbeatRunEvent } from "./heartbeat-run-events.js";
import { canContinueCancelledRun } from "./run-cancellation.js";
import { readQueuedInteractionResponse } from "./queued-interaction-response.js";
@@ -292,6 +293,8 @@ export async function admitExplicitNativeContinuation(input: {
});
}
const authorization = { actorId, commentId, ...(response ? { interactionId: response.source.interactionId } : {}), ...(retry ? { failedRunId: input.failedRunId } : {}),
...(comment ? { commentUpdatedAt: comment.updatedAt.toISOString(),
commentBodyHash: createHash("sha256").update(comment.body).digest("hex") } : {}),
...(queuedInterrupt ? { queuedCommentInterruptId: input.queuedCommentInterruptId } : {}),
...(queuedRequest ? { queuedCommentRequestId: input.queuedCommentRequestId } : {}), runId: input.successorRunId,
previousRunId: previous.id, recordedAt: new Date().toISOString() };
@@ -329,3 +332,91 @@ export async function admitExplicitNativeContinuation(input: {
});
return { previousRunId: previous.id, commentId, ...(retry ? { failedRunId: input.failedRunId! } : {}) };
}
/** Re-admit a bounded automatic retry without lending it its parent's receipt.
* The caller holds the task lock and creates the successor in this transaction.
* Only unchanged user messages are renewable; Retry/Interrupt intents keep their
* separate one-run contracts. The scheduler still owns retry and cleanup policy.
*/
export async function admitExplicitContinuationRetry(input: {
db: Db; companyId: string; issueId: string; agentId: string;
parentRunId: string; successorRunId: string; now: Date;
}): Promise<{ previousRunId: string; commentId: string } | null> {
const { db, companyId, issueId, agentId } = input;
const [task] = await db.select().from(issues).where(and(
eq(issues.companyId, companyId), eq(issues.id, issueId),
)).for("update");
const [parent] = await db.select().from(heartbeatRuns).where(and(
eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, input.parentRunId),
eq(heartbeatRuns.agentId, agentId),
)).for("update");
if (!task || task.assigneeAgentId !== agentId || task.executionRunId !== input.parentRunId ||
["done", "cancelled"].includes(task.status) || !parent ||
!["failed", "timed_out"].includes(parent.status) || !parent.finishedAt ||
parent.runtimeMode !== "legacy" || parent.contextSnapshot?.issueId !== issueId ||
(parent.nativeIssueId !== null && parent.nativeIssueId !== issueId) ||
adapterExecutionControls.has(parent.id) ||
await getExecutionBlocker(db, companyId, issueId)) return null;
const context = parent.contextSnapshot;
const explicit = context.explicitUserContinuation as Record<string, unknown> | undefined;
if (!explicit || !z.string().guid().safeParse(explicit.previousRunId).success ||
!z.string().guid().safeParse(explicit.commentId).success || explicit.failedRunId) return null;
const commentId = explicit.commentId as string;
const [comment] = await db.select().from(issueComments).where(and(
eq(issueComments.companyId, companyId), eq(issueComments.issueId, issueId),
eq(issueComments.id, commentId), eq(issueComments.authorType, "user"),
isNull(issueComments.createdByRunId), isNull(issueComments.deletedAt),
));
if (!comment?.authorUserId || !comment.body.trim()) return null;
const receipts = await db.select().from(issueRecoveryActions).where(and(
eq(issueRecoveryActions.companyId, companyId), eq(issueRecoveryActions.sourceIssueId, issueId),
eq(issueRecoveryActions.status, "resolved"),
sql`${issueRecoveryActions.evidence}->'explicitUserContinuation'->>'runId' = ${parent.id}`,
));
const receipt = receipts.find(row => {
const auth = row.evidence.explicitUserContinuation as Record<string, unknown> | undefined;
return auth && auth.previousRunId === explicit.previousRunId && auth.commentId === commentId &&
auth.actorId === comment.authorUserId && !auth.failedRunId && !auth.queuedCommentInterruptId &&
auth.commentUpdatedAt === comment.updatedAt.toISOString() &&
auth.commentBodyHash === createHash("sha256").update(comment.body).digest("hex");
});
if (!receipt) return null;
const [superseding] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(
eq(heartbeatRuns.companyId, companyId),
or(eq(heartbeatRuns.nativeIssueId, issueId), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`),
ne(heartbeatRuns.id, parent.id), ne(heartbeatRuns.id, input.successorRunId),
or(eq(heartbeatRuns.retryOfRunId, parent.id),
// Keep PostgreSQL's timestamp precision: Date would round the parent
// down and mistake an older run in the same millisecond for a successor.
sql`${heartbeatRuns.createdAt} > (select source.created_at from heartbeat_runs source
where source.company_id = ${companyId} and source.id = ${parent.id})`,
inArray(heartbeatRuns.status, ["queued", "running", "scheduled_retry"])),
)).limit(1);
if (superseding) return null;
// Reuse dispatch's complete source/actor/receipt validation on the parent.
// It proves only the parent; the new receipt below authorizes the successor.
try {
await buildExecutionContinuation({ db, companyId, issueId, agentId, runId: parent.id,
context, summary: null, exposeLowTrustRaw: false });
} catch (error) {
if (error instanceof Error && ["continuation_user_authorization_missing",
"continuation_source_context_missing", "continuation_task_ownership_changed"].includes(error.message)) return null;
throw error;
}
const continuation = { previousRunId: parent.id, commentId };
await db.insert(issueRecoveryActions).values({
companyId, sourceIssueId: issueId, kind: "active_run_watchdog",
cause: "explicit_user_continuation_retry", fingerprint: input.successorRunId,
status: "resolved", outcome: "retry_authorized", resolvedAt: input.now,
nextAction: "The bounded retry has its own authorization for the unchanged user message.",
evidence: { explicitUserContinuation: {
...continuation, actorId: comment.authorUserId, runId: input.successorRunId,
commentUpdatedAt: comment.updatedAt.toISOString(),
commentBodyHash: createHash("sha256").update(comment.body).digest("hex"),
recordedAt: input.now.toISOString(), automaticRetry: {
sourceRunId: parent.id, sourceAuthorizationId: receipt.id,
},
} },
});
return continuation;
}
+42 -3
View File
@@ -32,7 +32,7 @@ import { legacyControllerBootId, legacyControllerClaim, renewLegacyControllerLea
import { completeTerminatedRemoteNativeSessionCleanup } from "../vendor/paperclip-runner/index.js";
import { hasRemoteTerminationReceipt, remoteExecutionHasStopped, remoteTerminationReceipt, stoppedRemoteCleanupScopes } from "./remote-execution-termination.js";
import { applyConnectorSkills, prepareConnectorSkillDelivery, resolveConnectorAssignments } from "./connector-runtime.js";
import { admitExplicitNativeContinuation, undeliveredLegacyUserCommentIds } from "./explicit-native-continuation.js";
import { admitExplicitNativeContinuation, admitExplicitContinuationRetry, undeliveredLegacyUserCommentIds } from "./explicit-native-continuation.js";
import { canRetryStoppedRun, isCancelledNativeStartup } from "./cancelled-native-startup.js";
import { connectionIntentService } from "./connection-intents.js";
import { managedAiSessionFingerprintConfig, prepareManagedAiRuntime, assertManagedAiProjectAuth, stripAiAuthBindings, isAiConnectionBusy, AI_AUTH_ENV_KEYS } from "./ai-connection-runtime.js";
@@ -9649,6 +9649,7 @@ export function heartbeatService(
const recovery = recoveryService(db, {
enqueueWakeup,
liveRunExecutions,
settleExplicitContinuationRetry: releaseIssueExecutionAndPromote,
scheduleRecoveryRetry: async (runId) => {
const [run] = await db
.select()
@@ -9866,10 +9867,19 @@ export function heartbeatService(
));
const agent = source ? await getAgent(source.agentId) : null;
if (source && agent && agent.companyId === source.companyId) {
await scheduleBoundedRetryForRun(source, agent, effect.reviewParticipant ? {
const retry = await scheduleBoundedRetryForRun(source, agent, effect.reviewParticipant ? {
retryReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON,
wakeReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON,
} : undefined);
const issueId = readNonEmptyString(source.contextSnapshot?.issueId);
if (retry.outcome !== "scheduled" && source.contextSnapshot?.explicitUserContinuation && issueId &&
!adapterExecutionControls.has(source.id) && !(await getExecutionBlocker(db, source.companyId, issueId))) {
// Cleanup has settled, so exhaustion or revoked/missing authority
// must not retain a terminal claim. Do not request another retry.
const settled = await wakeQueue.releaseIssueExecution({ companyId: source.companyId,
runId: source.id, now: new Date(), suppressImmediateRecovery: true });
await applyWakeQueuePostCommitEffects(settled.postCommitEffects);
}
}
} else if (effect.kind === "run_queued") {
publishLiveEvent({
@@ -14347,7 +14357,9 @@ export function heartbeatService(
presentationDecision?: RunPresentationDecision | null,
) {
const contextSnapshot = parseObject(run.contextSnapshot);
if (readNonEmptyString(contextSnapshot.goalControlRequestId)) {
// The explicit receipt admitted one turn. A prose-only follow-up cannot
// reuse it or renew it under the separate transient-failure retry policy.
if (readNonEmptyString(contextSnapshot.goalControlRequestId) || contextSnapshot.explicitUserContinuation) {
if (run.issueCommentStatus !== "not_applicable") {
await patchRunIssueCommentStatus(run.id, {
issueCommentStatus: "not_applicable",
@@ -15702,6 +15714,7 @@ export function heartbeatService(
| "issue_cancelled"
| "issue_terminal_status"
| "issue_not_in_progress"
| "continuation_user_authorization_missing"
| "issue_execution_lock_changed";
issueId: string | null;
details: Record<string, unknown>;
@@ -15979,6 +15992,22 @@ export function heartbeatService(
}
}
const scheduledRunId = randomUUID();
if (contextSnapshot.explicitUserContinuation) {
const continuation = issueId && retryReason === "transient_failure" ? await admitExplicitContinuationRetry({
db: tx as unknown as Db, companyId: run.companyId, issueId, agentId: run.agentId,
parentRunId: run.id, successorRunId: scheduledRunId, now,
}) : null;
if (!continuation) return {
outcome: "not_scheduled", issueId,
errorCode: "continuation_user_authorization_missing",
reason: "The automatic retry could not revalidate the original user continuation.",
details: {},
};
retryContextSnapshot.explicitUserContinuation = continuation;
retryContextSnapshot.previousRunId = continuation.previousRunId;
}
const wakeupRequest = await tx
.insert(agentWakeupRequests)
.values({
@@ -16029,6 +16058,7 @@ export function heartbeatService(
const scheduledRun = await tx
.insert(heartbeatRuns)
.values({
id: scheduledRunId,
companyId: run.companyId,
agentId: run.agentId,
invocationSource: "automation",
@@ -26767,6 +26797,15 @@ export function heartbeatService(
}
// Terminalization precedes lease and adapter cleanup. Only now is the
// owner gone; retry pending input for ordinary completions as well as Stop.
if (latestRun?.runtimeMode === "legacy" && ["failed", "timed_out"].includes(latestRun.status) &&
latestRun.contextSnapshot?.explicitUserContinuation) {
// Re-run the same queue-first recovery decision after cleanup. Its
// earlier retry request could not authorize work while this executor
// still held its controller or environment lease.
await releaseIssueExecutionAndPromote(latestRun).catch(err => {
logger.error({ err, runId: run.id }, "failed to settle explicit continuation after cleanup");
});
}
if (latestRun?.runtimeMode === "legacy" && isHeartbeatRunTerminalStatus(latestRun.status)) {
const [pending] = await db.select({ id: agentWakeupRequests.id, payload: agentWakeupRequests.payload }).from(agentWakeupRequests).where(and(
eq(agentWakeupRequests.companyId, run.companyId), eq(agentWakeupRequests.agentId, run.agentId),
+28 -11
View File
@@ -9,6 +9,7 @@ import { executionProjectionsForRuns } from "./execution-projection.js";
import type { ExecutionProjection } from "@paperclipai/shared";
import { Buffer } from "node:buffer";
import { createHash, randomUUID } from "node:crypto";
import { isExplicitContinuationRetryClaim } from "./explicit-continuation-retry-claim.js";
import { markdownToPlainText, parseMarkdown } from "chat";
import {
and,
@@ -7497,6 +7498,7 @@ export function issueService(db: Db) {
const lockedIssue = await tx
.select({
id: issues.id,
companyId: issues.companyId,
status: issues.status,
assigneeAgentId: issues.assigneeAgentId,
checkoutRunId: issues.checkoutRunId,
@@ -7528,7 +7530,7 @@ export function issueService(db: Db) {
]);
const [existingRun, actorRun] = await Promise.all([
tx
.select({ status: heartbeatRuns.status })
.select()
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, input.expectedCheckoutRunId))
.then((rows) => rows[0] ?? null),
@@ -7540,6 +7542,17 @@ export function issueService(db: Db) {
]);
const stale =
!existingRun || TERMINAL_HEARTBEAT_RUN_STATUSES.has(existingRun.status);
if (isExplicitContinuationRetryClaim(lockedIssue, existingRun)) {
return { adopted: null, latest: lockedIssue };
}
if (lockedIssue.executionRunId && lockedIssue.executionRunId !== input.expectedCheckoutRunId) {
const executionRun = await tx.select().from(heartbeatRuns)
.where(eq(heartbeatRuns.id, lockedIssue.executionRunId)).for("update")
.then(rows => rows[0] ?? null);
if (isExplicitContinuationRetryClaim(lockedIssue, executionRun)) {
return { adopted: null, latest: lockedIssue };
}
}
const actorLive =
actorRun && !TERMINAL_HEARTBEAT_RUN_STATUSES.has(actorRun.status);
if (!stale || !actorLive) {
@@ -7649,7 +7662,7 @@ export function issueService(db: Db) {
sql`select ${issues.id} from ${issues} where ${issues.id} = ${issueId} for update`,
);
const issue = await tx
.select({ executionRunId: issues.executionRunId })
.select({ id: issues.id, companyId: issues.companyId, executionRunId: issues.executionRunId })
.from(issues)
.where(eq(issues.id, issueId))
.then((rows) => rows[0] ?? null);
@@ -7659,11 +7672,12 @@ export function issueService(db: Db) {
sql`select ${heartbeatRuns.id} from ${heartbeatRuns} where ${heartbeatRuns.id} = ${issue.executionRunId} for update`,
);
const run = await tx
.select({ status: heartbeatRuns.status })
.select()
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, issue.executionRunId))
.then((rows) => rows[0] ?? null);
if (run && !TERMINAL_HEARTBEAT_RUN_STATUSES.has(run.status)) return false;
if (isExplicitContinuationRetryClaim(issue, run)) return false;
const updated = await tx
.update(issues)
@@ -7689,8 +7703,7 @@ export function issueService(db: Db) {
// Symmetric to clearExecutionRunIfTerminal. Clears checkoutRunId (and the
// bundled execution lock cols) when the row's checkoutRunId points at a
// heartbeat run that is terminal or no longer exists. No assignee/status
// precondition: a terminal run holds no real claim regardless of who is
// assigned or what status the issue is currently in.
// precondition. Explicit retry claims remain owned by queue-first settlement.
async function clearCheckoutRunIfTerminal(issueId: string): Promise<boolean> {
return db.transaction(async (tx) => {
await tx.execute(
@@ -7698,6 +7711,8 @@ export function issueService(db: Db) {
);
const issue = await tx
.select({
id: issues.id,
companyId: issues.companyId,
checkoutRunId: issues.checkoutRunId,
executionRunId: issues.executionRunId,
})
@@ -7710,11 +7725,12 @@ export function issueService(db: Db) {
sql`select ${heartbeatRuns.id} from ${heartbeatRuns} where ${heartbeatRuns.id} = ${issue.checkoutRunId} for update`,
);
const run = await tx
.select({ status: heartbeatRuns.status })
.select()
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, issue.checkoutRunId))
.then((rows) => rows[0] ?? null);
if (run && !TERMINAL_HEARTBEAT_RUN_STATUSES.has(run.status)) return false;
if (isExplicitContinuationRetryClaim(issue, run)) return false;
if (
issue.executionRunId &&
@@ -7724,7 +7740,7 @@ export function issueService(db: Db) {
sql`select ${heartbeatRuns.id} from ${heartbeatRuns} where ${heartbeatRuns.id} = ${issue.executionRunId} for update`,
);
const executionRun = await tx
.select({ status: heartbeatRuns.status })
.select()
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, issue.executionRunId))
.then((rows) => rows[0] ?? null);
@@ -7733,6 +7749,7 @@ export function issueService(db: Db) {
!TERMINAL_HEARTBEAT_RUN_STATUSES.has(executionRun.status)
)
return false;
if (isExplicitContinuationRetryClaim(issue, executionRun)) return false;
}
const updated = await tx
@@ -11600,10 +11617,10 @@ export function issueService(db: Db) {
current.executionRunId !== checkoutRunId &&
(current.assigneeAgentId === agentId || current.assigneeAgentId == null)
) {
const stale = await isTerminalOrMissingHeartbeatRun(
current.executionRunId,
);
if (stale) {
const executionRun = await db.select().from(heartbeatRuns)
.where(eq(heartbeatRuns.id, current.executionRunId)).then(rows => rows[0] ?? null);
const stale = !executionRun || TERMINAL_HEARTBEAT_RUN_STATUSES.has(executionRun.status);
if (stale && !isExplicitContinuationRetryClaim({ ...current, companyId: issueCompany.companyId }, executionRun)) {
const now = new Date();
const adoptionSet: Record<string, unknown> = {
assigneeAgentId: agentId,
+16
View File
@@ -3,6 +3,7 @@ import { isNativeWorkspaceExportRepairCause } from "@paperclipai/shared";
import { settleSlackConversation } from "../slack-conversation-lifecycle.js";
import { externalConversationStateSql } from "../slack-conversation-state.js";
import { executionRetryAccounting } from "../execution-recovery-attempt.js";
import { isExplicitContinuationRetryClaim } from "../explicit-continuation-retry-claim.js";
import {
decideLegacyContinuation, legacyDispositionEpisode, legacyDispositionFingerprint,
LEGACY_DISPOSITION_REPAIR_INSTRUCTION, type LegacyDispositionEpisode,
@@ -912,6 +913,8 @@ export function recoveryService(
scheduleRecoveryRetry?: (
runId: string,
) => Promise<typeof heartbeatRuns.$inferSelect | null>;
/** Settle retained explicit retry claims through the queue-first release policy. */
settleExplicitContinuationRetry?: (run: typeof heartbeatRuns.$inferSelect) => Promise<void>;
/**
* Whether a failed or interrupted run has consumed every bounded
* transient retry, so `scheduleRecoveryRetry` can no longer produce a
@@ -6098,6 +6101,7 @@ export function recoveryService(
: [];
const runStatusById = new Map<string, string>();
for (const row of runRows) runStatusById.set(row.id, row.status);
const runById = new Map(runRows.map(row => [row.id, row]));
// Collect the runs that a non-terminal issue still references. Such a run is
// the live run of an active issue. A different, terminal issue can also hold
@@ -6159,6 +6163,18 @@ export function recoveryService(
};
for (const issue of candidates) {
const originalOwner = issue.executionRunId ? runById.get(issue.executionRunId) : undefined;
// The pre-pass can lose its terminal write to the executor and observe a
// newer terminal status. Do not test claim ownership using the old status.
const owner = originalOwner ? { ...originalOwner,
status: runStatusById.get(originalOwner.id) ?? originalOwner.status } : undefined;
if (owner && isExplicitContinuationRetryClaim(issue, owner)) {
// A terminal row can still own cleanup and a pending bounded retry.
// Re-enter the same policy rather than erasing its exact-owner proof.
// That policy handles pending cleanup, newer input, and final denial.
await deps.settleExplicitContinuationRetry?.(owner);
continue;
}
if (
!isCleanable(issue.checkoutRunId) ||
!isCleanable(issue.executionRunId)