mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-16 03:31:44 +01:00
fix(recovery): the isolated continuation runs before the saved follow-up
Releasing a run the fallback replaced promoted the person's saved follow-up while the fallback's isolated continuation was still waiting for delivery. The follow-up then became the task's next run without the isolation marker (refused at admission on a projectless task), and the delivery worker counted it as the continuation and skipped the isolated run. Release now leaves the follow-up deferred while that continuation is pending; the continuation's own release promotes it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
cc4dfa1696
commit
1791534795
3 files changed
+40
-8
No files matched your search
@@ -2,7 +2,7 @@ import { resolveExternalTestDatabaseUrl } from "../../../__tests__/helpers/exter
|
||||
import { DELIVERY_QUEUES } from "../../../services/delivery-work-notifications.js";
|
||||
import { instanceSettingsService } from "../../../services/instance-settings.js";
|
||||
import { randomBytes, randomUUID } from "node:crypto";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { createSystemRecovery } from "../../../services/system-recovery-state.js";
|
||||
@@ -506,8 +506,12 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => {
|
||||
});
|
||||
|
||||
it.each([
|
||||
["fresh_isolated_workspace", "promoted"], ["remote_epoch", "promoted"], ["writer_ended", "released"],
|
||||
] as const)("promotes saved input behind an exhausted restart hold once a %s continuation replaced the run (%s)", async (isolation, expected) => {
|
||||
["fresh_isolated_workspace", "promoted", "not_requested"], ["remote_epoch", "promoted", "not_requested"],
|
||||
["writer_ended", "released", "not_requested"],
|
||||
// The isolated continuation is still to be delivered: it runs first and binds the new workspace;
|
||||
// promoting the follow-up now would make it the task's next run, without the isolation.
|
||||
["fresh_isolated_workspace", "released", "pending"],
|
||||
] as const)("promotes saved input behind an exhausted restart hold once a %s continuation replaced the run (%s, continuation %s)", async (isolation, expected, continuationDelivery) => {
|
||||
// Restart recovery exhausted its retries (systemRecovery.state exhausted, no stop receipt), then the
|
||||
// fallback replaced the run. An isolated continuation reuses none of the old executor's workspace,
|
||||
// so the manual-admission hold no longer stands between the task and the person's follow-up.
|
||||
@@ -523,7 +527,8 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => {
|
||||
await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: issueId, kind: "active_run_watchdog",
|
||||
status: "resolved", outcome: "restored", ownerType: "system", returnOwnerAgentId: agentId,
|
||||
cause: "fresh_continuation", fingerprint: `fresh-continuation:${runId}`, nextAction: "Continuing.", resolvedAt: new Date(),
|
||||
evidence: { runId, freshContinuation: { sourceRunId: runId, isolation, delivery: "saved_input" }, continuationDelivery: "not_requested" } });
|
||||
evidence: { runId, freshContinuation: { sourceRunId: runId, isolation,
|
||||
delivery: continuationDelivery === "pending" ? "recorded" : "saved_input" }, continuationDelivery } });
|
||||
const [comment] = await db.insert(issueComments).values({ companyId, issueId, authorType: "user", authorUserId: "responsible-user", body: "Follow-up during the run" }).returning();
|
||||
const wakeId = await seedDeferredWake({ companyId, agentId, issueId, requestedByActorId: "responsible-user",
|
||||
payload: { _paperclipWakeContext: { issueId, wakeReason: "issue_commented", wakeCommentIds: [comment.id] } },
|
||||
@@ -536,6 +541,17 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => {
|
||||
const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeId));
|
||||
if (expected === "promoted") expect(wake.status).not.toBe("deferred_issue_execution");
|
||||
else expect(wake.status).toBe("deferred_issue_execution");
|
||||
if (continuationDelivery !== "pending") return;
|
||||
// The isolated continuation is delivered and runs; its release promotes the follow-up behind it.
|
||||
const continuationRunId = await seedRun({ companyId, agentId, status: "succeeded", contextSnapshot: { issueId } });
|
||||
await db.update(heartbeatRuns).set({ finishedAt: new Date() }).where(eq(heartbeatRuns.id, continuationRunId));
|
||||
await db.update(issueRecoveryActions).set({ evidence: sql`${issueRecoveryActions.evidence}
|
||||
|| ${JSON.stringify({ continuationDelivery: "delivered", continuationRunId })}::jsonb` })
|
||||
.where(eq(issueRecoveryActions.sourceIssueId, issueId));
|
||||
expect((await release({ companyId, runId: continuationRunId, now: new Date(), suppressImmediateRecovery: true })).outcome.kind)
|
||||
.toBe("promoted");
|
||||
expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeId)))[0].status)
|
||||
.not.toBe("deferred_issue_execution");
|
||||
});
|
||||
|
||||
it.each([
|
||||
|
||||
@@ -1303,7 +1303,7 @@ export function createPostgresWakeQueueAdapter(db: Db, deps: WakeQueuePostgresAd
|
||||
// reuses none of them, so they no longer hold the task's next turn; any
|
||||
// other replacement still waits for them.
|
||||
const isolatedReplacement = agentRunSupersededByFreshContinuation(run)
|
||||
&& await freshContinuationIsolatesRun(tx, { companyId: input.companyId, runId: run.id });
|
||||
? await freshContinuationIsolatesRun(tx, { companyId: input.companyId, runId: run.id }) : false;
|
||||
if (!isolatedReplacement && await nativeManualAdmissionHoldsExecution(tx, { companyId: input.companyId, runId: run.id })) {
|
||||
return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot };
|
||||
}
|
||||
@@ -1469,6 +1469,15 @@ export function createPostgresWakeQueueAdapter(db: Db, deps: WakeQueuePostgresAd
|
||||
// A release must leave deferred messages intact while execution is held.
|
||||
// The finishing conversation may still own its lease until finally cleanup.
|
||||
// Allow only bounded retry planning in that case; admission stays gated.
|
||||
// The fallback's isolated continuation runs before any saved input: it
|
||||
// alone carries the marker that prepares the new workspace and binds it.
|
||||
// Promoting the follow-up now would make it the task's next run without
|
||||
// that isolation (and the delivery worker would then skip the
|
||||
// continuation). The follow-up stays deferred and is promoted behind
|
||||
// the continuation when that run is released.
|
||||
if (isolatedReplacement && isolatedReplacement.continuationPending) {
|
||||
return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot };
|
||||
}
|
||||
const executionBlocker = await getExecutionBlocker(tx, issueRow.companyId, issueRow.id);
|
||||
const recoveryOnly = Boolean(executionBlocker &&
|
||||
executionBlocker.cause === "execution_owner_active" && executionBlocker.runId === run.id &&
|
||||
|
||||
@@ -326,12 +326,19 @@ export async function isFreshIsolatedContinuationWake(db: Db, input: { companyId
|
||||
* Holds that only protect the old executor's resources then no longer stand
|
||||
* between the task and its next turn.
|
||||
*/
|
||||
export async function freshContinuationIsolatesRun(db: Db, input: { companyId: string; runId: string }): Promise<boolean> {
|
||||
const [record] = await db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and(
|
||||
export async function freshContinuationIsolatesRun(db: Db, input: { companyId: string; runId: string }): Promise<false | {
|
||||
/** The isolated continuation is recorded but not yet delivered as a run. */
|
||||
continuationPending: boolean;
|
||||
}> {
|
||||
const [record] = await db.select({ isolation: sql<string>`${issueRecoveryActions.evidence}#>>'{freshContinuation,isolation}'`,
|
||||
delivery: sql<string | null>`${issueRecoveryActions.evidence}->>'continuationDelivery'` }).from(issueRecoveryActions).where(and(
|
||||
eq(issueRecoveryActions.companyId, input.companyId),
|
||||
sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,sourceRunId}' = ${input.runId}`,
|
||||
sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,isolation}' in ('fresh_isolated_workspace', 'remote_epoch')`)).limit(1);
|
||||
return Boolean(record);
|
||||
if (!record) return false;
|
||||
// Only the recorded continuation carries the isolation marker that prepares
|
||||
// the new workspace, so while it is pending it must be the task's next run.
|
||||
return { continuationPending: record.isolation === "fresh_isolated_workspace" && record.delivery === "pending" };
|
||||
}
|
||||
|
||||
export async function ensureFreshContinuation(db: Db, input: Input): Promise<FreshContinuationResult> {
|
||||
|
||||
Reference in new issue
Block a user