mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-07 07:55:47 +01:00
fix(recovery): continue verified preparation after lease interruption (#18892)
Use recorded system interruption and the existing execution, controller, stopped-lease, and saved-file proof instead of a shutdown-only error label. Preserve bounded retries, user Stops, task authority, and exactly one continuation. Keep the chat recovery fixture faithful to the preparation stage and terminal time written by the real lifecycle. Retain missing-field safety controls. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
db9be4c5a6
commit
7b1c53f8f1
10 files changed
+128
-34
No files matched your search
@@ -290,6 +290,9 @@ import { createHeartbeatQueueTestDependencies } from "./helpers/heartbeat-queue.
|
||||
async function interruptedBootstrap() {
|
||||
const f = await seed(); await f.finish(); const run = await f.run();
|
||||
const interrupted = await terminalizeLegacyExecution({ db, run, status: "interrupted", patch: {
|
||||
// The real controller claim records preparation before terminal status
|
||||
// records completion. The delivery fixture otherwise leaves both unset.
|
||||
executionStage: "preparing", finishedAt: new Date(),
|
||||
errorCode: "server_shutdown_interrupted", runnerProfileJson: { adapterDispatch: { adapterType: "paperclip_runner" } },
|
||||
} });
|
||||
expect(await getExecutionBlocker(db, f.companyId, f.sourceId)).toMatchObject({ runId: run.id });
|
||||
@@ -338,8 +341,10 @@ import { createHeartbeatQueueTestDependencies } from "./helpers/heartbeat-queue.
|
||||
expect(await settleInterruptedNativeBootstrap(db, { run, providerDispatchStarted: false })).not.toBeNull();
|
||||
expect(await issueService(db).getById(f.sourceId)).toMatchObject({ status: "blocked" });
|
||||
});
|
||||
it.each(["provider-entered", "runtime-selected", "other-adapter", "restore-unsafe", "active-lease", "cleanup-failed", "retry-exhausted"])("retains an interrupted bootstrap hold for %s", async kind => {
|
||||
it.each(["provider-entered", "runtime-selected", "other-adapter", "restore-unsafe", "active-lease", "cleanup-failed", "retry-exhausted", "missing-stage", "unfinished"])("retains an interrupted bootstrap hold for %s", async kind => {
|
||||
const { f, run } = await interruptedBootstrap();
|
||||
if (kind === "missing-stage") await db.update(heartbeatRuns).set({ executionStage: null }).where(eq(heartbeatRuns.id, run.id));
|
||||
if (kind === "unfinished") await db.update(heartbeatRuns).set({ finishedAt: null }).where(eq(heartbeatRuns.id, run.id));
|
||||
if (kind === "runtime-selected") await db.update(heartbeatRuns).set({ runtimeModeResolvedAt: new Date() }).where(eq(heartbeatRuns.id, run.id));
|
||||
if (kind === "other-adapter") await db.update(heartbeatRuns).set({ runnerProfileJson: { adapterDispatch: { adapterType: "process" } } }).where(eq(heartbeatRuns.id, run.id));
|
||||
if (kind === "restore-unsafe") await db.update(heartbeatRuns).set({ resultJson: { workspaceRestoreFailure: "restore_unsafe_archive" } }).where(eq(heartbeatRuns.id, run.id));
|
||||
|
||||
@@ -4,7 +4,7 @@ import { environmentLeases, heartbeatRunEvents, heartbeatRuns, nativeRunFinaliza
|
||||
import { claimedAdapterType } from "./conversation-continuation.js";
|
||||
import { PROCESS_IDENTITY_RECORDED, PROCESS_START_REQUESTED } from "./native-local-process-stop.js";
|
||||
import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js";
|
||||
import { canContinueCancelledRun } from "./run-cancellation.js";
|
||||
import { canContinueCancelledRun, readRunCancellation } from "./run-cancellation.js";
|
||||
import { legacyControllerBootId } from "./legacy-controller-lease.js";
|
||||
import { hasRequiredWorkspaceRecovery } from "./workspace-restore-recovery-state.js";
|
||||
import { hasVerifiedWorkspaceRepair } from "./workspace-recovery-receipt.js";
|
||||
@@ -37,14 +37,28 @@ export async function isFailedNativePreparation(db: Db, run: Run, coordinator: C
|
||||
return !lease && !result && !await nativeStartupHasExecutionEvidence(db, run);
|
||||
}
|
||||
|
||||
/** Candidate state only. Admission must also verify ownership, execution records,
|
||||
* stopped leases, and any saved-file obligation below. */
|
||||
export function isInterruptedNativePreparationCandidate(run: Run): boolean {
|
||||
const cancellation = readRunCancellation(run.resultJson);
|
||||
// Older shutdown records predate structured cancellation. Current records
|
||||
// identify unexpected system interruption without relying on an error label.
|
||||
const systemInterruption = cancellation
|
||||
? cancellation.initiator.type === "system" && cancellation.source !== "operator" &&
|
||||
(cancellation.source === "shutdown" || cancellation.expected === false)
|
||||
: run.errorCode === "server_shutdown_interrupted";
|
||||
const recovery = run.resultJson?.executionRecovery as Record<string, unknown> | undefined;
|
||||
return Boolean(systemInterruption && run.status === "interrupted" && run.finishedAt
|
||||
&& run.runtimeMode === "legacy" && !run.runtimeModeResolvedAt && run.executionStage === "preparing"
|
||||
&& claimedAdapterType(run) === "paperclip_runner" && !run.processPid && !run.processGroupId
|
||||
&& !run.processStartedAt && !run.sessionIdAfter && !run.nativeIssueId && !run.nativeSessionId
|
||||
&& run.runnerProfileJson?.nativeExecutionInput == null && recovery?.providerWorkStarted !== true);
|
||||
}
|
||||
|
||||
export async function isInterruptedNativePreparation(db: Db, run: Run): Promise<boolean> {
|
||||
const recovery = run.resultJson?.executionRecovery as Record<string, unknown> | undefined;
|
||||
const settled = recovery?.kind === "bootstrap" && recovery.providerWorkStarted === false;
|
||||
if (run.status !== "interrupted" || run.errorCode !== "server_shutdown_interrupted" || !run.finishedAt
|
||||
|| run.runtimeMode !== "legacy" || run.runtimeModeResolvedAt || run.executionStage !== "preparing"
|
||||
|| claimedAdapterType(run) !== "paperclip_runner" || run.processPid || run.processGroupId
|
||||
|| run.processStartedAt || run.sessionIdAfter || run.nativeIssueId || run.nativeSessionId
|
||||
|| run.runnerProfileJson?.nativeExecutionInput != null || recovery?.providerWorkStarted === true
|
||||
if (!isInterruptedNativePreparationCandidate(run)
|
||||
|| run.controllerLeaseExpiresAt && run.controllerLeaseExpiresAt > new Date()
|
||||
|| !settled && (!run.controllerBootId || run.controllerBootId === legacyControllerBootId)) return false;
|
||||
const [coordinator] = await db.select({ id: nativeRunFinalizations.runId }).from(nativeRunFinalizations).where(and(
|
||||
|
||||
@@ -7,7 +7,8 @@ import { notifyExecutionRunWork } from "./execution-control-notifications.js";
|
||||
import { hasWorkspaceRestoreFailure } from "@paperclipai/shared";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { canonicalExecutionUuid } from "./execution-identity.js";
|
||||
import { claimedAdapterType, conversationRecoveryActionPredicate, getConversationOwnershipBlocker } from "./conversation-continuation.js";
|
||||
import { conversationRecoveryActionPredicate, getConversationOwnershipBlocker } from "./conversation-continuation.js";
|
||||
import { isInterruptedNativePreparationCandidate } from "./cancelled-native-startup.js";
|
||||
import { persistActivity } from "./activity-log.js";
|
||||
import { appendHeartbeatRunEvent } from "./heartbeat-run-events.js";
|
||||
import { logger } from "../middleware/logger.js";
|
||||
@@ -612,8 +613,7 @@ export async function settleUnrecoverableExecutions(
|
||||
if (task.status !== "blocked" && run.runtimeMode === "native") {
|
||||
nativeFailureBlock = { runId: run.id, statusVersion: projected!.statusVersion };
|
||||
}
|
||||
if (task.status !== "blocked" && run.runtimeMode === "legacy" && !run.runtimeModeResolvedAt &&
|
||||
run.errorCode === "server_shutdown_interrupted" && claimedAdapterType(run) === "paperclip_runner") {
|
||||
if (task.status !== "blocked" && isInterruptedNativePreparationCandidate(run)) {
|
||||
nativeBootstrapFailureBlock = { runId: run.id, statusVersion: projected!.statusVersion, previousStatus: task.status };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3914,7 +3914,8 @@ export function heartbeatService(
|
||||
});
|
||||
}
|
||||
}
|
||||
if (latestRun?.status === "interrupted" && latestRun.errorCode === "server_shutdown_interrupted") {
|
||||
if (latestRun?.status === "interrupted" && latestRun.runtimeMode === "legacy" &&
|
||||
!latestRun.runtimeModeResolvedAt && latestRun.executionStage === "preparing") {
|
||||
latestRun = await settleInterruptedNativeBootstrap(db, { run: latestRun,
|
||||
providerDispatchStarted: legacyAdapterEntered || nativeDispatchStarted || nativeOwnershipHeld,
|
||||
}) ?? latestRun;
|
||||
|
||||
@@ -429,10 +429,10 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen
|
||||
}
|
||||
const now = opts?.now ?? new Date();
|
||||
const preparationRecovery = parseObject(run.resultJson?.executionRecovery);
|
||||
const interruptedPreparation = run.status === "interrupted" &&
|
||||
preparationRecovery.kind === "bootstrap" && preparationRecovery.providerWorkStarted === false;
|
||||
const retryReason = opts?.retryReason ??
|
||||
(run.status === "interrupted" && run.errorCode === "server_shutdown_interrupted" &&
|
||||
preparationRecovery.kind === "bootstrap" && preparationRecovery.providerWorkStarted === false
|
||||
? "interrupted_preparation" : BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON);
|
||||
(interruptedPreparation ? "interrupted_preparation" : BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON);
|
||||
const wakeReason =
|
||||
opts?.wakeReason ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON;
|
||||
const maxAttempts = Math.max(
|
||||
@@ -570,6 +570,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen
|
||||
: baseSchedule;
|
||||
|
||||
const requiresIssueGate =
|
||||
interruptedPreparation ||
|
||||
(run.status === "interrupted" && run.errorCode === "server_shutdown_interrupted") ||
|
||||
isTransientWorkspacePreparationCode(run.errorCode) ||
|
||||
hasConversationContinuationPolicy(run.resultJson) ||
|
||||
|
||||
@@ -572,7 +572,8 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
|
||||
async function resumeRemoteStopComments(run: NonNullable<Awaited<ReturnType<typeof getRunExecutionWait>>>, requestId?: string) {
|
||||
if (!isHeartbeatRunTerminalStatus(run.status) || adapterExecutionControls.has(run.id)) return;
|
||||
let interruptedPreparation = false;
|
||||
if (run.status === "interrupted" && run.errorCode === "server_shutdown_interrupted") {
|
||||
if (run.status === "interrupted" && run.runtimeMode === "legacy" &&
|
||||
!run.runtimeModeResolvedAt && run.executionStage === "preparing") {
|
||||
const current = await getRun(run.id, { includeExecutionEvidence: true });
|
||||
if (current && await isInterruptedNativePreparation(db, current)) {
|
||||
const settled = await settleInterruptedNativeBootstrap(db, { run: current, providerDispatchStarted: false });
|
||||
|
||||
@@ -20,7 +20,7 @@ import { executionFailureRetryCount } from "./execution-recovery-attempt.js";
|
||||
import { logActivity } from "./activity-log.js";
|
||||
import { isSupersededConversationRun } from "./agent-conversations.js";
|
||||
import { issueService } from "./issues.js";
|
||||
import { isInterruptedNativePreparation } from "./cancelled-native-startup.js";
|
||||
import { isInterruptedNativePreparation, isInterruptedNativePreparationCandidate } from "./cancelled-native-startup.js";
|
||||
import { hasVerifiedWorkspaceRepair } from "./workspace-recovery-receipt.js";
|
||||
|
||||
type Run = typeof heartbeatRuns.$inferSelect;
|
||||
@@ -318,10 +318,8 @@ export async function settleInterruptedNativeBootstrap(
|
||||
return db.transaction(async tx => {
|
||||
const [task] = await tx.select().from(issues).where(and(eq(issues.id, issueId), eq(issues.companyId, input.run.companyId))).for("update");
|
||||
const [run] = await tx.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, input.run.id), eq(heartbeatRuns.companyId, input.run.companyId))).for("update");
|
||||
if (!task || !run || run.runtimeMode !== "legacy" || run.runtimeModeResolvedAt ||
|
||||
if (!task || !run || !isInterruptedNativePreparationCandidate(run) ||
|
||||
run.contextSnapshot?.issueId !== task.id ||
|
||||
run.status !== "interrupted" || run.errorCode !== "server_shutdown_interrupted" ||
|
||||
claimedAdapterType(run) !== "paperclip_runner" || run.processPid || run.processGroupId ||
|
||||
executionFailureRetryCount(run) >= 2) return null;
|
||||
if (hasWorkspaceRestoreFailure(run.resultJson) && !await hasVerifiedWorkspaceRepair(tx as unknown as Db, run)) return null;
|
||||
const [native] = await tx.select({ id: nativeRunFinalizations.runId }).from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id)).limit(1);
|
||||
|
||||
@@ -393,16 +393,20 @@ describe("automatic retained workspace recovery", () => {
|
||||
return { ...f, source, workspaceId, resultId, git, commit };
|
||||
}
|
||||
|
||||
async function interruptedRemotePreparation() {
|
||||
async function interruptedRemotePreparation(interruption: "shutdown" | "lease" = "shutdown") {
|
||||
const f = await acceptedRemoteExport();
|
||||
await db.delete(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId));
|
||||
await db.delete(nativeRunResults).where(eq(nativeRunResults.runId, f.runId));
|
||||
const recovery = { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [f.lease.id] };
|
||||
await db.update(heartbeatRuns).set({ runtimeMode: "legacy", nativeIssueId: null, completionContractId: null,
|
||||
status: "interrupted", executionStage: "preparing", errorCode: "server_shutdown_interrupted",
|
||||
status: "interrupted", executionStage: "preparing",
|
||||
errorCode: interruption === "lease" ? "lease_released_before_terminal" : "server_shutdown_interrupted",
|
||||
controllerBootId: randomUUID(),
|
||||
error: "Interrupted by graceful server shutdown (SIGTERM)",
|
||||
resultJson: { workspaceRestoreFailure: "restore_failed", workspaceRestoreRecovery: recovery },
|
||||
error: interruption === "lease" ? "Environment lease released before preparation settled" : "Interrupted by graceful server shutdown (SIGTERM)",
|
||||
resultJson: { workspaceRestoreFailure: "restore_failed", workspaceRestoreRecovery: recovery,
|
||||
...(interruption === "lease" ? { cancellation: { source: "unknown", expected: false,
|
||||
initiator: { type: "system" }, reason: "Environment lease released before preparation settled",
|
||||
recordedAt: new Date().toISOString() } } : {}) },
|
||||
}).where(eq(heartbeatRuns.id, f.runId));
|
||||
await db.update(issues).set({ statusVersion: 2 }).where(eq(issues.id, f.issueId));
|
||||
await db.update(issueRecoveryActions).set({ cause: "legacy_execution_requires_reconciliation",
|
||||
@@ -416,8 +420,8 @@ describe("automatic retained workspace recovery", () => {
|
||||
return f;
|
||||
}
|
||||
|
||||
it("admits HTTP Retry once after an interrupted Runner preparation recovers its remote workspace", async () => {
|
||||
const f = await interruptedRemotePreparation();
|
||||
it.each(["shutdown", "lease"] as const)("admits HTTP Retry once after an interrupted Runner preparation recovers its remote workspace (%s)", async interruption => {
|
||||
const f = await interruptedRemotePreparation(interruption);
|
||||
await recoverRetainedWorkspace(f.input);
|
||||
expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(true);
|
||||
const intent = { observedRunId: f.runId, idempotencyKey: randomUUID() };
|
||||
@@ -441,8 +445,9 @@ describe("automatic retained workspace recovery", () => {
|
||||
finally { await saved.cleanup(); }
|
||||
}, 30_000);
|
||||
|
||||
it.each([false, true])("automatically retries interrupted preparation after preservation, including recovered history at startup=%s", async alreadyRecovered => {
|
||||
const f = await interruptedRemotePreparation();
|
||||
it.each([false, true].flatMap(alreadyRecovered => (["shutdown", "lease"] as const).map(interruption => ({ alreadyRecovered, interruption }))))(
|
||||
"automatically retries interrupted preparation after preservation ($interruption, startup=$alreadyRecovered)", async ({ alreadyRecovered, interruption }) => {
|
||||
const f = await interruptedRemotePreparation(interruption);
|
||||
if (alreadyRecovered) await recoverRetainedWorkspace(f.input);
|
||||
const restarted = createDb(connectionString);
|
||||
try {
|
||||
@@ -462,6 +467,60 @@ describe("automatic retained workspace recovery", () => {
|
||||
} finally { await restarted.$client.end(); }
|
||||
}, 30_000);
|
||||
|
||||
it("retries unpublished preparation from its preserved ancestor without an old failure-block receipt", async () => {
|
||||
const f = await seed("legacy", false);
|
||||
await requestWorkspaceRecovery({ ...f, idempotencyKey: randomUUID(), actorId: "board" });
|
||||
await recoverRetainedWorkspace(f.input);
|
||||
const ancestor = await run(f.runId), leaseId = randomUUID(), providerLeaseId = randomUUID();
|
||||
await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: ancestor.agentId, status: "running" });
|
||||
const app = boardRecoveryApp(), heartbeat = heartbeatService(db);
|
||||
const first = (await request(app).post(`/api/issues/${f.issueId}/retry`).send({
|
||||
observedRunId: f.runId, idempotencyKey: randomUUID(),
|
||||
}).expect(202)).body;
|
||||
await heartbeat.resumeTaskRetry(f.companyId, first.requestId);
|
||||
const admitted = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId),
|
||||
eq(heartbeatRuns.status, "queued")));
|
||||
expect(admitted).toHaveLength(1);
|
||||
const interruptedId = admitted[0].id;
|
||||
await db.update(heartbeatRuns).set({ runtimeMode: "legacy", status: "interrupted", executionStage: "preparing",
|
||||
startedAt: new Date(Date.now() - 60_000), finishedAt: new Date(), controllerBootId: randomUUID(),
|
||||
errorCode: "lease_released_before_terminal", error: "Lease released before preparation settled",
|
||||
runnerProfileJson: { adapterDispatch: { adapterType: "paperclip_runner" } },
|
||||
resultJson: { cancellation: { source: "unknown", expected: false, initiator: { type: "system" },
|
||||
reason: "Lease released before preparation settled", recordedAt: new Date().toISOString() } },
|
||||
}).where(eq(heartbeatRuns.id, interruptedId));
|
||||
await db.update(issues).set({ executionRunId: null, checkoutRunId: null }).where(eq(issues.id, f.issueId));
|
||||
const lease = { id: leaseId, companyId: f.companyId, heartbeatRunId: interruptedId,
|
||||
provider: "daytona", providerLeaseId };
|
||||
await db.insert(environmentLeases).values({ ...lease, issueId: f.issueId, environmentId: f.lease.environmentId,
|
||||
status: "released", cleanupStatus: "success", releasedAt: new Date(),
|
||||
metadata: { remoteExecutionTermination: remoteTerminationReceipt(lease, { providerLeaseId, state: "stopped" }) },
|
||||
});
|
||||
await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: f.issueId,
|
||||
kind: "active_run_watchdog", ownerType: "board", returnOwnerAgentId: ancestor.agentId,
|
||||
cause: "legacy_execution_requires_reconciliation", fingerprint: `legacy-execution:${interruptedId}`,
|
||||
status: "resolved", outcome: "blocked", nextAction: "Recorded actions require reconciliation",
|
||||
evidence: { runId: interruptedId, originalFailureCode: "lease_released_before_terminal",
|
||||
automaticRecovery: { policy: "preserve_without_replay_v1", runId: interruptedId, replay: "blocked", actionOutcome: "unknown" } },
|
||||
});
|
||||
const intent = { observedRunId: interruptedId, idempotencyKey: randomUUID() };
|
||||
const receipt = (await request(app).post(`/api/issues/${f.issueId}/retry`).send(intent).expect(202)).body;
|
||||
await Promise.all([heartbeat.resumeTaskRetry(f.companyId, receipt.requestId),
|
||||
heartbeat.resumeTaskRetry(f.companyId, receipt.requestId)]);
|
||||
const successors = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId),
|
||||
eq(heartbeatRuns.status, "queued")));
|
||||
expect(successors).toHaveLength(1);
|
||||
expect(successors[0].contextSnapshot).toMatchObject({ previousRunId: interruptedId });
|
||||
expect((await request(app).post(`/api/issues/${f.issueId}/retry`).send(intent).expect(202)).body.requestId).toBe(receipt.requestId);
|
||||
const restored = path.join(root, randomUUID()); await mkdir(restored);
|
||||
const copies = await attachPreservedWorkspaceSnapshots({ db, companyId: f.companyId, issueId: f.issueId,
|
||||
sourceRunIds: [interruptedId], cwd: restored });
|
||||
expect(copies).toHaveLength(1);
|
||||
expect(await readFile(path.join(copies[0]!.path, "files", "saved.txt"), "utf8")).toBe("finished work\n");
|
||||
expect(await readFile(path.join(f.host, "newer.txt"), "utf8")).toBe("newer local work\n");
|
||||
expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toBeNull();
|
||||
}, 30_000);
|
||||
|
||||
it.each(["local", "none"])("continues verified interrupted preparation with %s leases", async provider => {
|
||||
const f = await interruptedRemotePreparation();
|
||||
await db.update(heartbeatRuns).set({ runnerProfileJson: { adapterDispatch: { adapterType: "paperclip_runner" } },
|
||||
@@ -476,9 +535,10 @@ describe("automatic retained workspace recovery", () => {
|
||||
}, 30_000);
|
||||
|
||||
it.each(["provider_started", "provider_dispatch_recorded", "runtime_selected", "live_controller", "wrong_snapshot", "operator_stop",
|
||||
"operator_stop_metadata", "missing_stop_receipt", "unrecorded_interruption",
|
||||
"retry_exhausted", "human_reblock", "paused", "reassigned"] as const)(
|
||||
"keeps the interrupted preparation continuation behind %s", async gate => {
|
||||
const f = await interruptedRemotePreparation();
|
||||
const f = await interruptedRemotePreparation("lease");
|
||||
await recoverRetainedWorkspace(f.input);
|
||||
const source = await run(f.runId);
|
||||
if (gate === "provider_started") await db.insert(heartbeatRunEvents).values({ companyId: f.companyId,
|
||||
@@ -492,6 +552,17 @@ describe("automatic retained workspace recovery", () => {
|
||||
remoteWorkspace: { ...(source.runnerProfileJson!.remoteWorkspace as Record<string, unknown>), snapshotId: randomUUID() },
|
||||
} }).where(eq(heartbeatRuns.id, f.runId));
|
||||
if (gate === "operator_stop") await db.update(heartbeatRuns).set({ status: "cancelled", errorCode: "operator_interrupted" }).where(eq(heartbeatRuns.id, f.runId));
|
||||
if (gate === "operator_stop_metadata") await db.update(heartbeatRuns).set({ resultJson: { ...source.resultJson,
|
||||
cancellation: requestedRunCancellation({ cancelledByActorType: "user", cancelledByUserId: "operator" }, "Stop requested"),
|
||||
} }).where(eq(heartbeatRuns.id, f.runId));
|
||||
if (gate === "unrecorded_interruption") await db.update(heartbeatRuns).set({ resultJson: { ...source.resultJson,
|
||||
cancellation: null,
|
||||
} }).where(eq(heartbeatRuns.id, f.runId));
|
||||
if (gate === "missing_stop_receipt") {
|
||||
const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id));
|
||||
await db.update(environmentLeases).set({ metadata: { ...lease.metadata, remoteExecutionTermination: null } })
|
||||
.where(eq(environmentLeases.id, f.lease.id));
|
||||
}
|
||||
if (gate === "retry_exhausted") await db.update(heartbeatRuns).set({ scheduledRetryAttempt: 2 }).where(eq(heartbeatRuns.id, f.runId));
|
||||
if (gate === "human_reblock") await db.update(issues).set({ statusVersion: 3 }).where(eq(issues.id, f.issueId));
|
||||
if (gate === "paused") await db.update(agents).set({ status: "paused" }).where(eq(agents.id, source.agentId));
|
||||
@@ -500,7 +571,8 @@ describe("automatic retained workspace recovery", () => {
|
||||
await heartbeat.resumeRemoteStopComments(await run(f.runId));
|
||||
expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId),
|
||||
eq(heartbeatRuns.retryOfRunId, f.runId)))).toHaveLength(0);
|
||||
if (["provider_started", "provider_dispatch_recorded", "runtime_selected", "live_controller", "wrong_snapshot", "operator_stop"].includes(gate)) {
|
||||
if (["provider_started", "provider_dispatch_recorded", "runtime_selected", "live_controller", "wrong_snapshot", "operator_stop",
|
||||
"operator_stop_metadata", "missing_stop_receipt", "unrecorded_interruption"].includes(gate)) {
|
||||
const receipt = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issueId,
|
||||
actorId: "local-board", actorSource: "local_implicit", observedRunId: f.runId, idempotencyKey: randomUUID() });
|
||||
await heartbeat.resumeTaskRetry(f.companyId, receipt.requestId!);
|
||||
|
||||
@@ -447,7 +447,7 @@ export function retainedWorkspaceCandidates(db: Db, cursor?: string, unfinishedO
|
||||
and ${agentWakeupRequests.status} = 'deferred_issue_execution' and ${agentWakeupRequests.requestedByActorType} = 'user'
|
||||
and ${agentWakeupRequests.payload}->>'issueId' = ${issueRecoveryActions.sourceIssueId}::text)`;
|
||||
const interruptedPreparation = sql`(${heartbeatRuns.runtimeMode} = 'legacy'
|
||||
and ${heartbeatRuns.status} = 'interrupted' and ${heartbeatRuns.errorCode} = 'server_shutdown_interrupted'
|
||||
and ${heartbeatRuns.status} = 'interrupted'
|
||||
and ${heartbeatRuns.runtimeModeResolvedAt} is null and ${heartbeatRuns.executionStage} = 'preparing'
|
||||
and ${heartbeatRuns.runnerProfileJson}#>>'{adapterDispatch,adapterType}' = 'paperclip_runner'
|
||||
and ${issueRecoveryActions.evidence}#>>'{automaticRecovery,replay}' = 'blocked'
|
||||
|
||||
Reference in new issue
Block a user