diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 03f0044f38d..2c6fc2218d0 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1336,6 +1336,13 @@ does not reset the provider-attempt budget. Provider execution and control-plane finalization have different clocks. A healthy provider can think or execute a long tool without output. Once execution settles, recovery and finalization control steps have a 60-second deadline, checked on startup and every 15 seconds. With a healthy database and scheduler, an abandoned transition must be repaired or surfaced within 90 seconds. Terminal persistence must not wait on provider cleanup or publication; a late finalizer cannot change a reassigned or closed task or release another run's locks. Historical ambiguous runs are never automatically replayed after an upgrade. +When that deadline fails a native run with a prepared workspace, the same +transaction records the exact source's file-preservation obligation. Existing +stop-and-retain recovery contains the provider and saves the files before +ordinary Retry can continue. Finalized workspaces, mismatched sources and +accepted results keep their existing settlement paths. Reassignment does not +discard the stopped run's files or grant authority over the new assignee. + Every continuation carries the triggering request, ordered user direction, interaction outcomes, completed work, and explicit history coverage. A delivered message remains part of the task's request after its connection or approval resolves. The original title is background; a completed Notion read does not satisfy a later Gmail request. Author and source-trust boundaries survive rendering into both native and legacy prompts. Missing required history must be fetched before dispatch rather than described as complete. ### Interrupted conversation continuation diff --git a/maps/agent-access-map/recovery.md b/maps/agent-access-map/recovery.md index b2581d1d43b..7f212f4a3d2 100644 --- a/maps/agent-access-map/recovery.md +++ b/maps/agent-access-map/recovery.md @@ -4,6 +4,10 @@ Product map: [Recovery](../feature-map/recovery.md). Feature ID: `recovery`. +Native finalization deadline reconciliation records unfinished workspace +preservation through the existing server-owned recovery path. It grants no +additional agent API, provider-stop authority or permission to replay work. + Ordinary board Retry can reauthorize saved GitHub input for the same App after reconnect. It pins current runtime authority and preserves sender, repository, conversation and task checks. Pending input takes precedence over the older diff --git a/maps/feature-map/recovery.md b/maps/feature-map/recovery.md index c156b4d7cd7..8514cd446be 100644 --- a/maps/feature-map/recovery.md +++ b/maps/feature-map/recovery.md @@ -33,6 +33,8 @@ Agent access: [companion](../agent-access-map/recovery.md). workspace link when the destination is disabled. Local PostgreSQL and real filesystem regressions cover native and legacy recovery and one-time delivery; hosted provider recovery remains separate acceptance. + Native finalization deadline failures also retain the exact prepared source + for the same stop-and-save workflow before Retry admits another executor. - `unknown-outcome`: `recovery/unknown-outcome` — uncertain external side effects require inspection and an explicit decision, not automatic replay. - `user-continuation`: `recovery/user-continuation` — saved user messages after a stop are reconsidered once diff --git a/server/src/services/execution-control-reconciliation.test.ts b/server/src/services/execution-control-reconciliation.test.ts index 9d46777715c..1b873eb4e1e 100644 --- a/server/src/services/execution-control-reconciliation.test.ts +++ b/server/src/services/execution-control-reconciliation.test.ts @@ -2,7 +2,7 @@ import { enableRunFailureReporting } from "../__tests__/helpers/run-failure-repo import { randomUUID } from "node:crypto"; import { eq } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; -import { agents, companies, completionContracts, createDb, heartbeatRuns, issues, issueRecoveryActions, issueThreadInteractions, nativeRunResults, statusDecisions, workAssessments } from "@paperclipai/db"; +import { agents, companies, completionContracts, createDb, environmentLeases, environments, heartbeatRuns, issues, issueRecoveryActions, issueThreadInteractions, nativeRunFinalizations, nativeRunResults, statusDecisions, workAssessments } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, @@ -153,6 +153,65 @@ describeEmbeddedPostgres("reconcileAbandonedExecutionControl reports a genuine f expect(result.surfaced).toBe(1); expect(mockCaptureRunFailure.mock.calls.slice(captureCallsBefore)).toHaveLength(0); }); + + it.each(["prepared", "finalized", "mismatched", "reassigned", "accepted"] as const)("preserves only an exact unfinished native workspace after its deadline (%s)", async state => { + const f = await seedAbandonedRunFixture(); + const [environment] = await db.insert(environments).values({ name: `Deadline ${f.runId}`, driver: "sandbox" }).returning(); + const [lease] = await db.insert(environmentLeases).values({ companyId: f.companyId, issueId: f.issueId, + heartbeatRunId: f.runId, environmentId: environment.id, provider: "daytona", providerLeaseId: randomUUID(), + status: "active", leasePolicy: "reuse_by_environment", metadata: { remoteCwd: "/work" }, + }).returning(); + await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: f.issueId, runnerProfileJson: { + nativeWorkspaceSync: { schema: "paperclip.native-workspace-sync/v3", state: state === "finalized" ? state : "prepared", + descriptorSha256: "a".repeat(64), baselineSha256: "b".repeat(64), finalHostSha256: null, + workspaceId: f.runId, leaseId: lease.id, providerLeaseId: state === "mismatched" ? randomUUID() : lease.providerLeaseId, + remoteCwd: "/work", resourceDisposition: "keep_running" }, + } }).where(eq(heartbeatRuns.id, f.runId)); + await db.insert(nativeRunFinalizations).values({ runId: f.runId, companyId: f.companyId, issueId: f.issueId, + phase: "observed", leaseOwner: "abandoned-controller", controlDeadlineAt: new Date(Date.now() - 60_000) }); + if (state === "reassigned") { + const nextAgentId = randomUUID(); + await db.insert(agents).values({ id: nextAgentId, companyId: f.companyId, name: "New owner", adapterType: "codex_local" }); + await db.update(issues).set({ assigneeAgentId: nextAgentId, executionRunId: null, checkoutRunId: null }) + .where(eq(issues.id, f.issueId)); + } + if (state === "accepted") { + const contractId = randomUUID(), resultId = randomUUID(); + await db.insert(completionContracts).values({ id: contractId, companyId: f.companyId, issueId: f.issueId, + revision: 1, schemaVersion: "1", policyVersion: "test", risk: "low", completionAuthority: "agent_claim_policy", + incompleteCriteriaPolicy: "block", contractJson: {}, canonicalSha256: randomUUID(), createdByActorType: "agent", createdByActorId: f.agentId }); + await db.update(heartbeatRuns).set({ completionContractId: contractId }).where(eq(heartbeatRuns.id, f.runId)); + await db.insert(nativeRunResults).values({ id: resultId, companyId: f.companyId, issueId: f.issueId, runId: f.runId, + completionContractId: contractId, serverFingerprint: randomUUID(), schemaStatus: "valid", resultJson: {}, canonicalSha256: randomUUID() }); + await db.update(nativeRunFinalizations).set({ resultId }).where(eq(nativeRunFinalizations.runId, f.runId)); + } + + await reconcileAbandonedExecutionControl(db); + await reconcileAbandonedExecutionControl(db); + + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + const [retained] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, lease.id)); + expect(run).toMatchObject(state === "accepted" ? { status: "running", errorCode: null } + : { status: "failed", errorCode: "execution_finalization_deadline_exceeded" }); + if (state === "prepared" || state === "reassigned") { + expect(run.resultJson).toMatchObject({ workspaceRestoreRecovery: { leaseIds: [lease.id] } }); + expect(retained).toMatchObject({ status: "pending_cleanup", leasePolicy: "retain_on_failure", + metadata: { sandboxStopAndRetain: { runId: f.runId, providerLeaseId: lease.providerLeaseId } } }); + const recovery = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId)); + expect(recovery.filter(action => action.evidence?.workspaceRestoreRecovery)).toMatchObject([ + { evidence: { runId: f.runId, workspaceRestoreRecovery: { leaseIds: [lease.id] } } }, + ]); + } else { + expect(run.resultJson ?? {}).not.toHaveProperty("workspaceRestoreRecovery"); + expect(retained).toMatchObject({ status: "active", leasePolicy: "reuse_by_environment" }); + } + const [task] = await db.select().from(issues).where(eq(issues.id, f.issueId)); + if (state === "reassigned") { + expect(task.assigneeAgentId).not.toBe(f.agentId); + expect(task).toMatchObject({ status: "in_progress", executionRunId: null, checkoutRunId: null }); + } + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(1); + }); }); describeEmbeddedPostgres("native review execution reconciliation admission", () => { diff --git a/server/src/services/execution-control-reconciliation.ts b/server/src/services/execution-control-reconciliation.ts index 5b3a7aa3735..dd82e02b56a 100644 --- a/server/src/services/execution-control-reconciliation.ts +++ b/server/src/services/execution-control-reconciliation.ts @@ -14,6 +14,7 @@ import { issueRecoveryActionService } from "./issue-recovery-actions.js"; import { hasLiveLegacyController } from "./legacy-controller-lease.js"; import { reportRunFailure } from "./run-failure-report.js"; import { getNativeReviewAssignment, readNativeReviewAssignmentContext } from "./native-runtime/native-review-participant.js"; +import { preserveNativeFailedWorkspace } from "./native-runtime/native-failed-workspace-recovery.js"; /** Only newly recorded control deadlines are eligible. Upgrades never replay ambiguous historical runs. */ export async function reconcileAbandonedExecutionControl( @@ -233,8 +234,10 @@ export async function reconcileAbandonedExecutionControl( ["done", "cancelled"].includes(task.status) || (task.executionRunId && task.executionRunId !== run.id) || (task.checkoutRunId && task.checkoutRunId !== run.id) - ) + ) { + if (updatedRun) await preserveNativeFailedWorkspace(tx as unknown as Db, updatedRun); return true; + } await tx .update(issues) .set({ executionRunId: null, checkoutRunId: null, updatedAt: now }) @@ -261,6 +264,7 @@ export async function reconcileAbandonedExecutionControl( maxAttempts: 3, supersedeOnIdentityChange: true, }); + if (updatedRun) await preserveNativeFailedWorkspace(tx as unknown as Db, updatedRun); return true; }); if (repaired) surfaced += 1;