diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index a72120c36e6..5889dc4634a 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1724,6 +1724,20 @@ sources without this proof remain ineligible. The preservation owner still requires an independently stopped, exclusively bound allocation and no host writer; it saves files and resumes the existing finalizer without another provider turn. +Both the legacy native copyback descriptor and the remote-workspace descriptor +keep an unfinished export under the execution hold. Task Retry saves its intent +behind that hold; an accepted successful result resumes its existing finalizer +after verified preservation instead of admitting another task run. Pending human +review remains effective when the result commits. + +If a later failure supersedes the original recovery action, an explicit board +`recover-workspace` request can still preserve that exact historical export. +The cancelled action remains historical and the current action retains attention +ownership. Preservation checks current writers and source identity, records a +durable snapshot, and leaves the task's newer workspace binding unchanged. It +does not apply an older accepted result over a newer run or authorize continuation. +Cancelled history without a saved explicit recovery request remains ineligible. + Startup and the retained-workspace sweep reconstruct these obligations through the retained workspace index. Committed Retry, recovery admission, and recovery progress writes wake a separate preservation worker. It follows the persisted diff --git a/maps/agent-access-map/recovery.md b/maps/agent-access-map/recovery.md index d551c6b1824..18894d62573 100644 --- a/maps/agent-access-map/recovery.md +++ b/maps/agent-access-map/recovery.md @@ -326,6 +326,12 @@ API: `present` — POST /api/issues/{id}/recovery-actions/recover-workspace with Agent access: `restricted` — board task-read and issue:mutate; not runtime:manage. Agents denied through REST/native fallback. +An explicit request may also preserve a cancelled export-repair action that a +newer failure superseded. It retains the exact original source checks, current +writer exclusion and bounded maintenance budget. It does not change the newer +task workspace or recovery owner, apply an older result over newer execution, +or admit a provider turn. A cancelled record without that request is ineligible. + Policy: Exact terminal retained source/lease/reference; independent stop and physical ownership checks; no newer writer. Bounded queued repair preserves newer files and has retry state. Matching key/recovered state dedupes; permanent blocked source refuses. Recovery starts no provider turn. For native export repair, a reusable lease is eligible only with its exact descriptor and the finalizer-selected accepted successful result under the same company/task/run/completion contract. This does not change actor access or permit another allocation owner or host writer. Invalid Git metadata is a non-retryable persistence failure requiring service repair; it cannot use unsafe-archive omission to complete the result. Run-detail/activity projections retain the allowlisted code, including bounded large-result retrieval, and cancelled-preservation discovery honors the same hold. Evidence: `source-only` — [recovery routes](../../server/src/routes/issues.ts), [retained recovery](../../server/src/services/workspace-recovery.ts); reviewed at `3bbc2104b65caf9f7763938a01260a5673e12f7a`. No live provider proof. diff --git a/maps/feature-map/recovery.md b/maps/feature-map/recovery.md index b97ac0d9a08..1ba9594a4d7 100644 --- a/maps/feature-map/recovery.md +++ b/maps/feature-map/recovery.md @@ -429,6 +429,17 @@ Verify saved and newer local files, one preservation receipt, and no additional agent turn. The PostgreSQL regression covers this boundary; the opt-in provider probe also uses this runtime path with the protection marker. +Repeat Task Retry with a completed native result whose unfinished export uses +the remote-workspace descriptor. Before preservation, require the original run +to remain the only task run. After preservation, verify restored file bytes and +Git HEAD, commitment of that same result, and an unanswered human review keeping +the task In Review. For an action superseded by a newer failure, request exact +historical file recovery through the board API: verify a committed snapshot, +unchanged newer workspace binding and attention owner, no old-result commitment, +and no provider turn. An active current writer or no explicit request keeps the +historical source waiting. The PostgreSQL/HTTP regression is in +[workspace recovery](../../server/src/services/workspace-recovery.test.ts). + For a missing managed allocation, verify a fresh Cloud-signed absence observation under the exact physical recovery owner. The existing Retry may then select an independent workspace through normal admission. Verify that the original run diff --git a/server/src/services/execution-blocker.ts b/server/src/services/execution-blocker.ts index c22437f1a47..4e57d9b0c88 100644 --- a/server/src/services/execution-blocker.ts +++ b/server/src/services/execution-blocker.ts @@ -16,7 +16,7 @@ import { hasVerifiedWorkspaceRepair, workspaceRecoveryDisplay } from "./workspac const blockerContextKeys = ["issueId", "taskId", "source", "connectionIntentResolution", "interactionKind", "interactionStatus", "mutation", "interactionId"] as const; -const blockerProfileKeys = ["adapterDispatch", "nativeExecutionInput", "nativeWorkspaceSync"] as const; +const blockerProfileKeys = ["adapterDispatch", "nativeExecutionInput", "nativeWorkspaceSync", "remoteWorkspace"] as const; const blockerContextColumn = sql | null>`case when ${heartbeatRuns.contextSnapshot} is null then null else ( select coalesce(jsonb_object_agg(field.key, field.value), '{}'::jsonb) diff --git a/server/src/services/explicit-native-continuation.ts b/server/src/services/explicit-native-continuation.ts index 084a25e457f..c93ec6b8681 100644 --- a/server/src/services/explicit-native-continuation.ts +++ b/server/src/services/explicit-native-continuation.ts @@ -52,7 +52,7 @@ const preflightProfile = sql`case then jsonb_build_object('adapterType', field.value->'adapterType') else field.value end), '{}'::jsonb) from jsonb_each(case when jsonb_typeof(${heartbeatRuns.runnerProfileJson}) = 'object' then ${heartbeatRuns.runnerProfileJson} else '{}'::jsonb end) as field(key, value) - where field.key in ('adapterDispatch', 'nativeWorkspaceSync') + where field.key in ('adapterDispatch', 'nativeWorkspaceSync', 'remoteWorkspace') ) end`; const continuationRunFields = { id: heartbeatRuns.id, companyId: heartbeatRuns.companyId, agentId: heartbeatRuns.agentId, diff --git a/server/src/services/native-runtime/native-workspace-reconstruction.ts b/server/src/services/native-runtime/native-workspace-reconstruction.ts index 2daf462972f..aa20b852e14 100644 --- a/server/src/services/native-runtime/native-workspace-reconstruction.ts +++ b/server/src/services/native-runtime/native-workspace-reconstruction.ts @@ -36,7 +36,8 @@ export function unreconciledNativeExportPredicate() { and ${heartbeatRuns.nativeIssueId} = ${issueRecoveryActions.sourceIssueId} and ${heartbeatRuns.id}::text = ${issueRecoveryActions.evidence}->>'runId' and ${heartbeatRuns.runtimeMode} = 'native' and ${heartbeatRuns.status} = 'failed' - and ${heartbeatRuns.runnerProfileJson}->'nativeWorkspaceSync'->>'state' = 'prepared')`, + and (${heartbeatRuns.runnerProfileJson}->'nativeWorkspaceSync'->>'state' = 'prepared' + or ${heartbeatRuns.runnerProfileJson}->'remoteWorkspace'->>'state' = 'prepared'))`, ); } diff --git a/server/src/services/workspace-recovery-source.ts b/server/src/services/workspace-recovery-source.ts index 3cf14003e49..ad65d438d2b 100644 --- a/server/src/services/workspace-recovery-source.ts +++ b/server/src/services/workspace-recovery-source.ts @@ -25,6 +25,10 @@ export class WorkspaceSourceInspectionLimit extends Error {} export function workspaceRecoverySourceEligible(includeCompleted = false) { const actionEligible = sql`(${issueRecoveryActions.status} in ('active', 'escalated') or + (${issueRecoveryActions.status} = 'cancelled' + and ${inArray(issueRecoveryActions.cause, workspaceExportCauses)} + and jsonb_typeof(${issueRecoveryActions.evidence}#>'{workspaceRecovery,requestIds}') = 'array' + and ${issueRecoveryActions.evidence}#>'{workspaceRecovery,requestIds}' <> '[]'::jsonb) or (${issueRecoveryActions.status} = 'resolved' and ( (${issueRecoveryActions.evidence}#>>'{automaticRecovery,policy}' = 'preserve_without_replay_v1' and ${issueRecoveryActions.evidence}#>>'{automaticRecovery,replay}' = 'blocked' diff --git a/server/src/services/workspace-recovery.test.ts b/server/src/services/workspace-recovery.test.ts index 8648acad550..5c732d4f922 100644 --- a/server/src/services/workspace-recovery.test.ts +++ b/server/src/services/workspace-recovery.test.ts @@ -49,6 +49,10 @@ import { and, eq, inArray, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { agents, approvals, budgetPolicies, costEvents, completionContracts, issueApprovals, issueTreeHolds, issueTreeHoldMembers, companies, createDb, applyPendingMigrations, environmentLeases, environments, heartbeatRuns, issueComments, issueRecoveryActions, issues, nativeRunFinalizations, nativeRunResults, agentWakeupRequests, workspaceOperations } from "@paperclipai/db"; import { resumePreservedResultFinalization } from "./preserved-result-finalization.js"; +import { finalizeNativeRun } from "./native-runtime/native-run-finalizer.js"; +import { CONTROL_PLANE_CONFORMANCE_RESULT, CONTROL_PLANE_CONFORMANCE_TERMINAL } from "../vendor/paperclip-runner/testing.js"; +import { issueThreadInteractions } from "@paperclipai/db"; +import { issueRecoveryActionService } from "./issue-recovery-actions.js"; import type { AdapterSandboxExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; import type { EnvironmentLease } from "@paperclipai/shared"; import { startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; @@ -190,6 +194,149 @@ describe("automatic retained workspace recovery", () => { const action = async (id: string) => (await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, id)))[0]!; const run = async (id: string) => (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)))[0]!; + function boardRecoveryApp() { + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { req.actor = { type: "board", source: "local_implicit", userId: "local-board" }; next(); }); + app.use("/api", issueRoutes(db, {} as Parameters[1])); + app.use(errorHandler); + return app; + } + + async function acceptedRemoteExport() { + const f = await seed("native", false); + const source = await run(f.runId), workspaceId = randomUUID(); + const remoteCwd = await realpath(f.remote); + const git = (args: string[], cwd = remoteCwd) => new Promise((resolve, reject) => + execFile("git", args, { cwd }, (error, stdout) => error ? reject(error) : resolve(stdout.trim()))); + await git(["init"]); + await git(["add", "."]); + await git(["-c", "user.name=Recovery fixture", "-c", "user.email=recovery@example.invalid", "commit", "-m", "Completed work"]); + const commit = await git(["rev-parse", "HEAD"]); + targets.get(f.lease.id)!.remoteCwd = remoteCwd; + f.runtime.resumeRunLease.mockImplementation(async () => ({ providerLeaseId: f.providerLeaseId, metadata: { remoteCwd } })); + const authority = { schema: "paperclip.remote-workspace.v1", epoch: 1, snapshotId: null, + source: { kind: "empty" }, writerRunId: f.runId, dirty: true, + location: { environmentId: f.lease.environmentId!, leaseId: f.lease.id, providerLeaseId: f.providerLeaseId, remoteCwd } }; + await db.insert(executionWorkspaces).values({ id: workspaceId, companyId: f.companyId, sourceIssueId: f.issueId, + name: "Completed remote work", cwd: remoteCwd, providerType: "cloud_sandbox", mode: "isolated_workspace", + strategyType: "project_primary", status: "active", metadata: { remoteWorkspace: authority } }); + await db.update(issues).set({ executionWorkspaceId: workspaceId }).where(eq(issues.id, f.issueId)); + const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)); + await db.update(environmentLeases).set({ executionWorkspaceId: workspaceId, leasePolicy: "reuse_by_environment", + metadata: { ...lease.metadata, remoteCwd, agentId: source.agentId } }).where(eq(environmentLeases.id, lease.id)); + const contractId = randomUUID(), resultId = randomUUID(); + await db.insert(completionContracts).values({ id: contractId, companyId: f.companyId, issueId: f.issueId, + revision: 1, schemaVersion: "paperclip.completion-contract.v1", policyVersion: "test", risk: "low", + completionAuthority: "agent_claim_policy", incompleteCriteriaPolicy: "preserve_non_terminal", + contractJson: { revision: CONTROL_PLANE_CONFORMANCE_RESULT.completionClaim.contractRevision, + objective: "Complete the work", criteria: [{ id: "objective", requirement: "Complete the task" }] }, + canonicalSha256: "contract", createdByActorType: "system", createdByActorId: "test" }); + await db.update(heartbeatRuns).set({ completionContractId: contractId, resultJson: {}, runnerProfileJson: { + adapterDispatch: { adapterType: "paperclip_runner" }, + remoteWorkspace: { schema: "paperclip.remote-workspace-run.v1", workspaceId, leaseId: lease.id, + providerLeaseId: f.providerLeaseId, remoteCwd, epoch: 1, state: "prepared", snapshotId: null }, + } }).where(eq(heartbeatRuns.id, f.runId)); + await db.insert(nativeRunResults).values({ id: resultId, companyId: f.companyId, issueId: f.issueId, runId: f.runId, + completionContractId: contractId, schemaStatus: "accepted", serverFingerprint: "result", canonicalSha256: "result", + resultJson: { terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, result: CONTROL_PLANE_CONFORMANCE_RESULT } }); + await db.update(nativeRunFinalizations).set({ resultId, failureCode: "native_workspace_sync_out_retry_exhausted" }) + .where(eq(nativeRunFinalizations.runId, f.runId)); + await db.update(issueRecoveryActions).set({ cause: "native_workspace_sync_out_retry_exhausted", evidence: { runId: f.runId } }) + .where(eq(issueRecoveryActions.id, f.actionId)); + await adoptSystemRecovery(db, f); + return { ...f, source, workspaceId, resultId, git, commit }; + } + + it.each([false, true])("keeps Retry behind an accepted remote export and resumes only its saved result (review pending: %s)", async reviewPending => { + const f = await acceptedRemoteExport(); + const { source, workspaceId, resultId, git, commit } = f; + const interactionId = randomUUID(); + if (reviewPending) await db.insert(issueThreadInteractions).values({ id: interactionId, companyId: f.companyId, + issueId: f.issueId, sourceRunId: f.runId, createdByAgentId: source.agentId, kind: "request_confirmation", + title: "Review saved work", effectiveResolverPolicy: "human_only", + payload: { version: 1, prompt: "Review before publishing", rejectRequiresReason: true, + target: { type: "custom", key: "layout_review", label: "Review saved work" } } }); + await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: source.agentId, status: "running" }); + const app = boardRecoveryApp(); + const intent = { observedRunId: f.runId, idempotencyKey: randomUUID() }; + const retry = (await request(app).post(`/api/issues/${f.issueId}/retry`).send(intent).expect(202)).body; + expect((await request(app).post(`/api/issues/${f.issueId}/retry`).send(intent).expect(202)).body.requestId).toBe(retry.requestId); + await heartbeatService(db).resumeTaskRetry(f.companyId, retry.requestId!); + expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), + sql`coalesce(${heartbeatRuns.nativeIssueId}::text, ${heartbeatRuns.issueId}::text, + ${heartbeatRuns.contextSnapshot}->>'issueId') = ${f.issueId}`))).toHaveLength(1); + expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toMatchObject({ runId: f.runId, workspaceRepairRequired: true }); + await Promise.all(await reconcileRetainedWorkspaces(db, f.environmentRuntime, async () => { + await heartbeatService(db).resumeTaskRetry(f.companyId, retry.requestId!); + }) ?? []); + expect(readSystemRecovery((await action(f.actionId)).evidence)?.state).toBe("complete"); + const [workspace] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, workspaceId)); + const saved = readRemoteWorkspaceAuthority(workspace.metadata)!; + const restored = await createSavedWorkspaceLoader(db)({ companyId: f.companyId, workspaceId, snapshotId: saved.snapshotId! }); + try { + expect(await readFile(path.join(restored.root, "saved.txt"), "utf8")).toBe("finished work\n"); + expect(await git(["rev-parse", "HEAD"], restored.root)).toBe(commit); + } + finally { await restored.cleanup(); } + const [coordinator] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)); + expect(coordinator).toMatchObject({ phase: "workspace_finalizing", resultId }); + expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, retry.requestId!)))[0]) + .toMatchObject({ status: "coalesced", runId: f.runId, payload: { taskRetry: { strategy: "finalize_result" } } }); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(2); + await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true }); + expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0]) + .toMatchObject({ status: reviewPending ? "in_review" : "done", executionRunId: null, checkoutRunId: null }); + expect((await run(f.runId)).status).toBe("succeeded"); + if (reviewPending) expect((await db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.id, interactionId)))[0]) + .toMatchObject({ status: "pending", result: null, resolvedAt: null }); + }, 30_000); + + it("preserves an explicitly requested superseded remote export without applying its result over newer work", async () => { + const f = await acceptedRemoteExport(); + const [later] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.source.agentId, + issueId: f.issueId, status: "cancelled", runtimeMode: "legacy", finishedAt: new Date(), + contextSnapshot: { issueId: f.issueId }, createdAt: new Date(f.source.createdAt.getTime() + 1_000) }).returning(); + const [currentWorkspace] = await db.insert(executionWorkspaces).values({ companyId: f.companyId, + sourceIssueId: f.issueId, name: "Newer workspace", cwd: path.join(root, later.id), providerType: "cloud_sandbox", + mode: "isolated_workspace", strategyType: "project_primary", status: "active" }).returning(); + await db.update(issues).set({ executionWorkspaceId: currentWorkspace.id }).where(eq(issues.id, f.issueId)); + const laterAction = await issueRecoveryActionService(db).upsertSourceScoped({ companyId: f.companyId, + sourceIssueId: f.issueId, kind: "active_run_watchdog", cause: "legacy_execution_requires_reconciliation", + fingerprint: later.id, evidence: { runId: later.id }, nextAction: "Inspect the stopped work", + ownerType: "board", supersedeOnIdentityChange: true }); + expect((await action(f.actionId)).status).toBe("cancelled"); + await Promise.all(await reconcileRetainedWorkspaces(db, f.environmentRuntime, async () => {}) ?? []); + expect(f.runtime.resumeRunLease).not.toHaveBeenCalled(); + const app = boardRecoveryApp(); + const recoveryRequest = { actionId: f.actionId, runId: f.runId, idempotencyKey: randomUUID() }; + await request(app).post(`/api/issues/${f.issueId}/recovery-actions/recover-workspace`).send(recoveryRequest).expect(202); + const admitted = await action(f.actionId); + await request(app).post(`/api/issues/${f.issueId}/recovery-actions/recover-workspace`).send(recoveryRequest).expect(202); + expect(await action(f.actionId)).toEqual(admitted); + await db.update(heartbeatRuns).set({ status: "running", finishedAt: null }).where(eq(heartbeatRuns.id, later.id)); + await Promise.all(await reconcileRetainedWorkspaces(db, f.environmentRuntime, async () => {}) ?? []); + expect(readSystemRecovery((await action(f.actionId)).evidence)?.failure?.code).toBe("workspace_writer_active"); + expect(f.runtime.resumeRunLease).not.toHaveBeenCalled(); + await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date() }).where(eq(heartbeatRuns.id, later.id)); + await makeRetryDue(f.actionId); + await Promise.all(await reconcileRetainedWorkspaces(db, f.environmentRuntime, async () => {}) ?? []); + expect(readSystemRecovery((await action(f.actionId)).evidence)?.state).toBe("complete"); + const [workspace] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, f.workspaceId)); + const saved = readRemoteWorkspaceAuthority(workspace.metadata)!; + const restored = await createSavedWorkspaceLoader(db)({ companyId: f.companyId, workspaceId: f.workspaceId, snapshotId: saved.snapshotId! }); + try { + expect(await f.git(["rev-parse", "HEAD"], restored.root)).toBe(f.commit); + expect(await readFile(path.join(restored.root, "saved.txt"), "utf8")).toBe("finished work\n"); + } finally { await restored.cleanup(); } + expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0]) + .toMatchObject({ status: "blocked", executionWorkspaceId: currentWorkspace.id }); + expect(await action(laterAction.id)).toEqual(expect.objectContaining({ status: "active", evidence: { runId: later.id } })); + expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)))[0]) + .toMatchObject({ phase: "terminal_failure", resultId: f.resultId }); + expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId))).toEqual([]); + }, 30_000); + it.each([ { status: "failed", runtimeMode: "native", uploadFailure: "none" }, { status: "cancelled", runtimeMode: "native", uploadFailure: "none" },