diff --git a/server/src/__tests__/heartbeat-isolated-continuation-hold.test.ts b/server/src/__tests__/heartbeat-isolated-continuation-hold.test.ts new file mode 100644 index 00000000000..c4e14a9ad7c --- /dev/null +++ b/server/src/__tests__/heartbeat-isolated-continuation-hold.test.ts @@ -0,0 +1,250 @@ +import { randomUUID } from "node:crypto"; +import { and, eq, inArray, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { + agentWakeupRequests, agents, authUsers, companies, companyMemberships, createDb, executionWorkspaces, heartbeatRuns, issueRecoveryActions, issues, + nativeRunFinalizations, +} from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { getExecutionBlocker } from "../services/execution-blocker.ts"; +import { settleHoldIntoAbandonedEpochContinuation } from "../services/fresh-continuation-fallback.ts"; +import { HUMAN_GATE_EVIDENCE_KEYS, hasHumanGateEvidence } from "../services/recovery/human-gate-evidence.ts"; +import { recoverRetainedWorkspace } from "../services/workspace-recovery.ts"; +import { abandonUnverifiedRemoteWorkspaceEpoch } from "../services/remote-workspace-epoch-abandonment.ts"; +import { heartbeatService } from "../services/heartbeat.ts"; +import { runningProcesses } from "../adapters/index.ts"; +import { createSystemRecovery, failSystemRecovery, reserveSystemRecoveryAttempt, SystemRecoveryFailure } from "../services/system-recovery-state.ts"; + +const held = vi.hoisted(() => { + const state = { + release: () => undefined as void, + released: Promise.resolve(), + reset() { + state.released = new Promise((resolve) => { state.release = resolve; }); + }, + }; + return state; +}); + +vi.mock("../adapters/index.ts", async () => { + const actual = await vi.importActual("../adapters/index.ts"); + return { + ...actual, + getServerAdapter: vi.fn(() => ({ + supportsLocalAgentJwt: false, + execute: async () => { + await held.released; + return { exitCode: 0, signal: null, timedOut: false, errorMessage: null, summary: "done", provider: "test", model: "test" }; + }, + })), + }; +}); + +const support = await getEmbeddedPostgresTestSupport(); +const suite = support.supported ? describe : describe.skip; +const minutesAgo = (minutes: number) => new Date(Date.now() - minutes * 60_000); +const RESTORE_SCHEMA = "paperclip.workspace-restore-recovery.v1"; + +suite("a hold on a run that a fresh continuation already isolated", () => { + let db!: ReturnType; + let temporary: Awaited> | null = null; + + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("paperclip-isolated-continuation-hold-"); + db = createDb(temporary.connectionString); + }, 60_000); + + afterEach(async () => { + held.release(); + runningProcesses.clear(); + await db.execute(sql`truncate table ${companies} cascade`); + }); + + afterAll(async () => { await db?.$client.end(); await temporary?.cleanup(); }); + + type Shape = { + isolation?: "remote_epoch" | "fresh_isolated_workspace" | "writer_ended" | null; + personOwned?: boolean; + unsafeArchive?: boolean; + delivery?: "pending" | "delivered" | "delegated" | "invalidated" | "not_requested" | "coalesced"; + holdEvidence?: Record; + epoch?: "abandoned" | "still_owned" | "none"; + }; + + async function killedRunAfterFreshContinuation(shape: Shape = {}) { + const { isolation = "remote_epoch", personOwned = false, unsafeArchive = false, delivery = "delivered", holdEvidence = {}, epoch = "abandoned" } = shape; + const userId = randomUUID(), companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(); + const sourceRunId = randomUUID(), continuationId = randomUUID(), holdId = randomUUID(); + await db.insert(authUsers).values({ id: userId, name: "Board", email: `${userId}@example.test`, + emailVerified: true, createdAt: new Date(), updatedAt: new Date() }); + await db.insert(companies).values({ id: companyId, name: "Kill wave", issuePrefix: `K${companyId.replaceAll("-", "").slice(0, 6)}`, + requireBoardApprovalForNewAgents: false, defaultResponsibleUserId: userId }); + await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: userId, status: "active", membershipRole: "owner" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Worker", role: "engineer", status: "idle", + adapterType: "codex_local", adapterConfig: {}, + runtimeConfig: { heartbeat: { enabled: true, wakeOnDemand: true, maxConcurrentRuns: 1 } }, permissions: {} }); + await db.insert(issues).values({ id: issueId, companyId, title: "Long task across a hard kill", status: "in_progress", + assigneeAgentId: agentId, createdByUserId: userId }); + + const recordedAt = minutesAgo(4); + const restoreRecovery = { schema: RESTORE_SCHEMA, leaseIds: [randomUUID()] }; + await db.insert(heartbeatRuns).values({ id: sourceRunId, companyId, agentId, nativeIssueId: issueId, scopeKind: "issue", + runtimeMode: "native", nativePhase: "terminal_failure", status: "failed", errorCode: "native_restart_recovery_blocked", + invocationSource: "automation", startedAt: minutesAgo(15), finishedAt: minutesAgo(5), createdAt: minutesAgo(16), + contextSnapshot: { issueId, taskId: issueId, wakeReason: "issue_commented" }, + resultJson: { workspaceRestoreFailure: unsafeArchive ? "restore_unsafe_archive" : "restore_failed", + workspaceRestoreRecovery: restoreRecovery, + ...(isolation ? { freshContinuationSupersededAt: recordedAt.toISOString() } : {}) } }); + if (epoch !== "none") { + const workspaceId = randomUUID(), leaseId = randomUUID(), snapshotId = randomUUID(); + const location = { environmentId: randomUUID(), leaseId, providerLeaseId: `sandbox-${sourceRunId}`, + remoteCwd: `/home/daytona/paperclip-workspace/.paperclip-workspaces/${sourceRunId}` }; + await db.insert(executionWorkspaces).values({ id: workspaceId, companyId, mode: "isolated_workspace", + strategyType: "project_primary", name: "Task workspace", status: "active", providerType: "cloud_sandbox", + cwd: `/workspaces/${workspaceId}`, sourceIssueId: issueId, metadata: { remoteWorkspace: { + schema: "paperclip.remote-workspace.v1", epoch: 1, snapshotId, source: { kind: "empty" }, + writerRunId: sourceRunId, dirty: true, location } } }); + await db.update(heartbeatRuns).set({ runnerProfileJson: { remoteWorkspace: { schema: "paperclip.remote-workspace-run.v1", + workspaceId, leaseId, providerLeaseId: location.providerLeaseId, remoteCwd: location.remoteCwd, epoch: 1, + state: "prepared", snapshotId } } }).where(eq(heartbeatRuns.id, sourceRunId)); + if (epoch === "abandoned") await db.transaction(tx => abandonUnverifiedRemoteWorkspaceEpoch(tx, { companyId, workspaceId, + runId: sourceRunId, epoch: 1, snapshotId, location })); + } + await db.insert(nativeRunFinalizations).values({ companyId, issueId, runId: sourceRunId, phase: "terminal_failure", + attempt: 1, failureCode: "native_restart_recovery_blocked", + failureDetail: isolation ? { replacementDenied: "fresh_continuation" } : {} }); + + const queuedRunId = randomUUID(), wakeId = randomUUID(); + if (isolation) { + const freshContinuation = { schema: "paperclip.fresh-continuation.v1", sourceRunId, trigger: "continuation_wake", + failureCode: "native_restart_recovery_blocked", delivery: "recorded", recordedAt: recordedAt.toISOString(), isolation, + stopProof: isolation === "writer_ended" ? "writer_ended" : "unproven_epoch_abandoned", unverifiedActions: [], createdObjects: [] }; + const reconciliation = { runId: sourceRunId, providerStopped: true, + providerStopProof: isolation === "writer_ended" ? "writer_ended" : "isolated", actionOutcome: "mixed", + actorId: "fresh-continuation", recordedAt: recordedAt.toISOString() }; + await db.insert(issueRecoveryActions).values({ id: continuationId, companyId, sourceIssueId: issueId, + kind: "active_run_watchdog", status: "resolved", outcome: "restored", ownerType: "system", returnOwnerAgentId: agentId, + cause: "native_continuation_requires_reconciliation", fingerprint: `native-continuation:${sourceRunId}`, + nextAction: "Paperclip continues the task in a fresh run from its last saved work.", + resolutionNote: "continued_fresh_from_last_saved", resolvedAt: recordedAt, createdAt: recordedAt, updatedAt: recordedAt, + evidence: { runId: sourceRunId, executionReconciliation: reconciliation, freshContinuation, + continuationDelivery: delivery, + ...(delivery === "delivered" ? { continuationRunId: queuedRunId, continuationSupersededByRun: true } : {}) } }); + } + + const spent = failSystemRecovery(reserveSystemRecoveryAttempt(createSystemRecovery(sourceRunId, randomUUID()))!, + new SystemRecoveryFailure("source_unavailable", "use_alternative_strategy"), { now: Date.now() - 3 * 60_000 }); + await db.insert(issueRecoveryActions).values({ id: holdId, companyId, sourceIssueId: issueId, kind: "active_run_watchdog", + status: "active", ownerType: "board", ownerUserId: personOwned ? userId : null, returnOwnerAgentId: agentId, + cause: "native_continuation_requires_reconciliation", fingerprint: `workspace-restore:${sourceRunId}`, + nextAction: "Paperclip cannot recover the unsaved files from the previous run.", + createdAt: minutesAgo(3), updatedAt: minutesAgo(3), + evidence: { runId: sourceRunId, workspaceRestoreFailure: unsafeArchive ? "restore_unsafe_archive" : "restore_failed", + workspaceRestoreRecovery: restoreRecovery, systemRecovery: spent, + workspaceRecovery: { schema: "paperclip.workspace-recovery.v1", runId: sourceRunId, status: "blocked", stage: "verifying_stop", + attempt: 1, attemptLimit: 6, retryable: true, requestIds: [] }, ...holdEvidence } }); + await db.insert(agentWakeupRequests).values({ id: wakeId, companyId, agentId, source: "automation", triggerDetail: "system", + reason: "issue_children_completed", status: "queued", requestedByActorType: "system", requestedByActorId: "issue-children", + payload: { issueId } }); + await db.insert(heartbeatRuns).values({ id: queuedRunId, companyId, agentId, issueId, invocationSource: "automation", + triggerDetail: "system", status: "queued", wakeupRequestId: wakeId, createdAt: minutesAgo(2), + scheduledRetryAttempt: 1, scheduledRetryReason: "transient_failure", + contextSnapshot: { issueId, taskId: issueId, wakeReason: "issue_children_completed" } }); + await db.update(agentWakeupRequests).set({ runId: queuedRunId }).where(eq(agentWakeupRequests.id, wakeId)); + return { userId, companyId, agentId, issueId, sourceRunId, continuationId, holdId, queuedRunId }; + } + + async function runRow(id: string) { + return (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)))[0]!; + } + + async function startedRunIds(companyId: string) { + const rows = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns) + .where(and(eq(heartbeatRuns.companyId, companyId), sql`${heartbeatRuns.startedAt} is not null`, + inArray(heartbeatRuns.status, ["running", "succeeded"]))); + return rows.map(row => row.id); + } + + async function holdRow(id: string) { + return (await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, id)))[0]!; + } + + async function recoverHold(f: { companyId: string; issueId: string; sourceRunId: string; holdId: string }) { + await recoverRetainedWorkspace({ db, companyId: f.companyId, issueId: f.issueId, runId: f.sourceRunId, actionId: f.holdId, + environmentRuntime: {} as Parameters[0]["environmentRuntime"] }); + } + + it.each(["pending", "delivered", "delegated", "not_requested"] as const)( + "workspace recovery settles its late hold into a %s remote_epoch continuation of the same run", async delivery => { + const f = await killedRunAfterFreshContinuation({ isolation: "remote_epoch", delivery }); + expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toMatchObject({ recoveryActionId: f.holdId }); + await recoverHold(f); + expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toBeNull(); + expect(await holdRow(f.holdId)).toMatchObject({ status: "resolved", outcome: "restored", + resolutionNote: "continued_fresh_from_last_saved", evidence: { coalescedIntoRecoveryActionId: f.continuationId } }); + }); + + it.each([ + ["no continuation was recorded", { isolation: null }], + ["the continuation is remote_epoch but the run still owns its epoch", { epoch: "still_owned" }], + ["the continuation is remote_epoch but the run had no remote workspace", { epoch: "none" }], + ["the continuation is a local fresh_isolated_workspace", { isolation: "fresh_isolated_workspace" }], + ["the continuation reuses the old workspace", { isolation: "writer_ended" }], + ["the continuation was invalidated", { delivery: "invalidated" }], + ["the continuation was coalesced elsewhere", { delivery: "coalesced" }], + ["a person owns the hold", { personOwned: true }], + ["the saved archive is unsafe", { unsafeArchive: true }], + ["the hold records an owner-gone attestation", { holdEvidence: { ownerGoneAttestation: { attestedBy: "board" } } }], + ["the hold records an explicit continuation", { holdEvidence: { explicitUserContinuation: { runId: randomUUID() } } }], + ["the hold records a reconciliation", { holdEvidence: { executionReconciliation: { runId: randomUUID() } } }], + ["the hold records a null-valued owner-gone attestation", { holdEvidence: { ownerGoneAttestation: null } }], + ] as const)("keeps the hold when %s", async (_name, shape) => { + const f = await killedRunAfterFreshContinuation(shape); + expect(await settleHoldIntoAbandonedEpochContinuation(db, { companyId: f.companyId, issueId: f.issueId, + runId: f.sourceRunId, actionId: f.holdId })).toBe(false); + expect(await holdRow(f.holdId)).toMatchObject({ status: "active" }); + expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toMatchObject({ recoveryActionId: f.holdId }); + }); + + it("treats a present human-gate key as a person's decision in both SQL and TypeScript, whatever its value", () => { + for (const key of HUMAN_GATE_EVIDENCE_KEYS) { + expect(hasHumanGateEvidence({ [key]: null })).toBe(true); + expect(hasHumanGateEvidence({ [key]: false })).toBe(true); + } + expect(hasHumanGateEvidence({})).toBe(false); + expect(hasHumanGateEvidence(null)).toBe(false); + }); + + it("starts the queued continuation at the execution gate once recovery settled its late hold", async () => { + held.reset(); + const f = await killedRunAfterFreshContinuation({ isolation: "remote_epoch" }); + const heartbeat = heartbeatService(db); + try { + await recoverHold(f); + await heartbeat.resumeQueuedRunsForAgent(f.companyId, f.agentId); + await vi.waitFor(async () => expect((await runRow(f.queuedRunId)).status).not.toBe("queued"), { timeout: 20_000 }); + expect((await runRow(f.queuedRunId)).errorCode).not.toBe("execution_reconciliation_required"); + expect(await startedRunIds(f.companyId)).toEqual([f.queuedRunId]); + expect(await runRow(f.sourceRunId)).toMatchObject({ status: "failed", errorCode: "native_restart_recovery_blocked" }); + } finally { + held.release(); + await heartbeat.drainActiveRunExecutions(); + } + }, 60_000); + + it("still withholds the queued run when a person owns the late hold", async () => { + held.reset(); + const f = await killedRunAfterFreshContinuation({ isolation: "remote_epoch", personOwned: true }); + const heartbeat = heartbeatService(db); + try { + await recoverHold(f); + await heartbeat.resumeQueuedRunsForAgent(f.companyId, f.agentId); + await vi.waitFor(async () => expect((await runRow(f.queuedRunId)).status).toBe("cancelled"), { timeout: 20_000 }); + expect(await runRow(f.queuedRunId)).toMatchObject({ startedAt: null, errorCode: "execution_reconciliation_required" }); + expect(await startedRunIds(f.companyId)).toEqual([]); + } finally { + held.release(); + await heartbeat.drainActiveRunExecutions(); + } + }, 60_000); +}); diff --git a/server/src/services/fresh-continuation-fallback.ts b/server/src/services/fresh-continuation-fallback.ts index 138e819a929..bfc28ef8d31 100644 --- a/server/src/services/fresh-continuation-fallback.ts +++ b/server/src/services/fresh-continuation-fallback.ts @@ -21,7 +21,9 @@ import { issueService } from "./issues.js"; import { holdOwnsBlockedStatus } from "./recovery-hold-blocked-status.js"; import { proveRunProcessGone } from "./run-process-owner.js"; import { hasPendingRunCleanup } from "./execution-blocker.js"; +import { hasHumanGateEvidence } from "./recovery/human-gate-evidence.js"; import { abandonUnverifiedRemoteWorkspaceEpoch } from "./remote-workspace-epoch-abandonment.js"; +import { isAbandonedPreparedRemoteWorkspace } from "./remote-workspace-abandoned-preparation.js"; import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js"; import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { readRunCancellation } from "./run-cancellation.js"; @@ -294,8 +296,7 @@ async function fenceSourceWorkspace(tx: Db, run: typeof heartbeatRuns.$inferSele * gate again. */ export function isHumanGate(hold: Pick): boolean { - return Boolean(hold.ownerUserId || hold.evidence.ownerGoneAttestation || hold.evidence.explicitUserContinuation - || hold.evidence.executionReconciliation); + return Boolean(hold.ownerUserId) || hasHumanGateEvidence(hold.evidence); } const isRestoreHold = (hold: typeof issueRecoveryActions.$inferSelect) => @@ -702,6 +703,36 @@ export async function continueFreshInTransaction(tx: Db, input: Input): Promise< return { kind: "continued", recoveryActionId: primaryId, delivery, isolation }; } +export async function settleHoldIntoAbandonedEpochContinuation(db: Db, input: { + companyId: string; issueId: string; runId: string; actionId: string; now?: Date; +}): Promise { + const now = input.now ?? new Date(); + return db.transaction(async raw => { + const tx = raw as unknown as Db; + const [issue] = await tx.select().from(issues).where(and(eq(issues.companyId, input.companyId), eq(issues.id, input.issueId))).for("update"); + const [run] = await tx.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, input.companyId), + eq(heartbeatRuns.id, input.runId))); + if (!issue || !run) return false; + const [hold] = await tx.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, input.companyId), + eq(issueRecoveryActions.sourceIssueId, issue.id), eq(issueRecoveryActions.id, input.actionId))).for("update"); + if (!hold || !["active", "escalated"].includes(hold.status) || isHumanGate(hold) || hold.evidence.runId !== run.id + || hold.evidence.workspaceRestoreFailure === "restore_unsafe_archive" + || readSystemRecovery(hold.evidence)?.state !== "exhausted") return false; + const [prior] = await tx.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, input.companyId), + eq(issueRecoveryActions.sourceIssueId, issue.id), sql`${issueRecoveryActions.id} <> ${hold.id}`, + sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,schema}' = ${FRESH_CONTINUATION_SCHEMA}`, + sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,sourceRunId}' = ${run.id}`, + sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,isolation}' = 'remote_epoch'`, + inArray(sql`${issueRecoveryActions.evidence}->>'continuationDelivery'`, ["pending", "delivered", "delegated", "not_requested"]))) + .orderBy(desc(sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,recordedAt}'`)).limit(1).for("update"); + if (!prior || !await isAbandonedPreparedRemoteWorkspace(tx, run)) return false; + await settleLateHolds(tx, { companyId: input.companyId, issue, run, prior, holds: [hold], now }); + logger.info({ runId: run.id, issueId: issue.id, recoveryActionId: hold.id, continuationId: prior.id }, + "Workspace recovery hold settled into the continuation that already moved the task to a new workspace epoch"); + return true; + }); +} + async function settleLateHolds(tx: Db, input: { companyId: string; issue: typeof issues.$inferSelect; run: typeof heartbeatRuns.$inferSelect; prior: typeof issueRecoveryActions.$inferSelect; holds: Array; now: Date }) { const { companyId, issue, run, prior, holds, now } = input; diff --git a/server/src/services/interrupted-preparation-hold-settlement.ts b/server/src/services/interrupted-preparation-hold-settlement.ts index c6647d03ed8..fb18138f557 100644 --- a/server/src/services/interrupted-preparation-hold-settlement.ts +++ b/server/src/services/interrupted-preparation-hold-settlement.ts @@ -5,7 +5,7 @@ import { LEGACY_RECOVERY_CAUSE, settleInterruptedNativeBootstrap } from "./legac import { logger } from "../middleware/logger.js"; import type { OwnerProof } from "./owner-proof.js"; import type { InProgressRecord, SettleOutcome } from "./recovery/ownership-registry.js"; -import { OWNER_GONE_ATTESTATION_KEY } from "./recovery/owner-gone-attestation.js"; +import { humanGateEvidencePredicate } from "./recovery/human-gate-evidence.js"; import { abandonedPreparedRemoteWorkspaceExists } from "./remote-workspace-abandoned-preparation.js"; /** @@ -35,7 +35,7 @@ export function heldInterruptedPreparationWhere() { and hold.status = 'resolved' and hold.outcome = 'blocked' and hold.owner_user_id is null and hold.evidence->'automaticRecovery'->>'policy' = 'preserve_without_replay_v1' and hold.evidence->'automaticRecovery'->>'replay' = 'blocked' - and not (hold.evidence ?| array['executionReconciliation', ${OWNER_GONE_ATTESTATION_KEY}, 'explicitUserContinuation']) + and not ${humanGateEvidencePredicate(sql`hold.evidence`)} and (not (hold.evidence ?| array['workspaceRestoreFailure', 'workspaceRestoreRecovery']) or ${abandonedPreparedRemoteWorkspaceExists()}))`, sql`not exists (select 1 from environment_leases lease where lease.company_id = ${heartbeatRuns.companyId} diff --git a/server/src/services/native-runtime/completed-turn-cleanup-hold-settlement.ts b/server/src/services/native-runtime/completed-turn-cleanup-hold-settlement.ts index 89424d1b357..c2b9f4f3f20 100644 --- a/server/src/services/native-runtime/completed-turn-cleanup-hold-settlement.ts +++ b/server/src/services/native-runtime/completed-turn-cleanup-hold-settlement.ts @@ -7,7 +7,7 @@ import { issueService } from "../issues.js"; import { logger } from "../../middleware/logger.js"; import type { InProgressRecord, SettleOutcome } from "../recovery/ownership-registry.js"; import type { OwnerProof } from "../owner-proof.js"; -import { OWNER_GONE_ATTESTATION_KEY } from "../recovery/owner-gone-attestation.js"; +import { humanGateEvidencePredicate } from "../recovery/human-gate-evidence.js"; import { holdOwnsBlockedStatus } from "../recovery-hold-blocked-status.js"; import { notifyRoutineExecutionStatus } from "../routine-execution-status-notifications.js"; import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; @@ -66,7 +66,7 @@ function settleableHoldCondition(runId: SQL | string): SQL { isNull(issueRecoveryActions.ownerAgentId), sql`${issueRecoveryActions.evidence}->'automaticRecovery'->>'policy' = 'preserve_without_replay_v1'`, sql`${issueRecoveryActions.evidence}->'automaticRecovery'->>'replay' = 'blocked'`, - sql`not (${issueRecoveryActions.evidence} ?| array['executionReconciliation', ${OWNER_GONE_ATTESTATION_KEY}, 'explicitUserContinuation'])`, + sql`not ${humanGateEvidencePredicate(issueRecoveryActions.evidence)}`, sql`(not (${issueRecoveryActions.evidence} ?| array['workspaceRestoreFailure', 'workspaceRestoreRecovery']) or ${completedWorkspaceRecoveryCondition(runId)})`, )!; diff --git a/server/src/services/recovery/human-gate-evidence.ts b/server/src/services/recovery/human-gate-evidence.ts new file mode 100644 index 00000000000..8aa20c820fb --- /dev/null +++ b/server/src/services/recovery/human-gate-evidence.ts @@ -0,0 +1,13 @@ +import { sql, type SQL } from "drizzle-orm"; + +export const OWNER_GONE_ATTESTATION_KEY = "ownerGoneAttestation"; + +export const HUMAN_GATE_EVIDENCE_KEYS = ["executionReconciliation", OWNER_GONE_ATTESTATION_KEY, "explicitUserContinuation"] as const; + +export function humanGateEvidencePredicate(evidence: SQL | unknown): SQL { + return sql`(${evidence} ?| array[${sql.join(HUMAN_GATE_EVIDENCE_KEYS.map(key => sql`${key}`), sql`, `)}])`; +} + +export function hasHumanGateEvidence(evidence: Record | null | undefined): boolean { + return Boolean(evidence) && HUMAN_GATE_EVIDENCE_KEYS.some(key => Object.hasOwn(evidence!, key)); +} diff --git a/server/src/services/recovery/owner-gone-attestation.ts b/server/src/services/recovery/owner-gone-attestation.ts index bd5e4503cd8..0fe048136db 100644 --- a/server/src/services/recovery/owner-gone-attestation.ts +++ b/server/src/services/recovery/owner-gone-attestation.ts @@ -9,8 +9,7 @@ import type { InProgressRecord, OwnableKind } from "./ownership-registry.js"; /** Recovery action cause: a stuck record whose owner Paperclip cannot judge. */ export const OWNER_UNPROVABLE_CAUSE = "ownership_owner_unprovable"; -/** Lease metadata key the board's confirmation writes; the owner proof reads it as gone. */ -export const OWNER_GONE_ATTESTATION_KEY = "ownerGoneAttestation"; +export { OWNER_GONE_ATTESTATION_KEY } from "./human-gate-evidence.js"; /** Unprovable because the record names no usable owner: saved before owners were recorded. */ export const LEGACY_OWNER_REASONS: ReadonlySet = new Set(["no_owner_stamp", "process_space_unrecorded", "missing_identity"]); diff --git a/server/src/services/workspace-recovery.ts b/server/src/services/workspace-recovery.ts index f7dac753918..e4d9ddc8a42 100644 --- a/server/src/services/workspace-recovery.ts +++ b/server/src/services/workspace-recovery.ts @@ -41,7 +41,7 @@ import { capturePolicyRearmDueCondition, retainedSourceFallbackDueCondition, sto import { rearmExhaustedCapturePolicy } from "./workspace-recovery-capture-policy.js"; import { rearmExhaustedStopProof } from "./workspace-recovery-stop-proof.js"; import { settleRecoveredWorkspaceRestoreHolds } from "./workspace-restore-hold-settlement.js"; -import { expediteFreshContinuationChecks } from "./fresh-continuation-fallback.js"; +import { expediteFreshContinuationChecks, settleHoldIntoAbandonedEpochContinuation } from "./fresh-continuation-fallback.js"; import { reusableSandboxNotKeptAfterReplacementSql } from "./reusable-sandbox-kept-marker.js"; const exportCauses = ["native_workspace_sync_out_unrecoverable", "native_workspace_sync_out_retry_exhausted"]; @@ -206,6 +206,7 @@ export async function recoverRetainedWorkspace(input: Source & { db: Db; environ const { db } = input; await adoptSystemRecovery(db, input); let initial = await loadSystemRecovery(db, input); + if (initial?.state === "exhausted" && await settleHoldIntoAbandonedEpochContinuation(db, input)) return; // A hold whose files a sibling saved, or whose source is refused permanently // (including holds saved by older releases), settles without a person. // An adoptable native source keeps its retention path; only the sibling @@ -510,6 +511,7 @@ export async function recoverRetainedWorkspace(input: Source & { db: Db; environ ...(cause ? { cause } : {}), }); const recorded = await updateSystemRecovery(db, input, current!, failed); + if (recorded && failed.state === "exhausted" && await settleHoldIntoAbandonedEpochContinuation(db, input)) return; // A permanent refusal of the recorded source repeats on every retry. // Continue from the last save in this same pass, under this ownership. if (recorded && failed.state === "exhausted" && isUnverifiableSourceFailure(failed.failure)) {