diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index f77b10f3b04..da7020e5abe 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1812,6 +1812,23 @@ provider turn, and saved follow-up messages are reconsidered. A live owner, an owner without a recorded process, a lease of the current boot, and a remote workspace keep the lease until it expires. Lease age alone is never proof. +A recorded process is judged only in the PID space that recorded it. Native +controller rows store `controller_host_hash` and run processes store +`process_host_hash` (`"remote"` for a sandbox process). A dead PID on this host +never proves that a process on another host, in another PID namespace, or in a +sandbox is gone; such a record keeps its lease, fence or staleness TTL. + +Rows saved before those columns existed have no recorded host. For one release +only, a single-host server keeps the previous local proof for them, and each use +writes a `legacy_host_unknown_dead_pid_proof` activity-log entry, so the release that +removes this fallback can confirm that no rows still depend on it. An ephemeral +server (`PAPERCLIP_EPHEMERAL_SERVER=true`) never uses the fallback. Its native +controller rows with no recorded host wait as `controller_host_unrecorded`, with +a board recovery action to confirm that the previous server stopped. Its run +processes with no recorded host are left to the run's Stop control. The next +release removes the fallback and treats every row with no recorded host as +unprovable. + The stranded-work pass closes the gap where issue state survives a crash but the wake/run path does not. The silent-run scan covers the separate case where a live process exists but has stopped producing observable output. Native finalization reconciliation (part of native recovery and the orphan diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 4f351647499..36f2bf06e8a 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -1994,6 +1994,7 @@ export { taskRetrySchema, reconstructWorkspaceSchema, resumeWorkspaceFinalizationSchema, + confirmNativeControllerStoppedSchema, issueReviewRequestSchema, issueExecutionWorkspaceSettingsSchema, checkoutIssueSchema, diff --git a/packages/shared/src/validators/index.ts b/packages/shared/src/validators/index.ts index 6bf6703588c..9eb2f59fccb 100644 --- a/packages/shared/src/validators/index.ts +++ b/packages/shared/src/validators/index.ts @@ -451,6 +451,7 @@ export { taskRetrySchema, reconstructWorkspaceSchema, resumeWorkspaceFinalizationSchema, + confirmNativeControllerStoppedSchema, issueReviewRequestSchema, issueExecutionWorkspaceSettingsSchema, checkoutIssueSchema, diff --git a/packages/shared/src/validators/issue.ts b/packages/shared/src/validators/issue.ts index 3218666734a..1a3db303fec 100644 --- a/packages/shared/src/validators/issue.ts +++ b/packages/shared/src/validators/issue.ts @@ -591,6 +591,14 @@ export const resumeWorkspaceFinalizationSchema = z.object({ stopEvidence: z.string().trim().min(20).max(12000), }).strict(); +/** Board attestation that a native run's previous server stopped, for a + * controller row an ephemeral server cannot judge by PID. */ +export const confirmNativeControllerStoppedSchema = z.object({ + actionId: z.string().guid(), + runId: z.string().guid(), + stopEvidence: z.string().trim().min(20).max(12000), +}).strict(); + export const retryWorkspaceExportSchema = z.object({ actionId: z.string().guid(), runId: z.string().guid(), diff --git a/server/src/__tests__/recovery-stale-issue-lock-sweep.test.ts b/server/src/__tests__/recovery-stale-issue-lock-sweep.test.ts index 997ed16f5c4..9459d0572f0 100644 --- a/server/src/__tests__/recovery-stale-issue-lock-sweep.test.ts +++ b/server/src/__tests__/recovery-stale-issue-lock-sweep.test.ts @@ -2,7 +2,7 @@ import { persistHeartbeatRunProcessMetadata } from "../services/heartbeat/run-li import { currentProcessSpaceHash } from "../services/controller-process-identity.js"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { randomUUID } from "node:crypto"; -import { eq, sql } from "drizzle-orm"; +import { and, eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { subscribeDatabaseWork, @@ -396,6 +396,24 @@ describeEmbeddedPostgres("recovery sweepStaleIssueLocks", () => { expect(second.cleared).toBe(0); }); + it.each([[false, true], [true, false]] as const)( + "judges a pre-upgrade pid with no recorded space only on a single-host server (ephemeral: %s)", async (ephemeral, terminalized) => { + vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", ephemeral ? "true" : ""); + try { + const { companyId, agentId, runningRunId } = await seed(); + await db.update(heartbeatRuns).set({ processPid: 2_000_000_000, processHostHash: null }) + .where(eq(heartbeatRuns.id, runningRunId)); + await db.insert(issues).values({ id: randomUUID(), companyId, title: "Pre-upgrade orphan", status: "in_progress", + priority: "high", assigneeAgentId: agentId, checkoutRunId: runningRunId, executionRunId: runningRunId, + executionLockedAt: new Date() }); + const result = await heartbeatService(db).sweepStaleIssueLocks(); + expect(result.terminalizedRunIds.includes(runningRunId)).toBe(terminalized); + const events = await db.select({ details: activityLog.details }).from(activityLog) + .where(and(eq(activityLog.runId, runningRunId), eq(activityLog.action, "legacy_host_unknown_dead_pid_proof"))); + expect(events).toHaveLength(terminalized ? 1 : 0); + } finally { vi.unstubAllEnvs(); } + }); + it.each([["local", true], ["remote", false]] as const)("stamps a %s process pid with its PID space", async (targetKind, local) => { const { runningRunId } = await seed(); await persistHeartbeatRunProcessMetadata(db, runningRunId, { pid: process.pid, processGroupId: null, diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.ts index 4bf0e443318..a77ef522f1c 100644 --- a/server/src/routes/issues.ts +++ b/server/src/routes/issues.ts @@ -39,6 +39,7 @@ import { resolveConfirmationFromComment } from "../services/confirmation-comment import { createIssueReadTiming } from "../services/issue-read-timing.js"; import { isNativeWorkspaceExportRepairCause } from "@paperclipai/shared"; import { resumeNativeWorkspaceAfterOwnerStop } from "../services/native-runtime/native-workspace-owner-recovery.js"; +import { confirmNativeControllerStopped, NATIVE_CONTROLLER_HOST_UNVERIFIED } from "../services/native-runtime/native-controller-host-recovery.js"; import { retryNativeWorkspaceExport } from "../services/native-runtime/native-workspace-export-retry.js"; import { reconstructNativeWorkspace, unreconciledNativeExportPredicate } from "../services/native-runtime/native-workspace-reconstruction.js"; import { reconstructWorkspaceSchema } from "@paperclipai/shared"; @@ -122,6 +123,7 @@ import { recoverWorkspaceSchema, taskRetrySchema, resumeWorkspaceFinalizationSchema, + confirmNativeControllerStoppedSchema, runnerGoalActionRequestSchema, feedbackTargetTypeSchema, feedbackTraceStatusSchema, @@ -10026,6 +10028,17 @@ export function issueRoutes( res.status(202).json(receipt); }); + router.post("/issues/:id/recovery-actions/confirm-controller-stopped", validate(confirmNativeControllerStoppedSchema), async (req, res) => { + assertBoard(req); + const issue = await getAccessibleResource(req, res, svc.getById(req.params.id as string), "Issue not found"); + if (!issue || !(await assertIssueReadAllowed(req, res, issue))) return; + const decision = await access.decide({ actor: req.actor, action: "runtime:confirm_owner_stop", resource: { type: "company", companyId: issue.companyId } }); + if (!decision.allowed) throw forbidden(decision.explanation, authorizationDeniedDetails(decision)); + const receipt = await confirmNativeControllerStopped({ db, companyId: issue.companyId, issueId: issue.id, + ...req.body, actorId: getActorInfo(req).actorId }); + res.status(202).json(receipt); + }); + router.post("/issues/:id/recovery-actions/retry-workspace-export", validate(retryWorkspaceExportSchema), async (req, res) => { assertBoard(req); const issue = await getAccessibleResource(req, res, svc.getById(req.params.id as string), "Issue not found"); @@ -10290,6 +10303,10 @@ export function issueRoutes( throw conflict("Unsafe workspace export recovers automatically without another provider turn.", { code: "workspace_export_automatic_recovery" }); } + if ((outcome === "restored" || outcome === "false_positive") && activeRecoveryAction.cause === NATIVE_CONTROLLER_HOST_UNVERIFIED) { + throw conflict("Confirm that the previous server stopped to resume this run.", { code: "native_controller_stop_required" }); + } + if ((outcome === "restored" || outcome === "false_positive") && activeRecoveryAction.cause === "native_workspace_finalization_owner_unverified") { throw conflict("Verify the previous controller and its copyback processes stopped, then use Resume saved result.", { code: "workspace_owner_stop_required" }); } diff --git a/server/src/routes/openapi.ts b/server/src/routes/openapi.ts index 8e7d23d4c00..b52b5c716e7 100644 --- a/server/src/routes/openapi.ts +++ b/server/src/routes/openapi.ts @@ -260,6 +260,7 @@ import { retryWorkspaceExportSchema, reconstructWorkspaceSchema, resumeWorkspaceFinalizationSchema, + confirmNativeControllerStoppedSchema, cancelIssueThreadInteractionSchema, // Secret provider configs and remote import createSecretProviderConfigSchema, @@ -11026,6 +11027,15 @@ registerCurrentRoute({ responses: { 202: r.ok(), 400: r.badRequest, 401: r.unauthorized, 403: r.forbidden, 404: r.notFound, 409: r.conflict }, }); +registerCurrentRoute({ + method: "post", + path: "/api/issues/{id}/recovery-actions/confirm-controller-stopped", + tags: ["issues"], + summary: "Confirm that a native run's previous server stopped when its host was not recorded", + body: confirmNativeControllerStoppedSchema, + responses: { 202: r.ok(), 400: r.badRequest, 401: r.unauthorized, 403: r.forbidden, 404: r.notFound, 409: r.conflict }, +}); + registerCurrentRoute({ method: "post", path: "/api/issues/{id}/recovery-actions/retry-workspace-export", diff --git a/server/src/services/ephemeral-server-contract.ts b/server/src/services/ephemeral-server-contract.ts index f5b84347b78..c4d9ebc9daa 100644 --- a/server/src/services/ephemeral-server-contract.ts +++ b/server/src/services/ephemeral-server-contract.ts @@ -9,7 +9,9 @@ import { hasLocalNativeControllerObligations } from "./native-runtime/native-con import { hasUnsettledLocalInstructionCopies } from "./agent-instruction-copy-preservation.js"; import { hasLocalRuntimeSkillObligations } from "./native-runtime/runtime-skill-preservation.js"; -export const ephemeralServerEnabled = () => process.env.PAPERCLIP_EPHEMERAL_SERVER === "true"; +import { ephemeralServerEnabled } from "./ephemeral-server-mode.js"; + +export { ephemeralServerEnabled }; /** Validate before consumers can generate a replacement local identity. Error * messages name settings only, never credential values or connection URLs. */ diff --git a/server/src/services/ephemeral-server-mode.ts b/server/src/services/ephemeral-server-mode.ts new file mode 100644 index 00000000000..47e245cac99 --- /dev/null +++ b/server/src/services/ephemeral-server-mode.ts @@ -0,0 +1,4 @@ +/** Whether this server runs in volume-free ephemeral (multi-host) mode. A + * leaf module so recovery code can read the mode without importing the + * ephemeral data-eligibility checks and their dependency graph. */ +export const ephemeralServerEnabled = () => process.env.PAPERCLIP_EPHEMERAL_SERVER === "true"; diff --git a/server/src/services/native-runtime/native-controller-host-recovery.test.ts b/server/src/services/native-runtime/native-controller-host-recovery.test.ts new file mode 100644 index 00000000000..1d3c4c61e52 --- /dev/null +++ b/server/src/services/native-runtime/native-controller-host-recovery.test.ts @@ -0,0 +1,80 @@ +import { randomUUID } from "node:crypto"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { and, eq } from "drizzle-orm"; +import { + activityLog, agents, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRunEvents, heartbeatRuns, + issueRecoveryActions, issues, nativeRunFinalizations, +} from "@paperclipai/db"; +import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; +import { confirmNativeControllerStopped, NATIVE_CONTROLLER_HOST_UNVERIFIED, requestNativeControllerStopConfirmation } + from "./native-controller-host-recovery.js"; +import { evaluateNativeControllerTakeover, LEGACY_HOST_FALLBACK_EVENT, recordLegacyHostFallbackProof } + from "./native-restart-recovery.js"; + +describe("native controller rows saved before their host was recorded", () => { + let temporary: Awaited> | null = null; + let db: ReturnType; + + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("paperclip-native-controller-host-"); + db = createDb(temporary.connectionString); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + afterAll(async () => temporary?.cleanup()); + afterEach(() => vi.unstubAllEnvs()); + + async function seed() { + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Host", issuePrefix: `H${companyId.slice(0, 6)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Agent", adapterType: "codex_local", status: "running" }); + await db.insert(issues).values({ id: issueId, companyId, title: "Native task", status: "in_progress", assigneeAgentId: agentId }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: issueId, status: "running", + runtimeMode: "native", contextSnapshot: { issueId } }); + // The shape the previous release saved: a local controller and no host. + const [coordinator] = await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId, phase: "observed", + attempt: 1, leaseOwner: "old-controller", leaseExpiresAt: new Date(0), controllerBootId: randomUUID(), + controllerPid: 2_000_000_000, controllerProcessStartedAt: new Date() }).returning(); + return { companyId, agentId, issueId, runId, coordinator: coordinator! }; + } + + it("records each use of the single-host fallback in the run log", async () => { + vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", ""); + const { companyId, agentId, runId, coordinator } = await seed(); + const decision = await evaluateNativeControllerTakeover({ owner: coordinator, now: new Date() }); + expect(decision).toMatchObject({ allowed: true, legacyHostFallback: true }); + await recordLegacyHostFallbackProof(db, { companyId, id: runId, agentId }, decision, "test"); + const events = await db.select().from(activityLog).where(and(eq(activityLog.runId, runId), + eq(activityLog.action, LEGACY_HOST_FALLBACK_EVENT))); + expect(events).toEqual([expect.objectContaining({ details: expect.objectContaining({ reason: decision.reason, source: "test" }) })]); + // It is not run-log evidence, which restart recovery reads as progress. + expect(await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, runId))).toEqual([]); + }); + + it("asks the board on an ephemeral server and resumes only after it confirms the exact controller stopped", async () => { + vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "true"); + const { companyId, issueId, runId, coordinator } = await seed(); + expect(await evaluateNativeControllerTakeover({ owner: coordinator, now: new Date() })) + .toEqual({ allowed: false, reason: "controller_host_unrecorded" }); + await requestNativeControllerStopConfirmation(db, coordinator); + await requestNativeControllerStopConfirmation(db, coordinator); + const actions = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId)); + expect(actions).toEqual([expect.objectContaining({ cause: NATIVE_CONTROLLER_HOST_UNVERIFIED, ownerType: "board", + status: "active", evidence: expect.objectContaining({ runId }) })]); + + const stopEvidence = "The old server container was replaced and is not running."; + await expect(confirmNativeControllerStopped({ db, companyId, issueId, actionId: actions[0]!.id, runId: randomUUID(), + stopEvidence, actorId: "board-user" })).rejects.toMatchObject({ status: 409 }); + await expect(confirmNativeControllerStopped({ db, companyId, issueId, actionId: actions[0]!.id, runId, + stopEvidence, actorId: "board-user" })).resolves.toEqual({ runId, status: "queued" }); + + const [attested] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)); + expect(await evaluateNativeControllerTakeover({ owner: attested!, now: new Date() })) + .toEqual({ allowed: true, reason: "operator_attested_controller_stopped" }); + expect((await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, actions[0]!.id)))[0]) + .toMatchObject({ status: "resolved", evidence: expect.objectContaining({ controllerStop: expect.objectContaining({ stopEvidence }) }) }); + expect(await db.select({ action: activityLog.action }).from(activityLog).where(and(eq(activityLog.companyId, companyId), + eq(activityLog.action, "issue.native_controller_stop_confirmed")))).toHaveLength(1); + // The confirmation is single-use for that exact controller. + await expect(confirmNativeControllerStopped({ db, companyId, issueId, actionId: actions[0]!.id, runId, + stopEvidence, actorId: "board-user" })).rejects.toMatchObject({ status: 409 }); + }); +}); diff --git a/server/src/services/native-runtime/native-controller-host-recovery.ts b/server/src/services/native-runtime/native-controller-host-recovery.ts new file mode 100644 index 00000000000..84b482dcc83 --- /dev/null +++ b/server/src/services/native-runtime/native-controller-host-recovery.ts @@ -0,0 +1,83 @@ +import { and, eq, isNull } from "drizzle-orm"; +import { heartbeatRuns, issueRecoveryActions, nativeRunFinalizations, type Db } from "@paperclipai/db"; +import { confirmNativeControllerStoppedSchema } from "@paperclipai/shared"; +import { conflict } from "../../errors.js"; +import { persistActivity, publishActivity } from "../activity-log.js"; +import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js"; + +/** + * An ephemeral (multi-host) server cannot judge a native controller row saved + * before controller_host_hash existed: its PID may name a process on another + * host. Restart recovery leaves such a row waiting and asks the board, through + * this recovery action, to confirm that the previous server stopped. + */ +export const NATIVE_CONTROLLER_HOST_UNVERIFIED = "native_controller_host_unverified"; +/** controller_host_hash value written once the board confirms the stop. */ +export const OPERATOR_ATTESTED_CONTROLLER_HOST = "operator_attested"; + +const fingerprint = (runId: string) => `native-controller-host:${runId}`; +const changed = () => conflict("The recorded controller is no longer current. Refresh the task and inspect its run.", + { code: "native_controller_host_recovery_stale" }); + +type Coordinator = Pick; + +/** Idempotent. Another active recovery action on the task keeps precedence. */ +export async function requestNativeControllerStopConfirmation(tx: Db, coordinator: Coordinator) { + await tx.insert(issueRecoveryActions).values({ + companyId: coordinator.companyId, sourceIssueId: coordinator.issueId, kind: "active_run_watchdog", + status: "active", ownerType: "board", cause: NATIVE_CONTROLLER_HOST_UNVERIFIED, + fingerprint: fingerprint(coordinator.runId), attemptCount: 1, + evidence: { runId: coordinator.runId, controller: { bootId: coordinator.controllerBootId, pid: coordinator.controllerPid, + processStartedAt: coordinator.controllerProcessStartedAt?.toISOString() ?? null } }, + nextAction: "This run's previous server was recorded before Paperclip stored which host it ran on, so Paperclip " + + "cannot verify that it stopped. Confirm that the previous server process has stopped (for example, its " + + "container was replaced), and the run resumes from its saved state.", + }).onConflictDoNothing(); +} + +/** Board attestation. It marks only the exact recorded controller as stopped; + * restart recovery then takes over through its normal path. */ +export async function confirmNativeControllerStopped(input: { + db: Db; companyId: string; issueId: string; actionId: string; runId: string; stopEvidence: string; actorId: string; +}) { + const confirmation = confirmNativeControllerStoppedSchema.parse({ + actionId: input.actionId, runId: input.runId, stopEvidence: input.stopEvidence, + }); + const committed = await input.db.transaction(async transaction => { + const tx = transaction as unknown as Db; + const [coordinator] = await tx.select().from(nativeRunFinalizations).where(and( + eq(nativeRunFinalizations.companyId, input.companyId), eq(nativeRunFinalizations.issueId, input.issueId), + eq(nativeRunFinalizations.runId, input.runId))).for("update"); + const [action] = await tx.select().from(issueRecoveryActions).where(and( + eq(issueRecoveryActions.companyId, input.companyId), eq(issueRecoveryActions.sourceIssueId, input.issueId), + eq(issueRecoveryActions.id, input.actionId))).for("update"); + const [run] = await tx.select({ agentId: heartbeatRuns.agentId }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.runId))); + const recorded = (action?.evidence?.controller ?? {}) as Record; + if (!coordinator || !action || !run || action.cause !== NATIVE_CONTROLLER_HOST_UNVERIFIED || action.ownerType !== "board" + || !["active", "escalated"].includes(action.status) || action.evidence?.runId !== input.runId + || coordinator.controllerHostHash !== null || coordinator.controllerPid !== recorded.pid + || coordinator.controllerBootId !== recorded.bootId) throw changed(); + // Work intent before the write it announces (sleep-aware background work). + await notifyLiveRunRecovery(tx, { companyId: input.companyId, runId: input.runId, agentId: run.agentId }); + const now = new Date(); + const updated = await tx.update(nativeRunFinalizations).set({ controllerHostHash: OPERATOR_ATTESTED_CONTROLLER_HOST, updatedAt: now }) + .where(and(eq(nativeRunFinalizations.runId, input.runId), isNull(nativeRunFinalizations.controllerHostHash), + eq(nativeRunFinalizations.controllerPid, coordinator.controllerPid!), + eq(nativeRunFinalizations.controllerBootId, coordinator.controllerBootId!))) + .returning({ runId: nativeRunFinalizations.runId }); + if (!updated.length) throw changed(); + const proof = { controller: recorded, actorId: input.actorId, stopEvidence: confirmation.stopEvidence, confirmedAt: now.toISOString() }; + await tx.update(issueRecoveryActions).set({ status: "resolved", outcome: "restored", resolvedAt: now, updatedAt: now, + evidence: { ...action.evidence, controllerStop: proof }, + nextAction: "The previous server was confirmed stopped. Recovery resumes the run from its saved state.", + }).where(eq(issueRecoveryActions.id, action.id)); + const publication = await persistActivity(tx, { companyId: input.companyId, actorType: "user", actorId: input.actorId, + action: "issue.native_controller_stop_confirmed", entityType: "issue", entityId: input.issueId, runId: input.runId, + details: { recoveryActionId: input.actionId, ...proof } }); + return { publication }; + }); + publishActivity(committed.publication.publication); + return { runId: input.runId, status: "queued" as const }; +} diff --git a/server/src/services/native-runtime/native-dead-controller-result-lease.ts b/server/src/services/native-runtime/native-dead-controller-result-lease.ts index fcf922eb024..8d4f399f984 100644 --- a/server/src/services/native-runtime/native-dead-controller-result-lease.ts +++ b/server/src/services/native-runtime/native-dead-controller-result-lease.ts @@ -3,7 +3,7 @@ import { heartbeatRuns, nativeRunFinalizations, type Db } from "@paperclipai/db" import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js"; import { notifyNativeFinalizationWork } from "../native-finalization-notifications.js"; import { notifyNativeResultCommit } from "../native-result-commit-notifications.js"; -import { currentNativeControllerIdentity, evaluateNativeControllerTakeover } from "./native-restart-recovery.js"; +import { currentNativeControllerIdentity, evaluateNativeControllerTakeover, recordLegacyHostFallbackProof } from "./native-restart-recovery.js"; type TakeoverProbe = Pick[0], "isProcessAlive" | "readProcessStartedAt">; @@ -89,7 +89,12 @@ export async function releaseDeadControllerResultLeases(db: Db, input: { row.controllerPid === null ? isNull(nativeRunFinalizations.controllerPid) : eq(nativeRunFinalizations.controllerPid, row.controllerPid), )) - .returning({ runId: nativeRunFinalizations.runId }); + .returning({ runId: nativeRunFinalizations.runId }) + .then(async rows => { + if (rows.length) await recordLegacyHostFallbackProof(tx as unknown as Db, + { companyId: row.companyId, id: row.runId, agentId: row.agentId }, takeover, "dead_controller_result_lease"); + return rows; + }); }); if (updated.length) released.push({ runId: row.runId, reason: takeover.reason }); } diff --git a/server/src/services/native-runtime/native-restart-recovery.test.ts b/server/src/services/native-runtime/native-restart-recovery.test.ts index eaa8708e510..2e3d55ea235 100644 --- a/server/src/services/native-runtime/native-restart-recovery.test.ts +++ b/server/src/services/native-runtime/native-restart-recovery.test.ts @@ -1,5 +1,5 @@ import { currentProcessSpaceHash } from "../controller-process-identity.js"; -import { describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { classifyNativeRunnerRecoveryEvidence, @@ -281,12 +281,48 @@ describe("native controller takeover fencing", () => { } }); + describe("rows saved before the controller host was recorded", () => { + afterEach(() => vi.unstubAllEnvs()); + + it("keep the local dead-PID proof on a single-host server and flag the fallback", async () => { + vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", ""); + await expect(evaluateNativeControllerTakeover({ owner: { ...owner(), controllerHostHash: null }, now, + isProcessAlive: () => false })).resolves.toEqual({ allowed: true, reason: "controller_process_dead", legacyHostFallback: true }); + }); + + it("are unprovable on an ephemeral server, whatever a local probe says", async () => { + vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "true"); + await expect(evaluateNativeControllerTakeover({ owner: { ...owner(), controllerHostHash: null }, now, + isProcessAlive: () => false })).resolves.toEqual({ allowed: false, reason: "controller_host_unrecorded" }); + }); + + it("stay unprovable on another host in either mode", async () => { + for (const mode of ["", "true"]) { + vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", mode); + await expect(evaluateNativeControllerTakeover({ owner: { ...owner(), controllerHostHash: "another-host-pid-space" }, now, + isProcessAlive: () => false })).resolves.toEqual({ allowed: false, reason: "controller_host_unprovable" }); + } + }); + + it("accept a board confirmation that the exact controller stopped", async () => { + vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "true"); + await expect(evaluateNativeControllerTakeover({ owner: { ...owner(), controllerHostHash: "operator_attested" }, now, + isProcessAlive: () => true })).resolves.toEqual({ allowed: true, reason: "operator_attested_controller_stopped" }); + }); + + it("leave receipt owners that never stored a host unchanged", async () => { + vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "true"); + await expect(evaluateNativeControllerTakeover({ owner: owner(), now, isProcessAlive: () => false })) + .resolves.toEqual({ allowed: true, reason: "controller_process_dead" }); + }); + }); + it("proves a controller gone only in its own recorded PID space", async () => { await expect(evaluateNativeControllerTakeover({ owner: { ...owner(), controllerHostHash: currentProcessSpaceHash() }, now, isProcessAlive: () => false })).resolves.toEqual({ allowed: true, reason: "controller_process_dead" }); // A row saved before the PID space was recorded keeps the previous local proof for one release. await expect(evaluateNativeControllerTakeover({ owner: { ...owner(), controllerHostHash: null }, - now, isProcessAlive: () => false })).resolves.toEqual({ allowed: true, reason: "controller_process_dead" }); + now, isProcessAlive: () => false })).resolves.toEqual({ allowed: true, reason: "controller_process_dead", legacyHostFallback: true }); }); it.each([true, false])("requires a stopped idle controller even after its run lease was released: alive=%s", async alive => { diff --git a/server/src/services/native-runtime/native-restart-recovery.ts b/server/src/services/native-runtime/native-restart-recovery.ts index a5ed318b216..2dbeea7c9f5 100644 --- a/server/src/services/native-runtime/native-restart-recovery.ts +++ b/server/src/services/native-runtime/native-restart-recovery.ts @@ -35,6 +35,9 @@ import { } from "@paperclipai/db"; import { REMOTE_CONTROLLER_LEASE_MS } from "./native-runner-recovery-contract.js"; import { readProcessStartedAt } from "../hot-restart.js"; +import { ephemeralServerEnabled } from "../ephemeral-server-mode.js"; +import { persistActivity } from "../activity-log.js"; +import { OPERATOR_ATTESTED_CONTROLLER_HOST, requestNativeControllerStopConfirmation } from "./native-controller-host-recovery.js"; import { currentControllerBootId, currentProcessSpaceHash, currentProcessStartedAtMs, processStartsDefinitelyDiffer, sameProcessSpace } from "../controller-process-identity.js"; import { getServerProcessStartedAt } from "../../server-info.js"; import { redactSensitiveText } from "../../redaction.js"; @@ -360,6 +363,27 @@ function definitivelyDifferentProcessStart( return processStartsDefinitelyDiffer(left, right); } +export type NativeControllerTakeoverDecision = { + allowed: boolean; + reason: string; + /** The proof used the one-release local fallback for a row with no recorded host. */ + legacyHostFallback?: true; +}; + +/** Activity-log action for each use of the null-host fallback. The release + * that removes the fallback counts these to see whether any rows still need + * it. It is not a run-log event: run events are progress evidence for + * restart recovery, and this record must not change any recovery decision. */ +export const LEGACY_HOST_FALLBACK_EVENT = "legacy_host_unknown_dead_pid_proof"; + +export async function recordLegacyHostFallbackProof(db: Db, run: { companyId: string; id: string; agentId: string }, + decision: NativeControllerTakeoverDecision, source: string) { + if (!decision.legacyHostFallback) return; + await persistActivity(db, { companyId: run.companyId, actorType: "system", actorId: "native-restart-recovery", + action: LEGACY_HOST_FALLBACK_EVENT, entityType: "heartbeat_run", entityId: run.id, runId: run.id, agentId: run.agentId, + details: { reason: decision.reason, source } }); +} + export async function evaluateNativeControllerTakeover(input: { owner: Pick< typeof nativeRunFinalizations.$inferSelect, @@ -368,8 +392,10 @@ export async function evaluateNativeControllerTakeover(input: { | "controllerPid" | "controllerProcessStartedAt" > & { - /** Absent for owners without a stored PID space (receipts, rows saved - * before the column existed): those keep the previous local proof. */ + /** PID space of controllerPid. Absent for owners built from receipts + * that never stored one. Null on coordinator rows saved before the column + * existed: those keep the previous local proof for one release, outside + * ephemeral mode only (see LEGACY_HOST_FALLBACK_EVENT). */ controllerHostHash?: string | null; }; now: Date; @@ -381,7 +407,7 @@ export async function evaluateNativeControllerTakeover(input: { readProcessStartedAt?: (pid: number) => Promise; requireStoppedController?: boolean; remoteLeaseFenced?: boolean; -}): Promise<{ allowed: boolean; reason: string }> { +}): Promise { const { owner, now } = input; if (input.remoteLeaseFenced) { return owner.leaseExpiresAt && owner.leaseExpiresAt > now @@ -404,12 +430,26 @@ export async function evaluateNativeControllerTakeover(input: { : "live_legacy_lease_without_controller_identity", }; } + const recordsHost = Object.prototype.hasOwnProperty.call(owner, "controllerHostHash"); + // A board operator confirmed this exact controller stopped (see + // confirmNativeControllerStopped); that attestation is the proof. + if (recordsHost && owner.controllerHostHash === OPERATOR_ATTESTED_CONTROLLER_HOST) { + return { allowed: true, reason: "operator_attested_controller_stopped" }; + } // A PID can only be judged in the PID space that recorded it. A controller // on another host or namespace (or an unreadable local space) is never // proven gone by a local probe; its lease or fence decides instead. if (owner.controllerHostHash && !sameProcessSpace(owner.controllerHostHash)) { return { allowed: false, reason: "controller_host_unprovable" }; } + // A row saved before the host column existed. An ephemeral (multi-host) + // server can never assume it was written here; a board operator decides. + if (recordsHost && owner.controllerHostHash == null && ephemeralServerEnabled()) { + return { allowed: false, reason: "controller_host_unrecorded" }; + } + // Single-host servers keep the previous local proof for one release, and + // every use is recorded so the release that removes it can check for rows. + const fallback = recordsHost && owner.controllerHostHash == null ? { legacyHostFallback: true as const } : {}; if (!isAlive(priorPid)) { return { allowed: true, @@ -417,6 +457,7 @@ export async function evaluateNativeControllerTakeover(input: { owner.leaseExpiresAt !== null && owner.leaseExpiresAt <= now ? "expired_lease_controller_process_dead" : "controller_process_dead", + ...fallback, }; } @@ -429,7 +470,7 @@ export async function evaluateNativeControllerTakeover(input: { observedStartedAt, ) ) { - return { allowed: true, reason: "controller_pid_recycled" }; + return { allowed: true, reason: "controller_pid_recycled", ...fallback }; } const coordinated = input.coordinatedPreviousController; @@ -1110,6 +1151,10 @@ export async function claimNativeRestartRecoveries(input: { coordinatedPreviousController: input.coordinatedPreviousController ?? null, }); + await recordLegacyHostFallbackProof(tx as unknown as Db, row.run, takeover, "native_restart_recovery"); + if (!takeover.allowed && takeover.reason === "controller_host_unrecorded") { + await requestNativeControllerStopConfirmation(tx as unknown as Db, row.coordinator); + } if (!takeover.allowed) { const previous = row.coordinator.recoveryHistory.at(-1); const waitAlreadyRecorded = previous?.disposition === "awaiting_evidence" && previous.reason === takeover.reason; diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index cdc3427b280..331748456eb 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -309,6 +309,8 @@ import { import { currentNativeControllerIdentity, evaluateNativeControllerTakeover, + recordLegacyHostFallbackProof, + type NativeControllerTakeoverDecision, proveNativeLocalRunnerStopped, nativeProviderProcessIdentities, NATIVE_PROVIDER_PROCESS_EVENT_TYPES, @@ -5455,7 +5457,8 @@ const RETIREMENT_REASON_NAMES = { "recovery_needed"]), recoveryState: new Set(["blocked", "resuming_session", "awaiting_evidence", "awaiting_runner_reattach"]), controllerReason: new Set(["controller_still_alive", "coordinated_controller_still_alive", "remote_controller_lease_live", - "expired_lease_without_controller_identity", "live_legacy_lease_without_controller_identity"]), + "expired_lease_without_controller_identity", "live_legacy_lease_without_controller_identity", + "controller_host_unprovable", "controller_host_unrecorded"]), sandboxOutcome: new Set(["created", "resumed", "replacement"]), } as const; @@ -5543,10 +5546,11 @@ async function inspectFinishedPerTurnRunnerdOwner(input: { const finishedHere = () => isCurrentNativeControllerProcess(prior!.owner) && !activeNativeSessions.has(identity.runId) && !nativeSessionStartups.has(identity.runId); const profile = record(prior.run.runnerProfileJson); - const stopped: { allowed: boolean; reason?: string } = finishedHere() ? { allowed: true } : await evaluateNativeControllerTakeover({ owner: prior.owner, + const stopped: NativeControllerTakeoverDecision = finishedHere() ? { allowed: true, reason: "finished_here" } : await evaluateNativeControllerTakeover({ owner: prior.owner, now: new Date(), requireStoppedController: true, remoteLeaseFenced: readNativeControllerRecovery(profile.nativeControllerRecovery, profile.sessionCheckpoint) !== null }); if (!stopped.allowed) return refuse(`controller_${retirementReasonName("controllerReason", stopped.reason)}`); + await recordLegacyHostFallbackProof(input.db, prior.run, stopped, "native_retirement"); const provenFinishedHere = finishedHere(); const proveLocalRunnerStopped = async (): Promise => { const run = prior!.run; diff --git a/server/src/services/native-runtime/remote-warm-session-recovery.ts b/server/src/services/native-runtime/remote-warm-session-recovery.ts index 14bc3a791d7..40324702407 100644 --- a/server/src/services/native-runtime/remote-warm-session-recovery.ts +++ b/server/src/services/native-runtime/remote-warm-session-recovery.ts @@ -2,7 +2,7 @@ import { and, eq } from "drizzle-orm"; import { agents, companies, environmentLeases, heartbeatRuns, issues, nativeRunFinalizations, type Db } from "@paperclipai/db"; import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; import { parseNativeExecutionInput, type NativeExecutionInput, type PersistedNativeSession, type DurableRecoveryIdentity } from "../../vendor/paperclip-runner/index.js"; -import { currentNativeControllerIdentity, evaluateNativeControllerTakeover } from "./native-restart-recovery.js"; +import { currentNativeControllerIdentity, evaluateNativeControllerTakeover, recordLegacyHostFallbackProof } from "./native-restart-recovery.js"; import { NativeControllerRecoveryWaitingError, readNativeControllerRecovery } from "./native-controller-recovery.js"; export type RemoteWarmSessionRecovery = { @@ -143,6 +143,7 @@ export async function inspectRemoteWarmSessionRecovery(input: { const stopped = await evaluateNativeControllerTakeover({ owner: prior.owner, now: new Date(), requireStoppedController: true, remoteLeaseFenced: persistedRecovery !== null }); if (!stopped.allowed) return deny(); + await recordLegacyHostFallbackProof(input.db, prior.run, stopped, "remote_warm_session_recovery"); }; await assertPriorStopped(); const priorLeases = await input.db.select().from(environmentLeases).where(and( diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 7316605935c..63d9798ce47 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -109,10 +109,13 @@ import { isProcessGroupAlive, } from "../local-service-supervisor.js"; import { sameProcessSpace } from "../controller-process-identity.js"; +import { ephemeralServerEnabled } from "../ephemeral-server-mode.js"; +import { LEGACY_HOST_FALLBACK_EVENT } from "../native-runtime/native-restart-recovery.js"; import { redactSensitiveText } from "../../redaction.js"; import { isUniqueViolation } from "../../db-errors.js"; import { logActivity, + persistActivity, publishActivity, type ActivityPublication, } from "../activity-log.js"; @@ -6184,8 +6187,11 @@ export function recoveryService( // A recorded PID is judged only in the PID space that recorded it: a dead // local PID never proves that a process on another host, in another // namespace, or in a sandbox ("remote") is gone. Rows saved before the - // space was recorded keep the previous local proof for one release. - const processSpaceJudgeable = run.processHostHash == null || sameProcessSpace(run.processHostHash); + // space was recorded keep the previous local proof for one release, and + // only on a single-host server: an ephemeral server cannot assume the + // row was written here. Its operator path is the run's Stop control. + const legacyHostFallback = run.processHostHash == null && !ephemeralServerEnabled(); + const processSpaceJudgeable = legacyHostFallback || sameProcessSpace(run.processHostHash); if (!hasLiveExecution && processSpaceJudgeable) { if (typeof pid === "number" || typeof processGroupId === "number") { const processAlive = @@ -6331,6 +6337,17 @@ export function recoveryService( "failed to append recovery run event after terminalizing orphaned run; run stays terminal and the sweep clears the lock", ); } + // Counted by the release that removes the null-host fallback. Best-effort + // like the audit event above: it never holds the terminal write back. + if (authority === "process_gone" && legacyHostFallback) { + try { + await persistActivity(db, { companyId: run.companyId, actorType: "system", actorId: "orphan-reaper", + action: LEGACY_HOST_FALLBACK_EVENT, entityType: "heartbeat_run", entityId: run.id, runId: run.id, + agentId: run.agentId, details: { reason: "orphaned_running_run", source: "orphan_reaper" } }); + } catch (error) { + logger.error({ err: error, runId: run.id }, "failed to record the legacy host fallback event"); + } + } logger.warn( { runId: run.id, diff --git a/ui/src/api/issues.ts b/ui/src/api/issues.ts index 1e5a094076f..178c0a521a9 100644 --- a/ui/src/api/issues.ts +++ b/ui/src/api/issues.ts @@ -419,6 +419,8 @@ export const issuesApi = { `/issues/${id}/stalled-review-decision`, data, ), + confirmControllerStopped: (id: string, data: { actionId: string; runId: string; stopEvidence: string }) => + api.post<{ runId: string; status: "queued" }>(`/issues/${id}/recovery-actions/confirm-controller-stopped`, data), resumeWorkspaceFinalization: (id: string, data: { actionId: string; runId: string; ownerToken: string; controllerAndCopybackStopped: true; stopEvidence: string }) => api.post<{ runId: string; resultId: string | null; status: "queued" }>(`/issues/${id}/recovery-actions/resume-workspace-finalization`, data), retryWorkspaceExport: (id: string, data: { actionId: string; runId: string; repairNote: string }) => diff --git a/ui/src/components/WorkspaceExportRecovery.control.test.tsx b/ui/src/components/WorkspaceExportRecovery.control.test.tsx index 49ef3c4cb01..12dd3ff0008 100644 --- a/ui/src/components/WorkspaceExportRecovery.control.test.tsx +++ b/ui/src/components/WorkspaceExportRecovery.control.test.tsx @@ -9,9 +9,9 @@ import { queryKeys } from "../lib/queryKeys"; import { writePendingSend } from "../lib/pending-send"; import { WorkspaceExportRecovery } from "./WorkspaceExportRecovery"; -const api = vi.hoisted(() => ({ inspect: vi.fn(), recover: vi.fn(), resume: vi.fn() })); +const api = vi.hoisted(() => ({ inspect: vi.fn(), recover: vi.fn(), resume: vi.fn(), confirmController: vi.fn() })); vi.mock("../api/issues", () => ({ issuesApi: { recoveryActions: api.inspect, recoverWorkspace: api.recover, - resumeWorkspaceFinalization: api.resume } })); + resumeWorkspaceFinalization: api.resume, confirmControllerStopped: api.confirmController } })); (globalThis as unknown as { IS_REACT_ACT_ENVIRONMENT: boolean }).IS_REACT_ACT_ENVIRONMENT = true; const action = { @@ -473,6 +473,28 @@ describe("governed retained-file recovery", () => { expect(container.querySelector('[data-testid="retained-file-recovery"]')).not.toBeNull(); }); + it("confirms a pre-upgrade controller an ephemeral server cannot verify", async () => { + api.confirmController.mockResolvedValue({ runId: "run-1", status: "queued" }); + client.setQueryData(queryKeys.access.currentBoardAccess, { userId: "board-1", source: "session", isInstanceAdmin: false, + companyIds: ["company-1"], memberships: [{ companyId: "company-1", membershipRole: "owner", status: "active" }] }); + const controllerAction = { ...action, cause: "native_controller_host_unverified", ownerType: "board", + evidence: { runId: "run-1", controller: { bootId: "boot", pid: 7 } } } as unknown as IssueRecoveryAction; + await mount(controllerAction); + expect(container.textContent).toContain("before Paperclip stored which host it ran on"); + await click("Confirm the previous server stopped"); + const textarea = container.querySelector("textarea")!; + await act(async () => { + const setter = Object.getOwnPropertyDescriptor(HTMLTextAreaElement.prototype, "value")!.set!; + setter.call(textarea, "The old server container was replaced and is not running."); + textarea.dispatchEvent(new Event("input", { bubbles: true })); + }); + await click("Confirm and resume"); + expect(api.confirmController).toHaveBeenCalledExactlyOnceWith("issue-1", { actionId: "action-1", runId: "run-1", + stopEvidence: "The old server container was replaced and is not running." }); + expect(api.resume).not.toHaveBeenCalled(); + expect(container.textContent).toContain("Resuming the run…"); + }); + it("shows status only, without the action, once automatic recovery is resuming or for non-administrators", async () => { // The default fixture is an operator: it may resolve other recovery but not attest a controller stop. await mount({ ...ownerAction, evidence: { ...ownerAction.evidence, workspaceOwnerRecovery: { state: "resuming" } } } as IssueRecoveryAction); diff --git a/ui/src/components/WorkspaceExportRecovery.tsx b/ui/src/components/WorkspaceExportRecovery.tsx index e66e04707ad..03ebbc8377e 100644 --- a/ui/src/components/WorkspaceExportRecovery.tsx +++ b/ui/src/components/WorkspaceExportRecovery.tsx @@ -25,6 +25,14 @@ function ReportRecoveryControl({ owns, onChange }: { owns: boolean; onChange?: ( return null; } +const CONTROLLER_HOST_UNVERIFIED = "native_controller_host_unverified"; + +/** Causes whose one action is the board's previous-server stop confirmation; + * the generic recovery card must not offer other resolutions for them. */ +export function isPreviousServerStopCause(cause: string | null | undefined) { + return cause === "native_workspace_finalization_owner_unverified" || cause === CONTROLLER_HOST_UNVERIFIED; +} + /** Requests preservation; the existing worker owns stop, copyback and verification. */ export function WorkspaceExportRecovery({ action, className, issueId, companyId, showStatus = true, onRecoveryControlChange }: { className?: string; action: IssueRecoveryAction | null; issueId?: string; companyId?: string; showStatus?: boolean; @@ -44,6 +52,19 @@ export function WorkspaceExportRecovery({ action, className, issueId, companyId, action={action} issueId={issueId} companyId={companyId} className={className} onRecoveryControlChange={onRecoveryControlChange} />; } + // An ephemeral server cannot judge a pre-upgrade controller row by PID. + if (action?.cause === CONTROLLER_HOST_UNVERIFIED && issueId && companyId + && isControllerStopRequired(action, issueId, companyId)) { + return <> + issuesApi.confirmControllerStopped(issueId, { + actionId: action.id, runId: action.evidence.runId as string, stopEvidence })} + copy={{ waiting: "This run’s previous server was recorded before Paperclip stored which host it ran on, so Paperclip can’t verify that it stopped. The run’s saved state is kept.", + question: "How did you confirm that the previous server stopped?", + placeholder: "For example: the old server container was replaced and is no longer running.", + resuming: "Resuming the run…" }} />; + } const ownerRecovery = action?.cause === "native_workspace_finalization_owner_unverified"; // The task's interruption names this wait; the confirmation is its one action. if (ownerRecovery && issueId && companyId && isOwnerStopRequired(action, issueId, companyId)) { @@ -72,6 +93,12 @@ function isRetainedExport(action: IssueRecoveryAction, issueId: string, companyI && typeof action.evidence.runId === "string" && action.evidence.runId.length > 0; } +function isControllerStopRequired(action: IssueRecoveryAction, issueId: string, companyId: string) { + return action.companyId === companyId && action.sourceIssueId === issueId + && action.ownerType === "board" && ["active", "escalated"].includes(action.status) + && typeof action.evidence.runId === "string" && !action.evidence.controllerStop; +} + function isOwnerStopRequired(action: IssueRecoveryAction, issueId: string, companyId: string) { return action.companyId === companyId && action.sourceIssueId === issueId && action.ownerType === "board" && ["active", "escalated"].includes(action.status) @@ -89,8 +116,12 @@ function retainedOwnerStopToken(action: IssueRecoveryAction): string | null { /** A previous server that can no longer report cannot confirm its own stop. * The board confirms it once; the saved result then finishes without another agent turn. */ -function PreviousServerStopConfirmation({ action, ownerToken, issueId, companyId, className }: { - action: IssueRecoveryAction; ownerToken: string; issueId: string; companyId: string; className?: string; +function PreviousServerStopConfirmation({ action, ownerToken, issueId, companyId, className, confirmStop, copy }: { + action: IssueRecoveryAction; issueId: string; companyId: string; className?: string; + /** Saved-work copyback owner; used when no custom confirmation is given. */ + ownerToken?: string; + confirmStop?: (stopEvidence: string) => Promise; + copy?: { waiting: string; question: string; placeholder: string; resuming: string }; }) { const runId = action.evidence.runId as string; const client = useQueryClient(); @@ -101,8 +132,9 @@ function PreviousServerStopConfirmation({ action, ownerToken, issueId, companyId const accessView = useQueryView(access); const canConfirm = (accessView.kind === "ready" || accessView.kind === "stale") && canBoardConfirmOwnerStop(companyId, access.data); const confirm = useMutation({ retry: false, // query-error-ok: an explicit board confirmation; the error is shown below. - mutationFn: () => issuesApi.resumeWorkspaceFinalization(issueId, { actionId: action.id, runId, ownerToken, - controllerAndCopybackStopped: true, stopEvidence: evidence.trim() }), + mutationFn: () => confirmStop ? confirmStop(evidence.trim()) + : issuesApi.resumeWorkspaceFinalization(issueId, { actionId: action.id, runId, ownerToken: ownerToken!, + controllerAndCopybackStopped: true, stopEvidence: evidence.trim() }), onSuccess: () => { setOpen(false); void client.invalidateQueries({ queryKey: queryKeys.issues.detail(issueId) }); @@ -111,8 +143,8 @@ function PreviousServerStopConfirmation({ action, ownerToken, issueId, companyId } }); const ready = evidence.trim().length >= 20; return
-

{confirm.isSuccess ? "Restoring saved work…" - : "The previous server stopped while it was saving this task’s files, and Paperclip can’t verify that it stopped. The agent’s result is saved."}

+

{confirm.isSuccess ? copy?.resuming ?? "Restoring saved work…" + : copy?.waiting ?? "The previous server stopped while it was saving this task’s files, and Paperclip can’t verify that it stopped. The agent’s result is saved."}

{confirm.isSuccess ? null : !canConfirm ?

{accessView.kind === "loading" || accessView.kind === "reconnecting" ? "Checking access…" : "Paperclip is waiting to confirm that the previous server stopped."}

@@ -120,9 +152,9 @@ function PreviousServerStopConfirmation({ action, ownerToken, issueId, companyId Confirm the previous server stopped :
{ event.preventDefault(); if (ready && !confirm.isPending) confirm.mutate(); }}> + {copy?.question ?? "How did you confirm that the previous server and its file copy stopped?"}