diff --git a/doc/native-workspace-finalization-recovery.md b/doc/native-workspace-finalization-recovery.md index eaee145c11a..911815e092f 100644 --- a/doc/native-workspace-finalization-recovery.md +++ b/doc/native-workspace-finalization-recovery.md @@ -193,6 +193,28 @@ runs. Another agent's run and a recovery-action wake still prevent takeover. Publication checks repeat this validation, so a wake that starts during copyback stops publication. Row locks are never held across physical copyback I/O. +**Replaced deployment on a volume-backed service (automatic, no harness call).** +Railway mounts a service volume into one deployment at a time: it stops the +previous deployment before the new one starts, and a service with a volume has +no replicas. When this server runs on a Railway service with an attached volume +(`RAILWAY_VOLUME_ID` and `RAILWAY_VOLUME_MOUNT_PATH`), and the receipt names the +same project, environment and service with a different deployment ID, the +shared owner-liveness primitive (`proveOwnerGone`) returns `gone` with reason +`deployment_replaced`. That deployment's controller and every copyback child +in it have exited, on whichever host they ran. The finalizer then claims the +exact receipt as it does for a harness receipt, logs +`issue.workspace_owner_stop_verified` with `deploymentProof`, marks the +recovery `resuming`, and resumes the save with no provider turn. The same +deployment ID, another service or environment, or a server without a volume +gives no proof; the harness request below still applies. New receipts also +record the mounted volume ID, and the proof then requires this server to mount +that same volume. Receipts saved by earlier releases already record the +deployment, so they take this path after an upgrade. The finalizer reaches this +check only after it holds the run's copyback advisory lock, so the previous +owner's lock session has also ended. A wait that this rule and the harness +cannot settle names the Resume saved result action; its text never asks a +person to inspect a server. + The harness endpoint is `POST /v1/tenant/workspace-worker-stop`. The fleet-wide server token authorizes a read request, never stop evidence: responses must be signed by the harness. This endpoint requires the companion cloud harness change. diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.ts index a77ef522f1c..8b3d6254e3d 100644 --- a/server/src/routes/issues.ts +++ b/server/src/routes/issues.ts @@ -10308,7 +10308,7 @@ export function issueRoutes( } if ((outcome === "restored" || outcome === "false_positive") && activeRecoveryAction.cause === "native_workspace_finalization_owner_unverified") { - throw conflict("Verify the previous controller and its copyback processes stopped, then use Resume saved result.", { code: "workspace_owner_stop_required" }); + throw conflict("Paperclip is still confirming that the previous server stopped. Saving resumes without repeating the agent's work.", { code: "workspace_owner_stop_required" }); } if (outcome === "restored" && isNativeWorkspaceExportRepairCause(activeRecoveryAction.cause)) { diff --git a/server/src/services/explicit-native-continuation.ts b/server/src/services/explicit-native-continuation.ts index d48da1101de..c42208d44fe 100644 --- a/server/src/services/explicit-native-continuation.ts +++ b/server/src/services/explicit-native-continuation.ts @@ -415,7 +415,7 @@ export async function admitExplicitNativeContinuation(input: { sql`${issueRecoveryActions.evidence}->>'runId' = ${run.id}`, sql`${issueRecoveryActions.evidence} ? 'systemRecovery'`)); if (live.some(row => row.ownerHold)) - return blocked("workspace_owner_stop_required", "A board member must confirm that the previous server stopped copying files. Your Retry is saved."); + return blocked("workspace_owner_stop_required", "Paperclip is waiting to confirm that the previous server stopped saving files. Your Retry is saved."); // A spent recovery budget finishes nothing by itself. Another Retry // starts one new recovery generation (task-retry), so name that action // instead of an endless finalizing wait (RC-50b, s14). diff --git a/server/src/services/native-runtime/native-workspace-finalization-ownership.ts b/server/src/services/native-runtime/native-workspace-finalization-ownership.ts index 24827baeb57..fd355c38d94 100644 --- a/server/src/services/native-runtime/native-workspace-finalization-ownership.ts +++ b/server/src/services/native-runtime/native-workspace-finalization-ownership.ts @@ -7,7 +7,7 @@ import { beginRunPhase } from "../execution-preparation-timing.js"; import { logActivity } from "../activity-log.js"; import { issueRecoveryActionService } from "../issue-recovery-actions.js"; import { nativeSha256 } from "./canonical.js"; -import { NATIVE_WORKSPACE_OWNER_UNVERIFIED } from "./native-workspace-finalization-recovery-state.js"; +import { NATIVE_WORKSPACE_OWNER_UNVERIFIED, NATIVE_WORKSPACE_OWNER_WAIT_NEXT_ACTION } from "./native-workspace-finalization-recovery-state.js"; import { randomUUID } from "node:crypto"; import os from "node:os"; import { and, eq, inArray, sql } from "drizzle-orm"; @@ -22,6 +22,7 @@ import { import { currentWorkspaceWorker, requestWorkspaceWorkerStopProof, type WorkspaceWorker, type WorkspaceWorkerStopProof } from "./native-workspace-platform-proof.js"; import { currentProcessGroupId, proveLocalWorkspaceOwnerStopped, type LocalWorkspaceOwnerStop } from "./native-workspace-owner-stop-proof.js"; +import { currentDeploymentIdentity, proveOwnerGone } from "../owner-proof.js"; const OWNER_KEY = "nativeWorkspaceFinalizationOwner"; @@ -48,6 +49,10 @@ type Owner = { /** Copyback children inherit this group; a successor uses it to prove they stopped. */ processGroupId?: number; worker?: WorkspaceWorker; + /** The platform volume this deployment mounted; it binds a later deployment proof. */ + volumeId?: string; + /** Recorded on Railway receipts; false keeps a later deployment proof from applying. */ + exclusiveVolume?: boolean; }; export type NativeWorkspaceFinalizationWaitReason = | "busy" @@ -104,10 +109,14 @@ export async function withNativeWorkspaceFinalizationOwnership( ): Promise<{ acquired: false; reason: NativeWorkspaceFinalizationWaitReason } | { acquired: true; value: T }> { const identity = await currentNativeControllerIdentity(); const processGroupId = await currentProcessGroupId(); + const deployment = currentDeploymentIdentity(); const owner: Owner = { token: randomUUID(), controllerBootId: identity.bootId, hostname: os.hostname(), pid: identity.pid, processStartedAt: identity.processStartedAt.toISOString(), ...(processGroupId ? { processGroupId } : {}), worker: currentWorkspaceWorker(), + // Record a volume only when this deployment is volume-exclusive by the same rule the proof uses. + ...(deployment ? { exclusiveVolume: deployment.exclusiveVolume } : {}), + ...(deployment?.exclusiveVolume && deployment.volumeId ? { volumeId: deployment.volumeId } : {}), }; const scope = and(eq(heartbeatRuns.id, input.runId), eq(heartbeatRuns.companyId, input.companyId), input.retainedRepairActionId ? inArray(heartbeatRuns.runtimeMode, ["native", "legacy"]) : eq(heartbeatRuns.runtimeMode, "native")); @@ -168,13 +177,14 @@ export async function withNativeWorkspaceFinalizationOwnership( let platformStopPending = false; let localProof: Extract["proof"] | null = null; let localStopPending = false; + let deploymentProof: { reason: "deployment_replaced"; previousDeploymentId: string; worker: WorkspaceWorker } | null = null; const waitReason = () => platformStopPending ? "platform_stop_pending" as const : localStopPending ? "local_stop_pending" as const : "owner_unverified" as const; if (rawPrior != null) { const requireStopVerification = async () => { if (input.retainedRepairActionId) return { acquired: false, reason: "owner_unverified" } as const; const fingerprint = nativeSha256({ runId: input.runId, owner: rawPrior }); - const nextAction = "Verify the previous controller and its workspace-copyback processes have stopped, then release only this exact workspace owner receipt using doc/native-workspace-finalization-recovery.md. Resume workspace finalization without another provider turn."; + const nextAction = NATIVE_WORKSPACE_OWNER_WAIT_NEXT_ACTION; const recovery = issueRecoveryActionService(input.db); const existing = run.issueId ? await recovery.getActiveForIssue(input.companyId, run.issueId) : null; // Rechecking the same physical owner is not a new recovery attempt. @@ -211,16 +221,27 @@ export async function withNativeWorkspaceFinalizationOwnership( : activeRunOwners.get(runKey)?.db === input.db ? "owner_active" : "busy" } as const; } if (prior?.worker) { - const check = await requestWorkspaceWorkerStopProof({ runId: input.runId, ownerToken: prior.token, worker: prior.worker }); - platformProof = check.status === "verified" ? check.proof : null; - platformStopPending = check.status === "pending"; + // A restart onto a newer deployment of a volume-backed service is + // proof by itself: the platform stopped the old deployment first. + const replaced = await proveOwnerGone({ kind: "local", scope: "native_workspace_finalization", generation: null, + host: prior.hostname, processSpace: null, bootId: prior.controllerBootId ?? null, pid: prior.pid, + processStartedAt: prior.processStartedAt, worker: prior.worker, + volumeId: typeof prior.volumeId === "string" ? prior.volumeId : null, + exclusiveVolume: typeof prior.exclusiveVolume === "boolean" ? prior.exclusiveVolume : null }); + if (replaced.verdict === "gone" && replaced.reason === "deployment_replaced") + deploymentProof = { reason: "deployment_replaced", previousDeploymentId: prior.worker.deploymentId, worker: prior.worker }; + if (!deploymentProof) { + const check = await requestWorkspaceWorkerStopProof({ runId: input.runId, ownerToken: prior.token, worker: prior.worker }); + platformProof = check.status === "verified" ? check.proof : null; + platformStopPending = check.status === "pending"; + } } if (!prior) return requireStopVerification(); - if (!platformProof && prior.hostname !== owner.hostname) return requireStopVerification(); + if (!platformProof && !deploymentProof && prior.hostname !== owner.hostname) return requireStopVerification(); const knownJoinedHere = prior.controllerBootId === identity.bootId && prior.pid === identity.pid && joinedOwnerTokens.has(prior.token) && !activeOwnerTokens.has(prior.token); - if (!platformProof && !knownJoinedHere) { + if (!platformProof && !deploymentProof && !knownJoinedHere) { const takeover = await evaluateNativeControllerTakeover({ owner: { leaseOwner: prior.token, leaseExpiresAt: new Date(0), controllerPid: prior.pid, controllerProcessStartedAt: new Date(prior.processStartedAt) }, now: new Date(), @@ -265,7 +286,7 @@ export async function withNativeWorkspaceFinalizationOwnership( }; // A stop proof (platform or local) shows the old worker exited, not that its // task is still current. Revalidate after the proof and atomically with the CAS. - const claimedRun = platformProof || localProof ? await input.db.transaction(async tx => { + const claimedRun = platformProof || localProof || deploymentProof ? await input.db.transaction(async tx => { const result = await currentRecoveryResult(tx as unknown as Db); if (!result) return null; const claimedRun = await claimOwner(tx as unknown as Db); @@ -280,10 +301,10 @@ export async function withNativeWorkspaceFinalizationOwnership( claimed = false; return { acquired: false, reason: "binding_changed" } as const; } - if ((platformProof || localProof) && run.issueId) await logActivity(input.db, { + if ((platformProof || localProof || deploymentProof) && run.issueId) await logActivity(input.db, { companyId: input.companyId, actorType: "system", actorId: "workspace-recovery", action: "issue.workspace_owner_stop_verified", entityType: "issue", entityId: run.issueId, - runId: input.runId, details: platformProof ? { proof: platformProof } : { localProof }, + runId: input.runId, details: platformProof ? { proof: platformProof } : deploymentProof ? { deploymentProof } : { localProof }, }); if (prior) joinedOwnerTokens.delete(prior.token); const ownership: NativeWorkspaceFinalizationOwnership = { @@ -304,7 +325,7 @@ export async function withNativeWorkspaceFinalizationOwnership( }, }; await ownership.assertHeld(); - if ((platformProof || localProof) && run.issueId) await input.db.update(issueRecoveryActions).set({ + if ((platformProof || localProof || deploymentProof) && run.issueId) await input.db.update(issueRecoveryActions).set({ evidence: sql`jsonb_set(coalesce(${issueRecoveryActions.evidence}, '{}'::jsonb), '{workspaceOwnerRecovery}', '{"state":"resuming"}'::jsonb)`, updatedAt: new Date(), }).where(and( diff --git a/server/src/services/native-runtime/native-workspace-finalization-recovery-state.ts b/server/src/services/native-runtime/native-workspace-finalization-recovery-state.ts index a4798b23e55..dc686d50824 100644 --- a/server/src/services/native-runtime/native-workspace-finalization-recovery-state.ts +++ b/server/src/services/native-runtime/native-workspace-finalization-recovery-state.ts @@ -4,6 +4,8 @@ import { isUuidLike, type IssueRecoveryAction } from "@paperclipai/shared"; import { nativeSha256 } from "./canonical.js"; export const NATIVE_WORKSPACE_OWNER_UNVERIFIED = "native_workspace_finalization_owner_unverified"; +/** Describes the wait. It never asks a person to inspect a server or sandbox. */ +export const NATIVE_WORKSPACE_OWNER_WAIT_NEXT_ACTION = "Paperclip is holding this saved result until it can confirm that the previous server stopped, so the work is not saved twice. Saving then resumes without repeating the agent's work. If this platform cannot confirm it, an owner or administrator can resume the save from this task (Resume saved result)."; export function hasNativeWorkspaceFinalizationOwner(profile: Record | null | undefined): boolean { return profile?.nativeWorkspaceFinalizationOwner != null; diff --git a/server/src/services/native-runtime/native-workspace-finalizer.test.ts b/server/src/services/native-runtime/native-workspace-finalizer.test.ts index a60a0f05471..6897b7637b6 100644 --- a/server/src/services/native-runtime/native-workspace-finalizer.test.ts +++ b/server/src/services/native-runtime/native-workspace-finalizer.test.ts @@ -205,7 +205,7 @@ describe("native workspace finalization recovery", () => { await db.update(issueRecoveryActions).set({ nextAction: "Start a replacement run" }).where(eq(issueRecoveryActions.id, seed.actionId)); await withNativeWorkspaceFinalizationOwnership(seed, async () => "unexpected"); const [repaired] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, seed.actionId)); - expect(repaired.nextAction).toContain("workspace-copyback processes have stopped"); + expect(repaired.nextAction).toContain("confirm that the previous server stopped"); expect(repaired.attemptCount).toBe(action.attemptCount); expect(repaired.evidence?.recoveryBudget).toEqual(action.evidence?.recoveryBudget); }); @@ -703,7 +703,8 @@ describe("native workspace finalization recovery", () => { expect(work).not.toHaveBeenCalled(); const [action] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, seeded.issueId)); expect(action).toMatchObject({ ownerType: "board", cause: "native_workspace_finalization_owner_unverified", wakePolicy: null, evidence: { runId: seeded.runId, owner } }); - expect(action.nextAction).toContain("previous controller"); + expect(action.nextAction).toContain("confirm that the previous server stopped"); + expect(action.nextAction).not.toMatch(/\bverify\b/i); await expect(withNativeWorkspaceFinalizationOwnership({ db, companyId: foreignCompanyId, runId: seeded.runId }, work)) .rejects.toThrow("native_workspace_finalization_binding_missing"); const [held] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, seeded.runId)); diff --git a/server/src/services/native-runtime/native-workspace-owner-restart-recovery.test.ts b/server/src/services/native-runtime/native-workspace-owner-restart-recovery.test.ts index 297946aa2db..a4d41b06f7c 100644 --- a/server/src/services/native-runtime/native-workspace-owner-restart-recovery.test.ts +++ b/server/src/services/native-runtime/native-workspace-owner-restart-recovery.test.ts @@ -12,7 +12,7 @@ import { randomUUID } from "node:crypto"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { and, eq } from "drizzle-orm"; import { activityLog, agentWakeupRequests, agents, companies, completionContracts, createDb, heartbeatRuns, @@ -174,4 +174,118 @@ describe("native workspace owner recovery after a restart", () => { expect(await withNativeWorkspaceFinalizationOwnership({ db, companyId, runId: seeded.runId }, async () => "must not run")) .toEqual({ acquired: false, reason: "owner_unverified" }); }); + + // Staging 2026-11-07: a deploy restarted the server while three accepted + // results were being saved. The receipts name the previous Railway deployment + // on another container (e2d490c61 shape). A volume-backed service cannot run + // two deployments at once, so this server running on a newer deployment of the + // same service proves the previous one, and all its copyback children, exited. + describe("after a deploy on a volume-backed service", () => { + const worker = { provider: "railway" as const, projectId: randomUUID(), environmentId: randomUUID(), + serviceId: randomUUID(), deploymentId: randomUUID() }; + function currentDeployment(overrides: Partial> = {}) { + const env = { RAILWAY_PROJECT_ID: worker.projectId, RAILWAY_ENVIRONMENT_ID: worker.environmentId, + RAILWAY_SERVICE_ID: worker.serviceId, RAILWAY_DEPLOYMENT_ID: randomUUID(), + RAILWAY_VOLUME_ID: randomUUID(), RAILWAY_VOLUME_MOUNT_PATH: "/paperclip", ...overrides }; + for (const [key, value] of Object.entries(env)) vi.stubEnv(key, value ?? ""); + } + afterEach(() => vi.unstubAllEnvs()); + + async function previousDeploymentOwner() { + const seed = await seedAbandonedCopyback(); + const owner = { pid: 3, token: randomUUID(), worker, hostname: "3f58b3426ddc", + controllerBootId: randomUUID(), processStartedAt: "2026-11-07T16:32:33.070Z" }; + await db.update(heartbeatRuns).set({ runnerProfileJson: { nativeWorkspaceFinalizationOwner: owner, preserved: "workspace descriptor" } }) + .where(eq(heartbeatRuns.id, seed.runId)); + await db.update(issueRecoveryActions).set({ fingerprint: nativeSha256({ runId: seed.runId, owner }), + evidence: { runId: seed.runId, owner, workspaceOwnerRecovery: { state: "operator_required" } } }) + .where(eq(issueRecoveryActions.id, seed.actionId)); + return { ...seed, owner }; + } + + it("resumes the interrupted save automatically, with no board action", async () => { + currentDeployment(); + const seed = await previousDeploymentOwner(); + const [issue] = await db.select().from(issues).where(eq(issues.id, seed.issueId)); + const work = vi.fn(async () => { + const [recovery] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, seed.actionId)); + expect(recovery.evidence?.workspaceOwnerRecovery).toEqual({ state: "resuming" }); + expect(await taskInterruptionPresentation(db, issue)).toMatchObject({ responsibility: "system", userAction: null }); + return "saved workspace copied"; + }); + expect(await withNativeWorkspaceFinalizationOwnership(seed, work)).toEqual({ acquired: true, value: "saved workspace copied" }); + expect(work).toHaveBeenCalledOnce(); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, seed.runId)); + expect(run.runnerProfileJson?.nativeWorkspaceFinalizationOwner).toBeUndefined(); + expect(run.runnerProfileJson?.preserved).toBe("workspace descriptor"); + const verified = await db.select().from(activityLog).where(and(eq(activityLog.entityId, seed.issueId), + eq(activityLog.action, "issue.workspace_owner_stop_verified"))); + expect(verified).toHaveLength(1); + expect(verified[0].details).toMatchObject({ deploymentProof: { reason: "deployment_replaced", + previousDeploymentId: worker.deploymentId } }); + expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId))).toHaveLength(0); + }); + + it("records the volume on a new receipt only when the volume is fully configured", async () => { + const seed = await seedAbandonedCopyback(); + await db.update(heartbeatRuns).set({ runnerProfileJson: {} }).where(eq(heartbeatRuns.id, seed.runId)); + currentDeployment({ RAILWAY_VOLUME_ID: "half-configured", RAILWAY_VOLUME_MOUNT_PATH: "" }); + let recorded: Record | undefined; + await withNativeWorkspaceFinalizationOwnership(seed, async () => { + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, seed.runId)); + recorded = run.runnerProfileJson?.nativeWorkspaceFinalizationOwner as Record; + }); + expect(recorded).toBeDefined(); + expect(recorded!.volumeId).toBeUndefined(); + expect(recorded!.exclusiveVolume).toBe(false); + await db.update(heartbeatRuns).set({ runnerProfileJson: {} }).where(eq(heartbeatRuns.id, seed.runId)); + currentDeployment({ RAILWAY_VOLUME_ID: "mounted-volume" }); + await withNativeWorkspaceFinalizationOwnership(seed, async () => { + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, seed.runId)); + recorded = run.runnerProfileJson?.nativeWorkspaceFinalizationOwner as Record; + }); + expect(recorded!.volumeId).toBe("mounted-volume"); + expect(recorded!.exclusiveVolume).toBe(true); + }); + + it("keeps a receipt from a deployment that was not volume-exclusive fenced", async () => { + currentDeployment(); + const seed = await previousDeploymentOwner(); + const owner = { ...seed.owner, exclusiveVolume: false }; + await db.update(heartbeatRuns).set({ runnerProfileJson: { nativeWorkspaceFinalizationOwner: owner } }) + .where(eq(heartbeatRuns.id, seed.runId)); + const work = vi.fn(async () => "second physical writer"); + expect(await withNativeWorkspaceFinalizationOwnership(seed, work)).toEqual({ acquired: false, reason: "owner_unverified" }); + expect(work).not.toHaveBeenCalled(); + }); + + it("keeps the save fenced when the receipt names a different volume", async () => { + currentDeployment(); + const seed = await previousDeploymentOwner(); + const owner = { ...seed.owner, volumeId: randomUUID() }; + await db.update(heartbeatRuns).set({ runnerProfileJson: { nativeWorkspaceFinalizationOwner: owner } }) + .where(eq(heartbeatRuns.id, seed.runId)); + const work = vi.fn(async () => "second physical writer"); + expect(await withNativeWorkspaceFinalizationOwnership(seed, work)).toEqual({ acquired: false, reason: "owner_unverified" }); + expect(work).not.toHaveBeenCalled(); + const [action] = await db.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.sourceIssueId, seed.issueId), + eq(issueRecoveryActions.fingerprint, nativeSha256({ runId: seed.runId, owner })))); + expect(action.nextAction).toContain("Resume saved result"); + expect(action.nextAction).not.toMatch(/\bverify\b|\binspect\b/i); + }); + + it.each([ + ["the same deployment", () => ({ RAILWAY_DEPLOYMENT_ID: worker.deploymentId })], + ["a service without a volume", () => ({ RAILWAY_VOLUME_ID: "", RAILWAY_VOLUME_MOUNT_PATH: "" })], + ["another service", () => ({ RAILWAY_SERVICE_ID: randomUUID() })], + ["another environment", () => ({ RAILWAY_ENVIRONMENT_ID: randomUUID() })], + ["a half-configured volume", () => ({ RAILWAY_VOLUME_MOUNT_PATH: "" })], + ] as const)("keeps the save fenced on %s", async (_case, overrides) => { + currentDeployment(overrides()); + const seed = await previousDeploymentOwner(); + const work = vi.fn(async () => "second physical writer"); + expect(await withNativeWorkspaceFinalizationOwnership(seed, work)).toEqual({ acquired: false, reason: "owner_unverified" }); + expect(work).not.toHaveBeenCalled(); + }); + }); }); diff --git a/server/src/services/owner-proof.test.ts b/server/src/services/owner-proof.test.ts index d3ce7eb88d7..f5fdc9620de 100644 --- a/server/src/services/owner-proof.test.ts +++ b/server/src/services/owner-proof.test.ts @@ -3,6 +3,7 @@ import { currentProcessStartedAtMs, processExists, resetCurrentProcessStartedAtF import * as hotRestart from "./hot-restart.js"; import { currentLocalOwnerStamp, + currentDeploymentIdentity, currentProcessProbe, proveOwnerGone, type LocalOwnerStamp, @@ -169,6 +170,52 @@ describe("proveOwnerGone: local owners", () => { }); }); +describe("proveOwnerGone: replaced platform deployment", () => { + const worker = { provider: "railway" as const, projectId: "project", environmentId: "env", serviceId: "service", deploymentId: "old" }; + const current = (overrides: Partial = {}, exclusiveVolume = true) => + probe({ host: "new-container", processSpace: "space-new", deployment: { worker: { ...worker, deploymentId: "new", ...overrides }, exclusiveVolume } }); + + it("proves an owner on an earlier deployment of the same volume-backed service gone, on any host", async () => { + expect(await proveOwnerGone(stamp({ worker, host: "old-container", processSpace: null }), current())) + .toEqual({ verdict: "gone", reason: "deployment_replaced" }); + }); + + it("never treats this process's own boot as replaced", async () => { + expect(await proveOwnerGone(stamp({ worker, bootId: "current-boot" }), current())) + .toEqual({ verdict: "alive", reason: "current_process" }); + }); + + it("is no proof without an exclusive volume, for the same deployment, or for another service or environment", async () => { + const old = stamp({ worker, host: "old-container" }); + expect((await proveOwnerGone(old, current({}, false))).verdict).toBe("unprovable"); + expect((await proveOwnerGone(old, current({ deploymentId: "old" }))).verdict).toBe("unprovable"); + expect((await proveOwnerGone(old, current({ serviceId: "other" }))).verdict).toBe("unprovable"); + expect((await proveOwnerGone(old, current({ environmentId: "other" }))).verdict).toBe("unprovable"); + expect((await proveOwnerGone(stamp({ host: "old-container" }), current())).verdict).toBe("unprovable"); + }); + + it("reads the deployment and volume from the Railway environment", () => { + const env = { RAILWAY_PROJECT_ID: "p", RAILWAY_ENVIRONMENT_ID: "e", RAILWAY_SERVICE_ID: "s", RAILWAY_DEPLOYMENT_ID: "d" }; + expect(currentDeploymentIdentity(env)).toEqual({ worker: { provider: "railway", projectId: "p", environmentId: "e", + serviceId: "s", deploymentId: "d" }, exclusiveVolume: false, volumeId: null }); + expect(currentDeploymentIdentity({ ...env, RAILWAY_VOLUME_ID: "v", RAILWAY_VOLUME_MOUNT_PATH: "/data" })?.exclusiveVolume).toBe(true); + expect(currentDeploymentIdentity({})).toBeNull(); + // A half-configured volume is not exclusivity: fall back to other proof. + expect(currentDeploymentIdentity({ ...env, RAILWAY_VOLUME_ID: "v" })?.exclusiveVolume).toBe(false); + expect(currentDeploymentIdentity({ ...env, RAILWAY_VOLUME_MOUNT_PATH: "/data" })?.exclusiveVolume).toBe(false); + }); + + it("binds a stamp that recorded its volume to that same volume", async () => { + const withVolume = (volumeId: string) => probe({ host: "new-container", deployment: { + worker: { ...worker, deploymentId: "new" }, exclusiveVolume: true, volumeId } }); + expect(await proveOwnerGone(stamp({ worker, volumeId: "vol-a" }), withVolume("vol-a"))) + .toEqual({ verdict: "gone", reason: "deployment_replaced" }); + expect((await proveOwnerGone(stamp({ worker, host: "old", volumeId: "vol-a" }), withVolume("vol-b"))).verdict).toBe("unprovable"); + // A stamp from a deployment that was not volume-exclusive never qualifies. + expect((await proveOwnerGone(stamp({ worker, host: "old", exclusiveVolume: false }), withVolume("vol-a"))).verdict).toBe("unprovable"); + }); +}); + describe("proveOwnerGone: remote owners (fenced handoff)", () => { const remote = (overrides: Partial = {}): RemoteOwnerStamp => ({ kind: "remote", scope: "test", generation: 2, fenceToken: "boot:2:token", provider: "daytona", ...overrides, diff --git a/server/src/services/owner-proof.ts b/server/src/services/owner-proof.ts index 3093c8dd2dc..25f5bdb81f4 100644 --- a/server/src/services/owner-proof.ts +++ b/server/src/services/owner-proof.ts @@ -9,6 +9,7 @@ import { PROCESS_START_PRECISION_MS, UNKNOWN_PROCESS_SPACE, } from "./controller-process-identity.js"; +import { currentWorkspaceWorker, type WorkspaceWorker } from "./native-runtime/native-workspace-platform-proof.js"; /** * The owner recorded on an in-progress durable record (plan section A). @@ -35,6 +36,12 @@ export type LocalOwnerStamp = { /** Start time of the process group's leader (PID = `processGroupId`). When * the owner leads its own group this is `processStartedAt`. */ processGroupStartedAt?: string | null; + /** The platform deployment that ran the owner, when the platform names one. */ + worker?: WorkspaceWorker | null; + /** The platform volume mounted by that deployment, when the stamp recorded one. */ + volumeId?: string | null; + /** False when the owner's deployment was not volume-exclusive. Missing on older stamps. */ + exclusiveVolume?: boolean | null; }; export type RemoteOwnerStamp = { @@ -57,6 +64,7 @@ export type OwnerProofReason = | "fence_superseded" | "generation_superseded" | "provider_stopped" + | "deployment_replaced" // alive | "current_process" | "pid_alive" @@ -94,6 +102,10 @@ export type ProcessProbe = { pidExists(pid: number): boolean; processGroupExists(processGroupId: number): boolean; readStartedAt(pid: number): Promise; + /** This process's platform deployment. `exclusiveVolume` is true only when + * the platform stops every earlier deployment of the service before it + * starts this one (a Railway service with an attached volume). */ + deployment?: { worker: WorkspaceWorker; exclusiveVolume: boolean; volumeId?: string | null } | null; }; function processGroupExists(processGroupId: number): boolean { @@ -107,8 +119,19 @@ function processGroupExists(processGroupId: number): boolean { } } +/** Railway mounts a volume into one deployment of a service at a time: it stops + * the previous deployment before the new one starts, so the two never overlap. + * A service with a volume cannot have replicas. */ +export function currentDeploymentIdentity(env: NodeJS.ProcessEnv = process.env): ProcessProbe["deployment"] { + const worker = currentWorkspaceWorker(env); + if (!worker) return null; + const volumeId = env.RAILWAY_VOLUME_ID?.trim() || null; + return { worker, exclusiveVolume: Boolean(volumeId && env.RAILWAY_VOLUME_MOUNT_PATH?.trim()), volumeId }; +} + export function currentProcessProbe(): ProcessProbe { return { + deployment: currentDeploymentIdentity(), bootId: currentControllerBootId(), host: hostname(), processSpace: currentProcessSpaceHash(), @@ -182,6 +205,9 @@ export async function proveOwnerGone( async function proveLocalOwnerGone(stamp: LocalOwnerStamp, probe: ProcessProbe): Promise { if (stamp.bootId && stamp.bootId === probe.bootId) return { verdict: "alive", reason: "current_process" }; + // The platform stopped the owner's whole deployment, with every process and + // orphaned child in it, before it started this one. Host and PID are moot. + if (deploymentReplaced(stamp, probe.deployment)) return { verdict: "gone", reason: "deployment_replaced" }; if (!validPid(stamp.pid) || !stamp.processSpace) return { verdict: "unprovable", reason: "missing_identity" }; if (stamp.processSpace === UNKNOWN_PROCESS_SPACE || probe.processSpace === UNKNOWN_PROCESS_SPACE) { return { verdict: "unprovable", reason: "unknown_process_space" }; @@ -224,6 +250,20 @@ async function proveLocalOwnerGone(stamp: LocalOwnerStamp, probe: ProcessProbe): return { verdict: "gone", reason: processReason }; } +function deploymentReplaced(stamp: LocalOwnerStamp, current: ProcessProbe["deployment"]): boolean { + const stamped = stamp.worker; + if (!stamped || !current?.exclusiveVolume) return false; + // A stamp that recorded its volume must name this same volume. Older stamps + // predate the field; the service and environment then name the volume. + if (stamp.exclusiveVolume === false) return false; + if (stamp.volumeId != null && stamp.volumeId !== current.volumeId) return false; + const worker = current.worker; + return stamped.provider === worker.provider && stamped.projectId === worker.projectId + && stamped.environmentId === worker.environmentId && stamped.serviceId === worker.serviceId + && typeof stamped.deploymentId === "string" && stamped.deploymentId.length > 0 + && stamped.deploymentId !== worker.deploymentId; +} + /** "same" within `ps lstart` precision (1 s), "later" when the observed process * started definitely after the recorded one, "earlier" when definitely before, * "unknown" when either time is unreadable. */ diff --git a/server/src/services/task-interruption-presentation.ts b/server/src/services/task-interruption-presentation.ts index 6520d24f37d..e99c2a3bcb5 100644 --- a/server/src/services/task-interruption-presentation.ts +++ b/server/src/services/task-interruption-presentation.ts @@ -112,7 +112,7 @@ const retryWaitMessages: Record = { // Another click cannot release a copyback owner that stopped without reporting. // Retry starts a new recovery generation; the agent does not repeat its work. saved_work_recovery_failed: "Saving the agent's files failed after automatic retries. Retry to try again; the agent will not repeat its work.", - workspace_owner_stop_required: "Retry is waiting for a board member to confirm that the previous server stopped copying files. The agent's result and your Retry are saved.", + workspace_owner_stop_required: "Retry is waiting until Paperclip confirms that the previous server stopped saving files. The agent's result and your Retry are saved.", cleanup_quarantined: "Retry cannot continue this attempt. Send a new message after the previous execution has stopped.", cancelled_by_operator: "This attempt was stopped. Send a new message to continue from saved work.", // A new click replaces a Retry declined for its requester's access with a