diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 6feaf0177f2..ea215ff7344 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1886,7 +1886,10 @@ on the database clock (90 s TTL). use and extends the lease, but only if the boot stamped ownership in the last 90 s or still has an in-progress record stamped with its boot id: an environment lease or cleanup claim, a sandbox creation intent, a workspace - finalize attempt, or an agent lifecycle step. Otherwise the lease lapses and + finalize attempt, an agent lifecycle step, a native finalizer or cleanup + lease, an open native coordinator claim, or a running legacy run. A stamp + whose ownership write may still be pending also counts until the server's + tracked work settles (at most 10 minutes). Otherwise the lease lapses and the worker issues no statements until the next stamp, so an idle server's database can sleep. - **Renewal failure.** Each renewal has a 10 s bound. A failed or timed-out @@ -1911,9 +1914,33 @@ judged through that boot's row: - no row (a boot from before the lease): the record stays unprovable. A record with a recorded process group stays unprovable across hosts, because -agent processes may outlive their server. The lease is off on Railway, which -proves a replaced deployment from its own identity, and when -`PAPERCLIP_CONTROLLER_INSTANCE_LEASE=off`. +agent processes may outlive their server. The lease is off on a Railway +deployment with an exclusive volume, which proves a replaced deployment from its +own identity. A stateless Railway service keeps it. +`PAPERCLIP_CONTROLLER_INSTANCE_LEASE=on|off` overrides this. + +A record saved before its owner was fully recorded (`no_owner_stamp`, +`missing_identity`, `process_space_unrecorded`) stays unprovable and is never +released on its own. For a settling kind that opts in, recovery asks the board +on the record's task (`ownership_owner_unprovable`) to confirm that the previous +server stopped. The board action (`POST /api/issues/{id}/recovery-actions/confirm-owner-gone`) +is board-only, needs `runtime:confirm_owner_stop`, writes an +`issue.ownership_owner_gone_confirmed` activity entry and wakes the reconciler. +The confirmation then proves the owner gone (`operator_attested`) and the kind's +own compare-and-swap settle releases the record: + +- The environment run lease stores the confirmation on the lease. +- Every other opted-in kind (legacy controller lease, teardown claim, sandbox + creation intent, runtime-control operation, workspace finalize attempt, + native finalizer and cleanup leases) keeps it on the resolved recovery action, + bound to the record's last write. It applies only while the record is + unchanged and its recorded process is still unprovable for that legacy reason + (`missing_identity`, `process_space_unrecorded`). A record that changed since + is judged again, and recovery asks again about its current version. These + settles compare-and-swap on the recorded process, so a record of these kinds + with no owner stamp at all keeps its existing recovery path and is not offered. +- Kinds that are unowned by design, such as queued pending cleanup and + accounting debt, are never offered. The stranded-work pass closes the gap where issue state survives a crash but the wake/run path does not. The silent-run scan covers the separate case where a live process exists but has stopped producing observable output. diff --git a/packages/shared/src/validators/issue.ts b/packages/shared/src/validators/issue.ts index 7349a2ffbce..68ed55331c8 100644 --- a/packages/shared/src/validators/issue.ts +++ b/packages/shared/src/validators/issue.ts @@ -605,6 +605,8 @@ export const confirmOwnerGoneSchema = z.object({ actionId: z.string().guid(), recordId: z.string().guid(), stopEvidence: z.string().trim().min(20).max(12000), + /** The record version the board saw (`evidence.recordUpdatedAt`); required when the request names one. */ + recordVersion: z.string().datetime({ offset: true }).optional(), }).strict(); export const retryWorkspaceExportSchema = z.object({ diff --git a/server/src/__tests__/owner-gone-attestation.test.ts b/server/src/__tests__/owner-gone-attestation.test.ts index f61de25f6d0..ec91f065897 100644 --- a/server/src/__tests__/owner-gone-attestation.test.ts +++ b/server/src/__tests__/owner-gone-attestation.test.ts @@ -5,7 +5,9 @@ import { activityLog, agents, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, environmentLeases, heartbeatRuns, issueRecoveryActions, issues, } from "@paperclipai/db"; +import { subscribeDatabaseWork } from "@paperclipai/db/work-signals"; import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; import { ENVIRONMENT_LEASE_CONTROLLER_KEY } from "../services/environment-lease-controller.js"; import { confirmOwnerGone, OWNER_UNPROVABLE_CAUSE } from "../services/recovery/owner-gone-attestation.js"; import { createOwnershipReportState, runOwnershipReconciliationPass } from "../services/recovery/ownership-invariant-report.js"; @@ -54,7 +56,7 @@ describe("board confirmation that an unrecorded owner stopped", () => { expect((await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId)))).toHaveLength(1); await expect(confirmOwnerGone({ db, kinds: OWNABLE_KINDS, companyId, issueId, actionId: action!.id, recordId: leaseId, - stopEvidence, actorId: "board-user" })).resolves.toEqual({ recordId: leaseId, status: "queued" }); + stopEvidence, actorId: "board-user", recordVersion: action!.evidence.recordUpdatedAt as string })).resolves.toEqual({ recordId: leaseId, status: "queued" }); expect(await actionFor(issueId)).toMatchObject({ status: "resolved", evidence: { ownerGone: { actorId: "board-user", stopEvidence } } }); const [entry] = await db.select().from(activityLog).where(and(eq(activityLog.entityId, issueId), eq(activityLog.action, "issue.ownership_owner_gone_confirmed"))); @@ -71,12 +73,12 @@ describe("board confirmation that an unrecorded owner stopped", () => { await pass(); const action = (await actionFor(issueId))!; await expect(confirmOwnerGone({ db, kinds: OWNABLE_KINDS, companyId, issueId, actionId: action.id, recordId: randomUUID(), - stopEvidence, actorId: "board-user" })).rejects.toMatchObject({ status: 409 }); + stopEvidence, actorId: "board-user", recordVersion: action.evidence.recordUpdatedAt as string })).rejects.toMatchObject({ status: 409 }); // A holder stamped since the board was asked owns the lease now: its stamp decides. await db.update(environmentLeases).set({ metadata: { [ENVIRONMENT_LEASE_CONTROLLER_KEY]: { bootId: "new-boot", pid: 7, hostHash: "another-host", processStartedAt: null } } }).where(eq(environmentLeases.id, leaseId)); await expect(confirmOwnerGone({ db, kinds: OWNABLE_KINDS, companyId, issueId, actionId: action.id, recordId: leaseId, - stopEvidence, actorId: "board-user" })).rejects.toMatchObject({ status: 409 }); + stopEvidence, actorId: "board-user", recordVersion: action.evidence.recordUpdatedAt as string })).rejects.toMatchObject({ status: 409 }); expect(await actionFor(issueId)).toMatchObject({ status: "active" }); expect((await lease(leaseId)).status).toBe("active"); }); @@ -95,7 +97,7 @@ describe("board confirmation that an unrecorded owner stopped", () => { const action = (await actionFor(issueId))!; await db.update(environmentLeases).set({ status: "released" }).where(eq(environmentLeases.id, leaseId)); await expect(confirmOwnerGone({ db, kinds: OWNABLE_KINDS, companyId, issueId, actionId: action.id, recordId: leaseId, - stopEvidence, actorId: "board-user" })).resolves.toEqual({ recordId: leaseId, status: "released" }); + stopEvidence, actorId: "board-user", recordVersion: action.evidence.recordUpdatedAt as string })).resolves.toEqual({ recordId: leaseId, status: "released" }); const resolved = await actionFor(issueId); expect(resolved).toMatchObject({ status: "resolved", evidence: { ownerGoneNotNeeded: { actorId: "board-user" } } }); expect(resolved?.evidence).not.toHaveProperty("ownerGone"); @@ -112,4 +114,119 @@ describe("board confirmation that an unrecorded owner stopped", () => { await pass(); expect(await actionFor(orphan.issueId)).toBeUndefined(); }); + + describe("a settling kind with no attestation field (record-version attestation)", () => { + const LEGACY = "legacy_controller_lease"; + const legacyPass = async () => (await runOwnershipReconciliationPass(db, { state: createOwnershipReportState() })) + .kinds.find(kind => kind.kind === LEGACY)!; + + /** A legacy run whose controller was recorded before its PID space was: `missing_identity`. */ + async function seedLegacyRun() { + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID(), bootId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Legacy", issuePrefix: `L${companyId.slice(0, 6)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Agent", adapterType: "codex_local", status: "running" }); + await db.insert(issues).values({ id: issueId, companyId, title: "Task", status: "in_progress", assigneeAgentId: agentId }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, issueId, status: "running", invocationSource: "on_demand", + runtimeMode: "legacy", controllerBootId: bootId, controllerProcess: { bootId, pid: 4242, hostHash: "" }, + controllerLeaseExpiresAt: new Date(Date.now() + 3_600_000), updatedAt: new Date(Date.now() - 60_000) }); + return { companyId, issueId, runId }; + } + const leaseLive = async (runId: string) => + (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)))[0]!.controllerLeaseExpiresAt! > new Date(); + + it("asks the board, and settles through the kind's own settle once the board confirms the stop", async () => { + const { companyId, issueId, runId } = await seedLegacyRun(); + expect(await legacyPass()).toMatchObject({ settled: 0, reasons: { missing_identity: 1 } }); + const action = (await actionFor(issueId))!; + expect(action).toMatchObject({ ownerType: "board", status: "active", + evidence: { kind: LEGACY, recordId: runId, recordUpdatedAt: expect.any(String) } }); + // Still unprovable, and never released without the board. + expect(await legacyPass()).toMatchObject({ settled: 0 }); + expect(await leaseLive(runId)).toBe(true); + // An unprovable record is not watched: the confirmation itself wakes the reconciler. + const wakes: string[] = []; + const stopWatching = subscribeDatabaseWork(db, DELIVERY_QUEUES.ownershipInvariantReport, (event, hints) => { + if (event === "settled") wakes.push(...(hints ?? [])); + }); + try { + await expect(confirmOwnerGone({ db, kinds: OWNABLE_KINDS, companyId, issueId, actionId: action.id, recordId: runId, + stopEvidence, actorId: "board-user", recordVersion: action.evidence.recordUpdatedAt as string })).resolves.toEqual({ recordId: runId, status: "queued" }); + for (let wait = 0; wait < 50 && !wakes.some(hint => hint.includes(runId)); wait++) await new Promise(resolve => setTimeout(resolve, 20)); + } finally { stopWatching(); } + expect(wakes.some(hint => hint.includes(runId))).toBe(true); + expect(await legacyPass()).toMatchObject({ settled: 1, reasons: { operator_attested: 1 } }); + expect(await leaseLive(runId)).toBe(false); + }); + + it("does not ask about a record of these kinds with no owner stamp: its settle could never release it", async () => { + const { companyId, issueId, runId } = await seedLegacyRun(); + const kind = OWNABLE_KINDS.find(candidate => candidate.id === "sandbox_creation_intent")!; + expect(kind.ownerGoneAttestation).toBe("record_version"); + const record = { recordId: randomUUID(), companyId, runId, issueId, owner: { kind: "none" as const }, updatedAt: new Date() }; + await kind.onUnprovable!(db, record, { now: new Date(), proof: { verdict: "unprovable", reason: "no_owner_stamp" } as never }); + expect(await actionFor(issueId)).toBeUndefined(); + await db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, runId)); + }); + + it("never applies a confirmation to a record that changed since the board was asked", async () => { + const { companyId, issueId, runId } = await seedLegacyRun(); + await legacyPass(); + const action = (await actionFor(issueId))!; + await confirmOwnerGone({ db, kinds: OWNABLE_KINDS, companyId, issueId, actionId: action.id, recordId: runId, + stopEvidence, actorId: "board-user", recordVersion: action.evidence.recordUpdatedAt as string }); + // The record moved on (for example, a new controller took it over) after the confirmation. + await db.update(heartbeatRuns).set({ updatedAt: new Date() }).where(eq(heartbeatRuns.id, runId)); + expect(await legacyPass()).toMatchObject({ settled: 0, reasons: { missing_identity: 1 } }); + expect(await leaseLive(runId)).toBe(true); + // The board is asked again, about the current version. + const requests = await db.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.sourceIssueId, issueId), + eq(issueRecoveryActions.cause, OWNER_UNPROVABLE_CAUSE))); + expect(requests.map(request => request.status).sort()).toEqual(["active", "resolved"]); + // Confirming the current version applies, even with the older confirmation still on record. + const fresh = requests.find(request => request.status === "active")!; + await confirmOwnerGone({ db, kinds: OWNABLE_KINDS, companyId, issueId, actionId: fresh.id, recordId: runId, + stopEvidence, actorId: "board-user", recordVersion: fresh.evidence.recordUpdatedAt as string }); + expect(await legacyPass()).toMatchObject({ settled: 1, reasons: { operator_attested: 1 } }); + expect(await leaseLive(runId)).toBe(false); + }); + + it("refuses a confirmation for a version the record no longer has, then asks about the current one", async () => { + const { companyId, issueId, runId } = await seedLegacyRun(); + await legacyPass(); + const asked = (await actionFor(issueId))!; + await db.update(heartbeatRuns).set({ updatedAt: new Date() }).where(eq(heartbeatRuns.id, runId)); + await expect(confirmOwnerGone({ db, kinds: OWNABLE_KINDS, companyId, issueId, actionId: asked.id, recordId: runId, + stopEvidence, actorId: "board-user", recordVersion: asked.evidence.recordUpdatedAt as string })).rejects.toMatchObject({ status: 409 }); + // The next pass refreshes the open request to the current version; that confirmation applies. + await legacyPass(); + const refreshed = (await actionFor(issueId))!; + expect(refreshed.id).toBe(asked.id); + expect(refreshed.evidence.recordUpdatedAt).not.toBe(asked.evidence.recordUpdatedAt); + // A form opened before the refresh still names the old version: it cannot approve the new one. + await expect(confirmOwnerGone({ db, kinds: OWNABLE_KINDS, companyId, issueId, actionId: refreshed.id, recordId: runId, + stopEvidence, actorId: "board-user", recordVersion: asked.evidence.recordUpdatedAt as string })).rejects.toMatchObject({ status: 409 }); + await expect(confirmOwnerGone({ db, kinds: OWNABLE_KINDS, companyId, issueId, actionId: refreshed.id, recordId: runId, + stopEvidence, actorId: "board-user" })).rejects.toMatchObject({ status: 409 }); + await confirmOwnerGone({ db, kinds: OWNABLE_KINDS, companyId, issueId, actionId: refreshed.id, recordId: runId, + stopEvidence, actorId: "board-user", recordVersion: refreshed.evidence.recordUpdatedAt as string }); + expect(await legacyPass()).toMatchObject({ settled: 1, reasons: { operator_attested: 1 } }); + }); + + it("re-checks the confirmed version inside the settle's transaction", async () => { + const { companyId, issueId, runId } = await seedLegacyRun(); + await legacyPass(); + const kind = OWNABLE_KINDS.find(candidate => candidate.id === LEGACY)!; + const [row] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + const confirmedVersion = row!.updatedAt; + // The record moves on between the reconciler's listing and its settle. + await db.update(heartbeatRuns).set({ updatedAt: new Date() }).where(eq(heartbeatRuns.id, runId)); + const record = { recordId: runId, companyId, runId, issueId, updatedAt: confirmedVersion, + owner: { kind: "process" as const, stamp: { kind: "local" as const, scope: LEGACY, generation: null, host: null, + processSpace: "", bootId: row!.controllerBootId!, pid: 4242, processStartedAt: null, processGroupId: null } } }; + await expect(kind.settle!(db, record, { now: new Date(), proof: { verdict: "gone", reason: "operator_attested" } })) + .resolves.toMatchObject({ settled: false, reason: "record_changed_since_confirmation" }); + expect(await leaseLive(runId)).toBe(true); + await db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, runId)); + }); + }); }); diff --git a/server/src/services/recovery/owner-gone-attestation.ts b/server/src/services/recovery/owner-gone-attestation.ts index f8adb77e95f..bd5e4503cd8 100644 --- a/server/src/services/recovery/owner-gone-attestation.ts +++ b/server/src/services/recovery/owner-gone-attestation.ts @@ -1,8 +1,9 @@ -import { and, eq } from "drizzle-orm"; +import { and, eq, inArray, sql } from "drizzle-orm"; import { issueRecoveryActions, type Db } from "@paperclipai/db"; import { confirmOwnerGoneSchema } from "@paperclipai/shared"; import { conflict } from "../../errors.js"; import { persistActivity, publishActivity } from "../activity-log.js"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import type { OwnerProof } from "../owner-proof.js"; import type { InProgressRecord, OwnableKind } from "./ownership-registry.js"; @@ -12,7 +13,8 @@ export const OWNER_UNPROVABLE_CAUSE = "ownership_owner_unprovable"; export const OWNER_GONE_ATTESTATION_KEY = "ownerGoneAttestation"; /** Unprovable because the record names no usable owner: saved before owners were recorded. */ -const LEGACY_REASONS = new Set(["no_owner_stamp", "process_space_unrecorded", "missing_identity"]); +export const LEGACY_OWNER_REASONS: ReadonlySet = new Set(["no_owner_stamp", "process_space_unrecorded", "missing_identity"]); +const LEGACY_REASONS = LEGACY_OWNER_REASONS; const fingerprint = (kind: string, recordId: string) => `ownership-owner:${kind}:${recordId}`; const changed = () => conflict("This record changed since recovery asked. Refresh the task and inspect its run.", @@ -22,11 +24,25 @@ const changed = () => conflict("This record changed since recovery asked. Refres export function requestOwnerGoneConfirmation(kindId: string) { return async (db: Db, record: InProgressRecord, context: { now: Date; proof: OwnerProof }): Promise => { if (!LEGACY_REASONS.has(context.proof.reason) || !record.issueId || !record.companyId) return; + const recordUpdatedAt = record.updatedAt?.toISOString() ?? null; + // A record that changed while the board was asked: the open request names its current version, + // so a confirmation always binds to what the board can see now. + if (recordUpdatedAt) { + await db.update(issueRecoveryActions) + .set({ evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify({ recordUpdatedAt })}::jsonb`, updatedAt: context.now }) + .where(and(eq(issueRecoveryActions.companyId, record.companyId), eq(issueRecoveryActions.sourceIssueId, record.issueId), + eq(issueRecoveryActions.cause, OWNER_UNPROVABLE_CAUSE), eq(issueRecoveryActions.fingerprint, fingerprint(kindId, record.recordId)), + inArray(issueRecoveryActions.status, ["active", "escalated"]), + sql`${issueRecoveryActions.evidence}->>'recordUpdatedAt' is distinct from ${recordUpdatedAt}`)); + } await db.insert(issueRecoveryActions).values({ companyId: record.companyId, sourceIssueId: record.issueId, kind: "active_run_watchdog", status: "active", ownerType: "board", cause: OWNER_UNPROVABLE_CAUSE, fingerprint: fingerprint(kindId, record.recordId), attemptCount: 1, - evidence: { kind: kindId, recordId: record.recordId, runId: record.runId, proof: context.proof.reason }, + // The record version the board is asked about: a record-version attestation applies only while + // the record is unchanged, so it can never release an owner that took the record over since. + evidence: { kind: kindId, recordId: record.recordId, runId: record.runId, proof: context.proof.reason, + recordUpdatedAt }, nextAction: "A resource from this task's earlier run was recorded before Paperclip stored which server held it, " + "so Paperclip cannot verify that the server stopped. Confirm that the previous server process has stopped " + "(for example, its container was replaced), and recovery releases the resource.", @@ -34,13 +50,54 @@ export function requestOwnerGoneConfirmation(kindId: string) { }; } +/** + * The generic attestation for a kind whose record carries no field to hold it. The board's + * confirmation stays on the resolved recovery action, bound to the record version recovery asked + * about; the reconciler reads it back (`recordVersionAttestations`). A record with no version + * cannot be bound, so it keeps its other recovery path. + */ +export async function attestOwnerGoneByRecordVersion(tx: Db, target: { companyId: string; recordId: string; + requestedVersion: string | null }, lockVersion: (tx: Db, target: { companyId: string; recordId: string }) => Promise, +): Promise<"attested" | "released" | "changed"> { + const version = await lockVersion(tx, target); + if (!version) return "released"; + // The record changed since the board was asked: the confirmation would never apply, so refuse it. + return target.requestedVersion === version.toISOString() ? "attested" : "changed"; +} + +/** + * Board confirmations, by record id, for records of `kindId` whose owner is still unprovable for a + * legacy reason and whose version still equals the version the board confirmed. Only those + * records are read, so a kind with nothing stuck issues no statement. + */ +export async function recordVersionAttestations(db: Db, kindId: string, + candidates: readonly { recordId: string; companyId: string | null; updatedAt: Date | null }[]): Promise> { + const bound = candidates.filter(record => record.companyId && record.updatedAt); + if (!bound.length) return new Set(); + const rows = await db.select({ evidence: issueRecoveryActions.evidence }).from(issueRecoveryActions).where(and( + eq(issueRecoveryActions.cause, OWNER_UNPROVABLE_CAUSE), eq(issueRecoveryActions.status, "resolved"), + inArray(issueRecoveryActions.companyId, [...new Set(bound.map(record => record.companyId!))]), + inArray(issueRecoveryActions.fingerprint, bound.map(record => fingerprint(kindId, record.recordId))), + sql`${issueRecoveryActions.evidence} ? 'ownerGone'`)); + // A record can have several confirmations (one per version the board confirmed); any one of + // them that names the current version applies. + const confirmed = new Set(); + for (const { evidence } of rows) { + if (typeof evidence?.recordId === "string" && typeof evidence.recordUpdatedAt === "string") + confirmed.add(`${evidence.recordId}|${evidence.recordUpdatedAt}`); + } + return new Set(bound.filter(record => confirmed.has(`${record.recordId}|${record.updatedAt!.toISOString()}`)) + .map(record => record.recordId)); +} + /** Board attestation for exactly the record recovery asked about. Recovery then settles it on its normal path. */ export async function confirmOwnerGone(input: { db: Db; kinds: readonly OwnableKind[]; companyId: string; issueId: string; - actionId: string; recordId: string; stopEvidence: string; actorId: string; + actionId: string; recordId: string; stopEvidence: string; actorId: string; recordVersion?: string; }) { const confirmation = confirmOwnerGoneSchema.parse({ actionId: input.actionId, recordId: input.recordId, stopEvidence: input.stopEvidence, + ...(input.recordVersion === undefined ? {} : { recordVersion: input.recordVersion }), }); const committed = await input.db.transaction(async transaction => { const tx = transaction as unknown as Db; @@ -48,14 +105,23 @@ export async function confirmOwnerGone(input: { eq(issueRecoveryActions.companyId, input.companyId), eq(issueRecoveryActions.sourceIssueId, input.issueId), eq(issueRecoveryActions.id, input.actionId))).for("update"); const kind = input.kinds.find(candidate => candidate.id === action?.evidence?.kind); + // A request names the record version the board was shown; the confirmation must be for that + // same version, so a form opened before the request was refreshed cannot approve a newer one. + const requestedVersion = typeof action?.evidence?.recordUpdatedAt === "string" ? action.evidence.recordUpdatedAt : null; + if (requestedVersion && confirmation.recordVersion !== requestedVersion) throw changed(); if (!action || !kind?.attestOwnerGone || action.cause !== OWNER_UNPROVABLE_CAUSE || action.ownerType !== "board" || !["active", "escalated"].includes(action.status) || action.evidence?.recordId !== input.recordId) throw changed(); const now = new Date(); const proof = { actorId: input.actorId, stopEvidence: confirmation.stopEvidence, confirmedAt: now.toISOString() }; - const attestation = await kind.attestOwnerGone(tx, { companyId: input.companyId, recordId: input.recordId }, proof); + const attestation = await kind.attestOwnerGone(tx, { companyId: input.companyId, recordId: input.recordId, + requestedVersion }, proof); if (attestation === "changed") throw changed(); // Released by another path: close the request; nothing remains to attest. const released = attestation === "released"; + // An unprovable record is not watched, so wake the reconciler to settle it now (work intent + // before the write it announces). + if (!released) await notifyDeliveryWork(tx, DELIVERY_QUEUES.ownershipInvariantReport, + JSON.stringify({ companyId: input.companyId, runId: typeof action.evidence?.runId === "string" ? action.evidence.runId : null })); await tx.update(issueRecoveryActions).set({ status: "resolved", outcome: "restored", resolvedAt: now, updatedAt: now, evidence: { ...action.evidence, ...(released ? { ownerGoneNotNeeded: proof } : { ownerGone: proof }) }, nextAction: released ? "The resource was already released. No confirmation was needed." diff --git a/server/src/services/recovery/ownership-invariant-report.ts b/server/src/services/recovery/ownership-invariant-report.ts index b3379adb9d7..0bc8aecc468 100644 --- a/server/src/services/recovery/ownership-invariant-report.ts +++ b/server/src/services/recovery/ownership-invariant-report.ts @@ -11,6 +11,7 @@ import { deliveredQueuedCommentIds } from "../queued-comment-delivery.js"; import { logger } from "../../middleware/logger.js"; import { persistActivity } from "../activity-log.js"; import { currentProcessProbe, proveOwnerGone, type OwnerProof, type ProcessProbe } from "../owner-proof.js"; +import { LEGACY_OWNER_REASONS, recordVersionAttestations } from "./owner-gone-attestation.js"; import { OWNABLE_KINDS, resolveFenceHolders, @@ -192,6 +193,23 @@ async function classify(owner: RecordOwner, runProofs: Map { + if (kind.ownerGoneAttestation !== "record_version") return proofs; + const legacy = (record: InProgressRecord, proof: Classification) => record.owner.kind === "process" + && proof.verdict === "unprovable" && LEGACY_OWNER_REASONS.has(proof.reason); + const candidates = records.filter((record, index) => legacy(record, proofs[index]!)); + if (!candidates.length) return proofs; + const attested = await recordVersionAttestations(db, kind.id, candidates); + return proofs.map((proof, index) => legacy(records[index]!, proof) && attested.has(records[index]!.recordId) + ? { verdict: "gone", reason: "operator_attested" } : proof); +} + /** Settles a proven-gone record, or surfaces the operator action for an * unprovable one. A failure is logged and retried on a later pass. */ async function settleOrSurface(db: Db, kind: OwnableKind, record: InProgressRecord, proof: Classification, @@ -272,8 +290,11 @@ async function settleKind(db: Db, kindId: string, options: { const runProofs = await classifyRunOwners(db, unique); const settled: SettledRecord[] = []; let watched = 0; - for (const record of unique) { - let proof = await classify(record.owner, runProofs, options.probe, now, record.updatedAt); + const proofs: Classification[] = []; + for (const record of unique) proofs.push(await classify(record.owner, runProofs, options.probe, now, record.updatedAt)); + const attestedProofs = await applyOwnerGoneAttestations(db, kind, unique, proofs); + for (const [index, record] of unique.entries()) { + let proof = attestedProofs[index]!; if (options.currentProcessIdle && record.runId && proof.verdict === "alive" && proof.reason === "current_process") { proof = { verdict: "gone", reason: "current_process_idle" }; } @@ -620,8 +641,11 @@ export async function runOwnershipReconciliationPass(db: Db, options: { const unique = [...new Map(records.map(record => [record.recordId, record])).values()]; report.inProgress = unique.length; const runProofs = await classifyRunOwners(db, unique); - for (const record of unique) { - const proof = await classify(record.owner, runProofs, passProbe, now, record.updatedAt); + const classified: Classification[] = []; + for (const record of unique) classified.push(await classify(record.owner, runProofs, passProbe, now, record.updatedAt)); + const proofs = await applyOwnerGoneAttestations(db, kind, unique, classified); + for (const [index, record] of unique.entries()) { + const proof = proofs[index]!; report.verdicts[proof.verdict] += 1; report.reasons[proof.reason] = (report.reasons[proof.reason] ?? 0) + 1; const base = { kind: kind.id, recordId: record.recordId, companyId: record.companyId, diff --git a/server/src/services/recovery/ownership-registry.ts b/server/src/services/recovery/ownership-registry.ts index 92c31db0873..bbaca5f09ca 100644 --- a/server/src/services/recovery/ownership-registry.ts +++ b/server/src/services/recovery/ownership-registry.ts @@ -30,7 +30,7 @@ import { import type { LocalOwnerStamp, OwnerProof, RemoteFenceState, RemoteOwnerStamp } from "../owner-proof.js"; import { OPERATOR_ATTESTED_PROCESS_SPACE } from "../owner-proof.js"; import { writeEnvironmentCleanup } from "../environment-cleanup-notifications.js"; -import { OWNER_GONE_ATTESTATION_KEY, requestOwnerGoneConfirmation } from "./owner-gone-attestation.js"; +import { OWNER_GONE_ATTESTATION_KEY, attestOwnerGoneByRecordVersion, requestOwnerGoneConfirmation } from "./owner-gone-attestation.js"; import { requestNativeAcceptedResultLeaseOperatorAction, settleNativeAcceptedResultLease, @@ -153,8 +153,11 @@ export type OwnableKind = { onUnprovable?: UnprovableAction; /** Records the board's confirmation that this record's unrecorded owner stopped, so its * owner proves gone (`operator_attested`). Returns false when the record no longer qualifies. */ - attestOwnerGone?: (tx: Db, target: { companyId: string; recordId: string }, proof: Record) - => Promise<"attested" | "released" | "changed">; + attestOwnerGone?: (tx: Db, target: { companyId: string; recordId: string; requestedVersion: string | null }, + proof: Record) => Promise<"attested" | "released" | "changed">; + /** `record_version`: the board's confirmation stays on its recovery action, bound to the + * record version it was asked about, and the reconciler reads it back as `operator_attested`. */ + ownerGoneAttestation?: "record_version"; /** When a record of this kind next becomes settleable without a signal * (a retention window ending). The reconciler wakes then. */ nextDueAt?: (db: Db, now: Date) => Promise; @@ -185,6 +188,54 @@ function int(value: unknown): number | null { const runOwner = (runId: string | null | undefined): RecordOwner => runId ? { kind: "run", runId } : { kind: "none" }; +/** + * Lets the board confirm that a settling kind's unrecorded owner stopped. A process-owned record + * saved before its owner was fully recorded (`missing_identity`, `process_space_unrecorded`) is + * unprovable and is never released on its own; the board's confirmation, bound to the record + * version recovery asked about, proves its owner gone, and the kind's own settle releases it. + */ +function ownerGoneAttestable(kind: OwnableKind, lockVersion: RecordVersionLock): OwnableKind { + const ask = requestOwnerGoneConfirmation(kind.id); + const settle = kind.settle!; + return { ...kind, ownerGoneAttestation: "record_version", + attestOwnerGone: (tx, target) => attestOwnerGoneByRecordVersion(tx, target, lockVersion), + // These settles compare-and-swap on the recorded process, so only a process-owned record can be + // released; a record with no version cannot be bound to a confirmation. Recovery asks for neither. + onUnprovable: async (db, record, context) => { + if (record.owner.kind === "process" && record.updatedAt) await ask(db, record, context); + }, + // A confirmation proves the owner gone only for the version the board confirmed: lock the + // record and check that version in the same transaction as the kind's own settle. + settle: (db, record, context) => context.proof.reason !== "operator_attested" ? settle(db, record, context) + : db.transaction(async transaction => { + const tx = transaction as unknown as Db; + const version = record.companyId ? await lockVersion(tx, { companyId: record.companyId, recordId: record.recordId }) : null; + if (!version || !record.updatedAt || version.getTime() !== record.updatedAt.getTime()) { + return { settled: false, reason: "record_changed_since_confirmation" }; + } + return settle(tx, record, context); + }) }; +} + +/** Locks a record row for update and returns its last write time (null when the row is gone). */ +type RecordVersionLock = (tx: Db, target: { companyId: string; recordId: string }) => Promise; + +function versionLock(table: typeof heartbeatRuns | typeof environmentLeases | typeof sandboxCreationOperations + | typeof workspaceOperations): RecordVersionLock { + return async (tx, target) => { + const [row] = await tx.select({ updatedAt: table.updatedAt }).from(table) + .where(and(eq(table.id, target.recordId), eq(table.companyId, target.companyId))).for("update"); + return row?.updatedAt ?? null; + }; +} + +const nrfVersionLock: RecordVersionLock = async (tx, target) => { + const [row] = await tx.select({ updatedAt: nativeRunFinalizations.updatedAt }).from(nativeRunFinalizations) + .where(and(eq(nativeRunFinalizations.runId, target.recordId), eq(nativeRunFinalizations.companyId, target.companyId))) + .for("update"); + return row?.updatedAt ?? null; +}; + // ---------------------------------------------------------------------------- // native_run_finalizations @@ -634,19 +685,21 @@ export const OWNABLE_KINDS: readonly OwnableKind[] = [ // space is unprovable and keeps its lease until expiry, when the next claim // takes it over and the old holder's token compare-and-swap fails: that // fence is the handoff, never a host difference. - { + ownerGoneAttestable({ ...nrfKind("native_finalizer_lease", "Finalizer lease", "accepted_result", and(like(nativeRunFinalizations.leaseOwner, "native-finalizer:%"), notInArray(nativeRunFinalizations.phase, TERMINAL_PHASES), sql`${nativeRunFinalizations.leaseExpiresAt} > now()`), row => nrfTokenLeaseOwner(row, "native_finalizer_lease")), settle: (db, record, context) => settleNativeTokenLease("native_finalizer_lease", db, record, context), - }, - { + + }, nrfVersionLock), + ownerGoneAttestable({ ...nrfKind("native_cleanup_lease", "Cleanup lease", "workspace_preservation", and(like(nativeRunFinalizations.leaseOwner, "native-cleanup:%"), sql`${nativeRunFinalizations.leaseExpiresAt} > now()`), row => nrfTokenLeaseOwner(row, "native_cleanup_lease")), settle: (db, record, context) => settleNativeTokenLease("native_cleanup_lease", db, record, context), - }, + + }, nrfVersionLock), runKind("native_ownership_unverified_hold", "Ownership-unverified hold", "ownership", and(eq(heartbeatRuns.runtimeMode, "native"), eq(heartbeatRuns.status, "running"), or( and(eq(heartbeatRuns.errorCode, NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE), eq(heartbeatRuns.nativePhase, "terminal_failure")), @@ -714,7 +767,7 @@ export const OWNABLE_KINDS: readonly OwnableKind[] = [ }, // Environments and sandboxes - { + ownerGoneAttestable({ // Shared path since Phase 2: a creator proven gone makes its intent due // now; creation recovery then reconciles it with the provider. An intent // whose creator is already marked lost belongs to that recovery deadline. @@ -733,9 +786,9 @@ export const OWNABLE_KINDS: readonly OwnableKind[] = [ owner: controllerStamp(json(row.metadata).creationController, "sandbox_creation_intent"), updatedAt: row.updatedAt, })) }; }, - }, + }, versionLock(sandboxCreationOperations)), runLeaseKind(), - { + ownerGoneAttestable({ // Shared path since Phase 2: a claimant proven gone ends its fence now. ...leaseKind("environment_teardown_claim", "Teardown claim (in-flight cleanup)", "ownership", // Text compare: a malformed flag never makes the whole listing fail. @@ -743,7 +796,8 @@ export const OWNABLE_KINDS: readonly OwnableKind[] = [ sql`coalesce(${environmentLeases.metadata}->>'pendingCleanupInFlight', 'false') = 'true'`), teardownClaimOwner), settle: settleEnvironmentTeardownClaim, - }, + + }, versionLock(environmentLeases)), pendingCleanupKind(), leaseKind("environment_stop_and_retain_intent", "Stop-and-retain intent", "workspace_preservation", and(eq(environmentLeases.status, "pending_cleanup"), sql`${environmentLeases.metadata} ? 'sandboxStopAndRetain'`), @@ -772,7 +826,7 @@ export const OWNABLE_KINDS: readonly OwnableKind[] = [ }, // Workspaces - { + ownerGoneAttestable({ // Shared path since Phase 2: the owner stamp names the server process and, // since this kind moved, its boot id and start time, so a dead or reused // PID is proof. Another host or a stamp without a PID space stays @@ -784,8 +838,9 @@ export const OWNABLE_KINDS: readonly OwnableKind[] = [ return stamp ? { kind: "process", stamp } : { kind: "none" }; }), settle: settleRuntimeControlOperation, - }, - { + + }, versionLock(workspaceOperations)), + ownerGoneAttestable({ // Shared path since Phase 2: a finalize attempt whose process died (a // restart between start and copyback, or between copyback and done) is // settled so it cannot stay `running` forever (crash points @@ -796,7 +851,8 @@ export const OWNABLE_KINDS: readonly OwnableKind[] = [ row => json(row.metadata).finalizeOwner ? controllerStamp(json(row.metadata).finalizeOwner, "native_workspace_finalize_attempt") : runOwner(row.heartbeatRunId)), settle: settleAbandonedWorkspaceFinalizeAttempt, - }, + + }, versionLock(workspaceOperations)), operationKind("remote_provision_command", "Remote provision command intent", "workspace_preservation", and(eq(workspaceOperations.phase, "remote_workspace_provision"), or(eq(workspaceOperations.status, "running"), sql`(${workspaceOperations.metadata}->>'uncertain')::boolean is true`)), @@ -950,7 +1006,7 @@ export const OWNABLE_KINDS: readonly OwnableKind[] = [ })) }; }, }, - { + ownerGoneAttestable({ // Shared path since Phase 2: the claim records its controller process // (`controller_process`, bound to the boot id); settle expires the lease of // a controller proven gone, and live-run recovery settles the run now @@ -970,7 +1026,8 @@ export const OWNABLE_KINDS: readonly OwnableKind[] = [ return records; }, settle: settleLegacyControllerLease, - }, + + }, versionLock(heartbeatRuns)), runKind("detached_process_marker", "Detached-process marker", "ownership", and(eq(heartbeatRuns.status, "running"), eq(heartbeatRuns.errorCode, "process_detached")), row => row.processPid ? { kind: "process", stamp: runProcessStamp(row, "detached_process_marker") } : { kind: "none" }, diff --git a/ui/src/api/issues.ts b/ui/src/api/issues.ts index c81e0dedee2..9d207cbc15b 100644 --- a/ui/src/api/issues.ts +++ b/ui/src/api/issues.ts @@ -422,7 +422,7 @@ export const issuesApi = { ), confirmControllerStopped: (id: string, data: { actionId: string; runId: string; stopEvidence: string }) => api.post<{ runId: string; status: "queued" }>(`/issues/${id}/recovery-actions/confirm-controller-stopped`, data), - confirmOwnerGone: (id: string, data: { actionId: string; recordId: string; stopEvidence: string }) => + confirmOwnerGone: (id: string, data: { actionId: string; recordId: string; stopEvidence: string; recordVersion?: string }) => api.post<{ recordId: string; status: "queued" | "released" }>(`/issues/${id}/recovery-actions/confirm-owner-gone`, data), resumeWorkspaceFinalization: (id: string, data: { actionId: string; runId: string; ownerToken: string; controllerAndCopybackStopped: true; stopEvidence: string }) => api.post<{ runId: string; resultId: string | null; status: "queued" }>(`/issues/${id}/recovery-actions/resume-workspace-finalization`, data), diff --git a/ui/src/components/WorkspaceExportRecovery.control.test.tsx b/ui/src/components/WorkspaceExportRecovery.control.test.tsx index 5c2a158a6ec..c9654fb0d0d 100644 --- a/ui/src/components/WorkspaceExportRecovery.control.test.tsx +++ b/ui/src/components/WorkspaceExportRecovery.control.test.tsx @@ -516,7 +516,43 @@ describe("governed retained-file recovery", () => { await click("Confirm and resume"); expect(api.confirmOwnerGone).toHaveBeenCalledExactlyOnceWith("issue-1", { actionId: "action-1", recordId: "lease-1", stopEvidence: "The old server container was replaced and is not running." }); - expect(container.textContent).toContain("Releasing the resource…"); + expect(container.textContent).toContain("Releasing the sandbox lease…"); + }); + + it("resets an open confirmation when the request is refreshed to a newer record version", async () => { + api.confirmOwnerGone.mockResolvedValue({ recordId: "run-1", status: "queued" }); + client.setQueryData(queryKeys.access.currentBoardAccess, { userId: "board-1", source: "session", isInstanceAdmin: false, + companyIds: ["company-1"], memberships: [{ companyId: "company-1", membershipRole: "owner", status: "active" }] }); + const at = (version: string) => ({ ...action, cause: "ownership_owner_unprovable", ownerType: "board", + evidence: { kind: "legacy_controller_lease", recordId: "run-1", runId: "run-1", recordUpdatedAt: version } }) as unknown as IssueRecoveryAction; + await mount(at("2026-11-08T10:00:00.000Z")); + await click("Confirm the previous server stopped"); + expect(container.querySelector("textarea")).not.toBeNull(); + // The record changed and the request now names its new version: the open form starts over. + await mount(at("2026-11-08T10:05:00.000Z")); + expect(container.querySelector("textarea")).toBeNull(); + await click("Confirm the previous server stopped"); + const textarea = container.querySelector("textarea")!; + await act(async () => { + const setter = Object.getOwnPropertyDescriptor(HTMLTextAreaElement.prototype, "value")!.set!; + setter.call(textarea, "The old server container was replaced and is not running."); + textarea.dispatchEvent(new Event("input", { bubbles: true })); + }); + await click("Confirm and resume"); + expect(api.confirmOwnerGone).toHaveBeenCalledExactlyOnceWith("issue-1", { actionId: "action-1", recordId: "run-1", + stopEvidence: "The old server container was replaced and is not running.", recordVersion: "2026-11-08T10:05:00.000Z" }); + }); + + it("names the stuck record for a kind confirmed by record version, and a generic resource otherwise", async () => { + client.setQueryData(queryKeys.access.currentBoardAccess, { userId: "board-1", source: "session", isInstanceAdmin: false, + companyIds: ["company-1"], memberships: [{ companyId: "company-1", membershipRole: "owner", status: "active" }] }); + const ownerGone = (kind: string) => ({ ...action, cause: "ownership_owner_unprovable", ownerType: "board", + evidence: { kind, recordId: "run-1", runId: "run-1", recordUpdatedAt: "2026-11-08T10:00:00.000Z" } }) as unknown as IssueRecoveryAction; + await mount(ownerGone("legacy_controller_lease")); + expect(container.textContent).toContain("This task’s earlier run was recorded before Paperclip stored which server held it"); + expect(button("Confirm the previous server stopped")).toBeDefined(); + await mount(ownerGone("a_future_kind")); + expect(container.textContent).toContain("A resource from this task’s earlier run was recorded"); }); it("shows status only, without the action, once automatic recovery is resuming or for non-administrators", async () => { diff --git a/ui/src/components/WorkspaceExportRecovery.tsx b/ui/src/components/WorkspaceExportRecovery.tsx index 78684069b42..0c38829e65e 100644 --- a/ui/src/components/WorkspaceExportRecovery.tsx +++ b/ui/src/components/WorkspaceExportRecovery.tsx @@ -28,6 +28,19 @@ function ReportRecoveryControl({ owns, onChange }: { owns: boolean; onChange?: ( const CONTROLLER_HOST_UNVERIFIED = "native_controller_host_unverified"; const OWNER_UNPROVABLE = "ownership_owner_unprovable"; +/** What the stuck record is, by ownership kind, for the board's confirmation. */ +const OWNER_GONE_RESOURCE: Record = { + environment_run_lease_active: { what: "A sandbox lease from this task’s earlier run", releasing: "Releasing the sandbox lease…" }, + legacy_controller_lease: { what: "This task’s earlier run", releasing: "Recovering the run…" }, + environment_teardown_claim: { what: "A sandbox cleanup from this task’s earlier run", releasing: "Resuming the sandbox cleanup…" }, + sandbox_creation_intent: { what: "A sandbox this task’s earlier run was creating", releasing: "Reconciling the sandbox…" }, + workspace_runtime_control_operation: { what: "A workspace service change from this task’s earlier run", releasing: "Releasing the workspace…" }, + native_workspace_finalize_attempt: { what: "A workspace save from this task’s earlier run", releasing: "Releasing the workspace save…" }, + native_finalizer_lease: { what: "The result handling of this task’s earlier run", releasing: "Resuming the result handling…" }, + native_cleanup_lease: { what: "The cleanup of this task’s earlier run", releasing: "Resuming the cleanup…" }, +}; +const OWNER_GONE_DEFAULT = { what: "A resource from this task’s earlier run", releasing: "Releasing the resource…" }; + /** Causes whose one action is the board's previous-server stop confirmation; * the generic recovery card must not offer other resolutions for them. */ export function isPreviousServerStopCause(cause: string | null | undefined) { @@ -69,15 +82,20 @@ export function WorkspaceExportRecovery({ action, className, issueId, companyId, } // A resource held by a server recorded before Paperclip stored which server held it. if (action?.cause === OWNER_UNPROVABLE && issueId && companyId && isOwnerGoneRequired(action, issueId, companyId)) { + const resource = OWNER_GONE_RESOURCE[String(action.evidence.kind)] ?? OWNER_GONE_DEFAULT; return <> - issuesApi.confirmOwnerGone(issueId, { - actionId: action.id, recordId: action.evidence.recordId as string, stopEvidence })} - copy={{ waiting: "A resource from this task’s earlier run was recorded before Paperclip stored which server held it, so Paperclip can’t verify that the server stopped. The resource stays reserved until then.", + actionId: action.id, recordId: action.evidence.recordId as string, stopEvidence, + // The version shown here: a request refreshed since must be confirmed again. + ...(typeof action.evidence.recordUpdatedAt === "string" ? { recordVersion: action.evidence.recordUpdatedAt } : {}) })} + copy={{ waiting: `${resource.what} was recorded before Paperclip stored which server held it, so Paperclip can’t verify that the server stopped. It stays reserved until then.`, question: "How did you confirm that the previous server stopped?", placeholder: "For example: the old server container was replaced and is no longer running.", - resuming: "Releasing the resource…" }} />; + resuming: resource.releasing }} />; } const ownerRecovery = action?.cause === "native_workspace_finalization_owner_unverified"; // The task's interruption names this wait; the confirmation is its one action.