diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 04499d12da7..1cfff379faf 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -1996,6 +1996,7 @@ export { reconstructWorkspaceSchema, resumeWorkspaceFinalizationSchema, confirmNativeControllerStoppedSchema, + confirmOwnerGoneSchema, issueReviewRequestSchema, issueExecutionWorkspaceSettingsSchema, checkoutIssueSchema, diff --git a/packages/shared/src/validators/index.ts b/packages/shared/src/validators/index.ts index 9eb2f59fccb..d62b0678d5f 100644 --- a/packages/shared/src/validators/index.ts +++ b/packages/shared/src/validators/index.ts @@ -452,6 +452,7 @@ export { reconstructWorkspaceSchema, resumeWorkspaceFinalizationSchema, confirmNativeControllerStoppedSchema, + confirmOwnerGoneSchema, issueReviewRequestSchema, issueExecutionWorkspaceSettingsSchema, checkoutIssueSchema, diff --git a/packages/shared/src/validators/issue.ts b/packages/shared/src/validators/issue.ts index c0664bbc83c..7349a2ffbce 100644 --- a/packages/shared/src/validators/issue.ts +++ b/packages/shared/src/validators/issue.ts @@ -599,6 +599,14 @@ export const confirmNativeControllerStoppedSchema = z.object({ stopEvidence: z.string().trim().min(20).max(12000), }).strict(); +/** Board attestation that the owner of a record Paperclip cannot judge (saved + * before owners were recorded) has stopped, so recovery may settle the record. */ +export const confirmOwnerGoneSchema = z.object({ + actionId: z.string().guid(), + recordId: z.string().guid(), + stopEvidence: z.string().trim().min(20).max(12000), +}).strict(); + export const retryWorkspaceExportSchema = z.object({ actionId: z.string().guid(), runId: z.string().guid(), diff --git a/server/src/__tests__/owner-gone-attestation.test.ts b/server/src/__tests__/owner-gone-attestation.test.ts new file mode 100644 index 00000000000..f61de25f6d0 --- /dev/null +++ b/server/src/__tests__/owner-gone-attestation.test.ts @@ -0,0 +1,115 @@ +import { randomUUID } from "node:crypto"; +import { and, eq } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { + activityLog, agents, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, environmentLeases, heartbeatRuns, + issueRecoveryActions, issues, +} from "@paperclipai/db"; +import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { ENVIRONMENT_LEASE_CONTROLLER_KEY } from "../services/environment-lease-controller.js"; +import { confirmOwnerGone, OWNER_UNPROVABLE_CAUSE } from "../services/recovery/owner-gone-attestation.js"; +import { createOwnershipReportState, runOwnershipReconciliationPass } from "../services/recovery/ownership-invariant-report.js"; +import { OWNABLE_KINDS } from "../services/recovery/ownership-registry.js"; +import { recoveryService } from "../services/recovery/service.js"; + +const KIND = "environment_run_lease_active"; +const stopEvidence = "The old server container was replaced and is no longer running."; + +describe("board confirmation that an unrecorded owner stopped", () => { + let temporary: Awaited> | null = null; + let db: ReturnType; + + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("paperclip-owner-gone-attestation-"); + db = createDb(temporary.connectionString); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + afterAll(async () => temporary?.cleanup()); + afterEach(() => vi.unstubAllEnvs()); + + /** A run lease saved before holders were stamped, on an ephemeral server, after its run finished. */ + async function seed(holder: Record | null = null) { + vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "true"); + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID(), leaseId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Owner", issuePrefix: `O${companyId.slice(0, 6)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Agent", adapterType: "codex_local", status: "idle" }); + await db.insert(issues).values({ id: issueId, companyId, title: "Task", status: "in_progress", assigneeAgentId: agentId }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, issueId, status: "failed", invocationSource: "on_demand" }); + await db.insert(environmentLeases).values({ id: leaseId, companyId, issueId, heartbeatRunId: runId, status: "active", + leasePolicy: "ephemeral", metadata: holder ? { [ENVIRONMENT_LEASE_CONTROLLER_KEY]: holder } : {} }); + return { companyId, issueId, runId, leaseId }; + } + + const pass = async () => (await runOwnershipReconciliationPass(db, { state: createOwnershipReportState() })) + .kinds.find(kind => kind.kind === KIND)!; + const actionFor = async (issueId: string) => (await db.select().from(issueRecoveryActions) + .where(and(eq(issueRecoveryActions.sourceIssueId, issueId), eq(issueRecoveryActions.cause, OWNER_UNPROVABLE_CAUSE))))[0]; + const lease = async (leaseId: string) => (await db.select().from(environmentLeases).where(eq(environmentLeases.id, leaseId)))[0]!; + + it("asks the board on the task, and releases the lease once the board confirms the stop", async () => { + const { companyId, issueId, runId, leaseId } = await seed(); + expect(await pass()).toMatchObject({ settled: 0, reasons: { no_owner_stamp: 1 } }); + const action = await actionFor(issueId); + expect(action).toMatchObject({ ownerType: "board", status: "active", evidence: { kind: KIND, recordId: leaseId, runId } }); + await pass(); + expect((await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId)))).toHaveLength(1); + + await expect(confirmOwnerGone({ db, kinds: OWNABLE_KINDS, companyId, issueId, actionId: action!.id, recordId: leaseId, + stopEvidence, actorId: "board-user" })).resolves.toEqual({ recordId: leaseId, status: "queued" }); + expect(await actionFor(issueId)).toMatchObject({ status: "resolved", evidence: { ownerGone: { actorId: "board-user", stopEvidence } } }); + const [entry] = await db.select().from(activityLog).where(and(eq(activityLog.entityId, issueId), + eq(activityLog.action, "issue.ownership_owner_gone_confirmed"))); + expect(entry).toMatchObject({ actorType: "user", actorId: "board-user", runId, + details: expect.objectContaining({ kind: KIND, recordId: leaseId, stopEvidence }) }); + + expect(await pass()).toMatchObject({ settled: 1, reasons: { operator_attested: 1 } }); + expect((await lease(leaseId)).status).toBe("pending_cleanup"); + expect(await pass()).toMatchObject({ settled: 0 }); + }); + + it("refuses a stale or mismatched confirmation, and never attests a stamped holder", async () => { + const { companyId, issueId, leaseId } = await seed(); + await pass(); + const action = (await actionFor(issueId))!; + await expect(confirmOwnerGone({ db, kinds: OWNABLE_KINDS, companyId, issueId, actionId: action.id, recordId: randomUUID(), + stopEvidence, actorId: "board-user" })).rejects.toMatchObject({ status: 409 }); + // A holder stamped since the board was asked owns the lease now: its stamp decides. + await db.update(environmentLeases).set({ metadata: { [ENVIRONMENT_LEASE_CONTROLLER_KEY]: { + bootId: "new-boot", pid: 7, hostHash: "another-host", processStartedAt: null } } }).where(eq(environmentLeases.id, leaseId)); + await expect(confirmOwnerGone({ db, kinds: OWNABLE_KINDS, companyId, issueId, actionId: action.id, recordId: leaseId, + stopEvidence, actorId: "board-user" })).rejects.toMatchObject({ status: 409 }); + expect(await actionFor(issueId)).toMatchObject({ status: "active" }); + expect((await lease(leaseId)).status).toBe("active"); + }); + + it("is never closed by ordinary recovery, even once the task is done", async () => { + const { companyId, issueId } = await seed(); + await pass(); + await db.update(issues).set({ status: "done" }).where(eq(issues.id, issueId)); + await recoveryService(db, { enqueueWakeup: vi.fn() } as never).reconcileActiveRecoveryActions({ companyId }); + expect(await actionFor(issueId)).toMatchObject({ status: "active" }); + }); + + it("closes the request without attesting when the lease was released another way", async () => { + const { companyId, issueId, leaseId } = await seed(); + await pass(); + const action = (await actionFor(issueId))!; + await db.update(environmentLeases).set({ status: "released" }).where(eq(environmentLeases.id, leaseId)); + await expect(confirmOwnerGone({ db, kinds: OWNABLE_KINDS, companyId, issueId, actionId: action.id, recordId: leaseId, + stopEvidence, actorId: "board-user" })).resolves.toEqual({ recordId: leaseId, status: "released" }); + const resolved = await actionFor(issueId); + expect(resolved).toMatchObject({ status: "resolved", evidence: { ownerGoneNotNeeded: { actorId: "board-user" } } }); + expect(resolved?.evidence).not.toHaveProperty("ownerGone"); + expect((await lease(leaseId)).metadata).not.toHaveProperty("ownerGoneAttestation"); + }); + + it("does not ask about an owner Paperclip can judge, or a record with no task", async () => { + const stamped = await seed({ bootId: "boot", pid: 7, hostHash: "another-host", processStartedAt: null }); + await pass(); + expect(await actionFor(stamped.issueId)).toBeUndefined(); + const orphan = await seed(); + await db.update(environmentLeases).set({ issueId: null }).where(eq(environmentLeases.id, orphan.leaseId)); + await db.update(heartbeatRuns).set({ issueId: null }).where(eq(heartbeatRuns.id, orphan.runId)); + await pass(); + expect(await actionFor(orphan.issueId)).toBeUndefined(); + }); +}); diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.ts index 8cb780faa69..928d55caa0b 100644 --- a/server/src/routes/issues.ts +++ b/server/src/routes/issues.ts @@ -40,6 +40,8 @@ import { createIssueReadTiming } from "../services/issue-read-timing.js"; import { isNativeWorkspaceExportRepairCause } from "@paperclipai/shared"; import { resumeNativeWorkspaceAfterOwnerStop } from "../services/native-runtime/native-workspace-owner-recovery.js"; import { confirmNativeControllerStopped, NATIVE_CONTROLLER_HOST_UNVERIFIED } from "../services/native-runtime/native-controller-host-recovery.js"; +import { confirmOwnerGone, OWNER_UNPROVABLE_CAUSE } from "../services/recovery/owner-gone-attestation.js"; +import { OWNABLE_KINDS } from "../services/recovery/ownership-registry.js"; import { retryNativeWorkspaceExport } from "../services/native-runtime/native-workspace-export-retry.js"; import { reconstructNativeWorkspace, unreconciledNativeExportPredicate } from "../services/native-runtime/native-workspace-reconstruction.js"; import { reconstructWorkspaceSchema } from "@paperclipai/shared"; @@ -124,6 +126,7 @@ import { taskRetrySchema, resumeWorkspaceFinalizationSchema, confirmNativeControllerStoppedSchema, + confirmOwnerGoneSchema, runnerGoalActionRequestSchema, feedbackTargetTypeSchema, feedbackTraceStatusSchema, @@ -10112,6 +10115,17 @@ export function issueRoutes( res.status(202).json(receipt); }); + router.post("/issues/:id/recovery-actions/confirm-owner-gone", validate(confirmOwnerGoneSchema), async (req, res) => { + assertBoard(req); + const issue = await getAccessibleResource(req, res, svc.getById(req.params.id as string), "Issue not found"); + if (!issue || !(await assertIssueReadAllowed(req, res, issue))) return; + const decision = await access.decide({ actor: req.actor, action: "runtime:confirm_owner_stop", resource: { type: "company", companyId: issue.companyId } }); + if (!decision.allowed) throw forbidden(decision.explanation, authorizationDeniedDetails(decision)); + const receipt = await confirmOwnerGone({ db, kinds: OWNABLE_KINDS, companyId: issue.companyId, issueId: issue.id, + ...req.body, actorId: getActorInfo(req).actorId }); + res.status(202).json(receipt); + }); + router.post("/issues/:id/recovery-actions/retry-workspace-export", validate(retryWorkspaceExportSchema), async (req, res) => { assertBoard(req); const issue = await getAccessibleResource(req, res, svc.getById(req.params.id as string), "Issue not found"); @@ -10376,6 +10390,10 @@ export function issueRoutes( throw conflict("Unsafe workspace export recovers automatically without another provider turn.", { code: "workspace_export_automatic_recovery" }); } + if ((outcome === "restored" || outcome === "false_positive") && activeRecoveryAction.cause === OWNER_UNPROVABLE_CAUSE) { + throw conflict("Confirm that the previous server stopped to release this resource.", { code: "ownership_owner_stop_required" }); + } + if ((outcome === "restored" || outcome === "false_positive") && activeRecoveryAction.cause === NATIVE_CONTROLLER_HOST_UNVERIFIED) { throw conflict("Confirm that the previous server stopped to resume this run.", { code: "native_controller_stop_required" }); } diff --git a/server/src/routes/openapi.ts b/server/src/routes/openapi.ts index 0d75641826a..b2bc94cb5f3 100644 --- a/server/src/routes/openapi.ts +++ b/server/src/routes/openapi.ts @@ -261,6 +261,7 @@ import { reconstructWorkspaceSchema, resumeWorkspaceFinalizationSchema, confirmNativeControllerStoppedSchema, + confirmOwnerGoneSchema, cancelIssueThreadInteractionSchema, // Secret provider configs and remote import createSecretProviderConfigSchema, @@ -11064,6 +11065,15 @@ registerCurrentRoute({ responses: { 202: r.ok(), 400: r.badRequest, 401: r.unauthorized, 403: r.forbidden, 404: r.notFound, 409: r.conflict }, }); +registerCurrentRoute({ + method: "post", + path: "/api/issues/{id}/recovery-actions/confirm-owner-gone", + tags: ["issues"], + summary: "Confirm that the unrecorded server holding a stuck recovery record stopped, so recovery can release it", + body: confirmOwnerGoneSchema, + responses: { 202: r.ok(), 400: r.badRequest, 401: r.unauthorized, 403: r.forbidden, 404: r.notFound, 409: r.conflict }, +}); + registerCurrentRoute({ method: "post", path: "/api/issues/{id}/recovery-actions/retry-workspace-export", diff --git a/server/src/services/environment-run-lease-settlement.ts b/server/src/services/environment-run-lease-settlement.ts index cbbe4e87367..942f769a78f 100644 --- a/server/src/services/environment-run-lease-settlement.ts +++ b/server/src/services/environment-run-lease-settlement.ts @@ -6,7 +6,8 @@ import { pendingCleanupMetadataObjectSql } from "./environment-cleanup-state.js" import { readEnvironmentLeaseController } from "./environment-lease-controller.js"; import { allowLegacyShutdownWorkspaceCleanup } from "./legacy-shutdown-workspace-settlement.js"; import { nativeWorkspaceExportPendingForLease } from "./native-runtime/native-workspace-finalization-state.js"; -import type { OwnerProof } from "./owner-proof.js"; +import { OPERATOR_ATTESTED_PROCESS_SPACE, type OwnerProof } from "./owner-proof.js"; +import { OWNER_GONE_ATTESTATION_KEY } from "./recovery/owner-gone-attestation.js"; import type { InProgressRecord, SettleOutcome } from "./recovery/ownership-registry.js"; import { isBuiltinSandboxProvider } from "./sandbox-provider-runtime.js"; import { prepareSandboxStopAndRetain } from "./sandbox-stop-and-retain.js"; @@ -96,6 +97,8 @@ export function holderUnchanged(lease: LeaseRow, record: InProgressRecord): bool if (record.owner.kind === "run") return holder === null; if (record.owner.kind !== "process") return false; const stamp = record.owner.stamp; + // The board attested that the unrecorded holder stopped; still no holder stamped since. + if (stamp.processSpace === OPERATOR_ATTESTED_PROCESS_SPACE) return holder === null && Boolean(lease.metadata?.[OWNER_GONE_ATTESTATION_KEY]); return holder !== null && holder.bootId === stamp.bootId && holder.pid === stamp.pid && holder.hostHash === stamp.processSpace && holder.processStartedAt === (stamp.processStartedAt ?? null); } diff --git a/server/src/services/recovery/owner-gone-attestation.ts b/server/src/services/recovery/owner-gone-attestation.ts new file mode 100644 index 00000000000..f8adb77e95f --- /dev/null +++ b/server/src/services/recovery/owner-gone-attestation.ts @@ -0,0 +1,72 @@ +import { and, eq } from "drizzle-orm"; +import { issueRecoveryActions, type Db } from "@paperclipai/db"; +import { confirmOwnerGoneSchema } from "@paperclipai/shared"; +import { conflict } from "../../errors.js"; +import { persistActivity, publishActivity } from "../activity-log.js"; +import type { OwnerProof } from "../owner-proof.js"; +import type { InProgressRecord, OwnableKind } from "./ownership-registry.js"; + +/** Recovery action cause: a stuck record whose owner Paperclip cannot judge. */ +export const OWNER_UNPROVABLE_CAUSE = "ownership_owner_unprovable"; +/** Lease metadata key the board's confirmation writes; the owner proof reads it as gone. */ +export const OWNER_GONE_ATTESTATION_KEY = "ownerGoneAttestation"; + +/** Unprovable because the record names no usable owner: saved before owners were recorded. */ +const LEGACY_REASONS = new Set(["no_owner_stamp", "process_space_unrecorded", "missing_identity"]); + +const fingerprint = (kind: string, recordId: string) => `ownership-owner:${kind}:${recordId}`; +const changed = () => conflict("This record changed since recovery asked. Refresh the task and inspect its run.", + { code: "ownership_owner_attestation_stale" }); + +/** Idempotent. Asks the board on the record's task; a record with no task keeps its other recovery path. */ +export function requestOwnerGoneConfirmation(kindId: string) { + return async (db: Db, record: InProgressRecord, context: { now: Date; proof: OwnerProof }): Promise => { + if (!LEGACY_REASONS.has(context.proof.reason) || !record.issueId || !record.companyId) return; + await db.insert(issueRecoveryActions).values({ + companyId: record.companyId, sourceIssueId: record.issueId, kind: "active_run_watchdog", + status: "active", ownerType: "board", cause: OWNER_UNPROVABLE_CAUSE, + fingerprint: fingerprint(kindId, record.recordId), attemptCount: 1, + evidence: { kind: kindId, recordId: record.recordId, runId: record.runId, proof: context.proof.reason }, + nextAction: "A resource from this task's earlier run was recorded before Paperclip stored which server held it, " + + "so Paperclip cannot verify that the server stopped. Confirm that the previous server process has stopped " + + "(for example, its container was replaced), and recovery releases the resource.", + }).onConflictDoNothing(); + }; +} + +/** Board attestation for exactly the record recovery asked about. Recovery then settles it on its normal path. */ +export async function confirmOwnerGone(input: { + db: Db; kinds: readonly OwnableKind[]; companyId: string; issueId: string; + actionId: string; recordId: string; stopEvidence: string; actorId: string; +}) { + const confirmation = confirmOwnerGoneSchema.parse({ + actionId: input.actionId, recordId: input.recordId, stopEvidence: input.stopEvidence, + }); + const committed = await input.db.transaction(async transaction => { + const tx = transaction as unknown as Db; + const [action] = await tx.select().from(issueRecoveryActions).where(and( + eq(issueRecoveryActions.companyId, input.companyId), eq(issueRecoveryActions.sourceIssueId, input.issueId), + eq(issueRecoveryActions.id, input.actionId))).for("update"); + const kind = input.kinds.find(candidate => candidate.id === action?.evidence?.kind); + if (!action || !kind?.attestOwnerGone || action.cause !== OWNER_UNPROVABLE_CAUSE || action.ownerType !== "board" + || !["active", "escalated"].includes(action.status) || action.evidence?.recordId !== input.recordId) throw changed(); + const now = new Date(); + const proof = { actorId: input.actorId, stopEvidence: confirmation.stopEvidence, confirmedAt: now.toISOString() }; + const attestation = await kind.attestOwnerGone(tx, { companyId: input.companyId, recordId: input.recordId }, proof); + if (attestation === "changed") throw changed(); + // Released by another path: close the request; nothing remains to attest. + const released = attestation === "released"; + await tx.update(issueRecoveryActions).set({ status: "resolved", outcome: "restored", resolvedAt: now, updatedAt: now, + evidence: { ...action.evidence, ...(released ? { ownerGoneNotNeeded: proof } : { ownerGone: proof }) }, + nextAction: released ? "The resource was already released. No confirmation was needed." + : "The previous server was confirmed stopped. Recovery releases the resource.", + }).where(eq(issueRecoveryActions.id, action.id)); + const publication = await persistActivity(tx, { companyId: input.companyId, actorType: "user", actorId: input.actorId, + action: "issue.ownership_owner_gone_confirmed", entityType: "issue", entityId: input.issueId, + runId: typeof action.evidence?.runId === "string" ? action.evidence.runId : undefined, + details: { recoveryActionId: input.actionId, kind: kind.id, recordId: input.recordId, outcome: attestation, ...proof } }); + return { publication, released }; + }); + publishActivity(committed.publication.publication); + return { recordId: input.recordId, status: committed.released ? "released" as const : "queued" as const }; +} diff --git a/server/src/services/recovery/ownership-registry.ts b/server/src/services/recovery/ownership-registry.ts index 04f2e4aa6e5..92c31db0873 100644 --- a/server/src/services/recovery/ownership-registry.ts +++ b/server/src/services/recovery/ownership-registry.ts @@ -28,6 +28,9 @@ import { NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE, } from "../native-runtime/native-runner-ownership.js"; import type { LocalOwnerStamp, OwnerProof, RemoteFenceState, RemoteOwnerStamp } from "../owner-proof.js"; +import { OPERATOR_ATTESTED_PROCESS_SPACE } from "../owner-proof.js"; +import { writeEnvironmentCleanup } from "../environment-cleanup-notifications.js"; +import { OWNER_GONE_ATTESTATION_KEY, requestOwnerGoneConfirmation } from "./owner-gone-attestation.js"; import { requestNativeAcceptedResultLeaseOperatorAction, settleNativeAcceptedResultLease, @@ -148,6 +151,10 @@ export type OwnableKind = { settle: SettleRecord | null; /** Optional operator action for an unprovable owner of a settling kind. */ onUnprovable?: UnprovableAction; + /** Records the board's confirmation that this record's unrecorded owner stopped, so its + * owner proves gone (`operator_attested`). Returns false when the record no longer qualifies. */ + attestOwnerGone?: (tx: Db, target: { companyId: string; recordId: string }, proof: Record) + => Promise<"attested" | "released" | "changed">; /** When a record of this kind next becomes settleable without a signal * (a retention window ending). The reconciler wakes then. */ nextDueAt?: (db: Db, now: Date) => Promise; @@ -320,6 +327,10 @@ function runLeaseOwner(row: RunLeaseRow): RecordOwner { if (row.runStatus !== null && !TERMINAL_RUN_STATUSES.includes(row.runStatus)) return runOwner(row.heartbeatRunId); const holder = controllerStamp(json(row.metadata)[ENVIRONMENT_LEASE_CONTROLLER_KEY], "environment_run_lease_active"); if (holder.kind !== "none") return holder; + // The board confirmed that the server holding this unstamped lease stopped. + if (json(row.metadata)[OWNER_GONE_ATTESTATION_KEY]) return { kind: "process", stamp: { kind: "local", + scope: "environment_run_lease_active", generation: null, host: null, processSpace: OPERATOR_ATTESTED_PROCESS_SPACE, + bootId: null, pid: null, processStartedAt: null, processGroupId: null } }; return ephemeralServerEnabled() ? { kind: "none" } : runOwner(row.heartbeatRunId); } @@ -336,6 +347,23 @@ function runLeaseKind(): OwnableKind { return { id: "environment_run_lease_active", title: "Run lease (active)", obligation: "ownership", storage: "database", invariantChecks: [1, 2], settle: settleEnvironmentRunLease, + onUnprovable: requestOwnerGoneConfirmation("environment_run_lease_active"), + // Only an unstamped lease of a finished run: a stamped holder is judged by its stamp. + attestOwnerGone: async (tx, target, proof) => { + const attested = (await writeEnvironmentCleanup(tx, write => write.update(environmentLeases) + .set({ metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || ${JSON.stringify({ [OWNER_GONE_ATTESTATION_KEY]: proof })}::jsonb` }) + .where(and(eq(environmentLeases.id, target.recordId), eq(environmentLeases.companyId, target.companyId), + eq(environmentLeases.status, "active"), isNotNull(environmentLeases.heartbeatRunId), + sql`not (coalesce(${environmentLeases.metadata}, '{}'::jsonb) ? ${ENVIRONMENT_LEASE_CONTROLLER_KEY})`, + sql`exists (select 1 from heartbeat_runs run where run.id = ${environmentLeases.heartbeatRunId} + and run.status in ('succeeded', 'failed', 'cancelled', 'timed_out', 'interrupted'))`)) + .returning({ id: environmentLeases.id }), { companyId: target.companyId, leaseId: target.recordId })).length > 0; + if (attested) return "attested"; + // Released by another path since the board was asked: nothing is left to attest. + const [lease] = await tx.select({ status: environmentLeases.status }).from(environmentLeases) + .where(and(eq(environmentLeases.id, target.recordId), eq(environmentLeases.companyId, target.companyId))); + return !lease || lease.status !== "active" ? "released" : "changed"; + }, listInProgress: async (db, limit, offset) => { const rows = await select(db).where(where).orderBy(asc(environmentLeases.id)).limit(limit + 1).offset(offset); const result = page(rows, limit); diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 75e7657415c..16c9654b8c6 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -1,4 +1,5 @@ import { notifyScheduledRetryDispatch } from "../run-dispatch-notifications.js"; +import { OWNER_UNPROVABLE_CAUSE } from "./owner-gone-attestation.js"; import { promoteDeferredWorkIntents } from "../work-intent-outbox.js"; import { isContextCompactionRun } from "../context-compaction.js"; import { notifyNativeEnvironmentSettlement } from "../native-environment-settlement-notifications.js"; @@ -3490,6 +3491,12 @@ export function recoveryService( issueIds: [] as string[], }; for (const { action, issue } of rows) { + // Only the board's stop confirmation, or the resource's own release, settles this request: + // the reserved resource stays reserved however the task ends. + if (action.cause === OWNER_UNPROVABLE_CAUSE) { + result.skipped += 1; + continue; + } if (await hasUnverifiedNativeWorkspaceFinalizationOwner(db, action)) { result.skipped += 1; continue; diff --git a/ui/src/api/issues.ts b/ui/src/api/issues.ts index 803d206ef01..c81e0dedee2 100644 --- a/ui/src/api/issues.ts +++ b/ui/src/api/issues.ts @@ -422,6 +422,8 @@ export const issuesApi = { ), confirmControllerStopped: (id: string, data: { actionId: string; runId: string; stopEvidence: string }) => api.post<{ runId: string; status: "queued" }>(`/issues/${id}/recovery-actions/confirm-controller-stopped`, data), + confirmOwnerGone: (id: string, data: { actionId: string; recordId: string; stopEvidence: string }) => + api.post<{ recordId: string; status: "queued" | "released" }>(`/issues/${id}/recovery-actions/confirm-owner-gone`, data), resumeWorkspaceFinalization: (id: string, data: { actionId: string; runId: string; ownerToken: string; controllerAndCopybackStopped: true; stopEvidence: string }) => api.post<{ runId: string; resultId: string | null; status: "queued" }>(`/issues/${id}/recovery-actions/resume-workspace-finalization`, data), retryWorkspaceExport: (id: string, data: { actionId: string; runId: string; repairNote: string }) => diff --git a/ui/src/components/WorkspaceExportRecovery.control.test.tsx b/ui/src/components/WorkspaceExportRecovery.control.test.tsx index 12dd3ff0008..5c2a158a6ec 100644 --- a/ui/src/components/WorkspaceExportRecovery.control.test.tsx +++ b/ui/src/components/WorkspaceExportRecovery.control.test.tsx @@ -9,9 +9,9 @@ import { queryKeys } from "../lib/queryKeys"; import { writePendingSend } from "../lib/pending-send"; import { WorkspaceExportRecovery } from "./WorkspaceExportRecovery"; -const api = vi.hoisted(() => ({ inspect: vi.fn(), recover: vi.fn(), resume: vi.fn(), confirmController: vi.fn() })); +const api = vi.hoisted(() => ({ inspect: vi.fn(), recover: vi.fn(), resume: vi.fn(), confirmController: vi.fn(), confirmOwnerGone: vi.fn() })); vi.mock("../api/issues", () => ({ issuesApi: { recoveryActions: api.inspect, recoverWorkspace: api.recover, - resumeWorkspaceFinalization: api.resume, confirmControllerStopped: api.confirmController } })); + resumeWorkspaceFinalization: api.resume, confirmControllerStopped: api.confirmController, confirmOwnerGone: api.confirmOwnerGone } })); (globalThis as unknown as { IS_REACT_ACT_ENVIRONMENT: boolean }).IS_REACT_ACT_ENVIRONMENT = true; const action = { @@ -495,6 +495,30 @@ describe("governed retained-file recovery", () => { expect(container.textContent).toContain("Resuming the run…"); }); + it("confirms the unrecorded server holding a stuck resource, for board owners only", async () => { + api.confirmOwnerGone.mockResolvedValue({ recordId: "lease-1", status: "queued" }); + const ownerGoneAction = { ...action, cause: "ownership_owner_unprovable", ownerType: "board", + evidence: { kind: "environment_run_lease_active", recordId: "lease-1", runId: "run-1" } } as unknown as IssueRecoveryAction; + // An operator sees the wait but cannot attest the stop. + await mount(ownerGoneAction); + expect(container.textContent).toContain("before Paperclip stored which server held it"); + expect(button("Confirm the previous server stopped")).toBeUndefined(); + client.setQueryData(queryKeys.access.currentBoardAccess, { userId: "board-1", source: "session", isInstanceAdmin: false, + companyIds: ["company-1"], memberships: [{ companyId: "company-1", membershipRole: "owner", status: "active" }] }); + await mount(ownerGoneAction); + await click("Confirm the previous server stopped"); + const textarea = container.querySelector("textarea")!; + await act(async () => { + const setter = Object.getOwnPropertyDescriptor(HTMLTextAreaElement.prototype, "value")!.set!; + setter.call(textarea, "The old server container was replaced and is not running."); + textarea.dispatchEvent(new Event("input", { bubbles: true })); + }); + await click("Confirm and resume"); + expect(api.confirmOwnerGone).toHaveBeenCalledExactlyOnceWith("issue-1", { actionId: "action-1", recordId: "lease-1", + stopEvidence: "The old server container was replaced and is not running." }); + expect(container.textContent).toContain("Releasing the resource…"); + }); + it("shows status only, without the action, once automatic recovery is resuming or for non-administrators", async () => { // The default fixture is an operator: it may resolve other recovery but not attest a controller stop. await mount({ ...ownerAction, evidence: { ...ownerAction.evidence, workspaceOwnerRecovery: { state: "resuming" } } } as IssueRecoveryAction); diff --git a/ui/src/components/WorkspaceExportRecovery.tsx b/ui/src/components/WorkspaceExportRecovery.tsx index 03ebbc8377e..78684069b42 100644 --- a/ui/src/components/WorkspaceExportRecovery.tsx +++ b/ui/src/components/WorkspaceExportRecovery.tsx @@ -26,11 +26,13 @@ function ReportRecoveryControl({ owns, onChange }: { owns: boolean; onChange?: ( } const CONTROLLER_HOST_UNVERIFIED = "native_controller_host_unverified"; +const OWNER_UNPROVABLE = "ownership_owner_unprovable"; /** Causes whose one action is the board's previous-server stop confirmation; * the generic recovery card must not offer other resolutions for them. */ export function isPreviousServerStopCause(cause: string | null | undefined) { - return cause === "native_workspace_finalization_owner_unverified" || cause === CONTROLLER_HOST_UNVERIFIED; + return cause === "native_workspace_finalization_owner_unverified" || cause === CONTROLLER_HOST_UNVERIFIED + || cause === OWNER_UNPROVABLE; } /** Requests preservation; the existing worker owns stop, copyback and verification. */ @@ -65,6 +67,18 @@ export function WorkspaceExportRecovery({ action, className, issueId, companyId, placeholder: "For example: the old server container was replaced and is no longer running.", resuming: "Resuming the run…" }} />; } + // A resource held by a server recorded before Paperclip stored which server held it. + if (action?.cause === OWNER_UNPROVABLE && issueId && companyId && isOwnerGoneRequired(action, issueId, companyId)) { + return <> + issuesApi.confirmOwnerGone(issueId, { + actionId: action.id, recordId: action.evidence.recordId as string, stopEvidence })} + copy={{ waiting: "A resource from this task’s earlier run was recorded before Paperclip stored which server held it, so Paperclip can’t verify that the server stopped. The resource stays reserved until then.", + question: "How did you confirm that the previous server stopped?", + placeholder: "For example: the old server container was replaced and is no longer running.", + resuming: "Releasing the resource…" }} />; + } const ownerRecovery = action?.cause === "native_workspace_finalization_owner_unverified"; // The task's interruption names this wait; the confirmation is its one action. if (ownerRecovery && issueId && companyId && isOwnerStopRequired(action, issueId, companyId)) { @@ -99,6 +113,12 @@ function isControllerStopRequired(action: IssueRecoveryAction, issueId: string, && typeof action.evidence.runId === "string" && !action.evidence.controllerStop; } +function isOwnerGoneRequired(action: IssueRecoveryAction, issueId: string, companyId: string) { + return action.companyId === companyId && action.sourceIssueId === issueId + && action.ownerType === "board" && ["active", "escalated"].includes(action.status) + && typeof action.evidence.recordId === "string" && !action.evidence.ownerGone; +} + function isOwnerStopRequired(action: IssueRecoveryAction, issueId: string, companyId: string) { return action.companyId === companyId && action.sourceIssueId === issueId && action.ownerType === "board" && ["active", "escalated"].includes(action.status)