From 838e9bf3d2d860989ed3db119da598cebff9e547 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sun, 8 Nov 2026 01:07:58 -0600 Subject: [PATCH] feat(recovery): settle sandbox creation intents of dead creators on the shared ownership path (#19978) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Before it asks a provider to create a sandbox, the server writes a creation intent that names its creator process > - If the creator dies during the provider call, the intent waits for a 30-minute settle deadline before recovery reconciles it with the provider > - After a restart, creation recovery made such intents due early with its own process check (`promoteAbandonedSandboxCreations`) > - Phase 2 of the durable ownership plan moves each in-progress record kind onto one shared owner proof and deletes its bespoke check > - This pull request registers a settle for `sandbox_creation_intent` and calls the shared path where creation recovery called the bespoke promotion > - The benefit is one proof for every kind, while the provider absence proof and the cross-host handoff stay unchanged ## Linked Issues or Issue Description Refs #19964 Refs #19933 **What happened?** Sandbox creation recovery had its own liveness check (`controllerProcessExited`) for the creator of a pending intent. Each kind with its own check can drift from the shared rules (PID space, PID reuse, current boot, deployment replacement). **Expected behavior** The shared prover judges the creator. On gone, the intent becomes due now and its creator is marked lost. Provider absence stays inconclusive until the settle deadline. A creator on another host is never proven gone; its intent keeps the deadline. **Steps to reproduce** 1. Start a sandbox acquire so a creation intent is written. 2. Kill the server before the provider reply is committed. 3. Restart. The intent must become due through the shared ownership path, not a separate check. ## What Changed - `sandbox-creation-intent-settlement.ts` (new): compare-and-swap settle on the exact creator and a deadline that is still in the future. It sets `nextAttemptAt` to now, marks `creationControllerLost` with the proof reason, and wakes creation recovery in the same transaction. - `recovery/ownership-registry.ts`: `sandbox_creation_intent` registers the settle. An intent whose creator is already lost is no longer listed; it belongs to the creation recovery deadline. - `sandbox-creation-operations.ts`: creation recovery calls `settleOwnableKind(db, "sandbox_creation_intent")`. Deleted `promoteAbandonedSandboxCreations` and its use of `controllerProcessExited`. ## Verification - `cd server && npx vitest run src/__tests__/sandbox-creation-intent-ownership.test.ts src/__tests__/sandbox-creation-operations.test.ts src/__tests__/ownership-reconciliation-report.test.ts` (embedded Postgres). - The existing lost-creator matrix stays green on the shared path: destroy on the next sweep; page past unprovable creators; issue-bound creation after a later run; reused PID; live creator here; live creator in another process; creator on another host; early lookup finds nothing yet (absence inconclusive); committed lease untouched. - New, red on master and green here (written after the code): the reconciler pass makes a dead creator's intent due now and keeps the absence deadline. - New counter-cases (green): live creator, another host, and an intent without a creator keep their deadline; a read-only pass changes nothing. - Full typecheck is left to CI (local memory limit). ## Risks - Low risk. The settle writes the same state the deleted promotion wrote, plus the proof reason. - Previous-release rows: every creation intent already stored its creator with a PID space, so no fallback is needed. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, via Claude Code with tool use (shell, file edit, embedded Postgres tests). ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- .../ownership-reconciliation-report.test.ts | 4 +- .../sandbox-creation-intent-ownership.test.ts | 135 ++++++++++++++++++ .../sandbox-creation-operations.test.ts | 4 +- .../services/recovery/ownership-registry.ts | 10 +- .../sandbox-creation-intent-settlement.ts | 55 +++++++ .../services/sandbox-creation-operations.ts | 43 ++---- 6 files changed, 211 insertions(+), 40 deletions(-) create mode 100644 server/src/__tests__/sandbox-creation-intent-ownership.test.ts create mode 100644 server/src/services/sandbox-creation-intent-settlement.ts diff --git a/server/src/__tests__/ownership-reconciliation-report.test.ts b/server/src/__tests__/ownership-reconciliation-report.test.ts index e303af8eaef..f61af00bfe5 100644 --- a/server/src/__tests__/ownership-reconciliation-report.test.ts +++ b/server/src/__tests__/ownership-reconciliation-report.test.ts @@ -142,8 +142,8 @@ describe("report-only ownership reconciliation", () => { expect(new Set(names).size).toBe(names.length); expect(names.length).toBeGreaterThanOrEqual(40); expect(OWNABLE_KINDS.filter(kind => kind.settle !== null).map(kind => kind.id)) - .toEqual(["native_accepted_result_lease", "native_finalizer_lease", "native_cleanup_lease", "environment_run_lease_active", - "environment_teardown_claim", "native_workspace_finalize_attempt", "wake_request_claimed", "legacy_controller_lease"]); + .toEqual(["native_accepted_result_lease", "native_finalizer_lease", "native_cleanup_lease", "sandbox_creation_intent", + "environment_run_lease_active", "environment_teardown_claim", "native_workspace_finalize_attempt", "wake_request_claimed", "legacy_controller_lease"]); expect(OWNABLE_KINDS.filter(kind => !kind.listInProgress).every(kind => kind.coverageGap)).toBe(true); }); diff --git a/server/src/__tests__/sandbox-creation-intent-ownership.test.ts b/server/src/__tests__/sandbox-creation-intent-ownership.test.ts new file mode 100644 index 00000000000..55f6137a670 --- /dev/null +++ b/server/src/__tests__/sandbox-creation-intent-ownership.test.ts @@ -0,0 +1,135 @@ +import { spawn } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; +import { companies, createDb, sandboxCreationOperations } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { currentControllerProcessStamp, currentProcessSpaceHash } from "../services/controller-process-identity.js"; +import { nextEnvironmentCleanupAt } from "../services/environment-cleanup-state.js"; +import { createOwnershipReportState, runOwnershipReconciliationPass } from "../services/recovery/ownership-invariant-report.js"; +import { settleSandboxCreationIntent } from "../services/sandbox-creation-intent-settlement.js"; + +const support = await getEmbeddedPostgresTestSupport(); +const suite = support.supported ? describe : describe.skip; +const KIND = "sandbox_creation_intent"; + +async function exitedPid(): Promise { + const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { stdio: "ignore" }); + const pid = child.pid!; + await new Promise(resolve => { child.once("exit", () => resolve()); child.kill("SIGKILL"); }); + return pid; +} + +suite("sandbox_creation_intent on the shared ownership path", () => { + let db!: ReturnType; + let temporary: Awaited> | null = null; + let companyId = ""; + + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("paperclip-sandbox-creation-intent-ownership-"); + db = createDb(temporary.connectionString); + companyId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Paperclip", issuePrefix: "SCI", + requireBoardApprovalForNewAgents: false }); + }, 30_000); + + afterEach(async () => { + await db.delete(sandboxCreationOperations); + }); + + afterAll(async () => { + await temporary?.cleanup(); + }); + + /** The state a crash between the intent write and the provider reply leaves: + * a pending intent waiting for its 30-minute settle deadline. */ + async function intent(creationController: Record) { + const id = randomUUID(); + const settleAt = new Date(Date.now() + 30 * 60_000); + await db.insert(sandboxCreationOperations).values({ id, companyId, originalEnvironmentId: randomUUID(), + runId: randomUUID(), requestKey: "run", provider: "fixture", driverKey: "fixture", state: "pending", config: {}, + nextAttemptAt: settleAt, metadata: { creationController, creationSettleAt: settleAt.toISOString() } }); + return id; + } + + async function row(id: string) { + const [found] = await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.id, id)); + return found!; + } + + async function pass(record = true) { + const report = await runOwnershipReconciliationPass(db, { state: createOwnershipReportState(), record }); + return report.kinds.find(entry => entry.kind === KIND)!; + } + + it("makes a dead creator's intent due on the next pass, keeping the provider absence deadline", async () => { + const id = await intent({ bootId: "earlier-boot", pid: await exitedPid(), hostHash: currentProcessSpaceHash(), + processStartedAt: null }); + expect(await pass()).toMatchObject({ settled: 1, reasons: { pid_dead: 1 } }); + const settled = await row(id); + expect(settled.state).toBe("pending"); + expect(settled.nextAttemptAt.getTime()).toBeLessThanOrEqual(Date.now()); + expect(settled.metadata).toMatchObject({ creationControllerLost: true, creationControllerLostProof: "pid_dead" }); + // Creation recovery is due now; the saved settle deadline still guards absence. + expect(await nextEnvironmentCleanupAt(db)).toBeLessThanOrEqual(Date.now()); + expect(Date.parse(String(settled.metadata.creationSettleAt))).toBeGreaterThan(Date.now() + 25 * 60_000); + // A lost creator's intent belongs to creation recovery now. + expect(await pass()).toMatchObject({ inProgress: 0 }); + }); + + it("keeps the deadline of a live creator, a creator on another host, and an intent with no creator", async () => { + const ids = [ + await intent(currentControllerProcessStamp()), + await intent({ bootId: "other", pid: await exitedPid(), hostHash: "f".repeat(32), processStartedAt: null }), + await intent({}), + ]; + const before = await Promise.all(ids.map(row)); + expect(await pass()).toMatchObject({ settled: 0, + reasons: { current_process: 1, different_process_space: 1, no_owner_stamp: 1 } }); + expect(await Promise.all(ids.map(row))).toEqual(before); + }); + + describe("the settle compares and swaps on the exact creator it was given", () => { + async function record(id: string, creator: Record) { + return { recordId: id, companyId, runId: null, issueId: null, updatedAt: new Date(), + owner: { kind: "process" as const, stamp: { kind: "local" as const, scope: KIND, generation: null, host: null, + processSpace: String(creator.hostHash), bootId: String(creator.bootId), pid: Number(creator.pid), + processStartedAt: null, processGroupId: null } } }; + } + const proof = { verdict: "gone" as const, reason: "pid_dead" as const }; + + it.each([ + ["a creator that changed since the proof", "owner_changed_since_proof"], + ["an intent already due", "intent_already_due"], + ["an intent no longer pending", "intent_no_longer_pending"], + ["an intent whose creator is already lost", "creator_already_lost"], + ] as const)("leaves %s unchanged", async (label, reason) => { + const creator = { bootId: "earlier-boot", pid: await exitedPid(), hostHash: currentProcessSpaceHash(), processStartedAt: null }; + const id = await intent(creator); + if (label.startsWith("a creator")) { + await db.update(sandboxCreationOperations).set({ metadata: { creationController: { ...creator, bootId: "newer-boot" } } }) + .where(eq(sandboxCreationOperations.id, id)); + } else if (label.includes("already due")) { + await db.update(sandboxCreationOperations).set({ nextAttemptAt: new Date(Date.now() - 1_000) }) + .where(eq(sandboxCreationOperations.id, id)); + } else if (label.includes("no longer pending")) { + await db.update(sandboxCreationOperations).set({ state: "cleanup_pending" }).where(eq(sandboxCreationOperations.id, id)); + } else { + await db.update(sandboxCreationOperations).set({ metadata: { creationController: creator, creationControllerLost: true } }) + .where(eq(sandboxCreationOperations.id, id)); + } + const before = await row(id); + expect(await settleSandboxCreationIntent(db, await record(id, creator), { now: new Date(), proof })) + .toEqual({ settled: false, reason }); + expect(await row(id)).toEqual(before); + }); + }); + + it("never changes an intent on a read-only pass", async () => { + const id = await intent({ bootId: "earlier-boot", pid: await exitedPid(), hostHash: currentProcessSpaceHash(), + processStartedAt: null }); + const before = await row(id); + expect(await pass(false)).toMatchObject({ settled: 0, verdicts: { gone: 1 } }); + expect(await row(id)).toEqual(before); + }); +}); diff --git a/server/src/__tests__/sandbox-creation-operations.test.ts b/server/src/__tests__/sandbox-creation-operations.test.ts index d66102567f0..784f7983a7a 100644 --- a/server/src/__tests__/sandbox-creation-operations.test.ts +++ b/server/src/__tests__/sandbox-creation-operations.test.ts @@ -367,12 +367,12 @@ describe("durable sandbox creation through the runtime entry point", () => { await runtime().retryPendingSandboxTeardown({ environment: null, lease: (await environmentService(db).getLeaseById(lease!.id))! }); expect(resources.size).toBe(0); expect(destroys).toBe(1); expect(creates).toBe(1); }); - it("pages past more unprovable creations than one page to find an abandoned one", async () => { + it("pages past more unprovable creations than one settle page (200) to find an abandoned one", async () => { const pid = await exitedPid(); const operation = await interruptedCreation(async current => ({ ...current, bootId: randomUUID(), pid })); const { id: _id, runId: _runId, ...row } = operation; // Sorted ahead of the abandoned intent: creators on another host cannot be judged. - await db.insert(sandboxCreationOperations).values(Array.from({ length: 105 }, (_, index) => ({ ...row, + await db.insert(sandboxCreationOperations).values(Array.from({ length: 205 }, (_, index) => ({ ...row, id: `00000000-0000-4000-8000-${String(index).padStart(12, "0")}`, runId: randomUUID(), metadata: { ...row.metadata, creationController: { ...(row.metadata.creationController as object), bootId: randomUUID(), pid, hostHash: "f".repeat(32) } } }))); await runtime().flushDeferredOrphanCleanups(); diff --git a/server/src/services/recovery/ownership-registry.ts b/server/src/services/recovery/ownership-registry.ts index aee032246e4..2fde5d7d902 100644 --- a/server/src/services/recovery/ownership-registry.ts +++ b/server/src/services/recovery/ownership-registry.ts @@ -49,6 +49,7 @@ import { } from "../environment-cleanup-state.js"; import { settleEnvironmentRunLease } from "../environment-run-lease-settlement.js"; import { settleEnvironmentTeardownClaim } from "../environment-teardown-claim-settlement.js"; +import { settleSandboxCreationIntent } from "../sandbox-creation-intent-settlement.js"; /** * Registry of ownable record kinds (plan section C). @@ -633,13 +634,18 @@ export const OWNABLE_KINDS: readonly OwnableKind[] = [ // Environments and sandboxes { + // Shared path since Phase 2: a creator proven gone makes its intent due + // now; creation recovery then reconciles it with the provider. An intent + // whose creator is already marked lost belongs to that recovery deadline. id: "sandbox_creation_intent", title: "Sandbox creation intent", obligation: "ownership", storage: "database", - settle: null, invariantChecks: [1, 2, 3, 7], + settle: settleSandboxCreationIntent, invariantChecks: [1, 2, 3, 7], listInProgress: async (db, limit, offset) => { const rows = await db.select({ id: sandboxCreationOperations.id, companyId: sandboxCreationOperations.companyId, runId: sandboxCreationOperations.runId, issueId: sandboxCreationOperations.issueId, metadata: sandboxCreationOperations.metadata, updatedAt: sandboxCreationOperations.updatedAt }) - .from(sandboxCreationOperations).where(eq(sandboxCreationOperations.state, "pending")).orderBy(asc(sandboxCreationOperations.id)).limit(limit + 1).offset(offset); + .from(sandboxCreationOperations).where(and(eq(sandboxCreationOperations.state, "pending"), + sql`coalesce(${sandboxCreationOperations.metadata}->>'creationControllerLost', '') <> 'true'`)) + .orderBy(asc(sandboxCreationOperations.id)).limit(limit + 1).offset(offset); const result = page(rows, limit); return { truncated: result.truncated, records: result.rows.map(row => ({ recordId: row.id, companyId: row.companyId, runId: row.runId, issueId: row.issueId, diff --git a/server/src/services/sandbox-creation-intent-settlement.ts b/server/src/services/sandbox-creation-intent-settlement.ts new file mode 100644 index 00000000000..766aa2d30d9 --- /dev/null +++ b/server/src/services/sandbox-creation-intent-settlement.ts @@ -0,0 +1,55 @@ +import { and, eq, gt, sql } from "drizzle-orm"; +import { sandboxCreationOperations, type Db } from "@paperclipai/db"; +import { readControllerProcess } from "./controller-process-identity.js"; +import { notifyEnvironmentCleanup } from "./environment-cleanup-notifications.js"; +import type { OwnerProof } from "./owner-proof.js"; +import type { InProgressRecord, SettleOutcome } from "./recovery/ownership-registry.js"; + +/** + * Settle of the `sandbox_creation_intent` ownership kind (plan Phase 2). + * + * A creation intent is written before the provider call and names the server + * process that makes it. While that process lives, the intent waits for its + * settle deadline. Once the shared prover proves the creator gone, it cannot + * still be waiting for the provider, so this settle makes the intent due now + * and marks the creator lost; the creation recovery worker then reconciles it. + * + * It settles ownership only. Whether the sandbox exists is still decided by + * the provider: absence stays inconclusive until the creation settle deadline + * (`creationAbsenceInconclusive`), and an unidentifiable creation becomes an + * operator reconciliation. A creator on another host is never proven gone by + * a process proof; its intent keeps the deadline, which is the handoff. + * + * The update compares and swaps on the exact creator the proof judged and + * only moves a deadline that is still in the future. + */ +export async function settleSandboxCreationIntent(db: Db, record: InProgressRecord, + context: { now: Date; proof: OwnerProof }): Promise { + if (record.owner.kind !== "process") return { settled: false, reason: "owner_not_a_process" }; + const stamp = record.owner.stamp; + const now = context.now; + return db.transaction(async transaction => { + const tx = transaction as unknown as Db; + const [operation] = await tx.select().from(sandboxCreationOperations) + .where(and(eq(sandboxCreationOperations.id, record.recordId), eq(sandboxCreationOperations.state, "pending"))) + .for("update"); + if (!operation) return { settled: false, reason: "intent_no_longer_pending" }; + if (operation.metadata.creationControllerLost === true) return { settled: false, reason: "creator_already_lost" }; + const creator = readControllerProcess(operation.metadata.creationController); + if (!creator || creator.bootId !== stamp.bootId || creator.pid !== stamp.pid || creator.hostHash !== stamp.processSpace) { + return { settled: false, reason: "owner_changed_since_proof" }; + } + if (operation.nextAttemptAt <= now) return { settled: false, reason: "intent_already_due" }; + // Wake creation recovery before the write it announces. + await notifyEnvironmentCleanup(tx, { companyId: operation.companyId, runId: operation.runId }); + const updated = await tx.update(sandboxCreationOperations).set({ nextAttemptAt: now, updatedAt: now, + metadata: sql`coalesce(${sandboxCreationOperations.metadata}, '{}'::jsonb) || ${JSON.stringify({ + creationControllerLost: true, creationControllerLostProof: context.proof.reason })}::jsonb` }) + .where(and(eq(sandboxCreationOperations.id, operation.id), eq(sandboxCreationOperations.state, "pending"), + gt(sandboxCreationOperations.nextAttemptAt, now), + sql`coalesce(${sandboxCreationOperations.metadata}->>'creationControllerLost', '') <> 'true'`, + sql`${sandboxCreationOperations.metadata}->'creationController' = ${JSON.stringify(operation.metadata.creationController)}::jsonb`)) + .returning({ id: sandboxCreationOperations.id }); + return updated.length ? { settled: true, reason: context.proof.reason } : { settled: false, reason: "intent_no_longer_pending" }; + }); +} diff --git a/server/src/services/sandbox-creation-operations.ts b/server/src/services/sandbox-creation-operations.ts index 8e11ce48c37..bbc8369962e 100644 --- a/server/src/services/sandbox-creation-operations.ts +++ b/server/src/services/sandbox-creation-operations.ts @@ -1,7 +1,7 @@ import { collectSecretRefPaths, isUuidSecretRef, parseSecretRefBindingObject, readConfigValueAtPath } from "./json-schema-secret-refs.js"; import { createHash, randomUUID } from "node:crypto"; -import { and, asc, eq, gt, lte, or, sql } from "drizzle-orm"; -import { controllerProcessExited, currentControllerBootId, currentControllerProcess, readControllerProcess } from "./controller-process-identity.js"; +import { and, eq, lte, or, sql } from "drizzle-orm"; +import { currentControllerProcess } from "./controller-process-identity.js"; import { companies, environments, sandboxCreationOperations, type Db } from "@paperclipai/db"; import { readEnvironmentCreationCleanupError, type PluginEnvironmentLease, type PluginEnvironmentLeaseObservation } from "@paperclipai/plugin-sdk"; import { assertEnvironmentCompanyBinding, environmentService } from "./environments.js"; @@ -15,37 +15,6 @@ export type SandboxCreationOperation = typeof sandboxCreationOperations.$inferSe // A live creator may still be waiting for the provider, and a provider may still // materialize an aborted request. Absence is conclusive only after this delay. const CREATION_SETTLE_FLOOR_MS = 30 * 60_000; -const CREATION_PROMOTION_PAGE_SIZE = 100; - -/** After a restart, creation intents from an exited server cannot still be in - * flight there. Reconcile them now instead of at the settle deadline. Absence - * stays inconclusive until that deadline; see recoverSandboxCreations. */ -async function promoteAbandonedSandboxCreations(db: Db): Promise { - const now = new Date(); - // Page past live, foreign, and unidentified creators so they cannot hide an - // abandoned intent behind them. - let after: string | null = null; - for (;;) { - const waiting: Array> = await db - .select({ id: sandboxCreationOperations.id, metadata: sandboxCreationOperations.metadata }) - .from(sandboxCreationOperations).where(and( - eq(sandboxCreationOperations.state, "pending"), gt(sandboxCreationOperations.nextAttemptAt, now), - sql`${sandboxCreationOperations.metadata}->'creationController'->>'bootId' is distinct from ${currentControllerBootId()}`, - sql`coalesce(${sandboxCreationOperations.metadata}->>'creationControllerLost', '') <> 'true'`, - after === null ? undefined : gt(sandboxCreationOperations.id, after), - )).orderBy(asc(sandboxCreationOperations.id)).limit(CREATION_PROMOTION_PAGE_SIZE); - for (const operation of waiting) { - const controller = readControllerProcess(operation.metadata.creationController); - if (!controller || !await controllerProcessExited(controller)) continue; - await db.update(sandboxCreationOperations).set({ nextAttemptAt: now, updatedAt: now, - metadata: sql`coalesce(${sandboxCreationOperations.metadata}, '{}'::jsonb) || '{"creationControllerLost":true}'::jsonb` }) - .where(and(eq(sandboxCreationOperations.id, operation.id), eq(sandboxCreationOperations.state, "pending"), - gt(sandboxCreationOperations.nextAttemptAt, now))); - } - if (waiting.length < CREATION_PROMOTION_PAGE_SIZE) return; - after = waiting[waiting.length - 1]!.id; - } -} function creationAbsenceInconclusive(operation: SandboxCreationOperation): boolean { const settleAt = Date.parse(typeof operation.metadata.creationSettleAt === "string" ? operation.metadata.creationSettleAt : ""); @@ -199,7 +168,13 @@ export async function recoverSandboxCreations(db: Db, inspect: (operation: SandboxCreationOperation) => Promise, onTaskRecovered?: (operation: SandboxCreationOperation) => void | Promise, ): Promise { - await promoteAbandonedSandboxCreations(db); + // After a restart, an intent whose creator is proven gone cannot still be in + // flight there: the shared ownership path makes it due now. Absence stays + // inconclusive until the settle deadline (creationAbsenceInconclusive). + const { settleOwnableKind } = await import("./recovery/ownership-invariant-report.js"); + await settleOwnableKind(db, "sandbox_creation_intent").catch(() => { + logger.warn({ errorKind: "sandbox_creation_intent_settle_failed" }, "Sandbox creation intents keep their saved deadlines"); + }); const due = await db.select().from(sandboxCreationOperations).where(and( recoverableSandboxCreationCondition(), lte(sandboxCreationOperations.nextAttemptAt, new Date()), )).limit(100);