diff --git a/doc/DATABASE.md b/doc/DATABASE.md index 0b5a64bffd6..5b9623f1be0 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -2422,8 +2422,13 @@ the optional live environment FK uses SET NULL. Company/run deletion must not erase an outstanding remote-resource obligation. The unique company/environment/run/request index fences duplicate creation and -ambiguous successful lease commits. Normal lease attachment and operation transfer -share a transaction. Recovery transfers one claimed cleanup lease after provider +ambiguous successful lease commits. Since migration 0475 it covers only operations +that are not `resolved`. A later attempt for the same run and request that meets an +`attached` operation whose lease is closed checks the same stop proof same-run +recovery requires (a termination receipt for that sandbox, stored or derived from a +destroyed reusable successor); with it, one transaction resolves the operation and +records the replacement. +Normal lease attachment and operation transfer share a transaction. Recovery transfers one claimed cleanup lease after provider ownership lookup. The pending deadline index drives the existing cleanup worker. `needs_reconciliation` is a durable manual outcome, not an automatic retry or an idle blocker. See [cleanup ownership](idle-environment-cleanup.md). diff --git a/doc/idle-environment-cleanup.md b/doc/idle-environment-cleanup.md index d999536cdfc..8ff92df65bb 100644 --- a/doc/idle-environment-cleanup.md +++ b/doc/idle-environment-cleanup.md @@ -70,8 +70,18 @@ writers and another server process are not covered by these process-local signal A creation operation retains company, original environment, run, request identity, provider, and nonsecret configuration references. Its environment FK uses SET NULL; company/run/task identifiers remain after those records are removed. One operation -owns each company/environment/run/request. A retry of an attached operation returns -its active lease. Pending or unresolved ownership never authorizes another create. +owns each company/environment/run/request until it is resolved. A retry of an +attached operation returns its active lease. Pending or unresolved ownership never +authorizes another create. An attached operation resolves when its lease is proven +stopped, checked at the next creation with the same key: the closed lease must +carry the stop proof same-run recovery requires (`remoteExecutionHasStopped`), a +termination receipt for the sandbox this operation created, whether release, the +Runner `complete` operation, or a destroyed reusable successor supplied it. That +creation resolves the operation and records its replacement in one transaction, +so concurrent recoveries make one provider call. A recovered cleanup operation +resolves when its cleanup lease settles. Failed cleanup, success without a +receipt, a receipt for another sandbox, a retained lease, and pending cleanup keep +the operation and its key. Normal success transfers operation ownership and the lease in one transaction. Current company binding and run authority are checked again at attachment. Declared secret-reference fields retain only their references and nonsecret value diff --git a/packages/db/src/migrations/0475_sandbox_creation_owner_unresolved.sql b/packages/db/src/migrations/0475_sandbox_creation_owner_unresolved.sql new file mode 100644 index 00000000000..a753f04184a --- /dev/null +++ b/packages/db/src/migrations/0475_sandbox_creation_owner_unresolved.sql @@ -0,0 +1,3 @@ +-- paperclip:migration-compat online_compatible: The new index is the same unique key restricted to rows whose state is not 'resolved', so every write the previous release can make still satisfies it, and that release's untargeted ON CONFLICT DO NOTHING matches a partial unique index as well. The drop and rebuild share one transaction, so no write sees the key unfenced; the table holds one small row per sandbox creation, so the exclusive lock lasts only for that index build. +DROP INDEX "sandbox_creation_operations_owner_idx";--> statement-breakpoint +CREATE UNIQUE INDEX "sandbox_creation_operations_owner_idx" ON "sandbox_creation_operations" USING btree ("company_id","original_environment_id","run_id","request_key") WHERE "sandbox_creation_operations"."state" <> 'resolved'; diff --git a/packages/db/src/migrations/meta/0470_snapshot.json b/packages/db/src/migrations/meta/0475_snapshot.json similarity index 99% rename from packages/db/src/migrations/meta/0470_snapshot.json rename to packages/db/src/migrations/meta/0475_snapshot.json index 5cc01ce9202..7b5ecc397be 100644 --- a/packages/db/src/migrations/meta/0470_snapshot.json +++ b/packages/db/src/migrations/meta/0475_snapshot.json @@ -1,6 +1,6 @@ { - "id": "f0e17e47-6217-4080-b6b6-9d83b33e9ced", - "prevId": "ac25715a-74c6-4988-8525-63ad14781c74", + "id": "ee1f1f66-0cbe-4b18-b364-5e96708a1f0b", + "prevId": "a7108924-4d75-41e2-9139-f43f9396c02d", "version": "7", "dialect": "postgresql", "tables": { @@ -22342,6 +22342,18 @@ "type": "timestamp with time zone", "primaryKey": false, "notNull": false + }, + "dispatching_at": { + "name": "dispatching_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "dispatch_lease_expires_at": { + "name": "dispatch_lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false } }, "indexes": { @@ -53674,6 +53686,12 @@ "primaryKey": false, "notNull": true }, + "pending_delete_object_key": { + "name": "pending_delete_object_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, "created_at": { "name": "created_at", "type": "timestamp with time zone", @@ -53697,6 +53715,21 @@ "concurrently": false, "method": "btree", "with": {} + }, + "runner_api_response_reservations_asset_idx": { + "name": "runner_api_response_reservations_asset_idx", + "columns": [ + { + "expression": "asset_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} } }, "foreignKeys": { @@ -53909,6 +53942,7 @@ } ], "isUnique": true, + "where": "\"sandbox_creation_operations\".\"state\" <> 'resolved'", "concurrently": false, "method": "btree", "with": {} @@ -57502,6 +57536,12 @@ "notNull": true, "default": "'{}'::jsonb" }, + "ui_meta": { + "name": "ui_meta", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, "risk_level": { "name": "risk_level", "type": "text", @@ -59627,6 +59667,253 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.tool_mcp_app_views": { + "name": "tool_mcp_app_views", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "invocation_id": { + "name": "invocation_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "issue_id": { + "name": "issue_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "agent_id": { + "name": "agent_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "responsible_user_id": { + "name": "responsible_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "connection_id": { + "name": "connection_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "catalog_entry_id": { + "name": "catalog_entry_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "application_name": { + "name": "application_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tool_name": { + "name": "tool_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "upstream_tool_name": { + "name": "upstream_tool_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_uri": { + "name": "resource_uri", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tool_input": { + "name": "tool_input", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "tool_result": { + "name": "tool_result", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "tool_mcp_app_views_issue_idx": { + "name": "tool_mcp_app_views_issue_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "tool_mcp_app_views_connection_idx": { + "name": "tool_mcp_app_views_connection_idx", + "columns": [ + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "tool_mcp_app_views_company_id_companies_id_fk": { + "name": "tool_mcp_app_views_company_id_companies_id_fk", + "tableFrom": "tool_mcp_app_views", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "tool_mcp_app_views_invocation_id_tool_invocations_id_fk": { + "name": "tool_mcp_app_views_invocation_id_tool_invocations_id_fk", + "tableFrom": "tool_mcp_app_views", + "tableTo": "tool_invocations", + "columnsFrom": [ + "invocation_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "tool_mcp_app_views_issue_id_issues_id_fk": { + "name": "tool_mcp_app_views_issue_id_issues_id_fk", + "tableFrom": "tool_mcp_app_views", + "tableTo": "issues", + "columnsFrom": [ + "issue_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "tool_mcp_app_views_run_id_heartbeat_runs_id_fk": { + "name": "tool_mcp_app_views_run_id_heartbeat_runs_id_fk", + "tableFrom": "tool_mcp_app_views", + "tableTo": "heartbeat_runs", + "columnsFrom": [ + "run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "tool_mcp_app_views_agent_id_agents_id_fk": { + "name": "tool_mcp_app_views_agent_id_agents_id_fk", + "tableFrom": "tool_mcp_app_views", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "tool_mcp_app_views_connection_id_tool_connections_id_fk": { + "name": "tool_mcp_app_views_connection_id_tool_connections_id_fk", + "tableFrom": "tool_mcp_app_views", + "tableTo": "tool_connections", + "columnsFrom": [ + "connection_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "tool_mcp_app_views_catalog_entry_id_tool_catalog_entries_id_fk": { + "name": "tool_mcp_app_views_catalog_entry_id_tool_catalog_entries_id_fk", + "tableFrom": "tool_mcp_app_views", + "tableTo": "tool_catalog_entries", + "columnsFrom": [ + "catalog_entry_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.tool_mcp_gateway_tokens": { "name": "tool_mcp_gateway_tokens", "schema": "", @@ -60341,6 +60628,334 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.tool_mcp_skill_files": { + "name": "tool_mcp_skill_files", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "connection_id": { + "name": "connection_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "file_uri": { + "name": "file_uri", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "digest": { + "name": "digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "content_base64": { + "name": "content_base64", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "tool_mcp_skill_files_digest_idx": { + "name": "tool_mcp_skill_files_digest_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "file_uri", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "digest", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "tool_mcp_skill_files_company_id_companies_id_fk": { + "name": "tool_mcp_skill_files_company_id_companies_id_fk", + "tableFrom": "tool_mcp_skill_files", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "tool_mcp_skill_files_connection_id_tool_connections_id_fk": { + "name": "tool_mcp_skill_files_connection_id_tool_connections_id_fk", + "tableFrom": "tool_mcp_skill_files", + "tableTo": "tool_connections", + "columnsFrom": [ + "connection_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tool_mcp_skills": { + "name": "tool_mcp_skills", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "connection_id": { + "name": "connection_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "skill_uri": { + "name": "skill_uri", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "frontmatter": { + "name": "frontmatter", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "manifest": { + "name": "manifest", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "manifest_digest": { + "name": "manifest_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending_review'" + }, + "status_reason": { + "name": "status_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "approved_manifest_digest": { + "name": "approved_manifest_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "allowed_tools_approved": { + "name": "allowed_tools_approved", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "reviewed_at": { + "name": "reviewed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "reviewed_by_user_id": { + "name": "reviewed_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "tool_mcp_skills_connection_uri_idx": { + "name": "tool_mcp_skills_connection_uri_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "skill_uri", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "tool_mcp_skills_company_status_idx": { + "name": "tool_mcp_skills_company_status_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "tool_mcp_skills_company_id_companies_id_fk": { + "name": "tool_mcp_skills_company_id_companies_id_fk", + "tableFrom": "tool_mcp_skills", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "tool_mcp_skills_connection_id_tool_connections_id_fk": { + "name": "tool_mcp_skills_connection_id_tool_connections_id_fk", + "tableFrom": "tool_mcp_skills", + "tableTo": "tool_connections", + "columnsFrom": [ + "connection_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.tool_oauth_states": { "name": "tool_oauth_states", "schema": "", diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index 6e6a9dde60f..0d61b8089e7 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -3312,6 +3312,13 @@ "when": 1794524253117, "tag": "0474_milky_wolf_cub", "breakpoints": true + }, + { + "idx": 475, + "version": "7", + "when": 1794532772498, + "tag": "0475_sandbox_creation_owner_unresolved", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/db/src/schema/sandbox_creation_operations.ts b/packages/db/src/schema/sandbox_creation_operations.ts index bb81e503793..949f2f34b04 100644 --- a/packages/db/src/schema/sandbox_creation_operations.ts +++ b/packages/db/src/schema/sandbox_creation_operations.ts @@ -23,5 +23,8 @@ export const sandboxCreationOperations = pgTable("sandbox_creation_operations", updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), }, table => ({ pendingDeadline: index("sandbox_creation_operations_pending_deadline_idx").on(table.nextAttemptAt).where(sql`${table.state} = 'pending'`), - owner: uniqueIndex("sandbox_creation_operations_owner_idx").on(table.companyId, table.originalEnvironmentId, table.runId, table.requestKey), + // A resolved operation's sandbox is proven stopped, so it no longer holds its + // creation key: a later attempt for the same run and request may create again. + owner: uniqueIndex("sandbox_creation_operations_owner_idx").on(table.companyId, table.originalEnvironmentId, table.runId, table.requestKey) + .where(sql`${table.state} <> 'resolved'`), })); diff --git a/server/src/__tests__/sandbox-creation-operations.test.ts b/server/src/__tests__/sandbox-creation-operations.test.ts index af31da1b260..6a34fe65298 100644 --- a/server/src/__tests__/sandbox-creation-operations.test.ts +++ b/server/src/__tests__/sandbox-creation-operations.test.ts @@ -18,6 +18,8 @@ import * as secretOwner from "../services/secrets.js"; import { readIdleSleepSafety } from "../services/idle-sleep-safety.js"; import { nextEnvironmentCleanupAt } from "../services/environment-cleanup-state.js"; import { readProcessStartedAt } from "../services/hot-restart.js"; +import { hasRemoteTerminationReceipt, remoteTerminationReceipt, resolveRemoteTerminationLease } from "../services/remote-execution-termination.js"; +import { createReusableLeaseHandoff } from "../services/environment-lease-handoff.js"; import type { PluginWorkerManager } from "../services/plugin-worker-manager.js"; const external = resolveExternalTestDatabaseUrl("PAPERCLIP_HISTORY_TEST_DATABASE_URL"); @@ -164,6 +166,11 @@ describe("durable sandbox creation through the runtime entry point", () => { } expect(await db.select().from(sandboxCreationOperations)).toEqual([before]); expect(creates).toBe(1); + // The later migration narrowed the key to unresolved operations; a reapply keeps that. + const indexes = await db.execute(sql`select indexdef from pg_indexes where indexname = 'sandbox_creation_operations_owner_idx'`); + expect([...indexes].map(row => String((row as { indexdef: unknown }).indexdef))).toEqual([ + expect.stringContaining("WHERE (state <> 'resolved'::text)"), + ]); }); it("fails before the external action when the durable admission write fails", async () => { await rejectWrites("sandbox_creation_operations", "reject_fixture_creation"); @@ -178,6 +185,129 @@ describe("durable sandbox creation through the runtime entry point", () => { expect(await db.select().from(environmentLeases)).toHaveLength(1); expect(await nextEnvironmentCleanupAt(db)).toBeNull(); }); + describe("same-run recovery after the attached lease closes", () => { + type Lease = Awaited>["lease"]; + const stopReceipt = (lease: Lease) => remoteTerminationReceipt(lease, { providerLeaseId: lease.providerLeaseId, state: "stopped" }); + const operations = () => db.select().from(sandboxCreationOperations); + async function acquirePlugin() { + await db.update(environments).set({ driver: "plugin", config: { pluginKey: "paperclip.creation-fixture", driverKey: "creation-fixture", driverConfig: {} } }).where(eq(environments.id, environmentId)); + const [plugin] = await db.select().from(plugins).where(eq(plugins.id, pluginId)); + await db.update(plugins).set({ manifestJson: { ...plugin.manifestJson, environmentDrivers: [{ driverKey: "creation-fixture", kind: "environment_driver", displayName: "Computer", creationOperationVersion: 1, configSchema: { type: "object" } }] } }).where(eq(plugins.id, pluginId)); + return runtime().acquireRunLease({ companyId, environment: (await environmentService(db).getById(environmentId))!, + heartbeatRunId: runId, issueId: null, persistedExecutionWorkspace: null }); + } + /** The closing write left the operation attached; the next creation with the + * same key proves the stop, resolves it, and creates exactly one replacement. */ + async function expectOneReplacement(first: { lease: Lease }, acquireLease: () => Promise<{ lease: Lease }>) { + const [before] = await operations(); + expect(before).toMatchObject({ state: "attached", leaseId: first.lease.id }); + const second = await acquireLease(); + expect(second.lease.id).not.toBe(first.lease.id); + expect(second.lease.providerLeaseId).not.toBe(first.lease.providerLeaseId); + expect(creates).toBe(2); + expect(await operations()).toEqual(expect.arrayContaining([ + expect.objectContaining({ id: before!.id, state: "resolved", leaseId: first.lease.id }), + expect.objectContaining({ state: "attached", leaseId: second.lease.id, runId, requestKey: before!.requestKey }), + ])); + // A retry of the replacement replays it instead of creating again. + expect((await acquireLease()).lease.id).toBe(second.lease.id); + expect(creates).toBe(2); + } + + it.each([["sandbox", acquire], ["plugin", acquirePlugin]] as const)("creates one replacement once a stop receipt closes the %s lease", async (_driver, acquireLease) => { + const first = await acquireLease(); + expect(await environmentService(db).releaseLease(first.lease.id, "released", { cleanupStatus: "success", remoteExecutionTermination: stopReceipt(first.lease) })) + .toMatchObject({ status: "released" }); + await expectOneReplacement(first, acquireLease); + }); + + it("creates one replacement after the Runner complete operation stops a retained reusable lease", async () => { + const first = await acquirePlugin(); + await db.update(environmentLeases).set({ status: "retained", leasePolicy: "reuse_by_environment", releasedAt: null }) + .where(eq(environmentLeases.id, first.lease.id)); + // The same write the `complete` operation makes for each reusable owner of a + // stopped task: no releaseLease call. + const [owner] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, first.lease.id)); + await db.update(environmentLeases).set({ status: "released", releasedAt: new Date(), cleanupStatus: "success", + metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || ${JSON.stringify({ + remoteExecutionTermination: remoteTerminationReceipt(owner!, { providerLeaseId: owner!.providerLeaseId, state: "stopped" }), + runnerTaskExecutionStopped: true })}::jsonb` }).where(eq(environmentLeases.id, first.lease.id)); + await expectOneReplacement(first, acquirePlugin); + }); + + it("creates one replacement when only the destroyed successor proves an expired reusable lease stopped", async () => { + const first = await acquire(); + const [{ agentId }] = await db.select({ agentId: agents.id }).from(agents).where(eq(agents.companyId, companyId)); + const issueId = randomUUID(), successorRunId = randomUUID(); + await db.insert(issues).values({ id: issueId, companyId, title: "Reused sandbox", status: "done" }); + await db.insert(heartbeatRuns).values({ id: successorRunId, companyId, agentId, invocationSource: "on_demand", status: "succeeded" }); + const acquiredAt = new Date(Date.now() - 2000), handoffAt = new Date(Date.now() - 1000); + const [previous] = await db.update(environmentLeases).set({ status: "expired", leasePolicy: "reuse_by_environment", issueId, + acquiredAt, releasedAt: handoffAt, cleanupStatus: "success", metadata: sql`${environmentLeases.metadata} || ${JSON.stringify({ agentId })}::jsonb` }) + .where(eq(environmentLeases.id, first.lease.id)).returning(); + const successor = { ...previous!, id: randomUUID(), heartbeatRunId: successorRunId, acquiredAt: handoffAt, + status: "released" as const, releasedAt: new Date(), metadata: { agentId } }; + const handoff = createReusableLeaseHandoff(previous!, successor); + await db.update(environmentLeases).set({ metadata: sql`${environmentLeases.metadata} || ${JSON.stringify({ reusableLeaseSuccessor: handoff })}::jsonb` }) + .where(eq(environmentLeases.id, first.lease.id)); + await db.insert(environmentLeases).values({ ...successor, metadata: { agentId, reusableLeasePredecessor: handoff, + remoteExecutionTermination: remoteTerminationReceipt(successor, { providerLeaseId: successor.providerLeaseId, state: "destroyed" }) } }); + const [stored] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, first.lease.id)); + // No receipt is stored on the expired lease; recovery computes it from the successor. + expect(hasRemoteTerminationReceipt(stored!)).toBe(false); + expect(hasRemoteTerminationReceipt(await resolveRemoteTerminationLease(db, stored!))).toBe(true); + await expectOneReplacement(first, acquire); + }); + + it.each([ + ["failed cleanup", "failed", { cleanupStatus: "failed", receipt: true }], + ["cleanup success without a receipt", "released", { cleanupStatus: "success", receipt: false }], + ["a receipt for another lease", "released", { cleanupStatus: "success", receipt: "foreign" }], + ["a receipt for another provider task", "released", { cleanupStatus: "success", receipt: "other-task" }], + ["a retained lease", "retained", { cleanupStatus: "success", receipt: true }], + ["pending cleanup", "pending_cleanup", { cleanupStatus: "failed", receipt: false }], + ] as const)("keeps the creation key after %s", async (_case, status, options) => { + const first = await acquire(); + const receipt = options.receipt === "foreign" ? stopReceipt({ ...first.lease, id: randomUUID() }) + : options.receipt === "other-task" ? { ...stopReceipt(first.lease), providerLeaseId: "other-task" } + : options.receipt ? stopReceipt(first.lease) : undefined; + await environmentService(db).releaseLease(first.lease.id, status, { cleanupStatus: options.cleanupStatus, + ...(receipt ? { remoteExecutionTermination: receipt } : {}) }); + await expect(acquire()).rejects.toThrow("Sandbox creation has already transferred ownership"); + expect(await operations()).toEqual([expect.objectContaining({ state: "attached", leaseId: first.lease.id })]); + expect(creates).toBe(1); + }); + + it("keeps an earlier task's operation when the lease now holds a replacement task", async () => { + const first = await acquire(); + await db.update(sandboxCreationOperations).set({ metadata: sql`metadata || '{"creationProviderLeaseId":"earlier-task"}'::jsonb` }); + await environmentService(db).releaseLease(first.lease.id, "released", { cleanupStatus: "success", remoteExecutionTermination: stopReceipt(first.lease) }); + await expect(acquire()).rejects.toThrow("Sandbox creation has already transferred ownership"); + expect(await operations()).toEqual([expect.objectContaining({ state: "attached", leaseId: first.lease.id })]); + expect(creates).toBe(1); + }); + + it("lets only one of two concurrent recoveries create the replacement", async () => { + const first = await acquire(); + await environmentService(db).releaseLease(first.lease.id, "released", { cleanupStatus: "success", remoteExecutionTermination: stopReceipt(first.lease) }); + // Hold whichever recovery reaches the provider until the other one has settled. + let settleOther!: () => void; + const otherSettled = new Promise(resolve => { settleOther = resolve; }); + beforeReply = () => otherSettled; + const outcomes = [acquire(), acquire()].map(attempt => attempt.then( + value => ({ value, error: undefined }), (error: unknown) => ({ value: undefined, error }))); + const loser = await Promise.race(outcomes); + settleOther(); + const settled = await Promise.all(outcomes); + expect(loser.error).toEqual(expect.objectContaining({ message: "Sandbox creation requires reconciliation before another attempt" })); + expect(settled.filter(outcome => outcome.value)).toHaveLength(1); + expect(creates).toBe(2); + expect((await operations()).filter(operation => operation.state !== "resolved")).toEqual([ + expect.objectContaining({ state: "attached", leaseId: settled.find(outcome => outcome.value)!.value!.lease.id }), + ]); + expect(await operations()).toHaveLength(2); + }); + }); it("recovers a failed lease write with a fresh receiver and empty web disk", async () => { await rejectWrites("environment_leases", "reject_fixture_lease"); await expect(acquire()).rejects.toThrow(); diff --git a/server/src/services/sandbox-creation-operations.ts b/server/src/services/sandbox-creation-operations.ts index 4ab1d20048b..dd8ec2af093 100644 --- a/server/src/services/sandbox-creation-operations.ts +++ b/server/src/services/sandbox-creation-operations.ts @@ -1,14 +1,15 @@ import { collectSecretRefPaths, isUuidSecretRef, parseSecretRefBindingObject, readConfigValueAtPath } from "./json-schema-secret-refs.js"; import { createHash, randomUUID } from "node:crypto"; -import { and, eq, lte, or, sql } from "drizzle-orm"; +import { and, eq, lte, ne, or, sql } from "drizzle-orm"; import { currentControllerProcess } from "./controller-process-identity.js"; -import { companies, environments, heartbeatRuns, sandboxCreationOperations, type Db } from "@paperclipai/db"; +import { companies, environmentLeases, environments, heartbeatRuns, sandboxCreationOperations, type Db } from "@paperclipai/db"; import { readEnvironmentCreationCleanupError, type PluginEnvironmentLease, type PluginEnvironmentLeaseObservation } from "@paperclipai/plugin-sdk"; import { assertEnvironmentCompanyBinding, environmentService } from "./environments.js"; import { ENVIRONMENT_CLEANUP_RETRY_MS, recoverableSandboxCreationCondition } from "./environment-cleanup-state.js"; import { logger } from "../middleware/logger.js"; import { notifyEnvironmentCleanup, writeEnvironmentCleanup } from "./environment-cleanup-notifications.js"; import { crashPoint } from "@paperclipai/shared/crash-points"; +import { hasRemoteTerminationReceipt, resolveRemoteTerminationLease } from "./remote-execution-termination.js"; export type SandboxCreationOperation = typeof sandboxCreationOperations.$inferSelect; @@ -70,9 +71,14 @@ export async function createOwnedSandbox( const runId = owner.heartbeatRunId ?? randomUUID(); const settleAt = new Date(Date.now() + Math.max(CREATION_SETTLE_FLOOR_MS, (owner.timeoutMs ?? 0) + 60_000)); const creationController = await currentControllerProcess(); - const operation = await db.transaction(async tx => { + const admit = (stopped?: SandboxCreationOperation) => db.transaction(async tx => { await notifyEnvironmentCleanup(tx, { companyId: owner.companyId, runId: owner.heartbeatRunId ?? undefined }); await assertEnvironmentCompanyBinding(tx as unknown as Db, owner.environmentId, owner.companyId); + // Only one concurrent attempt resolves the stopped operation; any other one + // then meets the replacement below and is fenced exactly as before. + if (stopped) await tx.update(sandboxCreationOperations).set({ state: "resolved", updatedAt: new Date() }).where(and( + eq(sandboxCreationOperations.id, stopped.id), eq(sandboxCreationOperations.state, "attached"), + eq(sandboxCreationOperations.leaseId, stopped.leaseId!))); const [inserted] = await tx.insert(sandboxCreationOperations).values({ id, companyId: owner.companyId, environmentId: owner.environmentId, originalEnvironmentId: owner.environmentId, runId, requestKey: owner.requestKey ?? "run", issueId: owner.issueId, executionWorkspaceId: owner.executionWorkspaceId, @@ -85,10 +91,13 @@ export async function createOwnedSandbox( const [existing] = await tx.select().from(sandboxCreationOperations).where(and( eq(sandboxCreationOperations.companyId, owner.companyId), eq(sandboxCreationOperations.originalEnvironmentId, owner.environmentId), eq(sandboxCreationOperations.runId, runId), eq(sandboxCreationOperations.requestKey, owner.requestKey ?? "run"), + ne(sandboxCreationOperations.state, "resolved"), )); if (!existing) throw new Error("Sandbox creation ownership was not recorded"); return existing; }); + let operation = await admit(); + if (operation.id !== id && await createdSandboxStopped(db, operation)) operation = await admit(operation); if (operation.id !== id) { if (operation.state !== "attached" || !operation.leaseId) throw new Error("Sandbox creation requires reconciliation before another attempt"); const existing = await environmentService(db).getLeaseById(operation.leaseId); @@ -116,6 +125,20 @@ export async function createOwnedSandbox( } } +/** An attached operation is finished once its closed lease carries the same + * stop proof same-run recovery requires (remoteExecutionHasStopped) for the + * sandbox this operation created. A task replacement can leave an earlier + * operation attached to a lease that now holds another sandbox; that operation + * is not proven and keeps its key. */ +async function createdSandboxStopped(db: Db, operation: SandboxCreationOperation): Promise { + if (operation.state !== "attached" || !operation.leaseId) return false; + const [lease] = await db.select().from(environmentLeases).where(and( + eq(environmentLeases.id, operation.leaseId), eq(environmentLeases.companyId, operation.companyId))); + if (!lease || lease.status === "active" || !lease.providerLeaseId) return false; + const created = operation.metadata.creationProviderLeaseId ?? lease.providerLeaseId; + return created === lease.providerLeaseId && hasRemoteTerminationReceipt(await resolveRemoteTerminationLease(db, lease)); +} + /** Keep an already returned resource identity even when task attachment fails. * This enriches the existing owner; it never transfers or reopens ownership. */ export async function recordSandboxCreationResource(db: Db, input: {