From 5c2764634ff1c90e6cc0a1cb2c6ec8a73d601536 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 3 Nov 2026 22:32:10 -0600 Subject: [PATCH] feat(server): preserve native Runner state across ephemeral deploys Co-Authored-By: Paperclip --- doc/DATABASE.md | 27 + doc/RUNNER-INTEGRATION-PLAYBOOK.md | 7 + .../db/src/migrations/0429_high_freak.sql | 15 + ...{0424_snapshot.json => 0429_snapshot.json} | 484 +++++++++++++++++- packages/db/src/migrations/meta/_journal.json | 7 + packages/db/src/schema/index.ts | 1 + .../db/src/schema/native_harness_backups.ts | 19 + .../durable-prp-control-plane.test.ts | 35 ++ .../durable-prp-control-plane.ts | 27 + packages/paperclip-runner/src/index.ts | 1 + .../ephemeral-server-contract.test.ts | 12 +- .../maintenance/ephemeral-migration-data.ts | 13 +- server/src/services/environment-runtime.ts | 14 +- .../src/services/ephemeral-server-contract.ts | 4 + .../heartbeat/runtime-selection.test.ts | 47 +- .../services/heartbeat/runtime-selection.ts | 7 +- .../heartbeat/upgrade-readiness.test.ts | 7 +- .../services/heartbeat/upgrade-readiness.ts | 6 +- .../heartbeat/workspace-preparation.test.ts | 2 +- ...e-controller-migration.integration.test.ts | 106 ++++ .../native-controller-migration.ts | 142 +++++ .../native-controller-recovery-inventory.ts | 6 +- .../native-controller-recovery.ts | 52 +- .../native-harness-backup-archive.test.ts | 46 ++ .../native-harness-backup-archive.ts | 79 +++ .../native-harness-backup-expiration.ts | 78 ++- .../native-harness-backup-migration.ts | 76 +++ .../native-harness-backup-stamp.ts | 20 +- ...harness-backup-storage.integration.test.ts | 175 +++++++ .../native-harness-backup-storage.ts | 221 ++++++++ .../native-runner-checkpoint.ts | 7 + .../native-runtime/native-session-executor.ts | 36 +- .../native-runtime/native-session-resume.ts | 4 +- server/src/services/remote-workspaces.ts | 14 +- server/src/vendor/paperclip-runner/index.ts | 1 + tests/runner-e2e/README.md | 13 + tests/runner-e2e/catalog.ts | 52 +- .../controller-recovery-observer.test.ts | 11 +- .../controller-recovery-observer.ts | 15 +- .../controller-rolling-deployment.test.ts | 24 + .../controller-rolling-deployment.ts | 4 +- tests/runner-e2e/everyday-cases.ts | 1 + tests/runner-e2e/everyday-flow.ts | 13 +- .../everyday-remote-restart.test.ts | 12 + tests/runner-e2e/everyday-remote-restart.ts | 5 +- .../remote-workspace-storage.test.ts | 30 ++ tests/runner-e2e/remote-workspace-storage.ts | 31 +- 47 files changed, 1902 insertions(+), 107 deletions(-) create mode 100644 packages/db/src/migrations/0429_high_freak.sql rename packages/db/src/migrations/meta/{0424_snapshot.json => 0429_snapshot.json} (99%) create mode 100644 packages/db/src/schema/native_harness_backups.ts create mode 100644 server/src/services/native-runtime/native-controller-migration.integration.test.ts create mode 100644 server/src/services/native-runtime/native-controller-migration.ts create mode 100644 server/src/services/native-runtime/native-harness-backup-archive.test.ts create mode 100644 server/src/services/native-runtime/native-harness-backup-archive.ts create mode 100644 server/src/services/native-runtime/native-harness-backup-migration.ts create mode 100644 server/src/services/native-runtime/native-harness-backup-storage.integration.test.ts create mode 100644 server/src/services/native-runtime/native-harness-backup-storage.ts diff --git a/doc/DATABASE.md b/doc/DATABASE.md index b3f7070103c..00e16bd69be 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -1,5 +1,32 @@ # Database +### Durable native provider backups + +`native_harness_backups` journals encrypted provider-session archives in object +storage. One current row per company and session scope selects the published +backup. Pending uploads remain recorded after failure; a controller must still +own the native run when it publishes. Superseded controllers cannot replace the +selected backup. Archive bytes never enter the database. + +The dedicated volume migration imports verified surviving backups without +removing their source files or changing task-session pointers. Replacement +readiness rejects retained session pointers whose backups remain local. The +existing bounded startup retention pass retires closed sessions only after its +task, run, lease and age checks pass, then acknowledges object deletion before +removing the ledger row. It adds no idle polling. + +The same offline maintenance process imports surviving legacy controller state +into `native_runner_checkpoints` and binds its existing session snapshot through +`nativeControllerRecovery` metadata. Runner validates the original protocol +state and recorded artifact; Core requires a finished run with no live owner. +Blocked or pending recovery records remain unchanged while their surviving +state is preserved; migration does not mark their work completed. +The normal bounded continuation selector handles inherited, unstarted follow-ups. +Never-started queued turns keep their task pointers and remain queued. Original +files stay intact, canonical database checkpoints take precedence, and missing +source files are reported separately rather than fabricated. This migration +does not recover provider state that was already lost before enrollment. + ### Proactive agent chat messages Migration `0419_slippery_impossible_man.sql` adds an agent-scoped unique constraint diff --git a/doc/RUNNER-INTEGRATION-PLAYBOOK.md b/doc/RUNNER-INTEGRATION-PLAYBOOK.md index 1480ebc49c9..d3f39a9418a 100644 --- a/doc/RUNNER-INTEGRATION-PLAYBOOK.md +++ b/doc/RUNNER-INTEGRATION-PLAYBOOK.md @@ -781,6 +781,13 @@ case/flow, matcher, result validator, and report test. Follow this sequence: 2. Select existing environments and relevant journeys. Add a case only when a promised behavior lacks an independent oracle; keep one durable purpose per suite. Record expected turns, deadline, spending bound, and cleanup obligations. + Native sandbox integrations must pass `rolling-web-deployment` and + `rolling-web-deployment-warm` before cloud qualification. + The shared suite includes registered native Daytona profiles automatically. + It replaces the web controller during real work, deletes its entire home, + checks continuous reads and writes, and requires a later turn to retain the + native and provider session identity after another empty-home replacement. + A profile's presence in the matrix is not a passing qualification receipt. 3. Make setup create an isolated company, workspace, agent, and secrets through supported APIs. Execute the graded action through ordinary browser/API surfaces. Avoid direct database writes and test-only provider shortcuts. diff --git a/packages/db/src/migrations/0429_high_freak.sql b/packages/db/src/migrations/0429_high_freak.sql new file mode 100644 index 00000000000..7571d5f5566 --- /dev/null +++ b/packages/db/src/migrations/0429_high_freak.sql @@ -0,0 +1,15 @@ +CREATE TABLE IF NOT EXISTS "native_harness_backups" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "company_id" uuid NOT NULL, + "issue_id" uuid NOT NULL, + "run_id" uuid NOT NULL, + "scope_digest" text NOT NULL, + "manifest_sha256" text NOT NULL, + "status" text DEFAULT 'pending' NOT NULL, + "reference" jsonb NOT NULL, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE UNIQUE INDEX IF NOT EXISTS "native_harness_backups_current_idx" ON "native_harness_backups" USING btree ("company_id","scope_digest") WHERE "native_harness_backups"."status" = 'current';--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "native_harness_backups_scope_idx" ON "native_harness_backups" USING btree ("company_id","scope_digest","manifest_sha256"); diff --git a/packages/db/src/migrations/meta/0424_snapshot.json b/packages/db/src/migrations/meta/0429_snapshot.json similarity index 99% rename from packages/db/src/migrations/meta/0424_snapshot.json rename to packages/db/src/migrations/meta/0429_snapshot.json index d36bfdfe18c..ce1d9aceaee 100644 --- a/packages/db/src/migrations/meta/0424_snapshot.json +++ b/packages/db/src/migrations/meta/0429_snapshot.json @@ -1,6 +1,6 @@ { - "id": "cb093c1d-3e54-4ab2-8009-9c243097224c", - "prevId": "fe0b877c-d63e-46d9-8798-c7729fd13eb7", + "id": "809726a1-762c-4c9d-a7fd-55670f1c856f", + "prevId": "fb4383f6-cbe6-4aff-9fd2-6eeb69a6927a", "version": "7", "dialect": "postgresql", "tables": { @@ -28825,6 +28825,337 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.execution_admission_project_charges": { + "name": "execution_admission_project_charges", + "schema": "", + "columns": { + "admission_id": { + "name": "admission_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "project_id": { + "name": "project_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "execution_admission_project_charges_project_idx": { + "name": "execution_admission_project_charges_project_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "project_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "execution_admission_project_charges_company_admission_fk": { + "name": "execution_admission_project_charges_company_admission_fk", + "tableFrom": "execution_admission_project_charges", + "tableTo": "execution_admissions", + "columnsFrom": [ + "company_id", + "admission_id" + ], + "columnsTo": [ + "company_id", + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "execution_admission_project_charges_company_project_fk": { + "name": "execution_admission_project_charges_company_project_fk", + "tableFrom": "execution_admission_project_charges", + "tableTo": "projects", + "columnsFrom": [ + "company_id", + "project_id" + ], + "columnsTo": [ + "company_id", + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "execution_admission_project_charges_admission_id_project_id_pk": { + "name": "execution_admission_project_charges_admission_id_project_id_pk", + "columns": [ + "admission_id", + "project_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.execution_admissions": { + "name": "execution_admissions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "attempt": { + "name": "attempt", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "primary_project_id": { + "name": "primary_project_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "context_revision": { + "name": "context_revision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "admitted_project_ids": { + "name": "admitted_project_ids", + "type": "uuid[]", + "primaryKey": false, + "notNull": false + }, + "settlement_id": { + "name": "settlement_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "settled_at": { + "name": "settled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "execution_admissions_company_id_uq": { + "name": "execution_admissions_company_id_uq", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "execution_admissions_run_attempt_uq": { + "name": "execution_admissions_run_attempt_uq", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "attempt", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "execution_admissions_live_run_uq": { + "name": "execution_admissions_live_run_uq", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"execution_admissions\".\"settled_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "execution_admissions_company_id_companies_id_fk": { + "name": "execution_admissions_company_id_companies_id_fk", + "tableFrom": "execution_admissions", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "execution_admissions_run_id_heartbeat_runs_id_fk": { + "name": "execution_admissions_run_id_heartbeat_runs_id_fk", + "tableFrom": "execution_admissions", + "tableTo": "heartbeat_runs", + "columnsFrom": [ + "run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "execution_admissions_attempt_check": { + "name": "execution_admissions_attempt_check", + "value": "\"execution_admissions\".\"attempt\" > 0" + }, + "execution_admissions_settlement_check": { + "name": "execution_admissions_settlement_check", + "value": "(\"execution_admissions\".\"settlement_id\" is null) = (\"execution_admissions\".\"settled_at\" is null)" + }, + "execution_admissions_project_set_check": { + "name": "execution_admissions_project_set_check", + "value": "case when \"execution_admissions\".\"admitted_project_ids\" is null\n then \"execution_admissions\".\"settled_at\" is not null else array_position(\"execution_admissions\".\"admitted_project_ids\", null) is null and cardinality(\"execution_admissions\".\"admitted_project_ids\") <= 100 end" + } + }, + "isRLSEnabled": false + }, + "public.project_task_execution_policies": { + "name": "project_task_execution_policies", + "schema": "", + "columns": { + "project_id": { + "name": "project_id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "max_concurrent_tasks": { + "name": "max_concurrent_tasks", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "project_task_execution_policies_company_project_fk": { + "name": "project_task_execution_policies_company_project_fk", + "tableFrom": "project_task_execution_policies", + "tableTo": "projects", + "columnsFrom": [ + "company_id", + "project_id" + ], + "columnsTo": [ + "company_id", + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "project_task_execution_policies_maximum_check": { + "name": "project_task_execution_policies_maximum_check", + "value": "\"project_task_execution_policies\".\"max_concurrent_tasks\" is null or \"project_task_execution_policies\".\"max_concurrent_tasks\" > 0" + } + }, + "isRLSEnabled": false + }, "public.execution_workspace_repositories": { "name": "execution_workspace_repositories", "schema": "", @@ -29999,6 +30330,22 @@ "concurrently": false, "method": "btree", "with": {} + }, + "execution_workspace_snapshots_imported_children_idx": { + "name": "execution_workspace_snapshots_imported_children_idx", + "columns": [ + { + "expression": "(\"metadata\"->'importedWorkspaceChildren') jsonb_path_ops", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"execution_workspace_snapshots\".\"metadata\" ? 'importedWorkspaceChildren'", + "concurrently": false, + "method": "gin", + "with": {} } }, "foreignKeys": {}, @@ -43679,6 +44026,133 @@ }, "isRLSEnabled": false }, + "public.native_harness_backups": { + "name": "native_harness_backups", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "issue_id": { + "name": "issue_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "scope_digest": { + "name": "scope_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "manifest_sha256": { + "name": "manifest_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "reference": { + "name": "reference", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "native_harness_backups_current_idx": { + "name": "native_harness_backups_current_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "scope_digest", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"native_harness_backups\".\"status\" = 'current'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "native_harness_backups_scope_idx": { + "name": "native_harness_backups_scope_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "scope_digest", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "manifest_sha256", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.native_run_finalizations": { "name": "native_run_finalizations", "schema": "", @@ -62448,6 +62922,12 @@ "primaryKey": false, "notNull": false }, + "cleanup_verified_at": { + "name": "cleanup_verified_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, "created_at": { "name": "created_at", "type": "timestamp with time zone", diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index a2f1da3ce13..1d66ff4fa0c 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -2990,6 +2990,13 @@ "when": 1793743828770, "tag": "0428_retained_admitted_project_sets", "breakpoints": true + }, + { + "idx": 429, + "version": "7", + "when": 1793760539457, + "tag": "0429_high_freak", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/db/src/schema/index.ts b/packages/db/src/schema/index.ts index f12096d60cb..91c9770c6b1 100644 --- a/packages/db/src/schema/index.ts +++ b/packages/db/src/schema/index.ts @@ -146,6 +146,7 @@ export { completionContracts } from "./completion_contracts.js"; export { nativeRunResults } from "./native_run_results.js"; export { nativeRunFinalizations } from "./native_run_finalizations.js"; export { nativeRunnerCheckpoints } from "./native_runner_checkpoints.js"; +export { nativeHarnessBackups } from "./native_harness_backups.js"; export { workAssessments } from "./work_assessments.js"; export { statusDecisions } from "./status_decisions.js"; export { statusDecisionEffects } from "./status_decision_effects.js"; diff --git a/packages/db/src/schema/native_harness_backups.ts b/packages/db/src/schema/native_harness_backups.ts new file mode 100644 index 00000000000..9f39797534b --- /dev/null +++ b/packages/db/src/schema/native_harness_backups.ts @@ -0,0 +1,19 @@ +import { sql } from "drizzle-orm"; +import { index, jsonb, pgTable, text, timestamp, uniqueIndex, uuid } from "drizzle-orm/pg-core"; + +export const nativeHarnessBackups = pgTable("native_harness_backups", { + id: uuid("id").primaryKey().defaultRandom(), + companyId: uuid("company_id").notNull(), + issueId: uuid("issue_id").notNull(), + runId: uuid("run_id").notNull(), + scopeDigest: text("scope_digest").notNull(), + manifestSha256: text("manifest_sha256").notNull(), + status: text("status").notNull().default("pending"), + reference: jsonb("reference").$type>().notNull(), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), +}, table => ({ + current: uniqueIndex("native_harness_backups_current_idx").on(table.companyId, table.scopeDigest) + .where(sql`${table.status} = 'current'`), + scope: index("native_harness_backups_scope_idx").on(table.companyId, table.scopeDigest, table.manifestSha256), +})); diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts index 5c4bb9d17ad..3f9736ba32d 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts @@ -31,6 +31,7 @@ import { digestPaperclipSemanticContent } from "../semantic-tools/receipts.js"; import { DurablePrpControlPlane, decodeDurablePrpCheckpoint, + captureLegacyDurablePrpCheckpoint, inspectDurablePrpCheckpointHeader, inspectStagedRunAttachment, inspectRetainedRunInstructions, @@ -220,6 +221,40 @@ it("restores authentication and commands from opaque checkpoint records", async } }); +it("imports the authenticated legacy authority without changing its source and restores it in an empty home", async () => { + const root = mkdtempSync(resolve(tmpdir(), "runner-legacy-import-")); + const options = { stateDirectory: resolve(root, "old/control-plane"), identity, expectedRunnerVersion, expectedRunnerDigest }; + const core = new DurablePrpControlPlane(options); + let restored: DurablePrpControlPlane | undefined; + try { + await core.start(); + const client = (await authenticate(core, core.issueBootstrapTicket()))!; + core.queueCommand("session.snapshot", {}, "retained-maintenance"); + client.socket.destroy(); + await core.stop(); + const file = resolve(options.stateDirectory, "control-plane-state.json"); + const source = readFileSync(file, "utf8"); + const checkpoint = captureLegacyDurablePrpCheckpoint(source, identity); + expect(readFileSync(file, "utf8")).toBe(source); + expect(checkpoint.runnerDigest).toBe(expectedRunnerDigest); + expect(checkpoint.recoveryEvidence.authenticatedConnectionCount).toBe(1); + rmSync(resolve(root, "old"), { recursive: true }); + checkpoint.restoreSessionDirectory(resolve(root, "new")); + restored = new DurablePrpControlPlane({ ...options, stateDirectory: resolve(root, "new/control-plane") }); + expect(restored.getCommand("retained-maintenance")).toMatchObject({ type: "session.snapshot" }); + expect(() => captureLegacyDurablePrpCheckpoint(source, { ...identity, runId: "another-run" })).toThrow("binding_mismatch"); + const state = JSON.parse(source); + expect(() => captureLegacyDurablePrpCheckpoint(JSON.stringify({ ...state, tickets: {} }), identity)).toThrow("artifact_unproven"); + expect(captureLegacyDurablePrpCheckpoint(JSON.stringify({ ...state, tickets: {} }), identity, + { version: expectedRunnerVersion, digest: expectedRunnerDigest }).runnerDigest).toBe(expectedRunnerDigest); + expect(() => captureLegacyDurablePrpCheckpoint(source, identity, + { version: expectedRunnerVersion, digest: `sha256:${"0".repeat(64)}` })).toThrow("artifact_unproven"); + expect(() => captureLegacyDurablePrpCheckpoint(JSON.stringify({ schema: "paperclip.runner.external-control-plane-state.v1", identity }), identity)).toThrow("binding_mismatch"); + } finally { + await restored?.stop(); await core.stop(); rmSync(root, { recursive: true, force: true }); + } +}); + it.each(["missing record", "invalid state", "wrong sequence", "invalid preparation identity", "duplicate preparation", "malformed header", "malformed cursor"])("rejects %s at checkpoint decoding", async fault => { const root = mkdtempSync(resolve(tmpdir(), "runner-checkpoint-invalid-")); const core = new DurablePrpControlPlane({ stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest }); diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts index f0c727bac5b..87e7eae5042 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts @@ -392,6 +392,33 @@ function captureCheckpoint(state: StoredCoreState, runnerVersion: string, runner }; } +export function captureLegacyDurablePrpCheckpoint(contents: string, expected: { + runId: string; runnerInstanceId: string; normalizedSessionId: string; +}, verifiedArtifact?: { version: string; digest: string }): DurablePrpCheckpoint { + if (Buffer.byteLength(contents) > maxStateBytes) throw new Error("native_runner_checkpoint_invalid"); + const value: unknown = JSON.parse(contents); + if (!isRecord(value) || !exactIdentity(value.identity) || !isStoredCoreState(value, value.identity) + || value.identity.runId !== expected.runId || value.identity.runnerInstanceId !== expected.runnerInstanceId + || value.identity.normalizedSessionId !== expected.normalizedSessionId + || checkpointRunId(value) !== expected.runId || value.warmTransition + || value.stagedRunAttachment && !stagedTargetActivated(value)) { + throw new Error("native_runner_checkpoint_binding_mismatch"); + } + const artifacts: Array<{ runnerVersion: string; runnerDigest: string }> = value.completedWarmTransition + ? [value.completedWarmTransition.receipt] + : Object.values(value.tickets).filter(ticket => ticket.usedAt !== null + && Number.isFinite(Date.parse(ticket.usedAt)) + && canonicalJson(ticket.identity) === canonicalJson(value.identity)); + if (verifiedArtifact) artifacts.push({ runnerVersion: verifiedArtifact.version, runnerDigest: verifiedArtifact.digest }); + const artifact = artifacts[0]; + if (!artifact || typeof artifact.runnerVersion !== "string" || !artifact.runnerVersion + || !runnerDigestPattern.test(artifact.runnerDigest) + || artifacts.some(candidate => candidate.runnerVersion !== artifact.runnerVersion || candidate.runnerDigest !== artifact.runnerDigest)) { + throw new Error("native_runner_checkpoint_artifact_unproven"); + } + return decodeDurablePrpCheckpoint(captureCheckpoint(value, artifact.runnerVersion, artifact.runnerDigest).records); +} + function parseCheckpointHeader(contents: string): Record & { runnerVersion: string; runnerDigest: string; state: Record & { identity: DurableRecoveryIdentity }; diff --git a/packages/paperclip-runner/src/index.ts b/packages/paperclip-runner/src/index.ts index 8a1b75988e5..30e064085a0 100644 --- a/packages/paperclip-runner/src/index.ts +++ b/packages/paperclip-runner/src/index.ts @@ -46,6 +46,7 @@ export { type DurablePrpControlPlaneOptions, type DurablePrpCheckpoint, decodeDurablePrpCheckpoint, + captureLegacyDurablePrpCheckpoint, inspectDurablePrpCheckpointHeader, type PrpWireConnection, type PrpWireAttachment, diff --git a/server/src/__tests__/ephemeral-server-contract.test.ts b/server/src/__tests__/ephemeral-server-contract.test.ts index bfa5e4acae8..e677036a7e2 100644 --- a/server/src/__tests__/ephemeral-server-contract.test.ts +++ b/server/src/__tests__/ephemeral-server-contract.test.ts @@ -1,6 +1,6 @@ import { afterEach, expect, it, vi } from "vitest"; import { assertEphemeralServerConfig } from "../services/ephemeral-server-contract.js"; -import { assertEphemeralWorkspaceExecution, assertEphemeralNativeLifecycle } from "../services/remote-workspaces.js"; +import { assertEphemeralWorkspaceExecution, assertEphemeralNativeRuntime } from "../services/remote-workspaces.js"; import { assertEphemeralAgentInstructions } from "../services/agent-instructions.js"; const config = { databaseUrl: "postgresql://runtime:fixture@ep-example-pooler.neon.test/tenant", storageProvider: "s3" as const }; @@ -24,20 +24,20 @@ it("never connects the listener with migration privileges, another tenant, or th expect(() => assertEphemeralServerConfig(config, { ...env, PAPERCLIP_DATABASE_LISTEN_URL: url })).toThrow("runtime role"); } }); -it("rejects disk-dependent execution and instructions while retaining remote warm execution", () => { +it("requires remote workspace authority and native execution independently of provider lifecycle", () => { vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "true"); const remote = { externalConversation: false, adapterType: "paperclip_runner", pluginTaskWorkspace: false, remoteAuthority: true }; expect(() => assertEphemeralWorkspaceExecution(remote)).not.toThrow(); expect(() => assertEphemeralWorkspaceExecution({ ...remote, remoteAuthority: false, pluginTaskWorkspace: true })).not.toThrow(); for (const patch of [{ remoteAuthority: false }, { adapterType: "process" }]) { - expect(() => assertEphemeralWorkspaceExecution({ ...remote, ...patch })).toThrow("warm Paperclip Runner"); + expect(() => assertEphemeralWorkspaceExecution({ ...remote, ...patch })).toThrow("native Paperclip Runner"); } - expect(() => assertEphemeralNativeLifecycle("warm")).not.toThrow(); - expect(() => assertEphemeralNativeLifecycle("per_turn")).toThrow("warm Paperclip Runner"); - expect(() => assertEphemeralNativeLifecycle(undefined)).toThrow("warm Paperclip Runner"); + expect(() => assertEphemeralNativeRuntime(true)).not.toThrow(); + expect(() => assertEphemeralNativeRuntime(false)).toThrow("native Paperclip Runner"); const agent = { id: "agent", companyId: "company", name: "Agent", adapterConfig: {} }; expect(() => assertEphemeralAgentInstructions(agent)).not.toThrow(); expect(() => assertEphemeralAgentInstructions({ ...agent, adapterConfig: { instructionsBundleMode: "external", instructionsRootPath: "/old-volume" } })).toThrow("managed agent instructions"); vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "false"); expect(() => assertEphemeralWorkspaceExecution({ ...remote, remoteAuthority: false })).not.toThrow(); + expect(() => assertEphemeralNativeRuntime(false)).not.toThrow(); }); diff --git a/server/src/maintenance/ephemeral-migration-data.ts b/server/src/maintenance/ephemeral-migration-data.ts index a815a51db65..b6e705016c4 100644 --- a/server/src/maintenance/ephemeral-migration-data.ts +++ b/server/src/maintenance/ephemeral-migration-data.ts @@ -9,6 +9,8 @@ import { skillPackageMaintenance } from "../services/skill-package-maintenance.j import { hasUnsettledLocalInstructionCopies } from "../services/agent-instruction-working-copies.js"; import { providerTraceStore } from "../services/provider-trace-store.js"; import { migrateWorkspaceTrees } from "../services/remote-workspace-migration-batch.js"; +import { migrateNativeHarnessBackups } from "../services/native-runtime/native-harness-backup-migration.js"; +import { migrateNativeControllers } from "../services/native-runtime/native-controller-migration.js"; /** Runs only in the maintenance process after Cloud has confirmed all legacy * deployments stopped. Each owner retains its own migration/publication rules; @@ -25,12 +27,21 @@ export async function migrateEphemeralServerData(db: Db, input: { const before = await readEphemeralServerDataEligibility(db); const unsupported = before.blockers.filter(blocker => ![ "agent_files_require_migration", "run_logs_require_migration", "provider_traces_require_migration", "skills_require_migration", "workspaces_require_migration", + "native_harness_backups_require_migration", + "native_controllers_require_migration", ].includes(blocker)); if (unsupported.length) throw new Error(unsupported[0]); - phase("workspaces"); const storage = getStorageService(); if (storage.provider !== "s3") throw new Error("shared_object_storage_required"); + phase("native_controllers"); + const controllers = await migrateNativeControllers(db, { writersStopped: true, signal }); + counts.nativeControllers = controllers.imported; + counts.nativeControllersUnchanged = controllers.unchanged; + counts.nativeControllerSourcesAlreadyAbsent = controllers.sourceAlreadyAbsent; + phase("native_harness_backups"); + counts.nativeHarnessBackups = await migrateNativeHarnessBackups(db, { storage, writersStopped: true, signal }); + phase("workspaces"); counts.workspaces = await migrateWorkspaceTrees(db, { storage, signal, phase }); phase("agent_files"); diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index bb3c56d268f..6f53cbe45e9 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -56,7 +56,7 @@ import { import { environmentService } from "./environments.js"; import { instanceSettingsService } from "./instance-settings.js"; import { hasSandboxSnapshotQualification, qualifiedSandboxSnapshotId, usesManagedCloudSandboxImage, type SandboxTemplateIdentity } from "./sandbox-snapshot-qualification.js"; -import { verifyNativeHarnessBackupStamp } from "./native-runtime/native-harness-backup-stamp.js"; +import { verifyRecoverableNativeHarnessBackup } from "./native-runtime/native-harness-backup-storage.js"; import { collectEnvironmentSecretRefs, parseEnvironmentDriverConfig, @@ -1847,10 +1847,8 @@ function createSandboxEnvironmentDriver( if (!providerLease) { if ( input.adapterType === "paperclip_runner" && - !verifyNativeHarnessBackupStamp( - reusableLease.metadata?.nativeHarnessBackup, - reusableLease.providerLeaseId, - ) + !await verifyRecoverableNativeHarnessBackup({ db, companyId: reusableLease.companyId, + stamp: reusableLease.metadata?.nativeHarnessBackup, providerLeaseId: reusableLease.providerLeaseId }) ) { throw new RunnerHarnessBackupUnavailableError( reusableLease.providerLeaseId, @@ -3986,10 +3984,8 @@ export function environmentRuntimeService( "paperclip_runner" && leaseSnapshot.metadata?.sandboxLeaseAcquisition && (!leaseSnapshot.providerLeaseId || - !verifyNativeHarnessBackupStamp( - leaseSnapshot.metadata.nativeHarnessBackup, - leaseSnapshot.providerLeaseId, - )) + !await verifyRecoverableNativeHarnessBackup({ db, companyId: leaseSnapshot.companyId, + stamp: leaseSnapshot.metadata.nativeHarnessBackup, providerLeaseId: leaseSnapshot.providerLeaseId })) ) { throw new RunnerHarnessBackupUnavailableError( leaseSnapshot.providerLeaseId ?? leaseSnapshot.id, diff --git a/server/src/services/ephemeral-server-contract.ts b/server/src/services/ephemeral-server-contract.ts index 396885b3f57..78cc9c54502 100644 --- a/server/src/services/ephemeral-server-contract.ts +++ b/server/src/services/ephemeral-server-contract.ts @@ -4,6 +4,8 @@ import type { Config } from "../config.js"; import { isValidLocalEncryptedMasterKey } from "../secrets/local-encrypted-provider.js"; import { agentInstructionsBundleMode } from "./agent-instructions.js"; import { skillInstallationMaintenance } from "./skill-installation-maintenance.js"; +import { hasLocalNativeHarnessObligations } from "./native-runtime/native-harness-backup-migration.js"; +import { hasLocalNativeControllerObligations } from "./native-runtime/native-controller-migration.js"; export const ephemeralServerEnabled = () => process.env.PAPERCLIP_EPHEMERAL_SERVER === "true"; @@ -64,6 +66,8 @@ export async function readEphemeralServerDataEligibility(db: Db, options: { purp isNull(providerTraceRecords.deletedAt), sql`${providerTraceRecords.traceRef} not like 's3-segments-v1:%'`, )).limit(1); if (localTrace) blockers.push("provider_traces_require_migration"); + if (await hasLocalNativeHarnessObligations(db)) blockers.push("native_harness_backups_require_migration"); + if (await hasLocalNativeControllerObligations(db)) blockers.push("native_controllers_require_migration"); const skills = skillInstallationMaintenance(db); let after: string | undefined; for (;;) { diff --git a/server/src/services/heartbeat/runtime-selection.test.ts b/server/src/services/heartbeat/runtime-selection.test.ts index 032a80e73c6..3c7c3bab0d5 100644 --- a/server/src/services/heartbeat/runtime-selection.test.ts +++ b/server/src/services/heartbeat/runtime-selection.test.ts @@ -432,14 +432,14 @@ describe.skipIf(!support.supported)("heartbeat runtime selection boundary", () = const { input, run } = await fixture("process"); vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "true"); expect(input.nativeRuntimeResolution.kind).toBe("legacy"); - await expect(selectHeartbeatRuntime(db, input)).rejects.toThrow("warm Paperclip Runner"); + await expect(selectHeartbeatRuntime(db, input)).rejects.toThrow("native Paperclip Runner"); expect((await read(run.id)).runtimeMode).toBe("legacy"); expect(await coordinators(run.id)).toHaveLength(0); expect(input.onNativeLifecycleSelected).not.toHaveBeenCalled(); expect(prepareNativeWorkspaceSync).not.toHaveBeenCalled(); }); - it.each(["warm", "per_turn"] as const)("checks resolved %s policy for ephemeral admission", async mode => { + it.each(["warm", "per_turn"] as const)("preserves resolved %s policy on an ephemeral server", async mode => { const { input, run } = await fixture(); vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "true"); // Deliberately disagree with the resolved provider policy in both cases. @@ -447,14 +447,41 @@ describe.skipIf(!support.supported)("heartbeat runtime selection boundary", () = input.workspace.executionTarget = { ...sandbox, reusableLeaseConfigured: true, effectiveCapabilities: { reusableLeases: true } as never, runnerLifecyclePolicy: mode === "warm" ? { mode, idleTimeoutMs: 300_000 } : { mode, idleTimeoutMs: null } }; - if (mode === "per_turn") { - await expect(selectHeartbeatRuntime(db, input)).rejects.toThrow("warm Paperclip Runner"); - expect(await coordinators(run.id)).toHaveLength(0); - } else { - const selected = await selectHeartbeatRuntime(db, input); - expect(selected.selected && selected.nativeExecution?.session.lifecyclePolicy.mode).toBe("warm"); - expect((await read(run.id)).runnerProfileJson).toMatchObject({ nativeExecutionInput: { session: { lifecyclePolicy: { mode: "warm" } } } }); - } + const selected = await selectHeartbeatRuntime(db, input); + expect(selected.selected && selected.nativeExecution?.session.lifecyclePolicy.mode).toBe(mode); + expect((await read(run.id)).runnerProfileJson).toMatchObject({ nativeExecutionInput: { session: { lifecyclePolicy: { mode } } } }); + expect(await coordinators(run.id)).toHaveLength(1); + }); + + it.each([ + { provider: "acpx", acpxAgent: "claude", model: "claude-opus-5-5", credentialProvider: "anthropic" as const, method: "subscription" as const }, + { provider: "acpx", acpxAgent: "grok", model: "grok-4.7", credentialProvider: "xai" as const, method: "subscription" as const }, + { provider: "opencode", model: "openrouter/deepseek/deepseek-v4-flash-0731", credentialProvider: "openrouter" as const, method: "api_key" as const }, + ])("admits managed $provider/$acpxAgent on an ephemeral server without changing credential lifecycle", async profile => { + vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "true"); + const { input, run, issue } = await fixture(); + const { credentialProvider, method, ...providerConfig } = profile; + const adapterConfig = { ...providerConfig, lifecycleMode: "warm" }; + input.agent = { ...input.agent, adapterConfig }; + await db.update(agents).set({ adapterConfig }).where(eq(agents.id, input.agent.id)); + input.workspace.executionTarget = { ...sandbox, reusableLeaseConfigured: true, + effectiveCapabilities: { reusableLeases: true } as never, runnerLifecyclePolicy: { mode: "warm", idleTimeoutMs: 300_000 } }; + input.nativeRuntimeResolution = resolveHeartbeatNativeRuntimeMode({ persisted: run, enabled: true, runtimeConfig: {}, adapterConfig, + agent: input.agent, issue, target: input.workspace.executionTarget, workspaceId: null }); + expect(input.nativeRuntimeResolution.kind).toBe("native"); + const attribution = { connectionId: randomUUID(), grantId: randomUUID(), provider: credentialProvider, method, + mode: "responsible_user" as const, responsibleUserId: "task-owner" }; + input.config.managedAiRuntime = { sessionIdentity: "account:epoch-1", identity: "account:generation-1", + config: { env: {}, managedAiConnection: { ...attribution, identity: "account:generation-1", sessionIdentity: "account:epoch-1" } }, + attribution, accountName: "Saved account", accountOwnerUserId: "task-owner", home: instructionRoot, cleanup: async () => {} }; + const selected = await selectHeartbeatRuntime(db, input); + if (!selected.selected || !selected.nativeExecution) throw new Error("Expected native selection"); + expect(selected.nativeExecution.provider.kind).toBe(profile.provider); + expect(selected.nativeExecution.session.lifecyclePolicy).toEqual({ mode: "per_turn", idleTimeoutMs: null }); + expect((await read(run.id)).runnerProfileJson).toMatchObject({ + nativeExecutionInput: { session: { lifecyclePolicy: { mode: "per_turn" } } }, + }); + expect(await coordinators(run.id)).toHaveLength(1); }); it.each([ diff --git a/server/src/services/heartbeat/runtime-selection.ts b/server/src/services/heartbeat/runtime-selection.ts index b73ca32b3aa..d9613c3dc9b 100644 --- a/server/src/services/heartbeat/runtime-selection.ts +++ b/server/src/services/heartbeat/runtime-selection.ts @@ -1,7 +1,7 @@ import { resolveProviderApiPlacement } from "../native-runtime/provider-api-placement.js"; import { readClaimedNativeStopRecovery } from "../native-runtime/native-cancellation-request.js"; import { museRunnerBroker } from "../muse-runner-broker.js"; -import { assertEphemeralNativeLifecycle, readRemoteWorkspaceAuthority } from "../remote-workspaces.js"; +import { assertEphemeralNativeRuntime, readRemoteWorkspaceAuthority } from "../remote-workspaces.js"; import { measureRunTransaction } from "../execution-transaction-timing.js"; import { measureRunPhase } from "../execution-preparation-timing.js"; import { @@ -269,7 +269,7 @@ export async function selectHeartbeatRuntime(db: Db, input: HeartbeatRuntimeSele const { runtimeConfig, runtimeSkillEntries, runScopedMentionedSkillKeys, instructionCopy, agentIdentity, issueAssigneeOverrides, resolvedConfig, managedAiRuntime } = input.config; const runOptions = input.recovery; const { providerTraceRequested, providerTraceCapture } = input.trace; - if (nativeRuntimeResolution.kind !== "native" && !isDotRun) assertEphemeralNativeLifecycle(undefined); + if (!isDotRun) assertEphemeralNativeRuntime(nativeRuntimeResolution.kind === "native"); let nativeExecution: NativeExecutionInput | null = null; let getNativeFreshSessionHandoff: (() => Promise) | undefined; let nativeRunnerInstanceId: string | null = null; @@ -522,9 +522,6 @@ export async function selectHeartbeatRuntime(db: Db, input: HeartbeatRuntimeSele (["openai_dot_mcp", "grok_bot_mcp", "muse_external", "cursor_cloud_api"].includes(nativeRuntimeResolution.profile.backend) || managedAiRuntime && !supportsManagedWarmSession ? { mode: "per_turn" as const, idleTimeoutMs: null } : environmentLifecyclePolicy ?? agentLifecyclePolicy); - if (!["openai_dot_mcp", "grok_bot_mcp", "muse_external", "cursor_cloud_api"].includes(nativeRuntimeResolution.profile.backend)) { - assertEphemeralNativeLifecycle(effectiveLifecyclePolicy.mode); - } if ( effectiveLifecyclePolicy.mode === "warm" && executionTarget?.kind === "remote" && diff --git a/server/src/services/heartbeat/upgrade-readiness.test.ts b/server/src/services/heartbeat/upgrade-readiness.test.ts index 0dfb106e3c7..fb605c53be1 100644 --- a/server/src/services/heartbeat/upgrade-readiness.test.ts +++ b/server/src/services/heartbeat/upgrade-readiness.test.ts @@ -88,13 +88,14 @@ postgresTests("upgrade readiness persisted execution evidence", () => { finally { await core.stop(); await rm(root, { recursive: true, force: true }); } expect(await readUpgradeReadiness(db, input)).toMatchObject({ ready: true, recoverableRuns: 1 }); const cipher = createLocalEncryptedCipher("write"); - // Real staging headers contain retained launch metadata around 113 KiB. - const representativeHeader = JSON.stringify({ ...JSON.parse(header), padding: "x".repeat(100_000) }); + const parsedHeader = JSON.parse(header); + const representativeHeader = JSON.stringify({ ...parsedHeader, state: { ...parsedHeader.state, + runAttachTemplate: { provider: { instructions: "x".repeat(300_000) } } } }); await db.update(nativeRunnerCheckpoints).set({ material: cipher.encrypt(JSON.stringify({ ...binding, part: "header", contents: representativeHeader })) }) .where(eq(nativeRunnerCheckpoints.runId, runId)); expect((await readUpgradeReadiness(db, input)).ready).toBe(true); for (const contents of ["{", "{}", JSON.stringify({ ...JSON.parse(header), schema: "paperclip.runner.controller-checkpoint.v1" }), - JSON.stringify({ ...JSON.parse(header), padding: "x".repeat(400_000) }), + JSON.stringify({ ...JSON.parse(header), padding: "x".repeat(1_100_000) }), JSON.stringify({ ...JSON.parse(header), state: { ...JSON.parse(header).state, identity: { ...JSON.parse(header).state.identity, runId: randomUUID() } } })]) { await db.update(nativeRunnerCheckpoints).set({ material: cipher.encrypt(JSON.stringify({ ...binding, part: "header", contents })) }) .where(eq(nativeRunnerCheckpoints.runId, runId)); diff --git a/server/src/services/heartbeat/upgrade-readiness.ts b/server/src/services/heartbeat/upgrade-readiness.ts index 1ef14aed3b9..99da469b2ec 100644 --- a/server/src/services/heartbeat/upgrade-readiness.ts +++ b/server/src/services/heartbeat/upgrade-readiness.ts @@ -1,7 +1,8 @@ import type { Db } from "@paperclipai/db"; import { sql } from "drizzle-orm"; import { remoteWorkspaceRunReferenceSchema } from "@paperclipai/shared"; -import { inspectNativeRunnerUpgradeHeader } from "../native-runtime/native-runner-checkpoint.js"; +import { inspectNativeRunnerUpgradeHeader, NATIVE_CHECKPOINT_HEADER_INSPECTION_MAX_BYTES, + NATIVE_CHECKPOINT_INSPECTION_TOTAL_MAX_BYTES } from "../native-runtime/native-runner-checkpoint.js"; import { readNativeWorkspaceSyncReference } from "../native-runtime/native-workspace-sync.js"; import { REMOTE_RUNNER_RECONNECT_GRACE_MS } from "../native-runtime/native-runner-recovery-contract.js"; import { currentNativeControllerIdentity } from "../native-runtime/native-restart-recovery.js"; @@ -30,7 +31,8 @@ export async function readUpgradeReadiness(db: Db, input: { r.runner_profile_json->'nativeWorkspaceSync' as workspace, r.runner_profile_json->'remoteWorkspace' as "remoteWorkspace", r.runner_profile_json ? 'remoteWorkspace' as "hasRemoteWorkspace", p.id as "pluginLeaseId", - case when pg_column_size(c.material) <= 262144 and sum(pg_column_size(c.material)) over () <= 8388608 then c.material else null end as material, ( + case when pg_column_size(c.material) <= ${NATIVE_CHECKPOINT_HEADER_INSPECTION_MAX_BYTES} + and sum(pg_column_size(c.material)) over () <= ${NATIVE_CHECKPOINT_INSPECTION_TOTAL_MAX_BYTES} then c.material else null end as material, ( r.runtime_mode = 'native' and a.adapter_type = 'paperclip_runner' and r.runner_instance_id is not null and r.native_session_id is not null and r.process_pid is not null and r.process_started_at is not null diff --git a/server/src/services/heartbeat/workspace-preparation.test.ts b/server/src/services/heartbeat/workspace-preparation.test.ts index 3a218e50b8e..3134fb688d1 100644 --- a/server/src/services/heartbeat/workspace-preparation.test.ts +++ b/server/src/services/heartbeat/workspace-preparation.test.ts @@ -444,7 +444,7 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace const resolve = vi.spyOn(input.services, "resolveWorkspaceForRun").mockRejectedValue(new Error("local folder retained")); process.env.PAPERCLIP_EPHEMERAL_SERVER = String(ephemeral); await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow(ephemeral - ? "Volume-free servers require warm Paperclip Runner execution with remote workspace storage." : "local folder retained"); + ? "Volume-free servers require native Paperclip Runner execution with remote workspace storage." : "local folder retained"); if (ephemeral) expect(resolve).not.toHaveBeenCalled(); else expect(resolve).toHaveBeenCalledOnce(); expect(await db.select().from(executionWorkspaces)).toHaveLength(0); diff --git a/server/src/services/native-runtime/native-controller-migration.integration.test.ts b/server/src/services/native-runtime/native-controller-migration.integration.test.ts new file mode 100644 index 00000000000..c0b7829072b --- /dev/null +++ b/server/src/services/native-runtime/native-controller-migration.integration.test.ts @@ -0,0 +1,106 @@ +import { createHash, randomUUID } from "node:crypto"; +import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { eq } from "drizzle-orm"; +import { afterEach, expect, it, vi } from "vitest"; +import { agentTaskSessions, heartbeatRuns, issues, nativeRunFinalizations, nativeRunnerCheckpoints } from "@paperclipai/db"; +import { DurablePrpControlPlane, parseNativeExecutionInput } from "../../vendor/paperclip-runner/index.js"; +import { describeEmbeddedPostgres, nativeRestartFixture } from "../../__tests__/helpers/native-restart.js"; +import { nativeToolContractFingerprintForTarget } from "./native-session-resume.js"; +import { nativeControllerMigrationDescriptor } from "./native-session-executor.js"; +import { hasLocalNativeControllerObligations, migrateNativeControllers } from "./native-controller-migration.js"; +import { resolveNativeControllerRecovery } from "./native-controller-recovery.js"; +import { resolveRemoteProviderPackAuthority } from "./remote-provider-pack-authority.js"; +import { canonicalProviderPackJson } from "./remote-provider-pack-verification.js"; +import { nativeSha256 } from "./canonical.js"; + +describeEmbeddedPostgres("offline native controller migration", () => { + const environment = nativeRestartFixture(); + afterEach(() => vi.unstubAllEnvs()); + it("imports an exact settled authority once, preserves queued work, and resumes from an empty home", async () => { + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "73".repeat(32)); + const f = await environment.seedRun(randomUUID()); + const { companyId, agentId, paperclipHome, runtimeRoot } = environment.current; + const binding = { companyId, agentId, issueId: f.issueId, runId: f.runId, executionWorkspaceId: f.native.environmentLeaseId }; + const execution = parseNativeExecutionInput({ + schema: "paperclip.native-execution-input.v1", provider: { kind: "codex", model: null }, binding, + task: { identifier: "MIG-1", title: "Retained work", description: null, prompt: "Continue the work.", workMode: "standard" }, + workspace: { cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null }, + session: { normalizedSessionId: f.native.normalizedSessionId, driverKind: "codex_app_server", protocolVersion: 1, + lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } }, + completionContract: { id: randomUUID(), sha256: "fixture", schemaVersion: "paperclip.completion-contract.v1", + contract: { revision: "1", objective: "Continue", criteria: [{ id: "objective", requirement: "Save work." }] } }, + credentialBindings: [], interactionResponses: [], + }); + const fingerprint = nativeToolContractFingerprintForTarget("remote", "codex"); + const paths = nativeControllerMigrationDescriptor(execution, fingerprint); + const identity = { runId: f.runId, runnerInstanceId: f.native.runnerInstanceId, normalizedSessionId: f.native.normalizedSessionId, + environmentLeaseId: f.native.environmentLeaseId, turnId: f.native.turnId, itemId: f.native.itemId }; + const core = new DurablePrpControlPlane({ stateDirectory: join(paths.root, "control-plane"), identity, + expectedRunnerVersion: "migration-fixture", expectedRunnerDigest: `sha256:${"a".repeat(64)}` }); + core.issueBootstrapTicket(); core.queueCommand("session.snapshot", {}, "retained-command"); await core.stop(); + const file = join(paths.root, "control-plane/control-plane-state.json"); + const state = JSON.parse(await readFile(file, "utf8")); + state.tickets = {}; + const source = JSON.stringify(state); await writeFile(file, source, { mode: 0o600 }); + const packBinding = { companyId, agentId, workspaceId: binding.executionWorkspaceId, + sessionId: f.native.normalizedSessionId, runnerInstanceId: f.native.runnerInstanceId, + environmentLeaseId: f.native.environmentLeaseId, remoteCwd: "/remote/workspace", providerDigest: nativeSha256(execution.provider) }; + const payload = { + runnerSourceRevision: "a".repeat(40), target: { platform: "linux", architecture: "x64" }, + pins: { opencode: "1.18.34" }, artifacts: { + nodeCommand: { path: "node", sha256: `sha256:${"1".repeat(64)}` }, + opencodeCommand: { path: "opencode", sha256: `sha256:${"2".repeat(64)}` }, + }, + }; + const pack = resolveRemoteProviderPackAuthority({ stateDirectory: paths.root, reattach: false, binding: packBinding, + current: () => ({ schema: "paperclip-runner/remote-provider-pack/v1", payload, + digest: `sha256:${createHash("sha256").update(canonicalProviderPackJson(payload)).digest("hex")}` }), + runnerArtifact: () => ({ version: "migration-fixture", digest: `sha256:${"a".repeat(64)}` }) }); + pack.verified(); + const snapshot = { backendKind: "mock", sessionId: f.native.normalizedSessionId, + identity: { companyId, agentId, issueId: f.issueId, runId: f.runId, sessionId: f.native.normalizedSessionId } }; + const profile = { ...f.preparedProfile, nativeExecutionInput: execution, nativeToolContractFingerprint: fingerprint, sessionCheckpoint: snapshot, + nativeWorkspaceSync: { schema: "paperclip.native-workspace-sync/v1", state: "prepared", descriptorSha256: "a".repeat(64), + baselineSha256: "b".repeat(64), finalHostSha256: null, workspaceId: randomUUID(), leaseId: f.native.environmentLeaseId, + providerLeaseId: randomUUID(), remoteCwd: "/remote/workspace", resourceDisposition: "destroy" } }; + await f.db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date(), runnerProfileJson: profile }).where(eq(heartbeatRuns.id, f.runId)); + await f.db.update(nativeRunFinalizations).set({ phase: "terminal_failure", leaseOwner: null }).where(eq(nativeRunFinalizations.runId, f.runId)); + const queuedId = randomUUID(); + await f.db.insert(heartbeatRuns).values({ id: queuedId, companyId, agentId, issueId: f.issueId, status: "queued" }); + await f.db.update(issues).set({ executionRunId: queuedId }).where(eq(issues.id, f.issueId)); + const [session] = await f.db.insert(agentTaskSessions).values({ companyId, agentId, adapterType: "paperclip_runner", + taskKey: f.issueId, lastRunId: f.runId }).returning(); + expect(await hasLocalNativeControllerObligations(f.db)).toBe(true); + await f.db.update(heartbeatRuns).set({ runnerProfileJson: { ...profile, + nativeWorkspaceSync: { ...profile.nativeWorkspaceSync, leaseId: randomUUID() } } }).where(eq(heartbeatRuns.id, f.runId)); + await expect(migrateNativeControllers(f.db, { writersStopped: true, signal: AbortSignal.timeout(30_000) })) + .rejects.toThrow("native_controller_migration_artifact_binding"); + await f.db.update(heartbeatRuns).set({ runnerProfileJson: profile }).where(eq(heartbeatRuns.id, f.runId)); + await f.db.update(nativeRunFinalizations).set({ leaseOwner: "unsettled-owner" }).where(eq(nativeRunFinalizations.runId, f.runId)); + await expect(migrateNativeControllers(f.db, { writersStopped: true, signal: AbortSignal.timeout(30_000) })).rejects.toThrow("normal_ownership_recovery_required"); + expect(await f.db.select().from(nativeRunnerCheckpoints).where(eq(nativeRunnerCheckpoints.runId, f.runId))).toHaveLength(0); + await f.db.update(nativeRunFinalizations).set({ leaseOwner: "expired-owner", leaseExpiresAt: new Date(0), + phase: "retryable_failure", recoveryState: "resuming_session" }).where(eq(nativeRunFinalizations.runId, f.runId)); + const [preservedOwner] = await f.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)); + expect(await migrateNativeControllers(f.db, { writersStopped: true, signal: AbortSignal.timeout(30_000) })) + .toMatchObject({ imported: 1, sourceAlreadyAbsent: 0 }); + expect(await migrateNativeControllers(f.db, { writersStopped: true, signal: AbortSignal.timeout(30_000) })).toMatchObject({ unchanged: 1, imported: 0 }); + expect(await hasLocalNativeControllerObligations(f.db)).toBe(false); + expect((await f.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)))[0]).toEqual(preservedOwner); + expect(await readFile(file, "utf8")).toBe(source); + expect((await f.db.select().from(agentTaskSessions).where(eq(agentTaskSessions.id, session.id)))[0].lastRunId).toBe(f.runId); + expect((await f.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, queuedId)))[0].status).toBe("queued"); + await rm(paperclipHome, { recursive: true, force: true }); + await f.db.update(heartbeatRuns).set({ nativeIssueId: f.issueId, nativeSessionId: f.native.normalizedSessionId, + status: "running", runtimeMode: "native" }).where(eq(heartbeatRuns.id, queuedId)); + const recovered = await resolveNativeControllerRecovery(f.db, { companyId, agentId, issueId: f.issueId, + normalizedSessionId: f.native.normalizedSessionId, currentRunId: queuedId }); + expect(recovered.kind).toBe("recoverable"); + if (recovered.kind !== "recoverable") throw new Error(JSON.stringify(recovered)); + const replacement = join(runtimeRoot, "replacement"); await mkdir(replacement, { mode: 0o700 }); + recovered.checkpoint.restoreSessionDirectory(replacement); + expect(JSON.parse(await readFile(join(replacement, "control-plane/control-plane-state.json"), "utf8")).identity).toEqual(identity); + expect(recovered.snapshot).toEqual(snapshot); + }); +}); diff --git a/server/src/services/native-runtime/native-controller-migration.ts b/server/src/services/native-runtime/native-controller-migration.ts new file mode 100644 index 00000000000..830e837b855 --- /dev/null +++ b/server/src/services/native-runtime/native-controller-migration.ts @@ -0,0 +1,142 @@ +import { constants } from "node:fs"; +import { lstat, open, readdir } from "node:fs/promises"; +import { join } from "node:path"; +import { and, eq, sql } from "drizzle-orm"; +import { heartbeatRuns, type Db } from "@paperclipai/db"; +import { parseNativeExecutionInput } from "../../vendor/paperclip-runner/index.js"; +import { offlineMigrationExecutionWhere } from "../offline-migration-execution.js"; +import { findNativeSessionResumeRun } from "./native-session-resume.js"; +import { nativeControllerMigrationDescriptor } from "./native-session-executor.js"; +import { hasNativeRunnerCheckpoint, loadNativeRunnerCheckpoint } from "./native-runner-checkpoint.js"; +import { importNativeControllerRecovery, readNativeControllerRecovery } from "./native-controller-recovery.js"; +import { nativeHarnessStateBase } from "./native-harness-backup-stamp.js"; +import { providerPackWorkspaceId, resolveRemoteProviderPackAuthority } from "./remote-provider-pack-authority.js"; +import { nativeSha256 } from "./canonical.js"; +import { readNativeWorkspaceSyncReference } from "./native-workspace-sync.js"; + +type Selected = { id: string; companyId: string; agentId: string; issueId: string; sessionId: string; createdAt: Date }; + +async function* retainedSources(db: Db, signal?: AbortSignal) { + let after: string | undefined; + for (;;) { + signal?.throwIfAborted(); + const rows = await db.execute(sql`select s.id, s.company_id as "companyId", s.agent_id as "agentId", + r.native_issue_id as "issueId", r.native_session_id as "sessionId", r.created_at as "createdAt" + from agent_task_sessions s join heartbeat_runs r on r.id = s.last_run_id + and r.company_id = s.company_id and r.agent_id = s.agent_id + where s.adapter_type = 'paperclip_runner' and r.native_issue_id is not null and r.native_session_id is not null + ${after ? sql`and s.id > ${after}::uuid` : sql``} order by s.id limit 100`); + for (const selected of rows) { + signal?.throwIfAborted(); + const source = await findNativeSessionResumeRun(db, { companyId: selected.companyId, agentId: selected.agentId, + issueId: selected.issueId, normalizedSessionId: selected.sessionId, + beforeCreatedAt: new Date(selected.createdAt), includeProgressBarrier: true }); + if (source) yield { selected, source }; + } + if (rows.length < 100) return; + after = rows.at(-1)!.id; + } +} + +async function readPrivateSource(file: string, maxBytes: number): Promise { + const handle = await open(file, constants.O_RDONLY | constants.O_NOFOLLOW).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return undefined; throw error; + }); + if (!handle) return undefined; + try { + const stat = await handle.stat(); + if (!stat.isFile() || stat.nlink !== 1 || stat.size > maxBytes) throw new Error("native_controller_migration_unsafe_source"); + return await handle.readFile("utf8"); + } finally { await handle.close(); } +} + +async function localPaths(source: NonNullable>>) { + const execution = parseNativeExecutionInput(source.runnerProfileJson.nativeExecutionInput); + const descriptor = nativeControllerMigrationDescriptor(execution, source.runnerProfileJson.nativeToolContractFingerprint); + const root = await lstat(descriptor.root).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return undefined; throw error; + }); + if (root && (!root.isDirectory() || root.isSymbolicLink())) throw new Error("native_controller_migration_unsafe_source"); + const control = join(descriptor.root, "control-plane"); + const directory = await lstat(control).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return undefined; throw error; + }); + if (directory && (!directory.isDirectory() || directory.isSymbolicLink())) throw new Error("native_controller_migration_unsafe_source"); + return { ...descriptor, execution, controlFile: join(control, "control-plane-state.json") }; +} + +export async function hasLocalNativeControllerObligations(db: Db): Promise { + const roots = await readdir(nativeHarnessStateBase()).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return [] as string[]; throw error; + }); + if (!roots.length) return false; + for await (const { selected, source } of retainedSources(db)) { + const binding = { companyId: selected.companyId, issueId: selected.issueId, runId: source.id }; + if (readNativeControllerRecovery(source.runnerProfileJson.nativeControllerRecovery, source.runnerProfileJson.sessionCheckpoint) + && await hasNativeRunnerCheckpoint(db, binding)) continue; + const paths = await localPaths(source); + const present = await lstat(paths.controlFile).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return undefined; throw error; + }); + if (present || await hasNativeRunnerCheckpoint(db, binding)) return true; + } + return false; +} + +export async function migrateNativeControllers(db: Db, input: { writersStopped: true; signal: AbortSignal }) { + const [active] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(offlineMigrationExecutionWhere()).limit(1); + if (!input.writersStopped || active) throw new Error("native_controller_migration_requires_offline_writers"); + const counts = { inspected: 0, imported: 0, unchanged: 0, sourceAlreadyAbsent: 0 }; + for await (const { selected, source } of retainedSources(db, input.signal)) { + counts.inspected++; + const binding = { companyId: selected.companyId, issueId: selected.issueId, runId: source.id }; + const checkpoint = await loadNativeRunnerCheckpoint(db, binding); + const metadata = readNativeControllerRecovery(source.runnerProfileJson.nativeControllerRecovery, source.runnerProfileJson.sessionCheckpoint); + const snapshot = source.runnerProfileJson.sessionCheckpoint as { identity?: Record } | null; + if (checkpoint && metadata?.checkpoint?.runId === source.id + && metadata.checkpoint.runnerInstanceId === source.runnerInstanceId + && metadata.checkpoint.normalizedSessionId === selected.sessionId + && snapshot?.identity?.runId === source.id && snapshot.identity.companyId === selected.companyId + && snapshot.identity.agentId === selected.agentId && snapshot.identity.issueId === selected.issueId + && snapshot.identity.sessionId === selected.sessionId + && checkpoint.recoveryEvidence.runnerInstanceId === source.runnerInstanceId + && checkpoint.recoveryEvidence.normalizedSessionId === selected.sessionId) { counts.unchanged++; continue; } + const paths = await localPaths(source); + const legacyControllerState = checkpoint ? undefined : await readPrivateSource(paths.controlFile, 256 * 1024 * 1024); + if (!checkpoint && legacyControllerState === undefined) { counts.sourceAlreadyAbsent++; continue; } + let legacyRunnerArtifact: { version: string; digest: string } | undefined; + if (legacyControllerState !== undefined && await readPrivateSource(join(paths.root, "provider-pack-authority.json"), 512 * 1024)) { + const identity = JSON.parse(legacyControllerState).identity; + const execution = paths.execution; + const [admission] = await db.select({ workspace: sql`${heartbeatRuns.runnerProfileJson}->'nativeWorkspaceSync'` }) + .from(heartbeatRuns).where(and(eq(heartbeatRuns.id, source.id), eq(heartbeatRuns.companyId, selected.companyId), + eq(heartbeatRuns.agentId, selected.agentId))); + const workspace = readNativeWorkspaceSyncReference(admission?.workspace); + const remoteCwd = workspace?.remoteCwd ?? execution.workspace.cwd; + if (typeof identity?.environmentLeaseId !== "string" || !remoteCwd + || (workspace && workspace.leaseId !== identity.environmentLeaseId)) { + throw new Error("native_controller_migration_artifact_binding"); + } + const unavailable = (): never => { throw new Error("native_controller_migration_artifact_unavailable"); }; + legacyRunnerArtifact = resolveRemoteProviderPackAuthority({ stateDirectory: paths.root, reattach: true, + binding: { companyId: selected.companyId, agentId: selected.agentId, + workspaceId: providerPackWorkspaceId(execution.binding.executionWorkspaceId, source.id, identity.environmentLeaseId), + sessionId: selected.sessionId, runnerInstanceId: source.runnerInstanceId!, + environmentLeaseId: identity.environmentLeaseId, remoteCwd, + providerDigest: nativeSha256(execution.provider) }, + current: unavailable, runnerArtifact: unavailable }).runnerArtifact; + } + const supervisor = paths.execution.session.lifecyclePolicy.mode === "warm" + ? await readPrivateSource(paths.supervisorPath, 4 * 1024 * 1024) : undefined; + input.signal.throwIfAborted(); + const result = await importNativeControllerRecovery(db, { ...binding, agentId: selected.agentId, + normalizedSessionId: selected.sessionId, expectedConfigDigest: paths.configDigest, + supervisor: supervisor === undefined ? undefined : JSON.parse(supervisor), + offline: { writersStopped: true, legacyControllerState, legacyRunnerArtifact } }); + if (result.kind !== "imported" && result.kind !== "unchanged") { + throw new Error(`native_controller_migration_blocked:${source.id}:${"reason" in result ? result.reason : result.kind}`); + } + counts[result.kind]++; + } + return counts; +} diff --git a/server/src/services/native-runtime/native-controller-recovery-inventory.ts b/server/src/services/native-runtime/native-controller-recovery-inventory.ts index 88f0077f9bd..f965c569de8 100644 --- a/server/src/services/native-runtime/native-controller-recovery-inventory.ts +++ b/server/src/services/native-runtime/native-controller-recovery-inventory.ts @@ -1,6 +1,7 @@ import type { Db } from "@paperclipai/db"; import { sql } from "drizzle-orm"; -import { inspectNativeRunnerUpgradeHeader } from "./native-runner-checkpoint.js"; +import { inspectNativeRunnerUpgradeHeader, NATIVE_CHECKPOINT_HEADER_INSPECTION_MAX_BYTES, + NATIVE_CHECKPOINT_INSPECTION_TOTAL_MAX_BYTES } from "./native-runner-checkpoint.js"; /** An explicit, paginated diagnostic. Never called by an idle/background scan. * This inventories selected retained task-session pointers, not all history. @@ -23,7 +24,8 @@ export async function readNativeControllerRecoveryInventory(db: Db, input: { aft r.native_session_id as "sessionId", r.runner_instance_id as "runnerId", case when pg_column_size(r.runner_profile_json->'nativeControllerRecovery') <= 4096 then r.runner_profile_json->'nativeControllerRecovery' else null end as metadata, - case when pg_column_size(c.material) <= 262144 and sum(pg_column_size(c.material)) over () <= 8388608 + case when pg_column_size(c.material) <= ${NATIVE_CHECKPOINT_HEADER_INSPECTION_MAX_BYTES} + and sum(pg_column_size(c.material)) over () <= ${NATIVE_CHECKPOINT_INSPECTION_TOTAL_MAX_BYTES} then c.material else null end as material, (r.finished_at is null or r.status not in ('succeeded', 'failed', 'cancelled', 'interrupted', 'timed_out') or f.run_id is null or f.lease_owner is not null or f.recovery_state is not null diff --git a/server/src/services/native-runtime/native-controller-recovery.ts b/server/src/services/native-runtime/native-controller-recovery.ts index 223e5f69bdc..ffb11b9f1d7 100644 --- a/server/src/services/native-runtime/native-controller-recovery.ts +++ b/server/src/services/native-runtime/native-controller-recovery.ts @@ -1,12 +1,13 @@ import { and, eq, ne, sql } from "drizzle-orm"; import { environmentLeases, heartbeatRuns, issues, nativeRunFinalizations, nativeRunnerCheckpoints, type Db } from "@paperclipai/db"; import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; -import type { DurablePrpCheckpoint, PersistedNativeSession } from "../../vendor/paperclip-runner/index.js"; +import { captureLegacyDurablePrpCheckpoint, parseNativeExecutionInput, type DurablePrpCheckpoint, type PersistedNativeSession } from "../../vendor/paperclip-runner/index.js"; import { notifyNativeFinalizationWork } from "../native-finalization-notifications.js"; import { readRunCancellation } from "../run-cancellation.js"; import { nativeSha256 } from "./canonical.js"; import { findNativeSessionResumeRun, isNativeSessionId, isUnusedNativeSessionBootstrap, nativeSessionProviderEvidence } from "./native-session-resume.js"; -import { loadNativeRunnerCheckpoint } from "./native-runner-checkpoint.js"; +import { insertMigratedNativeRunnerCheckpoint, loadNativeRunnerCheckpoint } from "./native-runner-checkpoint.js"; +import { offlineMigrationExecutionWhere } from "../offline-migration-execution.js"; import { redactRunnerApiPayloads } from "./runner-api-log-redaction.js"; export const NATIVE_CONTROLLER_RECOVERY_MAX_BYTES = 4 * 1024; @@ -223,10 +224,11 @@ export async function resolveNativeControllerRecovery(db: Db, input: { export async function importNativeControllerRecovery(db: Db, input: { companyId: string; agentId: string; issueId: string; runId: string; normalizedSessionId: string; expectedConfigDigest: string; - supervisor: unknown; + supervisor?: unknown; + offline?: { writersStopped: true; legacyControllerState?: string; legacyRunnerArtifact?: { version: string; digest: string } }; }): Promise<{ kind: "imported" | "unchanged" } | Exclude> { const supervisor = record(input.supervisor); - if (supervisor.schema !== "paperclip.native-session-supervisor.v1" || supervisor.configDigest !== input.expectedConfigDigest) { + if (input.supervisor !== undefined && (supervisor.schema !== "paperclip.native-session-supervisor.v1" || supervisor.configDigest !== input.expectedConfigDigest)) { return { kind: "unsupported", reason: "supervisor_compatibility" }; } return db.transaction(async tx => { @@ -236,25 +238,48 @@ export async function importNativeControllerRecovery(db: Db, input: { eq(nativeRunFinalizations.companyId, input.companyId), eq(nativeRunFinalizations.issueId, input.issueId), eq(nativeRunFinalizations.runId, input.runId))).for("update"); if (!task || !owner) return { kind: "missing_evidence" as const, reason: "owner_binding" }; - if (owner.leaseOwner || task.executionRunId) { + if (input.offline) { + const [active] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(offlineMigrationExecutionWhere()).limit(1); + if (!input.offline.writersStopped || active) return { kind: "missing_evidence" as const, reason: "offline_writers_required" }; + } + const [clock] = await tx.execute<{ now: Date }>(sql`select clock_timestamp() as now`); + const liveOwner = owner.leaseOwner && (!owner.leaseExpiresAt || owner.leaseExpiresAt > new Date(clock.now)); + if (input.offline ? liveOwner : owner.leaseOwner || task.executionRunId) { return owner.leaseExpiresAt && owner.leaseExpiresAt > new Date() ? { kind: "waiting_for_owner" as const, deadline: owner.leaseExpiresAt } : { kind: "missing_evidence" as const, reason: "normal_ownership_recovery_required" }; } - if (owner.phase !== "committed" || !owner.resultId || owner.recoveryState) { + if (!input.offline && (!(owner.phase === "committed" && owner.resultId) || owner.recoveryState)) { return { kind: "missing_evidence" as const, reason: "pending_finalization" }; } const [run] = await tx.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, input.runId), eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.agentId, input.agentId), eq(heartbeatRuns.nativeIssueId, input.issueId), eq(heartbeatRuns.nativeSessionId, input.normalizedSessionId))).for("update"); const profile = record(run?.runnerProfileJson), snapshot = record(profile.sessionCheckpoint), identity = record(snapshot.identity); - if (run?.runtimeMode !== "native" || run.status !== "succeeded" || !run.finishedAt + if (run?.runtimeMode !== "native" || !(run.status === "succeeded" || input.offline && ["failed", "cancelled", "interrupted", "timed_out"].includes(run.status)) || !run.finishedAt || identity.runId !== input.runId || identity.companyId !== input.companyId || identity.issueId !== input.issueId || identity.agentId !== input.agentId || identity.sessionId !== input.normalizedSessionId - || nativeSha256(snapshot) !== nativeSha256(redactRunnerApiPayloads(supervisor.snapshot))) { + || input.supervisor !== undefined && nativeSha256(snapshot) !== nativeSha256(redactRunnerApiPayloads(supervisor.snapshot))) { return { kind: "missing_evidence" as const, reason: "stale_supervisor_snapshot" }; } - const checkpoint = await loadNativeRunnerCheckpoint(tx, input); + if (input.supervisor === undefined) { + try { + const admitted = parseNativeExecutionInput(profile.nativeExecutionInput); + if (!input.offline || admitted.session.lifecyclePolicy.mode !== "per_turn" + || admitted.binding.runId !== input.runId || admitted.binding.companyId !== input.companyId + || admitted.binding.agentId !== input.agentId || admitted.binding.issueId !== input.issueId) { + return { kind: "missing_evidence" as const, reason: "admitted_per_turn_binding" }; + } + } catch { return { kind: "unsupported" as const, reason: "admitted_input_format" }; } + } + let checkpoint = await loadNativeRunnerCheckpoint(tx, input); + const importing = !checkpoint && input.offline?.legacyControllerState !== undefined; + if (importing) { + if (!run.runnerInstanceId) return { kind: "missing_evidence" as const, reason: "checkpoint_binding" }; + checkpoint = captureLegacyDurablePrpCheckpoint(input.offline!.legacyControllerState!, { + runId: input.runId, runnerInstanceId: run.runnerInstanceId, normalizedSessionId: input.normalizedSessionId, + }, input.offline!.legacyRunnerArtifact); + } if (!checkpoint || checkpoint.recoveryEvidence.normalizedSessionId !== input.normalizedSessionId || checkpoint.recoveryEvidence.runnerInstanceId !== run.runnerInstanceId) { return { kind: "missing_evidence" as const, reason: "checkpoint_binding" }; @@ -265,10 +290,13 @@ export async function importNativeControllerRecovery(db: Db, input: { }, { runId: input.runId, runnerInstanceId: checkpoint.recoveryEvidence.runnerInstanceId, normalizedSessionId: input.normalizedSessionId }); if (!readNativeControllerRecovery(metadata, snapshot)) return { kind: "unsupported" as const, reason: "metadata_format" }; if (profile.nativeControllerRecovery) { - return nativeSha256(profile.nativeControllerRecovery) === nativeSha256(metadata) - ? { kind: "unchanged" as const } - : { kind: "missing_evidence" as const, reason: "existing_metadata_conflict" }; + if (nativeSha256(profile.nativeControllerRecovery) !== nativeSha256(metadata)) { + return { kind: "missing_evidence" as const, reason: "existing_metadata_conflict" }; + } + if (importing) await insertMigratedNativeRunnerCheckpoint(tx, input, checkpoint); + return { kind: importing ? "imported" as const : "unchanged" as const }; } + if (importing) await insertMigratedNativeRunnerCheckpoint(tx, input, checkpoint); await tx.update(heartbeatRuns).set({ runnerProfileJson: { ...profile, nativeControllerRecovery: metadata }, updatedAt: new Date() }) .where(eq(heartbeatRuns.id, input.runId)); return { kind: "imported" as const }; diff --git a/server/src/services/native-runtime/native-harness-backup-archive.test.ts b/server/src/services/native-runtime/native-harness-backup-archive.test.ts new file mode 100644 index 00000000000..0d54c5e51c3 --- /dev/null +++ b/server/src/services/native-runtime/native-harness-backup-archive.test.ts @@ -0,0 +1,46 @@ +import { mkdtemp, readFile, rm, stat, symlink, writeFile } from "node:fs/promises"; +import { createWriteStream } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { pipeline } from "node:stream/promises"; +import tar from "tar-stream"; +import { afterEach, expect, it } from "vitest"; +import { captureNativeHarnessArchive, restoreNativeHarnessArchive } from "./native-harness-backup-archive.js"; + +const roots: string[] = []; +afterEach(async () => { for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); }); +async function fixture() { const root = await mkdtemp(join(tmpdir(), "native-backup-archive-test-")); roots.push(root); return root; } +async function maliciousArchive(root: string, headers: Array) { + const archive = tar.pack(), file = join(root, "malicious.tar"); + const written = pipeline(archive, createWriteStream(file)); + for (const header of headers) archive.entry(header, Buffer.alloc(0)); + archive.finalize(); await written; return file; +} +it.each([ + { name: "../escaped", type: "file" as const }, + { name: "/absolute", type: "file" as const }, + { name: "link", type: "symlink" as const, linkname: "../outside" }, + { name: "device", type: "character-device" as const }, + { name: "parent/child", type: "file" as const }, +])("rejects unsafe archive entry $name", async header => { + const root = await fixture(), file = await maliciousArchive(root, [header]); + await expect(restoreNativeHarnessArchive(file, join(root, "destination"))).rejects.toThrow("native_harness_archive_invalid"); + await expect(stat(join(root, "escaped"))).rejects.toMatchObject({ code: "ENOENT" }); +}); +it("rejects duplicate paths instead of overwriting a restored file", async () => { + const root = await fixture(), file = await maliciousArchive(root, [{ name: "manifest.json", type: "file" }, { name: "manifest.json", type: "file" }]); + await expect(restoreNativeHarnessArchive(file, join(root, "destination"))).rejects.toThrow("native_harness_archive_invalid"); +}); +it("refuses a source symlink without reading its target", async () => { + const root = await fixture(), outside = join(root, "outside"); + await writeFile(outside, "private external file"); + const { mkdir } = await import("node:fs/promises"); const source = join(root, "source"); await mkdir(source); + await symlink(outside, join(source, "linked")); + await expect(captureNativeHarnessArchive(source, join(root, "archive"))).rejects.toThrow("native_harness_archive_invalid"); + expect(await readFile(outside, "utf8")).toBe("private external file"); +}); +it("honors cancellation before walking the checkpoint tree", async () => { + const root = await fixture(); + await expect(captureNativeHarnessArchive(root, join(root, "archive"), AbortSignal.abort(new Error("test cancelled")))) + .rejects.toThrow(); +}); diff --git a/server/src/services/native-runtime/native-harness-backup-archive.ts b/server/src/services/native-runtime/native-harness-backup-archive.ts new file mode 100644 index 00000000000..d4fb6882690 --- /dev/null +++ b/server/src/services/native-runtime/native-harness-backup-archive.ts @@ -0,0 +1,79 @@ +import { constants, createReadStream, createWriteStream } from "node:fs"; +import { chmod, lstat, mkdir, open, readdir } from "node:fs/promises"; +import { join, posix } from "node:path"; +import { pipeline } from "node:stream/promises"; +import tar from "tar-stream"; + +const MAX_BYTES = 256 * 1024 * 1024; +const MAX_ENTRIES = 40_000; +const fail = () => new Error("native_harness_archive_invalid"); +const safeName = (name: string) => name.length > 0 && !name.includes("\\") + && !/[\u0000-\u001f\u007f]/u.test(name) && !posix.isAbsolute(name) + && name.split("/").every(part => part !== ".." && part !== "." && part !== ""); + +export async function captureNativeHarnessArchive(root: string, target: string, signal?: AbortSignal) { + if (!(await lstat(root)).isDirectory()) throw fail(); + const archive = tar.pack(); + const output = pipeline(archive, createWriteStream(target, { flags: "wx", mode: 0o600 }), { signal }); + void output.catch(() => undefined); + let count = 0, bytes = 0; + const visit = async (relative: string): Promise => { + for (const name of (await readdir(join(root, relative))).sort()) { + signal?.throwIfAborted(); + const entry = relative ? `${relative}/${name}` : name; + if (!safeName(entry) || ++count > MAX_ENTRIES) throw fail(); + const file = join(root, entry), stat = await lstat(file); + if (stat.isDirectory()) { + await new Promise((resolve, reject) => archive.entry({ name: entry, type: "directory", mode: stat.mode & 0o777 }, + error => error ? reject(error) : resolve())); + await visit(entry); + } else if (stat.isFile()) { + bytes += stat.size; + if (bytes > MAX_BYTES) throw fail(); + const handle = await open(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + const before = await handle.stat(); + if (!before.isFile() || before.ino !== stat.ino || before.dev !== stat.dev || before.size !== stat.size) throw fail(); + const stream = archive.entry({ name: entry, size: stat.size, type: "file", mode: stat.mode & 0o777 }); + await pipeline(handle.createReadStream({ autoClose: false }), stream, { signal }); + const after = await handle.stat(); + if (before.size !== after.size || before.mtimeMs !== after.mtimeMs || before.ctimeMs !== after.ctimeMs) throw fail(); + } finally { await handle.close(); } + } else throw fail(); + } + }; + try { await visit(""); archive.finalize(); await output; } + catch (error) { archive.destroy(error as Error); await output.catch(() => undefined); throw error; } +} + +export async function restoreNativeHarnessArchive(source: string, target: string, signal?: AbortSignal) { + await mkdir(target, { mode: 0o700 }); + const archive = tar.extract(), names = new Set(); + const directories: Array<{ path: string; mode: number }> = [], directoryNames = new Set(); + let bytes = 0; + archive.on("entry", (header, body, next) => { + body.on("error", () => undefined); + void (async () => { + signal?.throwIfAborted(); + const name = header.type === "directory" ? header.name.replace(/\/$/u, "") : header.name; + if (!safeName(name) || names.has(name) || names.size >= MAX_ENTRIES + || !["directory", "file"].includes(header.type ?? "") || !Number.isSafeInteger(header.size ?? 0) + || (header.size ?? 0) < 0 || (bytes += header.size ?? 0) > MAX_BYTES) throw fail(); + names.add(name); + const file = join(target, name), mode = (header.mode ?? 0o600) & 0o777; + const parent = posix.dirname(name); + if (parent !== "." && !directoryNames.has(parent)) throw fail(); + if (header.type === "directory") { + if (header.size) throw fail(); + await mkdir(file, { mode: 0o700 }); directories.push({ path: file, mode }); directoryNames.add(name); + for await (const _ of body) throw fail(); + } else { + await pipeline(body, createWriteStream(file, { flags: "wx", mode }), { signal }); + await chmod(file, mode); + } + })().then(() => next(), error => archive.destroy(error as Error)); + }); + await pipeline(createReadStream(source), archive, { signal }); + if (!names.has("manifest.json")) throw fail(); + for (const directory of directories.reverse()) await chmod(directory.path, directory.mode); +} diff --git a/server/src/services/native-runtime/native-harness-backup-expiration.ts b/server/src/services/native-runtime/native-harness-backup-expiration.ts index 6e5ab7c424d..e61318333c3 100644 --- a/server/src/services/native-runtime/native-harness-backup-expiration.ts +++ b/server/src/services/native-runtime/native-harness-backup-expiration.ts @@ -2,7 +2,7 @@ import { notifyExternalConversationSettlement } from "../external-conversation-s import { randomUUID } from "node:crypto"; import { promises as fs } from "node:fs"; import path from "node:path"; -import { and, asc, eq, gt, inArray, isNull, lt, or, sql } from "drizzle-orm"; +import { and, asc, eq, gt, gte, inArray, isNull, lt, or, sql } from "drizzle-orm"; import { agentTaskSessions, environmentLeases, @@ -10,6 +10,7 @@ import { issues, nativeRunFinalizations, nativeRunnerCheckpoints, + nativeHarnessBackups, type Db, } from "@paperclipai/db"; import { logger } from "../../middleware/logger.js"; @@ -18,6 +19,8 @@ import { nativeSessionScopeDigestIsBusy } from "./native-session-executor.js"; import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; import { DELIVERY_QUEUES } from "../delivery-work-notifications.js"; import type { NativeHarnessBackupExpirationScope } from "../native-harness-backup-expiration-notifications.js"; +import { deleteRetiredNativeHarnessBackup, durableNativeHarnessStorage } from "./native-harness-backup-storage.js"; +import type { StorageService } from "../../storage/types.js"; export const NATIVE_HARNESS_BACKUP_EXPIRATION_INTERVAL_MS = 60 * 60 * 1000; export const NATIVE_HARNESS_BACKUP_EXPIRATION_STARTUP_DELAY_MS = 2 * 60 * 1000; @@ -51,6 +54,7 @@ export type NativeHarnessBackupExpirationOptions = { stateBase?: string; shouldContinue?: () => boolean; isSessionBusy?: (scopeDigest: string) => boolean; + storage?: StorageService; }; export type NativeHarnessBackupExpirationResult = { @@ -291,6 +295,67 @@ export function nativeHarnessBackupExpirationService( let evidenceDueAt = 0; let passDueAt = Number.POSITIVE_INFINITY; let checkpointCursor: string | null = null; + let durableBackupCursor: string | null = null; + let checkpointsEnumerated = false; + + async function expireDurableBackups(options: { + closedBefore: Date; dryRun: boolean; maxEntries: number; maxRemovals: number; shouldContinue: () => boolean; + }, result: NativeHarnessBackupExpirationResult) { + const storage = defaults.storage ?? durableNativeHarnessStorage(); + if (!storage) return; + const candidates = await db.select({ id: nativeHarnessBackups.id, scopeDigest: nativeHarnessBackups.scopeDigest, + runId: nativeHarnessBackups.runId, survivingRunId: heartbeatRuns.id, + sessionId: heartbeatRuns.nativeSessionId, status: nativeHarnessBackups.status }) + .from(nativeHarnessBackups).leftJoin(heartbeatRuns, eq(heartbeatRuns.id, nativeHarnessBackups.runId)) + .where(and(durableBackupCursor ? gt(nativeHarnessBackups.id, durableBackupCursor) : undefined, + or(eq(nativeHarnessBackups.status, "retired"), lt(nativeHarnessBackups.createdAt, options.closedBefore)))) + .orderBy(asc(nativeHarnessBackups.id)).limit(options.maxEntries + 1); + let processed = 0; + for (const candidate of candidates) { + if (!options.shouldContinue() || processed++ >= options.maxEntries || result.expired >= options.maxRemovals) { + result.truncated = true; return; + } + durableBackupCursor = candidate.id; + if (candidate.survivingRunId && !candidate.sessionId) continue; + try { + const retired = candidate.status === "retired" || await db.transaction(async tx => { + if (!candidate.survivingRunId) { + result.candidates++; + if (options.dryRun) return false; + await tx.update(nativeHarnessBackups).set({ status: "retired", updatedAt: new Date() }) + .where(eq(nativeHarnessBackups.id, candidate.id)); + return true; + } + const blocker = await sessionRetentionBlocker(tx, { scopeDigest: candidate.scopeDigest, checkpoint: true, + closedBefore: options.closedBefore, + identity: { runId: candidate.runId, normalizedSessionId: candidate.sessionId!, environmentLeaseId: null } }); + if ("reason" in blocker) return false; + const [recent] = await tx.select({ id: nativeHarnessBackups.id }).from(nativeHarnessBackups).where(and( + eq(nativeHarnessBackups.companyId, blocker.companyId), eq(nativeHarnessBackups.scopeDigest, candidate.scopeDigest), + gte(nativeHarnessBackups.createdAt, options.closedBefore))).limit(1); + if (recent) return false; + result.candidates++; + if (options.dryRun) return false; + await notifyExternalConversationSettlement(tx, { companyId: blocker.companyId }); + await tx.update(nativeRunFinalizations).set({ controllerGeneration: sql`${nativeRunFinalizations.controllerGeneration} + 1` }) + .where(inArray(nativeRunFinalizations.runId, blocker.runIds)); + await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${`${blocker.companyId}:${candidate.scopeDigest}`}, 0))`); + const cleared = await tx.delete(agentTaskSessions).where(and(eq(agentTaskSessions.companyId, blocker.companyId), + eq(agentTaskSessions.agentId, blocker.agentId), or(inArray(agentTaskSessions.lastRunId, blocker.runIds), + sql`${agentTaskSessions.sessionParamsJson}->>'sessionId' = ${candidate.sessionId}`))).returning({ id: agentTaskSessions.id }); + result.clearedTaskSessions += cleared.length; + await tx.update(nativeHarnessBackups).set({ status: "retired", updatedAt: new Date() }).where(and( + eq(nativeHarnessBackups.companyId, blocker.companyId), eq(nativeHarnessBackups.scopeDigest, candidate.scopeDigest))); + return true; + }); + if (retired && !options.dryRun && await deleteRetiredNativeHarnessBackup(db, storage, candidate.id, AbortSignal.timeout(60_000))) result.expired++; + } catch (err) { + result.failed++; + logger.warn({ err, backupId: candidate.id }, "durable native harness backup expiration skipped an object"); + } + } + durableBackupCursor = null; + } async function expireCheckpoints(options: { closedBefore: Date; dryRun: boolean; maxEntries: number; maxRemovals: number; shouldContinue: () => boolean; @@ -704,13 +769,20 @@ export function nativeHarnessBackupExpirationService( const result = emptyResult(); const shouldContinue = options.shouldContinue ?? defaults.shouldContinue ?? (() => true); if (!shouldContinue()) return { ...result, truncated: true }; - await expireCheckpoints({ + const pass = { closedBefore: new Date((options.now ?? defaults.now ?? new Date()).getTime() - closedForDays * DAY_MS), dryRun: options.dryRun ?? defaults.dryRun ?? false, maxEntries: Math.max(1, options.maxEntries ?? defaults.maxEntries ?? 200), maxRemovals: Math.max(1, options.maxRemovals ?? defaults.maxRemovals ?? 50), shouldContinue, - }, result); + }; + if (!checkpointsEnumerated) { + await expireCheckpoints(pass, result); + if (result.truncated) return result; + checkpointsEnumerated = true; + } + await expireDurableBackups(pass, result); + if (!result.truncated) checkpointsEnumerated = false; return result; } diff --git a/server/src/services/native-runtime/native-harness-backup-migration.ts b/server/src/services/native-runtime/native-harness-backup-migration.ts new file mode 100644 index 00000000000..99704200c7b --- /dev/null +++ b/server/src/services/native-runtime/native-harness-backup-migration.ts @@ -0,0 +1,76 @@ +import { createHash } from "node:crypto"; +import { lstat, readFile, readdir } from "node:fs/promises"; +import { join } from "node:path"; +import { and, eq, sql } from "drizzle-orm"; +import { environmentLeases, heartbeatRuns, nativeHarnessBackups, type Db } from "@paperclipai/db"; +import type { StorageService } from "../../storage/types.js"; +import { nativeHarnessStateBase, readNativeHarnessBackupStamp, verifyNativeHarnessBackupStamp } from "./native-harness-backup-stamp.js"; +import { persistNativeHarnessBackup, verifyDurableNativeHarnessBackup } from "./native-harness-backup-storage.js"; +import { offlineMigrationExecutionWhere } from "../offline-migration-execution.js"; + +export async function hasLocalNativeHarnessObligations(db: Db): Promise { + const rows = await db.execute(sql`select 1 from agent_task_sessions s + join heartbeat_runs r on r.id = s.last_run_id and r.company_id = s.company_id and r.agent_id = s.agent_id + join environment_leases e on e.company_id = r.company_id and e.issue_id = r.native_issue_id + and e.metadata->'nativeHarnessBackup'->>'normalizedSessionId' = r.native_session_id::text + where s.adapter_type = 'paperclip_runner' + and e.metadata->'nativeHarnessBackup'->>'schema' = 'paperclip.native-harness-backup-stamp.v2' + and not exists (select 1 from native_harness_backups b where b.company_id = r.company_id + and b.scope_digest = e.metadata->'nativeHarnessBackup'->>'sessionScopeSha256' and b.status = 'current') + limit 1`); + return rows.length > 0; +} + +export async function migrateNativeHarnessBackups(db: Db, input: { + storage: StorageService; writersStopped: true; signal: AbortSignal; +}): Promise { + if (!input.writersStopped) throw new Error("native_harness_migration_requires_offline_writers"); + const [active] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(offlineMigrationExecutionWhere()).limit(1); + if (active) throw new Error("native_harness_migration_requires_offline_writers"); + const base = nativeHarnessStateBase(); + const directories = await readdir(base).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return [] as string[]; throw error; + }); + let count = 0; + for (const scopeDigest of directories.filter(name => /^[a-f0-9]{64}$/.test(name)).sort()) { + input.signal.throwIfAborted(); + const root = join(base, scopeDigest, "failover-backups", "current"); + const stat = await lstat(root).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return null; throw error; + }); + if (!stat) continue; + if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error("native_harness_migration_unsafe_source"); + const manifestBytes = await readFile(join(root, "manifest.json")); + const manifest = JSON.parse(manifestBytes.toString("utf8")); + const state = JSON.parse(await readFile(join(root, "runner", "runner-state.json"), "utf8")); + if (state.lifecycle !== "suspended" || typeof state.runId !== "string" + || manifest.schema !== "paperclip.native-harness-backup.v1") throw new Error("native_harness_migration_source_unsettled"); + const [run] = await db.select({ companyId: heartbeatRuns.companyId, issueId: heartbeatRuns.nativeIssueId, + sessionId: heartbeatRuns.nativeSessionId, runnerId: heartbeatRuns.runnerInstanceId }).from(heartbeatRuns) + .where(eq(heartbeatRuns.id, state.runId)); + if (!run?.issueId || run.sessionId !== manifest.normalizedSessionId || run.runnerId !== manifest.runnerInstanceId) { + throw new Error("native_harness_migration_binding_mismatch"); + } + const [current] = await db.select().from(nativeHarnessBackups).where(and(eq(nativeHarnessBackups.companyId, run.companyId), + eq(nativeHarnessBackups.scopeDigest, scopeDigest), eq(nativeHarnessBackups.status, "current"))); + if (current) { + await verifyDurableNativeHarnessBackup({ db, storage: input.storage, companyId: run.companyId, + scopeDigest, manifestSha256: current.manifestSha256, signal: input.signal }); + count++; continue; + } + const manifestSha256 = `sha256:${createHash("sha256").update(manifestBytes).digest("hex")}`; + const leases = await db.select({ stamp: sql`${environmentLeases.metadata}->'nativeHarnessBackup'`, + providerLeaseId: environmentLeases.providerLeaseId }).from(environmentLeases).where(and( + eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.issueId, run.issueId), + sql`${environmentLeases.metadata}->'nativeHarnessBackup'->>'manifestSha256' = ${manifestSha256}`)); + const valid = leases.some(lease => { + const stamp = readNativeHarnessBackupStamp(lease.stamp, lease.providerLeaseId ?? ""); + return stamp?.sessionScopeSha256 === scopeDigest && verifyNativeHarnessBackupStamp(stamp, lease.providerLeaseId!); + }); + if (!valid) throw new Error("native_harness_migration_backup_unverified"); + await persistNativeHarnessBackup({ db, storage: input.storage, root, scopeDigest, signal: input.signal, + owner: { kind: "offline_migration", writersStopped: true, companyId: run.companyId, issueId: run.issueId, runId: state.runId } }); + count++; + } + return count; +} diff --git a/server/src/services/native-runtime/native-harness-backup-stamp.ts b/server/src/services/native-runtime/native-harness-backup-stamp.ts index 865a2b853cb..6ddebc6a8ec 100644 --- a/server/src/services/native-runtime/native-harness-backup-stamp.ts +++ b/server/src/services/native-runtime/native-harness-backup-stamp.ts @@ -142,12 +142,12 @@ export function createNativeHarnessBackupStamp(input: { }; } -export function verifyNativeHarnessBackupStamp( +export function readNativeHarnessBackupStamp( value: unknown, expectedProviderLeaseId: string, -): boolean { - if (!expectedProviderLeaseId) return false; - if (!value || typeof value !== "object" || Array.isArray(value)) return false; +): NativeHarnessBackupStamp | null { + if (!expectedProviderLeaseId) return null; + if (!value || typeof value !== "object" || Array.isArray(value)) return null; const stamp = value as Record; if ( stamp.schema !== "paperclip.native-harness-backup-stamp.v2" || @@ -160,9 +160,17 @@ export function verifyNativeHarnessBackupStamp( !stamp.sourceProviderLeaseId || stamp.authorizedProviderLeaseId !== expectedProviderLeaseId || typeof stamp.manifestSha256 !== "string" || - !stamp.manifestSha256.startsWith("sha256:") + !/^sha256:[a-f0-9]{64}$/.test(stamp.manifestSha256) || + !/^[a-f0-9]{64}$/.test(stamp.sessionScopeSha256) || + typeof stamp.completedAt !== "string" || !Number.isFinite(Date.parse(stamp.completedAt)) ) - return false; + return null; + return stamp as unknown as NativeHarnessBackupStamp; +} + +export function verifyNativeHarnessBackupStamp(value: unknown, expectedProviderLeaseId: string): boolean { + const stamp = readNativeHarnessBackupStamp(value, expectedProviderLeaseId); + if (!stamp) return false; const root = stateRootFromDigest(stamp.sessionScopeSha256); if (!root || !isRealDirectory(root)) return false; const backupRoot = resolve(root, "failover-backups"); diff --git a/server/src/services/native-runtime/native-harness-backup-storage.integration.test.ts b/server/src/services/native-runtime/native-harness-backup-storage.integration.test.ts new file mode 100644 index 00000000000..a6cce2b14df --- /dev/null +++ b/server/src/services/native-runtime/native-harness-backup-storage.integration.test.ts @@ -0,0 +1,175 @@ +import { createHash, randomUUID } from "node:crypto"; +import { chmod, mkdir, readFile, rename, stat, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { eq } from "drizzle-orm"; +import { agentTaskSessions, environmentLeases, heartbeatRuns, issues, nativeHarnessBackups, nativeRunFinalizations } from "@paperclipai/db"; +import { afterEach, expect, it, vi } from "vitest"; +import { describeEmbeddedPostgres, nativeRestartFixture } from "../../__tests__/helpers/native-restart.js"; +import { createLocalDiskStorageProvider } from "../../storage/local-disk-provider.js"; +import { createStorageService } from "../../storage/service.js"; +import { nativeHarnessBackupOwner, persistNativeHarnessBackup, restoreDurableNativeHarnessBackup, + verifyDurableNativeHarnessBackup } from "./native-harness-backup-storage.js"; +import { removeNativeHarnessBackup } from "./native-harness-backup-cleanup.js"; +import { buildNativeHarnessBackupManifest } from "./native-session-executor.js"; +import { createNativeHarnessBackupStamp } from "./native-harness-backup-stamp.js"; +import { hasLocalNativeHarnessObligations, migrateNativeHarnessBackups } from "./native-harness-backup-migration.js"; +import { nativeHarnessBackupExpirationService } from "./native-harness-backup-expiration.js"; + +describeEmbeddedPostgres("durable native provider backups", () => { + const environment = nativeRestartFixture(); + const ownedHomes: string[] = []; + afterEach(async () => { + for (const home of ownedHomes.splice(0)) await removeNativeHarnessBackup(home); + vi.unstubAllEnvs(); + }); + async function fixture(provider = "codex") { + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "51".repeat(32)); + const f = await environment.seedRun(randomUUID()); + const { companyId, runtimeRoot, successor } = environment.current; + await f.db.update(nativeRunFinalizations).set({ controllerBootId: successor.bootId, controllerGeneration: 1, + leaseOwner: "backup-writer", leaseExpiresAt: new Date(Date.now() + 120_000) }).where(eq(nativeRunFinalizations.runId, f.runId)); + const owner = await nativeHarnessBackupOwner(f.db, { companyId, issueId: f.issueId, runId: f.runId, bootId: successor.bootId }); + const home = join(runtimeRoot, randomUUID()), root = join(home, "failover-backups", "current"); + ownedHomes.push(home); + await mkdir(join(root, provider, "sessions"), { recursive: true }); + await mkdir(join(root, "runner")); + const identity = { normalizedSessionId: randomUUID(), providerSessionId: randomUUID(), provider }; + await writeFile(join(root, provider, "sessions", "history.json"), JSON.stringify(identity), { mode: 0o600 }); + await writeFile(join(root, "runner", "runner-state.json"), JSON.stringify({ runId: f.runId, lifecycle: "suspended" }), { mode: 0o600 }); + const manifest = JSON.stringify({ schema: "paperclip.native-harness-backup.v1", ...identity, directories: [provider, "runner"] }); + await writeFile(join(root, "manifest.json"), manifest, { mode: 0o600 }); + await chmod(join(root, provider, "sessions"), 0o500); + const sessionScopeId = randomUUID(), scopeDigest = createHash("sha256").update(sessionScopeId).digest("hex"); + const storage = createStorageService(createLocalDiskStorageProvider(join(runtimeRoot, "durable-objects"))); + return { ...f, home, root, companyId, owner, scopeDigest, sessionScopeId, storage, identity, + manifestSha256: `sha256:${createHash("sha256").update(manifest).digest("hex")}` }; + } + it.each(["codex", "claude", "opencode", "grok", "new-provider"])("restores %s session bytes after deleting the entire controller home", async provider => { + const f = await fixture(provider); + await persistNativeHarnessBackup(f); + const [row] = await f.db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.scopeDigest, f.scopeDigest)); + expect(row.status).toBe("current"); + const object = await f.storage.getObject(f.companyId, row.reference.objectKey as string); + const parts: Buffer[] = []; for await (const part of object.stream) parts.push(part); + expect(Buffer.concat(parts).includes(Buffer.from(f.identity.providerSessionId))).toBe(false); + await removeNativeHarnessBackup(f.home); + const replacement = join(environment.current.runtimeRoot, randomUUID(), "failover-backups"); + ownedHomes.push(replacement); + expect(await restoreDurableNativeHarnessBackup({ ...f, backupRoot: replacement })).toBe(true); + expect(JSON.parse(await readFile(join(replacement, "current", provider, "sessions", "history.json"), "utf8"))).toEqual(f.identity); + expect((await stat(join(replacement, "current", provider, "sessions"))).mode & 0o777).toBe(0o500); + expect(await verifyDurableNativeHarnessBackup(f)).toBe(true); + }); + it("keeps the last published backup when object storage fails", async () => { + const f = await fixture(); await persistNativeHarnessBackup(f); + const [before] = await f.db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.scopeDigest, f.scopeDigest)); + const put = vi.spyOn(f.storage, "putFile").mockRejectedValue(new Error("storage unavailable")); + await expect(persistNativeHarnessBackup(f)).rejects.toThrow("storage unavailable"); put.mockRestore(); + const rows = await f.db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.scopeDigest, f.scopeDigest)); + expect(rows.filter(row => row.status === "current").map(row => row.id)).toEqual([before.id]); + expect(rows.filter(row => row.status === "pending")).toHaveLength(1); + expect(await readFile(join(f.root, "manifest.json"), "utf8")).toContain("native-harness-backup"); + }); + it("fences an old controller whose upload finishes after ownership changes", async () => { + const f = await fixture(); await persistNativeHarnessBackup(f); + const [before] = await f.db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.scopeDigest, f.scopeDigest)); + const put = f.storage.putFile.bind(f.storage); + f.storage.putFile = async input => { + const result = await put(input); + await f.db.update(nativeRunFinalizations).set({ controllerBootId: randomUUID(), controllerGeneration: 2 }) + .where(eq(nativeRunFinalizations.runId, f.runId)); + return result; + }; + await expect(persistNativeHarnessBackup(f)).rejects.toThrow("native_runner_checkpoint_owner_lost"); + const rows = await f.db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.scopeDigest, f.scopeDigest)); + expect(rows.filter(row => row.status === "current").map(row => row.id)).toEqual([before.id]); + }); + it("rejects corrupt objects and cross-company lookups without publishing a local cache", async () => { + const f = await fixture(); await persistNativeHarnessBackup(f); + const replacement = join(environment.current.runtimeRoot, randomUUID(), "failover-backups"); + expect(await restoreDurableNativeHarnessBackup({ ...f, companyId: randomUUID(), backupRoot: replacement })).toBe(false); + const [row] = await f.db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.scopeDigest, f.scopeDigest)); + await f.storage.deleteObject(f.companyId, row.reference.objectKey as string); + await f.storage.putFile({ companyId: f.companyId, namespace: "native-harness-backups", objectKey: row.reference.objectKey as string, + originalFilename: null, contentType: "application/octet-stream", body: Buffer.from("corrupted") }); + await expect(restoreDurableNativeHarnessBackup({ ...f, backupRoot: replacement })).rejects.toThrow("hash_mismatch"); + await expect(stat(join(replacement, "current"))).rejects.toMatchObject({ code: "ENOENT" }); + }); + async function settle(f: { db: Awaited>["db"]; runId: string; issueId: string }, closedAt = new Date()) { + await f.db.update(heartbeatRuns).set({ status: "failed", finishedAt: closedAt }).where(eq(heartbeatRuns.id, f.runId)); + await f.db.update(nativeRunFinalizations).set({ phase: "terminal_failure", leaseOwner: null, leaseExpiresAt: null }) + .where(eq(nativeRunFinalizations.runId, f.runId)); + await f.db.update(issues).set({ status: "done", executionRunId: null, completedAt: closedAt }).where(eq(issues.id, f.issueId)); + } + it("migrates a lease-verified local backup once and keeps its selected session intact", async () => { + const f = await fixture("codex-home"); + const base = join(environment.current.runtimeRoot, randomUUID()); ownedHomes.push(base); await mkdir(base); + await rename(f.home, join(base, f.scopeDigest)); + const root = join(base, f.scopeDigest, "failover-backups", "current"); + vi.stubEnv("PAPERCLIP_RUNNER_STATE_DIR", base); + const runnerInstanceId = randomUUID(), providerLeaseId = randomUUID(); + const execution = { provider: { kind: "codex" }, binding: { companyId: f.companyId, issueId: f.issueId, + runId: f.runId, agentId: environment.current.agentId, executionWorkspaceId: randomUUID() }, + workspace: { cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null }, + session: { normalizedSessionId: f.identity.normalizedSessionId, driverKind: "codex_app_server", + protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } } } as never; + const manifest = buildNativeHarnessBackupManifest({ backupRoot: root, execution, runnerInstanceId, + providerSessionIdentity: { providerSessionId: f.identity.providerSessionId }, sourceProviderLeaseId: providerLeaseId }); + await writeFile(join(root, "manifest.json"), JSON.stringify(manifest)); + const stamp = createNativeHarnessBackupStamp({ manifestPath: join(root, "manifest.json"), sessionScopeId: f.sessionScopeId, + authorizedProviderLeaseId: providerLeaseId, runnerInstanceId, normalizedSessionId: f.identity.normalizedSessionId, completedAt: manifest.completedAt }); + await f.db.update(heartbeatRuns).set({ nativeSessionId: f.identity.normalizedSessionId, runnerInstanceId }).where(eq(heartbeatRuns.id, f.runId)); + await f.db.insert(environmentLeases).values({ companyId: f.companyId, issueId: f.issueId, heartbeatRunId: f.runId, + providerLeaseId, status: "expired", metadata: { nativeHarnessBackup: stamp } }); + const [selected] = await f.db.insert(agentTaskSessions).values({ companyId: f.companyId, agentId: environment.current.agentId, + adapterType: "paperclip_runner", taskKey: f.issueId, lastRunId: f.runId, + sessionParamsJson: { sessionId: f.identity.normalizedSessionId } }).returning(); + expect(await hasLocalNativeHarnessObligations(f.db)).toBe(true); + await expect(migrateNativeHarnessBackups(f.db, { storage: f.storage, writersStopped: true, signal: AbortSignal.timeout(10_000) })) + .rejects.toThrow("native_harness_migration_requires_offline_writers"); + const ownedRuns = await f.db.select({ id: heartbeatRuns.id, issueId: heartbeatRuns.nativeIssueId }).from(heartbeatRuns) + .where(eq(heartbeatRuns.companyId, f.companyId)); + for (const run of ownedRuns) await settle({ ...f, runId: run.id, issueId: run.issueId! }); + await f.db.update(nativeRunFinalizations).set({ recoveryState: "blocked" }).where(eq(nativeRunFinalizations.runId, f.runId)); + const [preservedOwner] = await f.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)); + const [queued] = await f.db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: environment.current.agentId, + status: "queued", invocationSource: "on_demand", issueId: f.issueId }).returning(); + await f.db.update(issues).set({ executionRunId: queued.id }).where(eq(issues.id, f.issueId)); + expect(await migrateNativeHarnessBackups(f.db, { storage: f.storage, writersStopped: true, signal: AbortSignal.timeout(10_000) })).toBe(1); + expect(await migrateNativeHarnessBackups(f.db, { storage: f.storage, writersStopped: true, signal: AbortSignal.timeout(10_000) })).toBe(1); + expect(await hasLocalNativeHarnessObligations(f.db)).toBe(false); + expect(await f.db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.scopeDigest, f.scopeDigest))).toHaveLength(1); + expect((await f.db.select().from(agentTaskSessions).where(eq(agentTaskSessions.id, selected.id)))[0]).toEqual(selected); + expect((await f.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)))[0]).toEqual(preservedOwner); + expect((await f.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, queued.id)))[0]).toEqual(queued); + expect((await f.db.select().from(issues).where(eq(issues.id, f.issueId)))[0].executionRunId).toBe(queued.id); + expect(JSON.parse(await readFile(join(root, "manifest.json"), "utf8"))).toEqual(manifest); + }); + it("retains live work and retries an uncertain object deletion through the existing startup retention pass", async () => { + const f = await fixture(); await persistNativeHarnessBackup(f); + const old = new Date(Date.now() - 10 * 86_400_000); + await f.db.update(nativeHarnessBackups).set({ createdAt: old }).where(eq(nativeHarnessBackups.scopeDigest, f.scopeDigest)); + const retention = nativeHarnessBackupExpirationService(f.db, { storage: f.storage }); + expect((await retention.tickCheckpoints()).expired).toBe(0); + await settle(f, old); + const drop = vi.spyOn(f.storage, "deleteObject").mockRejectedValueOnce(new Error("delete reply lost")); + expect((await retention.tickCheckpoints()).failed).toBe(1); + expect((await f.db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.scopeDigest, f.scopeDigest)))[0].status).toBe("retired"); + expect((await retention.tickCheckpoints()).expired).toBe(1); + expect(await f.db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.scopeDigest, f.scopeDigest))).toHaveLength(0); + expect(drop).toHaveBeenCalledTimes(2); + }); + it("keeps deleted-source objects unchanged during a retention dry run", async () => { + const f = await fixture(); await persistNativeHarnessBackup(f); + const old = new Date(Date.now() - 10 * 86_400_000); + await f.db.update(nativeHarnessBackups).set({ createdAt: old, runId: randomUUID() }) + .where(eq(nativeHarnessBackups.scopeDigest, f.scopeDigest)); + const [before] = await f.db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.scopeDigest, f.scopeDigest)); + const retention = nativeHarnessBackupExpirationService(f.db, { storage: f.storage }); + expect((await retention.tickCheckpoints({ dryRun: true })).expired).toBe(0); + expect((await f.db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.id, before.id)))[0]).toEqual(before); + expect(await verifyDurableNativeHarnessBackup(f)).toBe(true); + expect((await retention.tickCheckpoints()).expired).toBe(1); + expect(await f.db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.id, before.id))).toHaveLength(0); + }); +}); diff --git a/server/src/services/native-runtime/native-harness-backup-storage.ts b/server/src/services/native-runtime/native-harness-backup-storage.ts new file mode 100644 index 00000000000..629389ea0a6 --- /dev/null +++ b/server/src/services/native-runtime/native-harness-backup-storage.ts @@ -0,0 +1,221 @@ +import { createCipheriv, createDecipheriv, createHash, randomBytes, randomUUID } from "node:crypto"; +import { createReadStream, createWriteStream } from "node:fs"; +import { mkdtemp, readFile, rename } from "node:fs/promises"; +import { join } from "node:path"; +import os from "node:os"; +import { pipeline } from "node:stream/promises"; +import { Transform } from "node:stream"; +import { and, eq, inArray, sql } from "drizzle-orm"; +import { heartbeatRuns, issues, nativeHarnessBackups, nativeRunFinalizations, type Db } from "@paperclipai/db"; +import { createLocalEncryptedCipher } from "../../secrets/local-encrypted-provider.js"; +import type { StorageService } from "../../storage/types.js"; +import { assertNativeRunnerCheckpointOwner, type NativeRunnerCheckpointOwner } from "./native-runner-checkpoint.js"; +import { loadConfig } from "../../config.js"; +import { getStorageService } from "../../storage/index.js"; +import { readNativeHarnessBackupStamp, verifyNativeHarnessBackupStamp } from "./native-harness-backup-stamp.js"; +import { captureNativeHarnessArchive, restoreNativeHarnessArchive } from "./native-harness-backup-archive.js"; +import { removeNativeHarnessBackup } from "./native-harness-backup-cleanup.js"; +import { offlineMigrationExecutionWhere } from "../offline-migration-execution.js"; + +const SCHEMA = "paperclip.native-harness-object.v1"; +type Binding = { companyId: string; scopeDigest: string; id: string }; +type Reference = Binding & { + schema: typeof SCHEMA; objectKey: string; sha256: string; byteSize: number; + material: Record; manifestSha256: string; +}; +type Row = typeof nativeHarnessBackups.$inferSelect; +type OfflineOwner = { kind: "offline_migration"; companyId: string; issueId: string; runId: string; writersStopped: true }; +const fail = (reason: string): never => { throw new Error(`native_harness_storage_${reason}`); }; +const aad = (binding: Binding) => Buffer.from(JSON.stringify([SCHEMA, binding.companyId, binding.scopeDigest, binding.id])); +const scope = (binding: Pick) => and( + eq(nativeHarnessBackups.companyId, binding.companyId), eq(nativeHarnessBackups.scopeDigest, binding.scopeDigest)); + +export function durableNativeHarnessStorage(): StorageService | null { + if (loadConfig().storageProvider === "s3") return getStorageService(); + if (process.env.PAPERCLIP_EPHEMERAL_SERVER === "true") fail("durable_storage_required"); + return null; +} + +export async function verifyRecoverableNativeHarnessBackup(input: { + db: Db; companyId: string; stamp: unknown; providerLeaseId: string; +}): Promise { + const stamp = readNativeHarnessBackupStamp(input.stamp, input.providerLeaseId); + if (!stamp) return false; + const storage = durableNativeHarnessStorage(); + if (!storage) return verifyNativeHarnessBackupStamp(stamp, input.providerLeaseId); + const durable = await verifyDurableNativeHarnessBackup({ db: input.db, storage, companyId: input.companyId, + scopeDigest: stamp.sessionScopeSha256, manifestSha256: stamp.manifestSha256, signal: AbortSignal.timeout(300_000) }); + return durable || process.env.PAPERCLIP_EPHEMERAL_SERVER !== "true" && verifyNativeHarnessBackupStamp(stamp, input.providerLeaseId); +} +function reference(row: Row): Reference { + const ref = row.reference as Reference; + if (ref.schema !== SCHEMA || ref.id !== row.id || ref.companyId !== row.companyId || ref.scopeDigest !== row.scopeDigest + || !/^[a-f0-9]{64}$/.test(ref.scopeDigest) || !/^[a-f0-9]{64}$/.test(ref.sha256) + || ref.manifestSha256 !== row.manifestSha256 || !/^sha256:[a-f0-9]{64}$/.test(ref.manifestSha256) + || ref.objectKey !== `${row.companyId}/native-harness-backups/${row.scopeDigest}/${row.id}.pchn` + || !Number.isSafeInteger(ref.byteSize) || ref.byteSize <= 0 || ref.byteSize > 300 * 1024 * 1024 + || !ref.material || typeof ref.material !== "object") fail("reference_invalid"); + return ref; +} +async function digest(file: string, signal?: AbortSignal) { + const hash = createHash("sha256"); let byteSize = 0; + for await (const bytes of createReadStream(file, { signal })) { hash.update(bytes); byteSize += bytes.length; } + return { sha256: hash.digest("hex"), byteSize }; +} +async function receive(storage: StorageService, ref: Reference, target?: string, signal?: AbortSignal) { + const object = await storage.getObject(ref.companyId, ref.objectKey, { signal }); + const hash = createHash("sha256"); let byteSize = 0; + const observed = new Transform({ transform(bytes, _encoding, callback) { + hash.update(bytes); byteSize += bytes.length; + callback(byteSize > ref.byteSize ? new Error("native_harness_storage_size_mismatch") : null, bytes); + } }); + try { + if (target) await pipeline(object.stream, observed, createWriteStream(target, { flags: "wx", mode: 0o600 }), { signal }); + else for await (const bytes of object.stream) { + signal?.throwIfAborted(); hash.update(bytes); byteSize += bytes.length; + if (byteSize > ref.byteSize) fail("size_mismatch"); + } + if (byteSize !== ref.byteSize || hash.digest("hex") !== ref.sha256) fail("hash_mismatch"); + } finally { object.stream.destroy(); } +} + +export async function nativeHarnessBackupOwner(db: Db, input: { companyId: string; issueId: string; runId: string; bootId: string }) { + const [row] = await db.select().from(nativeRunFinalizations).where(and( + eq(nativeRunFinalizations.companyId, input.companyId), eq(nativeRunFinalizations.issueId, input.issueId), + eq(nativeRunFinalizations.runId, input.runId), eq(nativeRunFinalizations.controllerBootId, input.bootId))); + if (!row?.leaseOwner) fail("owner_missing"); + return { ...input, authorityRunId: input.runId, leaseOwner: row!.leaseOwner!, generation: row!.controllerGeneration } satisfies NativeRunnerCheckpointOwner; +} +async function fence(tx: Parameters[0]>[0], owner: NativeRunnerCheckpointOwner | OfflineOwner) { + const [task] = await tx.select({ executionRunId: issues.executionRunId }).from(issues) + .where(and(eq(issues.companyId, owner.companyId), eq(issues.id, owner.issueId))).for("share"); + const authorityRunId = "kind" in owner ? owner.runId : owner.authorityRunId; + if (!task || !("kind" in owner) && task.executionRunId && task.executionRunId !== authorityRunId) fail("owner_superseded"); + if ("kind" in owner) { + if (!owner.writersStopped) fail("offline_writers_required"); + const [active] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(offlineMigrationExecutionWhere()).limit(1); + if (active) fail("offline_writers_required"); + const [run] = await tx.select({ finishedAt: heartbeatRuns.finishedAt, status: heartbeatRuns.status }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.id, owner.runId), eq(heartbeatRuns.companyId, owner.companyId), + eq(heartbeatRuns.nativeIssueId, owner.issueId), eq(heartbeatRuns.runtimeMode, "native"))).for("share"); + const [coordinator] = await tx.select({ leaseOwner: nativeRunFinalizations.leaseOwner, + leaseExpiresAt: nativeRunFinalizations.leaseExpiresAt }) + .from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, owner.runId)).for("share"); + const [clock] = await tx.execute<{ now: Date }>(sql`select clock_timestamp() as now`); + if (!run?.finishedAt || !["succeeded", "failed", "cancelled", "interrupted", "timed_out"].includes(run.status) + || !coordinator || coordinator.leaseOwner && (!coordinator.leaseExpiresAt || coordinator.leaseExpiresAt > new Date(clock.now))) { + fail("offline_owner_unsettled"); + } + return; + } + await assertNativeRunnerCheckpointOwner(tx, owner, false); +} + +export async function persistNativeHarnessBackup(input: { + db: Db; storage: StorageService; owner: NativeRunnerCheckpointOwner | OfflineOwner; + scopeDigest: string; root: string; signal?: AbortSignal; +}): Promise { + if (!/^[a-f0-9]{64}$/.test(input.scopeDigest)) fail("scope_invalid"); + const manifestBytes = await readFile(join(input.root, "manifest.json")); + const manifestSha256 = `sha256:${createHash("sha256").update(manifestBytes).digest("hex")}`; + const binding = { companyId: input.owner.companyId, scopeDigest: input.scopeDigest, id: randomUUID() }; + const scratch = await mkdtemp(join(os.tmpdir(), "pc-harness-upload-")); + const key = randomBytes(32); + try { + const archive = join(scratch, "archive"), encrypted = join(scratch, "encrypted"); + await captureNativeHarnessArchive(input.root, archive, input.signal); + const plaintext = await digest(archive, input.signal), iv = randomBytes(12); + const cipher = createCipheriv("aes-256-gcm", key, iv); cipher.setAAD(aad(binding)); + await pipeline(createReadStream(archive), cipher, createWriteStream(encrypted, { flags: "wx", mode: 0o600 }), { signal: input.signal }); + const ref: Reference = { schema: SCHEMA, ...binding, manifestSha256, + objectKey: `${binding.companyId}/native-harness-backups/${binding.scopeDigest}/${binding.id}.pchn`, + ...await digest(encrypted, input.signal), + material: createLocalEncryptedCipher("write").encrypt(JSON.stringify({ schema: SCHEMA, ...binding, + key: key.toString("base64"), iv: iv.toString("base64"), tag: cipher.getAuthTag().toString("base64"), plaintext })) }; + await input.db.transaction(async tx => { + await fence(tx, input.owner); + await tx.insert(nativeHarnessBackups).values({ ...binding, issueId: input.owner.issueId, runId: input.owner.runId, + manifestSha256, reference: ref }); + }); + await input.storage.putFile({ companyId: ref.companyId, namespace: "native-harness-backups", objectKey: ref.objectKey, + originalFilename: null, contentType: "application/octet-stream", body: createReadStream(encrypted), + byteSize: ref.byteSize, sha256: ref.sha256, signal: input.signal }); + await receive(input.storage, ref, undefined, input.signal); + await input.db.transaction(async tx => { + await fence(tx, input.owner); + await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${`${ref.companyId}:${ref.scopeDigest}`}, 0))`); + const [successor] = await tx.select({ id: nativeHarnessBackups.id }).from(nativeHarnessBackups) + .innerJoin(heartbeatRuns, eq(heartbeatRuns.id, nativeHarnessBackups.runId)).where(and(scope(ref), + eq(nativeHarnessBackups.status, "current"), sql`${nativeHarnessBackups.runId} <> ${input.owner.runId}::uuid`, + sql`${heartbeatRuns.createdAt} >= (select created_at from heartbeat_runs where id = ${input.owner.runId}::uuid)`)).limit(1); + if (successor) fail("owner_superseded"); + await fence(tx, input.owner); + await tx.update(nativeHarnessBackups).set({ status: "superseded", updatedAt: new Date() }) + .where(and(scope(ref), eq(nativeHarnessBackups.status, "current"))); + await tx.update(nativeHarnessBackups).set({ status: "current", updatedAt: new Date() }) + .where(and(eq(nativeHarnessBackups.id, ref.id), eq(nativeHarnessBackups.status, "pending"))); + }); + } finally { key.fill(0); await removeNativeHarnessBackup(scratch); } +} + +export async function restoreDurableNativeHarnessBackup(input: { + db: Db; storage: StorageService; companyId: string; scopeDigest: string; + backupRoot: string; signal?: AbortSignal; +}): Promise { + const [row] = await input.db.select().from(nativeHarnessBackups).where(and(scope(input), eq(nativeHarnessBackups.status, "current"))); + if (!row) return false; + const ref = reference(row); + const wrapped = JSON.parse(createLocalEncryptedCipher("read").decrypt(ref.material)); + if (wrapped.schema !== SCHEMA || wrapped.id !== ref.id || wrapped.companyId !== ref.companyId + || wrapped.scopeDigest !== ref.scopeDigest) fail("binding_mismatch"); + const current = join(input.backupRoot, "current"); + const localManifest = await readFile(join(current, "manifest.json")).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return null; throw error; + }); + if (localManifest && `sha256:${createHash("sha256").update(localManifest).digest("hex")}` === ref.manifestSha256) return true; + const scratch = await mkdtemp(join(os.tmpdir(), "pc-harness-restore-")); + const key = Buffer.from(wrapped.key, "base64"); + try { + if (key.length !== 32) fail("key_invalid"); + const encrypted = join(scratch, "encrypted"), archive = join(scratch, "archive"); + await receive(input.storage, ref, encrypted, input.signal); + const decipher = createDecipheriv("aes-256-gcm", key, Buffer.from(wrapped.iv, "base64")); + decipher.setAAD(aad(ref)); decipher.setAuthTag(Buffer.from(wrapped.tag, "base64")); + await pipeline(createReadStream(encrypted), decipher, createWriteStream(archive, { flags: "wx", mode: 0o600 }), { signal: input.signal }); + const actual = await digest(archive, input.signal); + if (actual.sha256 !== wrapped.plaintext?.sha256 || actual.byteSize !== wrapped.plaintext?.byteSize) fail("plaintext_mismatch"); + const restored = join(scratch, "restored"); + await restoreNativeHarnessArchive(archive, restored, input.signal); + const manifest = await readFile(join(restored, "manifest.json")); + if (`sha256:${createHash("sha256").update(manifest).digest("hex")}` !== ref.manifestSha256) fail("manifest_mismatch"); + const { mkdir, cp } = await import("node:fs/promises"); + await mkdir(input.backupRoot, { recursive: true, mode: 0o700 }); + const pending = join(input.backupRoot, `.durable-${randomUUID()}`); + try { + await cp(restored, pending, { recursive: true, preserveTimestamps: true, errorOnExist: true, force: false }); + await removeNativeHarnessBackup(current); + await rename(pending, current); + } finally { await removeNativeHarnessBackup(pending); } + return true; + } finally { key.fill(0); await removeNativeHarnessBackup(scratch); } +} + +export async function verifyDurableNativeHarnessBackup(input: { + db: Db; storage: StorageService; companyId: string; scopeDigest: string; manifestSha256: string; signal?: AbortSignal; +}): Promise { + const [row] = await input.db.select().from(nativeHarnessBackups).where(and(scope(input), + eq(nativeHarnessBackups.manifestSha256, input.manifestSha256), inArray(nativeHarnessBackups.status, ["current", "superseded"]))).limit(1); + if (!row) return false; + await receive(input.storage, reference(row), undefined, input.signal); + return true; +} + +export async function deleteRetiredNativeHarnessBackup(db: Db, storage: StorageService, id: string, signal?: AbortSignal) { + const [row] = await db.select().from(nativeHarnessBackups).where(and( + eq(nativeHarnessBackups.id, id), eq(nativeHarnessBackups.status, "retired"))); + if (!row) return false; + const ref = reference(row); + await storage.deleteObject(ref.companyId, ref.objectKey, { allVersions: true, signal }); + await db.delete(nativeHarnessBackups).where(and(eq(nativeHarnessBackups.id, id), eq(nativeHarnessBackups.status, "retired"))); + return true; +} diff --git a/server/src/services/native-runtime/native-runner-checkpoint.ts b/server/src/services/native-runtime/native-runner-checkpoint.ts index 0a678a53bea..8c568cb6772 100644 --- a/server/src/services/native-runtime/native-runner-checkpoint.ts +++ b/server/src/services/native-runtime/native-runner-checkpoint.ts @@ -6,6 +6,8 @@ import { createLocalEncryptedCipher } from "../../secrets/local-encrypted-provid import { legacyControllerBootId } from "../legacy-controller-lease.js"; type Binding = { companyId: string; issueId: string; runId: string }; +export const NATIVE_CHECKPOINT_HEADER_INSPECTION_MAX_BYTES = 1024 * 1024; +export const NATIVE_CHECKPOINT_INSPECTION_TOTAL_MAX_BYTES = 8 * 1024 * 1024; export type NativeRunnerCheckpointOwner = Binding & { bootId: string; leaseOwner: string; generation: number; authorityRunId: string }; function encryptParts(binding: Binding, records: ReadonlyMap) { if (!records.size) return []; @@ -16,6 +18,11 @@ function encryptParts(binding: Binding, records: ReadonlyMap) { })); } +export async function insertMigratedNativeRunnerCheckpoint(tx: CheckpointTransaction, binding: Binding, checkpoint: DurablePrpCheckpoint) { + if (checkpoint.runId !== binding.runId) throw new Error("native_runner_checkpoint_binding_mismatch"); + await tx.insert(nativeRunnerCheckpoints).values(encryptParts(binding, checkpoint.records)); +} + function decryptPart(cipher: ReturnType, binding: Binding, part: string, material: Record): string { const envelope = JSON.parse(cipher.decrypt(material)); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index a2600506bd0..939e0415066 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -4,6 +4,7 @@ import { providerFailureFromError } from "../../vendor/paperclip-runner/index.js import { nativeCancellationHasUnsettledOwner, notifyNativeCancellation, waitForNativeCancellation } from "./native-cancellation-delivery.js"; import { readClaimedNativeStopRecovery } from "./native-cancellation-request.js"; import { publishNativeHarnessBackup } from "./native-harness-backup-publication.js"; +import { durableNativeHarnessStorage, nativeHarnessBackupOwner, persistNativeHarnessBackup, restoreDurableNativeHarnessBackup } from "./native-harness-backup-storage.js"; import { rehydratePinnedSkillAssets } from "./runtime-context.js"; import { createNativeUsageDelivery } from "./native-usage-delivery.js"; import { createNativeRecoveryUsageDelivery } from "./native-recovery-usage-delivery.js"; @@ -5964,6 +5965,7 @@ function nativeSessionConfigDigest( execution: NativeExecutionInput, executionTargetKind: "local" | "remote" = "local", legacyProjectlessRunId?: string, + recordedToolContractFingerprint?: string, ): string { const executionLocation = { executionKind: "local_process", @@ -5999,13 +6001,25 @@ function nativeSessionConfigDigest( // changes so an older thread is rotated instead of falsely resuming // without newly required tools. nativeToolContractFingerprint: - nativeToolContractFingerprintForTarget(executionTargetKind, execution.provider.kind), + recordedToolContractFingerprint ?? nativeToolContractFingerprintForTarget(executionTargetKind, execution.provider.kind), runtimeContract: nativeRuntimeContractForProvider(execution.provider), }), ) .digest("hex")}`; } +export function nativeControllerMigrationDescriptor(execution: NativeExecutionInput, recordedToolContractFingerprint: unknown) { + if (typeof recordedToolContractFingerprint !== "string" || !/^sha256:[a-f0-9]{64}$/.test(recordedToolContractFingerprint)) { + throw new Error("native_controller_migration_tool_contract_missing"); + } + return { + root: scopedRunnerdStateRoot(execution), + supervisorPath: resolve(resolvePaperclipInstanceRoot(), "runtime", "paperclip-runner", "sessions", + `${createHash("sha256").update(nativeSessionScopeKey(execution)).digest("hex")}.json`), + configDigest: nativeSessionConfigDigest(execution, "remote", undefined, recordedToolContractFingerprint), + }; +} + function idleWarmNativeSessionOwnerRunId(input: { execution: NativeExecutionInput; runnerExecutionTarget?: AdapterExecutionTarget | null; @@ -6027,6 +6041,7 @@ async function restorePerTurnRunnerdController(input: { if (input.restartRecovery || input.runnerExecutionTarget?.kind !== "remote" || input.execution.session.lifecyclePolicy.mode !== "per_turn") return; const root = scopedRunnerdStateRoot(input.execution); + await restoreDurableHarnessForExecution(input.db, input.execution, root); const identity = readRunnerdDurableIdentity(root); const reference = identity ?? readRunnerdDurableIdentity(root, true); if (!reference) { @@ -6091,6 +6106,7 @@ async function restoreWarmRunnerdController(input: { if (input.restartRecovery || input.runnerExecutionTarget?.kind !== "remote" || input.execution.session.lifecyclePolicy.mode !== "warm") return; const root = scopedRunnerdStateRoot(input.execution); + await restoreDurableHarnessForExecution(input.db, input.execution, root); const identity = readRunnerdDurableIdentity(root); const reference = identity ?? readRunnerdDurableIdentity(root, true); if (!idleWarmNativeSessionOwnerRunId(input) && (!reference || !existsSync(nativeSessionCheckpointPath(input.execution)))) { @@ -6599,6 +6615,14 @@ function harnessBackupRoot(root: string): string { return resolve(root, "failover-backups"); } +async function restoreDurableHarnessForExecution(db: Db, execution: NativeExecutionInput, root: string) { + const storage = durableNativeHarnessStorage(); + if (!storage) return; + await restoreDurableNativeHarnessBackup({ db, storage, companyId: execution.binding.companyId, + scopeDigest: createHash("sha256").update(nativeSessionScopeKey(execution)).digest("hex"), + backupRoot: harnessBackupRoot(root), signal: AbortSignal.timeout(300_000) }); +} + function harnessBackupCandidates(root: string): string[] { const backupRoot = harnessBackupRoot(root); return [resolve(backupRoot, "current"), resolve(backupRoot, "previous")]; @@ -14069,6 +14093,11 @@ async function createRunnerdBackendWithinSessionClaim( !remoteRunnerFilesystemRoot ) return; + const durableStorage = durableNativeHarnessStorage(); + if (durableStorage && !input.controllerPersistence) throw new Error("native_harness_storage_owner_missing"); + const backupOwner = durableStorage ? await nativeHarnessBackupOwner(input.db, { ...input.execution.binding, + runId: sessionToolAuthorityEpochs.get(sessionScopeId)?.runId ?? input.execution.binding.runId, + bootId: input.controllerPersistence!.bootId }) : null; // Transport release also runs when provider bootstrap failed. In that case // runnerd has no durable provider identity (and may not have created the // provider persistence directory at all), so attempting a failover backup @@ -14174,6 +14203,11 @@ async function createRunnerdBackendWithinSessionClaim( }); renameSync(temporaryManifest, manifestPath); + if (durableStorage && backupOwner) { + await persistNativeHarnessBackup({ db: input.db, storage: durableStorage, owner: backupOwner, + scopeDigest: createHash("sha256").update(sessionScopeId).digest("hex"), root: pendingRoot, + signal: AbortSignal.timeout(300_000) }); + } await publishNativeHarnessBackup({ db: input.db, companyId: input.execution.binding.companyId, scopeDigest: createHash("sha256").update(sessionScopeId).digest("hex"), backupRoot, pendingRoot, diff --git a/server/src/services/native-runtime/native-session-resume.ts b/server/src/services/native-runtime/native-session-resume.ts index 9e4fe58b486..fcf2a3e0856 100644 --- a/server/src/services/native-runtime/native-session-resume.ts +++ b/server/src/services/native-runtime/native-session-resume.ts @@ -225,7 +225,7 @@ type NativeSessionResumeSelection = { agentId: string; issueId: string; normalizedSessionId: string; - currentRunId: string; + currentRunId?: string; beforeCreatedAt: Date; /** Recovery discovery must retain an ineligible newest progress barrier. */ includeProgressBarrier?: boolean; @@ -306,7 +306,7 @@ async function selectNativeSessionResumeBarrier(db: Db, input: NativeSessionResu .from(heartbeatRuns) .where( and( - ne(heartbeatRuns.id, input.currentRunId), + input.currentRunId ? ne(heartbeatRuns.id, input.currentRunId) : undefined, eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.agentId, input.agentId), eq(heartbeatRuns.nativeIssueId, input.issueId), diff --git a/server/src/services/remote-workspaces.ts b/server/src/services/remote-workspaces.ts index 4712ae33568..521b6c365a0 100644 --- a/server/src/services/remote-workspaces.ts +++ b/server/src/services/remote-workspaces.ts @@ -73,28 +73,22 @@ export function shouldUseRemoteWorkspaces(input: { return configuredRemoteWorkspacePersistence() !== null; } -/** Deployment profile admission belongs beside workspace authority. A warm - * remote controller may replace its local projection; host-owned execution - * cannot make that promise. External conversation providers own no task disk. */ export function assertEphemeralWorkspaceExecution(input: { externalConversation: boolean; adapterType: string; pluginTaskWorkspace: boolean; remoteAuthority: boolean; }) { if (process.env.PAPERCLIP_EPHEMERAL_SERVER !== "true" || input.externalConversation) return; // Runtime selection has not promoted a fresh run yet. Validate storage here; - // the runtime selector separately admits only its resolved warm native policy. if (input.adapterType !== "paperclip_runner" || (!input.pluginTaskWorkspace && !input.remoteAuthority)) { - throw unprocessable("Volume-free servers require warm Paperclip Runner execution with remote workspace storage.", + throw unprocessable("Volume-free servers require native Paperclip Runner execution with remote workspace storage.", { code: "ephemeral_execution_configuration_required" }); } } -/** The runtime selector calls this after environment, credentials and saved - * execution policy have resolved. Raw agent settings are not lifecycle authority. */ -export function assertEphemeralNativeLifecycle(mode: "warm" | "per_turn" | undefined) { - if (process.env.PAPERCLIP_EPHEMERAL_SERVER === "true" && mode !== "warm") { - throw unprocessable("Volume-free servers require warm Paperclip Runner execution with remote workspace storage.", +export function assertEphemeralNativeRuntime(native: boolean) { + if (process.env.PAPERCLIP_EPHEMERAL_SERVER === "true" && !native) { + throw unprocessable("Volume-free servers require native Paperclip Runner execution with remote workspace storage.", { code: "ephemeral_execution_configuration_required" }); } } diff --git a/server/src/vendor/paperclip-runner/index.ts b/server/src/vendor/paperclip-runner/index.ts index 51ddeff3ea2..d28a01baba6 100644 --- a/server/src/vendor/paperclip-runner/index.ts +++ b/server/src/vendor/paperclip-runner/index.ts @@ -94,6 +94,7 @@ export const RUNNER_TOOL_SCOPE_TOKEN_HEADER = runner.RUNNER_TOOL_SCOPE_TOKEN_HEA export const DurablePrpControlPlane = runner.DurablePrpControlPlane; export const providerFailureFromError = runner.providerFailureFromError; export const decodeDurablePrpCheckpoint = runner.decodeDurablePrpCheckpoint; +export const captureLegacyDurablePrpCheckpoint = runner.captureLegacyDurablePrpCheckpoint; export const inspectDurablePrpCheckpointHeader = runner.inspectDurablePrpCheckpointHeader; export const RUNNER_PRP_PROTOCOL_MIN = runner.RUNNER_PRP_PROTOCOL_MIN; export const RUNNER_PRP_PROTOCOL_MAX = runner.RUNNER_PRP_PROTOCOL_MAX; diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 49181a8d3ec..147cae9f871 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -104,6 +104,19 @@ its DB checkpoint alone cannot replace those bytes. Per-turn application qualification under this disk loss awaits the provider-backup owner's durable storage contract; native controller recovery does not waive that dependency. +`rolling-web-deployment` definition 4 selects every registered native Daytona +profile in disposable per-turn environments; `rolling-web-deployment-warm` +uses the same profiles in reusable warm environments. Their +provider-backup path uses encrypted object storage and a fenced database ledger. +The earlier per-turn failure remains failed evidence; this definition requires +a fresh passing run, including the strict post-replacement continuation oracle. +Its revision task includes a 180-second foreground CLI reliability soak, giving +the replacement time to become ready while useful agent work remains active. +The observer rechecks the original live owner immediately before switching +traffic and fails if that turn already settled. A completed turn cannot count +as active takeover. Per-turn file inspection requires the exact committed saved +snapshot; the warm lane still requires the live workspace. + The ordinary Everyday matrix retains its 125-second active outage default. The two explicit continuity suites require 300 seconds and reject a shorter override. diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 81e7e4fdf85..7d5b470d0ca 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1,4 +1,4 @@ -import { ROLLING_WEB_DEPLOYMENT_SUITE } from "./controller-rolling-deployment.js"; +import { ROLLING_WEB_DEPLOYMENT_SUITE, ROLLING_WEB_DEPLOYMENT_WARM_SUITE } from "./controller-rolling-deployment.js"; import { FEEDBACK_PERSISTENCE_SUITE, feedbackPersistenceTask, feedbackPersistenceDigest } from "./feedback-persistence.js"; import { followupStopTask, followupStopDigest } from "./followup-stop-resume.js"; import { finalizationRecoveryTask, finalizationRecoveryDigest } from "./finalization-recovery.js"; @@ -1233,15 +1233,44 @@ const remoteWorkspaceStorageSuite: RunnerSuiteFixture = { source: "projectless-synthetic-files-and-model-created-git", authentication: "api-key", stagingQualification: false }, }; +export const kimiRunnerProfile = nativeProfile({ + id: "runner-acpx-kimi", label: "Runner Kimi Code", + provider: "acpx", acpxAgent: "kimi", qualificationCandidate: "kimi", + credential: "KIMI_API_KEY", model: "kimi-for-coding", + modelQualification: { source: "qualified_runner_profile", qualificationId: "kimi:2.3.3:native-qualified-profile17" }, +}); + +const rollingDeploymentProfiles = [...runnerProfiles, ...extendedHarnessProfiles, kimiRunnerProfile] + .filter(profile => profile.generation === "native" && profile.supportedEnvironments.includes("daytona")) + .map(original => { + const profile = productionStoryProfile(original); + return { ...profile, buildAgent(input: AgentFixtureBuildInput) { + const agent = profile.buildAgent(input); + const adapterConfig = { ...(agent.adapterConfig as Record) }; + delete adapterConfig.cwd; + return { ...agent, adapterConfig }; + } }; + }); +const rollingDeploymentSuites: RunnerSuiteFixture[] = [ + { id: ROLLING_WEB_DEPLOYMENT_SUITE, environment: runnerEnvironments.find(environment => environment.id === "daytona")! }, + { id: ROLLING_WEB_DEPLOYMENT_WARM_SUITE, environment: daytonaControllerOutageEnvironment }, +].map(({ id, environment }) => ({ + ...remoteWorkspaceStorageSuite, id, label: `Rolling control-plane web deployment (${environment.configurationKey ? "warm" : "per-turn"})`, + description: "Overlap independent ephemeral web servers while a remote execution stays active; keep ordinary reads, saves, files and a later native continuation working.", + profiles: rollingDeploymentProfiles, environments: [environment], + expectedMatrixSize: rollingDeploymentProfiles.length * remoteWorkspaceStorageSuite.tasks.length, + definitionMetadata: { ...remoteWorkspaceStorageSuite.definitionMetadata, version: 4, controllerDowntimeMs: 0, + activeWorkBoundary: "foreground-180-second-cli-reliability-soak-and-live-owner-recheck", + workspaceFileSource: environment.configurationKey ? "live" : "saved-exact-snapshot-after-per-turn-retirement", + lifecycleCoverage: environment.configurationKey ? "warm-reusable" : "per-turn-disposable", providerCoverage: "registered-native-daytona-profiles", + replacement: "ready-before-traffic-switch-sigterm-delete-entire-home", + upgradeReadiness: "active-remote-workspace-recoverable-before-handoff", + grading: "same-active-execution-ordinary-reads-saves-and-second-empty-home-continuation" }, +})); + export const runnerSuites: readonly RunnerSuiteFixture[] = [ remoteWorkspaceStorageSuite, - { ...remoteWorkspaceStorageSuite, id: ROLLING_WEB_DEPLOYMENT_SUITE, label: "Rolling control-plane web deployment", - description: "Overlap independent ephemeral web servers while a remote execution stays active; keep ordinary reads, saves, files and a later native continuation working.", - definitionMetadata: { ...remoteWorkspaceStorageSuite.definitionMetadata, version: 1, controllerDowntimeMs: 0, - replacement: "ready-before-traffic-switch-sigterm-delete-entire-home", - upgradeReadiness: "active-remote-workspace-recoverable-before-handoff", - grading: "same-active-execution-ordinary-reads-saves-and-second-empty-home-continuation" }, - }, + ...rollingDeploymentSuites, { id: CURSOR_CLOUD_SUITE, label: "Native Cursor Cloud", manualOnly: true, groups: [], description: "Local Paperclip controller with hosted Cursor work: verified artifact, same-agent follow-up, real active-run Stop and semantic plan revision/approval.", @@ -1537,12 +1566,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ id: "kimi-qualification", label: "Kimi Code Qualification", manualOnly: true, description: "Kimi Code replies, semantic planning and questions, verified file delivery, Stop and controller continuation through local and Daytona user journeys.", groups: ["native"], - profiles: [nativeProfile({ - id: "runner-acpx-kimi", label: "Runner Kimi Code", - provider: "acpx", acpxAgent: "kimi", qualificationCandidate: "kimi", - credential: "KIMI_API_KEY", model: "kimi-for-coding", - modelQualification: { source: "qualified_runner_profile", qualificationId: "kimi:2.3.3:native-qualified-profile17" }, - })], + profiles: [kimiRunnerProfile], environments: [localEnvironment, daytonaWarmEnvironment], environmentOverrides: Object.fromEntries(kimiLifecycleTasks.map(task => [task.id, { daytona: runnerEnvironments.find(environment => environment.id === "daytona")!, diff --git a/tests/runner-e2e/controller-recovery-observer.test.ts b/tests/runner-e2e/controller-recovery-observer.test.ts index 8597ee97a27..a84236c905a 100644 --- a/tests/runner-e2e/controller-recovery-observer.test.ts +++ b/tests/runner-e2e/controller-recovery-observer.test.ts @@ -1,10 +1,11 @@ import type { Db } from "../../packages/db/src/client.js"; import { describe, expect, it } from "vitest"; -import { advanceRecoveryCursor, parseRecoveryBinding, readControllerRecoverySnapshot, recoveryObservation, validControllerRecoveryReceipt, type RecoverySnapshot } from "./controller-recovery-observer.js"; +import { advanceRecoveryCursor, assertActiveRecoveryBoundary, parseRecoveryBinding, readControllerRecoverySnapshot, recoveryObservation, validControllerRecoveryReceipt, type RecoverySnapshot } from "./controller-recovery-observer.js"; import type { ControllerOutageReceipt } from "./controller-outage.js"; const binding = { companyId: "10000000-0000-0000-0000-000000000001", issueId: "10000000-0000-0000-0000-000000000002", runId: "10000000-0000-0000-0000-000000000003" }; const before: RecoverySnapshot = { runId: binding.runId, runnerInstanceId: "runner", nativeSessionId: "session", + status: "running", phase: "observed", resultId: null, bootId: "old", generation: 1, attempt: 1, leaseExpiresAt: "2026-10-24T00:01:00.000Z", sourceSeq: 42, history: [], authenticatedConnectionCount: 1, checkpointSourceSeq: 42 }; const fault: ControllerOutageReceipt = { schema: "paperclip.e2e.controller-outage/v1", requestId: "restart", previousPid: 1, @@ -13,6 +14,14 @@ const fault: ControllerOutageReceipt = { schema: "paperclip.e2e.controller-outag const after = (): RecoverySnapshot => ({ ...before, bootId: "new", generation: 2, sourceSeq: 47, authenticatedConnectionCount: 2, history: [{ controllerBootId: "new", controllerGeneration: 2, disposition: "reattach_remote_runner", at: "2026-10-24T00:01:00.001Z" }] }); describe("read-only controller recovery evidence", () => { + it("fails immediately when the original work settles while the replacement starts", () => { + const active = { ...before, leaseExpiresAt: new Date(Date.now() + 60_000).toISOString() }; + expect(() => assertActiveRecoveryBoundary(active)).not.toThrow(); + for (const patch of [{ status: "succeeded" }, { phase: "committed" }, { phase: "workspace_finalizing" }, + { resultId: "accepted" }, { leaseExpiresAt: null }, { leaseExpiresAt: new Date(0).toISOString() }]) { + expect(() => assertActiveRecoveryBoundary({ ...active, ...patch })).toThrow("boundary unexercised"); + } + }); it("does not count A's shutdown tail as B's processing even if B already claimed", () => { const stopped = { ...after(), checkpointSourceSeq: 48 }; const baseline = advanceRecoveryCursor(before, stopped); diff --git a/tests/runner-e2e/controller-recovery-observer.ts b/tests/runner-e2e/controller-recovery-observer.ts index a236b478c07..87f321425b6 100644 --- a/tests/runner-e2e/controller-recovery-observer.ts +++ b/tests/runner-e2e/controller-recovery-observer.ts @@ -13,11 +13,19 @@ import type { ControllerReplacementReceipt } from "./controller-replacement.js"; export interface RecoveryBinding { companyId: string; issueId: string; runId: string } export interface RecoverySnapshot { runId: string; runnerInstanceId: string; nativeSessionId: string; + status: string; phase: string; resultId: string | null; bootId: string; generation: number; attempt: number; leaseExpiresAt: string | null; sourceSeq: number; history: Array>; authenticatedConnectionCount: number; checkpointSourceSeq: number; } + +export function assertActiveRecoveryBoundary(snapshot: RecoverySnapshot): void { + if (snapshot.status !== "running" || snapshot.phase !== "observed" || snapshot.resultId !== null + || !snapshot.leaseExpiresAt || !Number.isFinite(Date.parse(snapshot.leaseExpiresAt)) || Date.parse(snapshot.leaseExpiresAt) <= Date.now()) { + throw new Error("Active controller handoff boundary unexercised: original execution settled before traffic switched"); + } +} export interface ControllerRecoveryReceipt { binding: RecoveryBinding; runnerInstanceId: string; @@ -98,7 +106,7 @@ export function validControllerRecoveryReceipt(fault: ControllerReplacementRecei export async function readControllerRecoverySnapshot(db: Db, binding: RecoveryBinding): Promise { return db.transaction(async tx => { - const [row] = await tx.execute(sql`select r.id as "runId", r.runner_instance_id as "runnerInstanceId", + const [row] = await tx.execute(sql`select r.id as "runId", r.status, f.phase, f.result_id as "resultId", r.runner_instance_id as "runnerInstanceId", r.native_session_id as "nativeSessionId", f.controller_boot_id as "bootId", f.controller_generation as generation, f.attempt, f.lease_expires_at::text as "leaseExpiresAt", f.recovery_history as history, coalesce((select max(e.source_seq) from heartbeat_run_events e where e.run_id = r.id @@ -139,7 +147,10 @@ export async function createControllerRecoveryObserver(configPath: string, bindi runnerInstanceId: before.runnerInstanceId, nativeSessionId: before.nativeSessionId }; }; return { - beforeHandoff: capture, + beforeHandoff: async (fault: ControllerReplacementReceipt) => { + await capture(fault); + assertActiveRecoveryBoundary(before!); + }, // Called only after the old server is confirmed dead. This also excludes // receipts committed while it was stopping from replacement evidence. stopped: capture, diff --git a/tests/runner-e2e/controller-rolling-deployment.test.ts b/tests/runner-e2e/controller-rolling-deployment.test.ts index 2dfa052eefb..d7310512412 100644 --- a/tests/runner-e2e/controller-rolling-deployment.test.ts +++ b/tests/runner-e2e/controller-rolling-deployment.test.ts @@ -3,6 +3,7 @@ import { rollController, validControllerRollingDeploymentReceipt } from "./contr import { validControllerReplacementReceipt } from "./controller-replacement.js"; import { controllerOutageForExecution, validControllerOutageReceipt } from "./controller-outage.js"; import { parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js"; +import { runnerProfiles, extendedHarnessProfiles, kimiRunnerProfile, runnerSuites } from "./catalog.js"; function fixture() { const order: string[] = []; @@ -55,4 +56,27 @@ describe("rolling controller qualification", () => { const [outage] = selectRunnerExecutions(parseRunnerSelectors(["--suite", "remote-workspace-storage"])); expect(controllerOutageForExecution(outage!, {})).toEqual({ signal: "SIGKILL", holdMs: 300_000 }); }); + it("requires both lifecycle modes for every registered native sandbox profile", () => { + const executions = ["rolling-web-deployment", "rolling-web-deployment-warm"] + .flatMap(suite => selectRunnerExecutions(parseRunnerSelectors(["--suite", suite]))); + const native = [...runnerProfiles, ...extendedHarnessProfiles, kimiRunnerProfile] + .filter(profile => profile.generation === "native" && profile.supportedEnvironments.includes("daytona")); + const kind = (profile: (typeof native)[number]) => { + const provider = profile.expectedRuntimeMetadata.provider; + const harness = "acpxAgent" in profile ? profile.acpxAgent + : provider === "acpx" && profile.provider !== "acpx" ? profile.provider : ""; + return `${provider}:${harness}`; + }; + const registeredKinds = new Set(runnerSuites.flatMap(suite => suite.profiles) + .filter(profile => profile.generation === "native" && profile.supportedEnvironments.includes("daytona")) + .map(kind)); + expect(new Set(native.map(kind))).toEqual(registeredKinds); + for (const profile of native) { + const cells = executions.filter(execution => execution.profile.id === profile.id); + expect(cells).toHaveLength(2); + expect(new Set(cells.map(cell => cell.environment.configurationKey ?? "per_turn"))) + .toEqual(new Set(["per_turn", "warm-controller-outage-v1"])); + } + expect(executions.every(execution => execution.suite.definitionMetadata?.retainedControllerContinuation === true)).toBe(true); + }); }); diff --git a/tests/runner-e2e/controller-rolling-deployment.ts b/tests/runner-e2e/controller-rolling-deployment.ts index 75ace959092..46e1570304e 100644 --- a/tests/runner-e2e/controller-rolling-deployment.ts +++ b/tests/runner-e2e/controller-rolling-deployment.ts @@ -3,7 +3,9 @@ import type { MatrixExecution } from "./types.js"; import type { ControllerOutageReceipt } from "./controller-outage.js"; export const ROLLING_WEB_DEPLOYMENT_SUITE = "rolling-web-deployment"; -export const isRollingWebDeployment = (execution: Pick) => execution.suite.id === ROLLING_WEB_DEPLOYMENT_SUITE; +export const ROLLING_WEB_DEPLOYMENT_WARM_SUITE = "rolling-web-deployment-warm"; +export const isRollingWebDeployment = (execution: Pick) => + [ROLLING_WEB_DEPLOYMENT_SUITE, ROLLING_WEB_DEPLOYMENT_WARM_SUITE].includes(execution.suite.id); export interface ControllerRollingDeploymentReceipt extends Omit { schema: "paperclip.e2e.controller-rolling-deployment/v1"; diff --git a/tests/runner-e2e/everyday-cases.ts b/tests/runner-e2e/everyday-cases.ts index 153ac13096c..10dc07d9db4 100644 --- a/tests/runner-e2e/everyday-cases.ts +++ b/tests/runner-e2e/everyday-cases.ts @@ -29,6 +29,7 @@ export function productionStoryProfile( export const SLUGIFY_REQUIREMENTS = `Build a small dependency-free Python command-line tool in slugify.py. It accepts one positional string and prints its slug: trim whitespace, lowercase, replace each sequence outside ASCII a-z and 0-9 with one hyphen, then trim hyphens. Expose slugify(text) for reuse. Include a README and automated tests. Deliver the source and tests as a downloadable ZIP. Work in the project workspace.`; export const SLUGIFY_REVISION = `Add an optional --separator argument that accepts either - or _. Keep - as the default and preserve the earlier behavior. Deliver an updated ZIP, keeping the previous download available.`; +export const SLUGIFY_DEPLOYMENT_SOAK = `Before editing, run a short command that prints the current workspace path. After implementing the revision, run a foreground reliability soak for a full 180 seconds: repeatedly invoke the CLI on varied inputs and check its output, sleeping briefly between iterations to avoid excessive CPU use. Wait for that command to finish; do not background it or shorten the duration. Include a small soak report with measured elapsed time and check count in the updated ZIP. Finish only after the soak and the regular tests pass.`; export const LATE_REQUIREMENT = `Also support --max-length as a positive integer. Truncate the final slug to that length, then remove any trailing separator. In particular, input " Queue--Ready!! " with --max-length 7 must print "queue-r". Preserve the default behavior.`; const definitions = [ diff --git a/tests/runner-e2e/everyday-flow.ts b/tests/runner-e2e/everyday-flow.ts index a699646cbee..ccd87e9ef89 100644 --- a/tests/runner-e2e/everyday-flow.ts +++ b/tests/runner-e2e/everyday-flow.ts @@ -39,7 +39,7 @@ import { StoryDecisionError, type StoryInteraction, } from "./everyday-decisions.js"; -import { LATE_REQUIREMENT, SLUGIFY_REVISION, requiresEverydayArtifactOracle } from "./everyday-cases.js"; +import { LATE_REQUIREMENT, SLUGIFY_REVISION, SLUGIFY_DEPLOYMENT_SOAK, requiresEverydayArtifactOracle } from "./everyday-cases.js"; import { isActiveStoryRun, isStoryWorkspaceDeferral, @@ -186,7 +186,8 @@ export async function runEverydayFlow(input: Input) { const controllerOutage = controllerOutageForExecution(execution, process.env); const requireRemoteRestart = controllerOutage !== undefined || isRollingWebDeployment(execution); const remoteWorkspaceStorage = isRemoteWorkspaceStorageExecution(execution); - if (remoteWorkspaceStorage) ev.remoteWorkspaceStorage = { browserReadPhases: [] }; + if (remoteWorkspaceStorage) ev.remoteWorkspaceStorage = { browserReadPhases: [], + expectedFileSource: execution.suite.definitionMetadata?.lifecycleCoverage === "per-turn-disposable" ? "saved" : "live" }; const providerChoice = caseId === "provider-decline" || caseId === "provider-second"; const nativeProviderCase = caseId === "provider-native"; let aggregatorFixture: Awaited> | undefined; @@ -562,7 +563,8 @@ export async function runEverydayFlow(input: Input) { const point = await pollUntil({ label: `remote workspace ${phase} saved version`, deadlineAt: input.deadlineAt, load: () => readRemoteWorkspaceStorageCheckpoint({ api, companyId: fixtures.company.id, - agentId: fixtures.agent.id, issueId: parent!.id, environmentId: fixtures.environment.id }), + agentId: fixtures.agent.id, issueId: parent!.id, environmentId: fixtures.environment.id, + expectedFileSource: ev.remoteWorkspaceStorage!.expectedFileSource }), accept: observed => Boolean(observed.point), }); ev.remoteWorkspaceStorage![phase] = point.point!; @@ -570,7 +572,8 @@ export async function runEverydayFlow(input: Input) { const browserPhase = phase === "initial" ? "before-restart" : "after-restart"; await page.goto(`${taskUrl(parent!)}?browse=1&q=${REMOTE_WORKSPACE_PROOF_FILE}`); await expect(page.getByRole("combobox", { name: "Search workspace files" })).toBeVisible(); - await expect(page.getByLabel("Workspace file source", { exact: true })).toHaveText("Live workspace"); + await expect(page.getByLabel("Workspace file source", { exact: true })).toHaveText( + ev.remoteWorkspaceStorage!.expectedFileSource === "saved" ? /^Saved copy(?: from .+)?$/ : "Live workspace"); await page.getByRole("treeitem").filter({ hasText: REMOTE_WORKSPACE_PROOF_FILE }).last().click(); await expect(page.getByText(point.content!.trim(), { exact: true })).toBeVisible(); await input.capture(`remote-workspace-${browserPhase}`, `Remote workspace file ${browserPhase}`, `remote-workspace-${browserPhase}.png`); @@ -936,7 +939,7 @@ export async function runEverydayFlow(input: Input) { throw new Error( "Remote fault boundary unexercised: no verified saved project", ); - await reply(SLUGIFY_REVISION); + await reply(isRollingWebDeployment(execution) ? `${SLUGIFY_REVISION}\n\n${SLUGIFY_DEPLOYMENT_SOAK}` : SLUGIFY_REVISION); note("remote-revision-submitted"); await pollUntil({ label: "remote revision executing", diff --git a/tests/runner-e2e/everyday-remote-restart.test.ts b/tests/runner-e2e/everyday-remote-restart.test.ts index 43e41fd6b54..f1a45b4fa19 100644 --- a/tests/runner-e2e/everyday-remote-restart.test.ts +++ b/tests/runner-e2e/everyday-remote-restart.test.ts @@ -195,6 +195,18 @@ describe("remote restart lifecycle evidence", () => { expect(gradeRemoteRestart(evidence).every(check => check.passed)).toBe(true); }); + it("accepts the same start notification after reattachment and rejects a different turn or payload", () => { + const f = fixture(); + const replay = f.event(5, "turn.started"); + const after = [...f.rows, replay]; + expect(observeRemoteRestartBoundary(f.active, after, f.leases, f.scope, true).identity).toEqual(f.evidence.before.identity); + expect(() => observeRemoteRestartBoundary(f.active, after, f.leases, f.scope)).toThrow("active original provider turn"); + expect(() => observeRemoteRestartBoundary(f.active, [...f.rows, f.event(5, "turn.started", { changed: true })], f.leases, f.scope, true)) + .toThrow("active original provider turn"); + replay.payload.prpEvent.turnId = "another-turn"; + expect(() => observeRemoteRestartBoundary(f.active, after, f.leases, f.scope, true)).toThrow("active original provider turn"); + }); + it.each(["replacement", "second submission", "changed tool receipt", "changed start receipt", "missing terminal"])("rejects %s at completion", fault => { const { evidence } = fixture(); if (fault === "replacement") evidence.final.run.id = "replacement"; diff --git a/tests/runner-e2e/everyday-remote-restart.ts b/tests/runner-e2e/everyday-remote-restart.ts index 60b12c19de3..cb2fd98f3f7 100644 --- a/tests/runner-e2e/everyday-remote-restart.ts +++ b/tests/runner-e2e/everyday-remote-restart.ts @@ -125,7 +125,10 @@ export function observeRemoteRestartBoundary(run: StoryRun, rows: readonly Row[] const events = projectRemoteRestartEvents(run, rows); const starts = events.filter(event => event.eventType === "turn.started" && event.sourceKind === "runner"); const completed = events.filter(event => event.toolCompleted && event.sourceKind === "runner" && event.toolExecutionId); - if (starts.length !== 1 || !id(starts[0]!.turnId) || completed.length === 0 + const originalStart = starts[0]; + const sameStart = originalStart && id(originalStart.turnId) && starts.every(event => + event.turnId === originalStart.turnId && event.payloadSha256 === originalStart.payloadSha256); + if ((!recovered && starts.length !== 1) || !sameStart || completed.length === 0 || completed.some(event => event.turnId !== starts[0]!.turnId) || (!recovered && events.some(event => ["turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted", "run.terminal"].includes(event.eventType)))) { throw new Error("Remote restart boundary has no completed tool in an active original provider turn"); diff --git a/tests/runner-e2e/remote-workspace-storage.test.ts b/tests/runner-e2e/remote-workspace-storage.test.ts index 64f9227b93c..4fa50e72462 100644 --- a/tests/runner-e2e/remote-workspace-storage.test.ts +++ b/tests/runner-e2e/remote-workspace-storage.test.ts @@ -56,6 +56,36 @@ describe("remote workspace Product E2E qualification", () => { expect(result.content).toBe(`${"a".repeat(32)}\n`); }); + it("reads the exact saved snapshot after per-turn sandbox retirement", async () => { + const f = checkpointApi(); + const originalGet = f.api.get; + const location = f.workspace.workspace.metadata.remoteWorkspace.location; + const detached = { ...f.workspace, workspace: { ...f.workspace.workspace, + metadata: { remoteWorkspace: { ...f.workspace.workspace.metadata.remoteWorkspace, location: null } } } }; + const file = { ...f.file, resource: { ...f.file.resource, source: { kind: "saved", snapshotId: f.receipt.snapshotId } } }; + const lease = { id: location.leaseId, companyId: f.context.companyId, issueId: f.context.issueId, + providerLeaseId: location.providerLeaseId }; + const api = { get: async (route: string): Promise => { + if (route.endsWith("/workspace")) return detached as T; + if (route.endsWith("/leases")) return [lease] as T; + if (route.includes("file-resources/content")) return file as T; + return originalGet(route); + } }; + const input = { ...f.context, api, expectedFileSource: "saved" as const }; + expect((await readRemoteWorkspaceStorageCheckpoint(input)).point).toMatchObject({ fileSource: "saved", fileSnapshotId: id("2") }); + expect(await readRemoteWorkspaceStorageCheckpoint({ ...input, expectedFileSource: "live" })).toEqual({}); + file.resource.source.snapshotId = id("5"); + expect(await readRemoteWorkspaceStorageCheckpoint(input)).toEqual({}); + file.resource.source.snapshotId = f.receipt.snapshotId; + lease.companyId = id("5"); + expect(await readRemoteWorkspaceStorageCheckpoint(input)).toEqual({}); + const evidence = goodEvidence(); evidence.expectedFileSource = "saved"; + for (const point of [evidence.initial!, evidence.final!]) Object.assign(point, { fileSource: "saved", fileSnapshotId: point.snapshotId }); + expect(gradeRemoteWorkspaceStorage(evidence).every(check => check.passed)).toBe(true); + evidence.final!.fileSnapshotId = id("9"); + expect(gradeRemoteWorkspaceStorage(evidence).every(check => check.passed)).toBe(false); + }); + it.each(["run-id", "company", "agent", "native-issue", "runtime", "status", "schema", "workspace", "epoch", "snapshot", "save-state", "lease", "provider", "remote-root"] as const)( "rejects a full-run %s mismatch even when its public summary succeeds", async mismatch => { diff --git a/tests/runner-e2e/remote-workspace-storage.ts b/tests/runner-e2e/remote-workspace-storage.ts index 7a820c60bd9..6275706a4e0 100644 --- a/tests/runner-e2e/remote-workspace-storage.ts +++ b/tests/runner-e2e/remote-workspace-storage.ts @@ -23,9 +23,11 @@ export interface RemoteWorkspaceStorageCheckpoint { providerType: string; proofSha256: string; fileSource: string; + fileSnapshotId?: string; } export interface RemoteWorkspaceStorageEvidence { + expectedFileSource?: "live" | "saved"; initial?: RemoteWorkspaceStorageCheckpoint; final?: RemoteWorkspaceStorageCheckpoint; browserReadPhases: string[]; @@ -47,6 +49,7 @@ export function remoteWorkspaceProofDigest(content: string): string { export async function readRemoteWorkspaceStorageCheckpoint(input: { api: Pick; companyId: string; agentId: string; issueId: string; environmentId: string; + expectedFileSource?: "live" | "saved"; }): Promise<{ point?: RemoteWorkspaceStorageCheckpoint; content?: string }> { const view = await input.api.get<{ issueId: string; workspace: ExecutionWorkspace | null }>(`/api/issues/${input.issueId}/workspace`); const workspace = view.workspace; @@ -54,8 +57,10 @@ export async function readRemoteWorkspaceStorageCheckpoint(input: { if (view.issueId !== input.issueId || !workspace || workspace.companyId !== input.companyId || workspace.providerType !== "cloud_sandbox" || !authorityResult.success) return {}; const authority = authorityResult.data; - if (!authority.snapshotId || authority.dirty || authority.writerRunId || !authority.location - || authority.location.environmentId !== input.environmentId) return {}; + const expectedSource = input.expectedFileSource ?? "live"; + if (!authority.snapshotId || authority.dirty || authority.writerRunId) return {}; + if (expectedSource === "live" && (!authority.location || authority.location.environmentId !== input.environmentId)) return {}; + if (expectedSource === "saved" && authority.location !== null) return {}; const runs = await input.api.get(`/api/issues/${input.issueId}/runs`); const latest = runs.filter(run => run.runtimeMode === "native" && run.status === "succeeded" && run.agentId === input.agentId && typeof run.runId === "string") @@ -68,19 +73,27 @@ export async function readRemoteWorkspaceStorageCheckpoint(input: { if (!receiptResult.success) return {}; const receipt = receiptResult.data; if (receipt.state !== "saved" || receipt.snapshotId !== authority.snapshotId - || receipt.workspaceId !== workspace.id || receipt.epoch !== authority.epoch - || receipt.leaseId !== authority.location.leaseId || receipt.providerLeaseId !== authority.location.providerLeaseId - || receipt.remoteCwd !== authority.location.remoteCwd) return {}; + || receipt.workspaceId !== workspace.id || receipt.epoch !== authority.epoch) return {}; + if (authority.location && (receipt.leaseId !== authority.location.leaseId + || receipt.providerLeaseId !== authority.location.providerLeaseId || receipt.remoteCwd !== authority.location.remoteCwd)) return {}; + if (expectedSource === "saved") { + const leases = await input.api.get>( + `/api/environments/${input.environmentId}/leases`); + if (!leases.some(lease => lease.id === receipt.leaseId && lease.companyId === input.companyId + && lease.issueId === input.issueId && lease.providerLeaseId === receipt.providerLeaseId)) return {}; + } const file = await input.api.get(`/api/issues/${input.issueId}/file-resources/content?workspace=execution&path=${REMOTE_WORKSPACE_PROOF_FILE}`); if (file.content?.encoding !== "utf8" || typeof file.content.data !== "string") throw new Error("Remote workspace file oracle did not return text bytes"); - if (file.resource?.source?.kind !== "live" || file.resource.workspaceKind !== "execution_workspace" + if (file.resource?.source?.kind !== expectedSource || file.resource.workspaceKind !== "execution_workspace" || file.resource.workspaceId !== workspace.id) return {}; + const fileSnapshotId = file.resource.source.kind === "saved" ? file.resource.source.snapshotId ?? undefined : undefined; + if (expectedSource === "saved" && fileSnapshotId !== authority.snapshotId) return {}; return { content: file.content.data, point: { workspaceId: workspace.id, snapshotId: authority.snapshotId, runId: run.id, runSnapshotId: receipt.snapshotId, runSaveState: receipt.state, authoritySchema: authority.schema, authorityDirty: authority.dirty, authorityWriter: authority.writerRunId, providerType: workspace.providerType, - proofSha256: remoteWorkspaceProofDigest(file.content.data), fileSource: file.resource?.source?.kind, + proofSha256: remoteWorkspaceProofDigest(file.content.data), fileSource: file.resource?.source?.kind, fileSnapshotId, } }; } @@ -91,7 +104,9 @@ export function gradeRemoteWorkspaceStorage(evidence: RemoteWorkspaceStorageEvid && point.snapshotId === point.runSnapshotId && point.runSaveState === "saved" && point.authoritySchema === "paperclip.remote-workspace.v1" && point.authorityDirty === false && point.authorityWriter === null && point.providerType === "cloud_sandbox" - && point.fileSource === "live" && /^[a-f0-9]{64}$/.test(point.proofSha256)); + && point.fileSource === (evidence.expectedFileSource ?? "live") + && (point.fileSource !== "saved" || point.fileSnapshotId === point.snapshotId) + && /^[a-f0-9]{64}$/.test(point.proofSha256)); return [ { id: "remote-workspace-initial-save", passed: committed(evidence.initial), detail: "The initial task admission commits a remote workspace snapshot and a matching saved run receipt." }, { id: "remote-workspace-final-save", passed: committed(evidence.final), detail: "The later admission commits its files before finalization succeeds." },