diff --git a/doc/opencode-cold-recovery.md b/doc/opencode-cold-recovery.md new file mode 100644 index 00000000000..842fd323cbc --- /dev/null +++ b/doc/opencode-cold-recovery.md @@ -0,0 +1,150 @@ +# OpenCode launch artifacts and cold recovery + +OpenCode records the absolute paths and hashes of its Node executable, proxy, +and native executable in its durable provider state. An image upgrade can change +these files. A saved session must keep its admitted files. The Runner still +rejects a different launch profile. + +For a new managed remote OpenCode session with durable object storage, Core puts +these three qualified files in the runtime account's private, content-addressed +launch directory. Core reads the home and UID from the runtime OS account +database. It does not infer ownership from `HOME`, a workspace path, or a provider +response. The launch receipt records this UID, the absolute directory, the pack +manifest, and the encrypted artifact identity. + +Core saves the files before it releases the first provider turn. Artifact capture +uses the current native coordinator claim and the recorded Core admission. A +brief admission lock waits for at most one second and checks the current owner +after it acquires the lock. A stale, revoked, or unknown owner cannot publish. +The artifact operation does not grant execution or tool authority. + +The archive is limited to 512 MiB of launch files. Core streams files and checks +hashes and file identity before and after capture. Pack-internal package links +can resolve to regular source files. Links outside the selected pack are refused. +The published launch directory contains regular files and private, owned +parents. Node and OpenCode must be qualified native ELF executables. A script +wrapper or a changed account fails closed. The selected runtime must still +supply the qualified native executables' platform and system dependencies. + +The artifact object uses authenticated encryption and a company-bound key +wrapper. A company can reuse a ready object with the same file descriptor. +Canonical run and harness-backup references keep the object alive. Large launch +files are excluded from each later harness archive. A pending upload records its +exact encrypted object reference before upload, so an interrupted reply can be +retired without guessing an object key. + +A ready company closure is reused. Concurrent first admissions may independently +upload that closure: each admission retains its own cancellation, ownership fence, +and pending receipt rather than depending on another controller's live transfer. +Each capture has the 512 MiB limit and five-minute deadline; a run can retain at +most three pending uploads. Publication selects one verified ready object under +the closure lock. A loser pins that winner before deleting only its own object. +This bounds each admitted run, not total simultaneous company egress. A burst can +therefore transfer duplicate bytes; distributed upload election is not included. + +On cold recovery, Core restores the canonical Runner checkpoint, workspace +snapshot, and native harness data through their normal paths. It also verifies +and decrypts the recorded launch files. The runtime must have the same owned +account UID and absolute home. Core stages the exact files at the recorded +content-addressed paths. It does not replace files in the current image. + +Published workspaces retain their recorded directory basename across lease +changes. A replacement allocation with the same provider base therefore keeps +the native session's absolute cwd. The workspace service can restore files to a +different base, but that alone does not authorize changing a native session's +cwd. This change adds no directory aliases and makes no change to the Rust +profile guard. + +An explicitly governed continuity break archives the controller launch cache +with the prior session state, receipt, and backups. The cache is not a new +execution authority and must not remain active for the fresh provider session. + +Live Runner reattachment does not download or replace the launch files. +Historical sessions that recorded image-rooted paths keep those paths. This +change cannot reconstruct files that were never saved. A governed fresh-session +transition can use a new pack after existing effects and receipts are reconciled. +It must not replay a previous turn or treat a missing result as proof that no +effect occurred. + +Artifact cleanup uses the existing bounded checkpoint and backup expiration +owners. It requires closed work, settled native ownership, and exact stopped +remote execution receipts. Backup retirement releases only that backup reference. +The separate run reference lasts until fenced checkpoint expiration, including +when an interrupted backup upload is retired while a resume handle still exists. +A remaining backup or run reference keeps the object. A failed external delete keeps its receipt and reference for a later normal +expiration attempt. There is no new idle scan or timer. + +Checkpoint expiration checks current resume handles under its existing locks. +It then fences old generations and atomically replaces the expired header with +a versioned, company/run-scoped retirement marker. An earlier handle keeps the +original checkpoint and files. After retirement, readers and header writers +refuse the old checkpoint. Source consumers also check retirement in their +existing ownership query, so a cached old snapshot cannot bypass the barrier. +The marker stays in the existing header index until +external artifact deletion succeeds, including after a server restart. No +transaction waits for a network delete. Families with more than 100 related +runs keep every pin and checkpoint. Missing object storage keeps the marker and +pins for a later normal cleanup attempt. + +The capture wait responds to Stop, startup completion, and the execution +deadline. Native bulk transfer does not offer a cancellation signal. Core keeps +that actual transfer registered as pending startup work after it stops waiting. +The late operation can write only its private local scratch files. It cannot +publish after the context closes. It removes its scratch files and releases its +worker responsibility only after actual settlement. Stop does not imply that +this transport has finished. + +## Local qualification + +Run the focused server suites with the repository's supported pnpm version: + +```sh +pnpm --filter @paperclipai/server exec vitest run --config vitest.config.ts \ + src/services/native-runtime/native-harness-backup-storage.integration.test.ts \ + src/services/native-runtime/native-harness-backup-archive.test.ts \ + src/services/native-runtime/remote-opencode-artifacts.test.ts \ + src/services/native-runtime/native-provider-pack-admission.test.ts \ + src/services/native-runtime/native-provider-artifact-capture.test.ts \ + src/services/native-runtime/native-runner-checkpoint-retirement.test.ts \ + src/services/native-runtime/native-session-executor.test.ts \ + src/__tests__/native-harness-backup-expiration.test.ts \ + src/__tests__/native-harness-backup-expiration-config.test.ts +``` + +The opt-in Linux qualification requires two locally cached Paperclip images with +full immutable image IDs and different OpenCode proxy hashes. Both images must +have the same Linux architecture and the standard qualified provider pack. The +new image must include the packaged Runner binary at +`/app/server/dist/vendor/paperclip-runner/bin/paperclip-runnerd`. +Build the current checkout's Runner TypeScript SDK first. Use the local Docker +context that has the cached images: + +```sh +pnpm --filter @paperclipai/paperclip-runner build:typescript +pnpm --filter @paperclipai/server exec tsx scripts/qualify-opencode-cold-recovery.mts \ + --context desktop-linux \ + --old-image "$OLD_IMAGE" \ + --new-image "$NEW_IMAGE" \ + --output /tmp/opencode-cold-recovery-result.json +``` + +The command accepts only full `sha256:` image IDs and a local Unix Docker +endpoint. It never pulls an image. It creates isolated PostgreSQL data and two +owned containers. The containers have no network, a read-only image filesystem, +resource limits, and a private fixture OS account. The synthetic model runs on +container loopback. No external model credential or API is used. + +The command uses production workspace save/restore, encrypted checkpoint and +harness storage, artifact publication, admission, SDK transport, Rust Runner, +and actual OpenCode. It destroys the original allocation's account and controller +files, changes the image and allocation lease, and restores the same provider +session. It requires the useful workspace file and earlier model context to +survive. Exactly one model request is allowed in each turn. The result records +image IDs, pack digests, checkout revision and dirty state, lease identities, +byte counts, session continuity, and useful output. The command removes only its +own containers and PostgreSQL server. It retains its temporary evidence directory. + +The independent object-store fixture uses the real StorageService and encrypted +streams with an owned local directory outside both destroyed allocations. This +qualifies the cold restore contract. It does not qualify a hosted S3 service, +a task's HTTP/actor workflow, or completion of an existing hosted task. diff --git a/packages/db/src/migrations/0438_mighty_outlaw_kid.sql b/packages/db/src/migrations/0438_mighty_outlaw_kid.sql new file mode 100644 index 00000000000..b15bb2a7090 --- /dev/null +++ b/packages/db/src/migrations/0438_mighty_outlaw_kid.sql @@ -0,0 +1,27 @@ +CREATE TABLE "native_provider_artifact_references" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "company_id" uuid NOT NULL, + "artifact_id" uuid NOT NULL, + "run_id" uuid NOT NULL, + "backup_id" uuid, + "created_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE TABLE "native_provider_artifacts" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "company_id" uuid NOT NULL, + "closure_digest" text NOT NULL, + "status" text DEFAULT 'pending' NOT NULL, + "upload_token" uuid, + "descriptor" jsonb NOT NULL, + "reference" jsonb, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +ALTER TABLE "native_provider_artifact_references" ADD CONSTRAINT "native_provider_artifact_references_artifact_id_native_provider_artifacts_id_fk" FOREIGN KEY ("artifact_id") REFERENCES "public"."native_provider_artifacts"("id") ON DELETE restrict ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "native_provider_artifact_references" ADD CONSTRAINT "native_provider_artifact_references_backup_id_native_harness_backups_id_fk" FOREIGN KEY ("backup_id") REFERENCES "public"."native_harness_backups"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +CREATE INDEX "native_provider_artifact_references_artifact_idx" ON "native_provider_artifact_references" USING btree ("company_id","artifact_id");--> statement-breakpoint +CREATE UNIQUE INDEX "native_provider_artifact_references_run_idx" ON "native_provider_artifact_references" USING btree ("company_id","run_id","artifact_id") WHERE "native_provider_artifact_references"."backup_id" is null;--> statement-breakpoint +CREATE UNIQUE INDEX "native_provider_artifact_references_backup_idx" ON "native_provider_artifact_references" USING btree ("company_id","artifact_id","backup_id") WHERE "native_provider_artifact_references"."backup_id" is not null;--> statement-breakpoint +CREATE UNIQUE INDEX "native_provider_artifacts_company_digest_idx" ON "native_provider_artifacts" USING btree ("company_id","closure_digest") WHERE "native_provider_artifacts"."status" = 'ready'; \ No newline at end of file diff --git a/packages/db/src/migrations/meta/0433_snapshot.json b/packages/db/src/migrations/meta/0438_snapshot.json similarity index 99% rename from packages/db/src/migrations/meta/0433_snapshot.json rename to packages/db/src/migrations/meta/0438_snapshot.json index 6b3488fc699..a2924101be1 100644 --- a/packages/db/src/migrations/meta/0433_snapshot.json +++ b/packages/db/src/migrations/meta/0438_snapshot.json @@ -1,6 +1,6 @@ { - "id": "10b23b32-3586-4a30-a993-618e7d3ca893", - "prevId": "a87278a8-15b3-4187-94a9-f4bb79d8640d", + "id": "69e6941f-890f-43d8-af7c-2e3e24ebfcd5", + "prevId": "2c0cb580-67fc-45d3-9ca5-cce4148dd1a4", "version": "7", "dialect": "postgresql", "tables": { @@ -4058,6 +4058,28 @@ "method": "btree", "with": {} }, + "agent_wakeup_requests_tree_hold_resume_uq": { + "name": "agent_wakeup_requests_tree_hold_resume_uq", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"agent_wakeup_requests\".\"idempotency_key\" LIKE 'tree-hold-resume:%' AND \"agent_wakeup_requests\".\"status\" <> 'skipped'", + "concurrently": true, + "method": "btree", + "with": {} + }, "agent_wakeup_requests_chat_completion_uq": { "name": "agent_wakeup_requests_chat_completion_uq", "columns": [ @@ -28962,6 +28984,22 @@ "method": "btree", "with": {} }, + "environments_managed_plugin_environment_idx": { + "name": "environments_managed_plugin_environment_idx", + "columns": [ + { + "expression": "(\"metadata\" ->> 'managedPluginEnvironment')", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "jsonb_typeof(\"environments\".\"metadata\" -> 'managedPluginEnvironment') = 'string'", + "concurrently": false, + "method": "btree", + "with": {} + }, "environments_name_idx": { "name": "environments_name_idx", "columns": [ @@ -35053,6 +35091,28 @@ "method": "btree", "with": {} }, + "heartbeat_runs_tree_hold_suppression_idx": { + "name": "heartbeat_runs_tree_hold_suppression_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "(\"result_json\"->'treeHoldSuppression'->>'holdId')", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"heartbeat_runs\".\"status\" = 'cancelled' and \"heartbeat_runs\".\"result_json\" ? 'treeHoldSuppression'", + "concurrently": false, + "method": "btree", + "with": {} + }, "heartbeat_runs_cost_accounting_pending_idx": { "name": "heartbeat_runs_cost_accounting_pending_idx", "columns": [ @@ -42492,6 +42552,12 @@ "primaryKey": false, "notNull": false }, + "origin_issue_id": { + "name": "origin_issue_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, "origin_identity_context_id": { "name": "origin_identity_context_id", "type": "uuid", @@ -42935,6 +43001,27 @@ "method": "btree", "with": {} }, + "issues_company_origin_issue_idx": { + "name": "issues_company_origin_issue_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "origin_issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, "issues_company_project_workspace_idx": { "name": "issues_company_project_workspace_idx", "columns": [ @@ -43534,6 +43621,19 @@ "onDelete": "no action", "onUpdate": "no action" }, + "issues_origin_issue_id_issues_id_fk": { + "name": "issues_origin_issue_id_issues_id_fk", + "tableFrom": "issues", + "tableTo": "issues", + "columnsFrom": [ + "origin_issue_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, "issues_execution_workspace_id_execution_workspaces_id_fk": { "name": "issues_execution_workspace_id_execution_workspaces_id_fk", "tableFrom": "issues", @@ -44357,6 +44457,256 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.native_provider_artifact_references": { + "name": "native_provider_artifact_references", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "artifact_id": { + "name": "artifact_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "backup_id": { + "name": "backup_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "native_provider_artifact_references_artifact_idx": { + "name": "native_provider_artifact_references_artifact_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "artifact_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "native_provider_artifact_references_run_idx": { + "name": "native_provider_artifact_references_run_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "artifact_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"native_provider_artifact_references\".\"backup_id\" is null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "native_provider_artifact_references_backup_idx": { + "name": "native_provider_artifact_references_backup_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "artifact_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "backup_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"native_provider_artifact_references\".\"backup_id\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "native_provider_artifact_references_artifact_id_native_provider_artifacts_id_fk": { + "name": "native_provider_artifact_references_artifact_id_native_provider_artifacts_id_fk", + "tableFrom": "native_provider_artifact_references", + "tableTo": "native_provider_artifacts", + "columnsFrom": [ + "artifact_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + }, + "native_provider_artifact_references_backup_id_native_harness_backups_id_fk": { + "name": "native_provider_artifact_references_backup_id_native_harness_backups_id_fk", + "tableFrom": "native_provider_artifact_references", + "tableTo": "native_harness_backups", + "columnsFrom": [ + "backup_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.native_provider_artifacts": { + "name": "native_provider_artifacts", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "closure_digest": { + "name": "closure_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "upload_token": { + "name": "upload_token", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "descriptor": { + "name": "descriptor", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "reference": { + "name": "reference", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "native_provider_artifacts_company_digest_idx": { + "name": "native_provider_artifacts_company_digest_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "closure_digest", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"native_provider_artifacts\".\"status\" = 'ready'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.native_run_finalizations": { "name": "native_run_finalizations", "schema": "", diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index 74008905afd..57b0de3aa33 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -3053,6 +3053,13 @@ "when": 1793982073367, "tag": "0437_workable_oracle", "breakpoints": true + }, + { + "idx": 438, + "version": "7", + "when": 1793996706927, + "tag": "0438_mighty_outlaw_kid", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/db/src/schema/index.ts b/packages/db/src/schema/index.ts index 57d474f5692..21948a1745b 100644 --- a/packages/db/src/schema/index.ts +++ b/packages/db/src/schema/index.ts @@ -278,3 +278,4 @@ export { issueThreadInteractionIdempotencyKeys } from "./issue_thread_interactio export { projectTaskExecutionPolicies, executionAdmissions, executionAdmissionProjectCharges } from "./execution_admissions.js"; export { budgetEvaluationWork } from "./budget_evaluation_work.js"; +export { nativeProviderArtifacts, nativeProviderArtifactReferences } from "./native_provider_artifacts.js"; diff --git a/packages/db/src/schema/native_provider_artifacts.ts b/packages/db/src/schema/native_provider_artifacts.ts new file mode 100644 index 00000000000..8336fe7d394 --- /dev/null +++ b/packages/db/src/schema/native_provider_artifacts.ts @@ -0,0 +1,65 @@ +import { sql } from "drizzle-orm"; +import { + index, + jsonb, + pgTable, + text, + timestamp, + uniqueIndex, + uuid, +} from "drizzle-orm/pg-core"; +import { nativeHarnessBackups } from "./native_harness_backups.js"; + +export const nativeProviderArtifacts = pgTable( + "native_provider_artifacts", + { + id: uuid("id").primaryKey().defaultRandom(), + companyId: uuid("company_id").notNull(), + closureDigest: text("closure_digest").notNull(), + status: text("status").notNull().default("pending"), + uploadToken: uuid("upload_token"), + descriptor: jsonb("descriptor").$type>().notNull(), + reference: jsonb("reference").$type>(), + createdAt: timestamp("created_at", { withTimezone: true }) + .notNull() + .defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true }) + .notNull() + .defaultNow(), + }, + (table) => ({ + digest: uniqueIndex("native_provider_artifacts_company_digest_idx") + .on(table.companyId, table.closureDigest) + .where(sql`${table.status} = 'ready'`), + }), +); + +export const nativeProviderArtifactReferences = pgTable( + "native_provider_artifact_references", + { + id: uuid("id").primaryKey().defaultRandom(), + companyId: uuid("company_id").notNull(), + artifactId: uuid("artifact_id") + .notNull() + .references(() => nativeProviderArtifacts.id, { onDelete: "restrict" }), + runId: uuid("run_id").notNull(), + backupId: uuid("backup_id").references(() => nativeHarnessBackups.id, { + onDelete: "cascade", + }), + createdAt: timestamp("created_at", { withTimezone: true }) + .notNull() + .defaultNow(), + }, + (table) => ({ + artifact: index("native_provider_artifact_references_artifact_idx").on( + table.companyId, + table.artifactId, + ), + run: uniqueIndex("native_provider_artifact_references_run_idx") + .on(table.companyId, table.runId, table.artifactId) + .where(sql`${table.backupId} is null`), + backup: uniqueIndex("native_provider_artifact_references_backup_idx") + .on(table.companyId, table.artifactId, table.backupId) + .where(sql`${table.backupId} is not null`), + }), +); diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs index 78e3b3e85f7..aa464d2c1a7 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs @@ -2493,6 +2493,140 @@ fn retains_opencode_state_when_a_valid_new_proxy_replaces_the_admitted_bytes() { fs::remove_dir_all(directory).unwrap(); } +#[test] +#[cfg_attr(windows, ignore = "verified launch is not supported on Windows yet")] +fn resumes_opencode_after_cold_state_and_artifact_restore_into_a_new_allocation() { + let root = temporary_directory("opencode-cold-artifact-restore"); + let account = root.join("owned-account"); + let old_state = root.join("old-controller-and-lease"); + let new_state = root.join("new-controller-and-lease"); + let old_cwd = + root.join(".paperclip-workspaces/workspace-1-42fd5504-d7d5-4374-a622-b06335b61e61"); + let new_cwd = old_cwd.clone(); + for path in [&account, &old_state, &new_state, &old_cwd, &new_cwd] { + fs::create_dir_all(path).unwrap(); + } + let original_config = opencode_config(&account); + let mut initial_config = original_config.clone(); + initial_config.state_dir = old_state.clone(); + let mut initial_payload = opencode_prepare_payload(&account); + initial_payload["provider"]["cwd"] = json!(old_cwd); + let mut first = NativeProviderCommandExecutor::with_runner_config(&old_state, &initial_config); + first + .execute(&command(1, "run.prepare", initial_payload)) + .unwrap(); + let opened = first + .execute(&command(2, "session.open", json!({}))) + .unwrap(); + first.shutdown().unwrap(); + drop(first); + let state = fs::read(old_state.join("codex-provider-state.json")).unwrap(); + let profile = original_config.opencode_launch_profile.as_ref().unwrap(); + let files = [ + &profile.command.path, + &profile.proxy_script.path, + &profile.executable.path, + ]; + let saved: Vec<_> = files.iter().map(|path| fs::read(path).unwrap()).collect(); + for path in files { + fs::remove_file(path).unwrap(); + } + fs::remove_dir_all(&old_state).unwrap(); + fs::remove_dir_all(&old_cwd).unwrap(); + fs::create_dir_all(&new_cwd).unwrap(); + fs::write(new_state.join("codex-provider-state.json"), &state).unwrap(); + #[cfg(unix)] + fs::set_permissions( + new_state.join("codex-provider-state.json"), + fs::Permissions::from_mode(0o600), + ) + .unwrap(); + fs::write( + new_cwd.join("retained-work.txt"), + "original useful workspace content", + ) + .unwrap(); + let mut recovered_config = original_config.clone(); + recovered_config.state_dir = new_state.clone(); + recovered_config.environment_lease_id = "new-allocation-lease".to_owned(); + let mut payload = opencode_prepare_payload(&account); + payload["provider"]["cwd"] = json!(new_cwd); + let calls_before = fs::read(account.join("fake-opencode-calls.log")).unwrap(); + let missing_state = root.join("rejected-controller-without-artifacts"); + fs::create_dir_all(&missing_state).unwrap(); + fs::write(missing_state.join("codex-provider-state.json"), &state).unwrap(); + #[cfg(unix)] + fs::set_permissions( + missing_state.join("codex-provider-state.json"), + fs::Permissions::from_mode(0o600), + ) + .unwrap(); + let mut missing_config = recovered_config.clone(); + missing_config.state_dir = missing_state.clone(); + let mut missing = + NativeProviderCommandExecutor::with_runner_config(&missing_state, &missing_config); + assert!(missing + .execute(&command(3, "run.prepare", payload.clone())) + .is_err()); + drop(missing); + assert_eq!( + fs::read(new_state.join("codex-provider-state.json")).unwrap(), + state + ); + assert_eq!( + fs::read(account.join("fake-opencode-calls.log")).unwrap(), + calls_before + ); + for (path, bytes) in files.iter().zip(saved) { + fs::write(path, bytes).unwrap(); + #[cfg(unix)] + fs::set_permissions(path, fs::Permissions::from_mode(0o500)).unwrap(); + } + let mut resumed = + NativeProviderCommandExecutor::with_runner_config(&new_state, &recovered_config); + resumed + .execute(&command(4, "run.prepare", payload)) + .unwrap(); + let reopened = resumed + .execute(&command(5, "session.open", json!({}))) + .unwrap(); + assert_eq!(reopened.result["provider"], opened.result["provider"]); + assert_eq!(opencode_call_count(&account, "thread/start"), 1); + assert_eq!(opencode_call_count(&account, "thread/resume"), 1); + assert_eq!(opencode_call_count(&account, "turn/start"), 0); + resumed + .execute(&command( + 6, + "turn.start", + json!({"text":"Deliver useful output from retained work."}), + )) + .unwrap(); + let deadline = std::time::Instant::now() + Duration::from_secs(5); + let mut terminal = false; + while std::time::Instant::now() < deadline { + let events = resumed.poll_events().unwrap(); + terminal |= events + .iter() + .any(|event| event.event_type == "run.terminal"); + resumed.acknowledge_events(events.len()).unwrap(); + if terminal { + break; + } + std::thread::sleep(Duration::from_millis(1)); + } + assert!( + terminal, + "the resumed native provider must finish useful work" + ); + assert_eq!(opencode_call_count(&account, "turn/start"), 1); + assert_eq!( + fs::read_to_string(new_cwd.join("retained-work.txt")).unwrap(), + "original useful workspace content" + ); + resumed.shutdown().unwrap(); + fs::remove_dir_all(root).unwrap(); +} + #[test] fn rejects_pi_without_an_explicit_model_before_starting_a_sidecar() { let directory = temporary_directory("pi-model"); diff --git a/server/scripts/fixtures/opencode-cold-recovery.mjs b/server/scripts/fixtures/opencode-cold-recovery.mjs new file mode 100644 index 00000000000..a9464607735 --- /dev/null +++ b/server/scripts/fixtures/opencode-cold-recovery.mjs @@ -0,0 +1,155 @@ +import { createServer } from "node:http"; +import { once } from "node:events"; +import { mkdir, readFile } from "node:fs/promises"; +const input = JSON.parse(process.argv[2]); +const { createRunnerdCodexTransport } = await import( + input.sdk + "/live/runnerd-codex-transport.js" +); +const seen = []; +const model = createServer(async (req, res) => { + const parts = []; + for await (const p of req) parts.push(p); + if (req.url != "/v1/chat/completions") { + res.writeHead(404).end(); + return; + } + const body = JSON.parse(Buffer.concat(parts).toString() || "{}"); + const context = JSON.stringify(body.messages); + seen.push(context); + const content = + input.phase === "resume" + ? context.includes("retained-context-token") + ? "cold-resume-useful-result" + : "MISSING_RETAINED_CONTEXT" + : "retained-context-token"; + res.writeHead(200, { "Content-Type": "text/event-stream" }); + for (const [delta, finish_reason] of [ + [{ role: "assistant", content }, null], + [{}, "stop"], + ]) + res.write( + "data: " + + JSON.stringify({ + id: "local-native-cold", + object: "chat.completion.chunk", + created: 1, + model: "qualification", + choices: [{ index: 0, delta, finish_reason }], + }) + + "\n\n", + ); + res.end("data: [DONE]\n\n"); +}); +model.listen(0, "127.0.0.1"); +await once(model, "listening"); +const root = input.artifactRoot, + a = input.manifest.payload.artifacts; +await mkdir(input.workspace, { recursive: true }); +const { transport } = createRunnerdCodexTransport({ + provider: "opencode", + runnerBinary: "/qualified-runnerd", + stateDirectory: input.state, + sourceCodexHome: "", + providerNodeCommand: root + "/" + a.nodeCommand.path, + providerNodeCommandSha256: a.nodeCommand.sha256, + opencodeCommand: root + "/" + a.opencodeExecutable.path, + opencodeCommandSha256: a.opencodeExecutable.sha256, + opencodeProxyPath: root + "/" + a.opencodeProxy.path, + opencodeProxySha256: a.opencodeProxy.sha256, + opencodeRuntimeDirectory: input.runtime, + prpIdentity: input.identity, + lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60000 }, + ...(input.thread + ? { + resumeProviderSession: { + driverSessionId: input.thread.id, + providerSessionId: input.thread.sessionId, + }, + } + : {}), + environment: { + PATH: "/usr/local/bin:/usr/bin:/bin", + HOME: input.home ?? "/root", + PAPERCLIP_AI_PROVIDER_URL: + "http://127.0.0.1:" + model.address().port + "/v1", + PAPERCLIP_AI_PROVIDER_KEY: "local-fixture-only", + OPENCODE_DISABLE_MODELS_FETCH: "true", + OPENCODE_DISABLE_DEFAULT_PLUGINS: "true", + }, +}); +const text = []; +let timer; +try { + const opened = await transport.request( + input.thread ? "thread/read" : "thread/start", + input.thread + ? {} + : { + cwd: input.workspace, + model: "paperclip/qualification", + dynamicTools: [], + completionContract: { + revision: "cold-closure-v1", + criterionIds: ["useful-output"], + }, + }, + ); + if (input.thread && opened.thread.sessionId !== input.thread.sessionId) + throw Error("provider session changed"); + if (input.phase !== "init") { + const completed = (async () => { + for await (const n of transport.notifications()) { + if (n.method === "item/agentMessage/delta") + text.push(n.params.delta ?? ""); + if (n.method === "turn/completed") return; + if (n.method === "turn/failed") throw Error("provider turn failed"); + } + throw Error("notifications ended"); + })(); + await transport.request("turn/start", { + input: [ + { + type: "text", + text: + input.phase === "resume" + ? "Return the earlier retained token as a useful cold-resume result." + : "Remember retained-context-token for the next turn.", + }, + ], + }); + await Promise.race([ + completed, + new Promise((_, reject) => { + timer = setTimeout( + () => reject(Error("native cold qualification deadline")), + 45000, + ); + }), + ]); + if (seen.length !== 1) + throw Error("expected exactly one synthetic model request"); + if ( + input.phase === "resume" && + !text.join("").includes("cold-resume-useful-result") + ) + throw Error("retained context not delivered"); + } + await transport.close(); + console.log( + JSON.stringify({ + phase: input.phase, + status: "passed", + thread: opened.thread, + text: text.join(""), + modelRequests: seen.length, + lifecycle: JSON.parse( + await readFile(input.state + "/runner/runner-state.json", "utf8"), + ).lifecycle, + }), + ); +} finally { + if (timer) clearTimeout(timer); + await transport.close().catch(() => {}); + model.closeAllConnections(); + await new Promise((r) => model.close(r)); +} diff --git a/server/scripts/qualify-opencode-cold-recovery.mts b/server/scripts/qualify-opencode-cold-recovery.mts new file mode 100644 index 00000000000..389b4dca1fb --- /dev/null +++ b/server/scripts/qualify-opencode-cold-recovery.mts @@ -0,0 +1,846 @@ +import { NativeExecutionContext } from "../src/services/native-runtime/native-execution-context.js"; +import { captureRemoteNativeProviderArtifacts } from "../src/services/native-runtime/native-provider-artifact-capture.js"; +import { execFile } from "node:child_process"; +import { promisify, parseArgs } from "node:util"; +import { fileURLToPath } from "node:url"; +import { createHash, randomBytes, randomUUID } from "node:crypto"; +import { + cp, + mkdir, + mkdtemp, + readFile, + writeFile, + chmod, + rm, + copyFile, +} from "node:fs/promises"; +import { tmpdir, userInfo } from "node:os"; +import { join, dirname, relative } from "node:path"; +import * as t from "../../packages/db/src/index.js"; +import { eq } from "drizzle-orm"; +import { startRecoveryPostgres } from "../src/__tests__/helpers/recovery-postgres.js"; +import { prepareRemoteOpenCodeArtifacts } from "../src/services/native-runtime/remote-opencode-artifacts.js"; +import { + retainNativeProviderArtifacts, + retainExistingNativeProviderArtifacts, +} from "../src/services/native-runtime/native-provider-artifacts.js"; +import { createStorageService } from "../src/storage/service.js"; +import { createLocalDiskStorageProvider } from "../src/storage/local-disk-provider.js"; +import { + reserveRemoteWorkspace, + prepareRemoteWorkspace, + saveRemoteWorkspace, + clearRemoteWorkspaceLocationForReleasedLease, +} from "../src/services/remote-workspaces.js"; +import { remoteTerminationReceipt } from "../src/services/remote-execution-termination.js"; +import { + nativeHarnessBackupOwner, + persistNativeHarnessBackup, + restoreDurableNativeHarnessBackup, +} from "../src/services/native-runtime/native-harness-backup-storage.js"; +import { + buildNativeHarnessBackupManifest, + resolveNativeHarnessPersistenceProfile, + providerSessionIdentityFromDurableProviderState, +} from "../src/services/native-runtime/native-session-executor.js"; +import { + nativeRunnerCheckpointWriter, + restoreNativeRunnerCheckpoint, +} from "../src/services/native-runtime/native-runner-checkpoint.js"; +import { + nativeProviderPackAdmissionPending, + rememberAdmittedNativeProviderPack, + recordNativeProviderPackAdmission, + rememberRecoveredNativeProviderPack, +} from "../src/services/native-runtime/native-provider-pack-admission.js"; +import { captureLegacyDurablePrpCheckpoint } from "../../packages/paperclip-runner/src/control-plane/durable-prp-control-plane.js"; + +const exec = promisify(execFile), + repo = fileURLToPath(new URL("../../", import.meta.url)); +const { values } = parseArgs({ + options: { + "old-image": { type: "string" }, + "new-image": { type: "string" }, + context: { type: "string", default: "desktop-linux" }, + output: { type: "string" }, + }, +}); +if (!values["old-image"] || !values["new-image"] || !values.output) + throw Error( + "Required: --old-image sha256:<64 hex> --new-image sha256:<64 hex> --output ", + ); +if (process.env.PAPERCLIP_TEST_DATABASE_URL) + throw Error( + "Qualification requires its own local embedded PostgreSQL server", + ); +const images = { old: values["old-image"], next: values["new-image"] }; +for (const image of Object.values(images)) + if (!/^sha256:[a-f0-9]{64}$/.test(image)) + throw Error("Only exact cached image IDs are supported"); +if (!values.output.startsWith("/")) throw Error("Output must be absolute"); +const context = values.context ?? "desktop-linux"; +const docker = async (args: string[]) => { + const result = await exec("docker", ["--context", context, ...args], { + maxBuffer: 8 * 1024 * 1024, + timeout: 180_000, + }); + return result.stdout.trim(); +}; +const contextMetadata = JSON.parse( + await docker(["context", "inspect", context]), +); +if (!contextMetadata[0]?.Endpoints?.docker?.Host?.startsWith("unix://")) + throw Error("Qualification requires a local Unix Docker endpoint"); +const metadata = {} as Record< + string, + { Id: string; Architecture: string; Os: string } +>; +for (const [name, image] of Object.entries(images)) { + const [details] = JSON.parse(await docker(["image", "inspect", image])); + if ( + details.Id !== image || + details.Os !== "linux" || + !["amd64", "arm64"].includes(details.Architecture) + ) + throw Error("Cached Linux image identity mismatch"); + metadata[name] = { + Id: details.Id, + Architecture: details.Architecture, + Os: details.Os, + }; +} +if (metadata.old.Architecture !== metadata.next.Architecture) + throw Error("Images must have matching architectures"); +const root = await mkdtemp(join(tmpdir(), "paperclip-native-cold-full-")); +process.env.PAPERCLIP_HOME = join(root, "fixture-home"); +process.env.PAPERCLIP_SECRETS_MASTER_KEY = randomBytes(32).toString("hex"); +const home = "/home/native-fixture", + base = home + "/paperclip-workspace", + account = userInfo(); +const passwd = join(root, "passwd"); +await writeFile( + passwd, + account.uid === 0 + ? `root:x:0:0:Owned fixture:${home}:/bin/sh\n` + : `root:x:0:0:root:/root:/bin/sh\nnative-fixture:x:${account.uid}:${account.gid}:Owned fixture:${home}:/bin/sh\n`, +); +const extract = async (image: string, source: string, destination: string) => { + const id = await docker([ + "create", + "--pull=never", + "--entrypoint", + "node", + image, + "--version", + ]); + try { + await docker(["cp", `${id}:${source}`, destination]); + } finally { + await docker(["rm", id]); + } +}; +await extract( + images.old, + "/opt/paperclip-runner/provider-pack/provider-pack.json", + join(root, "old-provider-pack.json"), +); +await extract( + images.next, + "/opt/paperclip-runner/provider-pack/provider-pack.json", + join(root, "new-provider-pack.json"), +); +await extract( + images.next, + "/app/server/dist/vendor/paperclip-runner/bin/paperclip-runnerd", + join(root, "paperclip-runnerd"), +); +const manifest = JSON.parse( + await readFile(join(root, "old-provider-pack.json"), "utf8"), +); +const nextManifest = JSON.parse( + await readFile(join(root, "new-provider-pack.json"), "utf8"), +); +if ( + manifest.payload.artifacts.opencodeProxy.sha256 === + nextManifest.payload.artifacts.opencodeProxy.sha256 +) + throw Error( + "Qualification requires an actual upgraded proxy with different bytes", + ); +const source = { + revision: ( + await exec("git", ["rev-parse", "HEAD"], { cwd: repo }) + ).stdout.trim(), + dirty: Boolean( + (await exec("git", ["status", "--porcelain"], { cwd: repo })).stdout.trim(), + ), + node: process.version, +}; +const containers = new Map(); +const create = async (first: boolean) => { + const path = join(root, first ? "original-account" : "replacement-account"); + await mkdir(join(path, "paperclip-workspace"), { recursive: true }); + await chmod(path, 0o700); + const id = await docker([ + "create", + "--pull=never", + "--platform", + "linux/" + metadata.old.Architecture, + "--network", + "none", + "--read-only", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges", + "--user", + `${account.uid}:${account.gid}`, + "--cpus", + "2", + "--memory", + "2g", + "--pids-limit", + "256", + "--tmpfs", + "/tmp:rw,size=536870912,mode=1777", + "--mount", + `type=bind,src=${repo},dst=${repo},readonly`, + "--mount", + `type=bind,src=${path},dst=${home}`, + "--mount", + `type=bind,src=${passwd},dst=/etc/passwd,readonly`, + "--mount", + `type=bind,src=${join(root, "paperclip-runnerd")},dst=/qualified-runnerd,readonly`, + "--mount", + `type=bind,src=${fileURLToPath(new URL("./fixtures/opencode-cold-recovery.mjs", import.meta.url))},dst=/native-probe.mjs,readonly`, + "--env", + `HOME=${home}`, + "--entrypoint", + "node", + first ? images.old : images.next, + "-e", + "setTimeout(()=>{},600000)", + ]); + containers.set(id, path); + await docker(["start", id]); + return { id, path }; +}; +const mapped = (path: string, file: string) => { + if (!file.startsWith(home + "/")) throw Error("unowned remote path"); + return join(path, relative(home, file)); +}; +const runner = (id: string, path: string) => { + const transfer = (out: boolean) => async (ops: any[]) => { + const results = []; + for (const o of ops) { + for (const f of o.files) { + const src = out ? mapped(path, f.sourcePath) : f.sourcePath, + dst = out ? f.targetPath : mapped(path, f.targetPath); + await mkdir(dirname(dst), { recursive: true }); + await cp(src, dst, { recursive: f.kind === "directory" }); + if (f.mode) await chmod(dst, f.mode); + } + results.push({ + operationId: o.operationId, + filesTransferred: o.files.length, + bytesTransferred: 0, + }); + } + return { operations: results }; + }; + return { + execute: async (input: any) => { + try { + const args = [ + "--context", + context, + "exec", + "--workdir", + input.cwd ?? base, + ]; + for (const [k, v] of Object.entries(input.env ?? {})) + args.push("--env", `${k}=${v}`); + const r = await exec( + "docker", + [...args, id, input.command, ...(input.args ?? [])], + { maxBuffer: 8 * 1024 * 1024, timeout: input.timeoutMs ?? 180000 }, + ); + return { + exitCode: 0, + stdout: r.stdout, + stderr: r.stderr, + signal: null, + timedOut: false, + }; + } catch (e: any) { + return { + exitCode: typeof e.code === "number" ? e.code : 1, + stdout: e.stdout ?? "", + stderr: e.stderr ?? "", + signal: null, + timedOut: false, + }; + } + }, + syncIn: transfer(false), + syncOut: transfer(true), + }; +}; +const stage = (path: string) => async (f: any) => { + await copyFile(f.sourcePath, mapped(path, f.targetPath), 1); + await chmod(mapped(path, f.targetPath), f.mode); +}; +const ids = { + company: randomUUID(), + agent: randomUUID(), + issue: randomUUID(), + run: randomUUID(), + environment: randomUUID(), + lease: randomUUID(), + boot: randomUUID(), +}; +const identity = { + runnerInstanceId: randomUUID(), + environmentLeaseId: ids.lease, + runId: ids.run, + normalizedSessionId: randomUUID(), + turnId: randomUUID(), + itemId: randomUUID(), +}; +const binding = { + companyId: ids.company, + agentId: ids.agent, + issueId: ids.issue, + runId: ids.run, + }, + scopeDigest = createHash("sha256") + .update(JSON.stringify(identity)) + .digest("hex"); +const database = await startRecoveryPostgres("opencode-native-cold-"); +process.env.DATABASE_URL = database.connectionString; +const db = t.createDb(database.connectionString); +const disk = createStorageService( + createLocalDiskStorageProvider(join(root, "objects")), + ), + uploads: any[] = []; +const storage = { + ...disk, + provider: "s3" as const, + putFile: async (input: any) => { + uploads.push({ namespace: input.namespace, byteSize: input.byteSize }); + return disk.putFile(input); + }, +}; +const claim = { + leaseOwner: "cold-owner", + attempt: 1, + controllerGeneration: 1, + controllerBootId: ids.boot, +}; +const facade = { + identity: () => ({ ...binding, sessionId: identity.normalizedSessionId }), +} as any; +let execution: any, workspace: string, runtime: string; +const probe = async (id: string, input: any) => { + const out = await docker([ + "exec", + id, + "node", + "/native-probe.mjs", + JSON.stringify({ + sdk: repo + "/packages/paperclip-runner/dist", + manifest, + workspace, + runtime, + identity, + home, + ...input, + }), + ]); + const line = out + .split("\n") + .filter((l) => l.startsWith("{")) + .at(-1); + if (!line) throw Error("missing probe receipt"); + await writeFile(join(root, `phase-${input.phase}.json`), line); + return JSON.parse(line); +}; +try { + await db.insert(t.companies).values({ + id: ids.company, + name: "Owned cold recovery qualification", + issuePrefix: "COLD", + }); + await db.insert(t.agents).values({ + id: ids.agent, + companyId: ids.company, + name: "Owned native worker", + adapterType: "paperclip_runner", + }); + await db.insert(t.environments).values({ + id: ids.environment, + name: ids.environment, + driver: "sandbox", + config: {}, + }); + await db.insert(t.issues).values({ + id: ids.issue, + companyId: ids.company, + title: "Preserve useful cold work", + status: "in_progress", + assigneeAgentId: ids.agent, + }); + const epoch = randomUUID(); + await db.insert(t.heartbeatRuns).values({ + id: ids.run, + companyId: ids.company, + agentId: ids.agent, + nativeIssueId: ids.issue, + runtimeMode: "native", + status: "running", + driverKind: "opencode_server", + nativeSessionId: identity.normalizedSessionId, + runnerInstanceId: identity.runnerInstanceId, + runnerProfileJson: { + nativeProviderPackAdmissionState: + nativeProviderPackAdmissionPending(epoch), + }, + }); + await db + .update(t.issues) + .set({ executionRunId: ids.run }) + .where(eq(t.issues.id, ids.issue)); + await db.insert(t.nativeRunFinalizations).values({ + runId: ids.run, + companyId: ids.company, + issueId: ids.issue, + phase: "observed", + attempt: 1, + controllerGeneration: 1, + controllerBootId: ids.boot, + leaseOwner: claim.leaseOwner, + leaseExpiresAt: new Date(Date.now() + 900000), + }); + await db.insert(t.environmentLeases).values({ + id: ids.lease, + companyId: ids.company, + environmentId: ids.environment, + heartbeatRunId: ids.run, + issueId: ids.issue, + provider: "daytona", + providerLeaseId: "allocation-" + ids.lease, + status: "active", + }); + const lease = async (id: string) => + ( + await db + .select() + .from(t.environmentLeases) + .where(eq(t.environmentLeases.id, id)) + )[0] as any; + const original = await create(true), + oldRunner = runner(original.id, original.path); + const target = (a: any, id: string) => + ({ + kind: "remote", + transport: "sandbox", + remoteCwd: base, + runner: runner(a.id, a.path), + provider: "daytona", + leaseId: id, + sandboxId: "allocation-" + id, + timeoutMs: 180000, + }) as any; + const oldTarget = target(original, ids.lease); + const reserve = (existingWorkspaceId: string | null) => + reserveRemoteWorkspace({ + db, + storage, + ...binding, + environmentId: ids.environment, + projectId: null, + projectWorkspaceId: null, + existingWorkspaceId, + source: { kind: "empty" }, + }); + const reserved = await reserve(null), + prepared = await prepareRemoteWorkspace({ + db, + storage, + workspace: reserved, + runId: ids.run, + lease: await lease(ids.lease), + target: oldTarget, + }); + workspace = prepared.remoteRoot; + runtime = + workspace + + "/.paperclip-runtime/paperclip-runner/sessions/retained/filesystem/opencode"; + execution = { + binding: { ...binding, executionWorkspaceId: reserved.id }, + provider: { kind: "opencode", model: "paperclip/qualification" }, + workspace: { + cwd: workspace, + repoUrl: null, + repoRef: null, + branchName: null, + }, + session: { + normalizedSessionId: identity.normalizedSessionId, + driverKind: "opencode_server", + protocolVersion: 1, + lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60000 }, + }, + }; + const launch = await prepareRemoteOpenCodeArtifacts({ + runner: oldRunner as never, + bootstrapNode: "/usr/local/bin/node", + manifest, + nonce: randomUUID(), + sourceRoot: "/opt/paperclip-runner/provider-pack", + stageFile: stage(original.path), + }); + const originalState = home + "/controller-original", + initialized = await probe(original.id, { + phase: "init", + state: originalState, + artifactRoot: launch.root, + }); + if ( + !rememberAdmittedNativeProviderPack(facade, { + identity: facade.identity(), + runnerInstanceId: identity.runnerInstanceId, + environmentLeaseId: identity.environmentLeaseId, + manifest, + }) + ) + throw Error("admission unavailable"); + if ( + !(await recordNativeProviderPackAdmission(db, { + session: facade, + identity: facade.identity(), + admissionEpoch: epoch, + claim, + })) + ) + throw Error("admission not recorded"); + const owner = await nativeHarnessBackupOwner(db, { + ...binding, + bootId: ids.boot, + }); + const captureStartedAt = performance.now(); + const captureContext = new NativeExecutionContext(execution, 300_000); + let receipt: Awaited>; + try { + receipt = await captureRemoteNativeProviderArtifacts( + { + runner: oldRunner as never, + launch, + manifest, + signal: captureContext.signal, + context: captureContext, + }, + (source) => + retainNativeProviderArtifacts({ + db, + storage, + owner, + source, + signal: captureContext.signal, + }), + ); + } finally { + captureContext.finish(); + } + const artifactCaptureMs = Math.round(performance.now() - captureStartedAt); + const first = await probe(original.id, { + phase: "first", + state: originalState, + artifactRoot: launch.root, + thread: initialized.thread, + }); + if (first.lifecycle !== "suspended") + throw Error("source runner not suspended"); + await writeFile( + mapped(original.path, workspace + "/useful-result.txt"), + "retained useful workspace result", + ); + const saved = await saveRemoteWorkspace({ + db, + storage, + workspace: prepared.workspace, + runId: ids.run, + lease: prepared.lease, + target: oldTarget, + assertOwned: async () => {}, + }); + const checkpoint = captureLegacyDurablePrpCheckpoint( + await readFile( + mapped( + original.path, + originalState + "/control-plane/control-plane-state.json", + ), + "utf8", + ), + { + runId: ids.run, + runnerInstanceId: identity.runnerInstanceId, + normalizedSessionId: identity.normalizedSessionId, + }, + ); + await nativeRunnerCheckpointWriter(db, { + ...owner, + leaseOwner: claim.leaseOwner, + })(checkpoint); + const nativeRoot = join(root, "old-controller-backup"); + await mkdir(nativeRoot); + for (const d of resolveNativeHarnessPersistenceProfile(execution) + .directories) { + const src = mapped( + original.path, + d.name === "runner" ? originalState + "/runner" : runtime, + ); + await cp(src, join(nativeRoot, d.name), { + recursive: true, + filter: (path) => + !(d.excludeEntries ?? []).some( + (e) => + relative(src, path) === e || + relative(src, path).startsWith(e + "/"), + ), + }); + } + const providerState = JSON.parse( + await readFile( + join(nativeRoot, "runner/codex-provider-state.json"), + "utf8", + ), + ); + const nativeManifest = buildNativeHarnessBackupManifest({ + backupRoot: nativeRoot, + execution, + runnerInstanceId: identity.runnerInstanceId, + providerSessionIdentity: providerSessionIdentityFromDurableProviderState({ + execution, + providerState, + }), + sourceProviderLeaseId: prepared.lease.providerLeaseId, + }); + nativeManifest.providerArtifacts = receipt; + await writeFile( + join(nativeRoot, "manifest.json"), + JSON.stringify(nativeManifest), + ); + await persistNativeHarnessBackup({ + db, + storage, + owner, + scopeDigest, + root: nativeRoot, + signal: AbortSignal.timeout(180000), + }); + const [recorded] = await db + .select() + .from(t.heartbeatRuns) + .where(eq(t.heartbeatRuns.id, ids.run)); + await docker(["stop", "--time", "5", original.id]); + await docker(["rm", original.id]); + containers.delete(original.id); + const stopped = { + ...prepared.lease, + status: "released", + releasedAt: new Date(), + cleanupStatus: "success", + metadata: { + ...prepared.lease.metadata, + remoteExecutionTermination: remoteTerminationReceipt(prepared.lease, { + providerLeaseId: prepared.lease.providerLeaseId, + state: "destroyed", + }), + }, + } as any; + await db + .update(t.environmentLeases) + .set({ + status: stopped.status, + releasedAt: stopped.releasedAt, + cleanupStatus: "success", + metadata: stopped.metadata, + }) + .where(eq(t.environmentLeases.id, ids.lease)); + await db.transaction((tx) => + clearRemoteWorkspaceLocationForReleasedLease({ db: tx, lease: stopped }), + ); + await rm(original.path, { recursive: true, force: true }); + await rm(nativeRoot, { recursive: true, force: true }); + const replacement = await create(false), + nextLease = randomUUID(); + await db.insert(t.environmentLeases).values({ + id: nextLease, + companyId: ids.company, + environmentId: ids.environment, + heartbeatRunId: ids.run, + issueId: ids.issue, + provider: "daytona", + providerLeaseId: "allocation-" + nextLease, + status: "active", + }); + const restoredWorkspace = await prepareRemoteWorkspace({ + db, + storage, + workspace: await reserve(reserved.id), + runId: ids.run, + lease: await lease(nextLease), + target: target(replacement, nextLease), + }); + if (restoredWorkspace.remoteRoot !== workspace) + throw Error("published cwd changed"); + if ( + (await readFile( + mapped(replacement.path, workspace + "/useful-result.txt"), + "utf8", + )) !== "retained useful workspace result" + ) + throw Error("workspace file lost"); + const restored = join(root, "new-controller"); + await mkdir(restored, { mode: 0o700 }); + await restoreNativeRunnerCheckpoint( + db, + binding, + restored, + identity.normalizedSessionId, + ); + if ( + !(await restoreDurableNativeHarnessBackup({ + db, + storage, + companyId: ids.company, + scopeDigest, + backupRoot: join(restored, "failover-backups"), + signal: AbortSignal.timeout(180000), + })) + ) + throw Error("durable harness unavailable"); + const current = join(restored, "failover-backups/current"), + nextState = home + "/controller-replacement"; + await mkdir(mapped(replacement.path, nextState), { + recursive: true, + mode: 0o700, + }); + await cp( + join(restored, "control-plane"), + mapped(replacement.path, nextState + "/control-plane"), + { recursive: true }, + ); + await cp( + join(current, "runner"), + mapped(replacement.path, nextState + "/runner"), + { recursive: true }, + ); + await mkdir(dirname(mapped(replacement.path, runtime)), { recursive: true }); + await cp(join(current, "opencode"), mapped(replacement.path, runtime), { + recursive: true, + }); + const recovered = await prepareRemoteOpenCodeArtifacts({ + runner: runner(replacement.id, replacement.path) as never, + bootstrapNode: "/usr/local/bin/node", + manifest, + nonce: randomUUID(), + restored: { receipt, directory: join(current, "provider-artifacts") }, + stageFile: stage(replacement.path), + }); + const nextEpoch = randomUUID(), + nextBoot = randomUUID(); + await db + .update(t.nativeRunFinalizations) + .set({ + controllerBootId: nextBoot, + controllerGeneration: 2, + leaseOwner: "cold-successor", + leaseExpiresAt: new Date(Date.now() + 900000), + }) + .where(eq(t.nativeRunFinalizations.runId, ids.run)); + await db + .update(t.heartbeatRuns) + .set({ + runnerProfileJson: { + ...recorded.runnerProfileJson, + nativeProviderPackAdmissionState: + nativeProviderPackAdmissionPending(nextEpoch), + }, + }) + .where(eq(t.heartbeatRuns.id, ids.run)); + if ( + !(await rememberRecoveredNativeProviderPack(db, facade, { + identity: facade.identity(), + sourceRunId: ids.run, + runnerInstanceId: identity.runnerInstanceId, + environmentLeaseId: identity.environmentLeaseId, + source: { + runId: ids.run, + receipt: recorded.runnerProfileJson?.nativeProviderPackAdmission, + state: recorded.runnerProfileJson?.nativeProviderPackAdmissionState, + }, + })) + ) + throw Error("recovered admission unavailable"); + if ( + !(await recordNativeProviderPackAdmission(db, { + session: facade, + identity: facade.identity(), + admissionEpoch: nextEpoch, + claim: { + ...claim, + leaseOwner: "cold-successor", + controllerGeneration: 2, + controllerBootId: nextBoot, + }, + })) + ) + throw Error("recovered admission not recorded"); + await retainExistingNativeProviderArtifacts({ + db, + owner: await nativeHarnessBackupOwner(db, { ...binding, bootId: nextBoot }), + receipt, + }); + const resumed = await probe(replacement.id, { + phase: "resume", + state: nextState, + artifactRoot: recovered.root, + thread: first.thread, + }); + const result = { + root, + status: "passed", + images, + metadata, + source, + oldPack: manifest.digest, + newPack: nextManifest.digest, + first, + resumed, + sameLaunchPaths: + JSON.stringify(launch.paths) === JSON.stringify(recovered.paths), + canonicalCwd: workspace, + oldLease: ids.lease, + newLease: nextLease, + workspaceSnapshot: saved.snapshotId, + artifactCaptureMs, + uploads, + controllerCheckpoint: "encrypted real PostgreSQL reload", + harness: "AES authenticated object reload", + network: "none; local synthetic model only", + }; + await writeFile(values.output, JSON.stringify(result, null, 2)); + console.log(JSON.stringify(result)); +} finally { + for (const [id] of containers) { + await docker(["stop", "--time", "5", id]).catch(() => {}); + await docker(["rm", id]).catch(() => {}); + } + await t.closeRegisteredClients(database.connectionString, { + scope: "database", + }); + await database.cleanup(); + console.error("Owned qualification files retained at " + root); +} diff --git a/server/src/services/native-runtime/native-harness-backup-archive.ts b/server/src/services/native-runtime/native-harness-backup-archive.ts index b36d7ab07b4..86821b8f4dd 100644 --- a/server/src/services/native-runtime/native-harness-backup-archive.ts +++ b/server/src/services/native-runtime/native-harness-backup-archive.ts @@ -44,7 +44,9 @@ const boundedArchive = (limit = MAX_NATIVE_HARNESS_OBJECT_BYTES) => { } }); }; -export async function captureNativeHarnessArchive(root: string, target: string, signal?: AbortSignal, maxArchiveBytes = MAX_NATIVE_HARNESS_OBJECT_BYTES) { +export async function captureNativeHarnessArchive(root: string, target: string, signal?: AbortSignal, maxArchiveBytes = MAX_NATIVE_HARNESS_OBJECT_BYTES, + excludeRootEntries: readonly string[] = []) { + if (excludeRootEntries.some(entry => entry !== "provider-artifacts")) throw fail(); if (!Number.isSafeInteger(maxArchiveBytes) || maxArchiveBytes <= 0 || maxArchiveBytes > MAX_NATIVE_HARNESS_OBJECT_BYTES) throw fail(); if (!(await readNativeHarnessBackupSourceInput(() => lstat(root))).isDirectory()) throw fail(); const archive = tar.pack(); @@ -55,6 +57,7 @@ export async function captureNativeHarnessArchive(root: string, target: string, const links = new Map(); const visit = async (relative: string): Promise => { for (const name of (await readNativeHarnessBackupSourceInput(() => readdir(join(root, relative)))).sort()) { + if (!relative && excludeRootEntries.includes(name)) continue; signal?.throwIfAborted(); const entry = relative ? `${relative}/${name}` : name; if (!safeName(entry) || ++count > MAX_NATIVE_HARNESS_ENTRIES) throw fail(); diff --git a/server/src/services/native-runtime/native-harness-backup-expiration.ts b/server/src/services/native-runtime/native-harness-backup-expiration.ts index c326d232098..c51bcbd30ff 100644 --- a/server/src/services/native-runtime/native-harness-backup-expiration.ts +++ b/server/src/services/native-runtime/native-harness-backup-expiration.ts @@ -1,3 +1,4 @@ +import { readNativeCheckpointRetirement, nativeCheckpointRetirement, MAX_NATIVE_CHECKPOINT_RETIREMENT_RUNS } from "./native-runner-checkpoint-retirement.js"; import { notifyExternalConversationSettlement } from "../external-conversation-settlement-notifications.js"; import { randomUUID } from "node:crypto"; import { promises as fs } from "node:fs"; @@ -11,6 +12,7 @@ import { nativeRunFinalizations, nativeRunnerCheckpoints, nativeHarnessBackups, + nativeProviderArtifactReferences, type Db, } from "@paperclipai/db"; import { logger } from "../../middleware/logger.js"; @@ -22,6 +24,7 @@ import type { NativeHarnessBackupExpirationScope } from "../native-harness-backu import { deleteRetiredNativeHarnessBackup, durableNativeHarnessStorage } from "./native-harness-backup-storage.js"; import { retireObsoleteNativeHarnessBackup } from "./native-harness-backup-retention.js"; import type { StorageService } from "../../storage/types.js"; +import { releaseExpiredNativeProviderArtifactRun } from "./native-provider-artifacts.js"; export const NATIVE_HARNESS_BACKUP_EXPIRATION_INTERVAL_MS = 60 * 60 * 1000; export const NATIVE_HARNESS_BACKUP_EXPIRATION_STARTUP_DELAY_MS = 2 * 60 * 1000; @@ -369,7 +372,8 @@ export function nativeHarnessBackupExpirationService( }, result: NativeHarnessBackupExpirationResult) { // The partial header index enumerates retained checkpoints without reading // event payloads or scanning runs that have no checkpoint left to expire. - const candidates = await db.select({ runId: heartbeatRuns.id, sessionId: heartbeatRuns.nativeSessionId }) + const candidates = await db.select({ runId: heartbeatRuns.id, companyId: heartbeatRuns.companyId, sessionId: heartbeatRuns.nativeSessionId, + retirement: sql | null>`case when ${nativeRunnerCheckpoints.material}->>'schema' like 'paperclip.native-runner-checkpoint-retirement.%' then ${nativeRunnerCheckpoints.material} else null end` }) .from(nativeRunnerCheckpoints) .innerJoin(nativeRunFinalizations, eq(nativeRunFinalizations.runId, nativeRunnerCheckpoints.runId)) .innerJoin(heartbeatRuns, eq(heartbeatRuns.id, nativeRunFinalizations.runId)) @@ -392,12 +396,14 @@ export function nativeHarnessBackupExpirationService( checkpointCursor = candidate.runId; if (!candidate.sessionId) continue; // Unknown identity is never expiration evidence. try { - const expired = await db.transaction(async tx => { + const previousRetirement = readNativeCheckpointRetirement(candidate.retirement, candidate); + if (previousRetirement && previousRetirement.sessionId !== candidate.sessionId) throw new Error("native_runner_checkpoint_retirement_invalid"); + const retirement = previousRetirement ?? await db.transaction(async tx => { const blocker = await sessionRetentionBlocker(tx, { scopeDigest: null, checkpoint: true, closedBefore: options.closedBefore, identity: { runId: candidate.runId, normalizedSessionId: candidate.sessionId!, environmentLeaseId: null }, }); - if ("reason" in blocker) return false; + if ("reason" in blocker || blocker.runIds.length > MAX_NATIVE_CHECKPOINT_RETIREMENT_RUNS) return false; const [resume] = await tx.select({ id: agentTaskSessions.id }).from(agentTaskSessions).where(and( eq(agentTaskSessions.companyId, blocker.companyId), eq(agentTaskSessions.agentId, blocker.agentId), or(inArray(agentTaskSessions.lastRunId, blocker.runIds), @@ -411,11 +417,36 @@ export function nativeHarnessBackupExpirationService( await tx.update(nativeRunFinalizations).set({ controllerGeneration: sql`${nativeRunFinalizations.controllerGeneration} + 1`, }).where(inArray(nativeRunFinalizations.runId, blocker.runIds)); - const rows = await tx.delete(nativeRunnerCheckpoints).where(and( - eq(nativeRunnerCheckpoints.companyId, blocker.companyId), eq(nativeRunnerCheckpoints.runId, candidate.runId), - )).returning({ part: nativeRunnerCheckpoints.part }); - return rows.length > 0; + const retired = nativeCheckpointRetirement({ companyId: blocker.companyId, agentId: blocker.agentId, + sessionId: candidate.sessionId!, runIds: blocker.runIds }); + await tx.update(nativeRunnerCheckpoints).set({ material: retired, updatedAt: new Date() }).where(and( + eq(nativeRunnerCheckpoints.companyId, blocker.companyId), inArray(nativeRunnerCheckpoints.runId, blocker.runIds), + eq(nativeRunnerCheckpoints.part, "header"))); + return retired; }); + let expired = false; + if (retirement && !options.dryRun) { + const storage = defaults.storage ?? durableNativeHarnessStorage(); + if (!storage) { + const [pending] = await db.select({ id: nativeProviderArtifactReferences.id }).from(nativeProviderArtifactReferences).where(and( + eq(nativeProviderArtifactReferences.companyId, retirement.companyId), inArray(nativeProviderArtifactReferences.runId, retirement.runIds), + isNull(nativeProviderArtifactReferences.backupId))).limit(1); + if (pending) throw new Error("native_provider_artifacts_storage_unavailable"); + } + if (storage) for (const runId of retirement.runIds) await releaseExpiredNativeProviderArtifactRun({ db, storage, + companyId: retirement.companyId, runId, signal: AbortSignal.timeout(60_000) }); + expired = !options.dryRun && await db.transaction(async tx => { + const [header] = await tx.select().from(nativeRunnerCheckpoints).where(and( + eq(nativeRunnerCheckpoints.companyId, retirement.companyId), eq(nativeRunnerCheckpoints.runId, candidate.runId), + eq(nativeRunnerCheckpoints.part, "header"))).for("update"); + const current = header && readNativeCheckpointRetirement(header.material, candidate); + if (!current || current.retiredAt !== retirement.retiredAt || current.agentId !== retirement.agentId + || current.sessionId !== retirement.sessionId || current.runIds.join(",") !== retirement.runIds.join(",")) return false; + const rows = await tx.delete(nativeRunnerCheckpoints).where(and(eq(nativeRunnerCheckpoints.companyId, retirement.companyId), + eq(nativeRunnerCheckpoints.runId, candidate.runId))).returning({ part: nativeRunnerCheckpoints.part }); + return rows.length > 0; + }); + } if (expired) { removed++; result.expiredCheckpoints++; } } catch (err) { result.failed++; diff --git a/server/src/services/native-runtime/native-harness-backup-storage.integration.test.ts b/server/src/services/native-runtime/native-harness-backup-storage.integration.test.ts index 182f14d03fb..fcc6d5b5709 100644 --- a/server/src/services/native-runtime/native-harness-backup-storage.integration.test.ts +++ b/server/src/services/native-runtime/native-harness-backup-storage.integration.test.ts @@ -1,8 +1,10 @@ +import fs from "node:fs/promises"; import { createHash, randomUUID } from "node:crypto"; import { chmod, mkdir, readFile, rename, stat, symlink, readlink, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { and, eq } from "drizzle-orm"; -import { agentTaskSessions, environmentLeases, heartbeatRuns, issues, nativeHarnessBackups, nativeRunFinalizations, nativeRunResults } from "@paperclipai/db"; +import { and, eq, inArray } from "drizzle-orm"; +import { agentTaskSessions, environmentLeases, heartbeatRuns, issues, nativeHarnessBackups, nativeRunFinalizations, nativeRunResults, + nativeProviderArtifacts, nativeProviderArtifactReferences, nativeRunnerCheckpoints } from "@paperclipai/db"; import { afterEach, expect, it, vi } from "vitest"; import { describeEmbeddedPostgres, nativeRestartFixture } from "../../__tests__/helpers/native-restart.js"; import { createLocalDiskStorageProvider } from "../../storage/local-disk-provider.js"; @@ -15,7 +17,17 @@ import { createNativeHarnessBackupStamp, verifyNativeHarnessBackupStamp, verifyN import { hasLocalNativeHarnessObligations, migrateNativeHarnessBackups, prestageNativeHarnessBackups } from "./native-harness-backup-migration.js"; import { nativeHarnessBackupExpirationService } from "./native-harness-backup-expiration.js"; import { legacyControllerBootId } from "../legacy-controller-lease.js"; +import { DurablePrpControlPlane, captureLegacyDurablePrpCheckpoint } from "../../vendor/paperclip-runner/index.js"; +import { loadNativeRunnerCheckpoint, nativeRunnerCheckpointWriter } from "./native-runner-checkpoint.js"; +import { restoreNativeProviderArtifacts } from "./native-provider-artifacts.js"; import * as backupArchive from "./native-harness-backup-archive.js"; +import { canonicalProviderPackJson } from "./remote-provider-pack-verification.js"; + +vi.mock("node:fs/promises", async importOriginal => { + const original = await importOriginal(); + const files = { ...original }; + return { ...original, default: files, cp: (...args: Parameters) => files.cp(...args) }; +}); describeEmbeddedPostgres("durable native provider backups", () => { const environment = nativeRestartFixture(); @@ -160,6 +172,309 @@ describeEmbeddedPostgres("durable native provider backups", () => { expect((await stat(join(replacement, "current", provider, "sessions"))).mode & 0o777).toBe(0o500); expect(await verifyDurableNativeHarnessBackup(f)).toBe(true); }); + async function artifactFixture(artifactVersion = randomUUID()) { + const f = await fixture("opencode"); + const sourceRoot = join(f.home, "qualified-provider-pack"); + await mkdir(sourceRoot); + const original = { nodeCommand: "qualified Node executable", opencodeExecutable: "qualified OpenCode executable", + opencodeProxy: `qualified original proxy bytes ${artifactVersion}` }; + const artifacts = Object.fromEntries(await Promise.all(Object.entries(original).map(async ([name, bytes]) => { + await writeFile(join(sourceRoot, name), bytes, { mode: 0o500 }); + return [name, { path: name, sha256: `sha256:${createHash("sha256").update(bytes).digest("hex")}` }]; + }))); + const payload = { runnerSourceRevision: "a".repeat(40), target: { platform: "linux", architecture: "x64" }, + pins: { opencode: "1.18.34" }, artifacts: { ...artifacts, opencodeCommand: artifacts.opencodeExecutable } }; + const manifest = { schema: "paperclip-runner/remote-provider-pack/v1", payload, + digest: `sha256:${createHash("sha256").update(canonicalProviderPackJson(payload)).digest("hex")}` }; + const launchRoot = `/home/daytona/.paperclip-runtime/paperclip-runner/provider-artifacts/${manifest.digest.slice(7)}`; + await f.db.update(heartbeatRuns).set({ nativeSessionId: f.identity.normalizedSessionId, driverKind: "opencode_server", + runnerProfileJson: { ...f.preparedProfile, nativeProviderPackAdmissionState: { + schema: "paperclip.native-provider-pack-admission-state.v1", state: "verified", admissionEpoch: randomUUID() }, + nativeProviderPackAdmission: { schema: "paperclip.native-provider-pack-admission.v1", stage: "session_admitted", + authority: "core_verified_selection", companyId: f.companyId, agentId: environment.current.agentId, + runId: f.runId, sessionId: f.identity.normalizedSessionId, runnerInstanceId: "artifact-fixture-runner", + runnerEnvironmentLeaseId: "artifact-fixture-lease", providerPackDigest: manifest.digest, + opencodeProxySha256: artifacts.opencodeProxy.sha256, sourceRevision: payload.runnerSourceRevision } }, + }).where(eq(heartbeatRuns.id, f.runId)); + return { ...f, original, manifest, launchRoot, providerArtifacts: { sourceRoot, manifest, launchRoot, accountUid: 1000 } }; + } + it("restores the admitted OpenCode launch bytes after the controller and sandbox image change", async () => { + const publication = await artifactFixture(), f = publication; + const { original, manifest, launchRoot } = publication; + await persistNativeHarnessBackup(publication); + await removeNativeHarnessBackup(f.home); + const newImage = join(environment.current.runtimeRoot, randomUUID()); ownedHomes.push(newImage); + await mkdir(newImage); + await writeFile(join(newImage, "opencodeProxy"), "qualified newer proxy bytes"); + const replacement = join(environment.current.runtimeRoot, randomUUID(), "failover-backups"); ownedHomes.push(replacement); + expect(await restoreDurableNativeHarnessBackup({ ...f, backupRoot: replacement })).toBe(true); + expect(JSON.parse(await readFile(join(replacement, "current/opencode/sessions/history.json"), "utf8"))).toEqual(f.identity); + for (const [name, bytes] of Object.entries(original)) { + expect(await readFile(join(replacement, "current/provider-artifacts", name), "utf8")).toBe(bytes); + } + const restored = JSON.parse(await readFile(join(replacement, "current/manifest.json"), "utf8")); + expect(restored.providerArtifacts).toMatchObject({ launchRoot, providerPackDigest: manifest.digest }); + }); + it("repairs a missing local launch cache without adding the binaries to each harness backup", async () => { + const f = await artifactFixture(); await persistNativeHarnessBackup(f); + const [before] = await f.db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.scopeDigest, f.scopeDigest)); + const replacement = join(environment.current.runtimeRoot, randomUUID()); ownedHomes.push(replacement); + expect(await restoreDurableNativeHarnessBackup({ ...f, backupRoot: replacement })).toBe(true); + await removeNativeHarnessBackup(join(replacement, "current/provider-artifacts")); + expect(await restoreDurableNativeHarnessBackup({ ...f, backupRoot: replacement })).toBe(true); + expect(await readFile(join(replacement, "current/provider-artifacts/opencodeProxy"), "utf8")).toBe(f.original.opencodeProxy); + await persistNativeHarnessBackup({ ...f, providerArtifacts: undefined, root: join(replacement, "current") }); + const [after] = await f.db.select().from(nativeHarnessBackups).where(and(eq(nativeHarnessBackups.scopeDigest, f.scopeDigest), + eq(nativeHarnessBackups.status, "current"))); + expect(after.reference.byteSize).toBe(before.reference.byteSize); + }); + it("retries an interrupted artifact cache copy without publishing partial launch files", async () => { + const f = await artifactFixture(); await persistNativeHarnessBackup(f); + const receipt = JSON.parse(await readFile(join(f.root, "manifest.json"), "utf8")).providerArtifacts; + const home = join(environment.current.runtimeRoot, randomUUID()); ownedHomes.push(home); await mkdir(home); + const targetRoot = join(home, "provider-artifacts"), copy = fs.cp.bind(fs); + const interrupted = vi.spyOn(fs, "cp").mockImplementation(async (source, destination, options) => { + if (String(destination).startsWith(targetRoot)) { + await mkdir(destination, { recursive: true }); + await writeFile(join(String(destination), "opencodeProxy"), "partial copy"); + throw Object.assign(new Error("controlled disk full during cache publication"), { code: "ENOSPC" }); + } + return copy(source, destination, options); + }); + try { await expect(restoreNativeProviderArtifacts({ ...f, receipt, targetRoot })).rejects.toThrow("controlled disk full"); } + finally { interrupted.mockRestore(); } + await expect(fs.stat(targetRoot)).rejects.toMatchObject({ code: "ENOENT" }); + await expect(restoreNativeProviderArtifacts({ ...f, receipt, targetRoot })).resolves.toBeUndefined(); + expect(await readFile(join(targetRoot, "opencodeProxy"), "utf8")).toBe(f.original.opencodeProxy); + expect(await fs.readdir(home)).toEqual(["provider-artifacts"]); + }); + it("reuses a sequentially admitted closure and releases it only after the last stopped checkpoint retires", async () => { + const version = randomUUID(), first = await artifactFixture(version), other = await artifactFixture(version); + const second = { ...other, storage: first.storage }; + const put = vi.spyOn(first.storage, "putFile"); + await persistNativeHarnessBackup(first); await persistNativeHarnessBackup(second); + expect(put.mock.calls.filter(([input]) => input.namespace === "native-provider-artifacts")).toHaveLength(1); + const [firstBackup] = await first.db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.scopeDigest, first.scopeDigest)); + const [secondBackup] = await first.db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.scopeDigest, second.scopeDigest)); + const artifactId = (firstBackup.reference.providerArtifacts as { artifactId: string }).artifactId; + const [artifact] = await first.db.select().from(nativeProviderArtifacts).where(eq(nativeProviderArtifacts.id, artifactId)); + const stop = async (f: typeof first) => { + await settle(f, new Date(Date.now() - 10 * 86_400_000)); + const leaseId = randomUUID(), providerLeaseId = randomUUID(), confirmedAt = new Date(); + await f.db.insert(environmentLeases).values({ id: leaseId, companyId: f.companyId, heartbeatRunId: f.runId, + issueId: f.issueId, provider: "daytona", providerLeaseId, status: "expired", cleanupStatus: "success", releasedAt: confirmedAt, + metadata: { remoteExecutionTermination: { schema: "paperclip.remote-termination.v1", companyId: f.companyId, + runId: f.runId, leaseId, provider: "daytona", providerLeaseId, state: "destroyed", confirmedAt: confirmedAt.toISOString() } } }); + }; + await stop(first); + await first.db.update(nativeHarnessBackups).set({ status: "retired" }).where(eq(nativeHarnessBackups.id, firstBackup.id)); + expect(await deleteRetiredNativeHarnessBackup(first.db, first.storage, firstBackup.id)).toBe(true); + expect(await first.db.select().from(nativeProviderArtifacts).where(eq(nativeProviderArtifacts.id, artifact.id))).toHaveLength(1); + const replacement = join(environment.current.runtimeRoot, randomUUID()); ownedHomes.push(replacement); + expect(await restoreDurableNativeHarnessBackup({ ...second, backupRoot: replacement })).toBe(true); + await stop(second); + await second.db.update(nativeHarnessBackups).set({ status: "retired" }).where(eq(nativeHarnessBackups.id, secondBackup.id)); + expect(await deleteRetiredNativeHarnessBackup(second.db, second.storage, secondBackup.id)).toBe(true); + expect(await first.db.select().from(nativeProviderArtifacts).where(eq(nativeProviderArtifacts.id, artifact.id))).toHaveLength(1); + const runPins = await first.db.select().from(nativeProviderArtifactReferences).where(eq(nativeProviderArtifactReferences.artifactId, artifact.id)); + expect(runPins).toHaveLength(2); expect(runPins.every(pin => pin.backupId === null)).toBe(true); + await first.db.insert(nativeRunnerCheckpoints).values([first, second].map(f => ({ companyId: f.companyId, + issueId: f.issueId, runId: f.runId, part: "header", material: { retained: "canonical checkpoint expiry fixture" } }))); + expect((await nativeHarnessBackupExpirationService(first.db, { storage: first.storage }).tickCheckpoints()).expiredCheckpoints).toBe(2); + expect(await first.db.select().from(nativeProviderArtifacts).where(eq(nativeProviderArtifacts.id, artifact.id))).toHaveLength(0); + expect(await first.db.select().from(nativeProviderArtifactReferences).where(eq(nativeProviderArtifactReferences.artifactId, artifact.id))).toHaveLength(0); + }); + it("keeps independent overlapping uploads bounded and retains only their verified winner", async () => { + const version = randomUUID(), first = await artifactFixture(version), other = await artifactFixture(version); + const second = { ...other, storage: first.storage }, put = first.storage.putFile.bind(first.storage); + let release!: () => void, arrivals = 0; + const together = new Promise(resolve => { release = resolve; }); + const uploaded: string[] = []; + const calls = vi.spyOn(first.storage, "putFile").mockImplementation(async input => { + if (input.namespace === "native-provider-artifacts") { + expect(input.objectKey).toBeTypeOf("string"); uploaded.push(input.objectKey!); if (++arrivals === 2) release(); + await together; expect("byteSize" in input ? input.byteSize : input.body.length).toBeLessThan(512 * 1024 * 1024 + 1024 * 1024); + } + return put(input); + }); + const deleted = vi.spyOn(first.storage, "deleteObject"); + const transfers = Promise.all([persistNativeHarnessBackup(first), persistNativeHarnessBackup(second)]); + void transfers.catch(() => undefined); + try { + try { await vi.waitFor(() => expect(arrivals).toBe(2), { timeout: 10_000 }); } finally { release(); } + await transfers; + } finally { calls.mockRestore(); } + expect(uploaded).toHaveLength(2); + const objects = await first.db.select().from(nativeProviderArtifacts).where(and(eq(nativeProviderArtifacts.companyId, first.companyId), + inArray(nativeProviderArtifacts.id, uploaded.map(key => key.split("/").at(-2)!)))); + expect(objects).toHaveLength(1); expect(objects[0].status).toBe("ready"); + expect(deleted.mock.calls.map(([, key]) => key)).toEqual(uploaded.filter(key => key !== objects[0].reference!.objectKey)); + for (const f of [first, second]) { + const [run] = await f.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + expect(run.runnerProfileJson?.nativeProviderArtifacts).toMatchObject({ artifactId: objects[0].id }); + const replacement = join(environment.current.runtimeRoot, randomUUID()); ownedHomes.push(replacement); + expect(await restoreDurableNativeHarnessBackup({ ...f, backupRoot: replacement })).toBe(true); + expect(await readFile(join(replacement, "current/provider-artifacts/opencodeProxy"), "utf8")).toBe(f.original.opencodeProxy); + } + }); + it("keeps a resumable run pin when an interrupted backup is retired", async () => { + const f = await artifactFixture(), put = f.storage.putFile.bind(f.storage); + const interrupted = vi.spyOn(f.storage, "putFile").mockImplementation(async input => { + const result = await put(input); + if (input.namespace === "native-harness-backups") throw new Error("harness upload reply lost"); + return result; + }); + await expect(persistNativeHarnessBackup(f)).rejects.toThrow("harness upload reply lost"); interrupted.mockRestore(); + const [pending] = await f.db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.scopeDigest, f.scopeDigest)); + expect(pending.status).toBe("pending"); + const receipt = pending.reference.providerArtifacts as NonNullable[0]["receipt"]>; + const identity = { runnerInstanceId: f.native.runnerInstanceId, environmentLeaseId: f.native.environmentLeaseId, + runId: f.runId, normalizedSessionId: f.identity.normalizedSessionId, turnId: f.native.turnId, itemId: f.native.itemId }; + const controller = new DurablePrpControlPlane({ stateDirectory: join(f.home, "resumable-controller"), identity, + expectedRunnerVersion: "0.0.0", expectedRunnerDigest: "sha256:" + "1".repeat(64) }); + const checkpoint = captureLegacyDurablePrpCheckpoint(JSON.stringify(controller.store.state), { + runId: f.runId, runnerInstanceId: identity.runnerInstanceId, normalizedSessionId: identity.normalizedSessionId }, + { version: "0.0.0", digest: "sha256:" + "1".repeat(64) }); + await nativeRunnerCheckpointWriter(f.db, { ...f.owner, leaseOwner: f.owner.leaseOwner! })(checkpoint); + const old = new Date(Date.now() - 10 * 86_400_000); await settle(f, old); + await f.db.update(nativeHarnessBackups).set({ createdAt: old, updatedAt: old }).where(eq(nativeHarnessBackups.id, pending.id)); + const leaseId = randomUUID(), providerLeaseId = randomUUID(), confirmedAt = new Date(); + await f.db.insert(environmentLeases).values({ id: leaseId, companyId: f.companyId, heartbeatRunId: f.runId, + issueId: f.issueId, provider: "daytona", providerLeaseId, status: "expired", cleanupStatus: "success", releasedAt: confirmedAt, + metadata: { remoteExecutionTermination: { schema: "paperclip.remote-termination.v1", companyId: f.companyId, + runId: f.runId, leaseId, provider: "daytona", providerLeaseId, state: "destroyed", confirmedAt: confirmedAt.toISOString() } } }); + const [selected] = await f.db.insert(agentTaskSessions).values({ companyId: f.companyId, agentId: environment.current.agentId, + adapterType: "paperclip_runner", taskKey: f.issueId, lastRunId: f.runId, + sessionParamsJson: { sessionId: f.identity.normalizedSessionId } }).returning(); + const retention = nativeHarnessBackupExpirationService(f.db, { storage: f.storage }); + const first = await retention.tickCheckpoints(); + expect(first.expired).toBe(1); expect(first.expiredCheckpoints).toBe(0); + expect(await f.db.select().from(nativeProviderArtifactReferences).where(eq(nativeProviderArtifactReferences.runId, f.runId))) + .toMatchObject([{ backupId: null, artifactId: receipt.artifactId }]); + expect((await loadNativeRunnerCheckpoint(f.db, { companyId: f.companyId, issueId: f.issueId, runId: f.runId }))?.recoveryEvidence.normalizedSessionId) + .toBe(f.identity.normalizedSessionId); + await removeNativeHarnessBackup(f.providerArtifacts.sourceRoot); + const restored = join(environment.current.runtimeRoot, randomUUID()); ownedHomes.push(restored); + await restoreNativeProviderArtifacts({ ...f, receipt, targetRoot: restored }); + expect(await readFile(join(restored, "opencodeProxy"), "utf8")).toBe(f.original.opencodeProxy); + await f.db.delete(agentTaskSessions).where(eq(agentTaskSessions.id, selected.id)); + expect((await retention.tickCheckpoints()).expiredCheckpoints).toBe(1); + expect(await f.db.select().from(nativeProviderArtifacts).where(eq(nativeProviderArtifacts.id, receipt.artifactId))).toEqual([]); + }); + it("keeps the exact encrypted upload receipt when its artifact owner changes after upload", async () => { + const f = await artifactFixture(), put = f.storage.putFile.bind(f.storage); + vi.spyOn(f.storage, "putFile").mockImplementation(async input => { + const result = await put(input); + if (input.namespace === "native-provider-artifacts") await f.db.update(nativeRunFinalizations) + .set({ controllerGeneration: 2 }).where(eq(nativeRunFinalizations.runId, f.runId)); + return result; + }); + await expect(persistNativeHarnessBackup(f)).rejects.toThrow("native_runner_checkpoint_owner_lost"); + const [pin] = await f.db.select().from(nativeProviderArtifactReferences).where(eq(nativeProviderArtifactReferences.runId, f.runId)); + const [artifact] = await f.db.select().from(nativeProviderArtifacts).where(eq(nativeProviderArtifacts.id, pin.artifactId)); + expect(artifact.status).toBe("pending"); + expect(artifact.reference).toMatchObject({ companyId: f.companyId, artifactId: artifact.id, objectKey: expect.any(String), material: expect.any(Object) }); + expect(await readFile(join(f.providerArtifacts.sourceRoot, "opencodeProxy"), "utf8")).toBe(f.original.opencodeProxy); + }); + it("retires an abandoned artifact upload through checkpoint expiration without losing deletion retries", async () => { + const f = await artifactFixture(), put = f.storage.putFile.bind(f.storage); + const lostReply = vi.spyOn(f.storage, "putFile").mockImplementation(async input => { + await put(input); throw new Error("artifact upload reply lost"); + }); + await expect(persistNativeHarnessBackup(f)).rejects.toThrow("artifact upload reply lost"); lostReply.mockRestore(); + const [pin] = await f.db.select().from(nativeProviderArtifactReferences).where(eq(nativeProviderArtifactReferences.runId, f.runId)); + const [artifact] = await f.db.select().from(nativeProviderArtifacts).where(eq(nativeProviderArtifacts.id, pin.artifactId)); + expect(artifact.reference).toMatchObject({ objectKey: expect.any(String), material: expect.any(Object) }); + expect(await f.db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.scopeDigest, f.scopeDigest))).toEqual([]); + await f.db.insert(nativeRunnerCheckpoints).values({ companyId: f.companyId, runId: f.runId, + issueId: f.issueId, part: "header", material: { retained: "canonical header index fixture" } }); + const retention = nativeHarnessBackupExpirationService(f.db, { storage: f.storage }); + expect((await retention.tickCheckpoints()).expiredCheckpoints).toBe(0); + const old = new Date(Date.now() - 10 * 86_400_000); + await settle(f, old); + const leaseId = randomUUID(), providerLeaseId = randomUUID(), confirmedAt = new Date(); + await f.db.insert(environmentLeases).values({ id: leaseId, companyId: f.companyId, heartbeatRunId: f.runId, + issueId: f.issueId, provider: "daytona", providerLeaseId, status: "expired", cleanupStatus: "success", releasedAt: confirmedAt, + metadata: { remoteExecutionTermination: { schema: "paperclip.remote-termination.v1", companyId: f.companyId, + runId: f.runId, leaseId, provider: "daytona", providerLeaseId, state: "destroyed", confirmedAt: confirmedAt.toISOString() } } }); + const drop = vi.spyOn(f.storage, "deleteObject").mockRejectedValueOnce(new Error("delete reply lost")); + const [earlier] = await f.db.insert(agentTaskSessions).values({ companyId: f.companyId, agentId: environment.current.agentId, + adapterType: "paperclip_runner", taskKey: f.issueId, lastRunId: f.runId }).returning(); + expect((await retention.tickCheckpoints()).expiredCheckpoints).toBe(0); + expect(drop).not.toHaveBeenCalled(); + expect((await f.db.select().from(nativeRunnerCheckpoints).where(eq(nativeRunnerCheckpoints.runId, f.runId)))[0].material) + .toEqual({ retained: "canonical header index fixture" }); + await f.db.delete(agentTaskSessions).where(eq(agentTaskSessions.id, earlier.id)); + expect((await retention.tickCheckpoints()).failed).toBe(1); + const [retired] = await f.db.select().from(nativeRunnerCheckpoints).where(eq(nativeRunnerCheckpoints.runId, f.runId)); + expect(retired.material).toMatchObject({ schema: "paperclip.native-runner-checkpoint-retirement.v1", companyId: f.companyId, + runIds: [f.runId], sessionId: f.identity.normalizedSessionId }); + expect(await f.db.select().from(nativeProviderArtifactReferences).where(eq(nativeProviderArtifactReferences.artifactId, artifact.id))).toHaveLength(1); + await f.db.insert(agentTaskSessions).values({ companyId: f.companyId, agentId: environment.current.agentId, + adapterType: "paperclip_runner", taskKey: f.issueId, lastRunId: f.runId }); + await expect(loadNativeRunnerCheckpoint(f.db, { companyId: f.companyId, issueId: f.issueId, runId: f.runId })) + .rejects.toThrow("native_runner_checkpoint_retired"); + const missingStorage = nativeHarnessBackupExpirationService(f.db); + expect((await missingStorage.tickCheckpoints()).failed).toBe(1); + expect(drop).toHaveBeenCalledTimes(1); + const [priorOwner] = await f.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)); + const currentOwner = { ...f.owner, generation: priorOwner.controllerGeneration, leaseOwner: "fresh-retired-source-claim" }; + await f.db.update(nativeRunFinalizations).set({ phase: "observed", leaseOwner: currentOwner.leaseOwner, + leaseExpiresAt: new Date(Date.now() + 60_000) }).where(eq(nativeRunFinalizations.runId, f.runId)); + const identity = { runnerInstanceId: randomUUID(), environmentLeaseId: randomUUID(), runId: f.runId, + normalizedSessionId: f.identity.normalizedSessionId, turnId: randomUUID(), itemId: randomUUID() }; + const controller = new DurablePrpControlPlane({ stateDirectory: join(f.home, "new-source-controller"), identity, + expectedRunnerVersion: "0.0.0", expectedRunnerDigest: "sha256:" + "1".repeat(64) }); + const checkpoint = captureLegacyDurablePrpCheckpoint(JSON.stringify(controller.store.state), { + runId: f.runId, runnerInstanceId: identity.runnerInstanceId, normalizedSessionId: identity.normalizedSessionId }, + { version: "0.0.0", digest: "sha256:" + "1".repeat(64) }); + try { await expect(nativeRunnerCheckpointWriter(f.db, currentOwner)(checkpoint)).rejects.toThrow("native_runner_checkpoint_retired"); } + finally { await f.db.update(nativeRunFinalizations).set({ phase: priorOwner.phase, leaseOwner: null, + leaseExpiresAt: null }).where(eq(nativeRunFinalizations.runId, f.runId)); } + const consumerRunId = randomUUID(), consumerLeaseId = randomUUID(); + await f.db.insert(heartbeatRuns).values({ id: consumerRunId, companyId: f.companyId, agentId: environment.current.agentId, + issueId: f.issueId, nativeIssueId: f.issueId, nativeSessionId: f.identity.normalizedSessionId, status: "running" }); + await f.db.insert(nativeRunFinalizations).values({ runId: consumerRunId, companyId: f.companyId, issueId: f.issueId, + phase: "observed", controllerBootId: f.owner.bootId, controllerGeneration: 1, leaseOwner: "late-source-consumer", + leaseExpiresAt: new Date(Date.now() + 60_000) }); + await f.db.update(issues).set({ status: "in_progress", executionRunId: consumerRunId }).where(eq(issues.id, f.issueId)); + await f.db.update(environmentLeases).set({ metadata: { remoteExecutionTermination: { + schema: "paperclip.remote-termination.v1", companyId: f.companyId, runId: f.runId, leaseId, provider: "daytona", + providerLeaseId, state: "destroyed", confirmedAt: confirmedAt.toISOString() }, + runnerTaskSourceConsumer: { leaseId: consumerLeaseId, runId: consumerRunId, accountIdentity: "owned-fixture-account", identity } } }) + .where(eq(environmentLeases.id, leaseId)); + await f.db.insert(environmentLeases).values({ id: consumerLeaseId, companyId: f.companyId, heartbeatRunId: consumerRunId, + issueId: f.issueId, provider: "daytona", providerLeaseId: randomUUID(), status: "active", metadata: { runnerTaskSourceRecovery: { + schema: "paperclip.runner.source-recovery.v1", sourceLeaseId: leaseId, consumerLeaseId, consumerRunId, + accountIdentity: "owned-fixture-account", controllerBootId: f.owner.bootId, controllerGeneration: 1, identity } } }); + const lateCheckpoint = { ...checkpoint, reset: false, changedRecords: new Map([["cursor", checkpoint.records.get("cursor")!]]), removedRecords: [] }; + await expect(nativeRunnerCheckpointWriter(f.db, { ...f.owner, authorityRunId: consumerRunId, generation: 1, + leaseOwner: "late-source-consumer" })(lateCheckpoint)).rejects.toThrow("native_runner_checkpoint_retired"); + const restarted = nativeHarnessBackupExpirationService(f.db, { storage: f.storage }); + expect((await restarted.tickCheckpoints()).expiredCheckpoints).toBe(1); + expect(await f.db.select().from(nativeProviderArtifacts).where(eq(nativeProviderArtifacts.id, artifact.id))).toEqual([]); + expect(await f.db.select().from(nativeRunnerCheckpoints).where(eq(nativeRunnerCheckpoints.runId, f.runId))).toEqual([]); + expect((await restarted.tickCheckpoints()).expiredCheckpoints).toBe(0); + expect(drop).toHaveBeenCalledTimes(2); + }); + it("retains every artifact and checkpoint when a retirement family exceeds its run bound", async () => { + const f = await artifactFixture(), put = f.storage.putFile.bind(f.storage); + const lost = vi.spyOn(f.storage, "putFile").mockImplementation(async input => { await put(input); throw new Error("upload reply lost"); }); + await expect(persistNativeHarnessBackup(f)).rejects.toThrow("upload reply lost"); lost.mockRestore(); + const old = new Date(Date.now() - 10 * 86_400_000); await settle(f, old); + await f.db.insert(nativeRunnerCheckpoints).values({ companyId: f.companyId, issueId: f.issueId, runId: f.runId, + part: "header", material: { retained: "original checkpoint" } }); + const relatives = Array.from({ length: 100 }, () => randomUUID()); + await f.db.insert(heartbeatRuns).values(relatives.map(id => ({ id, companyId: f.companyId, agentId: environment.current.agentId, + issueId: f.issueId, nativeIssueId: f.issueId, nativeSessionId: f.identity.normalizedSessionId, status: "failed", finishedAt: old }))); + await f.db.insert(nativeRunFinalizations).values(relatives.map(runId => ({ runId, companyId: f.companyId, issueId: f.issueId, phase: "terminal_failure" }))); + const remove = vi.spyOn(f.storage, "deleteObject"); + const result = await nativeHarnessBackupExpirationService(f.db, { storage: f.storage }).tickCheckpoints(); + expect(result.expiredCheckpoints).toBe(0); expect(remove).not.toHaveBeenCalled(); + expect((await f.db.select().from(nativeRunnerCheckpoints).where(eq(nativeRunnerCheckpoints.runId, f.runId)))[0].material) + .toEqual({ retained: "original checkpoint" }); + expect(await f.db.select().from(nativeProviderArtifactReferences).where(eq(nativeProviderArtifactReferences.runId, f.runId))).toHaveLength(1); + }); it("keeps the last published backup when object storage fails", async () => { const f = await fixture(); await persistNativeHarnessBackup(f); const [before] = await f.db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.scopeDigest, f.scopeDigest)); diff --git a/server/src/services/native-runtime/native-harness-backup-storage.ts b/server/src/services/native-runtime/native-harness-backup-storage.ts index 4d35d2d5482..74bdd963e1c 100644 --- a/server/src/services/native-runtime/native-harness-backup-storage.ts +++ b/server/src/services/native-runtime/native-harness-backup-storage.ts @@ -1,7 +1,7 @@ import { publishNativeHarnessBackup } from "./native-harness-backup-publication.js"; import { createCipheriv, createDecipheriv, createHash, randomBytes, randomUUID } from "node:crypto"; import { createReadStream, createWriteStream } from "node:fs"; -import { mkdtemp, readFile } from "node:fs/promises"; +import { mkdtemp, readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; import os from "node:os"; import { pipeline } from "node:stream/promises"; @@ -19,12 +19,16 @@ import { removeNativeHarnessBackup } from "./native-harness-backup-cleanup.js"; import { hasRemoteTerminationReceipt, resolveRemoteTerminationLease } from "../remote-execution-termination.js"; import { offlineMigrationExecutionWhere } from "../offline-migration-execution.js"; import { readNativeHarnessBackupSource } from "./native-harness-backup-source.js"; +import { pinNativeProviderArtifactsForBackup, releaseRetiredNativeProviderArtifacts, restoreNativeProviderArtifacts, retainNativeProviderArtifacts, + type NativeProviderArtifactReceipt, type NativeProviderArtifactSource } from "./native-provider-artifacts.js"; +import { canonicalProviderPackJson } from "./remote-provider-pack-verification.js"; const SCHEMA = "paperclip.native-harness-object.v1"; type Binding = { companyId: string; scopeDigest: string; id: string }; type Reference = Binding & { schema: typeof SCHEMA; objectKey: string; sha256: string; byteSize: number; material: Record; manifestSha256: string; + providerArtifacts?: NativeProviderArtifactReceipt; }; type Row = typeof nativeHarnessBackups.$inferSelect; type OfflineOwner = { kind: "offline_migration"; companyId: string; issueId: string; runId: string; @@ -182,11 +186,20 @@ export async function persistNativeHarnessBackup(input: { db: Db; storage: StorageService; owner: NativeRunnerCheckpointFenceOwner | OfflineOwner; scopeDigest: string; root: string; signal?: AbortSignal; maxArchiveBytes?: number; inactiveSource?: { normalizedSessionId: string; runnerInstanceId: string; providerLeaseId: string }; + providerArtifacts?: NativeProviderArtifactSource; }): Promise { if (!/^[a-f0-9]{64}$/.test(input.scopeDigest)) fail("scope_invalid"); + if (input.providerArtifacts) { + if ("kind" in input.owner) fail("provider_artifacts_require_runtime_owner"); + const receipt = await retainNativeProviderArtifacts({ ...input, owner: input.owner, source: input.providerArtifacts }); + const manifestPath = join(input.root, "manifest.json"); + const manifest = JSON.parse(await readFile(manifestPath, "utf8")); + await writeFile(manifestPath, JSON.stringify({ ...manifest, providerArtifacts: receipt }), { mode: 0o600 }); + } const source = input.inactiveSource ? await readNativeHarnessBackupSource(input.root) : undefined; if (source === null) fail("prestage_source_changed"); const manifestBytes = source?.manifestBytes ?? await readFile(join(input.root, "manifest.json")); + const providerArtifacts = JSON.parse(manifestBytes.toString("utf8")).providerArtifacts as NativeProviderArtifactReceipt | undefined; const manifestSha256 = `sha256:${createHash("sha256").update(manifestBytes).digest("hex")}`; const inactiveSource = input.inactiveSource ? { ...input.inactiveSource, root: input.root, manifestSha256, scopeDigest: input.scopeDigest } : undefined; const binding = { companyId: input.owner.companyId, scopeDigest: input.scopeDigest, id: randomUUID() }; @@ -194,11 +207,11 @@ export async function persistNativeHarnessBackup(input: { const key = randomBytes(32); try { const archive = join(scratch, "archive"), encrypted = join(scratch, "encrypted"); - await captureNativeHarnessArchive(input.root, archive, input.signal, input.maxArchiveBytes); + await captureNativeHarnessArchive(input.root, archive, input.signal, input.maxArchiveBytes, providerArtifacts ? ["provider-artifacts"] : []); const plaintext = await digest(archive, input.signal), iv = randomBytes(12); const cipher = createCipheriv("aes-256-gcm", key, iv); cipher.setAAD(aad(binding)); await pipeline(createReadStream(archive), cipher, createWriteStream(encrypted, { flags: "wx", mode: 0o600 }), { signal: input.signal }); - const ref: Reference = { schema: SCHEMA, ...binding, manifestSha256, + const ref: Reference = { schema: SCHEMA, ...binding, manifestSha256, ...(providerArtifacts ? { providerArtifacts } : {}), objectKey: `${binding.companyId}/native-harness-backups/${binding.scopeDigest}/${binding.id}.pchn`, ...await digest(encrypted, input.signal), material: createLocalEncryptedCipher("write").encrypt(JSON.stringify({ schema: SCHEMA, ...binding, @@ -208,6 +221,8 @@ export async function persistNativeHarnessBackup(input: { await fence(tx, input.owner, inactiveSource); await tx.insert(nativeHarnessBackups).values({ ...binding, issueId: input.owner.issueId, runId: input.owner.runId, manifestSha256, reference: ref }); + if (providerArtifacts) await pinNativeProviderArtifactsForBackup(tx, { companyId: input.owner.companyId, + runId: input.owner.runId, backupId: binding.id, receipt: providerArtifacts }); }); await input.storage.putFile({ companyId: ref.companyId, namespace: "native-harness-backups", objectKey: ref.objectKey, originalFilename: null, contentType: "application/octet-stream", body: createReadStream(encrypted), @@ -247,7 +262,13 @@ export async function restoreDurableNativeHarnessBackup(input: { const localManifest = await readFile(join(current, "manifest.json")).catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return null; throw error; }); - if (localManifest && `sha256:${createHash("sha256").update(localManifest).digest("hex")}` === ref.manifestSha256) return true; + if (localManifest && `sha256:${createHash("sha256").update(localManifest).digest("hex")}` === ref.manifestSha256) { + if (canonicalProviderPackJson(JSON.parse(localManifest.toString("utf8")).providerArtifacts) + !== canonicalProviderPackJson(ref.providerArtifacts)) fail("provider_artifacts_mismatch"); + if (ref.providerArtifacts) await restoreNativeProviderArtifacts({ ...input, receipt: ref.providerArtifacts, + targetRoot: join(current, "provider-artifacts") }); + return true; + } const scratch = await mkdtemp(join(os.tmpdir(), "pc-harness-restore-")); const key = Buffer.from(wrapped.key, "base64"); try { @@ -263,6 +284,10 @@ export async function restoreDurableNativeHarnessBackup(input: { await restoreNativeHarnessArchive(archive, restored, input.signal); const manifest = await readFile(join(restored, "manifest.json")); if (`sha256:${createHash("sha256").update(manifest).digest("hex")}` !== ref.manifestSha256) fail("manifest_mismatch"); + const restoredManifest = JSON.parse(manifest.toString("utf8")); + if (canonicalProviderPackJson(restoredManifest.providerArtifacts) !== canonicalProviderPackJson(ref.providerArtifacts)) fail("provider_artifacts_mismatch"); + if (ref.providerArtifacts) await restoreNativeProviderArtifacts({ ...input, receipt: ref.providerArtifacts, + targetRoot: join(restored, "provider-artifacts") }); const { mkdir, cp } = await import("node:fs/promises"); await mkdir(input.backupRoot, { recursive: true, mode: 0o700 }); const pending = join(input.backupRoot, `.durable-${randomUUID()}`); @@ -290,6 +315,8 @@ export async function deleteRetiredNativeHarnessBackup(db: Db, storage: StorageS eq(nativeHarnessBackups.id, id), eq(nativeHarnessBackups.status, "retired"))); if (!row) return false; const ref = reference(row); + if (ref.providerArtifacts) await releaseRetiredNativeProviderArtifacts({ db, storage, companyId: row.companyId, + runId: row.runId, backupId: row.id, receipt: ref.providerArtifacts, signal }); await storage.deleteObject(ref.companyId, ref.objectKey, { allVersions: true, signal }); await db.delete(nativeHarnessBackups).where(and(eq(nativeHarnessBackups.id, id), eq(nativeHarnessBackups.status, "retired"))); return true; diff --git a/server/src/services/native-runtime/native-provider-artifact-capture.test.ts b/server/src/services/native-runtime/native-provider-artifact-capture.test.ts new file mode 100644 index 00000000000..6fcc707cd26 --- /dev/null +++ b/server/src/services/native-runtime/native-provider-artifact-capture.test.ts @@ -0,0 +1,112 @@ +import { randomUUID } from "node:crypto"; +import { stat, writeFile } from "node:fs/promises"; +import { dirname } from "node:path"; +import { afterEach, expect, it, vi } from "vitest"; +import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; +import type { NativeExecutionInput } from "../../vendor/paperclip-runner/index.js"; +import type { NativeProviderArtifactSource } from "./native-provider-artifacts.js"; +import { NativeExecutionContext } from "./native-execution-context.js"; +import { captureRemoteNativeProviderArtifacts } from "./native-provider-artifact-capture.js"; + +afterEach(() => vi.useRealTimers()); +it.each(["stop", "deadline"])( + "bounds %s while retaining the actual late transfer and scratch owner", + async (mode) => { + vi.useFakeTimers(); + const context = new NativeExecutionContext( + { + binding: { + companyId: randomUUID(), + runId: randomUUID(), + agentId: randomUUID(), + }, + } as NativeExecutionInput, + 1_000, + ); + let release!: () => void, + entered!: () => void, + root = "", + outcome = "pending"; + const gate = new Promise((done) => { + release = done; + }), + started = new Promise((done) => { + entered = done; + }); + const runner = { + execute: vi.fn(async () => { + throw new Error("unexpected provider command"); + }), + syncOut: async (operations) => { + const destination = operations[0].files[0].targetPath; + root = dirname(destination); + await writeFile(destination, "partial qualified download"); + entered(); + await gate; + await writeFile(destination, "late qualified download finished"); + return { operations: [] }; + }, + } as CommandManagedRuntimeRunner; + const manifest = { + payload: { + artifacts: Object.fromEntries( + ["nodeCommand", "opencodeProxy", "opencodeExecutable"].map((name) => [ + name, + { path: name }, + ]), + ), + }, + } as NativeProviderArtifactSource["manifest"]; + const publish = vi.fn(async () => "must not publish"); + const result = captureRemoteNativeProviderArtifacts( + { + runner, + manifest, + context, + signal: context.signal, + launch: { + root: "/owned/qualified", + accountUid: 1000, + paths: { + nodeCommand: "/qualified/node", + opencodeProxy: "/qualified/proxy", + opencodeExecutable: "/qualified/opencode", + }, + }, + }, + publish, + ).then( + () => { + outcome = "published"; + }, + (error) => { + outcome = error.message; + }, + ); + try { + await started; + if (mode === "stop") context.abort(new Error("controlled Stop")); + await vi.advanceTimersByTimeAsync(mode === "deadline" ? 1_000 : 0); + expect(outcome).toBe( + mode === "stop" + ? "controlled Stop" + : "native_execution_startup_expired", + ); + expect(context.pending).toBe(true); + expect((await stat(root)).isDirectory()).toBe(true); + expect(publish).not.toHaveBeenCalled(); + } finally { + release(); + await result; + vi.useRealTimers(); + await vi.waitFor(() => expect(context.pending).toBe(false), { + timeout: 1_000, + interval: 5, + }); + context.finish(); + } + expect(context.pending).toBe(false); + await expect(stat(root)).rejects.toMatchObject({ code: "ENOENT" }); + expect(publish).not.toHaveBeenCalled(); + }, +); diff --git a/server/src/services/native-runtime/native-provider-artifact-capture.ts b/server/src/services/native-runtime/native-provider-artifact-capture.ts new file mode 100644 index 00000000000..9c267ea2069 --- /dev/null +++ b/server/src/services/native-runtime/native-provider-artifact-capture.ts @@ -0,0 +1,92 @@ +import { logger } from "../../middleware/logger.js"; +import { randomUUID } from "node:crypto"; +import { mkdtemp, mkdir } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, dirname } from "node:path"; +import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; +import type { NativeProviderArtifactSource } from "./native-provider-artifacts.js"; +import type { OpenCodeLaunchArtifacts } from "./remote-opencode-artifacts.js"; +import type { NativeExecutionContext } from "./native-execution-context.js"; +import { removeNativeHarnessBackup } from "./native-harness-backup-cleanup.js"; + +export async function captureRemoteNativeProviderArtifacts( + input: { + runner: CommandManagedRuntimeRunner; + launch: OpenCodeLaunchArtifacts; + manifest: NativeProviderArtifactSource["manifest"]; + signal: AbortSignal; + context?: NativeExecutionContext; + }, + publish: (source: NativeProviderArtifactSource) => Promise, +): Promise { + const root = await mkdtemp(join(tmpdir(), "paperclip-provider-launch-")); + let ownedByOperation = false; + try { + const files = await Promise.all( + (["nodeCommand", "opencodeProxy", "opencodeExecutable"] as const).map( + async (name) => { + const targetPath = join( + root, + input.manifest.payload.artifacts[name].path, + ); + await mkdir(dirname(targetPath), { recursive: true, mode: 0o700 }); + return { + sourcePath: input.launch.paths[name], + targetPath, + kind: "file" as const, + mode: 0o500, + }; + }, + ), + ); + input.signal.throwIfAborted(); + const settled = input.context?.trackWorker(); + ownedByOperation = true; + const operation = Promise.resolve() + .then(async () => { + input.signal.throwIfAborted(); + input.context?.assertOpen(); + await input.runner.syncOut!([ + { operationId: `provider-artifacts-${randomUUID()}`, files }, + ]); + input.signal.throwIfAborted(); + input.context?.assertOpen(); + return publish({ + sourceRoot: root, + manifest: input.manifest, + launchRoot: input.launch.root, + accountUid: input.launch.accountUid, + }); + }) + .finally(async () => { + try { + await removeNativeHarnessBackup(root); + } catch (err) { + logger.warn( + { err }, + "native provider artifact capture cleanup failed", + ); + throw err; + } finally { + settled?.(); + } + }); + void operation.catch(() => undefined); + let abort!: () => void; + const stopped = new Promise((_resolve, reject) => { + abort = () => reject(input.signal.reason); + input.signal.addEventListener("abort", abort, { once: true }); + if (input.signal.aborted) abort(); + }); + try { + return await Promise.race([ + input.context ? input.context.wait(() => operation) : operation, + stopped, + ]); + } finally { + input.signal.removeEventListener("abort", abort); + } + } finally { + if (!ownedByOperation) await removeNativeHarnessBackup(root); + } +} diff --git a/server/src/services/native-runtime/native-provider-artifact-files.ts b/server/src/services/native-runtime/native-provider-artifact-files.ts new file mode 100644 index 00000000000..1fffb059e04 --- /dev/null +++ b/server/src/services/native-runtime/native-provider-artifact-files.ts @@ -0,0 +1,384 @@ +import { createHash } from "node:crypto"; +import { constants, createWriteStream } from "node:fs"; +import { lstat, open, readdir } from "node:fs/promises"; +import { join, posix } from "node:path"; +import { Transform } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import type { StorageService } from "../../storage/types.js"; +import { + canonicalProviderPackJson, + type ProviderPackIdentity, +} from "./remote-provider-pack-verification.js"; + +export const schema = "paperclip.native-provider-artifacts.v1"; +export const names = [ + "nodeCommand", + "opencodeProxy", + "opencodeExecutable", +] as const; +export const MAX_NATIVE_PROVIDER_ARTIFACT_BYTES = 512 * 1024 * 1024; +export type Name = (typeof names)[number]; +export type Entry = { + name: Name; + path: string; + sha256: string; + byteSize: number; +}; +export type Descriptor = { + schema: typeof schema; + target: ProviderPackIdentity["payload"]["target"]; + entries: Entry[]; +}; +export type NativeProviderArtifactSource = { + sourceRoot: string; + manifest: ProviderPackIdentity; + launchRoot: string; + accountUid: number; +}; +export type NativeProviderArtifactReceipt = { + schema: typeof schema; + artifactId: string; + closureDigest: string; + providerPackDigest: string; + launchRoot: string; + accountUid: number; + manifest: ProviderPackIdentity; +}; +export type BlobReference = { + schema: typeof schema; + companyId: string; + artifactId: string; + closureDigest: string; + objectKey: string; + sha256: string; + byteSize: number; + material: Record; +}; +export const digest = (value: string | Buffer) => + `sha256:${createHash("sha256").update(value).digest("hex")}`; +export const fail = (reason: string): never => { + throw new Error(`native_provider_artifacts_${reason}`); +}; +export const aad = ( + companyId: string, + artifactId: string, + closureDigest: string, +) => + Buffer.from(JSON.stringify([schema, companyId, artifactId, closureDigest])); +export const safeRelative = (value: unknown): value is string => + typeof value === "string" && + value.length > 0 && + value.length <= 4096 && + !value.includes("\\") && + !/[\u0000-\u001f\u007f]/u.test(value) && + !posix.isAbsolute(value) && + value + .split("/") + .every((part) => part !== ".." && part !== "." && part !== ""); +export const validDigest = (value: unknown): value is string => + typeof value === "string" && /^sha256:[a-f0-9]{64}$/.test(value); + +export function receiptFor( + artifactId: string, + closureDigest: string, + source: NativeProviderArtifactSource, +): NativeProviderArtifactReceipt { + return { + schema, + artifactId, + closureDigest, + launchRoot: source.launchRoot, + accountUid: source.accountUid, + providerPackDigest: source.manifest.digest, + manifest: structuredClone(source.manifest), + }; +} + +export function manifestIdentity(manifest: ProviderPackIdentity) { + if ( + manifest?.schema !== "paperclip-runner/remote-provider-pack/v1" || + !manifest.payload || + manifest.digest !== digest(canonicalProviderPackJson(manifest.payload)) + ) + fail("manifest_invalid"); + if ( + manifest.payload.target.platform !== "linux" || + !["x64", "arm64"].includes(manifest.payload.target.architecture) + ) + fail("target_invalid"); + for (const name of names) { + const artifact = manifest.payload.artifacts[name]; + if ( + !artifact || + !safeRelative(artifact.path) || + !validDigest(artifact.sha256) + ) + fail("manifest_invalid"); + } +} + +export function launchRootIdentity( + root: string, + manifest: ProviderPackIdentity, + accountUid: number, +) { + if (!Number.isSafeInteger(accountUid) || accountUid < 0) + fail("account_invalid"); + if ( + typeof root !== "string" || + root.length > 4096 || + !posix.isAbsolute(root) || + posix.normalize(root) !== root || + /[\\\u0000-\u001f\u007f]/u.test(root) || + !root.endsWith( + `/.paperclip-runtime/paperclip-runner/provider-artifacts/${manifest.digest.slice(7)}`, + ) + ) + fail("launch_root_invalid"); +} + +export function descriptorIdentity( + descriptor: Descriptor, + receipt: NativeProviderArtifactReceipt, +) { + if ( + descriptor.schema !== schema || + !Array.isArray(descriptor.entries) || + descriptor.entries.length !== names.length || + canonicalProviderPackJson( + descriptor.entries.map((entry) => entry.name).sort(), + ) !== canonicalProviderPackJson([...names].sort()) || + canonicalProviderPackJson(descriptor.target) !== + canonicalProviderPackJson(receipt.manifest.payload.target) + ) + fail("descriptor_invalid"); + let bytes = 0; + for (const entry of descriptor.entries) { + if ( + entry.path !== receipt.manifest.payload.artifacts[entry.name]?.path || + entry.sha256 !== receipt.manifest.payload.artifacts[entry.name]?.sha256 || + !Number.isSafeInteger(entry.byteSize) || + entry.byteSize < 0 + ) + fail("descriptor_invalid"); + bytes += entry.byteSize; + } + if (bytes > MAX_NATIVE_PROVIDER_ARTIFACT_BYTES) fail("size_limit"); +} + +export async function fileDigest(path: string, signal?: AbortSignal) { + const stat = await lstat(path); + if (!stat.isFile() || stat.isSymbolicLink()) fail("file_unsafe"); + const hash = createHash("sha256"); + let byteSize = 0; + const handle = await open( + path, + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, + ); + try { + for await (const bytes of handle.createReadStream({ + signal, + autoClose: false, + })) { + byteSize += bytes.length; + if (byteSize > MAX_NATIVE_PROVIDER_ARTIFACT_BYTES + 1024 * 1024) + fail("size_limit"); + hash.update(bytes); + } + } finally { + await handle.close(); + } + return { sha256: `sha256:${hash.digest("hex")}`, byteSize }; +} + +export async function readDescriptor(path: string): Promise { + const handle = await open( + path, + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, + ); + try { + const stat = await handle.stat(); + if (!stat.isFile() || stat.size > 512 * 1024) fail("descriptor_invalid"); + try { + return JSON.parse(await handle.readFile("utf8")) as Descriptor; + } catch { + return fail("descriptor_invalid"); + } + } finally { + await handle.close(); + } +} + +export function artifactReference( + value: Record, + companyId: string, + artifactId: string, + closureDigest: string, +): BlobReference { + const reference = value as BlobReference; + if ( + reference.schema !== schema || + reference.companyId !== companyId || + reference.artifactId !== artifactId || + reference.closureDigest !== closureDigest || + !validDigest(reference.sha256) || + !Number.isSafeInteger(reference.byteSize) || + reference.byteSize <= 0 || + reference.byteSize > MAX_NATIVE_PROVIDER_ARTIFACT_BYTES + 1024 * 1024 || + typeof reference.objectKey !== "string" || + !reference.objectKey.startsWith( + `${companyId}/native-provider-artifacts/${artifactId}/`, + ) || + !/^[a-f0-9-]{36}\.pcpa$/.test(reference.objectKey.split("/").at(-1)!) || + reference.objectKey.split("/").length !== 4 + ) + fail("reference_invalid"); + return reference; +} + +export async function verifyMaterialized( + root: string, + descriptor: Descriptor, + signal?: AbortSignal, +): Promise { + const stat = await lstat(root).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return null; + throw error; + }); + if (!stat) return false; + if ( + !stat.isDirectory() || + stat.isSymbolicLink() || + canonicalProviderPackJson((await readdir(root)).sort()) !== + canonicalProviderPackJson(["manifest.json", ...names].sort()) + ) + fail("cache_changed"); + if ( + canonicalProviderPackJson( + await readDescriptor(join(root, "manifest.json")), + ) !== canonicalProviderPackJson(descriptor) + ) + fail("cache_changed"); + for (const entry of descriptor.entries) { + const actual = await fileDigest(join(root, entry.name), signal); + if (actual.sha256 !== entry.sha256 || actual.byteSize !== entry.byteSize) + fail("cache_changed"); + } + return true; +} + +export async function capture( + source: NativeProviderArtifactSource, + destination: string, + signal?: AbortSignal, +): Promise { + manifestIdentity(source.manifest); + const root = await lstat(source.sourceRoot); + if (!root.isDirectory() || root.isSymbolicLink()) fail("source_unsafe"); + const entries: Entry[] = []; + let total = 0; + for (const name of names) { + signal?.throwIfAborted(); + const artifact = source.manifest.payload.artifacts[name]; + let path = source.sourceRoot; + for (const part of artifact.path.split("/")) { + path = join(path, part); + if ((await lstat(path)).isSymbolicLink()) fail("source_unsafe"); + } + const handle = await open( + path, + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, + ); + try { + const before = await handle.stat(); + total += before.size; + if (!before.isFile() || total > MAX_NATIVE_PROVIDER_ARTIFACT_BYTES) + fail("size_limit"); + const hash = createHash("sha256"); + let copied = 0; + const observe = new Transform({ + transform(bytes, _encoding, callback) { + copied += bytes.length; + hash.update(bytes); + callback( + copied > before.size + ? new Error("native_provider_artifacts_source_changed") + : null, + bytes, + ); + }, + }); + await pipeline( + handle.createReadStream({ autoClose: false }), + observe, + createWriteStream(join(destination, name), { + flags: "wx", + mode: 0o500, + }), + { signal }, + ); + const after = await handle.stat(); + if ( + before.ino !== after.ino || + before.dev !== after.dev || + before.size !== after.size || + before.mtimeMs !== after.mtimeMs || + before.ctimeMs !== after.ctimeMs || + copied !== before.size || + `sha256:${hash.digest("hex")}` !== artifact.sha256 + ) + fail("source_changed"); + entries.push({ + name, + path: artifact.path, + sha256: artifact.sha256, + byteSize: copied, + }); + } finally { + await handle.close(); + } + } + return { schema, target: source.manifest.payload.target, entries }; +} + +export async function receive( + storage: StorageService, + reference: BlobReference, + target: string, + signal?: AbortSignal, +) { + const object = await storage.getObject( + reference.companyId, + reference.objectKey, + { signal }, + ); + const hash = createHash("sha256"); + let bytes = 0; + const observe = new Transform({ + transform(chunk, _encoding, callback) { + hash.update(chunk); + bytes += chunk.length; + callback( + bytes > reference.byteSize + ? new Error("native_provider_artifacts_size_mismatch") + : null, + chunk, + ); + }, + }); + try { + await pipeline( + object.stream, + observe, + createWriteStream(target, { flags: "wx", mode: 0o600 }), + { signal }, + ); + if ( + bytes !== reference.byteSize || + `sha256:${hash.digest("hex")}` !== reference.sha256 + ) + fail("object_changed"); + } finally { + object.stream.destroy(); + } +} diff --git a/server/src/services/native-runtime/native-provider-artifacts.ts b/server/src/services/native-runtime/native-provider-artifacts.ts new file mode 100644 index 00000000000..d7ff6b6a815 --- /dev/null +++ b/server/src/services/native-runtime/native-provider-artifacts.ts @@ -0,0 +1,831 @@ +import { + schema, + names, + MAX_NATIVE_PROVIDER_ARTIFACT_BYTES, + digest, + fail, + aad, + validDigest, + receiptFor, + manifestIdentity, + launchRootIdentity, + descriptorIdentity, + fileDigest, + readDescriptor, + artifactReference, + verifyMaterialized, + capture, + receive, + type Descriptor, + type BlobReference, + type NativeProviderArtifactSource, + type NativeProviderArtifactReceipt, +} from "./native-provider-artifact-files.js"; +export { + MAX_NATIVE_PROVIDER_ARTIFACT_BYTES, + type NativeProviderArtifactSource, + type NativeProviderArtifactReceipt, +} from "./native-provider-artifact-files.js"; +import { + createCipheriv, + createDecipheriv, + randomBytes, + randomUUID, +} from "node:crypto"; +import { createReadStream, createWriteStream } from "node:fs"; +import { cp, mkdir, mkdtemp, rename, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pipeline } from "node:stream/promises"; +import { and, eq, inArray, not, sql } from "drizzle-orm"; +import { + environmentLeases, + heartbeatRuns, + nativeRunFinalizations, + nativeProviderArtifacts, + nativeProviderArtifactReferences, + type Db, +} from "@paperclipai/db"; +import type { StorageService } from "../../storage/types.js"; +import { createLocalEncryptedCipher } from "../../secrets/local-encrypted-provider.js"; +import { + captureNativeHarnessArchive, + restoreNativeHarnessArchive, +} from "./native-harness-backup-archive.js"; +import { + lockNativeRunnerCheckpointOwner, + validateNativeRunnerCheckpointOwner, + type NativeRunnerCheckpointFenceOwner, +} from "./native-runner-checkpoint.js"; +import { + canonicalProviderPackJson, + type ProviderPackIdentity, +} from "./remote-provider-pack-verification.js"; +import { hasRemoteTerminationReceipt } from "../remote-execution-termination.js"; + +async function recordRunReceipt( + tx: Parameters[0]>[0], + owner: NativeRunnerCheckpointFenceOwner, + receipt: NativeProviderArtifactReceipt, +) { + await tx + .update(heartbeatRuns) + .set({ + runnerProfileJson: sql`coalesce(${heartbeatRuns.runnerProfileJson}, '{}'::jsonb) || ${JSON.stringify({ nativeProviderArtifacts: receipt })}::jsonb`, + }) + .where( + and( + eq(heartbeatRuns.companyId, owner.companyId), + eq(heartbeatRuns.id, owner.authorityRunId), + ), + ); +} + +async function artifactOwnerFence( + tx: Parameters[0]>[0], + owner: NativeRunnerCheckpointFenceOwner, + manifest: ProviderPackIdentity, +) { + await validateNativeRunnerCheckpointOwner( + tx, + owner, + await lockNativeRunnerCheckpointOwner(tx, owner), + false, + ); + const [run] = await tx + .select({ + id: heartbeatRuns.id, + agentId: heartbeatRuns.agentId, + issueId: heartbeatRuns.nativeIssueId, + sessionId: heartbeatRuns.nativeSessionId, + driver: heartbeatRuns.driverKind, + profile: heartbeatRuns.runnerProfileJson, + }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, owner.companyId), + eq(heartbeatRuns.id, owner.authorityRunId), + ), + ) + .for("share"); + const receipt = run?.profile?.nativeProviderPackAdmission as + Record | undefined; + const state = run?.profile?.nativeProviderPackAdmissionState as + Record | undefined; + if ( + !run || + run.issueId !== owner.issueId || + run.driver !== "opencode_server" || + !run.sessionId || + state?.schema !== "paperclip.native-provider-pack-admission-state.v1" || + state.state !== "verified" || + receipt?.schema !== "paperclip.native-provider-pack-admission.v1" || + receipt.stage !== "session_admitted" || + receipt.authority !== "core_verified_selection" || + receipt.companyId !== owner.companyId || + receipt.runId !== run.id || + receipt.agentId !== run.agentId || + receipt.sessionId !== run.sessionId || + receipt.providerPackDigest !== manifest.digest || + receipt.sourceRevision !== manifest.payload.runnerSourceRevision || + receipt.opencodeProxySha256 !== + manifest.payload.artifacts.opencodeProxy.sha256 + ) + fail("admission_unverified"); +} + +export async function retainNativeProviderArtifacts(input: { + db: Db; + storage: StorageService; + owner: NativeRunnerCheckpointFenceOwner; + source: NativeProviderArtifactSource; + signal?: AbortSignal; +}): Promise { + const scratch = await mkdtemp(join(tmpdir(), "pc-provider-artifacts-")); + const files = join(scratch, "files"); + await mkdir(files, { mode: 0o700 }); + const key = randomBytes(32); + try { + manifestIdentity(input.source.manifest); + launchRootIdentity( + input.source.launchRoot, + input.source.manifest, + input.source.accountUid, + ); + await input.db.transaction((tx) => + artifactOwnerFence(tx, input.owner, input.source.manifest), + ); + const descriptor = await capture(input.source, files, input.signal); + const closureDigest = digest(canonicalProviderPackJson(descriptor)); + const uploadToken = randomUUID(); + const selected = await input.db.transaction(async (tx) => { + await artifactOwnerFence(tx, input.owner, input.source.manifest); + await tx.execute( + sql`select pg_advisory_xact_lock(hashtextextended(${`${input.owner.companyId}:${closureDigest}`}, 0))`, + ); + let [row] = await tx + .select() + .from(nativeProviderArtifacts) + .where( + and( + eq(nativeProviderArtifacts.companyId, input.owner.companyId), + eq(nativeProviderArtifacts.closureDigest, closureDigest), + eq(nativeProviderArtifacts.status, "ready"), + ), + ) + .for("update"); + if (!row) { + const pending = await tx + .select({ id: nativeProviderArtifacts.id }) + .from(nativeProviderArtifacts) + .innerJoin( + nativeProviderArtifactReferences, + eq( + nativeProviderArtifactReferences.artifactId, + nativeProviderArtifacts.id, + ), + ) + .where( + and( + eq(nativeProviderArtifacts.companyId, input.owner.companyId), + eq(nativeProviderArtifacts.status, "pending"), + eq( + nativeProviderArtifactReferences.companyId, + input.owner.companyId, + ), + eq(nativeProviderArtifactReferences.runId, input.owner.runId), + ), + ) + .limit(4); + if (pending.length >= 3) fail("pending_limit"); + [row] = await tx + .insert(nativeProviderArtifacts) + .values({ + companyId: input.owner.companyId, + closureDigest, + descriptor, + uploadToken, + }) + .returning(); + } + if ( + !row || + canonicalProviderPackJson(row.descriptor) !== + canonicalProviderPackJson(descriptor) + ) + fail("unavailable"); + await tx + .insert(nativeProviderArtifactReferences) + .values({ + companyId: input.owner.companyId, + artifactId: row.id, + runId: input.owner.runId, + }) + .onConflictDoNothing(); + if (row.status === "ready") + await recordRunReceipt( + tx, + input.owner, + receiptFor(row.id, closureDigest, input.source), + ); + return row; + }); + if (selected.status !== "ready") { + const archive = join(scratch, "archive"), + encrypted = join(scratch, "encrypted"); + await writeFile( + join(files, "manifest.json"), + JSON.stringify(descriptor), + { mode: 0o600 }, + ); + await captureNativeHarnessArchive( + files, + archive, + input.signal, + MAX_NATIVE_PROVIDER_ARTIFACT_BYTES + 1024 * 1024, + ); + const plaintext = await fileDigest(archive, input.signal), + iv = randomBytes(12); + const cipher = createCipheriv("aes-256-gcm", key, iv); + cipher.setAAD(aad(input.owner.companyId, selected.id, closureDigest)); + await pipeline( + createReadStream(archive), + cipher, + createWriteStream(encrypted, { flags: "wx", mode: 0o600 }), + { signal: input.signal }, + ); + const reference: BlobReference = { + schema, + companyId: input.owner.companyId, + artifactId: selected.id, + closureDigest, + objectKey: `${input.owner.companyId}/native-provider-artifacts/${selected.id}/${uploadToken}.pcpa`, + ...(await fileDigest(encrypted, input.signal)), + material: createLocalEncryptedCipher("write").encrypt( + JSON.stringify({ + schema, + companyId: input.owner.companyId, + artifactId: selected.id, + closureDigest, + key: key.toString("base64"), + iv: iv.toString("base64"), + tag: cipher.getAuthTag().toString("base64"), + plaintext, + }), + ), + }; + await input.db.transaction(async (tx) => { + await artifactOwnerFence(tx, input.owner, input.source.manifest); + const [owned] = await tx + .update(nativeProviderArtifacts) + .set({ reference, updatedAt: new Date() }) + .where( + and( + eq(nativeProviderArtifacts.companyId, input.owner.companyId), + eq(nativeProviderArtifacts.id, selected.id), + eq(nativeProviderArtifacts.uploadToken, uploadToken), + eq(nativeProviderArtifacts.status, "pending"), + ), + ) + .returning({ id: nativeProviderArtifacts.id }); + if (!owned) fail("owner_changed"); + }); + await input.storage.putFile({ + companyId: input.owner.companyId, + namespace: "native-provider-artifacts", + objectKey: reference.objectKey, + originalFilename: null, + contentType: "application/octet-stream", + body: createReadStream(encrypted), + byteSize: reference.byteSize, + sha256: reference.sha256.slice(7), + signal: input.signal, + }); + await receive( + input.storage, + reference, + join(scratch, "verified"), + input.signal, + ); + const published = await input.db.transaction(async (tx) => { + await artifactOwnerFence(tx, input.owner, input.source.manifest); + await tx.execute( + sql`select pg_advisory_xact_lock(hashtextextended(${`${input.owner.companyId}:${closureDigest}`}, 0))`, + ); + const [winner] = await tx + .select({ id: nativeProviderArtifacts.id }) + .from(nativeProviderArtifacts) + .where( + and( + eq(nativeProviderArtifacts.companyId, input.owner.companyId), + eq(nativeProviderArtifacts.closureDigest, closureDigest), + eq(nativeProviderArtifacts.status, "ready"), + ), + ) + .for("update"); + if (winner) { + await tx + .insert(nativeProviderArtifactReferences) + .values({ + companyId: input.owner.companyId, + artifactId: winner.id, + runId: input.owner.runId, + }) + .onConflictDoNothing(); + await recordRunReceipt( + tx, + input.owner, + receiptFor(winner.id, closureDigest, input.source), + ); + return winner.id; + } + const rows = await tx + .update(nativeProviderArtifacts) + .set({ + reference, + status: "ready", + uploadToken: null, + updatedAt: new Date(), + }) + .where( + and( + eq(nativeProviderArtifacts.companyId, input.owner.companyId), + eq(nativeProviderArtifacts.id, selected.id), + eq(nativeProviderArtifacts.uploadToken, uploadToken), + eq(nativeProviderArtifacts.status, "pending"), + ), + ) + .returning({ id: nativeProviderArtifacts.id }); + if (rows.length) + await recordRunReceipt( + tx, + input.owner, + receiptFor(selected.id, closureDigest, input.source), + ); + return rows[0]?.id ?? null; + }); + if (!published) { + await input.storage.deleteObject( + input.owner.companyId, + reference.objectKey, + { allVersions: true, signal: input.signal }, + ); + fail("owner_changed"); + } + if (published !== selected.id) { + await input.storage.deleteObject( + input.owner.companyId, + reference.objectKey, + { allVersions: true, signal: input.signal }, + ); + await input.db.transaction(async (tx) => { + await tx + .delete(nativeProviderArtifactReferences) + .where( + and( + eq( + nativeProviderArtifactReferences.companyId, + input.owner.companyId, + ), + eq(nativeProviderArtifactReferences.artifactId, selected.id), + ), + ); + await tx + .delete(nativeProviderArtifacts) + .where( + and( + eq(nativeProviderArtifacts.companyId, input.owner.companyId), + eq(nativeProviderArtifacts.id, selected.id), + eq(nativeProviderArtifacts.status, "pending"), + ), + ); + }); + selected.id = published; + } + } + return receiptFor(selected.id, closureDigest, input.source); + } finally { + key.fill(0); + await rm(scratch, { recursive: true, force: true }); + } +} + +export async function retainExistingNativeProviderArtifacts(input: { + db: Db; + owner: NativeRunnerCheckpointFenceOwner; + receipt: NativeProviderArtifactReceipt; +}) { + manifestIdentity(input.receipt.manifest); + launchRootIdentity( + input.receipt.launchRoot, + input.receipt.manifest, + input.receipt.accountUid, + ); + if ( + input.receipt.schema !== schema || + input.receipt.providerPackDigest !== input.receipt.manifest.digest + ) + fail("receipt_invalid"); + await input.db.transaction(async (tx) => { + await artifactOwnerFence(tx, input.owner, input.receipt.manifest); + const [row] = await tx + .select() + .from(nativeProviderArtifacts) + .where( + and( + eq(nativeProviderArtifacts.companyId, input.owner.companyId), + eq(nativeProviderArtifacts.id, input.receipt.artifactId), + eq( + nativeProviderArtifacts.closureDigest, + input.receipt.closureDigest, + ), + eq(nativeProviderArtifacts.status, "ready"), + ), + ) + .for("update"); + if ( + !row || + digest(canonicalProviderPackJson(row.descriptor)) !== + input.receipt.closureDigest + ) + fail("unavailable"); + descriptorIdentity(row.descriptor as Descriptor, input.receipt); + await tx + .insert(nativeProviderArtifactReferences) + .values({ + companyId: input.owner.companyId, + artifactId: row.id, + runId: input.owner.runId, + }) + .onConflictDoNothing(); + await recordRunReceipt(tx, input.owner, input.receipt); + }); +} + +export async function restoreNativeProviderArtifacts(input: { + db: Db; + storage: StorageService; + companyId: string; + receipt: NativeProviderArtifactReceipt; + targetRoot: string; + signal?: AbortSignal; +}) { + const receipt = input.receipt; + if (receipt?.schema !== schema || !validDigest(receipt.closureDigest)) + fail("receipt_invalid"); + manifestIdentity(receipt.manifest); + launchRootIdentity(receipt.launchRoot, receipt.manifest, receipt.accountUid); + if (receipt.providerPackDigest !== receipt.manifest.digest) + fail("receipt_invalid"); + const [row] = await input.db + .select() + .from(nativeProviderArtifacts) + .where( + and( + eq(nativeProviderArtifacts.companyId, input.companyId), + eq(nativeProviderArtifacts.id, receipt.artifactId), + eq(nativeProviderArtifacts.closureDigest, receipt.closureDigest), + eq(nativeProviderArtifacts.status, "ready"), + ), + ); + if (!row?.reference) fail("unavailable"); + const reference = artifactReference( + row.reference!, + input.companyId, + row.id, + receipt.closureDigest, + ); + const expectedDescriptor = row.descriptor as Descriptor; + if ( + digest(canonicalProviderPackJson(expectedDescriptor)) !== + receipt.closureDigest + ) + fail("descriptor_invalid"); + descriptorIdentity(expectedDescriptor, receipt); + if ( + await verifyMaterialized(input.targetRoot, expectedDescriptor, input.signal) + ) + return; + const wrapped = JSON.parse( + createLocalEncryptedCipher("read").decrypt(reference.material), + ); + if ( + wrapped.schema !== schema || + wrapped.companyId !== input.companyId || + wrapped.artifactId !== row.id || + wrapped.closureDigest !== receipt.closureDigest + ) + fail("binding_mismatch"); + const key = Buffer.from(wrapped.key, "base64"); + const scratch = await mkdtemp( + join(tmpdir(), "pc-provider-artifacts-restore-"), + ); + try { + if (key.length !== 32) fail("key_invalid"); + const encrypted = join(scratch, "encrypted"), + archive = join(scratch, "archive"), + restored = join(scratch, "restored"); + await receive(input.storage, reference, encrypted, input.signal); + const decipher = createDecipheriv( + "aes-256-gcm", + key, + Buffer.from(wrapped.iv, "base64"), + ); + decipher.setAAD(aad(input.companyId, row.id, receipt.closureDigest)); + decipher.setAuthTag(Buffer.from(wrapped.tag, "base64")); + await pipeline( + createReadStream(encrypted), + decipher, + createWriteStream(archive, { flags: "wx", mode: 0o600 }), + { signal: input.signal }, + ); + const actual = await fileDigest(archive, input.signal); + if ( + actual.sha256 !== wrapped.plaintext?.sha256 || + actual.byteSize !== wrapped.plaintext?.byteSize + ) + fail("object_changed"); + await restoreNativeHarnessArchive(archive, restored, input.signal); + const descriptor = await readDescriptor(join(restored, "manifest.json")); + if ( + digest(canonicalProviderPackJson(descriptor)) !== receipt.closureDigest || + canonicalProviderPackJson(descriptor) !== + canonicalProviderPackJson(row.descriptor) + ) + fail("descriptor_invalid"); + for (const entry of descriptor.entries) { + if ( + !names.includes(entry.name) || + entry.path !== receipt.manifest.payload.artifacts[entry.name]?.path || + entry.sha256 !== receipt.manifest.payload.artifacts[entry.name]?.sha256 + ) + fail("descriptor_invalid"); + const actual = await fileDigest(join(restored, entry.name), input.signal); + if (actual.sha256 !== entry.sha256 || actual.byteSize !== entry.byteSize) + fail("object_changed"); + } + const staging = `${input.targetRoot}.${randomUUID()}.pending`; + try { + await cp(restored, staging, { + recursive: true, + force: false, + errorOnExist: true, + }); + await verifyMaterialized(staging, expectedDescriptor, input.signal); + input.signal?.throwIfAborted(); + try { + await rename(staging, input.targetRoot); + } catch (error) { + if ( + !["EEXIST", "ENOTEMPTY"].includes( + (error as NodeJS.ErrnoException).code ?? "", + ) + ) + throw error; + if ( + !(await verifyMaterialized( + input.targetRoot, + expectedDescriptor, + input.signal, + )) + ) + throw error; + } + } finally { + await rm(staging, { recursive: true, force: true }); + } + } finally { + key.fill(0); + await rm(scratch, { recursive: true, force: true }); + } +} + +export async function pinNativeProviderArtifactsForBackup( + tx: Parameters[0]>[0], + input: { + companyId: string; + runId: string; + backupId: string; + receipt: NativeProviderArtifactReceipt; + }, +) { + const [artifact] = await tx + .select({ id: nativeProviderArtifacts.id }) + .from(nativeProviderArtifacts) + .where( + and( + eq(nativeProviderArtifacts.companyId, input.companyId), + eq(nativeProviderArtifacts.id, input.receipt.artifactId), + eq(nativeProviderArtifacts.closureDigest, input.receipt.closureDigest), + eq(nativeProviderArtifacts.status, "ready"), + ), + ) + .for("update"); + if (!artifact) fail("unavailable"); + await tx + .insert(nativeProviderArtifactReferences) + .values({ + companyId: input.companyId, + artifactId: artifact.id, + runId: input.runId, + backupId: input.backupId, + }) + .onConflictDoNothing(); +} + +export async function releaseRetiredNativeProviderArtifacts(input: { + db: Db; + storage: StorageService; + companyId: string; + runId: string; + backupId?: string; + receipt: Pick; + signal?: AbortSignal; +}) { + const deletion = await input.db.transaction(async (tx) => { + const [owner] = await tx + .select() + .from(nativeRunFinalizations) + .where( + and( + eq(nativeRunFinalizations.companyId, input.companyId), + eq(nativeRunFinalizations.runId, input.runId), + ), + ) + .for("share"); + const [run] = await tx + .select() + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, input.companyId), + eq(heartbeatRuns.id, input.runId), + ), + ) + .for("share"); + const [artifact] = await tx + .select() + .from(nativeProviderArtifacts) + .where( + and( + eq(nativeProviderArtifacts.companyId, input.companyId), + eq(nativeProviderArtifacts.id, input.receipt.artifactId), + ), + ) + .for("update"); + if (!artifact) return null; + if (artifact.closureDigest !== input.receipt.closureDigest) + fail("binding_mismatch"); + const leases = await tx + .select() + .from(environmentLeases) + .where( + and( + eq(environmentLeases.companyId, input.companyId), + eq(environmentLeases.heartbeatRunId, input.runId), + ), + ) + .limit(101); + if ( + !run?.finishedAt || + ![ + "succeeded", + "failed", + "cancelled", + "interrupted", + "timed_out", + ].includes(run.status) || + !owner || + !["committed", "terminal_failure"].includes(owner.phase) || + (owner.leaseOwner && + (!owner.leaseExpiresAt || owner.leaseExpiresAt > new Date())) || + !leases.length || + leases.length > 100 || + leases.some((lease) => !hasRemoteTerminationReceipt(lease)) + ) + fail("retained_owner_unsettled"); + const ownedReferences = and( + eq(nativeProviderArtifactReferences.companyId, input.companyId), + eq(nativeProviderArtifactReferences.artifactId, artifact.id), + eq(nativeProviderArtifactReferences.runId, input.runId), + input.backupId + ? sql`${nativeProviderArtifactReferences.backupId} is not distinct from ${input.backupId}::uuid` + : sql`${nativeProviderArtifactReferences.backupId} is null`, + ); + const [retained] = await tx + .select({ id: nativeProviderArtifactReferences.id }) + .from(nativeProviderArtifactReferences) + .where( + and( + eq(nativeProviderArtifactReferences.companyId, input.companyId), + eq(nativeProviderArtifactReferences.artifactId, artifact.id), + not(ownedReferences!), + ), + ) + .limit(1); + if (retained) { + await tx.delete(nativeProviderArtifactReferences).where(ownedReferences); + return null; + } + const pins = await tx + .select({ id: nativeProviderArtifactReferences.id }) + .from(nativeProviderArtifactReferences) + .where(ownedReferences); + await tx + .update(nativeProviderArtifacts) + .set({ status: "retiring", updatedAt: new Date() }) + .where(eq(nativeProviderArtifacts.id, artifact.id)); + return { + id: artifact.id, + reference: artifact.reference + ? artifactReference( + artifact.reference, + input.companyId, + artifact.id, + artifact.closureDigest, + ) + : null, + pins: pins.map((pin) => pin.id), + }; + }); + if (!deletion) return; + if (deletion.reference) + await input.storage.deleteObject( + input.companyId, + deletion.reference.objectKey, + { allVersions: true, signal: input.signal }, + ); + await input.db.transaction(async (tx) => { + const [artifact] = await tx + .select({ id: nativeProviderArtifacts.id }) + .from(nativeProviderArtifacts) + .where( + and( + eq(nativeProviderArtifacts.companyId, input.companyId), + eq(nativeProviderArtifacts.id, deletion.id), + eq(nativeProviderArtifacts.status, "retiring"), + ), + ) + .for("update"); + if (!artifact) return; + if (deletion.pins.length) + await tx + .delete(nativeProviderArtifactReferences) + .where( + and( + eq(nativeProviderArtifactReferences.companyId, input.companyId), + eq(nativeProviderArtifactReferences.artifactId, artifact.id), + inArray(nativeProviderArtifactReferences.id, deletion.pins), + ), + ); + const [retained] = await tx + .select({ id: nativeProviderArtifactReferences.id }) + .from(nativeProviderArtifactReferences) + .where( + and( + eq(nativeProviderArtifactReferences.companyId, input.companyId), + eq(nativeProviderArtifactReferences.artifactId, artifact.id), + ), + ) + .limit(1); + if (retained) fail("retirement_conflict"); + await tx + .delete(nativeProviderArtifacts) + .where(eq(nativeProviderArtifacts.id, artifact.id)); + }); +} + +export async function releaseExpiredNativeProviderArtifactRun(input: { + db: Db; + storage: StorageService; + companyId: string; + runId: string; + signal?: AbortSignal; +}) { + const references = await input.db + .select({ + artifactId: nativeProviderArtifacts.id, + closureDigest: nativeProviderArtifacts.closureDigest, + }) + .from(nativeProviderArtifactReferences) + .innerJoin( + nativeProviderArtifacts, + eq( + nativeProviderArtifacts.id, + nativeProviderArtifactReferences.artifactId, + ), + ) + .where( + and( + eq(nativeProviderArtifactReferences.companyId, input.companyId), + eq(nativeProviderArtifacts.companyId, input.companyId), + eq(nativeProviderArtifactReferences.runId, input.runId), + sql`${nativeProviderArtifactReferences.backupId} is null`, + ), + ) + .limit(101); + if (references.length > 100) fail("retirement_limit"); + for (const receipt of references) + await releaseRetiredNativeProviderArtifacts({ ...input, receipt }); +} diff --git a/server/src/services/native-runtime/native-provider-pack-admission.test.ts b/server/src/services/native-runtime/native-provider-pack-admission.test.ts index b5380d173ac..5d5af36841e 100644 --- a/server/src/services/native-runtime/native-provider-pack-admission.test.ts +++ b/server/src/services/native-runtime/native-provider-pack-admission.test.ts @@ -1,6 +1,6 @@ import { createHash, randomUUID } from "node:crypto"; -import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import { eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { eq, sql } from "drizzle-orm"; import { agents, companies, createDb, heartbeatRuns, issues, nativeRunFinalizations, type Db } from "@paperclipai/db"; import type { NativeSession } from "../../vendor/paperclip-runner/index.js"; import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; @@ -226,4 +226,33 @@ describe("provider pack admission PostgreSQL fencing", () => { } finally { release(); await owner; } await expect(recordNativeProviderPackAdmission(db, f)).resolves.toBe(true); }); + it.each([false, true])("waits within the durable-capture bound and rechecks the claim after its lock (revoked: %s)", async revoked => { + const f = await fixture(); + let release!: () => void, acquired!: () => void; + const holding = new Promise(resolve => { acquired = resolve; }); + const gate = new Promise(resolve => { release = resolve; }); + const owner = db.transaction(async tx => { + await tx.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.identity.runId)).for("update"); + acquired(); await gate; + if (revoked) await tx.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 1000) }) + .where(eq(nativeRunFinalizations.runId, f.identity.runId)); + }); + let settled = false; + await holding; + const recording = recordNativeProviderPackAdmission(db, { ...f, waitForClaim: true } as Parameters[1]) + .then(value => { settled = true; return { value }; }, error => { settled = true; return { error }; }); + try { + await vi.waitFor(async () => { + const [row] = await db.execute<{ waiting: boolean }>(sql`select exists(select 1 from pg_stat_activity + where datname = current_database() and wait_event_type = 'Lock' and state = 'active' + and query like '%native_run_finalizations%') as waiting`); + expect(settled || row.waiting).toBe(true); + }, { timeout: 800 }); + expect(settled).toBe(false); + } finally { release(); await owner; } + if (revoked) { + expect(await recording).toHaveProperty("error"); + expect((await read(f.identity.runId)).runnerProfileJson?.nativeProviderPackAdmissionState).toEqual(f.profile.nativeProviderPackAdmissionState); + } else expect(await recording).toEqual({ value: true }); + }); }); diff --git a/server/src/services/native-runtime/native-provider-pack-admission.ts b/server/src/services/native-runtime/native-provider-pack-admission.ts index b50a57bbfef..8476f852b10 100644 --- a/server/src/services/native-runtime/native-provider-pack-admission.ts +++ b/server/src/services/native-runtime/native-provider-pack-admission.ts @@ -3,6 +3,7 @@ import { createHash } from "node:crypto"; import { heartbeatRuns, nativeRunFinalizations, type Db } from "@paperclipai/db"; import type { NativeSession } from "../../vendor/paperclip-runner/index.js"; import { canonicalProviderPackJson, type ProviderPackIdentity } from "./remote-provider-pack-verification.js"; +import { assertNativeRunnerCheckpointOwner } from "./native-runner-checkpoint.js"; type Selection = { admittedRunId: string; @@ -29,7 +30,7 @@ const admitted = new WeakMap>(); const digest = /^sha256:[a-f0-9]{64}$/; type NativeRunIdentity = ReturnType; -export type NativeProviderPackAdmissionSource = { runId: string; receipt: unknown; state: unknown }; +export type NativeProviderPackAdmissionSource = { runId: string; receipt: unknown; state: unknown; artifacts?: unknown }; export function nativeProviderPackAdmissionPending(admissionEpoch: string) { return { schema: "paperclip.native-provider-pack-admission-state.v1" as const, state: "pending" as const, admissionEpoch }; @@ -127,9 +128,17 @@ export async function recordNativeProviderPackAdmission(db: Db, input: { identity: NativeRunIdentity; admissionEpoch: string; claim: { leaseOwner: string; attempt: number; controllerGeneration: number; controllerBootId: string }; + waitForClaim?: boolean; }): Promise { const receipt = admittedNativeProviderPack(input.session, input.identity); return db.transaction(async tx => { + if (input.waitForClaim) { + await tx.execute(sql`select set_config('lock_timeout', '1000ms', true)`); + await assertNativeRunnerCheckpointOwner(tx, { companyId: input.identity.companyId, + issueId: input.identity.issueId, runId: input.identity.runId, authorityRunId: input.identity.runId, + bootId: input.claim.controllerBootId, leaseOwner: input.claim.leaseOwner, + generation: input.claim.controllerGeneration }, false); + } const [owner] = await tx.select({ runId: nativeRunFinalizations.runId }).from(nativeRunFinalizations).where(and( eq(nativeRunFinalizations.runId, input.identity.runId), eq(nativeRunFinalizations.companyId, input.identity.companyId), eq(nativeRunFinalizations.issueId, input.identity.issueId), eq(nativeRunFinalizations.leaseOwner, input.claim.leaseOwner), diff --git a/server/src/services/native-runtime/native-runner-checkpoint-retirement.test.ts b/server/src/services/native-runtime/native-runner-checkpoint-retirement.test.ts new file mode 100644 index 00000000000..371245eacd0 --- /dev/null +++ b/server/src/services/native-runtime/native-runner-checkpoint-retirement.test.ts @@ -0,0 +1,49 @@ +import { randomUUID } from "node:crypto"; +import { expect, it } from "vitest"; +import { + nativeCheckpointRetirement, + readNativeCheckpointRetirement, +} from "./native-runner-checkpoint-retirement.js"; + +it.each([ + "invalid_uuid", + "foreign_company", + "foreign_run", + "duplicate", + "overflow", + "missing_agent", + "future_schema", + "invalid_time", + "unknown_key", +])("refuses malformed or foreign checkpoint retirement: %s", (kind) => { + const companyId = randomUUID(), + runId = randomUUID(); + const value: Record = nativeCheckpointRetirement({ + companyId, + runIds: [runId], + agentId: randomUUID(), + sessionId: randomUUID(), + }); + if (kind === "invalid_uuid") value.agentId = "------------------------------------"; + if (kind === "foreign_company") value.companyId = randomUUID(); + if (kind === "foreign_run") value.runIds = [randomUUID()]; + if (kind === "duplicate") value.runIds = [runId, runId]; + if (kind === "overflow") + value.runIds = [runId, ...Array.from({ length: 100 }, () => randomUUID())]; + if (kind === "missing_agent") delete value.agentId; + if (kind === "future_schema") + value.schema = "paperclip.native-runner-checkpoint-retirement.v2"; + if (kind === "invalid_time") value.retiredAt = "not a timestamp"; + if (kind === "unknown_key") value.extra = true; + expect(() => + readNativeCheckpointRetirement(value, { companyId, runId }), + ).toThrow("native_runner_checkpoint_retirement_invalid"); +}); +it("keeps ordinary encrypted checkpoint materials outside the retirement contract", () => { + expect( + readNativeCheckpointRetirement( + { provider: "local_encrypted", ciphertext: "existing material" }, + { companyId: randomUUID(), runId: randomUUID() }, + ), + ).toBeNull(); +}); diff --git a/server/src/services/native-runtime/native-runner-checkpoint-retirement.ts b/server/src/services/native-runtime/native-runner-checkpoint-retirement.ts new file mode 100644 index 00000000000..880383d7f47 --- /dev/null +++ b/server/src/services/native-runtime/native-runner-checkpoint-retirement.ts @@ -0,0 +1,60 @@ +const schema = "paperclip.native-runner-checkpoint-retirement.v1"; +export const MAX_NATIVE_CHECKPOINT_RETIREMENT_RUNS = 100; +export type NativeCheckpointRetirement = { + schema: typeof schema; + companyId: string; + agentId: string; + sessionId: string; + runIds: string[]; + retiredAt: string; +}; +const uuid = /^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/; +export function readNativeCheckpointRetirement( + value: unknown, + binding: { companyId: string; runId: string }, +): NativeCheckpointRetirement | null { + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const record = value as Record; + if ( + typeof record.schema !== "string" || + !record.schema.startsWith("paperclip.native-runner-checkpoint-retirement.") + ) + return null; + if ( + record.schema !== schema || + Object.keys(record).sort().join(",") !== + "agentId,companyId,retiredAt,runIds,schema,sessionId" || + record.companyId !== binding.companyId || + typeof record.agentId !== "string" || + !uuid.test(record.agentId) || + typeof record.sessionId !== "string" || + !record.sessionId || + record.sessionId.length > 4096 || + !Array.isArray(record.runIds) || + !record.runIds.length || + record.runIds.length > MAX_NATIVE_CHECKPOINT_RETIREMENT_RUNS || + !record.runIds.every((id) => typeof id === "string" && uuid.test(id)) || + new Set(record.runIds).size !== record.runIds.length || + !record.runIds.includes(binding.runId) || + typeof record.retiredAt !== "string" || + !Number.isFinite(Date.parse(record.retiredAt)) + ) { + throw new Error("native_runner_checkpoint_retirement_invalid"); + } + return record as NativeCheckpointRetirement; +} +export function nativeCheckpointRetirement( + input: Omit, +): NativeCheckpointRetirement { + const record: NativeCheckpointRetirement = { + ...input, + runIds: [...input.runIds], + schema, + retiredAt: new Date().toISOString(), + }; + readNativeCheckpointRetirement(record, { + companyId: input.companyId, + runId: input.runIds[0], + }); + return record; +} diff --git a/server/src/services/native-runtime/native-runner-checkpoint.ts b/server/src/services/native-runtime/native-runner-checkpoint.ts index f72e9123c6c..fce1a2a1f34 100644 --- a/server/src/services/native-runtime/native-runner-checkpoint.ts +++ b/server/src/services/native-runtime/native-runner-checkpoint.ts @@ -1,3 +1,4 @@ +import { readNativeCheckpointRetirement } from "./native-runner-checkpoint-retirement.js"; import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js"; import { and, eq, exists, gt, isNull, lte, inArray, notExists, or, sql } from "drizzle-orm"; import { checkpointDb, heartbeatRuns, issues, nativeRunFinalizations, nativeRunnerCheckpoints, type Db } from "@paperclipai/db"; @@ -127,6 +128,12 @@ export function nativeRunnerCheckpointWriter(db: Db, owner: NativeRunnerCheckpoi await assertNativeSourceCheckpointConsumer(tx, owner, checkpoint); } await assertNativeRunnerCheckpointOwner(tx, owner, checkpoint.workCommandIds.some(id => !previousWork.has(id))); + if (checkpoint.reset || checkpoint.changedRecords.has("header") || checkpoint.removedRecords.includes("header")) { + const [header] = await tx.select({ material: nativeRunnerCheckpoints.material }).from(nativeRunnerCheckpoints).where(and( + eq(nativeRunnerCheckpoints.companyId, owner.companyId), eq(nativeRunnerCheckpoints.runId, owner.runId), + eq(nativeRunnerCheckpoints.part, "header"))).for("share").limit(1); + if (header && readNativeCheckpointRetirement(header.material, owner)) throw new Error("native_runner_checkpoint_retired"); + } if (checkpoint.predecessor) { const [prior] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( eq(heartbeatRuns.id, checkpoint.predecessor.runId), eq(heartbeatRuns.companyId, owner.companyId), @@ -216,6 +223,7 @@ export async function loadNativeRunnerCheckpointHeader(db: Pick, b eq(nativeRunnerCheckpoints.companyId, binding.companyId), eq(nativeRunnerCheckpoints.issueId, binding.issueId), eq(nativeRunnerCheckpoints.runId, binding.runId), eq(nativeRunnerCheckpoints.part, "header"))).limit(1); if (!row) return null; + if (readNativeCheckpointRetirement(row.material, binding)) throw new Error("native_runner_checkpoint_retired"); try { return decryptPart(createLocalEncryptedCipher("read"), binding, "header", row.material); } catch { return null; } } @@ -225,6 +233,8 @@ export async function loadNativeRunnerCheckpoint(db: Pick, binding eq(nativeRunnerCheckpoints.companyId, binding.companyId), eq(nativeRunnerCheckpoints.issueId, binding.issueId), eq(nativeRunnerCheckpoints.runId, binding.runId))); if (!rows.length) return null; + const header = rows.find(row => row.part === "header"); + if (header && readNativeCheckpointRetirement(header.material, binding)) throw new Error("native_runner_checkpoint_retired"); const cipher = createLocalEncryptedCipher("read"); const parts = new Map(rows.map(row => [row.part, decryptPart(cipher, binding, row.part, row.material)] as const)); diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index d140b35a031..21159604164 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -116,6 +116,8 @@ import { acknowledgedNativeStopExecutionHasStopped, isAcknowledgedNativeStop } f import { admittedNativeProviderPack, nativeProviderPackAdmissionPending, rememberAdmittedNativeProviderPack } from "./native-provider-pack-admission.js"; import { canonicalProviderPackJson } from "./remote-provider-pack-verification.js"; import * as providerPackAdmission from "./native-provider-pack-admission.js"; +import * as providerArtifactStorage from "./native-provider-artifacts.js"; +import * as providerArtifactCapture from "./native-provider-artifact-capture.js"; import * as nativeExecutionDriver from "./native-runner-execution-driver.js"; vi.mock("node:fs", async (importOriginal) => { @@ -1880,7 +1882,8 @@ describe("remote provider pack manifest", () => { return JSON.stringify(value); }; - it.each(["opencode", "opencode-continuation", "opencode-continuation-missing", "opencode-continuation-no-receipt", "opencode-continuation-replacement", "opencode-dirty", "opencode-diagnostics", "opencode-diagnostics-stale", "cursor", "claude-x64", "claude-arm64"] as const)("uses the bundled Linux image authority for remote %s without controller overrides", async provider => { + it.each(["opencode", "opencode-new-durable", "opencode-continuation", "opencode-continuation-missing", "opencode-continuation-no-receipt", "opencode-continuation-replacement", "opencode-continuation-cold", "opencode-continuation-legacy-durable", "opencode-dirty", "opencode-diagnostics", "opencode-diagnostics-stale", "cursor", "claude-x64", "claude-arm64"] as const)("uses the bundled Linux image authority for remote %s without controller overrides", async provider => { + const firstDurable = provider === "opencode-new-durable"; const architecture = provider === "claude-arm64" ? "arm64" : "x64"; const claude = provider.startsWith("claude-"); const opencode = provider.startsWith("opencode"); @@ -1892,7 +1895,7 @@ describe("remote provider pack manifest", () => { productionLock: "pnpm-lock.yaml", opencodeCommand: "node_modules/.bin/opencode", opencodeExecutable: "node_modules/opencode-ai/bin/opencode.exe", opencodeProxy: "dist/cli/opencode-app-server-proxy.cjs", acpxSidecar: "dist/cli/acpx-runtime-sidecar.cjs", - }).map(([key, path]) => [key, { path, sha256: digest }])) as Record<"nodeCommand" | "opencodeCommand" | "opencodeProxy", { path: string; sha256: string }>; + }).map(([key, path]) => [key, { path, sha256: digest }])) as Record<"nodeCommand" | "opencodeCommand" | "opencodeExecutable" | "opencodeProxy", { path: string; sha256: string }>; const payload = { compatibility: { opencode: 1, acpx: 1 }, contractDigest: digest, target: { platform: "linux", architecture }, runnerSourceRevision: "a".repeat(40), @@ -1931,6 +1934,44 @@ describe("remote provider pack manifest", () => { if (provider !== "opencode-diagnostics") installed.payload.artifacts.opencodeProxy.sha256 = "sha256:" + "e".repeat(64); installed.digest = "sha256:" + createHash("sha256").update(canonical(installed.payload)).digest("hex"); } + const launchRoot = `/home/runtime-account/.paperclip-runtime/paperclip-runner/provider-artifacts/${installed.digest.slice(7)}`; + const newReceipt = { schema: "paperclip.native-provider-artifacts.v1" as const, artifactId: randomUUID(), + closureDigest: "sha256:" + "9".repeat(64), providerPackDigest: installed.digest, manifest: installed, launchRoot, accountUid: 1000 }; + const providerState = { schema: "paperclip.runner.codex-provider-state.v1", lifecycle: "prepared", threadId: "new-opencode-thread", + providerSessionId: null, activeProviderTurnId: null, config: { provider: "opencode", driver: "opencode_server" } }; + const suspendedState = durableRunnerState({ runId: remoteExecution.binding.runId, runnerInstanceId: "bundled-image-runner", + environmentLeaseId: "lease", normalizedSessionId: remoteExecution.session.normalizedSessionId }, "suspended"); + const syncOut = vi.fn(async (operations: Array<{ files: Array<{ kind: string; targetPath: string }> }>) => { + for (const operation of operations) for (const file of operation.files) { + if (file.kind === "directory") { + await mkdir(file.targetPath, { recursive: true }); + await writeFile(join(file.targetPath, "runner-state.json"), JSON.stringify(suspendedState)); + await writeFile(join(file.targetPath, "codex-provider-state.json"), JSON.stringify(providerState)); + } else { await mkdir(dirname(file.targetPath), { recursive: true }); await writeFile(file.targetPath, "qualified launch fixture"); } + } + }); + const preparedDb = leaseDb(remoteExecution), select = preparedDb.select.bind(preparedDb); + if (firstDurable) preparedDb.select = ((fields?: Record) => ({ from: (table: unknown) => { + if (table !== environmentLeases) return select(fields as never).from(table as never); + const rows = [{ metadata: {}, providerLeaseId: "new-session-sandbox", companyId: remoteExecution.binding.companyId, + heartbeatRunId: remoteExecution.binding.runId }]; + const query = { then: Promise.resolve(rows).then.bind(Promise.resolve(rows)), where: () => query, limit: () => query }; + return query; + } })) as unknown as Db["select"]; + const driver = vi.spyOn(nativeExecutionDriver, "createNativeRunnerExecutionDriver"); + onTestFinished(() => driver.mockRestore()); + if (firstDurable) { + const storage = vi.spyOn(harnessStorage, "durableNativeHarnessStorage").mockReturnValue({ provider: "s3" } as never); + const owner = vi.spyOn(harnessStorage, "nativeHarnessBackupOwner").mockResolvedValue({ runId: remoteExecution.binding.runId } as never); + const validate = vi.spyOn(harnessStorage, "assertNativeHarnessBackupOwner").mockResolvedValue(undefined); + const retain = vi.spyOn(providerArtifactStorage, "retainNativeProviderArtifacts").mockResolvedValue(newReceipt); + const capture = vi.spyOn(providerArtifactCapture, "captureRemoteNativeProviderArtifacts"); + const backup = vi.spyOn(harnessStorage, "persistNativeHarnessBackup").mockImplementation(async input => { + expect(JSON.parse(await readFile(join(input.root, "manifest.json"), "utf8")).providerArtifacts).toEqual(newReceipt); + return undefined as never; + }); + onTestFinished(() => { storage.mockRestore(); owner.mockRestore(); validate.mockRestore(); retain.mockRestore(); capture.mockRestore(); backup.mockRestore(); }); + } const execute = vi.fn(async (command: { command: string; args?: string[] }) => ({ exitCode: 0, signal: null, timedOut: false, stderr: "", stdout: command.args?.[0] === "--build-metadata" ? JSON.stringify({ @@ -1938,16 +1979,19 @@ describe("remote provider pack manifest", () => { packageName: "@paperclipai/paperclip-runner", binaryContractVersion: 2, durableSessionCapabilities: ["unlimited_runtime", "connection_lease_renewal"], prpTransportModes: ["listen_ws"], providerPacks: [{ root: "/opt/paperclip-runner/provider-pack", manifest: installed, immutable: true }], - }) : "", + }) : firstDurable && command.args?.[1]?.startsWith("const os=require") ? JSON.stringify({ home: "/home/runtime-account", uid: 1000 }) + : firstDurable && command.args?.[1]?.includes("base64 <") ? Buffer.from(JSON.stringify(command.args[1].includes("codex-provider-state.json") + && !command.args[1].includes("runner-state.json") ? providerState : suspendedState)).toString("base64") : "", })); try { let coldAdmission: ReturnType = null; - const prepare = createRunnerdBackend({ db: leaseDb(remoteExecution), execution: remoteExecution, + const prepare = createRunnerdBackend({ db: preparedDb, execution: remoteExecution, runnerInstanceId: "bundled-image-runner", runnerIngressAuthorized: true, + ...(firstDurable ? { controllerPersistence: { bootId: randomUUID() } } : {}), ...(opencode ? { runnerEnvironment: { ...configuredEnvironmentProjection(diagnostics ? { PAPERCLIP_OPENCODE_PERFORMANCE_DIAGNOSTICS: "1" } : {}), PAPERCLIP_NATIVE_MCP_NAME: "assigned-fixture", PAPERCLIP_NATIVE_MCP_URL: "http://127.0.0.1/mcp/gateways/assigned-fixture", PAPERCLIP_NATIVE_MCP_TOKEN: "private-fixture" } } : {}), runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: "/workspace", environmentId: "environment", leaseId: "lease", providerKey: "daytona", - effectiveCapabilities: { runnerWebSocketIngress: true }, runner: { execute, syncIn } } as never }); + effectiveCapabilities: { runnerWebSocketIngress: true }, runner: { execute, syncIn, ...(firstDurable ? { syncOut } : {}) } } as never }); if (provider === "opencode-diagnostics-stale") { await expect(prepare).rejects.toThrow("image provider pack did not match the bundled release identity"); expect(state.createBackend).not.toHaveBeenCalled(); @@ -1978,9 +2022,9 @@ describe("remote provider pack manifest", () => { expect(execute.mock.calls.some(([call]) => call.args?.[1] === "uname -s; uname -m")).toBe(false); expect(existsSync(join(transport.stateDirectory!, "provider-pack-authority.json"))).toBe(false); expect(transport).toMatchObject({ providerPackAuthorityDigest: installed.digest, - providerNodeCommand: "/opt/paperclip-runner/provider-pack/node_modules/node/bin/node", + providerNodeCommand: firstDurable ? `${launchRoot}/${installed.payload.artifacts.nodeCommand.path}` : "/opt/paperclip-runner/provider-pack/node_modules/node/bin/node", providerNodeCommandSha256: installed.payload.artifacts.nodeCommand.sha256 }); - if (provider === "opencode" || provider.startsWith("opencode-continuation") || provider === "opencode-dirty") { + if (provider === "opencode" || firstDurable || provider.startsWith("opencode-continuation") || provider === "opencode-dirty") { const identity = { ...remoteExecution.binding, sessionId: remoteExecution.session.normalizedSessionId! }; const session = { identity: () => identity } as never; const backend = state.createBackend.mock.results.at(-1)!.value; @@ -2003,6 +2047,16 @@ describe("remote provider pack manifest", () => { coldAdmission = admittedNativeProviderPack(opened, identity); } expect(execute.mock.calls.filter(([call]) => call.args?.[0] === "--build-metadata")).toHaveLength(1); + if (firstDurable) { + expect(managedBackend.providerAdmissionRequired?.()).toBe(true); + await managedBackend.providerAdmissionPersisted!(); + expect(providerArtifactCapture.captureRemoteNativeProviderArtifacts).toHaveBeenCalledTimes(1); + expect(providerArtifactStorage.retainNativeProviderArtifacts).toHaveBeenCalledWith(expect.objectContaining({ + owner: expect.objectContaining({ runId: remoteExecution.binding.runId }), source: expect.objectContaining({ manifest: installed, launchRoot, accountUid: 1000 }) })); + await driver.mock.calls.at(-1)![0].checkpoint!("settled"); + expect(harnessStorage.persistNativeHarnessBackup).toHaveBeenCalledTimes(1); + return; + } expect(execute.mock.calls.some(([call]) => call.args?.[0] === "-e" || call.args?.[0] === "--version")).toBe(false); expect(syncIn.mock.calls.every(([operations]) => operations.every(operation => operation.files.every(file => file.kind === "file")))).toBe(true); if (provider.startsWith("opencode-continuation")) { @@ -2026,23 +2080,43 @@ describe("remote provider pack manifest", () => { const receiptPath = join(transport.stateDirectory!, "provider-pack-authority.json"); const receiptBefore = await readFile(receiptPath, "utf8"); if (provider === "opencode-continuation-no-receipt") await rm(receiptPath); + const cold = provider === "opencode-continuation-cold"; + const legacyDurable = provider === "opencode-continuation-legacy-durable"; + const launchRoot = `/home/runtime-account/.paperclip-runtime/paperclip-runner/provider-artifacts/${installed.digest.slice(7)}`; + const artifactReceipt = { schema: "paperclip.native-provider-artifacts.v1" as const, artifactId: randomUUID(), + closureDigest: "sha256:" + "9".repeat(64), providerPackDigest: installed.digest, manifest: installed, launchRoot, accountUid: 1000 }; + if (cold || legacyDurable) { + if (cold) await rm(receiptPath); + const storageMode = vi.spyOn(harnessStorage, "durableNativeHarnessStorage").mockReturnValue({} as never); + const backupOwner = vi.spyOn(harnessStorage, "nativeHarnessBackupOwner").mockResolvedValue({} as never); + const restore = vi.spyOn(providerArtifactStorage, "restoreNativeProviderArtifacts").mockResolvedValue(undefined); + const capture = legacyDurable ? vi.spyOn(providerArtifactStorage, "retainNativeProviderArtifacts").mockResolvedValue(artifactReceipt) : null; + onTestFinished(() => { storageMode.mockRestore(); backupOwner.mockRestore(); restore.mockRestore(); capture?.mockRestore(); }); + } const prepareRemote = vi.spyOn(nativeExecutionDriver, "prepareNativeRemoteRunner"); const backendsBefore = state.createBackend.mock.calls.length; - const selectedExecute = provider === "opencode-continuation-missing" + const selectedExecute = provider === "opencode-continuation-missing" || cold ? async (command: { command: string; args?: string[] }) => command.args?.[0] === "--build-metadata" ? { ...await execute(command), stdout: JSON.stringify({ schema: "paperclip-runner/runnerd-build-metadata/v1", binaryName: "paperclip-runnerd", packageName: "@paperclipai/paperclip-runner", binaryContractVersion: 2, durableSessionCapabilities: ["unlimited_runtime", "connection_lease_renewal"], prpTransportModes: ["listen_ws"], providerPacks: [{ root: "/opt/paperclip-runner/provider-pack", manifest: next, immutable: true }], - }) } : execute(command) - : execute; + }) } : (cold || legacyDurable) && command.args?.[1]?.startsWith("const os=require") + ? { ...await execute(command), stdout: JSON.stringify({ home: "/home/runtime-account", uid: 1000 }) } : execute(command) + : legacyDurable ? async (command: { command: string; args?: string[] }) => command.args?.[1]?.startsWith("const os=require") + ? { ...await execute(command), stdout: JSON.stringify({ home: "/home/runtime-account", uid: 1000 }) } : execute(command) : execute; const continuation = createRunnerdBackend({ db: leaseDb(remoteExecution), execution: remoteExecution, runnerInstanceId: identity.runnerInstanceId, runnerIngressAuthorized: true, + ...(legacyDurable ? { controllerPersistence: { bootId: randomUUID() } } : {}), + ...(cold ? { controllerPersistence: { bootId: randomUUID() }, providerPackAdmissionSource: { + runId: remoteExecution.binding.runId, receipt: coldAdmission, state: { schema: "paperclip.native-provider-pack-admission-state.v1", + state: "verified", admissionEpoch: randomUUID() }, artifacts: artifactReceipt } } : {}), runnerEnvironment: configuredEnvironmentProjection(provider === "opencode-continuation-replacement" ? {} : { PAPERCLIP_OPENCODE_PERFORMANCE_DIAGNOSTICS: "1" }), - runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: "/workspace", + runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: cold ? "/workspace-new-allocation-lease" : "/workspace", environmentId: "environment", leaseId: "lease", providerKey: "daytona", - effectiveCapabilities: { runnerWebSocketIngress: true }, runner: { execute: selectedExecute, syncIn } } as never }); + effectiveCapabilities: { runnerWebSocketIngress: true }, runner: { execute: selectedExecute, syncIn, + ...(cold || legacyDurable ? { syncOut: vi.fn() } : {}) } } as never }); if (provider === "opencode-continuation-missing" || provider === "opencode-continuation-no-receipt") { await expect(continuation).rejects.toThrow("image provider pack did not match the bundled release identity"); expect(state.createBackend).toHaveBeenCalledTimes(backendsBefore); @@ -2051,13 +2125,26 @@ describe("remote provider pack manifest", () => { return; } const continued = await continuation; + if (legacyDurable) { + await expect(continued.providerAdmissionPersisted?.()).resolves.toBeUndefined(); + expect(providerArtifactStorage.retainNativeProviderArtifacts).not.toHaveBeenCalled(); + } state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!(); expect(state.createTransport.mock.calls.at(-1)![0]).toMatchObject({ providerPackAuthorityDigest: installed.digest, opencodeProxySha256: installed.payload.artifacts.opencodeProxy.sha256, - opencodeProxyPath: "/opt/paperclip-runner/provider-pack/dist/cli/opencode-app-server-proxy.cjs", + opencodeProxyPath: cold ? `${launchRoot}/${installed.payload.artifacts.opencodeProxy.path}` + : "/opt/paperclip-runner/provider-pack/dist/cli/opencode-app-server-proxy.cjs", runnerArtifact: { version: transport.runnerArtifact!.version, digest: "sha256:" + createHash("sha256").update(currentDaemon).digest("hex") }, }); + if (cold) { + expect(providerArtifactStorage.restoreNativeProviderArtifacts).toHaveBeenCalledWith(expect.objectContaining({ + companyId: remoteExecution.binding.companyId, receipt: artifactReceipt })); + expect(state.createTransport.mock.calls.at(-1)![0]).toMatchObject({ + providerNodeCommand: `${launchRoot}/${installed.payload.artifacts.nodeCommand.path}`, + opencodeCommand: `${launchRoot}/${installed.payload.artifacts.opencodeExecutable.path}`, + }); + } if (provider === "opencode-continuation-replacement") { const preparation = prepareRemote.mock.calls.at(-1)![0]; const backend = state.createBackend.mock.results.at(-1)!.value; @@ -2128,8 +2215,17 @@ describe("remote provider pack manifest", () => { const result = await recoveredBackend.recoverSession!(checkpoint, { signal: new AbortController().signal }); expect(result.recovered).toBe(true); expect(admittedNativeProviderPack(result.session!, nativeIdentity)).toBeNull(); + const storageMode = vi.spyOn(harnessStorage, "durableNativeHarnessStorage").mockReturnValue({} as never); + const backupOwner = vi.spyOn(harnessStorage, "nativeHarnessBackupOwner").mockResolvedValue({} as never); + const restoreArtifacts = vi.spyOn(providerArtifactStorage, "restoreNativeProviderArtifacts") + .mockRejectedValue(new Error("Object storage must not gate a live runner reattachment")); + const captureArtifacts = vi.spyOn(providerArtifactStorage, "retainNativeProviderArtifacts"); + onTestFinished(() => { storageMode.mockRestore(); backupOwner.mockRestore(); restoreArtifacts.mockRestore(); captureArtifacts.mockRestore(); }); const legacyBackend = await createRunnerdBackend({ ...recoveryInput, controllerPersistence: { bootId: "replacement-controller", runnerArtifact: artifact } }); + await expect(legacyBackend.providerAdmissionPersisted?.()).resolves.toBeUndefined(); + expect(restoreArtifacts).not.toHaveBeenCalled(); + expect(captureArtifacts).not.toHaveBeenCalled(); const legacy = { identity: () => nativeIdentity } as never; Object.assign(state.createBackend.mock.results.at(-1)!.value, { recoverSession: async () => ({ recovered: true, session: legacy }), @@ -9088,7 +9184,7 @@ describe("native session execution lease fencing", () => { expect(profileWrites).toBe(2); expect(nullWrites).toBe(0); const claim = updates.find(update => update.table === nativeRunFinalizations && update.values.phase === "observed" && update.values.leaseOwner)!.values; - expect(recordAdmission).toHaveBeenCalledExactlyOnceWith(db, { session, identity, + expect(recordAdmission).toHaveBeenCalledExactlyOnceWith(db, { session, identity, waitForClaim: false, admissionEpoch: (profile.nativeProviderPackAdmissionState as Record).admissionEpoch, claim: { leaseOwner: claim.leaseOwner, attempt: claim.attempt, controllerGeneration: claim.controllerGeneration, controllerBootId: claim.controllerBootId } }); }); @@ -18014,6 +18110,8 @@ describe("runnerd provider runtime wiring", () => { await mkdir(join(root, "failover-backups", name), { recursive: true }); await writeFile(join(root, "failover-backups", name, "manifest.json"), JSON.stringify({ priorSession: name })); } + await mkdir(join(root, "provider-artifacts"), { recursive: true }); + await writeFile(join(root, "provider-artifacts", "opencodeProxy"), "retained prior launch bytes"); // Ambiguous ordinary recovery must still fail closed. Only the runtime's // explicitly admitted replacement may retire these prior-session backups. await expect(prepareReplacement()).rejects.toThrow("runner_harness_state_mismatch: backup_without_reusable_lease"); @@ -18024,6 +18122,9 @@ describe("runnerd provider runtime wiring", () => { await expect(access(join(root, "failover-backups"))).rejects.toThrow(); const archives = await readdir(join(root, "continuity-breaks")); expect(archives).toHaveLength(1); + await expect(access(join(root, "provider-artifacts"))).rejects.toThrow(); + expect(await readFile(join(root, "continuity-breaks", archives[0]!, "provider-artifacts", "opencodeProxy"), "utf8")) + .toBe("retained prior launch bytes"); for (const name of ["current", "previous"]) { expect(JSON.parse(await readFile(join(root, "continuity-breaks", archives[0]!, "failover-backups", name, "manifest.json"), "utf8"))) .toEqual({ priorSession: name }); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index d0243073dd5..8db3c94d7c4 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -1,3 +1,4 @@ +import { captureRemoteNativeProviderArtifacts } from "./native-provider-artifact-capture.js"; import { NativeSessionDiagnostics, nativeSessionDiagnostics, type NativeDiagnosticLease } from "./native-session-diagnostics.js"; import { recordAgentProviderRateLimits } from "../agent-provider-rate-limits.js"; import { nativePostResultFailureCategory, type NativePostResultFailureCategory } from "./native-post-result-failure.js"; @@ -12,6 +13,9 @@ import { nativeCancellationHasUnsettledOwner, notifyNativeCancellation, waitForN import { readClaimedNativeStopRecovery } from "./native-cancellation-request.js"; import { publishNativeHarnessBackup } from "./native-harness-backup-publication.js"; import { assertNativeHarnessBackupOwner, durableNativeHarnessStorage, nativeHarnessBackupOwner, persistNativeHarnessBackup, restoreDurableNativeHarnessBackup } from "./native-harness-backup-storage.js"; +import { restoreNativeProviderArtifacts, retainNativeProviderArtifacts, retainExistingNativeProviderArtifacts, + type NativeProviderArtifactReceipt } from "./native-provider-artifacts.js"; +import { prepareRemoteOpenCodeArtifacts, type OpenCodeLaunchArtifacts } from "./remote-opencode-artifacts.js"; import { rehydratePinnedSkillAssets } from "./runtime-context.js"; import { createNativeUsageDelivery } from "./native-usage-delivery.js"; import { createNativeRecoveryUsageDelivery } from "./native-recovery-usage-delivery.js"; @@ -6524,6 +6528,7 @@ export interface NativeHarnessBackupManifest { bytes: number; }>; completedAt: string; + providerArtifacts?: NativeProviderArtifactReceipt; } export function resolveNativeHarnessPersistenceProfile( @@ -9351,7 +9356,8 @@ async function executePaperclipNativeSessionWithinScope( if (!claimed) throw new Error("native_session_lease_lost"); providerPackAdmissionSource = { runId: coordinator.runId, receipt: boundRun.runnerProfileJson?.nativeProviderPackAdmission, - state: boundRun.runnerProfileJson?.nativeProviderPackAdmissionState }; + state: boundRun.runnerProfileJson?.nativeProviderPackAdmissionState, + artifacts: boundRun.runnerProfileJson?.nativeProviderArtifacts }; controllerGeneration = nextControllerGeneration; claimedStopCleanup = stopRecovery; await tx @@ -10683,10 +10689,12 @@ async function executePaperclipNativeSessionWithinScope( agentId: input.execution.binding.agentId, issueId: input.execution.binding.issueId, sessionId: nativeSessionKey(input.execution) }, claim: { leaseOwner, attempt, controllerGeneration, controllerBootId: controller.bootId }, + waitForClaim: runnerdBackend?.providerAdmissionRequired?.() === true, }); } catch { try { await input.onLog?.("stderr", "[paperclip-runner] native provider-pack admission receipt could not be saved\n"); } catch {} } + await runnerdBackend?.providerAdmissionPersisted?.(); if (session.resolveRuntimeRequest) await liveQuestions.attach(); if (active.cancelRequested) { await active.cancellationSettled; @@ -13520,6 +13528,8 @@ type ManagedRunnerdBackend = NativeSessionBackend & { bindManagedSession(session: NativeSession): NativeSession; bindDiagnosticSession?(session: NativeSession): void; sessionCheckpointPersisted?(snapshot: PersistedNativeSession): void; + providerAdmissionPersisted?(): Promise; + providerAdmissionRequired?(): boolean; }; async function createRunnerdBackendWithinSessionClaim( @@ -13944,6 +13954,22 @@ async function createRunnerdBackendWithinSessionClaim( remoteTarget !== null && (input.execution.provider.kind === "opencode" || input.execution.provider.kind === "acpx"); + const artifactStorage = remoteTarget?.transport === "sandbox" && input.controllerPersistence + && input.execution.provider.kind === "opencode" ? durableNativeHarnessStorage() : null; + const providerArtifactsSignal = () => input.executionContext?.signal + ? AbortSignal.any([input.executionContext.signal, AbortSignal.timeout(300_000)]) : AbortSignal.timeout(300_000); + const liveProviderRecovery = input.restartRecovery?.kind === "reattach_remote_runner" || input.remoteWarmRecovery !== undefined; + const artifactBackup = artifactStorage ? verifyNativeHarnessBackup({ root, execution: input.execution, + runnerInstanceId: input.runnerInstanceId }) : null; + let providerArtifacts = artifactStorage ? artifactBackup?.manifest.providerArtifacts + ?? input.providerPackAdmissionSource?.artifacts as NativeProviderArtifactReceipt | undefined : undefined; + let legacyProviderContinuity = durableBinding !== null && !providerArtifacts; + if (providerArtifacts && artifactStorage && !liveProviderRecovery) await restoreNativeProviderArtifacts({ db: input.db, storage: artifactStorage, + companyId: input.execution.binding.companyId, receipt: providerArtifacts, + targetRoot: artifactBackup ? join(artifactBackup.root, "provider-artifacts") : join(root, "provider-artifacts"), + signal: providerArtifactsSignal() }); + const restoredArtifactDirectory = artifactBackup ? join(artifactBackup.root, "provider-artifacts") : join(root, "provider-artifacts"); + let openCodeLaunchArtifacts: OpenCodeLaunchArtifacts | undefined; const adoptingPersistedRemoteRunner = remoteTarget !== null && input.restartRecovery?.kind === "reattach_remote_runner" && input.controllerPersistence?.runnerArtifact !== undefined; @@ -14025,12 +14051,13 @@ async function createRunnerdBackendWithinSessionClaim( runnerInstanceId: effectiveRunnerInstanceId, environmentLeaseId: effectiveEnvironmentLeaseId, remoteCwd: remoteTarget!.remoteCwd, providerDigest: nativeSha256(input.execution.provider) }, reattach, - retainProviderPack: reattach || retainProviderPack, + retainProviderPack: !providerArtifacts && (reattach || retainProviderPack), runnerArtifact: () => readRunnerdArtifactBinding(controllerRunnerBinary), - current: resolveCurrentProviderPackManifest, + current: () => providerArtifacts?.manifest as RemoteProviderPackManifest ?? resolveCurrentProviderPackManifest(), }) : null; let providerPackAuthority = adoptingPersistedRemoteRunner ? null : selectProviderPackAuthority(reattachProviderPack); - let expectedProviderPackManifest = providerPackAuthority?.manifest ?? null; + let expectedProviderPackManifest: RemoteProviderPackManifest | null = providerPackAuthority?.manifest + ?? providerArtifacts?.manifest as RemoteProviderPackManifest ?? null; let providerPackVerified = false; const explicitRemoteCodex = input.runnerRemoteCodexPath?.trim() || null; const remoteCodexNpmSpec = input.runnerRemoteCodexNpmSpec?.trim() || null; @@ -14180,7 +14207,7 @@ async function createRunnerdBackendWithinSessionClaim( return false; }; - const prepareRemoteRunner = async ( + const prepareRemoteRunnerAssets = async ( requiredMode: "dial_wss" | "listen_ws", ) => { if ( @@ -14209,6 +14236,11 @@ async function createRunnerdBackendWithinSessionClaim( return; } await remoteNativeArtifacts?.prepare(requiredMode); + if (artifactStorage && providerArtifacts) { + providerPackVerified = true; + remotePrepared = true; + return; + } if (requiresRemoteProviderPack && await useInstalledProviderPack()) { remotePrepared = true; return; @@ -14228,6 +14260,25 @@ async function createRunnerdBackendWithinSessionClaim( remotePrepared = true; }; + const prepareRemoteRunner = async (mode: "dial_wss" | "listen_ws") => { + await prepareRemoteRunnerAssets(mode); + if (!artifactStorage || openCodeLaunchArtifacts || !remoteCommandRunner || !remoteTarget + || !expectedProviderPackManifest || liveProviderRecovery || legacyProviderContinuity) return; + if (!remoteCommandRunner.syncIn || !remoteCommandRunner.syncOut) throw new Error("native_provider_artifacts_bulk_transport_required"); + const bootstrap = activeRemoteProviderPackRoot + ? { root: activeRemoteProviderPackRoot, manifest: expectedProviderPackManifest } + : remoteNativeArtifacts?.providerPackInventory().find(pack => pack.immutable); + if (!bootstrap) throw new Error("native_provider_artifacts_bootstrap_unavailable"); + const bootstrapManifest = bootstrap.manifest as RemoteProviderPackManifest; + openCodeLaunchArtifacts = await prepareRemoteOpenCodeArtifacts({ runner: remoteCommandRunner, + bootstrapNode: posix.join(bootstrap.root, bootstrapManifest.payload.artifacts.nodeCommand.path), + manifest: expectedProviderPackManifest, nonce: randomUUID(), + ...(providerArtifacts ? { restored: { receipt: providerArtifacts, directory: restoredArtifactDirectory } } + : { sourceRoot: activeRemoteProviderPackRoot! }), + stageFile: asset => stageRemoteRunnerFile({ target: remoteTarget, runner: remoteCommandRunner, ...asset }), + }); + }; + /** * Inspect the remote harness state. Only in-place host preparation may use * this invocation's archived rotation snapshot; every other caller, @@ -14903,6 +14954,7 @@ async function createRunnerdBackendWithinSessionClaim( input.durableEnvironmentLeaseId ?? "unknown", }); + if (providerArtifacts) manifest.providerArtifacts = providerArtifacts; backupSpanAttributes.bytesTransferred = manifest.directories.reduce( (total, directory) => total + directory.bytes, @@ -15166,6 +15218,9 @@ async function createRunnerdBackendWithinSessionClaim( } remotePrepared = false; retainProviderPack = false; + legacyProviderContinuity = false; + providerArtifacts = undefined; + openCodeLaunchArtifacts = undefined; providerPackAuthority = selectProviderPackAuthority(false); expectedProviderPackManifest = providerPackAuthority?.manifest ?? null; providerPackVerified = false; @@ -15390,6 +15445,15 @@ async function createRunnerdBackendWithinSessionClaim( .sha256, } : {}), + ...(openCodeLaunchArtifacts ? { + providerNodeCommand: openCodeLaunchArtifacts.paths.nodeCommand, + providerNodeCommandSha256: expectedProviderPackManifest!.payload.artifacts.nodeCommand.sha256, + opencodeCommand: openCodeLaunchArtifacts.paths.opencodeExecutable, + opencodeCommandSha256: expectedProviderPackManifest!.payload.artifacts.opencodeExecutable.sha256, + opencodeProxyPath: openCodeLaunchArtifacts.paths.opencodeProxy, + opencodeProxySha256: expectedProviderPackManifest!.payload.artifacts.opencodeProxy.sha256, + providerPackAuthorityDigest: expectedProviderPackManifest!.digest, + } : {}), stateDirectory: root, runnerBinary: controllerRunnerBinary, runnerArtifact: input.controllerPersistence?.runnerArtifact ?? providerPackAuthority?.runnerArtifact, @@ -15681,6 +15745,24 @@ async function createRunnerdBackendWithinSessionClaim( }; return { bindManagedSession: wrapManagedSession, + providerAdmissionRequired: () => Boolean(artifactStorage + && (providerArtifacts || !legacyProviderContinuity && !liveProviderRecovery)), + providerAdmissionPersisted: artifactStorage ? async () => { + const backupOwner = sessionHarnessBackupOwners.get(authorityEpoch); + if ((liveProviderRecovery || legacyProviderContinuity) && !providerArtifacts) return; + if (!backupOwner || !expectedProviderPackManifest || !remoteCommandRunner) { + throw new Error("native_provider_artifacts_admission_missing"); + } + if (providerArtifacts) { + await retainExistingNativeProviderArtifacts({ db: input.db, owner: backupOwner, receipt: providerArtifacts }); + return; + } + if (!openCodeLaunchArtifacts) throw new Error("native_provider_artifacts_admission_missing"); + const signal = providerArtifactsSignal(); + providerArtifacts = await captureRemoteNativeProviderArtifacts({ runner: remoteCommandRunner, + launch: openCodeLaunchArtifacts, manifest: expectedProviderPackManifest, signal, context: input.executionContext }, + source => retainNativeProviderArtifacts({ db: input.db, storage: artifactStorage, owner: backupOwner, source, signal })); + } : undefined, bindDiagnosticSession: session => { const retained = nativeSessionDiagnostics(session); if (retained && retained !== diagnostics) { diff --git a/server/src/services/native-runtime/native-session-root-entries.test.ts b/server/src/services/native-runtime/native-session-root-entries.test.ts index 8ac6ab64dad..f34d28fc856 100644 --- a/server/src/services/native-runtime/native-session-root-entries.test.ts +++ b/server/src/services/native-runtime/native-session-root-entries.test.ts @@ -69,7 +69,7 @@ describe("native session root entry contract", () => { expect(unresolved).toEqual([]); // Guard the scan itself: these writers exist today. for (const name of ["control-plane", "runner", "runtime-context.json", "authority-epochs", "launch-preparation.json", - "provider-pack-authority.json", "cleanup-activation.json", "plugin-runner-state.json", "codex-home", "opencode", + "provider-pack-authority.json", "provider-artifacts", "cleanup-activation.json", "plugin-runner-state.json", "codex-home", "opencode", "acpx", "cursor-cloud", "diagnostics", "failover-backups", "continuity-breaks"]) { expect(written).toContain(name); } diff --git a/server/src/services/native-runtime/native-session-root-entries.ts b/server/src/services/native-runtime/native-session-root-entries.ts index 3a04144c570..25653eb2d40 100644 --- a/server/src/services/native-runtime/native-session-root-entries.ts +++ b/server/src/services/native-runtime/native-session-root-entries.ts @@ -17,6 +17,7 @@ export const NATIVE_CONTINUITY_STATE_ENTRIES: readonly string[] = [ "acpx", "failover-backups", "provider-pack-authority.json", + "provider-artifacts", // Runner transport: each cold authority rotation archives the settled prior // epoch here (rotateAuthorityEpoch). "authority-epochs", diff --git a/server/src/services/native-runtime/native-source-recovery-ownership.ts b/server/src/services/native-runtime/native-source-recovery-ownership.ts index eeec0f1196f..51b4eb56f0c 100644 --- a/server/src/services/native-runtime/native-source-recovery-ownership.ts +++ b/server/src/services/native-runtime/native-source-recovery-ownership.ts @@ -1,5 +1,5 @@ -import { and, asc, eq, inArray, sql } from "drizzle-orm"; -import { environmentLeases, nativeRunFinalizations, type Db } from "@paperclipai/db"; +import { and, asc, eq, exists, inArray, sql } from "drizzle-orm"; +import { environmentLeases, nativeRunFinalizations, nativeRunnerCheckpoints, type Db } from "@paperclipai/db"; import { inspectDurablePrpCheckpointHeader, type DurablePrpCheckpoint } from "../../vendor/paperclip-runner/index.js"; type Transaction = Parameters[0]>[0]; @@ -12,7 +12,11 @@ export async function lockSourceRecoveryCoordinators(tx: Transaction, companyId: } export async function readNativeSourceConsumers(tx: Transaction, companyId: string, runId: string) { - return tx.select({ id: environmentLeases.id, metadata: environmentLeases.metadata }).from(environmentLeases).where(and( + const retired = exists(tx.select({ runId: nativeRunnerCheckpoints.runId }).from(nativeRunnerCheckpoints).where(and( + eq(nativeRunnerCheckpoints.companyId, companyId), eq(nativeRunnerCheckpoints.runId, runId), + eq(nativeRunnerCheckpoints.part, "header"), + sql`${nativeRunnerCheckpoints.material}->>'schema' like 'paperclip.native-runner-checkpoint-retirement.%'`))); + return tx.select({ id: environmentLeases.id, metadata: environmentLeases.metadata, retired }).from(environmentLeases).where(and( eq(environmentLeases.companyId, companyId), eq(environmentLeases.heartbeatRunId, runId), sql`${environmentLeases.metadata}->'runnerTaskSourceConsumer' is not null`)); } @@ -22,6 +26,7 @@ export async function assertNativeSourceCheckpointConsumer(tx: Transaction, owne companyId: string; runId: string; authorityRunId: string; bootId: string; generation: number; }, checkpoint?: DurablePrpCheckpoint) { const rows = await readNativeSourceConsumers(tx, owner.companyId, owner.runId); + if (rows.some(row => row.retired)) throw new Error("native_runner_checkpoint_retired"); if (!rows.length) return; const deny = (): never => { throw new Error("native_runner_checkpoint_owner_lost"); }; if (owner.runId === owner.authorityRunId || rows.length !== 1 || !checkpoint) return deny(); diff --git a/server/src/services/native-runtime/remote-opencode-artifacts.test.ts b/server/src/services/native-runtime/remote-opencode-artifacts.test.ts new file mode 100644 index 00000000000..c1ff0bf8756 --- /dev/null +++ b/server/src/services/native-runtime/remote-opencode-artifacts.test.ts @@ -0,0 +1,279 @@ +import { execFile } from "node:child_process"; +import { createHash, randomUUID } from "node:crypto"; +import { + chmod, + cp, + lstat, + mkdir, + mkdtemp, + readFile, + readdir, + realpath, + rename, + rm, + symlink, + writeFile, +} from "node:fs/promises"; +import { tmpdir, userInfo } from "node:os"; +import { dirname, join } from "node:path"; +import { promisify } from "node:util"; +import { afterEach, expect, it } from "vitest"; +import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; +import { prepareRemoteOpenCodeArtifacts } from "./remote-opencode-artifacts.js"; +import { + canonicalProviderPackJson, + type ProviderPackIdentity, +} from "./remote-provider-pack-verification.js"; +import type { NativeProviderArtifactReceipt } from "./native-provider-artifacts.js"; + +const exec = promisify(execFile), + homes: string[] = []; +afterEach(async () => { + for (const root of homes.splice(0)) + await rm(root, { recursive: true, force: true }); +}); +async function fixture() { + const root = await realpath( + await mkdtemp(join(tmpdir(), "paperclip-closed-launch-")), + ); + homes.push(root); + const accountHome = join(root, "account"), + sourceRoot = join(root, "image-provider-pack"); + await mkdir(accountHome, { mode: 0o700 }); + await mkdir(sourceRoot); + const values = { + nodeCommand: "\u007fELForiginal qualified Node", + opencodeProxy: "original qualified proxy", + opencodeExecutable: "\u007fELForiginal qualified OpenCode", + }; + const artifacts: Record = {}; + for (const [name, value] of Object.entries(values)) { + const path = `qualified/${name}`; + await mkdir(dirname(join(sourceRoot, path)), { recursive: true }); + await writeFile(join(sourceRoot, path), value, { mode: 0o500 }); + artifacts[name] = { + path, + sha256: `sha256:${createHash("sha256").update(value).digest("hex")}`, + }; + } + const payload = { + artifacts, + target: { platform: "linux", architecture: "x64" }, + pins: { opencode: "1.18.34" }, + runnerSourceRevision: "a".repeat(40), + }; + const manifest = { + schema: "paperclip-runner/remote-provider-pack/v1", + payload, + digest: `sha256:${createHash("sha256").update(canonicalProviderPackJson(payload)).digest("hex")}`, + } as ProviderPackIdentity; + let activeHome = accountHome; + const runner = { + execute: async (input) => { + const result = input.args?.[1]?.startsWith("const os=require") + ? { + stdout: JSON.stringify({ home: activeHome, uid: userInfo().uid }), + stderr: "", + } + : await exec(input.command, input.args ?? []).catch((error) => ({ + stdout: error.stdout ?? "", + stderr: error.stderr ?? "", + code: error.code, + })); + return { + ...result, + exitCode: "code" in result ? 1 : 0, + signal: null, + timedOut: false, + }; + }, + } as CommandManagedRuntimeRunner; + const stageFile = async (file: { + sourcePath: string; + targetPath: string; + }) => { + await cp(file.sourcePath, file.targetPath, { + errorOnExist: true, + force: false, + }); + }; + return { + root, + accountHome, + sourceRoot, + values, + manifest, + runner, + stageFile, + changeAccount: (home: string) => { + activeHome = home; + }, + }; +} +it("restores exact launch paths independently of a replacement lease cwd and upgraded image", async () => { + const f = await fixture(); + const first = await prepareRemoteOpenCodeArtifacts({ + ...f, + bootstrapNode: process.execPath, + nonce: randomUUID(), + }); + const cache = join(f.root, "decrypted-backup"); + await mkdir(cache); + for (const [name, path] of Object.entries(first.paths)) + await cp(path, join(cache, name)); + const receipt = { + launchRoot: first.root, + accountUid: userInfo().uid, + providerPackDigest: f.manifest.digest, + } as NativeProviderArtifactReceipt; + const oldCwd = join(f.root, "workspace-old-lease"), + newCwd = join(f.root, "workspace-new-lease"); + await mkdir(oldCwd); + await mkdir(newCwd); + await rm(oldCwd, { recursive: true }); + await rm(first.root, { recursive: true }); + await rm(f.sourceRoot, { recursive: true }); + await mkdir(f.sourceRoot); + await writeFile(join(f.sourceRoot, "new-proxy"), "upgraded image proxy"); + const recovered = await prepareRemoteOpenCodeArtifacts({ + ...f, + sourceRoot: undefined, + restored: { receipt, directory: cache }, + bootstrapNode: process.execPath, + nonce: randomUUID(), + }); + expect(recovered).toEqual(first); + for (const [name, value] of Object.entries(f.values)) + expect( + await readFile( + recovered.paths[name as keyof typeof recovered.paths], + "utf8", + ), + ).toBe(value); + expect(await readFile(join(f.sourceRoot, "new-proxy"), "utf8")).toBe( + "upgraded image proxy", + ); +}); +it("refuses a different account instead of rewriting the recorded profile path", async () => { + const f = await fixture(), + first = await prepareRemoteOpenCodeArtifacts({ + ...f, + bootstrapNode: process.execPath, + nonce: randomUUID(), + }); + const next = join(f.root, "other-account"); + await mkdir(next, { mode: 0o700 }); + f.changeAccount(next); + await expect( + prepareRemoteOpenCodeArtifacts({ + ...f, + bootstrapNode: process.execPath, + nonce: randomUUID(), + restored: { + receipt: { + launchRoot: first.root, + accountUid: userInfo().uid, + providerPackDigest: f.manifest.digest, + } as NativeProviderArtifactReceipt, + directory: f.sourceRoot, + }, + }), + ).rejects.toThrow("account_changed"); +}); +it("rejects another account UID even when its absolute home path is unchanged", async () => { + const f = await fixture(), + first = await prepareRemoteOpenCodeArtifacts({ + ...f, + bootstrapNode: process.execPath, + nonce: randomUUID(), + }); + await expect( + prepareRemoteOpenCodeArtifacts({ + ...f, + bootstrapNode: process.execPath, + nonce: randomUUID(), + restored: { + receipt: { + launchRoot: first.root, + providerPackDigest: f.manifest.digest, + accountUid: userInfo().uid + 1, + } as unknown as NativeProviderArtifactReceipt, + directory: f.sourceRoot, + }, + }), + ).rejects.toThrow("account_changed"); +}); +it("materializes qualified internal package links as regular closed launch files", async () => { + const f = await fixture(); + await rename( + join(f.sourceRoot, "qualified"), + join(f.sourceRoot, "package-content"), + ); + await symlink("package-content", join(f.sourceRoot, "qualified")); + const launch = await prepareRemoteOpenCodeArtifacts({ + ...f, + bootstrapNode: process.execPath, + nonce: randomUUID(), + }); + for (const [name, path] of Object.entries(launch.paths)) { + expect((await lstat(path)).isSymbolicLink()).toBe(false); + expect(await readFile(path, "utf8")).toBe( + f.values[name as keyof typeof f.values], + ); + } +}); +it("holds a corrupted retained closure and never replaces it behind provider state", async () => { + const f = await fixture(), + first = await prepareRemoteOpenCodeArtifacts({ + ...f, + bootstrapNode: process.execPath, + nonce: randomUUID(), + }); + await chmod(first.paths.opencodeProxy, 0o700); + await writeFile(first.paths.opencodeProxy, "changed proxy bytes"); + await expect( + prepareRemoteOpenCodeArtifacts({ + ...f, + bootstrapNode: process.execPath, + nonce: randomUUID(), + }), + ).rejects.toThrow("remote_unavailable"); + expect(await readFile(first.paths.opencodeProxy, "utf8")).toBe( + "changed proxy bytes", + ); +}); +it("rejects administrative symlinks and source hash drift before publication", async () => { + const f = await fixture(), + source = join( + f.sourceRoot, + f.manifest.payload.artifacts.opencodeProxy.path, + ); + const outside = join(f.root, "outside"); + await cp(source, outside); + await rm(source); + await symlink(outside, source); + await expect( + prepareRemoteOpenCodeArtifacts({ + ...f, + bootstrapNode: process.execPath, + nonce: randomUUID(), + }), + ).rejects.toThrow("remote_unavailable"); + await rm(source); + await writeFile(source, "different source bytes"); + await expect( + prepareRemoteOpenCodeArtifacts({ + ...f, + bootstrapNode: process.execPath, + nonce: randomUUID(), + }), + ).rejects.toThrow("remote_unavailable"); + expect( + await readdir( + join( + f.accountHome, + ".paperclip-runtime/paperclip-runner/provider-artifacts", + ), + ), + ).toEqual([]); +}); diff --git a/server/src/services/native-runtime/remote-opencode-artifacts.ts b/server/src/services/native-runtime/remote-opencode-artifacts.ts new file mode 100644 index 00000000000..8c4937f0b82 --- /dev/null +++ b/server/src/services/native-runtime/remote-opencode-artifacts.ts @@ -0,0 +1,160 @@ +import { posix } from "node:path"; +import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; +import type { NativeProviderArtifactReceipt } from "./native-provider-artifacts.js"; +import type { ProviderPackIdentity } from "./remote-provider-pack-verification.js"; + +const names = ["nodeCommand", "opencodeProxy", "opencodeExecutable"] as const; +export type OpenCodeLaunchArtifacts = { + root: string; + accountUid: number; + paths: Record<(typeof names)[number], string>; +}; + +const accountProbe = `const os=require('node:os');const a=os.userInfo();process.stdout.write(JSON.stringify({home:a.homedir,uid:a.uid}));`; +const prepare = String.raw` +const fs=require('node:fs'),p=require('node:path'),c=require('node:crypto'); +const x=JSON.parse(Buffer.from(process.argv[1],'base64').toString('utf8')); +const bad=()=>{throw Error('native_provider_artifacts_remote_unsafe')}; +const checkDir=(d)=>{const s=fs.lstatSync(d);if(!s.isDirectory()||s.isSymbolicLink()||s.uid!==x.uid||(s.mode&0o022))bad()}; +let current='/';for(const part of x.home.split('/').filter(Boolean)){current=p.join(current,part);const s=fs.lstatSync(current);if(s.isSymbolicLink()||!s.isDirectory())bad()} +checkDir(x.home); +for(const part of ['.paperclip-runtime','paperclip-runner','provider-artifacts']){current=p.join(current,part);try{fs.mkdirSync(current,{mode:0o700})}catch(e){if(e.code!=='EEXIST')throw e}checkDir(current)} +if(p.join(current,x.digest)!==x.root)bad(); +if(x.mode==='cleanup'){try{checkDir(x.pending);fs.rmSync(x.pending,{recursive:true})}catch(e){if(e.code!=='ENOENT')throw e}process.exit(0)} +const hash=(f,expected)=>{const fd=fs.openSync(f,fs.constants.O_RDONLY|fs.constants.O_NOFOLLOW|fs.constants.O_NONBLOCK);try{const a=fs.fstatSync(fd);if(!a.isFile()||a.size>512*1024*1024)bad();const h=c.createHash('sha256'),b=Buffer.alloc(1024*1024);let n,total=0;while((n=fs.readSync(fd,b,0,b.length,null))){total+=n;h.update(b.subarray(0,n))}const z=fs.fstatSync(fd);if(total!==a.size||a.ino!==z.ino||a.size!==z.size||a.mtimeMs!==z.mtimeMs||a.ctimeMs!==z.ctimeMs||'sha256:'+h.digest('hex')!==expected)bad()}finally{fs.closeSync(fd)}}; +const confinedSource=(root,relative)=>{ + root=fs.realpathSync(root);let current=root,parts=relative.split('/'),links=0; + while(parts.length){current=p.join(current,parts.shift());const s=fs.lstatSync(current); + if(s.isSymbolicLink()){ + if(++links>32)bad();const target=p.resolve(p.dirname(current),fs.readlinkSync(current)); + if(target!==root&&!target.startsWith(root+p.sep))bad(); + parts=(target===root?[]:p.relative(root,target).split('/')).concat(parts);current=root; + }else if(parts.length&&!s.isDirectory())bad(); + } + return current; +}; +const copyQualified=(source,target,expected,limit)=>{ + const input=fs.openSync(source,fs.constants.O_RDONLY|fs.constants.O_NOFOLLOW|fs.constants.O_NONBLOCK);let output; + try{const before=fs.fstatSync(input);if(!before.isFile()||before.size>limit)bad(); + output=fs.openSync(target,fs.constants.O_WRONLY|fs.constants.O_CREAT|fs.constants.O_EXCL|fs.constants.O_NOFOLLOW,0o500); + const h=c.createHash('sha256'),b=Buffer.alloc(1024*1024);let n,total=0; + while((n=fs.readSync(input,b,0,b.length,null))){if((total+=n)>before.size)bad();h.update(b.subarray(0,n));let written=0;while(written{checkDir(root);let total=0;for(const e of x.entries){let d=root;const parts=e.path.split('/');for(const part of parts.slice(0,-1)){d=p.join(d,part);checkDir(d)}const f=p.join(root,e.path),s=fs.lstatSync(f);if(s.isSymbolicLink()||!s.isFile()||s.uid!==x.uid||(s.mode&0o022)||(total+=s.size)>512*1024*1024)bad();hash(f,e.sha256);if(e.name!=='opencodeProxy'){const fd=fs.openSync(f,fs.constants.O_RDONLY|fs.constants.O_NOFOLLOW),b=Buffer.alloc(4);try{if(fs.readSync(fd,b,0,4,0)!==4||b.toString('hex')!=='7f454c46')bad()}finally{fs.closeSync(fd)}}}}; +if(x.mode==='inspect'){try{verify(x.root);process.stdout.write('ready')}catch(e){if(e.code!=='ENOENT')throw e}process.exit(0)} +if(x.mode==='publish'){verify(x.pending);try{fs.renameSync(x.pending,x.root)}catch(e){if(e.code!=='EEXIST'&&e.code!=='ENOTEMPTY')throw e;verify(x.root);fs.rmSync(x.pending,{recursive:true})}verify(x.root);process.exit(0)} +try{fs.mkdirSync(x.pending,{mode:0o700})}catch(e){if(e.code!=='EEXIST')throw e}checkDir(x.pending); +let total=0;for(const e of x.entries){let d=x.pending;for(const part of e.path.split('/').slice(0,-1)){d=p.join(d,part);try{fs.mkdirSync(d,{mode:0o700})}catch(e){if(e.code!=='EEXIST')throw e}checkDir(d)} + if(x.source){const src=confinedSource(x.source,e.path);total+=copyQualified(src,p.join(x.pending,e.path),e.sha256,512*1024*1024-total)}} +`; + +export async function prepareRemoteOpenCodeArtifacts(input: { + runner: CommandManagedRuntimeRunner; + bootstrapNode: string; + manifest: ProviderPackIdentity; + nonce: string; + sourceRoot?: string; + restored?: { receipt: NativeProviderArtifactReceipt; directory: string }; + stageFile: (input: { + sourcePath: string; + targetPath: string; + mode: number; + }) => Promise; +}): Promise { + if ( + !/^[a-f0-9-]{36}$/.test(input.nonce) || + !/^sha256:[a-f0-9]{64}$/.test(input.manifest.digest) + ) + throw new Error("native_provider_artifacts_manifest_invalid"); + const run = async (args: string[]) => { + const result = await input.runner.execute({ + command: input.bootstrapNode, + args, + bypassSession: true, + timeoutMs: 120_000, + }); + if (result.exitCode !== 0 || result.timedOut) + throw new Error("native_provider_artifacts_remote_unavailable"); + return result.stdout; + }; + const account = JSON.parse(await run(["-e", accountProbe])); + if ( + typeof account.home !== "string" || + !posix.isAbsolute(account.home) || + account.home === "/" || + posix.normalize(account.home) !== account.home || + /[\\\u0000-\u001f\u007f]/u.test(account.home) || + !Number.isSafeInteger(account.uid) || + account.uid < 0 + ) + throw new Error("native_provider_artifacts_account_invalid"); + const root = posix.join( + account.home, + ".paperclip-runtime/paperclip-runner/provider-artifacts", + input.manifest.digest.slice(7), + ); + if ( + input.restored && + (input.restored.receipt.accountUid !== account.uid || + input.restored.receipt.launchRoot !== root || + input.restored.receipt.providerPackDigest !== input.manifest.digest) + ) { + throw new Error("native_provider_artifacts_account_changed"); + } + const entries = names.map((name) => { + const e = input.manifest.payload.artifacts[name]; + if ( + !e || + !/^[a-zA-Z0-9._/-]+$/.test(e.path) || + e.path.split("/").some((p) => !p || p === "." || p === "..") || + posix.isAbsolute(e.path) || + !/^sha256:[a-f0-9]{64}$/.test(e.sha256) + ) + throw new Error("native_provider_artifacts_manifest_invalid"); + return { name, ...e }; + }); + const pending = `${root}.${input.nonce}.pending`; + const command = (mode: string) => [ + "-e", + prepare, + Buffer.from( + JSON.stringify({ + ...account, + root, + pending, + digest: input.manifest.digest.slice(7), + entries, + source: input.restored ? null : input.sourceRoot, + mode, + }), + ).toString("base64"), + ]; + if ((await run(command("inspect"))) !== "ready") { + if (!input.restored && !input.sourceRoot) + throw new Error("native_provider_artifacts_source_missing"); + try { + await run(command("prepare")); + if (input.restored) + for (const entry of entries) + await input.stageFile({ + sourcePath: posix.join(input.restored.directory, entry.name), + targetPath: posix.join(pending, entry.path), + mode: 0o500, + }); + await run(command("publish")); + } catch (error) { + await run(command("cleanup")).catch(() => undefined); + throw error; + } + } + return { + root, + accountUid: account.uid, + paths: Object.fromEntries( + entries.map((e) => [e.name, posix.join(root, e.path)]), + ) as OpenCodeLaunchArtifacts["paths"], + }; +}