mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-05 03:51:54 +01:00
feat(server): preserve native Runner state across ephemeral deploys
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
1a39dea9aa
commit
5c2764634f
47 files changed
+1902
-107
No files matched your search
@@ -1,5 +1,32 @@
|
||||
# Database
|
||||
|
||||
### Durable native provider backups
|
||||
|
||||
`native_harness_backups` journals encrypted provider-session archives in object
|
||||
storage. One current row per company and session scope selects the published
|
||||
backup. Pending uploads remain recorded after failure; a controller must still
|
||||
own the native run when it publishes. Superseded controllers cannot replace the
|
||||
selected backup. Archive bytes never enter the database.
|
||||
|
||||
The dedicated volume migration imports verified surviving backups without
|
||||
removing their source files or changing task-session pointers. Replacement
|
||||
readiness rejects retained session pointers whose backups remain local. The
|
||||
existing bounded startup retention pass retires closed sessions only after its
|
||||
task, run, lease and age checks pass, then acknowledges object deletion before
|
||||
removing the ledger row. It adds no idle polling.
|
||||
|
||||
The same offline maintenance process imports surviving legacy controller state
|
||||
into `native_runner_checkpoints` and binds its existing session snapshot through
|
||||
`nativeControllerRecovery` metadata. Runner validates the original protocol
|
||||
state and recorded artifact; Core requires a finished run with no live owner.
|
||||
Blocked or pending recovery records remain unchanged while their surviving
|
||||
state is preserved; migration does not mark their work completed.
|
||||
The normal bounded continuation selector handles inherited, unstarted follow-ups.
|
||||
Never-started queued turns keep their task pointers and remain queued. Original
|
||||
files stay intact, canonical database checkpoints take precedence, and missing
|
||||
source files are reported separately rather than fabricated. This migration
|
||||
does not recover provider state that was already lost before enrollment.
|
||||
|
||||
### Proactive agent chat messages
|
||||
|
||||
Migration `0419_slippery_impossible_man.sql` adds an agent-scoped unique constraint
|
||||
|
||||
@@ -781,6 +781,13 @@ case/flow, matcher, result validator, and report test. Follow this sequence:
|
||||
2. Select existing environments and relevant journeys. Add a case only when a
|
||||
promised behavior lacks an independent oracle; keep one durable purpose per
|
||||
suite. Record expected turns, deadline, spending bound, and cleanup obligations.
|
||||
Native sandbox integrations must pass `rolling-web-deployment` and
|
||||
`rolling-web-deployment-warm` before cloud qualification.
|
||||
The shared suite includes registered native Daytona profiles automatically.
|
||||
It replaces the web controller during real work, deletes its entire home,
|
||||
checks continuous reads and writes, and requires a later turn to retain the
|
||||
native and provider session identity after another empty-home replacement.
|
||||
A profile's presence in the matrix is not a passing qualification receipt.
|
||||
3. Make setup create an isolated company, workspace, agent, and secrets through
|
||||
supported APIs. Execute the graded action through ordinary browser/API
|
||||
surfaces. Avoid direct database writes and test-only provider shortcuts.
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
CREATE TABLE IF NOT EXISTS "native_harness_backups" (
|
||||
"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
|
||||
"company_id" uuid NOT NULL,
|
||||
"issue_id" uuid NOT NULL,
|
||||
"run_id" uuid NOT NULL,
|
||||
"scope_digest" text NOT NULL,
|
||||
"manifest_sha256" text NOT NULL,
|
||||
"status" text DEFAULT 'pending' NOT NULL,
|
||||
"reference" jsonb NOT NULL,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS "native_harness_backups_current_idx" ON "native_harness_backups" USING btree ("company_id","scope_digest") WHERE "native_harness_backups"."status" = 'current';--> statement-breakpoint
|
||||
CREATE INDEX IF NOT EXISTS "native_harness_backups_scope_idx" ON "native_harness_backups" USING btree ("company_id","scope_digest","manifest_sha256");
|
||||
+482
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"id": "cb093c1d-3e54-4ab2-8009-9c243097224c",
|
||||
"prevId": "fe0b877c-d63e-46d9-8798-c7729fd13eb7",
|
||||
"id": "809726a1-762c-4c9d-a7fd-55670f1c856f",
|
||||
"prevId": "fb4383f6-cbe6-4aff-9fd2-6eeb69a6927a",
|
||||
"version": "7",
|
||||
"dialect": "postgresql",
|
||||
"tables": {
|
||||
@@ -28825,6 +28825,337 @@
|
||||
"checkConstraints": {},
|
||||
"isRLSEnabled": false
|
||||
},
|
||||
"public.execution_admission_project_charges": {
|
||||
"name": "execution_admission_project_charges",
|
||||
"schema": "",
|
||||
"columns": {
|
||||
"admission_id": {
|
||||
"name": "admission_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"company_id": {
|
||||
"name": "company_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"project_id": {
|
||||
"name": "project_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
}
|
||||
},
|
||||
"indexes": {
|
||||
"execution_admission_project_charges_project_idx": {
|
||||
"name": "execution_admission_project_charges_project_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "company_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "project_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
}
|
||||
},
|
||||
"foreignKeys": {
|
||||
"execution_admission_project_charges_company_admission_fk": {
|
||||
"name": "execution_admission_project_charges_company_admission_fk",
|
||||
"tableFrom": "execution_admission_project_charges",
|
||||
"tableTo": "execution_admissions",
|
||||
"columnsFrom": [
|
||||
"company_id",
|
||||
"admission_id"
|
||||
],
|
||||
"columnsTo": [
|
||||
"company_id",
|
||||
"id"
|
||||
],
|
||||
"onDelete": "cascade",
|
||||
"onUpdate": "no action"
|
||||
},
|
||||
"execution_admission_project_charges_company_project_fk": {
|
||||
"name": "execution_admission_project_charges_company_project_fk",
|
||||
"tableFrom": "execution_admission_project_charges",
|
||||
"tableTo": "projects",
|
||||
"columnsFrom": [
|
||||
"company_id",
|
||||
"project_id"
|
||||
],
|
||||
"columnsTo": [
|
||||
"company_id",
|
||||
"id"
|
||||
],
|
||||
"onDelete": "no action",
|
||||
"onUpdate": "no action"
|
||||
}
|
||||
},
|
||||
"compositePrimaryKeys": {
|
||||
"execution_admission_project_charges_admission_id_project_id_pk": {
|
||||
"name": "execution_admission_project_charges_admission_id_project_id_pk",
|
||||
"columns": [
|
||||
"admission_id",
|
||||
"project_id"
|
||||
]
|
||||
}
|
||||
},
|
||||
"uniqueConstraints": {},
|
||||
"policies": {},
|
||||
"checkConstraints": {},
|
||||
"isRLSEnabled": false
|
||||
},
|
||||
"public.execution_admissions": {
|
||||
"name": "execution_admissions",
|
||||
"schema": "",
|
||||
"columns": {
|
||||
"id": {
|
||||
"name": "id",
|
||||
"type": "uuid",
|
||||
"primaryKey": true,
|
||||
"notNull": true
|
||||
},
|
||||
"company_id": {
|
||||
"name": "company_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"run_id": {
|
||||
"name": "run_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"attempt": {
|
||||
"name": "attempt",
|
||||
"type": "integer",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"primary_project_id": {
|
||||
"name": "primary_project_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"context_revision": {
|
||||
"name": "context_revision",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"admitted_project_ids": {
|
||||
"name": "admitted_project_ids",
|
||||
"type": "uuid[]",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"settlement_id": {
|
||||
"name": "settlement_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"settled_at": {
|
||||
"name": "settled_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"created_at": {
|
||||
"name": "created_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "now()"
|
||||
}
|
||||
},
|
||||
"indexes": {
|
||||
"execution_admissions_company_id_uq": {
|
||||
"name": "execution_admissions_company_id_uq",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "company_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": true,
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"execution_admissions_run_attempt_uq": {
|
||||
"name": "execution_admissions_run_attempt_uq",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "run_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "attempt",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": true,
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"execution_admissions_live_run_uq": {
|
||||
"name": "execution_admissions_live_run_uq",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "run_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": true,
|
||||
"where": "\"execution_admissions\".\"settled_at\" is null",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
}
|
||||
},
|
||||
"foreignKeys": {
|
||||
"execution_admissions_company_id_companies_id_fk": {
|
||||
"name": "execution_admissions_company_id_companies_id_fk",
|
||||
"tableFrom": "execution_admissions",
|
||||
"tableTo": "companies",
|
||||
"columnsFrom": [
|
||||
"company_id"
|
||||
],
|
||||
"columnsTo": [
|
||||
"id"
|
||||
],
|
||||
"onDelete": "no action",
|
||||
"onUpdate": "no action"
|
||||
},
|
||||
"execution_admissions_run_id_heartbeat_runs_id_fk": {
|
||||
"name": "execution_admissions_run_id_heartbeat_runs_id_fk",
|
||||
"tableFrom": "execution_admissions",
|
||||
"tableTo": "heartbeat_runs",
|
||||
"columnsFrom": [
|
||||
"run_id"
|
||||
],
|
||||
"columnsTo": [
|
||||
"id"
|
||||
],
|
||||
"onDelete": "no action",
|
||||
"onUpdate": "no action"
|
||||
}
|
||||
},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"policies": {},
|
||||
"checkConstraints": {
|
||||
"execution_admissions_attempt_check": {
|
||||
"name": "execution_admissions_attempt_check",
|
||||
"value": "\"execution_admissions\".\"attempt\" > 0"
|
||||
},
|
||||
"execution_admissions_settlement_check": {
|
||||
"name": "execution_admissions_settlement_check",
|
||||
"value": "(\"execution_admissions\".\"settlement_id\" is null) = (\"execution_admissions\".\"settled_at\" is null)"
|
||||
},
|
||||
"execution_admissions_project_set_check": {
|
||||
"name": "execution_admissions_project_set_check",
|
||||
"value": "case when \"execution_admissions\".\"admitted_project_ids\" is null\n then \"execution_admissions\".\"settled_at\" is not null else array_position(\"execution_admissions\".\"admitted_project_ids\", null) is null and cardinality(\"execution_admissions\".\"admitted_project_ids\") <= 100 end"
|
||||
}
|
||||
},
|
||||
"isRLSEnabled": false
|
||||
},
|
||||
"public.project_task_execution_policies": {
|
||||
"name": "project_task_execution_policies",
|
||||
"schema": "",
|
||||
"columns": {
|
||||
"project_id": {
|
||||
"name": "project_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": true,
|
||||
"notNull": true
|
||||
},
|
||||
"company_id": {
|
||||
"name": "company_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"max_concurrent_tasks": {
|
||||
"name": "max_concurrent_tasks",
|
||||
"type": "integer",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"created_at": {
|
||||
"name": "created_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "now()"
|
||||
},
|
||||
"updated_at": {
|
||||
"name": "updated_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "now()"
|
||||
}
|
||||
},
|
||||
"indexes": {},
|
||||
"foreignKeys": {
|
||||
"project_task_execution_policies_company_project_fk": {
|
||||
"name": "project_task_execution_policies_company_project_fk",
|
||||
"tableFrom": "project_task_execution_policies",
|
||||
"tableTo": "projects",
|
||||
"columnsFrom": [
|
||||
"company_id",
|
||||
"project_id"
|
||||
],
|
||||
"columnsTo": [
|
||||
"company_id",
|
||||
"id"
|
||||
],
|
||||
"onDelete": "cascade",
|
||||
"onUpdate": "no action"
|
||||
}
|
||||
},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"policies": {},
|
||||
"checkConstraints": {
|
||||
"project_task_execution_policies_maximum_check": {
|
||||
"name": "project_task_execution_policies_maximum_check",
|
||||
"value": "\"project_task_execution_policies\".\"max_concurrent_tasks\" is null or \"project_task_execution_policies\".\"max_concurrent_tasks\" > 0"
|
||||
}
|
||||
},
|
||||
"isRLSEnabled": false
|
||||
},
|
||||
"public.execution_workspace_repositories": {
|
||||
"name": "execution_workspace_repositories",
|
||||
"schema": "",
|
||||
@@ -29999,6 +30330,22 @@
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"execution_workspace_snapshots_imported_children_idx": {
|
||||
"name": "execution_workspace_snapshots_imported_children_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "(\"metadata\"->'importedWorkspaceChildren') jsonb_path_ops",
|
||||
"asc": true,
|
||||
"isExpression": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"execution_workspace_snapshots\".\"metadata\" ? 'importedWorkspaceChildren'",
|
||||
"concurrently": false,
|
||||
"method": "gin",
|
||||
"with": {}
|
||||
}
|
||||
},
|
||||
"foreignKeys": {},
|
||||
@@ -43679,6 +44026,133 @@
|
||||
},
|
||||
"isRLSEnabled": false
|
||||
},
|
||||
"public.native_harness_backups": {
|
||||
"name": "native_harness_backups",
|
||||
"schema": "",
|
||||
"columns": {
|
||||
"id": {
|
||||
"name": "id",
|
||||
"type": "uuid",
|
||||
"primaryKey": true,
|
||||
"notNull": true,
|
||||
"default": "gen_random_uuid()"
|
||||
},
|
||||
"company_id": {
|
||||
"name": "company_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"issue_id": {
|
||||
"name": "issue_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"run_id": {
|
||||
"name": "run_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"scope_digest": {
|
||||
"name": "scope_digest",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"manifest_sha256": {
|
||||
"name": "manifest_sha256",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"status": {
|
||||
"name": "status",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "'pending'"
|
||||
},
|
||||
"reference": {
|
||||
"name": "reference",
|
||||
"type": "jsonb",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"created_at": {
|
||||
"name": "created_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "now()"
|
||||
},
|
||||
"updated_at": {
|
||||
"name": "updated_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "now()"
|
||||
}
|
||||
},
|
||||
"indexes": {
|
||||
"native_harness_backups_current_idx": {
|
||||
"name": "native_harness_backups_current_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "company_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "scope_digest",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": true,
|
||||
"where": "\"native_harness_backups\".\"status\" = 'current'",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"native_harness_backups_scope_idx": {
|
||||
"name": "native_harness_backups_scope_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "company_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "scope_digest",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "manifest_sha256",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
}
|
||||
},
|
||||
"foreignKeys": {},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"policies": {},
|
||||
"checkConstraints": {},
|
||||
"isRLSEnabled": false
|
||||
},
|
||||
"public.native_run_finalizations": {
|
||||
"name": "native_run_finalizations",
|
||||
"schema": "",
|
||||
@@ -62448,6 +62922,12 @@
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"cleanup_verified_at": {
|
||||
"name": "cleanup_verified_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"created_at": {
|
||||
"name": "created_at",
|
||||
"type": "timestamp with time zone",
|
||||
+7
@@ -2990,6 +2990,13 @@
|
||||
"when": 1793743828770,
|
||||
"tag": "0428_retained_admitted_project_sets",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 429,
|
||||
"version": "7",
|
||||
"when": 1793760539457,
|
||||
"tag": "0429_high_freak",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -146,6 +146,7 @@ export { completionContracts } from "./completion_contracts.js";
|
||||
export { nativeRunResults } from "./native_run_results.js";
|
||||
export { nativeRunFinalizations } from "./native_run_finalizations.js";
|
||||
export { nativeRunnerCheckpoints } from "./native_runner_checkpoints.js";
|
||||
export { nativeHarnessBackups } from "./native_harness_backups.js";
|
||||
export { workAssessments } from "./work_assessments.js";
|
||||
export { statusDecisions } from "./status_decisions.js";
|
||||
export { statusDecisionEffects } from "./status_decision_effects.js";
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
import { sql } from "drizzle-orm";
|
||||
import { index, jsonb, pgTable, text, timestamp, uniqueIndex, uuid } from "drizzle-orm/pg-core";
|
||||
|
||||
export const nativeHarnessBackups = pgTable("native_harness_backups", {
|
||||
id: uuid("id").primaryKey().defaultRandom(),
|
||||
companyId: uuid("company_id").notNull(),
|
||||
issueId: uuid("issue_id").notNull(),
|
||||
runId: uuid("run_id").notNull(),
|
||||
scopeDigest: text("scope_digest").notNull(),
|
||||
manifestSha256: text("manifest_sha256").notNull(),
|
||||
status: text("status").notNull().default("pending"),
|
||||
reference: jsonb("reference").$type<Record<string, unknown>>().notNull(),
|
||||
createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
|
||||
updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(),
|
||||
}, table => ({
|
||||
current: uniqueIndex("native_harness_backups_current_idx").on(table.companyId, table.scopeDigest)
|
||||
.where(sql`${table.status} = 'current'`),
|
||||
scope: index("native_harness_backups_scope_idx").on(table.companyId, table.scopeDigest, table.manifestSha256),
|
||||
}));
|
||||
@@ -31,6 +31,7 @@ import { digestPaperclipSemanticContent } from "../semantic-tools/receipts.js";
|
||||
import {
|
||||
DurablePrpControlPlane,
|
||||
decodeDurablePrpCheckpoint,
|
||||
captureLegacyDurablePrpCheckpoint,
|
||||
inspectDurablePrpCheckpointHeader,
|
||||
inspectStagedRunAttachment,
|
||||
inspectRetainedRunInstructions,
|
||||
@@ -220,6 +221,40 @@ it("restores authentication and commands from opaque checkpoint records", async
|
||||
}
|
||||
});
|
||||
|
||||
it("imports the authenticated legacy authority without changing its source and restores it in an empty home", async () => {
|
||||
const root = mkdtempSync(resolve(tmpdir(), "runner-legacy-import-"));
|
||||
const options = { stateDirectory: resolve(root, "old/control-plane"), identity, expectedRunnerVersion, expectedRunnerDigest };
|
||||
const core = new DurablePrpControlPlane(options);
|
||||
let restored: DurablePrpControlPlane | undefined;
|
||||
try {
|
||||
await core.start();
|
||||
const client = (await authenticate(core, core.issueBootstrapTicket()))!;
|
||||
core.queueCommand("session.snapshot", {}, "retained-maintenance");
|
||||
client.socket.destroy();
|
||||
await core.stop();
|
||||
const file = resolve(options.stateDirectory, "control-plane-state.json");
|
||||
const source = readFileSync(file, "utf8");
|
||||
const checkpoint = captureLegacyDurablePrpCheckpoint(source, identity);
|
||||
expect(readFileSync(file, "utf8")).toBe(source);
|
||||
expect(checkpoint.runnerDigest).toBe(expectedRunnerDigest);
|
||||
expect(checkpoint.recoveryEvidence.authenticatedConnectionCount).toBe(1);
|
||||
rmSync(resolve(root, "old"), { recursive: true });
|
||||
checkpoint.restoreSessionDirectory(resolve(root, "new"));
|
||||
restored = new DurablePrpControlPlane({ ...options, stateDirectory: resolve(root, "new/control-plane") });
|
||||
expect(restored.getCommand("retained-maintenance")).toMatchObject({ type: "session.snapshot" });
|
||||
expect(() => captureLegacyDurablePrpCheckpoint(source, { ...identity, runId: "another-run" })).toThrow("binding_mismatch");
|
||||
const state = JSON.parse(source);
|
||||
expect(() => captureLegacyDurablePrpCheckpoint(JSON.stringify({ ...state, tickets: {} }), identity)).toThrow("artifact_unproven");
|
||||
expect(captureLegacyDurablePrpCheckpoint(JSON.stringify({ ...state, tickets: {} }), identity,
|
||||
{ version: expectedRunnerVersion, digest: expectedRunnerDigest }).runnerDigest).toBe(expectedRunnerDigest);
|
||||
expect(() => captureLegacyDurablePrpCheckpoint(source, identity,
|
||||
{ version: expectedRunnerVersion, digest: `sha256:${"0".repeat(64)}` })).toThrow("artifact_unproven");
|
||||
expect(() => captureLegacyDurablePrpCheckpoint(JSON.stringify({ schema: "paperclip.runner.external-control-plane-state.v1", identity }), identity)).toThrow("binding_mismatch");
|
||||
} finally {
|
||||
await restored?.stop(); await core.stop(); rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it.each(["missing record", "invalid state", "wrong sequence", "invalid preparation identity", "duplicate preparation", "malformed header", "malformed cursor"])("rejects %s at checkpoint decoding", async fault => {
|
||||
const root = mkdtempSync(resolve(tmpdir(), "runner-checkpoint-invalid-"));
|
||||
const core = new DurablePrpControlPlane({ stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest });
|
||||
|
||||
@@ -392,6 +392,33 @@ function captureCheckpoint(state: StoredCoreState, runnerVersion: string, runner
|
||||
};
|
||||
}
|
||||
|
||||
export function captureLegacyDurablePrpCheckpoint(contents: string, expected: {
|
||||
runId: string; runnerInstanceId: string; normalizedSessionId: string;
|
||||
}, verifiedArtifact?: { version: string; digest: string }): DurablePrpCheckpoint {
|
||||
if (Buffer.byteLength(contents) > maxStateBytes) throw new Error("native_runner_checkpoint_invalid");
|
||||
const value: unknown = JSON.parse(contents);
|
||||
if (!isRecord(value) || !exactIdentity(value.identity) || !isStoredCoreState(value, value.identity)
|
||||
|| value.identity.runId !== expected.runId || value.identity.runnerInstanceId !== expected.runnerInstanceId
|
||||
|| value.identity.normalizedSessionId !== expected.normalizedSessionId
|
||||
|| checkpointRunId(value) !== expected.runId || value.warmTransition
|
||||
|| value.stagedRunAttachment && !stagedTargetActivated(value)) {
|
||||
throw new Error("native_runner_checkpoint_binding_mismatch");
|
||||
}
|
||||
const artifacts: Array<{ runnerVersion: string; runnerDigest: string }> = value.completedWarmTransition
|
||||
? [value.completedWarmTransition.receipt]
|
||||
: Object.values(value.tickets).filter(ticket => ticket.usedAt !== null
|
||||
&& Number.isFinite(Date.parse(ticket.usedAt))
|
||||
&& canonicalJson(ticket.identity) === canonicalJson(value.identity));
|
||||
if (verifiedArtifact) artifacts.push({ runnerVersion: verifiedArtifact.version, runnerDigest: verifiedArtifact.digest });
|
||||
const artifact = artifacts[0];
|
||||
if (!artifact || typeof artifact.runnerVersion !== "string" || !artifact.runnerVersion
|
||||
|| !runnerDigestPattern.test(artifact.runnerDigest)
|
||||
|| artifacts.some(candidate => candidate.runnerVersion !== artifact.runnerVersion || candidate.runnerDigest !== artifact.runnerDigest)) {
|
||||
throw new Error("native_runner_checkpoint_artifact_unproven");
|
||||
}
|
||||
return decodeDurablePrpCheckpoint(captureCheckpoint(value, artifact.runnerVersion, artifact.runnerDigest).records);
|
||||
}
|
||||
|
||||
function parseCheckpointHeader(contents: string): Record<string, unknown> & {
|
||||
runnerVersion: string; runnerDigest: string;
|
||||
state: Record<string, unknown> & { identity: DurableRecoveryIdentity };
|
||||
|
||||
@@ -46,6 +46,7 @@ export {
|
||||
type DurablePrpControlPlaneOptions,
|
||||
type DurablePrpCheckpoint,
|
||||
decodeDurablePrpCheckpoint,
|
||||
captureLegacyDurablePrpCheckpoint,
|
||||
inspectDurablePrpCheckpointHeader,
|
||||
type PrpWireConnection,
|
||||
type PrpWireAttachment,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import { assertEphemeralServerConfig } from "../services/ephemeral-server-contract.js";
|
||||
import { assertEphemeralWorkspaceExecution, assertEphemeralNativeLifecycle } from "../services/remote-workspaces.js";
|
||||
import { assertEphemeralWorkspaceExecution, assertEphemeralNativeRuntime } from "../services/remote-workspaces.js";
|
||||
import { assertEphemeralAgentInstructions } from "../services/agent-instructions.js";
|
||||
|
||||
const config = { databaseUrl: "postgresql://runtime:fixture@ep-example-pooler.neon.test/tenant", storageProvider: "s3" as const };
|
||||
@@ -24,20 +24,20 @@ it("never connects the listener with migration privileges, another tenant, or th
|
||||
expect(() => assertEphemeralServerConfig(config, { ...env, PAPERCLIP_DATABASE_LISTEN_URL: url })).toThrow("runtime role");
|
||||
}
|
||||
});
|
||||
it("rejects disk-dependent execution and instructions while retaining remote warm execution", () => {
|
||||
it("requires remote workspace authority and native execution independently of provider lifecycle", () => {
|
||||
vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "true");
|
||||
const remote = { externalConversation: false, adapterType: "paperclip_runner", pluginTaskWorkspace: false, remoteAuthority: true };
|
||||
expect(() => assertEphemeralWorkspaceExecution(remote)).not.toThrow();
|
||||
expect(() => assertEphemeralWorkspaceExecution({ ...remote, remoteAuthority: false, pluginTaskWorkspace: true })).not.toThrow();
|
||||
for (const patch of [{ remoteAuthority: false }, { adapterType: "process" }]) {
|
||||
expect(() => assertEphemeralWorkspaceExecution({ ...remote, ...patch })).toThrow("warm Paperclip Runner");
|
||||
expect(() => assertEphemeralWorkspaceExecution({ ...remote, ...patch })).toThrow("native Paperclip Runner");
|
||||
}
|
||||
expect(() => assertEphemeralNativeLifecycle("warm")).not.toThrow();
|
||||
expect(() => assertEphemeralNativeLifecycle("per_turn")).toThrow("warm Paperclip Runner");
|
||||
expect(() => assertEphemeralNativeLifecycle(undefined)).toThrow("warm Paperclip Runner");
|
||||
expect(() => assertEphemeralNativeRuntime(true)).not.toThrow();
|
||||
expect(() => assertEphemeralNativeRuntime(false)).toThrow("native Paperclip Runner");
|
||||
const agent = { id: "agent", companyId: "company", name: "Agent", adapterConfig: {} };
|
||||
expect(() => assertEphemeralAgentInstructions(agent)).not.toThrow();
|
||||
expect(() => assertEphemeralAgentInstructions({ ...agent, adapterConfig: { instructionsBundleMode: "external", instructionsRootPath: "/old-volume" } })).toThrow("managed agent instructions");
|
||||
vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "false");
|
||||
expect(() => assertEphemeralWorkspaceExecution({ ...remote, remoteAuthority: false })).not.toThrow();
|
||||
expect(() => assertEphemeralNativeRuntime(false)).not.toThrow();
|
||||
});
|
||||
@@ -9,6 +9,8 @@ import { skillPackageMaintenance } from "../services/skill-package-maintenance.j
|
||||
import { hasUnsettledLocalInstructionCopies } from "../services/agent-instruction-working-copies.js";
|
||||
import { providerTraceStore } from "../services/provider-trace-store.js";
|
||||
import { migrateWorkspaceTrees } from "../services/remote-workspace-migration-batch.js";
|
||||
import { migrateNativeHarnessBackups } from "../services/native-runtime/native-harness-backup-migration.js";
|
||||
import { migrateNativeControllers } from "../services/native-runtime/native-controller-migration.js";
|
||||
|
||||
/** Runs only in the maintenance process after Cloud has confirmed all legacy
|
||||
* deployments stopped. Each owner retains its own migration/publication rules;
|
||||
@@ -25,12 +27,21 @@ export async function migrateEphemeralServerData(db: Db, input: {
|
||||
const before = await readEphemeralServerDataEligibility(db);
|
||||
const unsupported = before.blockers.filter(blocker => ![
|
||||
"agent_files_require_migration", "run_logs_require_migration", "provider_traces_require_migration", "skills_require_migration", "workspaces_require_migration",
|
||||
"native_harness_backups_require_migration",
|
||||
"native_controllers_require_migration",
|
||||
].includes(blocker));
|
||||
if (unsupported.length) throw new Error(unsupported[0]);
|
||||
|
||||
phase("workspaces");
|
||||
const storage = getStorageService();
|
||||
if (storage.provider !== "s3") throw new Error("shared_object_storage_required");
|
||||
phase("native_controllers");
|
||||
const controllers = await migrateNativeControllers(db, { writersStopped: true, signal });
|
||||
counts.nativeControllers = controllers.imported;
|
||||
counts.nativeControllersUnchanged = controllers.unchanged;
|
||||
counts.nativeControllerSourcesAlreadyAbsent = controllers.sourceAlreadyAbsent;
|
||||
phase("native_harness_backups");
|
||||
counts.nativeHarnessBackups = await migrateNativeHarnessBackups(db, { storage, writersStopped: true, signal });
|
||||
phase("workspaces");
|
||||
counts.workspaces = await migrateWorkspaceTrees(db, { storage, signal, phase });
|
||||
|
||||
phase("agent_files");
|
||||
|
||||
@@ -56,7 +56,7 @@ import {
|
||||
import { environmentService } from "./environments.js";
|
||||
import { instanceSettingsService } from "./instance-settings.js";
|
||||
import { hasSandboxSnapshotQualification, qualifiedSandboxSnapshotId, usesManagedCloudSandboxImage, type SandboxTemplateIdentity } from "./sandbox-snapshot-qualification.js";
|
||||
import { verifyNativeHarnessBackupStamp } from "./native-runtime/native-harness-backup-stamp.js";
|
||||
import { verifyRecoverableNativeHarnessBackup } from "./native-runtime/native-harness-backup-storage.js";
|
||||
import {
|
||||
collectEnvironmentSecretRefs,
|
||||
parseEnvironmentDriverConfig,
|
||||
@@ -1847,10 +1847,8 @@ function createSandboxEnvironmentDriver(
|
||||
if (!providerLease) {
|
||||
if (
|
||||
input.adapterType === "paperclip_runner" &&
|
||||
!verifyNativeHarnessBackupStamp(
|
||||
reusableLease.metadata?.nativeHarnessBackup,
|
||||
reusableLease.providerLeaseId,
|
||||
)
|
||||
!await verifyRecoverableNativeHarnessBackup({ db, companyId: reusableLease.companyId,
|
||||
stamp: reusableLease.metadata?.nativeHarnessBackup, providerLeaseId: reusableLease.providerLeaseId })
|
||||
) {
|
||||
throw new RunnerHarnessBackupUnavailableError(
|
||||
reusableLease.providerLeaseId,
|
||||
@@ -3986,10 +3984,8 @@ export function environmentRuntimeService(
|
||||
"paperclip_runner" &&
|
||||
leaseSnapshot.metadata?.sandboxLeaseAcquisition &&
|
||||
(!leaseSnapshot.providerLeaseId ||
|
||||
!verifyNativeHarnessBackupStamp(
|
||||
leaseSnapshot.metadata.nativeHarnessBackup,
|
||||
leaseSnapshot.providerLeaseId,
|
||||
))
|
||||
!await verifyRecoverableNativeHarnessBackup({ db, companyId: leaseSnapshot.companyId,
|
||||
stamp: leaseSnapshot.metadata.nativeHarnessBackup, providerLeaseId: leaseSnapshot.providerLeaseId }))
|
||||
) {
|
||||
throw new RunnerHarnessBackupUnavailableError(
|
||||
leaseSnapshot.providerLeaseId ?? leaseSnapshot.id,
|
||||
|
||||
@@ -4,6 +4,8 @@ import type { Config } from "../config.js";
|
||||
import { isValidLocalEncryptedMasterKey } from "../secrets/local-encrypted-provider.js";
|
||||
import { agentInstructionsBundleMode } from "./agent-instructions.js";
|
||||
import { skillInstallationMaintenance } from "./skill-installation-maintenance.js";
|
||||
import { hasLocalNativeHarnessObligations } from "./native-runtime/native-harness-backup-migration.js";
|
||||
import { hasLocalNativeControllerObligations } from "./native-runtime/native-controller-migration.js";
|
||||
|
||||
export const ephemeralServerEnabled = () => process.env.PAPERCLIP_EPHEMERAL_SERVER === "true";
|
||||
|
||||
@@ -64,6 +66,8 @@ export async function readEphemeralServerDataEligibility(db: Db, options: { purp
|
||||
isNull(providerTraceRecords.deletedAt), sql`${providerTraceRecords.traceRef} not like 's3-segments-v1:%'`,
|
||||
)).limit(1);
|
||||
if (localTrace) blockers.push("provider_traces_require_migration");
|
||||
if (await hasLocalNativeHarnessObligations(db)) blockers.push("native_harness_backups_require_migration");
|
||||
if (await hasLocalNativeControllerObligations(db)) blockers.push("native_controllers_require_migration");
|
||||
const skills = skillInstallationMaintenance(db);
|
||||
let after: string | undefined;
|
||||
for (;;) {
|
||||
|
||||
@@ -432,14 +432,14 @@ describe.skipIf(!support.supported)("heartbeat runtime selection boundary", () =
|
||||
const { input, run } = await fixture("process");
|
||||
vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "true");
|
||||
expect(input.nativeRuntimeResolution.kind).toBe("legacy");
|
||||
await expect(selectHeartbeatRuntime(db, input)).rejects.toThrow("warm Paperclip Runner");
|
||||
await expect(selectHeartbeatRuntime(db, input)).rejects.toThrow("native Paperclip Runner");
|
||||
expect((await read(run.id)).runtimeMode).toBe("legacy");
|
||||
expect(await coordinators(run.id)).toHaveLength(0);
|
||||
expect(input.onNativeLifecycleSelected).not.toHaveBeenCalled();
|
||||
expect(prepareNativeWorkspaceSync).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(["warm", "per_turn"] as const)("checks resolved %s policy for ephemeral admission", async mode => {
|
||||
it.each(["warm", "per_turn"] as const)("preserves resolved %s policy on an ephemeral server", async mode => {
|
||||
const { input, run } = await fixture();
|
||||
vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "true");
|
||||
// Deliberately disagree with the resolved provider policy in both cases.
|
||||
@@ -447,14 +447,41 @@ describe.skipIf(!support.supported)("heartbeat runtime selection boundary", () =
|
||||
input.workspace.executionTarget = { ...sandbox, reusableLeaseConfigured: true,
|
||||
effectiveCapabilities: { reusableLeases: true } as never,
|
||||
runnerLifecyclePolicy: mode === "warm" ? { mode, idleTimeoutMs: 300_000 } : { mode, idleTimeoutMs: null } };
|
||||
if (mode === "per_turn") {
|
||||
await expect(selectHeartbeatRuntime(db, input)).rejects.toThrow("warm Paperclip Runner");
|
||||
expect(await coordinators(run.id)).toHaveLength(0);
|
||||
} else {
|
||||
const selected = await selectHeartbeatRuntime(db, input);
|
||||
expect(selected.selected && selected.nativeExecution?.session.lifecyclePolicy.mode).toBe("warm");
|
||||
expect((await read(run.id)).runnerProfileJson).toMatchObject({ nativeExecutionInput: { session: { lifecyclePolicy: { mode: "warm" } } } });
|
||||
}
|
||||
const selected = await selectHeartbeatRuntime(db, input);
|
||||
expect(selected.selected && selected.nativeExecution?.session.lifecyclePolicy.mode).toBe(mode);
|
||||
expect((await read(run.id)).runnerProfileJson).toMatchObject({ nativeExecutionInput: { session: { lifecyclePolicy: { mode } } } });
|
||||
expect(await coordinators(run.id)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ provider: "acpx", acpxAgent: "claude", model: "claude-opus-5-5", credentialProvider: "anthropic" as const, method: "subscription" as const },
|
||||
{ provider: "acpx", acpxAgent: "grok", model: "grok-4.7", credentialProvider: "xai" as const, method: "subscription" as const },
|
||||
{ provider: "opencode", model: "openrouter/deepseek/deepseek-v4-flash-0731", credentialProvider: "openrouter" as const, method: "api_key" as const },
|
||||
])("admits managed $provider/$acpxAgent on an ephemeral server without changing credential lifecycle", async profile => {
|
||||
vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "true");
|
||||
const { input, run, issue } = await fixture();
|
||||
const { credentialProvider, method, ...providerConfig } = profile;
|
||||
const adapterConfig = { ...providerConfig, lifecycleMode: "warm" };
|
||||
input.agent = { ...input.agent, adapterConfig };
|
||||
await db.update(agents).set({ adapterConfig }).where(eq(agents.id, input.agent.id));
|
||||
input.workspace.executionTarget = { ...sandbox, reusableLeaseConfigured: true,
|
||||
effectiveCapabilities: { reusableLeases: true } as never, runnerLifecyclePolicy: { mode: "warm", idleTimeoutMs: 300_000 } };
|
||||
input.nativeRuntimeResolution = resolveHeartbeatNativeRuntimeMode({ persisted: run, enabled: true, runtimeConfig: {}, adapterConfig,
|
||||
agent: input.agent, issue, target: input.workspace.executionTarget, workspaceId: null });
|
||||
expect(input.nativeRuntimeResolution.kind).toBe("native");
|
||||
const attribution = { connectionId: randomUUID(), grantId: randomUUID(), provider: credentialProvider, method,
|
||||
mode: "responsible_user" as const, responsibleUserId: "task-owner" };
|
||||
input.config.managedAiRuntime = { sessionIdentity: "account:epoch-1", identity: "account:generation-1",
|
||||
config: { env: {}, managedAiConnection: { ...attribution, identity: "account:generation-1", sessionIdentity: "account:epoch-1" } },
|
||||
attribution, accountName: "Saved account", accountOwnerUserId: "task-owner", home: instructionRoot, cleanup: async () => {} };
|
||||
const selected = await selectHeartbeatRuntime(db, input);
|
||||
if (!selected.selected || !selected.nativeExecution) throw new Error("Expected native selection");
|
||||
expect(selected.nativeExecution.provider.kind).toBe(profile.provider);
|
||||
expect(selected.nativeExecution.session.lifecyclePolicy).toEqual({ mode: "per_turn", idleTimeoutMs: null });
|
||||
expect((await read(run.id)).runnerProfileJson).toMatchObject({
|
||||
nativeExecutionInput: { session: { lifecyclePolicy: { mode: "per_turn" } } },
|
||||
});
|
||||
expect(await coordinators(run.id)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it.each([
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { resolveProviderApiPlacement } from "../native-runtime/provider-api-placement.js";
|
||||
import { readClaimedNativeStopRecovery } from "../native-runtime/native-cancellation-request.js";
|
||||
import { museRunnerBroker } from "../muse-runner-broker.js";
|
||||
import { assertEphemeralNativeLifecycle, readRemoteWorkspaceAuthority } from "../remote-workspaces.js";
|
||||
import { assertEphemeralNativeRuntime, readRemoteWorkspaceAuthority } from "../remote-workspaces.js";
|
||||
import { measureRunTransaction } from "../execution-transaction-timing.js";
|
||||
import { measureRunPhase } from "../execution-preparation-timing.js";
|
||||
import {
|
||||
@@ -269,7 +269,7 @@ export async function selectHeartbeatRuntime(db: Db, input: HeartbeatRuntimeSele
|
||||
const { runtimeConfig, runtimeSkillEntries, runScopedMentionedSkillKeys, instructionCopy, agentIdentity, issueAssigneeOverrides, resolvedConfig, managedAiRuntime } = input.config;
|
||||
const runOptions = input.recovery;
|
||||
const { providerTraceRequested, providerTraceCapture } = input.trace;
|
||||
if (nativeRuntimeResolution.kind !== "native" && !isDotRun) assertEphemeralNativeLifecycle(undefined);
|
||||
if (!isDotRun) assertEphemeralNativeRuntime(nativeRuntimeResolution.kind === "native");
|
||||
let nativeExecution: NativeExecutionInput | null = null;
|
||||
let getNativeFreshSessionHandoff: (() => Promise<string | null>) | undefined;
|
||||
let nativeRunnerInstanceId: string | null = null;
|
||||
@@ -522,9 +522,6 @@ export async function selectHeartbeatRuntime(db: Db, input: HeartbeatRuntimeSele
|
||||
(["openai_dot_mcp", "grok_bot_mcp", "muse_external", "cursor_cloud_api"].includes(nativeRuntimeResolution.profile.backend) || managedAiRuntime && !supportsManagedWarmSession
|
||||
? { mode: "per_turn" as const, idleTimeoutMs: null }
|
||||
: environmentLifecyclePolicy ?? agentLifecyclePolicy);
|
||||
if (!["openai_dot_mcp", "grok_bot_mcp", "muse_external", "cursor_cloud_api"].includes(nativeRuntimeResolution.profile.backend)) {
|
||||
assertEphemeralNativeLifecycle(effectiveLifecyclePolicy.mode);
|
||||
}
|
||||
if (
|
||||
effectiveLifecyclePolicy.mode === "warm" &&
|
||||
executionTarget?.kind === "remote" &&
|
||||
|
||||
@@ -88,13 +88,14 @@ postgresTests("upgrade readiness persisted execution evidence", () => {
|
||||
finally { await core.stop(); await rm(root, { recursive: true, force: true }); }
|
||||
expect(await readUpgradeReadiness(db, input)).toMatchObject({ ready: true, recoverableRuns: 1 });
|
||||
const cipher = createLocalEncryptedCipher("write");
|
||||
// Real staging headers contain retained launch metadata around 113 KiB.
|
||||
const representativeHeader = JSON.stringify({ ...JSON.parse(header), padding: "x".repeat(100_000) });
|
||||
const parsedHeader = JSON.parse(header);
|
||||
const representativeHeader = JSON.stringify({ ...parsedHeader, state: { ...parsedHeader.state,
|
||||
runAttachTemplate: { provider: { instructions: "x".repeat(300_000) } } } });
|
||||
await db.update(nativeRunnerCheckpoints).set({ material: cipher.encrypt(JSON.stringify({ ...binding, part: "header", contents: representativeHeader })) })
|
||||
.where(eq(nativeRunnerCheckpoints.runId, runId));
|
||||
expect((await readUpgradeReadiness(db, input)).ready).toBe(true);
|
||||
for (const contents of ["{", "{}", JSON.stringify({ ...JSON.parse(header), schema: "paperclip.runner.controller-checkpoint.v1" }),
|
||||
JSON.stringify({ ...JSON.parse(header), padding: "x".repeat(400_000) }),
|
||||
JSON.stringify({ ...JSON.parse(header), padding: "x".repeat(1_100_000) }),
|
||||
JSON.stringify({ ...JSON.parse(header), state: { ...JSON.parse(header).state, identity: { ...JSON.parse(header).state.identity, runId: randomUUID() } } })]) {
|
||||
await db.update(nativeRunnerCheckpoints).set({ material: cipher.encrypt(JSON.stringify({ ...binding, part: "header", contents })) })
|
||||
.where(eq(nativeRunnerCheckpoints.runId, runId));
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { sql } from "drizzle-orm";
|
||||
import { remoteWorkspaceRunReferenceSchema } from "@paperclipai/shared";
|
||||
import { inspectNativeRunnerUpgradeHeader } from "../native-runtime/native-runner-checkpoint.js";
|
||||
import { inspectNativeRunnerUpgradeHeader, NATIVE_CHECKPOINT_HEADER_INSPECTION_MAX_BYTES,
|
||||
NATIVE_CHECKPOINT_INSPECTION_TOTAL_MAX_BYTES } from "../native-runtime/native-runner-checkpoint.js";
|
||||
import { readNativeWorkspaceSyncReference } from "../native-runtime/native-workspace-sync.js";
|
||||
import { REMOTE_RUNNER_RECONNECT_GRACE_MS } from "../native-runtime/native-runner-recovery-contract.js";
|
||||
import { currentNativeControllerIdentity } from "../native-runtime/native-restart-recovery.js";
|
||||
@@ -30,7 +31,8 @@ export async function readUpgradeReadiness(db: Db, input: {
|
||||
r.runner_profile_json->'nativeWorkspaceSync' as workspace,
|
||||
r.runner_profile_json->'remoteWorkspace' as "remoteWorkspace",
|
||||
r.runner_profile_json ? 'remoteWorkspace' as "hasRemoteWorkspace", p.id as "pluginLeaseId",
|
||||
case when pg_column_size(c.material) <= 262144 and sum(pg_column_size(c.material)) over () <= 8388608 then c.material else null end as material, (
|
||||
case when pg_column_size(c.material) <= ${NATIVE_CHECKPOINT_HEADER_INSPECTION_MAX_BYTES}
|
||||
and sum(pg_column_size(c.material)) over () <= ${NATIVE_CHECKPOINT_INSPECTION_TOTAL_MAX_BYTES} then c.material else null end as material, (
|
||||
r.runtime_mode = 'native' and a.adapter_type = 'paperclip_runner'
|
||||
and r.runner_instance_id is not null and r.native_session_id is not null
|
||||
and r.process_pid is not null and r.process_started_at is not null
|
||||
|
||||
@@ -444,7 +444,7 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace
|
||||
const resolve = vi.spyOn(input.services, "resolveWorkspaceForRun").mockRejectedValue(new Error("local folder retained"));
|
||||
process.env.PAPERCLIP_EPHEMERAL_SERVER = String(ephemeral);
|
||||
await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow(ephemeral
|
||||
? "Volume-free servers require warm Paperclip Runner execution with remote workspace storage." : "local folder retained");
|
||||
? "Volume-free servers require native Paperclip Runner execution with remote workspace storage." : "local folder retained");
|
||||
if (ephemeral) expect(resolve).not.toHaveBeenCalled();
|
||||
else expect(resolve).toHaveBeenCalledOnce();
|
||||
expect(await db.select().from(executionWorkspaces)).toHaveLength(0);
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import { agentTaskSessions, heartbeatRuns, issues, nativeRunFinalizations, nativeRunnerCheckpoints } from "@paperclipai/db";
|
||||
import { DurablePrpControlPlane, parseNativeExecutionInput } from "../../vendor/paperclip-runner/index.js";
|
||||
import { describeEmbeddedPostgres, nativeRestartFixture } from "../../__tests__/helpers/native-restart.js";
|
||||
import { nativeToolContractFingerprintForTarget } from "./native-session-resume.js";
|
||||
import { nativeControllerMigrationDescriptor } from "./native-session-executor.js";
|
||||
import { hasLocalNativeControllerObligations, migrateNativeControllers } from "./native-controller-migration.js";
|
||||
import { resolveNativeControllerRecovery } from "./native-controller-recovery.js";
|
||||
import { resolveRemoteProviderPackAuthority } from "./remote-provider-pack-authority.js";
|
||||
import { canonicalProviderPackJson } from "./remote-provider-pack-verification.js";
|
||||
import { nativeSha256 } from "./canonical.js";
|
||||
|
||||
describeEmbeddedPostgres("offline native controller migration", () => {
|
||||
const environment = nativeRestartFixture();
|
||||
afterEach(() => vi.unstubAllEnvs());
|
||||
it("imports an exact settled authority once, preserves queued work, and resumes from an empty home", async () => {
|
||||
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "73".repeat(32));
|
||||
const f = await environment.seedRun(randomUUID());
|
||||
const { companyId, agentId, paperclipHome, runtimeRoot } = environment.current;
|
||||
const binding = { companyId, agentId, issueId: f.issueId, runId: f.runId, executionWorkspaceId: f.native.environmentLeaseId };
|
||||
const execution = parseNativeExecutionInput({
|
||||
schema: "paperclip.native-execution-input.v1", provider: { kind: "codex", model: null }, binding,
|
||||
task: { identifier: "MIG-1", title: "Retained work", description: null, prompt: "Continue the work.", workMode: "standard" },
|
||||
workspace: { cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null },
|
||||
session: { normalizedSessionId: f.native.normalizedSessionId, driverKind: "codex_app_server", protocolVersion: 1,
|
||||
lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } },
|
||||
completionContract: { id: randomUUID(), sha256: "fixture", schemaVersion: "paperclip.completion-contract.v1",
|
||||
contract: { revision: "1", objective: "Continue", criteria: [{ id: "objective", requirement: "Save work." }] } },
|
||||
credentialBindings: [], interactionResponses: [],
|
||||
});
|
||||
const fingerprint = nativeToolContractFingerprintForTarget("remote", "codex");
|
||||
const paths = nativeControllerMigrationDescriptor(execution, fingerprint);
|
||||
const identity = { runId: f.runId, runnerInstanceId: f.native.runnerInstanceId, normalizedSessionId: f.native.normalizedSessionId,
|
||||
environmentLeaseId: f.native.environmentLeaseId, turnId: f.native.turnId, itemId: f.native.itemId };
|
||||
const core = new DurablePrpControlPlane({ stateDirectory: join(paths.root, "control-plane"), identity,
|
||||
expectedRunnerVersion: "migration-fixture", expectedRunnerDigest: `sha256:${"a".repeat(64)}` });
|
||||
core.issueBootstrapTicket(); core.queueCommand("session.snapshot", {}, "retained-command"); await core.stop();
|
||||
const file = join(paths.root, "control-plane/control-plane-state.json");
|
||||
const state = JSON.parse(await readFile(file, "utf8"));
|
||||
state.tickets = {};
|
||||
const source = JSON.stringify(state); await writeFile(file, source, { mode: 0o600 });
|
||||
const packBinding = { companyId, agentId, workspaceId: binding.executionWorkspaceId,
|
||||
sessionId: f.native.normalizedSessionId, runnerInstanceId: f.native.runnerInstanceId,
|
||||
environmentLeaseId: f.native.environmentLeaseId, remoteCwd: "/remote/workspace", providerDigest: nativeSha256(execution.provider) };
|
||||
const payload = {
|
||||
runnerSourceRevision: "a".repeat(40), target: { platform: "linux", architecture: "x64" },
|
||||
pins: { opencode: "1.18.34" }, artifacts: {
|
||||
nodeCommand: { path: "node", sha256: `sha256:${"1".repeat(64)}` },
|
||||
opencodeCommand: { path: "opencode", sha256: `sha256:${"2".repeat(64)}` },
|
||||
},
|
||||
};
|
||||
const pack = resolveRemoteProviderPackAuthority({ stateDirectory: paths.root, reattach: false, binding: packBinding,
|
||||
current: () => ({ schema: "paperclip-runner/remote-provider-pack/v1", payload,
|
||||
digest: `sha256:${createHash("sha256").update(canonicalProviderPackJson(payload)).digest("hex")}` }),
|
||||
runnerArtifact: () => ({ version: "migration-fixture", digest: `sha256:${"a".repeat(64)}` }) });
|
||||
pack.verified();
|
||||
const snapshot = { backendKind: "mock", sessionId: f.native.normalizedSessionId,
|
||||
identity: { companyId, agentId, issueId: f.issueId, runId: f.runId, sessionId: f.native.normalizedSessionId } };
|
||||
const profile = { ...f.preparedProfile, nativeExecutionInput: execution, nativeToolContractFingerprint: fingerprint, sessionCheckpoint: snapshot,
|
||||
nativeWorkspaceSync: { schema: "paperclip.native-workspace-sync/v1", state: "prepared", descriptorSha256: "a".repeat(64),
|
||||
baselineSha256: "b".repeat(64), finalHostSha256: null, workspaceId: randomUUID(), leaseId: f.native.environmentLeaseId,
|
||||
providerLeaseId: randomUUID(), remoteCwd: "/remote/workspace", resourceDisposition: "destroy" } };
|
||||
await f.db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date(), runnerProfileJson: profile }).where(eq(heartbeatRuns.id, f.runId));
|
||||
await f.db.update(nativeRunFinalizations).set({ phase: "terminal_failure", leaseOwner: null }).where(eq(nativeRunFinalizations.runId, f.runId));
|
||||
const queuedId = randomUUID();
|
||||
await f.db.insert(heartbeatRuns).values({ id: queuedId, companyId, agentId, issueId: f.issueId, status: "queued" });
|
||||
await f.db.update(issues).set({ executionRunId: queuedId }).where(eq(issues.id, f.issueId));
|
||||
const [session] = await f.db.insert(agentTaskSessions).values({ companyId, agentId, adapterType: "paperclip_runner",
|
||||
taskKey: f.issueId, lastRunId: f.runId }).returning();
|
||||
expect(await hasLocalNativeControllerObligations(f.db)).toBe(true);
|
||||
await f.db.update(heartbeatRuns).set({ runnerProfileJson: { ...profile,
|
||||
nativeWorkspaceSync: { ...profile.nativeWorkspaceSync, leaseId: randomUUID() } } }).where(eq(heartbeatRuns.id, f.runId));
|
||||
await expect(migrateNativeControllers(f.db, { writersStopped: true, signal: AbortSignal.timeout(30_000) }))
|
||||
.rejects.toThrow("native_controller_migration_artifact_binding");
|
||||
await f.db.update(heartbeatRuns).set({ runnerProfileJson: profile }).where(eq(heartbeatRuns.id, f.runId));
|
||||
await f.db.update(nativeRunFinalizations).set({ leaseOwner: "unsettled-owner" }).where(eq(nativeRunFinalizations.runId, f.runId));
|
||||
await expect(migrateNativeControllers(f.db, { writersStopped: true, signal: AbortSignal.timeout(30_000) })).rejects.toThrow("normal_ownership_recovery_required");
|
||||
expect(await f.db.select().from(nativeRunnerCheckpoints).where(eq(nativeRunnerCheckpoints.runId, f.runId))).toHaveLength(0);
|
||||
await f.db.update(nativeRunFinalizations).set({ leaseOwner: "expired-owner", leaseExpiresAt: new Date(0),
|
||||
phase: "retryable_failure", recoveryState: "resuming_session" }).where(eq(nativeRunFinalizations.runId, f.runId));
|
||||
const [preservedOwner] = await f.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId));
|
||||
expect(await migrateNativeControllers(f.db, { writersStopped: true, signal: AbortSignal.timeout(30_000) }))
|
||||
.toMatchObject({ imported: 1, sourceAlreadyAbsent: 0 });
|
||||
expect(await migrateNativeControllers(f.db, { writersStopped: true, signal: AbortSignal.timeout(30_000) })).toMatchObject({ unchanged: 1, imported: 0 });
|
||||
expect(await hasLocalNativeControllerObligations(f.db)).toBe(false);
|
||||
expect((await f.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)))[0]).toEqual(preservedOwner);
|
||||
expect(await readFile(file, "utf8")).toBe(source);
|
||||
expect((await f.db.select().from(agentTaskSessions).where(eq(agentTaskSessions.id, session.id)))[0].lastRunId).toBe(f.runId);
|
||||
expect((await f.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, queuedId)))[0].status).toBe("queued");
|
||||
await rm(paperclipHome, { recursive: true, force: true });
|
||||
await f.db.update(heartbeatRuns).set({ nativeIssueId: f.issueId, nativeSessionId: f.native.normalizedSessionId,
|
||||
status: "running", runtimeMode: "native" }).where(eq(heartbeatRuns.id, queuedId));
|
||||
const recovered = await resolveNativeControllerRecovery(f.db, { companyId, agentId, issueId: f.issueId,
|
||||
normalizedSessionId: f.native.normalizedSessionId, currentRunId: queuedId });
|
||||
expect(recovered.kind).toBe("recoverable");
|
||||
if (recovered.kind !== "recoverable") throw new Error(JSON.stringify(recovered));
|
||||
const replacement = join(runtimeRoot, "replacement"); await mkdir(replacement, { mode: 0o700 });
|
||||
recovered.checkpoint.restoreSessionDirectory(replacement);
|
||||
expect(JSON.parse(await readFile(join(replacement, "control-plane/control-plane-state.json"), "utf8")).identity).toEqual(identity);
|
||||
expect(recovered.snapshot).toEqual(snapshot);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,142 @@
|
||||
import { constants } from "node:fs";
|
||||
import { lstat, open, readdir } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { heartbeatRuns, type Db } from "@paperclipai/db";
|
||||
import { parseNativeExecutionInput } from "../../vendor/paperclip-runner/index.js";
|
||||
import { offlineMigrationExecutionWhere } from "../offline-migration-execution.js";
|
||||
import { findNativeSessionResumeRun } from "./native-session-resume.js";
|
||||
import { nativeControllerMigrationDescriptor } from "./native-session-executor.js";
|
||||
import { hasNativeRunnerCheckpoint, loadNativeRunnerCheckpoint } from "./native-runner-checkpoint.js";
|
||||
import { importNativeControllerRecovery, readNativeControllerRecovery } from "./native-controller-recovery.js";
|
||||
import { nativeHarnessStateBase } from "./native-harness-backup-stamp.js";
|
||||
import { providerPackWorkspaceId, resolveRemoteProviderPackAuthority } from "./remote-provider-pack-authority.js";
|
||||
import { nativeSha256 } from "./canonical.js";
|
||||
import { readNativeWorkspaceSyncReference } from "./native-workspace-sync.js";
|
||||
|
||||
type Selected = { id: string; companyId: string; agentId: string; issueId: string; sessionId: string; createdAt: Date };
|
||||
|
||||
async function* retainedSources(db: Db, signal?: AbortSignal) {
|
||||
let after: string | undefined;
|
||||
for (;;) {
|
||||
signal?.throwIfAborted();
|
||||
const rows = await db.execute<Selected>(sql`select s.id, s.company_id as "companyId", s.agent_id as "agentId",
|
||||
r.native_issue_id as "issueId", r.native_session_id as "sessionId", r.created_at as "createdAt"
|
||||
from agent_task_sessions s join heartbeat_runs r on r.id = s.last_run_id
|
||||
and r.company_id = s.company_id and r.agent_id = s.agent_id
|
||||
where s.adapter_type = 'paperclip_runner' and r.native_issue_id is not null and r.native_session_id is not null
|
||||
${after ? sql`and s.id > ${after}::uuid` : sql``} order by s.id limit 100`);
|
||||
for (const selected of rows) {
|
||||
signal?.throwIfAborted();
|
||||
const source = await findNativeSessionResumeRun(db, { companyId: selected.companyId, agentId: selected.agentId,
|
||||
issueId: selected.issueId, normalizedSessionId: selected.sessionId,
|
||||
beforeCreatedAt: new Date(selected.createdAt), includeProgressBarrier: true });
|
||||
if (source) yield { selected, source };
|
||||
}
|
||||
if (rows.length < 100) return;
|
||||
after = rows.at(-1)!.id;
|
||||
}
|
||||
}
|
||||
|
||||
async function readPrivateSource(file: string, maxBytes: number): Promise<string | undefined> {
|
||||
const handle = await open(file, constants.O_RDONLY | constants.O_NOFOLLOW).catch((error: NodeJS.ErrnoException) => {
|
||||
if (error.code === "ENOENT") return undefined; throw error;
|
||||
});
|
||||
if (!handle) return undefined;
|
||||
try {
|
||||
const stat = await handle.stat();
|
||||
if (!stat.isFile() || stat.nlink !== 1 || stat.size > maxBytes) throw new Error("native_controller_migration_unsafe_source");
|
||||
return await handle.readFile("utf8");
|
||||
} finally { await handle.close(); }
|
||||
}
|
||||
|
||||
async function localPaths(source: NonNullable<Awaited<ReturnType<typeof findNativeSessionResumeRun>>>) {
|
||||
const execution = parseNativeExecutionInput(source.runnerProfileJson.nativeExecutionInput);
|
||||
const descriptor = nativeControllerMigrationDescriptor(execution, source.runnerProfileJson.nativeToolContractFingerprint);
|
||||
const root = await lstat(descriptor.root).catch((error: NodeJS.ErrnoException) => {
|
||||
if (error.code === "ENOENT") return undefined; throw error;
|
||||
});
|
||||
if (root && (!root.isDirectory() || root.isSymbolicLink())) throw new Error("native_controller_migration_unsafe_source");
|
||||
const control = join(descriptor.root, "control-plane");
|
||||
const directory = await lstat(control).catch((error: NodeJS.ErrnoException) => {
|
||||
if (error.code === "ENOENT") return undefined; throw error;
|
||||
});
|
||||
if (directory && (!directory.isDirectory() || directory.isSymbolicLink())) throw new Error("native_controller_migration_unsafe_source");
|
||||
return { ...descriptor, execution, controlFile: join(control, "control-plane-state.json") };
|
||||
}
|
||||
|
||||
export async function hasLocalNativeControllerObligations(db: Db): Promise<boolean> {
|
||||
const roots = await readdir(nativeHarnessStateBase()).catch((error: NodeJS.ErrnoException) => {
|
||||
if (error.code === "ENOENT") return [] as string[]; throw error;
|
||||
});
|
||||
if (!roots.length) return false;
|
||||
for await (const { selected, source } of retainedSources(db)) {
|
||||
const binding = { companyId: selected.companyId, issueId: selected.issueId, runId: source.id };
|
||||
if (readNativeControllerRecovery(source.runnerProfileJson.nativeControllerRecovery, source.runnerProfileJson.sessionCheckpoint)
|
||||
&& await hasNativeRunnerCheckpoint(db, binding)) continue;
|
||||
const paths = await localPaths(source);
|
||||
const present = await lstat(paths.controlFile).catch((error: NodeJS.ErrnoException) => {
|
||||
if (error.code === "ENOENT") return undefined; throw error;
|
||||
});
|
||||
if (present || await hasNativeRunnerCheckpoint(db, binding)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export async function migrateNativeControllers(db: Db, input: { writersStopped: true; signal: AbortSignal }) {
|
||||
const [active] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(offlineMigrationExecutionWhere()).limit(1);
|
||||
if (!input.writersStopped || active) throw new Error("native_controller_migration_requires_offline_writers");
|
||||
const counts = { inspected: 0, imported: 0, unchanged: 0, sourceAlreadyAbsent: 0 };
|
||||
for await (const { selected, source } of retainedSources(db, input.signal)) {
|
||||
counts.inspected++;
|
||||
const binding = { companyId: selected.companyId, issueId: selected.issueId, runId: source.id };
|
||||
const checkpoint = await loadNativeRunnerCheckpoint(db, binding);
|
||||
const metadata = readNativeControllerRecovery(source.runnerProfileJson.nativeControllerRecovery, source.runnerProfileJson.sessionCheckpoint);
|
||||
const snapshot = source.runnerProfileJson.sessionCheckpoint as { identity?: Record<string, unknown> } | null;
|
||||
if (checkpoint && metadata?.checkpoint?.runId === source.id
|
||||
&& metadata.checkpoint.runnerInstanceId === source.runnerInstanceId
|
||||
&& metadata.checkpoint.normalizedSessionId === selected.sessionId
|
||||
&& snapshot?.identity?.runId === source.id && snapshot.identity.companyId === selected.companyId
|
||||
&& snapshot.identity.agentId === selected.agentId && snapshot.identity.issueId === selected.issueId
|
||||
&& snapshot.identity.sessionId === selected.sessionId
|
||||
&& checkpoint.recoveryEvidence.runnerInstanceId === source.runnerInstanceId
|
||||
&& checkpoint.recoveryEvidence.normalizedSessionId === selected.sessionId) { counts.unchanged++; continue; }
|
||||
const paths = await localPaths(source);
|
||||
const legacyControllerState = checkpoint ? undefined : await readPrivateSource(paths.controlFile, 256 * 1024 * 1024);
|
||||
if (!checkpoint && legacyControllerState === undefined) { counts.sourceAlreadyAbsent++; continue; }
|
||||
let legacyRunnerArtifact: { version: string; digest: string } | undefined;
|
||||
if (legacyControllerState !== undefined && await readPrivateSource(join(paths.root, "provider-pack-authority.json"), 512 * 1024)) {
|
||||
const identity = JSON.parse(legacyControllerState).identity;
|
||||
const execution = paths.execution;
|
||||
const [admission] = await db.select({ workspace: sql<unknown>`${heartbeatRuns.runnerProfileJson}->'nativeWorkspaceSync'` })
|
||||
.from(heartbeatRuns).where(and(eq(heartbeatRuns.id, source.id), eq(heartbeatRuns.companyId, selected.companyId),
|
||||
eq(heartbeatRuns.agentId, selected.agentId)));
|
||||
const workspace = readNativeWorkspaceSyncReference(admission?.workspace);
|
||||
const remoteCwd = workspace?.remoteCwd ?? execution.workspace.cwd;
|
||||
if (typeof identity?.environmentLeaseId !== "string" || !remoteCwd
|
||||
|| (workspace && workspace.leaseId !== identity.environmentLeaseId)) {
|
||||
throw new Error("native_controller_migration_artifact_binding");
|
||||
}
|
||||
const unavailable = (): never => { throw new Error("native_controller_migration_artifact_unavailable"); };
|
||||
legacyRunnerArtifact = resolveRemoteProviderPackAuthority({ stateDirectory: paths.root, reattach: true,
|
||||
binding: { companyId: selected.companyId, agentId: selected.agentId,
|
||||
workspaceId: providerPackWorkspaceId(execution.binding.executionWorkspaceId, source.id, identity.environmentLeaseId),
|
||||
sessionId: selected.sessionId, runnerInstanceId: source.runnerInstanceId!,
|
||||
environmentLeaseId: identity.environmentLeaseId, remoteCwd,
|
||||
providerDigest: nativeSha256(execution.provider) },
|
||||
current: unavailable, runnerArtifact: unavailable }).runnerArtifact;
|
||||
}
|
||||
const supervisor = paths.execution.session.lifecyclePolicy.mode === "warm"
|
||||
? await readPrivateSource(paths.supervisorPath, 4 * 1024 * 1024) : undefined;
|
||||
input.signal.throwIfAborted();
|
||||
const result = await importNativeControllerRecovery(db, { ...binding, agentId: selected.agentId,
|
||||
normalizedSessionId: selected.sessionId, expectedConfigDigest: paths.configDigest,
|
||||
supervisor: supervisor === undefined ? undefined : JSON.parse(supervisor),
|
||||
offline: { writersStopped: true, legacyControllerState, legacyRunnerArtifact } });
|
||||
if (result.kind !== "imported" && result.kind !== "unchanged") {
|
||||
throw new Error(`native_controller_migration_blocked:${source.id}:${"reason" in result ? result.reason : result.kind}`);
|
||||
}
|
||||
counts[result.kind]++;
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { sql } from "drizzle-orm";
|
||||
import { inspectNativeRunnerUpgradeHeader } from "./native-runner-checkpoint.js";
|
||||
import { inspectNativeRunnerUpgradeHeader, NATIVE_CHECKPOINT_HEADER_INSPECTION_MAX_BYTES,
|
||||
NATIVE_CHECKPOINT_INSPECTION_TOTAL_MAX_BYTES } from "./native-runner-checkpoint.js";
|
||||
|
||||
/** An explicit, paginated diagnostic. Never called by an idle/background scan.
|
||||
* This inventories selected retained task-session pointers, not all history.
|
||||
@@ -23,7 +24,8 @@ export async function readNativeControllerRecoveryInventory(db: Db, input: { aft
|
||||
r.native_session_id as "sessionId", r.runner_instance_id as "runnerId",
|
||||
case when pg_column_size(r.runner_profile_json->'nativeControllerRecovery') <= 4096
|
||||
then r.runner_profile_json->'nativeControllerRecovery' else null end as metadata,
|
||||
case when pg_column_size(c.material) <= 262144 and sum(pg_column_size(c.material)) over () <= 8388608
|
||||
case when pg_column_size(c.material) <= ${NATIVE_CHECKPOINT_HEADER_INSPECTION_MAX_BYTES}
|
||||
and sum(pg_column_size(c.material)) over () <= ${NATIVE_CHECKPOINT_INSPECTION_TOTAL_MAX_BYTES}
|
||||
then c.material else null end as material,
|
||||
(r.finished_at is null or r.status not in ('succeeded', 'failed', 'cancelled', 'interrupted', 'timed_out')
|
||||
or f.run_id is null or f.lease_owner is not null or f.recovery_state is not null
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
import { and, eq, ne, sql } from "drizzle-orm";
|
||||
import { environmentLeases, heartbeatRuns, issues, nativeRunFinalizations, nativeRunnerCheckpoints, type Db } from "@paperclipai/db";
|
||||
import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target";
|
||||
import type { DurablePrpCheckpoint, PersistedNativeSession } from "../../vendor/paperclip-runner/index.js";
|
||||
import { captureLegacyDurablePrpCheckpoint, parseNativeExecutionInput, type DurablePrpCheckpoint, type PersistedNativeSession } from "../../vendor/paperclip-runner/index.js";
|
||||
import { notifyNativeFinalizationWork } from "../native-finalization-notifications.js";
|
||||
import { readRunCancellation } from "../run-cancellation.js";
|
||||
import { nativeSha256 } from "./canonical.js";
|
||||
import { findNativeSessionResumeRun, isNativeSessionId, isUnusedNativeSessionBootstrap, nativeSessionProviderEvidence } from "./native-session-resume.js";
|
||||
import { loadNativeRunnerCheckpoint } from "./native-runner-checkpoint.js";
|
||||
import { insertMigratedNativeRunnerCheckpoint, loadNativeRunnerCheckpoint } from "./native-runner-checkpoint.js";
|
||||
import { offlineMigrationExecutionWhere } from "../offline-migration-execution.js";
|
||||
import { redactRunnerApiPayloads } from "./runner-api-log-redaction.js";
|
||||
|
||||
export const NATIVE_CONTROLLER_RECOVERY_MAX_BYTES = 4 * 1024;
|
||||
@@ -223,10 +224,11 @@ export async function resolveNativeControllerRecovery(db: Db, input: {
|
||||
export async function importNativeControllerRecovery(db: Db, input: {
|
||||
companyId: string; agentId: string; issueId: string; runId: string; normalizedSessionId: string;
|
||||
expectedConfigDigest: string;
|
||||
supervisor: unknown;
|
||||
supervisor?: unknown;
|
||||
offline?: { writersStopped: true; legacyControllerState?: string; legacyRunnerArtifact?: { version: string; digest: string } };
|
||||
}): Promise<{ kind: "imported" | "unchanged" } | Exclude<NativeControllerRecoveryOutcome, { kind: "recoverable" } | { kind: "no_prior_execution" }>> {
|
||||
const supervisor = record(input.supervisor);
|
||||
if (supervisor.schema !== "paperclip.native-session-supervisor.v1" || supervisor.configDigest !== input.expectedConfigDigest) {
|
||||
if (input.supervisor !== undefined && (supervisor.schema !== "paperclip.native-session-supervisor.v1" || supervisor.configDigest !== input.expectedConfigDigest)) {
|
||||
return { kind: "unsupported", reason: "supervisor_compatibility" };
|
||||
}
|
||||
return db.transaction(async tx => {
|
||||
@@ -236,25 +238,48 @@ export async function importNativeControllerRecovery(db: Db, input: {
|
||||
eq(nativeRunFinalizations.companyId, input.companyId), eq(nativeRunFinalizations.issueId, input.issueId),
|
||||
eq(nativeRunFinalizations.runId, input.runId))).for("update");
|
||||
if (!task || !owner) return { kind: "missing_evidence" as const, reason: "owner_binding" };
|
||||
if (owner.leaseOwner || task.executionRunId) {
|
||||
if (input.offline) {
|
||||
const [active] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(offlineMigrationExecutionWhere()).limit(1);
|
||||
if (!input.offline.writersStopped || active) return { kind: "missing_evidence" as const, reason: "offline_writers_required" };
|
||||
}
|
||||
const [clock] = await tx.execute<{ now: Date }>(sql`select clock_timestamp() as now`);
|
||||
const liveOwner = owner.leaseOwner && (!owner.leaseExpiresAt || owner.leaseExpiresAt > new Date(clock.now));
|
||||
if (input.offline ? liveOwner : owner.leaseOwner || task.executionRunId) {
|
||||
return owner.leaseExpiresAt && owner.leaseExpiresAt > new Date()
|
||||
? { kind: "waiting_for_owner" as const, deadline: owner.leaseExpiresAt }
|
||||
: { kind: "missing_evidence" as const, reason: "normal_ownership_recovery_required" };
|
||||
}
|
||||
if (owner.phase !== "committed" || !owner.resultId || owner.recoveryState) {
|
||||
if (!input.offline && (!(owner.phase === "committed" && owner.resultId) || owner.recoveryState)) {
|
||||
return { kind: "missing_evidence" as const, reason: "pending_finalization" };
|
||||
}
|
||||
const [run] = await tx.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, input.runId),
|
||||
eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.agentId, input.agentId),
|
||||
eq(heartbeatRuns.nativeIssueId, input.issueId), eq(heartbeatRuns.nativeSessionId, input.normalizedSessionId))).for("update");
|
||||
const profile = record(run?.runnerProfileJson), snapshot = record(profile.sessionCheckpoint), identity = record(snapshot.identity);
|
||||
if (run?.runtimeMode !== "native" || run.status !== "succeeded" || !run.finishedAt
|
||||
if (run?.runtimeMode !== "native" || !(run.status === "succeeded" || input.offline && ["failed", "cancelled", "interrupted", "timed_out"].includes(run.status)) || !run.finishedAt
|
||||
|| identity.runId !== input.runId || identity.companyId !== input.companyId || identity.issueId !== input.issueId
|
||||
|| identity.agentId !== input.agentId || identity.sessionId !== input.normalizedSessionId
|
||||
|| nativeSha256(snapshot) !== nativeSha256(redactRunnerApiPayloads(supervisor.snapshot))) {
|
||||
|| input.supervisor !== undefined && nativeSha256(snapshot) !== nativeSha256(redactRunnerApiPayloads(supervisor.snapshot))) {
|
||||
return { kind: "missing_evidence" as const, reason: "stale_supervisor_snapshot" };
|
||||
}
|
||||
const checkpoint = await loadNativeRunnerCheckpoint(tx, input);
|
||||
if (input.supervisor === undefined) {
|
||||
try {
|
||||
const admitted = parseNativeExecutionInput(profile.nativeExecutionInput);
|
||||
if (!input.offline || admitted.session.lifecyclePolicy.mode !== "per_turn"
|
||||
|| admitted.binding.runId !== input.runId || admitted.binding.companyId !== input.companyId
|
||||
|| admitted.binding.agentId !== input.agentId || admitted.binding.issueId !== input.issueId) {
|
||||
return { kind: "missing_evidence" as const, reason: "admitted_per_turn_binding" };
|
||||
}
|
||||
} catch { return { kind: "unsupported" as const, reason: "admitted_input_format" }; }
|
||||
}
|
||||
let checkpoint = await loadNativeRunnerCheckpoint(tx, input);
|
||||
const importing = !checkpoint && input.offline?.legacyControllerState !== undefined;
|
||||
if (importing) {
|
||||
if (!run.runnerInstanceId) return { kind: "missing_evidence" as const, reason: "checkpoint_binding" };
|
||||
checkpoint = captureLegacyDurablePrpCheckpoint(input.offline!.legacyControllerState!, {
|
||||
runId: input.runId, runnerInstanceId: run.runnerInstanceId, normalizedSessionId: input.normalizedSessionId,
|
||||
}, input.offline!.legacyRunnerArtifact);
|
||||
}
|
||||
if (!checkpoint || checkpoint.recoveryEvidence.normalizedSessionId !== input.normalizedSessionId
|
||||
|| checkpoint.recoveryEvidence.runnerInstanceId !== run.runnerInstanceId) {
|
||||
return { kind: "missing_evidence" as const, reason: "checkpoint_binding" };
|
||||
@@ -265,10 +290,13 @@ export async function importNativeControllerRecovery(db: Db, input: {
|
||||
}, { runId: input.runId, runnerInstanceId: checkpoint.recoveryEvidence.runnerInstanceId, normalizedSessionId: input.normalizedSessionId });
|
||||
if (!readNativeControllerRecovery(metadata, snapshot)) return { kind: "unsupported" as const, reason: "metadata_format" };
|
||||
if (profile.nativeControllerRecovery) {
|
||||
return nativeSha256(profile.nativeControllerRecovery) === nativeSha256(metadata)
|
||||
? { kind: "unchanged" as const }
|
||||
: { kind: "missing_evidence" as const, reason: "existing_metadata_conflict" };
|
||||
if (nativeSha256(profile.nativeControllerRecovery) !== nativeSha256(metadata)) {
|
||||
return { kind: "missing_evidence" as const, reason: "existing_metadata_conflict" };
|
||||
}
|
||||
if (importing) await insertMigratedNativeRunnerCheckpoint(tx, input, checkpoint);
|
||||
return { kind: importing ? "imported" as const : "unchanged" as const };
|
||||
}
|
||||
if (importing) await insertMigratedNativeRunnerCheckpoint(tx, input, checkpoint);
|
||||
await tx.update(heartbeatRuns).set({ runnerProfileJson: { ...profile, nativeControllerRecovery: metadata }, updatedAt: new Date() })
|
||||
.where(eq(heartbeatRuns.id, input.runId));
|
||||
return { kind: "imported" as const };
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
import { mkdtemp, readFile, rm, stat, symlink, writeFile } from "node:fs/promises";
|
||||
import { createWriteStream } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { pipeline } from "node:stream/promises";
|
||||
import tar from "tar-stream";
|
||||
import { afterEach, expect, it } from "vitest";
|
||||
import { captureNativeHarnessArchive, restoreNativeHarnessArchive } from "./native-harness-backup-archive.js";
|
||||
|
||||
const roots: string[] = [];
|
||||
afterEach(async () => { for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); });
|
||||
async function fixture() { const root = await mkdtemp(join(tmpdir(), "native-backup-archive-test-")); roots.push(root); return root; }
|
||||
async function maliciousArchive(root: string, headers: Array<tar.Headers>) {
|
||||
const archive = tar.pack(), file = join(root, "malicious.tar");
|
||||
const written = pipeline(archive, createWriteStream(file));
|
||||
for (const header of headers) archive.entry(header, Buffer.alloc(0));
|
||||
archive.finalize(); await written; return file;
|
||||
}
|
||||
it.each([
|
||||
{ name: "../escaped", type: "file" as const },
|
||||
{ name: "/absolute", type: "file" as const },
|
||||
{ name: "link", type: "symlink" as const, linkname: "../outside" },
|
||||
{ name: "device", type: "character-device" as const },
|
||||
{ name: "parent/child", type: "file" as const },
|
||||
])("rejects unsafe archive entry $name", async header => {
|
||||
const root = await fixture(), file = await maliciousArchive(root, [header]);
|
||||
await expect(restoreNativeHarnessArchive(file, join(root, "destination"))).rejects.toThrow("native_harness_archive_invalid");
|
||||
await expect(stat(join(root, "escaped"))).rejects.toMatchObject({ code: "ENOENT" });
|
||||
});
|
||||
it("rejects duplicate paths instead of overwriting a restored file", async () => {
|
||||
const root = await fixture(), file = await maliciousArchive(root, [{ name: "manifest.json", type: "file" }, { name: "manifest.json", type: "file" }]);
|
||||
await expect(restoreNativeHarnessArchive(file, join(root, "destination"))).rejects.toThrow("native_harness_archive_invalid");
|
||||
});
|
||||
it("refuses a source symlink without reading its target", async () => {
|
||||
const root = await fixture(), outside = join(root, "outside");
|
||||
await writeFile(outside, "private external file");
|
||||
const { mkdir } = await import("node:fs/promises"); const source = join(root, "source"); await mkdir(source);
|
||||
await symlink(outside, join(source, "linked"));
|
||||
await expect(captureNativeHarnessArchive(source, join(root, "archive"))).rejects.toThrow("native_harness_archive_invalid");
|
||||
expect(await readFile(outside, "utf8")).toBe("private external file");
|
||||
});
|
||||
it("honors cancellation before walking the checkpoint tree", async () => {
|
||||
const root = await fixture();
|
||||
await expect(captureNativeHarnessArchive(root, join(root, "archive"), AbortSignal.abort(new Error("test cancelled"))))
|
||||
.rejects.toThrow();
|
||||
});
|
||||
@@ -0,0 +1,79 @@
|
||||
import { constants, createReadStream, createWriteStream } from "node:fs";
|
||||
import { chmod, lstat, mkdir, open, readdir } from "node:fs/promises";
|
||||
import { join, posix } from "node:path";
|
||||
import { pipeline } from "node:stream/promises";
|
||||
import tar from "tar-stream";
|
||||
|
||||
const MAX_BYTES = 256 * 1024 * 1024;
|
||||
const MAX_ENTRIES = 40_000;
|
||||
const fail = () => new Error("native_harness_archive_invalid");
|
||||
const safeName = (name: string) => name.length > 0 && !name.includes("\\")
|
||||
&& !/[\u0000-\u001f\u007f]/u.test(name) && !posix.isAbsolute(name)
|
||||
&& name.split("/").every(part => part !== ".." && part !== "." && part !== "");
|
||||
|
||||
export async function captureNativeHarnessArchive(root: string, target: string, signal?: AbortSignal) {
|
||||
if (!(await lstat(root)).isDirectory()) throw fail();
|
||||
const archive = tar.pack();
|
||||
const output = pipeline(archive, createWriteStream(target, { flags: "wx", mode: 0o600 }), { signal });
|
||||
void output.catch(() => undefined);
|
||||
let count = 0, bytes = 0;
|
||||
const visit = async (relative: string): Promise<void> => {
|
||||
for (const name of (await readdir(join(root, relative))).sort()) {
|
||||
signal?.throwIfAborted();
|
||||
const entry = relative ? `${relative}/${name}` : name;
|
||||
if (!safeName(entry) || ++count > MAX_ENTRIES) throw fail();
|
||||
const file = join(root, entry), stat = await lstat(file);
|
||||
if (stat.isDirectory()) {
|
||||
await new Promise<void>((resolve, reject) => archive.entry({ name: entry, type: "directory", mode: stat.mode & 0o777 },
|
||||
error => error ? reject(error) : resolve()));
|
||||
await visit(entry);
|
||||
} else if (stat.isFile()) {
|
||||
bytes += stat.size;
|
||||
if (bytes > MAX_BYTES) throw fail();
|
||||
const handle = await open(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
||||
try {
|
||||
const before = await handle.stat();
|
||||
if (!before.isFile() || before.ino !== stat.ino || before.dev !== stat.dev || before.size !== stat.size) throw fail();
|
||||
const stream = archive.entry({ name: entry, size: stat.size, type: "file", mode: stat.mode & 0o777 });
|
||||
await pipeline(handle.createReadStream({ autoClose: false }), stream, { signal });
|
||||
const after = await handle.stat();
|
||||
if (before.size !== after.size || before.mtimeMs !== after.mtimeMs || before.ctimeMs !== after.ctimeMs) throw fail();
|
||||
} finally { await handle.close(); }
|
||||
} else throw fail();
|
||||
}
|
||||
};
|
||||
try { await visit(""); archive.finalize(); await output; }
|
||||
catch (error) { archive.destroy(error as Error); await output.catch(() => undefined); throw error; }
|
||||
}
|
||||
|
||||
export async function restoreNativeHarnessArchive(source: string, target: string, signal?: AbortSignal) {
|
||||
await mkdir(target, { mode: 0o700 });
|
||||
const archive = tar.extract(), names = new Set<string>();
|
||||
const directories: Array<{ path: string; mode: number }> = [], directoryNames = new Set<string>();
|
||||
let bytes = 0;
|
||||
archive.on("entry", (header, body, next) => {
|
||||
body.on("error", () => undefined);
|
||||
void (async () => {
|
||||
signal?.throwIfAborted();
|
||||
const name = header.type === "directory" ? header.name.replace(/\/$/u, "") : header.name;
|
||||
if (!safeName(name) || names.has(name) || names.size >= MAX_ENTRIES
|
||||
|| !["directory", "file"].includes(header.type ?? "") || !Number.isSafeInteger(header.size ?? 0)
|
||||
|| (header.size ?? 0) < 0 || (bytes += header.size ?? 0) > MAX_BYTES) throw fail();
|
||||
names.add(name);
|
||||
const file = join(target, name), mode = (header.mode ?? 0o600) & 0o777;
|
||||
const parent = posix.dirname(name);
|
||||
if (parent !== "." && !directoryNames.has(parent)) throw fail();
|
||||
if (header.type === "directory") {
|
||||
if (header.size) throw fail();
|
||||
await mkdir(file, { mode: 0o700 }); directories.push({ path: file, mode }); directoryNames.add(name);
|
||||
for await (const _ of body) throw fail();
|
||||
} else {
|
||||
await pipeline(body, createWriteStream(file, { flags: "wx", mode }), { signal });
|
||||
await chmod(file, mode);
|
||||
}
|
||||
})().then(() => next(), error => archive.destroy(error as Error));
|
||||
});
|
||||
await pipeline(createReadStream(source), archive, { signal });
|
||||
if (!names.has("manifest.json")) throw fail();
|
||||
for (const directory of directories.reverse()) await chmod(directory.path, directory.mode);
|
||||
}
|
||||
@@ -2,7 +2,7 @@ import { notifyExternalConversationSettlement } from "../external-conversation-s
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { promises as fs } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { and, asc, eq, gt, inArray, isNull, lt, or, sql } from "drizzle-orm";
|
||||
import { and, asc, eq, gt, gte, inArray, isNull, lt, or, sql } from "drizzle-orm";
|
||||
import {
|
||||
agentTaskSessions,
|
||||
environmentLeases,
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
issues,
|
||||
nativeRunFinalizations,
|
||||
nativeRunnerCheckpoints,
|
||||
nativeHarnessBackups,
|
||||
type Db,
|
||||
} from "@paperclipai/db";
|
||||
import { logger } from "../../middleware/logger.js";
|
||||
@@ -18,6 +19,8 @@ import { nativeSessionScopeDigestIsBusy } from "./native-session-executor.js";
|
||||
import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js";
|
||||
import { DELIVERY_QUEUES } from "../delivery-work-notifications.js";
|
||||
import type { NativeHarnessBackupExpirationScope } from "../native-harness-backup-expiration-notifications.js";
|
||||
import { deleteRetiredNativeHarnessBackup, durableNativeHarnessStorage } from "./native-harness-backup-storage.js";
|
||||
import type { StorageService } from "../../storage/types.js";
|
||||
|
||||
export const NATIVE_HARNESS_BACKUP_EXPIRATION_INTERVAL_MS = 60 * 60 * 1000;
|
||||
export const NATIVE_HARNESS_BACKUP_EXPIRATION_STARTUP_DELAY_MS = 2 * 60 * 1000;
|
||||
@@ -51,6 +54,7 @@ export type NativeHarnessBackupExpirationOptions = {
|
||||
stateBase?: string;
|
||||
shouldContinue?: () => boolean;
|
||||
isSessionBusy?: (scopeDigest: string) => boolean;
|
||||
storage?: StorageService;
|
||||
};
|
||||
|
||||
export type NativeHarnessBackupExpirationResult = {
|
||||
@@ -291,6 +295,67 @@ export function nativeHarnessBackupExpirationService(
|
||||
let evidenceDueAt = 0;
|
||||
let passDueAt = Number.POSITIVE_INFINITY;
|
||||
let checkpointCursor: string | null = null;
|
||||
let durableBackupCursor: string | null = null;
|
||||
let checkpointsEnumerated = false;
|
||||
|
||||
async function expireDurableBackups(options: {
|
||||
closedBefore: Date; dryRun: boolean; maxEntries: number; maxRemovals: number; shouldContinue: () => boolean;
|
||||
}, result: NativeHarnessBackupExpirationResult) {
|
||||
const storage = defaults.storage ?? durableNativeHarnessStorage();
|
||||
if (!storage) return;
|
||||
const candidates = await db.select({ id: nativeHarnessBackups.id, scopeDigest: nativeHarnessBackups.scopeDigest,
|
||||
runId: nativeHarnessBackups.runId, survivingRunId: heartbeatRuns.id,
|
||||
sessionId: heartbeatRuns.nativeSessionId, status: nativeHarnessBackups.status })
|
||||
.from(nativeHarnessBackups).leftJoin(heartbeatRuns, eq(heartbeatRuns.id, nativeHarnessBackups.runId))
|
||||
.where(and(durableBackupCursor ? gt(nativeHarnessBackups.id, durableBackupCursor) : undefined,
|
||||
or(eq(nativeHarnessBackups.status, "retired"), lt(nativeHarnessBackups.createdAt, options.closedBefore))))
|
||||
.orderBy(asc(nativeHarnessBackups.id)).limit(options.maxEntries + 1);
|
||||
let processed = 0;
|
||||
for (const candidate of candidates) {
|
||||
if (!options.shouldContinue() || processed++ >= options.maxEntries || result.expired >= options.maxRemovals) {
|
||||
result.truncated = true; return;
|
||||
}
|
||||
durableBackupCursor = candidate.id;
|
||||
if (candidate.survivingRunId && !candidate.sessionId) continue;
|
||||
try {
|
||||
const retired = candidate.status === "retired" || await db.transaction(async tx => {
|
||||
if (!candidate.survivingRunId) {
|
||||
result.candidates++;
|
||||
if (options.dryRun) return false;
|
||||
await tx.update(nativeHarnessBackups).set({ status: "retired", updatedAt: new Date() })
|
||||
.where(eq(nativeHarnessBackups.id, candidate.id));
|
||||
return true;
|
||||
}
|
||||
const blocker = await sessionRetentionBlocker(tx, { scopeDigest: candidate.scopeDigest, checkpoint: true,
|
||||
closedBefore: options.closedBefore,
|
||||
identity: { runId: candidate.runId, normalizedSessionId: candidate.sessionId!, environmentLeaseId: null } });
|
||||
if ("reason" in blocker) return false;
|
||||
const [recent] = await tx.select({ id: nativeHarnessBackups.id }).from(nativeHarnessBackups).where(and(
|
||||
eq(nativeHarnessBackups.companyId, blocker.companyId), eq(nativeHarnessBackups.scopeDigest, candidate.scopeDigest),
|
||||
gte(nativeHarnessBackups.createdAt, options.closedBefore))).limit(1);
|
||||
if (recent) return false;
|
||||
result.candidates++;
|
||||
if (options.dryRun) return false;
|
||||
await notifyExternalConversationSettlement(tx, { companyId: blocker.companyId });
|
||||
await tx.update(nativeRunFinalizations).set({ controllerGeneration: sql`${nativeRunFinalizations.controllerGeneration} + 1` })
|
||||
.where(inArray(nativeRunFinalizations.runId, blocker.runIds));
|
||||
await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${`${blocker.companyId}:${candidate.scopeDigest}`}, 0))`);
|
||||
const cleared = await tx.delete(agentTaskSessions).where(and(eq(agentTaskSessions.companyId, blocker.companyId),
|
||||
eq(agentTaskSessions.agentId, blocker.agentId), or(inArray(agentTaskSessions.lastRunId, blocker.runIds),
|
||||
sql`${agentTaskSessions.sessionParamsJson}->>'sessionId' = ${candidate.sessionId}`))).returning({ id: agentTaskSessions.id });
|
||||
result.clearedTaskSessions += cleared.length;
|
||||
await tx.update(nativeHarnessBackups).set({ status: "retired", updatedAt: new Date() }).where(and(
|
||||
eq(nativeHarnessBackups.companyId, blocker.companyId), eq(nativeHarnessBackups.scopeDigest, candidate.scopeDigest)));
|
||||
return true;
|
||||
});
|
||||
if (retired && !options.dryRun && await deleteRetiredNativeHarnessBackup(db, storage, candidate.id, AbortSignal.timeout(60_000))) result.expired++;
|
||||
} catch (err) {
|
||||
result.failed++;
|
||||
logger.warn({ err, backupId: candidate.id }, "durable native harness backup expiration skipped an object");
|
||||
}
|
||||
}
|
||||
durableBackupCursor = null;
|
||||
}
|
||||
|
||||
async function expireCheckpoints(options: {
|
||||
closedBefore: Date; dryRun: boolean; maxEntries: number; maxRemovals: number; shouldContinue: () => boolean;
|
||||
@@ -704,13 +769,20 @@ export function nativeHarnessBackupExpirationService(
|
||||
const result = emptyResult();
|
||||
const shouldContinue = options.shouldContinue ?? defaults.shouldContinue ?? (() => true);
|
||||
if (!shouldContinue()) return { ...result, truncated: true };
|
||||
await expireCheckpoints({
|
||||
const pass = {
|
||||
closedBefore: new Date((options.now ?? defaults.now ?? new Date()).getTime() - closedForDays * DAY_MS),
|
||||
dryRun: options.dryRun ?? defaults.dryRun ?? false,
|
||||
maxEntries: Math.max(1, options.maxEntries ?? defaults.maxEntries ?? 200),
|
||||
maxRemovals: Math.max(1, options.maxRemovals ?? defaults.maxRemovals ?? 50),
|
||||
shouldContinue,
|
||||
}, result);
|
||||
};
|
||||
if (!checkpointsEnumerated) {
|
||||
await expireCheckpoints(pass, result);
|
||||
if (result.truncated) return result;
|
||||
checkpointsEnumerated = true;
|
||||
}
|
||||
await expireDurableBackups(pass, result);
|
||||
if (!result.truncated) checkpointsEnumerated = false;
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { lstat, readFile, readdir } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { environmentLeases, heartbeatRuns, nativeHarnessBackups, type Db } from "@paperclipai/db";
|
||||
import type { StorageService } from "../../storage/types.js";
|
||||
import { nativeHarnessStateBase, readNativeHarnessBackupStamp, verifyNativeHarnessBackupStamp } from "./native-harness-backup-stamp.js";
|
||||
import { persistNativeHarnessBackup, verifyDurableNativeHarnessBackup } from "./native-harness-backup-storage.js";
|
||||
import { offlineMigrationExecutionWhere } from "../offline-migration-execution.js";
|
||||
|
||||
export async function hasLocalNativeHarnessObligations(db: Db): Promise<boolean> {
|
||||
const rows = await db.execute(sql`select 1 from agent_task_sessions s
|
||||
join heartbeat_runs r on r.id = s.last_run_id and r.company_id = s.company_id and r.agent_id = s.agent_id
|
||||
join environment_leases e on e.company_id = r.company_id and e.issue_id = r.native_issue_id
|
||||
and e.metadata->'nativeHarnessBackup'->>'normalizedSessionId' = r.native_session_id::text
|
||||
where s.adapter_type = 'paperclip_runner'
|
||||
and e.metadata->'nativeHarnessBackup'->>'schema' = 'paperclip.native-harness-backup-stamp.v2'
|
||||
and not exists (select 1 from native_harness_backups b where b.company_id = r.company_id
|
||||
and b.scope_digest = e.metadata->'nativeHarnessBackup'->>'sessionScopeSha256' and b.status = 'current')
|
||||
limit 1`);
|
||||
return rows.length > 0;
|
||||
}
|
||||
|
||||
export async function migrateNativeHarnessBackups(db: Db, input: {
|
||||
storage: StorageService; writersStopped: true; signal: AbortSignal;
|
||||
}): Promise<number> {
|
||||
if (!input.writersStopped) throw new Error("native_harness_migration_requires_offline_writers");
|
||||
const [active] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(offlineMigrationExecutionWhere()).limit(1);
|
||||
if (active) throw new Error("native_harness_migration_requires_offline_writers");
|
||||
const base = nativeHarnessStateBase();
|
||||
const directories = await readdir(base).catch((error: NodeJS.ErrnoException) => {
|
||||
if (error.code === "ENOENT") return [] as string[]; throw error;
|
||||
});
|
||||
let count = 0;
|
||||
for (const scopeDigest of directories.filter(name => /^[a-f0-9]{64}$/.test(name)).sort()) {
|
||||
input.signal.throwIfAborted();
|
||||
const root = join(base, scopeDigest, "failover-backups", "current");
|
||||
const stat = await lstat(root).catch((error: NodeJS.ErrnoException) => {
|
||||
if (error.code === "ENOENT") return null; throw error;
|
||||
});
|
||||
if (!stat) continue;
|
||||
if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error("native_harness_migration_unsafe_source");
|
||||
const manifestBytes = await readFile(join(root, "manifest.json"));
|
||||
const manifest = JSON.parse(manifestBytes.toString("utf8"));
|
||||
const state = JSON.parse(await readFile(join(root, "runner", "runner-state.json"), "utf8"));
|
||||
if (state.lifecycle !== "suspended" || typeof state.runId !== "string"
|
||||
|| manifest.schema !== "paperclip.native-harness-backup.v1") throw new Error("native_harness_migration_source_unsettled");
|
||||
const [run] = await db.select({ companyId: heartbeatRuns.companyId, issueId: heartbeatRuns.nativeIssueId,
|
||||
sessionId: heartbeatRuns.nativeSessionId, runnerId: heartbeatRuns.runnerInstanceId }).from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, state.runId));
|
||||
if (!run?.issueId || run.sessionId !== manifest.normalizedSessionId || run.runnerId !== manifest.runnerInstanceId) {
|
||||
throw new Error("native_harness_migration_binding_mismatch");
|
||||
}
|
||||
const [current] = await db.select().from(nativeHarnessBackups).where(and(eq(nativeHarnessBackups.companyId, run.companyId),
|
||||
eq(nativeHarnessBackups.scopeDigest, scopeDigest), eq(nativeHarnessBackups.status, "current")));
|
||||
if (current) {
|
||||
await verifyDurableNativeHarnessBackup({ db, storage: input.storage, companyId: run.companyId,
|
||||
scopeDigest, manifestSha256: current.manifestSha256, signal: input.signal });
|
||||
count++; continue;
|
||||
}
|
||||
const manifestSha256 = `sha256:${createHash("sha256").update(manifestBytes).digest("hex")}`;
|
||||
const leases = await db.select({ stamp: sql<unknown>`${environmentLeases.metadata}->'nativeHarnessBackup'`,
|
||||
providerLeaseId: environmentLeases.providerLeaseId }).from(environmentLeases).where(and(
|
||||
eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.issueId, run.issueId),
|
||||
sql`${environmentLeases.metadata}->'nativeHarnessBackup'->>'manifestSha256' = ${manifestSha256}`));
|
||||
const valid = leases.some(lease => {
|
||||
const stamp = readNativeHarnessBackupStamp(lease.stamp, lease.providerLeaseId ?? "");
|
||||
return stamp?.sessionScopeSha256 === scopeDigest && verifyNativeHarnessBackupStamp(stamp, lease.providerLeaseId!);
|
||||
});
|
||||
if (!valid) throw new Error("native_harness_migration_backup_unverified");
|
||||
await persistNativeHarnessBackup({ db, storage: input.storage, root, scopeDigest, signal: input.signal,
|
||||
owner: { kind: "offline_migration", writersStopped: true, companyId: run.companyId, issueId: run.issueId, runId: state.runId } });
|
||||
count++;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
@@ -142,12 +142,12 @@ export function createNativeHarnessBackupStamp(input: {
|
||||
};
|
||||
}
|
||||
|
||||
export function verifyNativeHarnessBackupStamp(
|
||||
export function readNativeHarnessBackupStamp(
|
||||
value: unknown,
|
||||
expectedProviderLeaseId: string,
|
||||
): boolean {
|
||||
if (!expectedProviderLeaseId) return false;
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
||||
): NativeHarnessBackupStamp | null {
|
||||
if (!expectedProviderLeaseId) return null;
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
|
||||
const stamp = value as Record<string, unknown>;
|
||||
if (
|
||||
stamp.schema !== "paperclip.native-harness-backup-stamp.v2" ||
|
||||
@@ -160,9 +160,17 @@ export function verifyNativeHarnessBackupStamp(
|
||||
!stamp.sourceProviderLeaseId ||
|
||||
stamp.authorizedProviderLeaseId !== expectedProviderLeaseId ||
|
||||
typeof stamp.manifestSha256 !== "string" ||
|
||||
!stamp.manifestSha256.startsWith("sha256:")
|
||||
!/^sha256:[a-f0-9]{64}$/.test(stamp.manifestSha256) ||
|
||||
!/^[a-f0-9]{64}$/.test(stamp.sessionScopeSha256) ||
|
||||
typeof stamp.completedAt !== "string" || !Number.isFinite(Date.parse(stamp.completedAt))
|
||||
)
|
||||
return false;
|
||||
return null;
|
||||
return stamp as unknown as NativeHarnessBackupStamp;
|
||||
}
|
||||
|
||||
export function verifyNativeHarnessBackupStamp(value: unknown, expectedProviderLeaseId: string): boolean {
|
||||
const stamp = readNativeHarnessBackupStamp(value, expectedProviderLeaseId);
|
||||
if (!stamp) return false;
|
||||
const root = stateRootFromDigest(stamp.sessionScopeSha256);
|
||||
if (!root || !isRealDirectory(root)) return false;
|
||||
const backupRoot = resolve(root, "failover-backups");
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { chmod, mkdir, readFile, rename, stat, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { agentTaskSessions, environmentLeases, heartbeatRuns, issues, nativeHarnessBackups, nativeRunFinalizations } from "@paperclipai/db";
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import { describeEmbeddedPostgres, nativeRestartFixture } from "../../__tests__/helpers/native-restart.js";
|
||||
import { createLocalDiskStorageProvider } from "../../storage/local-disk-provider.js";
|
||||
import { createStorageService } from "../../storage/service.js";
|
||||
import { nativeHarnessBackupOwner, persistNativeHarnessBackup, restoreDurableNativeHarnessBackup,
|
||||
verifyDurableNativeHarnessBackup } from "./native-harness-backup-storage.js";
|
||||
import { removeNativeHarnessBackup } from "./native-harness-backup-cleanup.js";
|
||||
import { buildNativeHarnessBackupManifest } from "./native-session-executor.js";
|
||||
import { createNativeHarnessBackupStamp } from "./native-harness-backup-stamp.js";
|
||||
import { hasLocalNativeHarnessObligations, migrateNativeHarnessBackups } from "./native-harness-backup-migration.js";
|
||||
import { nativeHarnessBackupExpirationService } from "./native-harness-backup-expiration.js";
|
||||
|
||||
describeEmbeddedPostgres("durable native provider backups", () => {
|
||||
const environment = nativeRestartFixture();
|
||||
const ownedHomes: string[] = [];
|
||||
afterEach(async () => {
|
||||
for (const home of ownedHomes.splice(0)) await removeNativeHarnessBackup(home);
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
async function fixture(provider = "codex") {
|
||||
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "51".repeat(32));
|
||||
const f = await environment.seedRun(randomUUID());
|
||||
const { companyId, runtimeRoot, successor } = environment.current;
|
||||
await f.db.update(nativeRunFinalizations).set({ controllerBootId: successor.bootId, controllerGeneration: 1,
|
||||
leaseOwner: "backup-writer", leaseExpiresAt: new Date(Date.now() + 120_000) }).where(eq(nativeRunFinalizations.runId, f.runId));
|
||||
const owner = await nativeHarnessBackupOwner(f.db, { companyId, issueId: f.issueId, runId: f.runId, bootId: successor.bootId });
|
||||
const home = join(runtimeRoot, randomUUID()), root = join(home, "failover-backups", "current");
|
||||
ownedHomes.push(home);
|
||||
await mkdir(join(root, provider, "sessions"), { recursive: true });
|
||||
await mkdir(join(root, "runner"));
|
||||
const identity = { normalizedSessionId: randomUUID(), providerSessionId: randomUUID(), provider };
|
||||
await writeFile(join(root, provider, "sessions", "history.json"), JSON.stringify(identity), { mode: 0o600 });
|
||||
await writeFile(join(root, "runner", "runner-state.json"), JSON.stringify({ runId: f.runId, lifecycle: "suspended" }), { mode: 0o600 });
|
||||
const manifest = JSON.stringify({ schema: "paperclip.native-harness-backup.v1", ...identity, directories: [provider, "runner"] });
|
||||
await writeFile(join(root, "manifest.json"), manifest, { mode: 0o600 });
|
||||
await chmod(join(root, provider, "sessions"), 0o500);
|
||||
const sessionScopeId = randomUUID(), scopeDigest = createHash("sha256").update(sessionScopeId).digest("hex");
|
||||
const storage = createStorageService(createLocalDiskStorageProvider(join(runtimeRoot, "durable-objects")));
|
||||
return { ...f, home, root, companyId, owner, scopeDigest, sessionScopeId, storage, identity,
|
||||
manifestSha256: `sha256:${createHash("sha256").update(manifest).digest("hex")}` };
|
||||
}
|
||||
it.each(["codex", "claude", "opencode", "grok", "new-provider"])("restores %s session bytes after deleting the entire controller home", async provider => {
|
||||
const f = await fixture(provider);
|
||||
await persistNativeHarnessBackup(f);
|
||||
const [row] = await f.db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.scopeDigest, f.scopeDigest));
|
||||
expect(row.status).toBe("current");
|
||||
const object = await f.storage.getObject(f.companyId, row.reference.objectKey as string);
|
||||
const parts: Buffer[] = []; for await (const part of object.stream) parts.push(part);
|
||||
expect(Buffer.concat(parts).includes(Buffer.from(f.identity.providerSessionId))).toBe(false);
|
||||
await removeNativeHarnessBackup(f.home);
|
||||
const replacement = join(environment.current.runtimeRoot, randomUUID(), "failover-backups");
|
||||
ownedHomes.push(replacement);
|
||||
expect(await restoreDurableNativeHarnessBackup({ ...f, backupRoot: replacement })).toBe(true);
|
||||
expect(JSON.parse(await readFile(join(replacement, "current", provider, "sessions", "history.json"), "utf8"))).toEqual(f.identity);
|
||||
expect((await stat(join(replacement, "current", provider, "sessions"))).mode & 0o777).toBe(0o500);
|
||||
expect(await verifyDurableNativeHarnessBackup(f)).toBe(true);
|
||||
});
|
||||
it("keeps the last published backup when object storage fails", async () => {
|
||||
const f = await fixture(); await persistNativeHarnessBackup(f);
|
||||
const [before] = await f.db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.scopeDigest, f.scopeDigest));
|
||||
const put = vi.spyOn(f.storage, "putFile").mockRejectedValue(new Error("storage unavailable"));
|
||||
await expect(persistNativeHarnessBackup(f)).rejects.toThrow("storage unavailable"); put.mockRestore();
|
||||
const rows = await f.db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.scopeDigest, f.scopeDigest));
|
||||
expect(rows.filter(row => row.status === "current").map(row => row.id)).toEqual([before.id]);
|
||||
expect(rows.filter(row => row.status === "pending")).toHaveLength(1);
|
||||
expect(await readFile(join(f.root, "manifest.json"), "utf8")).toContain("native-harness-backup");
|
||||
});
|
||||
it("fences an old controller whose upload finishes after ownership changes", async () => {
|
||||
const f = await fixture(); await persistNativeHarnessBackup(f);
|
||||
const [before] = await f.db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.scopeDigest, f.scopeDigest));
|
||||
const put = f.storage.putFile.bind(f.storage);
|
||||
f.storage.putFile = async input => {
|
||||
const result = await put(input);
|
||||
await f.db.update(nativeRunFinalizations).set({ controllerBootId: randomUUID(), controllerGeneration: 2 })
|
||||
.where(eq(nativeRunFinalizations.runId, f.runId));
|
||||
return result;
|
||||
};
|
||||
await expect(persistNativeHarnessBackup(f)).rejects.toThrow("native_runner_checkpoint_owner_lost");
|
||||
const rows = await f.db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.scopeDigest, f.scopeDigest));
|
||||
expect(rows.filter(row => row.status === "current").map(row => row.id)).toEqual([before.id]);
|
||||
});
|
||||
it("rejects corrupt objects and cross-company lookups without publishing a local cache", async () => {
|
||||
const f = await fixture(); await persistNativeHarnessBackup(f);
|
||||
const replacement = join(environment.current.runtimeRoot, randomUUID(), "failover-backups");
|
||||
expect(await restoreDurableNativeHarnessBackup({ ...f, companyId: randomUUID(), backupRoot: replacement })).toBe(false);
|
||||
const [row] = await f.db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.scopeDigest, f.scopeDigest));
|
||||
await f.storage.deleteObject(f.companyId, row.reference.objectKey as string);
|
||||
await f.storage.putFile({ companyId: f.companyId, namespace: "native-harness-backups", objectKey: row.reference.objectKey as string,
|
||||
originalFilename: null, contentType: "application/octet-stream", body: Buffer.from("corrupted") });
|
||||
await expect(restoreDurableNativeHarnessBackup({ ...f, backupRoot: replacement })).rejects.toThrow("hash_mismatch");
|
||||
await expect(stat(join(replacement, "current"))).rejects.toMatchObject({ code: "ENOENT" });
|
||||
});
|
||||
async function settle(f: { db: Awaited<ReturnType<typeof fixture>>["db"]; runId: string; issueId: string }, closedAt = new Date()) {
|
||||
await f.db.update(heartbeatRuns).set({ status: "failed", finishedAt: closedAt }).where(eq(heartbeatRuns.id, f.runId));
|
||||
await f.db.update(nativeRunFinalizations).set({ phase: "terminal_failure", leaseOwner: null, leaseExpiresAt: null })
|
||||
.where(eq(nativeRunFinalizations.runId, f.runId));
|
||||
await f.db.update(issues).set({ status: "done", executionRunId: null, completedAt: closedAt }).where(eq(issues.id, f.issueId));
|
||||
}
|
||||
it("migrates a lease-verified local backup once and keeps its selected session intact", async () => {
|
||||
const f = await fixture("codex-home");
|
||||
const base = join(environment.current.runtimeRoot, randomUUID()); ownedHomes.push(base); await mkdir(base);
|
||||
await rename(f.home, join(base, f.scopeDigest));
|
||||
const root = join(base, f.scopeDigest, "failover-backups", "current");
|
||||
vi.stubEnv("PAPERCLIP_RUNNER_STATE_DIR", base);
|
||||
const runnerInstanceId = randomUUID(), providerLeaseId = randomUUID();
|
||||
const execution = { provider: { kind: "codex" }, binding: { companyId: f.companyId, issueId: f.issueId,
|
||||
runId: f.runId, agentId: environment.current.agentId, executionWorkspaceId: randomUUID() },
|
||||
workspace: { cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null },
|
||||
session: { normalizedSessionId: f.identity.normalizedSessionId, driverKind: "codex_app_server",
|
||||
protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } } } as never;
|
||||
const manifest = buildNativeHarnessBackupManifest({ backupRoot: root, execution, runnerInstanceId,
|
||||
providerSessionIdentity: { providerSessionId: f.identity.providerSessionId }, sourceProviderLeaseId: providerLeaseId });
|
||||
await writeFile(join(root, "manifest.json"), JSON.stringify(manifest));
|
||||
const stamp = createNativeHarnessBackupStamp({ manifestPath: join(root, "manifest.json"), sessionScopeId: f.sessionScopeId,
|
||||
authorizedProviderLeaseId: providerLeaseId, runnerInstanceId, normalizedSessionId: f.identity.normalizedSessionId, completedAt: manifest.completedAt });
|
||||
await f.db.update(heartbeatRuns).set({ nativeSessionId: f.identity.normalizedSessionId, runnerInstanceId }).where(eq(heartbeatRuns.id, f.runId));
|
||||
await f.db.insert(environmentLeases).values({ companyId: f.companyId, issueId: f.issueId, heartbeatRunId: f.runId,
|
||||
providerLeaseId, status: "expired", metadata: { nativeHarnessBackup: stamp } });
|
||||
const [selected] = await f.db.insert(agentTaskSessions).values({ companyId: f.companyId, agentId: environment.current.agentId,
|
||||
adapterType: "paperclip_runner", taskKey: f.issueId, lastRunId: f.runId,
|
||||
sessionParamsJson: { sessionId: f.identity.normalizedSessionId } }).returning();
|
||||
expect(await hasLocalNativeHarnessObligations(f.db)).toBe(true);
|
||||
await expect(migrateNativeHarnessBackups(f.db, { storage: f.storage, writersStopped: true, signal: AbortSignal.timeout(10_000) }))
|
||||
.rejects.toThrow("native_harness_migration_requires_offline_writers");
|
||||
const ownedRuns = await f.db.select({ id: heartbeatRuns.id, issueId: heartbeatRuns.nativeIssueId }).from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.companyId, f.companyId));
|
||||
for (const run of ownedRuns) await settle({ ...f, runId: run.id, issueId: run.issueId! });
|
||||
await f.db.update(nativeRunFinalizations).set({ recoveryState: "blocked" }).where(eq(nativeRunFinalizations.runId, f.runId));
|
||||
const [preservedOwner] = await f.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId));
|
||||
const [queued] = await f.db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: environment.current.agentId,
|
||||
status: "queued", invocationSource: "on_demand", issueId: f.issueId }).returning();
|
||||
await f.db.update(issues).set({ executionRunId: queued.id }).where(eq(issues.id, f.issueId));
|
||||
expect(await migrateNativeHarnessBackups(f.db, { storage: f.storage, writersStopped: true, signal: AbortSignal.timeout(10_000) })).toBe(1);
|
||||
expect(await migrateNativeHarnessBackups(f.db, { storage: f.storage, writersStopped: true, signal: AbortSignal.timeout(10_000) })).toBe(1);
|
||||
expect(await hasLocalNativeHarnessObligations(f.db)).toBe(false);
|
||||
expect(await f.db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.scopeDigest, f.scopeDigest))).toHaveLength(1);
|
||||
expect((await f.db.select().from(agentTaskSessions).where(eq(agentTaskSessions.id, selected.id)))[0]).toEqual(selected);
|
||||
expect((await f.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)))[0]).toEqual(preservedOwner);
|
||||
expect((await f.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, queued.id)))[0]).toEqual(queued);
|
||||
expect((await f.db.select().from(issues).where(eq(issues.id, f.issueId)))[0].executionRunId).toBe(queued.id);
|
||||
expect(JSON.parse(await readFile(join(root, "manifest.json"), "utf8"))).toEqual(manifest);
|
||||
});
|
||||
it("retains live work and retries an uncertain object deletion through the existing startup retention pass", async () => {
|
||||
const f = await fixture(); await persistNativeHarnessBackup(f);
|
||||
const old = new Date(Date.now() - 10 * 86_400_000);
|
||||
await f.db.update(nativeHarnessBackups).set({ createdAt: old }).where(eq(nativeHarnessBackups.scopeDigest, f.scopeDigest));
|
||||
const retention = nativeHarnessBackupExpirationService(f.db, { storage: f.storage });
|
||||
expect((await retention.tickCheckpoints()).expired).toBe(0);
|
||||
await settle(f, old);
|
||||
const drop = vi.spyOn(f.storage, "deleteObject").mockRejectedValueOnce(new Error("delete reply lost"));
|
||||
expect((await retention.tickCheckpoints()).failed).toBe(1);
|
||||
expect((await f.db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.scopeDigest, f.scopeDigest)))[0].status).toBe("retired");
|
||||
expect((await retention.tickCheckpoints()).expired).toBe(1);
|
||||
expect(await f.db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.scopeDigest, f.scopeDigest))).toHaveLength(0);
|
||||
expect(drop).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
it("keeps deleted-source objects unchanged during a retention dry run", async () => {
|
||||
const f = await fixture(); await persistNativeHarnessBackup(f);
|
||||
const old = new Date(Date.now() - 10 * 86_400_000);
|
||||
await f.db.update(nativeHarnessBackups).set({ createdAt: old, runId: randomUUID() })
|
||||
.where(eq(nativeHarnessBackups.scopeDigest, f.scopeDigest));
|
||||
const [before] = await f.db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.scopeDigest, f.scopeDigest));
|
||||
const retention = nativeHarnessBackupExpirationService(f.db, { storage: f.storage });
|
||||
expect((await retention.tickCheckpoints({ dryRun: true })).expired).toBe(0);
|
||||
expect((await f.db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.id, before.id)))[0]).toEqual(before);
|
||||
expect(await verifyDurableNativeHarnessBackup(f)).toBe(true);
|
||||
expect((await retention.tickCheckpoints()).expired).toBe(1);
|
||||
expect(await f.db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.id, before.id))).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,221 @@
|
||||
import { createCipheriv, createDecipheriv, createHash, randomBytes, randomUUID } from "node:crypto";
|
||||
import { createReadStream, createWriteStream } from "node:fs";
|
||||
import { mkdtemp, readFile, rename } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import os from "node:os";
|
||||
import { pipeline } from "node:stream/promises";
|
||||
import { Transform } from "node:stream";
|
||||
import { and, eq, inArray, sql } from "drizzle-orm";
|
||||
import { heartbeatRuns, issues, nativeHarnessBackups, nativeRunFinalizations, type Db } from "@paperclipai/db";
|
||||
import { createLocalEncryptedCipher } from "../../secrets/local-encrypted-provider.js";
|
||||
import type { StorageService } from "../../storage/types.js";
|
||||
import { assertNativeRunnerCheckpointOwner, type NativeRunnerCheckpointOwner } from "./native-runner-checkpoint.js";
|
||||
import { loadConfig } from "../../config.js";
|
||||
import { getStorageService } from "../../storage/index.js";
|
||||
import { readNativeHarnessBackupStamp, verifyNativeHarnessBackupStamp } from "./native-harness-backup-stamp.js";
|
||||
import { captureNativeHarnessArchive, restoreNativeHarnessArchive } from "./native-harness-backup-archive.js";
|
||||
import { removeNativeHarnessBackup } from "./native-harness-backup-cleanup.js";
|
||||
import { offlineMigrationExecutionWhere } from "../offline-migration-execution.js";
|
||||
|
||||
const SCHEMA = "paperclip.native-harness-object.v1";
|
||||
type Binding = { companyId: string; scopeDigest: string; id: string };
|
||||
type Reference = Binding & {
|
||||
schema: typeof SCHEMA; objectKey: string; sha256: string; byteSize: number;
|
||||
material: Record<string, unknown>; manifestSha256: string;
|
||||
};
|
||||
type Row = typeof nativeHarnessBackups.$inferSelect;
|
||||
type OfflineOwner = { kind: "offline_migration"; companyId: string; issueId: string; runId: string; writersStopped: true };
|
||||
const fail = (reason: string): never => { throw new Error(`native_harness_storage_${reason}`); };
|
||||
const aad = (binding: Binding) => Buffer.from(JSON.stringify([SCHEMA, binding.companyId, binding.scopeDigest, binding.id]));
|
||||
const scope = (binding: Pick<Binding, "companyId" | "scopeDigest">) => and(
|
||||
eq(nativeHarnessBackups.companyId, binding.companyId), eq(nativeHarnessBackups.scopeDigest, binding.scopeDigest));
|
||||
|
||||
export function durableNativeHarnessStorage(): StorageService | null {
|
||||
if (loadConfig().storageProvider === "s3") return getStorageService();
|
||||
if (process.env.PAPERCLIP_EPHEMERAL_SERVER === "true") fail("durable_storage_required");
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function verifyRecoverableNativeHarnessBackup(input: {
|
||||
db: Db; companyId: string; stamp: unknown; providerLeaseId: string;
|
||||
}): Promise<boolean> {
|
||||
const stamp = readNativeHarnessBackupStamp(input.stamp, input.providerLeaseId);
|
||||
if (!stamp) return false;
|
||||
const storage = durableNativeHarnessStorage();
|
||||
if (!storage) return verifyNativeHarnessBackupStamp(stamp, input.providerLeaseId);
|
||||
const durable = await verifyDurableNativeHarnessBackup({ db: input.db, storage, companyId: input.companyId,
|
||||
scopeDigest: stamp.sessionScopeSha256, manifestSha256: stamp.manifestSha256, signal: AbortSignal.timeout(300_000) });
|
||||
return durable || process.env.PAPERCLIP_EPHEMERAL_SERVER !== "true" && verifyNativeHarnessBackupStamp(stamp, input.providerLeaseId);
|
||||
}
|
||||
function reference(row: Row): Reference {
|
||||
const ref = row.reference as Reference;
|
||||
if (ref.schema !== SCHEMA || ref.id !== row.id || ref.companyId !== row.companyId || ref.scopeDigest !== row.scopeDigest
|
||||
|| !/^[a-f0-9]{64}$/.test(ref.scopeDigest) || !/^[a-f0-9]{64}$/.test(ref.sha256)
|
||||
|| ref.manifestSha256 !== row.manifestSha256 || !/^sha256:[a-f0-9]{64}$/.test(ref.manifestSha256)
|
||||
|| ref.objectKey !== `${row.companyId}/native-harness-backups/${row.scopeDigest}/${row.id}.pchn`
|
||||
|| !Number.isSafeInteger(ref.byteSize) || ref.byteSize <= 0 || ref.byteSize > 300 * 1024 * 1024
|
||||
|| !ref.material || typeof ref.material !== "object") fail("reference_invalid");
|
||||
return ref;
|
||||
}
|
||||
async function digest(file: string, signal?: AbortSignal) {
|
||||
const hash = createHash("sha256"); let byteSize = 0;
|
||||
for await (const bytes of createReadStream(file, { signal })) { hash.update(bytes); byteSize += bytes.length; }
|
||||
return { sha256: hash.digest("hex"), byteSize };
|
||||
}
|
||||
async function receive(storage: StorageService, ref: Reference, target?: string, signal?: AbortSignal) {
|
||||
const object = await storage.getObject(ref.companyId, ref.objectKey, { signal });
|
||||
const hash = createHash("sha256"); let byteSize = 0;
|
||||
const observed = new Transform({ transform(bytes, _encoding, callback) {
|
||||
hash.update(bytes); byteSize += bytes.length;
|
||||
callback(byteSize > ref.byteSize ? new Error("native_harness_storage_size_mismatch") : null, bytes);
|
||||
} });
|
||||
try {
|
||||
if (target) await pipeline(object.stream, observed, createWriteStream(target, { flags: "wx", mode: 0o600 }), { signal });
|
||||
else for await (const bytes of object.stream) {
|
||||
signal?.throwIfAborted(); hash.update(bytes); byteSize += bytes.length;
|
||||
if (byteSize > ref.byteSize) fail("size_mismatch");
|
||||
}
|
||||
if (byteSize !== ref.byteSize || hash.digest("hex") !== ref.sha256) fail("hash_mismatch");
|
||||
} finally { object.stream.destroy(); }
|
||||
}
|
||||
|
||||
export async function nativeHarnessBackupOwner(db: Db, input: { companyId: string; issueId: string; runId: string; bootId: string }) {
|
||||
const [row] = await db.select().from(nativeRunFinalizations).where(and(
|
||||
eq(nativeRunFinalizations.companyId, input.companyId), eq(nativeRunFinalizations.issueId, input.issueId),
|
||||
eq(nativeRunFinalizations.runId, input.runId), eq(nativeRunFinalizations.controllerBootId, input.bootId)));
|
||||
if (!row?.leaseOwner) fail("owner_missing");
|
||||
return { ...input, authorityRunId: input.runId, leaseOwner: row!.leaseOwner!, generation: row!.controllerGeneration } satisfies NativeRunnerCheckpointOwner;
|
||||
}
|
||||
async function fence(tx: Parameters<Parameters<Db["transaction"]>[0]>[0], owner: NativeRunnerCheckpointOwner | OfflineOwner) {
|
||||
const [task] = await tx.select({ executionRunId: issues.executionRunId }).from(issues)
|
||||
.where(and(eq(issues.companyId, owner.companyId), eq(issues.id, owner.issueId))).for("share");
|
||||
const authorityRunId = "kind" in owner ? owner.runId : owner.authorityRunId;
|
||||
if (!task || !("kind" in owner) && task.executionRunId && task.executionRunId !== authorityRunId) fail("owner_superseded");
|
||||
if ("kind" in owner) {
|
||||
if (!owner.writersStopped) fail("offline_writers_required");
|
||||
const [active] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(offlineMigrationExecutionWhere()).limit(1);
|
||||
if (active) fail("offline_writers_required");
|
||||
const [run] = await tx.select({ finishedAt: heartbeatRuns.finishedAt, status: heartbeatRuns.status }).from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.id, owner.runId), eq(heartbeatRuns.companyId, owner.companyId),
|
||||
eq(heartbeatRuns.nativeIssueId, owner.issueId), eq(heartbeatRuns.runtimeMode, "native"))).for("share");
|
||||
const [coordinator] = await tx.select({ leaseOwner: nativeRunFinalizations.leaseOwner,
|
||||
leaseExpiresAt: nativeRunFinalizations.leaseExpiresAt })
|
||||
.from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, owner.runId)).for("share");
|
||||
const [clock] = await tx.execute<{ now: Date }>(sql`select clock_timestamp() as now`);
|
||||
if (!run?.finishedAt || !["succeeded", "failed", "cancelled", "interrupted", "timed_out"].includes(run.status)
|
||||
|| !coordinator || coordinator.leaseOwner && (!coordinator.leaseExpiresAt || coordinator.leaseExpiresAt > new Date(clock.now))) {
|
||||
fail("offline_owner_unsettled");
|
||||
}
|
||||
return;
|
||||
}
|
||||
await assertNativeRunnerCheckpointOwner(tx, owner, false);
|
||||
}
|
||||
|
||||
export async function persistNativeHarnessBackup(input: {
|
||||
db: Db; storage: StorageService; owner: NativeRunnerCheckpointOwner | OfflineOwner;
|
||||
scopeDigest: string; root: string; signal?: AbortSignal;
|
||||
}): Promise<void> {
|
||||
if (!/^[a-f0-9]{64}$/.test(input.scopeDigest)) fail("scope_invalid");
|
||||
const manifestBytes = await readFile(join(input.root, "manifest.json"));
|
||||
const manifestSha256 = `sha256:${createHash("sha256").update(manifestBytes).digest("hex")}`;
|
||||
const binding = { companyId: input.owner.companyId, scopeDigest: input.scopeDigest, id: randomUUID() };
|
||||
const scratch = await mkdtemp(join(os.tmpdir(), "pc-harness-upload-"));
|
||||
const key = randomBytes(32);
|
||||
try {
|
||||
const archive = join(scratch, "archive"), encrypted = join(scratch, "encrypted");
|
||||
await captureNativeHarnessArchive(input.root, archive, input.signal);
|
||||
const plaintext = await digest(archive, input.signal), iv = randomBytes(12);
|
||||
const cipher = createCipheriv("aes-256-gcm", key, iv); cipher.setAAD(aad(binding));
|
||||
await pipeline(createReadStream(archive), cipher, createWriteStream(encrypted, { flags: "wx", mode: 0o600 }), { signal: input.signal });
|
||||
const ref: Reference = { schema: SCHEMA, ...binding, manifestSha256,
|
||||
objectKey: `${binding.companyId}/native-harness-backups/${binding.scopeDigest}/${binding.id}.pchn`,
|
||||
...await digest(encrypted, input.signal),
|
||||
material: createLocalEncryptedCipher("write").encrypt(JSON.stringify({ schema: SCHEMA, ...binding,
|
||||
key: key.toString("base64"), iv: iv.toString("base64"), tag: cipher.getAuthTag().toString("base64"), plaintext })) };
|
||||
await input.db.transaction(async tx => {
|
||||
await fence(tx, input.owner);
|
||||
await tx.insert(nativeHarnessBackups).values({ ...binding, issueId: input.owner.issueId, runId: input.owner.runId,
|
||||
manifestSha256, reference: ref });
|
||||
});
|
||||
await input.storage.putFile({ companyId: ref.companyId, namespace: "native-harness-backups", objectKey: ref.objectKey,
|
||||
originalFilename: null, contentType: "application/octet-stream", body: createReadStream(encrypted),
|
||||
byteSize: ref.byteSize, sha256: ref.sha256, signal: input.signal });
|
||||
await receive(input.storage, ref, undefined, input.signal);
|
||||
await input.db.transaction(async tx => {
|
||||
await fence(tx, input.owner);
|
||||
await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${`${ref.companyId}:${ref.scopeDigest}`}, 0))`);
|
||||
const [successor] = await tx.select({ id: nativeHarnessBackups.id }).from(nativeHarnessBackups)
|
||||
.innerJoin(heartbeatRuns, eq(heartbeatRuns.id, nativeHarnessBackups.runId)).where(and(scope(ref),
|
||||
eq(nativeHarnessBackups.status, "current"), sql`${nativeHarnessBackups.runId} <> ${input.owner.runId}::uuid`,
|
||||
sql`${heartbeatRuns.createdAt} >= (select created_at from heartbeat_runs where id = ${input.owner.runId}::uuid)`)).limit(1);
|
||||
if (successor) fail("owner_superseded");
|
||||
await fence(tx, input.owner);
|
||||
await tx.update(nativeHarnessBackups).set({ status: "superseded", updatedAt: new Date() })
|
||||
.where(and(scope(ref), eq(nativeHarnessBackups.status, "current")));
|
||||
await tx.update(nativeHarnessBackups).set({ status: "current", updatedAt: new Date() })
|
||||
.where(and(eq(nativeHarnessBackups.id, ref.id), eq(nativeHarnessBackups.status, "pending")));
|
||||
});
|
||||
} finally { key.fill(0); await removeNativeHarnessBackup(scratch); }
|
||||
}
|
||||
|
||||
export async function restoreDurableNativeHarnessBackup(input: {
|
||||
db: Db; storage: StorageService; companyId: string; scopeDigest: string;
|
||||
backupRoot: string; signal?: AbortSignal;
|
||||
}): Promise<boolean> {
|
||||
const [row] = await input.db.select().from(nativeHarnessBackups).where(and(scope(input), eq(nativeHarnessBackups.status, "current")));
|
||||
if (!row) return false;
|
||||
const ref = reference(row);
|
||||
const wrapped = JSON.parse(createLocalEncryptedCipher("read").decrypt(ref.material));
|
||||
if (wrapped.schema !== SCHEMA || wrapped.id !== ref.id || wrapped.companyId !== ref.companyId
|
||||
|| wrapped.scopeDigest !== ref.scopeDigest) fail("binding_mismatch");
|
||||
const current = join(input.backupRoot, "current");
|
||||
const localManifest = await readFile(join(current, "manifest.json")).catch((error: NodeJS.ErrnoException) => {
|
||||
if (error.code === "ENOENT") return null; throw error;
|
||||
});
|
||||
if (localManifest && `sha256:${createHash("sha256").update(localManifest).digest("hex")}` === ref.manifestSha256) return true;
|
||||
const scratch = await mkdtemp(join(os.tmpdir(), "pc-harness-restore-"));
|
||||
const key = Buffer.from(wrapped.key, "base64");
|
||||
try {
|
||||
if (key.length !== 32) fail("key_invalid");
|
||||
const encrypted = join(scratch, "encrypted"), archive = join(scratch, "archive");
|
||||
await receive(input.storage, ref, encrypted, input.signal);
|
||||
const decipher = createDecipheriv("aes-256-gcm", key, Buffer.from(wrapped.iv, "base64"));
|
||||
decipher.setAAD(aad(ref)); decipher.setAuthTag(Buffer.from(wrapped.tag, "base64"));
|
||||
await pipeline(createReadStream(encrypted), decipher, createWriteStream(archive, { flags: "wx", mode: 0o600 }), { signal: input.signal });
|
||||
const actual = await digest(archive, input.signal);
|
||||
if (actual.sha256 !== wrapped.plaintext?.sha256 || actual.byteSize !== wrapped.plaintext?.byteSize) fail("plaintext_mismatch");
|
||||
const restored = join(scratch, "restored");
|
||||
await restoreNativeHarnessArchive(archive, restored, input.signal);
|
||||
const manifest = await readFile(join(restored, "manifest.json"));
|
||||
if (`sha256:${createHash("sha256").update(manifest).digest("hex")}` !== ref.manifestSha256) fail("manifest_mismatch");
|
||||
const { mkdir, cp } = await import("node:fs/promises");
|
||||
await mkdir(input.backupRoot, { recursive: true, mode: 0o700 });
|
||||
const pending = join(input.backupRoot, `.durable-${randomUUID()}`);
|
||||
try {
|
||||
await cp(restored, pending, { recursive: true, preserveTimestamps: true, errorOnExist: true, force: false });
|
||||
await removeNativeHarnessBackup(current);
|
||||
await rename(pending, current);
|
||||
} finally { await removeNativeHarnessBackup(pending); }
|
||||
return true;
|
||||
} finally { key.fill(0); await removeNativeHarnessBackup(scratch); }
|
||||
}
|
||||
|
||||
export async function verifyDurableNativeHarnessBackup(input: {
|
||||
db: Db; storage: StorageService; companyId: string; scopeDigest: string; manifestSha256: string; signal?: AbortSignal;
|
||||
}): Promise<boolean> {
|
||||
const [row] = await input.db.select().from(nativeHarnessBackups).where(and(scope(input),
|
||||
eq(nativeHarnessBackups.manifestSha256, input.manifestSha256), inArray(nativeHarnessBackups.status, ["current", "superseded"]))).limit(1);
|
||||
if (!row) return false;
|
||||
await receive(input.storage, reference(row), undefined, input.signal);
|
||||
return true;
|
||||
}
|
||||
|
||||
export async function deleteRetiredNativeHarnessBackup(db: Db, storage: StorageService, id: string, signal?: AbortSignal) {
|
||||
const [row] = await db.select().from(nativeHarnessBackups).where(and(
|
||||
eq(nativeHarnessBackups.id, id), eq(nativeHarnessBackups.status, "retired")));
|
||||
if (!row) return false;
|
||||
const ref = reference(row);
|
||||
await storage.deleteObject(ref.companyId, ref.objectKey, { allVersions: true, signal });
|
||||
await db.delete(nativeHarnessBackups).where(and(eq(nativeHarnessBackups.id, id), eq(nativeHarnessBackups.status, "retired")));
|
||||
return true;
|
||||
}
|
||||
@@ -6,6 +6,8 @@ import { createLocalEncryptedCipher } from "../../secrets/local-encrypted-provid
|
||||
import { legacyControllerBootId } from "../legacy-controller-lease.js";
|
||||
|
||||
type Binding = { companyId: string; issueId: string; runId: string };
|
||||
export const NATIVE_CHECKPOINT_HEADER_INSPECTION_MAX_BYTES = 1024 * 1024;
|
||||
export const NATIVE_CHECKPOINT_INSPECTION_TOTAL_MAX_BYTES = 8 * 1024 * 1024;
|
||||
export type NativeRunnerCheckpointOwner = Binding & { bootId: string; leaseOwner: string; generation: number; authorityRunId: string };
|
||||
function encryptParts(binding: Binding, records: ReadonlyMap<string, string>) {
|
||||
if (!records.size) return [];
|
||||
@@ -16,6 +18,11 @@ function encryptParts(binding: Binding, records: ReadonlyMap<string, string>) {
|
||||
}));
|
||||
}
|
||||
|
||||
export async function insertMigratedNativeRunnerCheckpoint(tx: CheckpointTransaction, binding: Binding, checkpoint: DurablePrpCheckpoint) {
|
||||
if (checkpoint.runId !== binding.runId) throw new Error("native_runner_checkpoint_binding_mismatch");
|
||||
await tx.insert(nativeRunnerCheckpoints).values(encryptParts(binding, checkpoint.records));
|
||||
}
|
||||
|
||||
function decryptPart(cipher: ReturnType<typeof createLocalEncryptedCipher>, binding: Binding,
|
||||
part: string, material: Record<string, unknown>): string {
|
||||
const envelope = JSON.parse(cipher.decrypt(material));
|
||||
|
||||
@@ -4,6 +4,7 @@ import { providerFailureFromError } from "../../vendor/paperclip-runner/index.js
|
||||
import { nativeCancellationHasUnsettledOwner, notifyNativeCancellation, waitForNativeCancellation } from "./native-cancellation-delivery.js";
|
||||
import { readClaimedNativeStopRecovery } from "./native-cancellation-request.js";
|
||||
import { publishNativeHarnessBackup } from "./native-harness-backup-publication.js";
|
||||
import { durableNativeHarnessStorage, nativeHarnessBackupOwner, persistNativeHarnessBackup, restoreDurableNativeHarnessBackup } from "./native-harness-backup-storage.js";
|
||||
import { rehydratePinnedSkillAssets } from "./runtime-context.js";
|
||||
import { createNativeUsageDelivery } from "./native-usage-delivery.js";
|
||||
import { createNativeRecoveryUsageDelivery } from "./native-recovery-usage-delivery.js";
|
||||
@@ -5964,6 +5965,7 @@ function nativeSessionConfigDigest(
|
||||
execution: NativeExecutionInput,
|
||||
executionTargetKind: "local" | "remote" = "local",
|
||||
legacyProjectlessRunId?: string,
|
||||
recordedToolContractFingerprint?: string,
|
||||
): string {
|
||||
const executionLocation = {
|
||||
executionKind: "local_process",
|
||||
@@ -5999,13 +6001,25 @@ function nativeSessionConfigDigest(
|
||||
// changes so an older thread is rotated instead of falsely resuming
|
||||
// without newly required tools.
|
||||
nativeToolContractFingerprint:
|
||||
nativeToolContractFingerprintForTarget(executionTargetKind, execution.provider.kind),
|
||||
recordedToolContractFingerprint ?? nativeToolContractFingerprintForTarget(executionTargetKind, execution.provider.kind),
|
||||
runtimeContract: nativeRuntimeContractForProvider(execution.provider),
|
||||
}),
|
||||
)
|
||||
.digest("hex")}`;
|
||||
}
|
||||
|
||||
export function nativeControllerMigrationDescriptor(execution: NativeExecutionInput, recordedToolContractFingerprint: unknown) {
|
||||
if (typeof recordedToolContractFingerprint !== "string" || !/^sha256:[a-f0-9]{64}$/.test(recordedToolContractFingerprint)) {
|
||||
throw new Error("native_controller_migration_tool_contract_missing");
|
||||
}
|
||||
return {
|
||||
root: scopedRunnerdStateRoot(execution),
|
||||
supervisorPath: resolve(resolvePaperclipInstanceRoot(), "runtime", "paperclip-runner", "sessions",
|
||||
`${createHash("sha256").update(nativeSessionScopeKey(execution)).digest("hex")}.json`),
|
||||
configDigest: nativeSessionConfigDigest(execution, "remote", undefined, recordedToolContractFingerprint),
|
||||
};
|
||||
}
|
||||
|
||||
function idleWarmNativeSessionOwnerRunId(input: {
|
||||
execution: NativeExecutionInput;
|
||||
runnerExecutionTarget?: AdapterExecutionTarget | null;
|
||||
@@ -6027,6 +6041,7 @@ async function restorePerTurnRunnerdController(input: {
|
||||
if (input.restartRecovery || input.runnerExecutionTarget?.kind !== "remote" ||
|
||||
input.execution.session.lifecyclePolicy.mode !== "per_turn") return;
|
||||
const root = scopedRunnerdStateRoot(input.execution);
|
||||
await restoreDurableHarnessForExecution(input.db, input.execution, root);
|
||||
const identity = readRunnerdDurableIdentity(root);
|
||||
const reference = identity ?? readRunnerdDurableIdentity(root, true);
|
||||
if (!reference) {
|
||||
@@ -6091,6 +6106,7 @@ async function restoreWarmRunnerdController(input: {
|
||||
if (input.restartRecovery || input.runnerExecutionTarget?.kind !== "remote" ||
|
||||
input.execution.session.lifecyclePolicy.mode !== "warm") return;
|
||||
const root = scopedRunnerdStateRoot(input.execution);
|
||||
await restoreDurableHarnessForExecution(input.db, input.execution, root);
|
||||
const identity = readRunnerdDurableIdentity(root);
|
||||
const reference = identity ?? readRunnerdDurableIdentity(root, true);
|
||||
if (!idleWarmNativeSessionOwnerRunId(input) && (!reference || !existsSync(nativeSessionCheckpointPath(input.execution)))) {
|
||||
@@ -6599,6 +6615,14 @@ function harnessBackupRoot(root: string): string {
|
||||
return resolve(root, "failover-backups");
|
||||
}
|
||||
|
||||
async function restoreDurableHarnessForExecution(db: Db, execution: NativeExecutionInput, root: string) {
|
||||
const storage = durableNativeHarnessStorage();
|
||||
if (!storage) return;
|
||||
await restoreDurableNativeHarnessBackup({ db, storage, companyId: execution.binding.companyId,
|
||||
scopeDigest: createHash("sha256").update(nativeSessionScopeKey(execution)).digest("hex"),
|
||||
backupRoot: harnessBackupRoot(root), signal: AbortSignal.timeout(300_000) });
|
||||
}
|
||||
|
||||
function harnessBackupCandidates(root: string): string[] {
|
||||
const backupRoot = harnessBackupRoot(root);
|
||||
return [resolve(backupRoot, "current"), resolve(backupRoot, "previous")];
|
||||
@@ -14069,6 +14093,11 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
!remoteRunnerFilesystemRoot
|
||||
)
|
||||
return;
|
||||
const durableStorage = durableNativeHarnessStorage();
|
||||
if (durableStorage && !input.controllerPersistence) throw new Error("native_harness_storage_owner_missing");
|
||||
const backupOwner = durableStorage ? await nativeHarnessBackupOwner(input.db, { ...input.execution.binding,
|
||||
runId: sessionToolAuthorityEpochs.get(sessionScopeId)?.runId ?? input.execution.binding.runId,
|
||||
bootId: input.controllerPersistence!.bootId }) : null;
|
||||
// Transport release also runs when provider bootstrap failed. In that case
|
||||
// runnerd has no durable provider identity (and may not have created the
|
||||
// provider persistence directory at all), so attempting a failover backup
|
||||
@@ -14174,6 +14203,11 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
});
|
||||
renameSync(temporaryManifest, manifestPath);
|
||||
|
||||
if (durableStorage && backupOwner) {
|
||||
await persistNativeHarnessBackup({ db: input.db, storage: durableStorage, owner: backupOwner,
|
||||
scopeDigest: createHash("sha256").update(sessionScopeId).digest("hex"), root: pendingRoot,
|
||||
signal: AbortSignal.timeout(300_000) });
|
||||
}
|
||||
await publishNativeHarnessBackup({
|
||||
db: input.db, companyId: input.execution.binding.companyId,
|
||||
scopeDigest: createHash("sha256").update(sessionScopeId).digest("hex"), backupRoot, pendingRoot,
|
||||
|
||||
@@ -225,7 +225,7 @@ type NativeSessionResumeSelection = {
|
||||
agentId: string;
|
||||
issueId: string;
|
||||
normalizedSessionId: string;
|
||||
currentRunId: string;
|
||||
currentRunId?: string;
|
||||
beforeCreatedAt: Date;
|
||||
/** Recovery discovery must retain an ineligible newest progress barrier. */
|
||||
includeProgressBarrier?: boolean;
|
||||
@@ -306,7 +306,7 @@ async function selectNativeSessionResumeBarrier(db: Db, input: NativeSessionResu
|
||||
.from(heartbeatRuns)
|
||||
.where(
|
||||
and(
|
||||
ne(heartbeatRuns.id, input.currentRunId),
|
||||
input.currentRunId ? ne(heartbeatRuns.id, input.currentRunId) : undefined,
|
||||
eq(heartbeatRuns.companyId, input.companyId),
|
||||
eq(heartbeatRuns.agentId, input.agentId),
|
||||
eq(heartbeatRuns.nativeIssueId, input.issueId),
|
||||
|
||||
@@ -73,28 +73,22 @@ export function shouldUseRemoteWorkspaces(input: {
|
||||
return configuredRemoteWorkspacePersistence() !== null;
|
||||
}
|
||||
|
||||
/** Deployment profile admission belongs beside workspace authority. A warm
|
||||
* remote controller may replace its local projection; host-owned execution
|
||||
* cannot make that promise. External conversation providers own no task disk. */
|
||||
export function assertEphemeralWorkspaceExecution(input: {
|
||||
externalConversation: boolean; adapterType: string;
|
||||
pluginTaskWorkspace: boolean; remoteAuthority: boolean;
|
||||
}) {
|
||||
if (process.env.PAPERCLIP_EPHEMERAL_SERVER !== "true" || input.externalConversation) return;
|
||||
// Runtime selection has not promoted a fresh run yet. Validate storage here;
|
||||
// the runtime selector separately admits only its resolved warm native policy.
|
||||
if (input.adapterType !== "paperclip_runner"
|
||||
|| (!input.pluginTaskWorkspace && !input.remoteAuthority)) {
|
||||
throw unprocessable("Volume-free servers require warm Paperclip Runner execution with remote workspace storage.",
|
||||
throw unprocessable("Volume-free servers require native Paperclip Runner execution with remote workspace storage.",
|
||||
{ code: "ephemeral_execution_configuration_required" });
|
||||
}
|
||||
}
|
||||
|
||||
/** The runtime selector calls this after environment, credentials and saved
|
||||
* execution policy have resolved. Raw agent settings are not lifecycle authority. */
|
||||
export function assertEphemeralNativeLifecycle(mode: "warm" | "per_turn" | undefined) {
|
||||
if (process.env.PAPERCLIP_EPHEMERAL_SERVER === "true" && mode !== "warm") {
|
||||
throw unprocessable("Volume-free servers require warm Paperclip Runner execution with remote workspace storage.",
|
||||
export function assertEphemeralNativeRuntime(native: boolean) {
|
||||
if (process.env.PAPERCLIP_EPHEMERAL_SERVER === "true" && !native) {
|
||||
throw unprocessable("Volume-free servers require native Paperclip Runner execution with remote workspace storage.",
|
||||
{ code: "ephemeral_execution_configuration_required" });
|
||||
}
|
||||
}
|
||||
|
||||
+1
@@ -94,6 +94,7 @@ export const RUNNER_TOOL_SCOPE_TOKEN_HEADER = runner.RUNNER_TOOL_SCOPE_TOKEN_HEA
|
||||
export const DurablePrpControlPlane = runner.DurablePrpControlPlane;
|
||||
export const providerFailureFromError = runner.providerFailureFromError;
|
||||
export const decodeDurablePrpCheckpoint = runner.decodeDurablePrpCheckpoint;
|
||||
export const captureLegacyDurablePrpCheckpoint = runner.captureLegacyDurablePrpCheckpoint;
|
||||
export const inspectDurablePrpCheckpointHeader = runner.inspectDurablePrpCheckpointHeader;
|
||||
export const RUNNER_PRP_PROTOCOL_MIN = runner.RUNNER_PRP_PROTOCOL_MIN;
|
||||
export const RUNNER_PRP_PROTOCOL_MAX = runner.RUNNER_PRP_PROTOCOL_MAX;
|
||||
|
||||
@@ -104,6 +104,19 @@ its DB checkpoint alone cannot replace those bytes. Per-turn application
|
||||
qualification under this disk loss awaits the provider-backup owner's durable
|
||||
storage contract; native controller recovery does not waive that dependency.
|
||||
|
||||
`rolling-web-deployment` definition 4 selects every registered native Daytona
|
||||
profile in disposable per-turn environments; `rolling-web-deployment-warm`
|
||||
uses the same profiles in reusable warm environments. Their
|
||||
provider-backup path uses encrypted object storage and a fenced database ledger.
|
||||
The earlier per-turn failure remains failed evidence; this definition requires
|
||||
a fresh passing run, including the strict post-replacement continuation oracle.
|
||||
Its revision task includes a 180-second foreground CLI reliability soak, giving
|
||||
the replacement time to become ready while useful agent work remains active.
|
||||
The observer rechecks the original live owner immediately before switching
|
||||
traffic and fails if that turn already settled. A completed turn cannot count
|
||||
as active takeover. Per-turn file inspection requires the exact committed saved
|
||||
snapshot; the warm lane still requires the live workspace.
|
||||
|
||||
The ordinary Everyday matrix retains its 125-second active outage default. The
|
||||
two explicit continuity suites require 300 seconds and reject a shorter override.
|
||||
|
||||
|
||||
+38
-14
@@ -1,4 +1,4 @@
|
||||
import { ROLLING_WEB_DEPLOYMENT_SUITE } from "./controller-rolling-deployment.js";
|
||||
import { ROLLING_WEB_DEPLOYMENT_SUITE, ROLLING_WEB_DEPLOYMENT_WARM_SUITE } from "./controller-rolling-deployment.js";
|
||||
import { FEEDBACK_PERSISTENCE_SUITE, feedbackPersistenceTask, feedbackPersistenceDigest } from "./feedback-persistence.js";
|
||||
import { followupStopTask, followupStopDigest } from "./followup-stop-resume.js";
|
||||
import { finalizationRecoveryTask, finalizationRecoveryDigest } from "./finalization-recovery.js";
|
||||
@@ -1233,15 +1233,44 @@ const remoteWorkspaceStorageSuite: RunnerSuiteFixture = {
|
||||
source: "projectless-synthetic-files-and-model-created-git", authentication: "api-key", stagingQualification: false },
|
||||
};
|
||||
|
||||
export const kimiRunnerProfile = nativeProfile({
|
||||
id: "runner-acpx-kimi", label: "Runner Kimi Code",
|
||||
provider: "acpx", acpxAgent: "kimi", qualificationCandidate: "kimi",
|
||||
credential: "KIMI_API_KEY", model: "kimi-for-coding",
|
||||
modelQualification: { source: "qualified_runner_profile", qualificationId: "kimi:2.3.3:native-qualified-profile17" },
|
||||
});
|
||||
|
||||
const rollingDeploymentProfiles = [...runnerProfiles, ...extendedHarnessProfiles, kimiRunnerProfile]
|
||||
.filter(profile => profile.generation === "native" && profile.supportedEnvironments.includes("daytona"))
|
||||
.map(original => {
|
||||
const profile = productionStoryProfile(original);
|
||||
return { ...profile, buildAgent(input: AgentFixtureBuildInput) {
|
||||
const agent = profile.buildAgent(input);
|
||||
const adapterConfig = { ...(agent.adapterConfig as Record<string, unknown>) };
|
||||
delete adapterConfig.cwd;
|
||||
return { ...agent, adapterConfig };
|
||||
} };
|
||||
});
|
||||
const rollingDeploymentSuites: RunnerSuiteFixture[] = [
|
||||
{ id: ROLLING_WEB_DEPLOYMENT_SUITE, environment: runnerEnvironments.find(environment => environment.id === "daytona")! },
|
||||
{ id: ROLLING_WEB_DEPLOYMENT_WARM_SUITE, environment: daytonaControllerOutageEnvironment },
|
||||
].map(({ id, environment }) => ({
|
||||
...remoteWorkspaceStorageSuite, id, label: `Rolling control-plane web deployment (${environment.configurationKey ? "warm" : "per-turn"})`,
|
||||
description: "Overlap independent ephemeral web servers while a remote execution stays active; keep ordinary reads, saves, files and a later native continuation working.",
|
||||
profiles: rollingDeploymentProfiles, environments: [environment],
|
||||
expectedMatrixSize: rollingDeploymentProfiles.length * remoteWorkspaceStorageSuite.tasks.length,
|
||||
definitionMetadata: { ...remoteWorkspaceStorageSuite.definitionMetadata, version: 4, controllerDowntimeMs: 0,
|
||||
activeWorkBoundary: "foreground-180-second-cli-reliability-soak-and-live-owner-recheck",
|
||||
workspaceFileSource: environment.configurationKey ? "live" : "saved-exact-snapshot-after-per-turn-retirement",
|
||||
lifecycleCoverage: environment.configurationKey ? "warm-reusable" : "per-turn-disposable", providerCoverage: "registered-native-daytona-profiles",
|
||||
replacement: "ready-before-traffic-switch-sigterm-delete-entire-home",
|
||||
upgradeReadiness: "active-remote-workspace-recoverable-before-handoff",
|
||||
grading: "same-active-execution-ordinary-reads-saves-and-second-empty-home-continuation" },
|
||||
}));
|
||||
|
||||
export const runnerSuites: readonly RunnerSuiteFixture[] = [
|
||||
remoteWorkspaceStorageSuite,
|
||||
{ ...remoteWorkspaceStorageSuite, id: ROLLING_WEB_DEPLOYMENT_SUITE, label: "Rolling control-plane web deployment",
|
||||
description: "Overlap independent ephemeral web servers while a remote execution stays active; keep ordinary reads, saves, files and a later native continuation working.",
|
||||
definitionMetadata: { ...remoteWorkspaceStorageSuite.definitionMetadata, version: 1, controllerDowntimeMs: 0,
|
||||
replacement: "ready-before-traffic-switch-sigterm-delete-entire-home",
|
||||
upgradeReadiness: "active-remote-workspace-recoverable-before-handoff",
|
||||
grading: "same-active-execution-ordinary-reads-saves-and-second-empty-home-continuation" },
|
||||
},
|
||||
...rollingDeploymentSuites,
|
||||
{
|
||||
id: CURSOR_CLOUD_SUITE, label: "Native Cursor Cloud", manualOnly: true, groups: [],
|
||||
description: "Local Paperclip controller with hosted Cursor work: verified artifact, same-agent follow-up, real active-run Stop and semantic plan revision/approval.",
|
||||
@@ -1537,12 +1566,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
|
||||
id: "kimi-qualification", label: "Kimi Code Qualification", manualOnly: true,
|
||||
description: "Kimi Code replies, semantic planning and questions, verified file delivery, Stop and controller continuation through local and Daytona user journeys.",
|
||||
groups: ["native"],
|
||||
profiles: [nativeProfile({
|
||||
id: "runner-acpx-kimi", label: "Runner Kimi Code",
|
||||
provider: "acpx", acpxAgent: "kimi", qualificationCandidate: "kimi",
|
||||
credential: "KIMI_API_KEY", model: "kimi-for-coding",
|
||||
modelQualification: { source: "qualified_runner_profile", qualificationId: "kimi:2.3.3:native-qualified-profile17" },
|
||||
})],
|
||||
profiles: [kimiRunnerProfile],
|
||||
environments: [localEnvironment, daytonaWarmEnvironment],
|
||||
environmentOverrides: Object.fromEntries(kimiLifecycleTasks.map(task => [task.id, {
|
||||
daytona: runnerEnvironments.find(environment => environment.id === "daytona")!,
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
import type { Db } from "../../packages/db/src/client.js";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { advanceRecoveryCursor, parseRecoveryBinding, readControllerRecoverySnapshot, recoveryObservation, validControllerRecoveryReceipt, type RecoverySnapshot } from "./controller-recovery-observer.js";
|
||||
import { advanceRecoveryCursor, assertActiveRecoveryBoundary, parseRecoveryBinding, readControllerRecoverySnapshot, recoveryObservation, validControllerRecoveryReceipt, type RecoverySnapshot } from "./controller-recovery-observer.js";
|
||||
import type { ControllerOutageReceipt } from "./controller-outage.js";
|
||||
|
||||
const binding = { companyId: "10000000-0000-0000-0000-000000000001", issueId: "10000000-0000-0000-0000-000000000002", runId: "10000000-0000-0000-0000-000000000003" };
|
||||
const before: RecoverySnapshot = { runId: binding.runId, runnerInstanceId: "runner", nativeSessionId: "session",
|
||||
status: "running", phase: "observed", resultId: null,
|
||||
bootId: "old", generation: 1, attempt: 1, leaseExpiresAt: "2026-10-24T00:01:00.000Z", sourceSeq: 42, history: [],
|
||||
authenticatedConnectionCount: 1, checkpointSourceSeq: 42 };
|
||||
const fault: ControllerOutageReceipt = { schema: "paperclip.e2e.controller-outage/v1", requestId: "restart", previousPid: 1,
|
||||
@@ -13,6 +14,14 @@ const fault: ControllerOutageReceipt = { schema: "paperclip.e2e.controller-outag
|
||||
const after = (): RecoverySnapshot => ({ ...before, bootId: "new", generation: 2, sourceSeq: 47, authenticatedConnectionCount: 2,
|
||||
history: [{ controllerBootId: "new", controllerGeneration: 2, disposition: "reattach_remote_runner", at: "2026-10-24T00:01:00.001Z" }] });
|
||||
describe("read-only controller recovery evidence", () => {
|
||||
it("fails immediately when the original work settles while the replacement starts", () => {
|
||||
const active = { ...before, leaseExpiresAt: new Date(Date.now() + 60_000).toISOString() };
|
||||
expect(() => assertActiveRecoveryBoundary(active)).not.toThrow();
|
||||
for (const patch of [{ status: "succeeded" }, { phase: "committed" }, { phase: "workspace_finalizing" },
|
||||
{ resultId: "accepted" }, { leaseExpiresAt: null }, { leaseExpiresAt: new Date(0).toISOString() }]) {
|
||||
expect(() => assertActiveRecoveryBoundary({ ...active, ...patch })).toThrow("boundary unexercised");
|
||||
}
|
||||
});
|
||||
it("does not count A's shutdown tail as B's processing even if B already claimed", () => {
|
||||
const stopped = { ...after(), checkpointSourceSeq: 48 };
|
||||
const baseline = advanceRecoveryCursor(before, stopped);
|
||||
|
||||
@@ -13,11 +13,19 @@ import type { ControllerReplacementReceipt } from "./controller-replacement.js";
|
||||
export interface RecoveryBinding { companyId: string; issueId: string; runId: string }
|
||||
export interface RecoverySnapshot {
|
||||
runId: string; runnerInstanceId: string; nativeSessionId: string;
|
||||
status: string; phase: string; resultId: string | null;
|
||||
bootId: string; generation: number; attempt: number; leaseExpiresAt: string | null;
|
||||
sourceSeq: number; history: Array<Record<string, unknown>>;
|
||||
authenticatedConnectionCount: number;
|
||||
checkpointSourceSeq: number;
|
||||
}
|
||||
|
||||
export function assertActiveRecoveryBoundary(snapshot: RecoverySnapshot): void {
|
||||
if (snapshot.status !== "running" || snapshot.phase !== "observed" || snapshot.resultId !== null
|
||||
|| !snapshot.leaseExpiresAt || !Number.isFinite(Date.parse(snapshot.leaseExpiresAt)) || Date.parse(snapshot.leaseExpiresAt) <= Date.now()) {
|
||||
throw new Error("Active controller handoff boundary unexercised: original execution settled before traffic switched");
|
||||
}
|
||||
}
|
||||
export interface ControllerRecoveryReceipt {
|
||||
binding: RecoveryBinding;
|
||||
runnerInstanceId: string;
|
||||
@@ -98,7 +106,7 @@ export function validControllerRecoveryReceipt(fault: ControllerReplacementRecei
|
||||
|
||||
export async function readControllerRecoverySnapshot(db: Db, binding: RecoveryBinding): Promise<RecoverySnapshot> {
|
||||
return db.transaction(async tx => {
|
||||
const [row] = await tx.execute(sql`select r.id as "runId", r.runner_instance_id as "runnerInstanceId",
|
||||
const [row] = await tx.execute(sql`select r.id as "runId", r.status, f.phase, f.result_id as "resultId", r.runner_instance_id as "runnerInstanceId",
|
||||
r.native_session_id as "nativeSessionId", f.controller_boot_id as "bootId", f.controller_generation as generation,
|
||||
f.attempt, f.lease_expires_at::text as "leaseExpiresAt", f.recovery_history as history,
|
||||
coalesce((select max(e.source_seq) from heartbeat_run_events e where e.run_id = r.id
|
||||
@@ -139,7 +147,10 @@ export async function createControllerRecoveryObserver(configPath: string, bindi
|
||||
runnerInstanceId: before.runnerInstanceId, nativeSessionId: before.nativeSessionId };
|
||||
};
|
||||
return {
|
||||
beforeHandoff: capture,
|
||||
beforeHandoff: async (fault: ControllerReplacementReceipt) => {
|
||||
await capture(fault);
|
||||
assertActiveRecoveryBoundary(before!);
|
||||
},
|
||||
// Called only after the old server is confirmed dead. This also excludes
|
||||
// receipts committed while it was stopping from replacement evidence.
|
||||
stopped: capture,
|
||||
|
||||
@@ -3,6 +3,7 @@ import { rollController, validControllerRollingDeploymentReceipt } from "./contr
|
||||
import { validControllerReplacementReceipt } from "./controller-replacement.js";
|
||||
import { controllerOutageForExecution, validControllerOutageReceipt } from "./controller-outage.js";
|
||||
import { parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js";
|
||||
import { runnerProfiles, extendedHarnessProfiles, kimiRunnerProfile, runnerSuites } from "./catalog.js";
|
||||
|
||||
function fixture() {
|
||||
const order: string[] = [];
|
||||
@@ -55,4 +56,27 @@ describe("rolling controller qualification", () => {
|
||||
const [outage] = selectRunnerExecutions(parseRunnerSelectors(["--suite", "remote-workspace-storage"]));
|
||||
expect(controllerOutageForExecution(outage!, {})).toEqual({ signal: "SIGKILL", holdMs: 300_000 });
|
||||
});
|
||||
it("requires both lifecycle modes for every registered native sandbox profile", () => {
|
||||
const executions = ["rolling-web-deployment", "rolling-web-deployment-warm"]
|
||||
.flatMap(suite => selectRunnerExecutions(parseRunnerSelectors(["--suite", suite])));
|
||||
const native = [...runnerProfiles, ...extendedHarnessProfiles, kimiRunnerProfile]
|
||||
.filter(profile => profile.generation === "native" && profile.supportedEnvironments.includes("daytona"));
|
||||
const kind = (profile: (typeof native)[number]) => {
|
||||
const provider = profile.expectedRuntimeMetadata.provider;
|
||||
const harness = "acpxAgent" in profile ? profile.acpxAgent
|
||||
: provider === "acpx" && profile.provider !== "acpx" ? profile.provider : "";
|
||||
return `${provider}:${harness}`;
|
||||
};
|
||||
const registeredKinds = new Set(runnerSuites.flatMap(suite => suite.profiles)
|
||||
.filter(profile => profile.generation === "native" && profile.supportedEnvironments.includes("daytona"))
|
||||
.map(kind));
|
||||
expect(new Set(native.map(kind))).toEqual(registeredKinds);
|
||||
for (const profile of native) {
|
||||
const cells = executions.filter(execution => execution.profile.id === profile.id);
|
||||
expect(cells).toHaveLength(2);
|
||||
expect(new Set(cells.map(cell => cell.environment.configurationKey ?? "per_turn")))
|
||||
.toEqual(new Set(["per_turn", "warm-controller-outage-v1"]));
|
||||
}
|
||||
expect(executions.every(execution => execution.suite.definitionMetadata?.retainedControllerContinuation === true)).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -3,7 +3,9 @@ import type { MatrixExecution } from "./types.js";
|
||||
import type { ControllerOutageReceipt } from "./controller-outage.js";
|
||||
|
||||
export const ROLLING_WEB_DEPLOYMENT_SUITE = "rolling-web-deployment";
|
||||
export const isRollingWebDeployment = (execution: Pick<MatrixExecution, "suite">) => execution.suite.id === ROLLING_WEB_DEPLOYMENT_SUITE;
|
||||
export const ROLLING_WEB_DEPLOYMENT_WARM_SUITE = "rolling-web-deployment-warm";
|
||||
export const isRollingWebDeployment = (execution: Pick<MatrixExecution, "suite">) =>
|
||||
[ROLLING_WEB_DEPLOYMENT_SUITE, ROLLING_WEB_DEPLOYMENT_WARM_SUITE].includes(execution.suite.id);
|
||||
|
||||
export interface ControllerRollingDeploymentReceipt extends Omit<ControllerOutageReceipt, "schema" | "holdMs" | "signal"> {
|
||||
schema: "paperclip.e2e.controller-rolling-deployment/v1";
|
||||
|
||||
@@ -29,6 +29,7 @@ export function productionStoryProfile(
|
||||
|
||||
export const SLUGIFY_REQUIREMENTS = `Build a small dependency-free Python command-line tool in slugify.py. It accepts one positional string and prints its slug: trim whitespace, lowercase, replace each sequence outside ASCII a-z and 0-9 with one hyphen, then trim hyphens. Expose slugify(text) for reuse. Include a README and automated tests. Deliver the source and tests as a downloadable ZIP. Work in the project workspace.`;
|
||||
export const SLUGIFY_REVISION = `Add an optional --separator argument that accepts either - or _. Keep - as the default and preserve the earlier behavior. Deliver an updated ZIP, keeping the previous download available.`;
|
||||
export const SLUGIFY_DEPLOYMENT_SOAK = `Before editing, run a short command that prints the current workspace path. After implementing the revision, run a foreground reliability soak for a full 180 seconds: repeatedly invoke the CLI on varied inputs and check its output, sleeping briefly between iterations to avoid excessive CPU use. Wait for that command to finish; do not background it or shorten the duration. Include a small soak report with measured elapsed time and check count in the updated ZIP. Finish only after the soak and the regular tests pass.`;
|
||||
export const LATE_REQUIREMENT = `Also support --max-length as a positive integer. Truncate the final slug to that length, then remove any trailing separator. In particular, input " Queue--Ready!! " with --max-length 7 must print "queue-r". Preserve the default behavior.`;
|
||||
|
||||
const definitions = [
|
||||
|
||||
@@ -39,7 +39,7 @@ import {
|
||||
StoryDecisionError,
|
||||
type StoryInteraction,
|
||||
} from "./everyday-decisions.js";
|
||||
import { LATE_REQUIREMENT, SLUGIFY_REVISION, requiresEverydayArtifactOracle } from "./everyday-cases.js";
|
||||
import { LATE_REQUIREMENT, SLUGIFY_REVISION, SLUGIFY_DEPLOYMENT_SOAK, requiresEverydayArtifactOracle } from "./everyday-cases.js";
|
||||
import {
|
||||
isActiveStoryRun,
|
||||
isStoryWorkspaceDeferral,
|
||||
@@ -186,7 +186,8 @@ export async function runEverydayFlow(input: Input) {
|
||||
const controllerOutage = controllerOutageForExecution(execution, process.env);
|
||||
const requireRemoteRestart = controllerOutage !== undefined || isRollingWebDeployment(execution);
|
||||
const remoteWorkspaceStorage = isRemoteWorkspaceStorageExecution(execution);
|
||||
if (remoteWorkspaceStorage) ev.remoteWorkspaceStorage = { browserReadPhases: [] };
|
||||
if (remoteWorkspaceStorage) ev.remoteWorkspaceStorage = { browserReadPhases: [],
|
||||
expectedFileSource: execution.suite.definitionMetadata?.lifecycleCoverage === "per-turn-disposable" ? "saved" : "live" };
|
||||
const providerChoice = caseId === "provider-decline" || caseId === "provider-second";
|
||||
const nativeProviderCase = caseId === "provider-native";
|
||||
let aggregatorFixture: Awaited<ReturnType<typeof setupAggregatorFixture>> | undefined;
|
||||
@@ -562,7 +563,8 @@ export async function runEverydayFlow(input: Input) {
|
||||
const point = await pollUntil({
|
||||
label: `remote workspace ${phase} saved version`, deadlineAt: input.deadlineAt,
|
||||
load: () => readRemoteWorkspaceStorageCheckpoint({ api, companyId: fixtures.company.id,
|
||||
agentId: fixtures.agent.id, issueId: parent!.id, environmentId: fixtures.environment.id }),
|
||||
agentId: fixtures.agent.id, issueId: parent!.id, environmentId: fixtures.environment.id,
|
||||
expectedFileSource: ev.remoteWorkspaceStorage!.expectedFileSource }),
|
||||
accept: observed => Boolean(observed.point),
|
||||
});
|
||||
ev.remoteWorkspaceStorage![phase] = point.point!;
|
||||
@@ -570,7 +572,8 @@ export async function runEverydayFlow(input: Input) {
|
||||
const browserPhase = phase === "initial" ? "before-restart" : "after-restart";
|
||||
await page.goto(`${taskUrl(parent!)}?browse=1&q=${REMOTE_WORKSPACE_PROOF_FILE}`);
|
||||
await expect(page.getByRole("combobox", { name: "Search workspace files" })).toBeVisible();
|
||||
await expect(page.getByLabel("Workspace file source", { exact: true })).toHaveText("Live workspace");
|
||||
await expect(page.getByLabel("Workspace file source", { exact: true })).toHaveText(
|
||||
ev.remoteWorkspaceStorage!.expectedFileSource === "saved" ? /^Saved copy(?: from .+)?$/ : "Live workspace");
|
||||
await page.getByRole("treeitem").filter({ hasText: REMOTE_WORKSPACE_PROOF_FILE }).last().click();
|
||||
await expect(page.getByText(point.content!.trim(), { exact: true })).toBeVisible();
|
||||
await input.capture(`remote-workspace-${browserPhase}`, `Remote workspace file ${browserPhase}`, `remote-workspace-${browserPhase}.png`);
|
||||
@@ -936,7 +939,7 @@ export async function runEverydayFlow(input: Input) {
|
||||
throw new Error(
|
||||
"Remote fault boundary unexercised: no verified saved project",
|
||||
);
|
||||
await reply(SLUGIFY_REVISION);
|
||||
await reply(isRollingWebDeployment(execution) ? `${SLUGIFY_REVISION}\n\n${SLUGIFY_DEPLOYMENT_SOAK}` : SLUGIFY_REVISION);
|
||||
note("remote-revision-submitted");
|
||||
await pollUntil({
|
||||
label: "remote revision executing",
|
||||
|
||||
@@ -195,6 +195,18 @@ describe("remote restart lifecycle evidence", () => {
|
||||
expect(gradeRemoteRestart(evidence).every(check => check.passed)).toBe(true);
|
||||
});
|
||||
|
||||
it("accepts the same start notification after reattachment and rejects a different turn or payload", () => {
|
||||
const f = fixture();
|
||||
const replay = f.event(5, "turn.started");
|
||||
const after = [...f.rows, replay];
|
||||
expect(observeRemoteRestartBoundary(f.active, after, f.leases, f.scope, true).identity).toEqual(f.evidence.before.identity);
|
||||
expect(() => observeRemoteRestartBoundary(f.active, after, f.leases, f.scope)).toThrow("active original provider turn");
|
||||
expect(() => observeRemoteRestartBoundary(f.active, [...f.rows, f.event(5, "turn.started", { changed: true })], f.leases, f.scope, true))
|
||||
.toThrow("active original provider turn");
|
||||
replay.payload.prpEvent.turnId = "another-turn";
|
||||
expect(() => observeRemoteRestartBoundary(f.active, after, f.leases, f.scope, true)).toThrow("active original provider turn");
|
||||
});
|
||||
|
||||
it.each(["replacement", "second submission", "changed tool receipt", "changed start receipt", "missing terminal"])("rejects %s at completion", fault => {
|
||||
const { evidence } = fixture();
|
||||
if (fault === "replacement") evidence.final.run.id = "replacement";
|
||||
|
||||
@@ -125,7 +125,10 @@ export function observeRemoteRestartBoundary(run: StoryRun, rows: readonly Row[]
|
||||
const events = projectRemoteRestartEvents(run, rows);
|
||||
const starts = events.filter(event => event.eventType === "turn.started" && event.sourceKind === "runner");
|
||||
const completed = events.filter(event => event.toolCompleted && event.sourceKind === "runner" && event.toolExecutionId);
|
||||
if (starts.length !== 1 || !id(starts[0]!.turnId) || completed.length === 0
|
||||
const originalStart = starts[0];
|
||||
const sameStart = originalStart && id(originalStart.turnId) && starts.every(event =>
|
||||
event.turnId === originalStart.turnId && event.payloadSha256 === originalStart.payloadSha256);
|
||||
if ((!recovered && starts.length !== 1) || !sameStart || completed.length === 0
|
||||
|| completed.some(event => event.turnId !== starts[0]!.turnId)
|
||||
|| (!recovered && events.some(event => ["turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted", "run.terminal"].includes(event.eventType)))) {
|
||||
throw new Error("Remote restart boundary has no completed tool in an active original provider turn");
|
||||
|
||||
@@ -56,6 +56,36 @@ describe("remote workspace Product E2E qualification", () => {
|
||||
expect(result.content).toBe(`${"a".repeat(32)}\n`);
|
||||
});
|
||||
|
||||
it("reads the exact saved snapshot after per-turn sandbox retirement", async () => {
|
||||
const f = checkpointApi();
|
||||
const originalGet = f.api.get;
|
||||
const location = f.workspace.workspace.metadata.remoteWorkspace.location;
|
||||
const detached = { ...f.workspace, workspace: { ...f.workspace.workspace,
|
||||
metadata: { remoteWorkspace: { ...f.workspace.workspace.metadata.remoteWorkspace, location: null } } } };
|
||||
const file = { ...f.file, resource: { ...f.file.resource, source: { kind: "saved", snapshotId: f.receipt.snapshotId } } };
|
||||
const lease = { id: location.leaseId, companyId: f.context.companyId, issueId: f.context.issueId,
|
||||
providerLeaseId: location.providerLeaseId };
|
||||
const api = { get: async <T>(route: string): Promise<T> => {
|
||||
if (route.endsWith("/workspace")) return detached as T;
|
||||
if (route.endsWith("/leases")) return [lease] as T;
|
||||
if (route.includes("file-resources/content")) return file as T;
|
||||
return originalGet<T>(route);
|
||||
} };
|
||||
const input = { ...f.context, api, expectedFileSource: "saved" as const };
|
||||
expect((await readRemoteWorkspaceStorageCheckpoint(input)).point).toMatchObject({ fileSource: "saved", fileSnapshotId: id("2") });
|
||||
expect(await readRemoteWorkspaceStorageCheckpoint({ ...input, expectedFileSource: "live" })).toEqual({});
|
||||
file.resource.source.snapshotId = id("5");
|
||||
expect(await readRemoteWorkspaceStorageCheckpoint(input)).toEqual({});
|
||||
file.resource.source.snapshotId = f.receipt.snapshotId;
|
||||
lease.companyId = id("5");
|
||||
expect(await readRemoteWorkspaceStorageCheckpoint(input)).toEqual({});
|
||||
const evidence = goodEvidence(); evidence.expectedFileSource = "saved";
|
||||
for (const point of [evidence.initial!, evidence.final!]) Object.assign(point, { fileSource: "saved", fileSnapshotId: point.snapshotId });
|
||||
expect(gradeRemoteWorkspaceStorage(evidence).every(check => check.passed)).toBe(true);
|
||||
evidence.final!.fileSnapshotId = id("9");
|
||||
expect(gradeRemoteWorkspaceStorage(evidence).every(check => check.passed)).toBe(false);
|
||||
});
|
||||
|
||||
it.each(["run-id", "company", "agent", "native-issue", "runtime", "status", "schema", "workspace",
|
||||
"epoch", "snapshot", "save-state", "lease", "provider", "remote-root"] as const)(
|
||||
"rejects a full-run %s mismatch even when its public summary succeeds", async mismatch => {
|
||||
|
||||
@@ -23,9 +23,11 @@ export interface RemoteWorkspaceStorageCheckpoint {
|
||||
providerType: string;
|
||||
proofSha256: string;
|
||||
fileSource: string;
|
||||
fileSnapshotId?: string;
|
||||
}
|
||||
|
||||
export interface RemoteWorkspaceStorageEvidence {
|
||||
expectedFileSource?: "live" | "saved";
|
||||
initial?: RemoteWorkspaceStorageCheckpoint;
|
||||
final?: RemoteWorkspaceStorageCheckpoint;
|
||||
browserReadPhases: string[];
|
||||
@@ -47,6 +49,7 @@ export function remoteWorkspaceProofDigest(content: string): string {
|
||||
|
||||
export async function readRemoteWorkspaceStorageCheckpoint(input: {
|
||||
api: Pick<RunnerApi, "get">; companyId: string; agentId: string; issueId: string; environmentId: string;
|
||||
expectedFileSource?: "live" | "saved";
|
||||
}): Promise<{ point?: RemoteWorkspaceStorageCheckpoint; content?: string }> {
|
||||
const view = await input.api.get<{ issueId: string; workspace: ExecutionWorkspace | null }>(`/api/issues/${input.issueId}/workspace`);
|
||||
const workspace = view.workspace;
|
||||
@@ -54,8 +57,10 @@ export async function readRemoteWorkspaceStorageCheckpoint(input: {
|
||||
if (view.issueId !== input.issueId || !workspace || workspace.companyId !== input.companyId
|
||||
|| workspace.providerType !== "cloud_sandbox" || !authorityResult.success) return {};
|
||||
const authority = authorityResult.data;
|
||||
if (!authority.snapshotId || authority.dirty || authority.writerRunId || !authority.location
|
||||
|| authority.location.environmentId !== input.environmentId) return {};
|
||||
const expectedSource = input.expectedFileSource ?? "live";
|
||||
if (!authority.snapshotId || authority.dirty || authority.writerRunId) return {};
|
||||
if (expectedSource === "live" && (!authority.location || authority.location.environmentId !== input.environmentId)) return {};
|
||||
if (expectedSource === "saved" && authority.location !== null) return {};
|
||||
const runs = await input.api.get<RunSummary[]>(`/api/issues/${input.issueId}/runs`);
|
||||
const latest = runs.filter(run => run.runtimeMode === "native" && run.status === "succeeded"
|
||||
&& run.agentId === input.agentId && typeof run.runId === "string")
|
||||
@@ -68,19 +73,27 @@ export async function readRemoteWorkspaceStorageCheckpoint(input: {
|
||||
if (!receiptResult.success) return {};
|
||||
const receipt = receiptResult.data;
|
||||
if (receipt.state !== "saved" || receipt.snapshotId !== authority.snapshotId
|
||||
|| receipt.workspaceId !== workspace.id || receipt.epoch !== authority.epoch
|
||||
|| receipt.leaseId !== authority.location.leaseId || receipt.providerLeaseId !== authority.location.providerLeaseId
|
||||
|| receipt.remoteCwd !== authority.location.remoteCwd) return {};
|
||||
|| receipt.workspaceId !== workspace.id || receipt.epoch !== authority.epoch) return {};
|
||||
if (authority.location && (receipt.leaseId !== authority.location.leaseId
|
||||
|| receipt.providerLeaseId !== authority.location.providerLeaseId || receipt.remoteCwd !== authority.location.remoteCwd)) return {};
|
||||
if (expectedSource === "saved") {
|
||||
const leases = await input.api.get<Array<{ id: string; companyId: string; issueId: string; providerLeaseId: string }>>(
|
||||
`/api/environments/${input.environmentId}/leases`);
|
||||
if (!leases.some(lease => lease.id === receipt.leaseId && lease.companyId === input.companyId
|
||||
&& lease.issueId === input.issueId && lease.providerLeaseId === receipt.providerLeaseId)) return {};
|
||||
}
|
||||
const file = await input.api.get<WorkspaceFileContent>(`/api/issues/${input.issueId}/file-resources/content?workspace=execution&path=${REMOTE_WORKSPACE_PROOF_FILE}`);
|
||||
if (file.content?.encoding !== "utf8" || typeof file.content.data !== "string") throw new Error("Remote workspace file oracle did not return text bytes");
|
||||
if (file.resource?.source?.kind !== "live" || file.resource.workspaceKind !== "execution_workspace"
|
||||
if (file.resource?.source?.kind !== expectedSource || file.resource.workspaceKind !== "execution_workspace"
|
||||
|| file.resource.workspaceId !== workspace.id) return {};
|
||||
const fileSnapshotId = file.resource.source.kind === "saved" ? file.resource.source.snapshotId ?? undefined : undefined;
|
||||
if (expectedSource === "saved" && fileSnapshotId !== authority.snapshotId) return {};
|
||||
return { content: file.content.data, point: {
|
||||
workspaceId: workspace.id, snapshotId: authority.snapshotId, runId: run.id,
|
||||
runSnapshotId: receipt.snapshotId, runSaveState: receipt.state,
|
||||
authoritySchema: authority.schema, authorityDirty: authority.dirty,
|
||||
authorityWriter: authority.writerRunId, providerType: workspace.providerType,
|
||||
proofSha256: remoteWorkspaceProofDigest(file.content.data), fileSource: file.resource?.source?.kind,
|
||||
proofSha256: remoteWorkspaceProofDigest(file.content.data), fileSource: file.resource?.source?.kind, fileSnapshotId,
|
||||
} };
|
||||
}
|
||||
|
||||
@@ -91,7 +104,9 @@ export function gradeRemoteWorkspaceStorage(evidence: RemoteWorkspaceStorageEvid
|
||||
&& point.snapshotId === point.runSnapshotId && point.runSaveState === "saved"
|
||||
&& point.authoritySchema === "paperclip.remote-workspace.v1" && point.authorityDirty === false
|
||||
&& point.authorityWriter === null && point.providerType === "cloud_sandbox"
|
||||
&& point.fileSource === "live" && /^[a-f0-9]{64}$/.test(point.proofSha256));
|
||||
&& point.fileSource === (evidence.expectedFileSource ?? "live")
|
||||
&& (point.fileSource !== "saved" || point.fileSnapshotId === point.snapshotId)
|
||||
&& /^[a-f0-9]{64}$/.test(point.proofSha256));
|
||||
return [
|
||||
{ id: "remote-workspace-initial-save", passed: committed(evidence.initial), detail: "The initial task admission commits a remote workspace snapshot and a matching saved run receipt." },
|
||||
{ id: "remote-workspace-final-save", passed: committed(evidence.final), detail: "The later admission commits its files before finalization succeeds." },
|
||||
|
||||
Reference in new issue
Block a user