diff --git a/packages/db/src/migrations/0211_famous_guardsmen.sql b/packages/db/src/migrations/0211_famous_guardsmen.sql index b07c4ad2961..8533d3712cd 100644 --- a/packages/db/src/migrations/0211_famous_guardsmen.sql +++ b/packages/db/src/migrations/0211_famous_guardsmen.sql @@ -28,7 +28,7 @@ CREATE TABLE "native_run_finalizations" ( "assessment_id" uuid, "decision_id" uuid, "failure_code" text, - "failure_detail" text, + "failure_detail" jsonb, "next_attempt_at" timestamp with time zone, "created_at" timestamp with time zone DEFAULT now() NOT NULL, "updated_at" timestamp with time zone DEFAULT now() NOT NULL @@ -76,14 +76,14 @@ CREATE TABLE "status_decisions" ( "company_id" uuid NOT NULL, "issue_id" uuid NOT NULL, "assessment_id" uuid NOT NULL, - "decision_version" integer NOT NULL, + "decision_version" bigint NOT NULL, "policy_version" text NOT NULL, "from_status" text NOT NULL, "to_status" text NOT NULL, "reason_code" text NOT NULL, "decision_json" jsonb NOT NULL, "decision_digest" text NOT NULL, - "application_state" text DEFAULT 'pending' NOT NULL, + "application_state" text DEFAULT 'proposed' NOT NULL, "supersedes_decision_id" uuid, "applied_at" timestamp with time zone, "created_at" timestamp with time zone DEFAULT now() NOT NULL @@ -133,16 +133,35 @@ ALTER TABLE "heartbeat_runs" ADD COLUMN "native_phase" text;--> statement-breakp ALTER TABLE "heartbeat_runs" ADD COLUMN "native_phase_updated_at" timestamp with time zone;--> statement-breakpoint ALTER TABLE "issues" ADD COLUMN "status_version" bigint DEFAULT 0 NOT NULL;--> statement-breakpoint ALTER TABLE "issues" ADD COLUMN "last_status_decision_id" uuid;--> statement-breakpoint --- Historical event writers computed max(seq) outside their insert transaction. Repair any --- collisions deterministically before enforcing the native runner's single-writer invariant. -WITH resequenced AS ( - SELECT "id", row_number() OVER (PARTITION BY "run_id" ORDER BY "created_at", "id") AS "new_seq" +-- Historical event writers computed max(seq) outside their insert transaction. Preserve every +-- existing unique cursor and the first row at each duplicated cursor. Only later colliding rows +-- move above the run's pre-migration maximum, in deterministic legacy order. +WITH ranked AS ( + SELECT + "id", + "run_id", + "seq", + "created_at", + row_number() OVER ( + PARTITION BY "run_id", "seq" + ORDER BY "created_at", "id" + ) AS "duplicate_ordinal", + max("seq") OVER (PARTITION BY "run_id") AS "old_run_max" FROM "heartbeat_run_events" +), duplicate_repairs AS ( + SELECT + "id", + "old_run_max" + row_number() OVER ( + PARTITION BY "run_id" + ORDER BY "seq", "created_at", "id" + ) AS "new_seq" + FROM ranked + WHERE "duplicate_ordinal" > 1 ) UPDATE "heartbeat_run_events" AS event -SET "seq" = resequenced."new_seq" -FROM resequenced -WHERE event."id" = resequenced."id";--> statement-breakpoint +SET "seq" = duplicate_repairs."new_seq" +FROM duplicate_repairs +WHERE event."id" = duplicate_repairs."id";--> statement-breakpoint UPDATE "heartbeat_runs" AS run SET "next_event_seq" = COALESCE(( SELECT max(event."seq") + 1 @@ -189,10 +208,11 @@ CREATE UNIQUE INDEX "native_run_results_run_fingerprint_uq" ON "native_run_resul CREATE UNIQUE INDEX "native_run_results_run_caller_result_uq" ON "native_run_results" USING btree ("run_id","caller_result_id");--> statement-breakpoint CREATE UNIQUE INDEX "native_run_results_run_caller_dedupe_uq" ON "native_run_results" USING btree ("run_id","caller_dedupe_key");--> statement-breakpoint CREATE UNIQUE INDEX "status_decision_effects_decision_ordinal_uq" ON "status_decision_effects" USING btree ("decision_id","ordinal");--> statement-breakpoint -CREATE UNIQUE INDEX "status_decision_effects_idempotency_uq" ON "status_decision_effects" USING btree ("idempotency_key");--> statement-breakpoint -CREATE UNIQUE INDEX "status_decisions_assessment_version_uq" ON "status_decisions" USING btree ("assessment_id","decision_version");--> statement-breakpoint -CREATE UNIQUE INDEX "status_decisions_issue_digest_uq" ON "status_decisions" USING btree ("issue_id","decision_digest");--> statement-breakpoint -CREATE UNIQUE INDEX "work_assessments_result_input_uq" ON "work_assessments" USING btree ("result_id","input_digest");--> statement-breakpoint +CREATE UNIQUE INDEX "status_decision_effects_company_idempotency_uq" ON "status_decision_effects" USING btree ("company_id","idempotency_key");--> statement-breakpoint +CREATE UNIQUE INDEX "status_decisions_company_issue_version_uq" ON "status_decisions" USING btree ("company_id","issue_id","decision_version");--> statement-breakpoint +CREATE UNIQUE INDEX "status_decisions_company_assessment_uq" ON "status_decisions" USING btree ("company_id","assessment_id");--> statement-breakpoint +CREATE UNIQUE INDEX "status_decisions_company_issue_digest_uq" ON "status_decisions" USING btree ("company_id","issue_id","decision_digest");--> statement-breakpoint +CREATE UNIQUE INDEX "work_assessments_company_issue_input_uq" ON "work_assessments" USING btree ("company_id","issue_id","input_digest");--> statement-breakpoint -- paperclip:migration-safety-ignore large-create-index-not-concurrently: duplicate rows are deterministically resequenced above before this invariant is enabled in the same transaction. CREATE UNIQUE INDEX "heartbeat_run_events_run_seq_uq" ON "heartbeat_run_events" USING btree ("run_id","seq");--> statement-breakpoint -- paperclip:migration-safety-ignore large-create-index-not-concurrently: nullable native source IDs mean historical rows need no backfill and the invariant must commit atomically with the new columns. diff --git a/packages/db/src/migrations/meta/0211_snapshot.json b/packages/db/src/migrations/meta/0211_snapshot.json index de954b9a31a..098db4f0250 100644 --- a/packages/db/src/migrations/meta/0211_snapshot.json +++ b/packages/db/src/migrations/meta/0211_snapshot.json @@ -24318,7 +24318,7 @@ }, "failure_detail": { "name": "failure_detail", - "type": "text", + "type": "jsonb", "primaryKey": false, "notNull": false }, @@ -32244,9 +32244,15 @@ "method": "btree", "with": {} }, - "status_decision_effects_idempotency_uq": { - "name": "status_decision_effects_idempotency_uq", + "status_decision_effects_company_idempotency_uq": { + "name": "status_decision_effects_company_idempotency_uq", "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, { "expression": "idempotency_key", "isExpression": false, @@ -32338,7 +32344,7 @@ }, "decision_version": { "name": "decision_version", - "type": "integer", + "type": "bigint", "primaryKey": false, "notNull": true }, @@ -32383,7 +32389,7 @@ "type": "text", "primaryKey": false, "notNull": true, - "default": "'pending'" + "default": "'proposed'" }, "supersedes_decision_id": { "name": "supersedes_decision_id", @@ -32406,11 +32412,17 @@ } }, "indexes": { - "status_decisions_assessment_version_uq": { - "name": "status_decisions_assessment_version_uq", + "status_decisions_company_issue_version_uq": { + "name": "status_decisions_company_issue_version_uq", "columns": [ { - "expression": "assessment_id", + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", "isExpression": false, "asc": true, "nulls": "last" @@ -32427,9 +32439,36 @@ "method": "btree", "with": {} }, - "status_decisions_issue_digest_uq": { - "name": "status_decisions_issue_digest_uq", + "status_decisions_company_assessment_uq": { + "name": "status_decisions_company_assessment_uq", "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "assessment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "status_decisions_company_issue_digest_uq": { + "name": "status_decisions_company_issue_digest_uq", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, { "expression": "issue_id", "isExpression": false, @@ -39258,11 +39297,17 @@ } }, "indexes": { - "work_assessments_result_input_uq": { - "name": "work_assessments_result_input_uq", + "work_assessments_company_issue_input_uq": { + "name": "work_assessments_company_issue_input_uq", "columns": [ { - "expression": "result_id", + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", "isExpression": false, "asc": true, "nulls": "last" @@ -40060,4 +40105,4 @@ "schemas": {}, "tables": {} } -} \ No newline at end of file +} diff --git a/packages/db/src/schema/native_run_finalizations.ts b/packages/db/src/schema/native_run_finalizations.ts index 1ec4b8ba69c..8985dfb5f85 100644 --- a/packages/db/src/schema/native_run_finalizations.ts +++ b/packages/db/src/schema/native_run_finalizations.ts @@ -1,4 +1,4 @@ -import { pgTable, uuid, text, timestamp, integer } from "drizzle-orm/pg-core"; +import { pgTable, uuid, text, timestamp, integer, jsonb } from "drizzle-orm/pg-core"; import { companies } from "./companies.js"; import { issues } from "./issues.js"; import { heartbeatRuns } from "./heartbeat_runs.js"; @@ -16,7 +16,7 @@ export const nativeRunFinalizations = pgTable("native_run_finalizations", { assessmentId: uuid("assessment_id"), decisionId: uuid("decision_id"), failureCode: text("failure_code"), - failureDetail: text("failure_detail"), + failureDetail: jsonb("failure_detail").$type>(), nextAttemptAt: timestamp("next_attempt_at", { withTimezone: true }), createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), diff --git a/packages/db/src/schema/status_decision_effects.ts b/packages/db/src/schema/status_decision_effects.ts index 22187fc9531..b8dd68df2fb 100644 --- a/packages/db/src/schema/status_decision_effects.ts +++ b/packages/db/src/schema/status_decision_effects.ts @@ -23,5 +23,6 @@ export const statusDecisionEffects = pgTable("status_decision_effects", { updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), }, (table) => ({ decisionOrdinalUq: uniqueIndex("status_decision_effects_decision_ordinal_uq").on(table.decisionId, table.ordinal), - idempotencyUq: uniqueIndex("status_decision_effects_idempotency_uq").on(table.idempotencyKey), + idempotencyUq: uniqueIndex("status_decision_effects_company_idempotency_uq") + .on(table.companyId, table.idempotencyKey), })); diff --git a/packages/db/src/schema/status_decisions.ts b/packages/db/src/schema/status_decisions.ts index ee94ff8cf75..e94d3ea36b2 100644 --- a/packages/db/src/schema/status_decisions.ts +++ b/packages/db/src/schema/status_decisions.ts @@ -1,4 +1,4 @@ -import { pgTable, uuid, text, timestamp, integer, jsonb, uniqueIndex } from "drizzle-orm/pg-core"; +import { pgTable, uuid, text, timestamp, bigint, jsonb, uniqueIndex } from "drizzle-orm/pg-core"; import { companies } from "./companies.js"; import { issues } from "./issues.js"; import { workAssessments } from "./work_assessments.js"; @@ -8,18 +8,21 @@ export const statusDecisions = pgTable("status_decisions", { companyId: uuid("company_id").notNull().references(() => companies.id), issueId: uuid("issue_id").notNull().references(() => issues.id), assessmentId: uuid("assessment_id").notNull().references(() => workAssessments.id), - decisionVersion: integer("decision_version").notNull(), + decisionVersion: bigint("decision_version", { mode: "number" }).notNull(), policyVersion: text("policy_version").notNull(), fromStatus: text("from_status").notNull(), toStatus: text("to_status").notNull(), reasonCode: text("reason_code").notNull(), decisionJson: jsonb("decision_json").$type>().notNull(), decisionDigest: text("decision_digest").notNull(), - applicationState: text("application_state").notNull().default("pending"), + applicationState: text("application_state").notNull().default("proposed"), supersedesDecisionId: uuid("supersedes_decision_id"), appliedAt: timestamp("applied_at", { withTimezone: true }), createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), }, (table) => ({ - assessmentVersionUq: uniqueIndex("status_decisions_assessment_version_uq").on(table.assessmentId, table.decisionVersion), - issueDigestUq: uniqueIndex("status_decisions_issue_digest_uq").on(table.issueId, table.decisionDigest), + issueVersionUq: uniqueIndex("status_decisions_company_issue_version_uq") + .on(table.companyId, table.issueId, table.decisionVersion), + assessmentUq: uniqueIndex("status_decisions_company_assessment_uq").on(table.companyId, table.assessmentId), + issueDigestUq: uniqueIndex("status_decisions_company_issue_digest_uq") + .on(table.companyId, table.issueId, table.decisionDigest), })); diff --git a/packages/db/src/schema/work_assessments.ts b/packages/db/src/schema/work_assessments.ts index ec766b30fb2..5c6986dfb22 100644 --- a/packages/db/src/schema/work_assessments.ts +++ b/packages/db/src/schema/work_assessments.ts @@ -26,5 +26,6 @@ export const workAssessments = pgTable("work_assessments", { supersedesAssessmentId: uuid("supersedes_assessment_id"), createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), }, (table) => ({ - resultInputUq: uniqueIndex("work_assessments_result_input_uq").on(table.resultId, table.inputDigest), + issueInputUq: uniqueIndex("work_assessments_company_issue_input_uq") + .on(table.companyId, table.issueId, table.inputDigest), })); diff --git a/packages/paperclip-runner/docs/design/phase-6-thin-paperclip-adapter.md b/packages/paperclip-runner/docs/design/phase-6-thin-paperclip-adapter.md index c7d08613d24..8a73b397d1d 100644 --- a/packages/paperclip-runner/docs/design/phase-6-thin-paperclip-adapter.md +++ b/packages/paperclip-runner/docs/design/phase-6-thin-paperclip-adapter.md @@ -695,6 +695,29 @@ allowlist amendment and CTO review. In particular, Phase 6 must not change a concrete legacy adapter, approval authority, browser UI, workspace policy, budget policy, or activity semantics merely to make the tracer pass. +### Remediation allowlist amendment (2026-08-09) + +The implementation review found four seam-local paths whose names differed +from the provisional list above. They are added without expanding authority: + +```text +packages/paperclip-runner/src/native-session-runtime.ts +packages/paperclip-runner/src/native-session-runtime.test.ts +server/src/services/native-runtime/native-session-executor.ts +server/src/services/native-runtime/native-session-executor.test.ts +server/src/services/native-runtime/paperclip-control-plane-port.test.ts +server/src/services/native-runtime/status-arbiter.test.ts +packages/paperclip-runner/docs/design/phase-6-thin-paperclip-adapter.md +``` + +The package runtime files own persisted provider-session recovery. The server +executor files own only the run-scoped cancellation handle and coordinator +lease around that package runtime. The colocated port and arbiter tests exercise +the same approved database and pure-policy seams exposed through the named +`server/src/__tests__` matrix entry points. This amendment does not authorize +changes to a concrete legacy adapter, approval authority, UI, workspace policy, +budget policy, or runner/provider/session behavior outside the package. + ## Commands the implementation must make runnable These commands are the acceptance contract for the implementation issue. They diff --git a/packages/paperclip-runner/docs/phase-06-thin-paperclip-adapter.md b/packages/paperclip-runner/docs/phase-06-thin-paperclip-adapter.md index 8f07d6d28a4..5d9eed2df22 100644 --- a/packages/paperclip-runner/docs/phase-06-thin-paperclip-adapter.md +++ b/packages/paperclip-runner/docs/phase-06-thin-paperclip-adapter.md @@ -32,6 +32,8 @@ contract are persisted before provider execution. `PaperclipControlPlanePort` implements the package's `ControlPlanePort`: - `openRun` verifies company/run/issue/agent/contract binding; +- `loadSessionCheckpoint` and `checkpointSession` bind restart state to the + same company/run/issue/agent/session identity; - `appendEvent` validates PRP, commits through the shared per-run allocator, deduplicates source identity by canonical digest, and acknowledges afterward; - `replayEvents` uses an exclusive source cursor; @@ -51,21 +53,41 @@ versions, native run metadata, and native source identity on heartbeat events. All event writers allocate from the row-locked `heartbeat_runs.next_event_seq`. Finalization runs only after result persistence and workspace finalization. The -pure arbiter marks `done` only when the terminal state succeeded, the objective -and every criterion are satisfied, verification passed, and no blocking work -remains. It marks `blocked` only for a first-class reported blocker. A failed -workspace barrier preserves the immutable native result, fails the run, and -leaves issue status unchanged. Review/yield/incomplete/failure/cancellation -remain non-terminal. -Status projection is a versioned CAS and emits an audit record. +server verifies cited evidence against company- and issue-scoped durable +events, work products, approvals, interactions, or attachments; model claims +alone cannot complete an issue. The pure arbiter marks `done` only when the +terminal state succeeded, every criterion and verification has accepted +durable evidence, and no blocking work remains. It marks `blocked` only for a +task-wide first-class blocker with a named owner and action. Review, retry, +continuation, blocker, and recovery effects materialize atomically with the +status-version CAS and audit record. ## Recovery and safety The finalization reconciler selects persisted `runtime_mode=native` rows; it -does not consult the current flag. Cancellation uses a run-scoped normalized -session handle before the existing process cleanup. Native execution does not -construct a Paperclip JWT, managed MCP access, legacy context, or raw adapter -environment. +does not consult the current flag. Per-attempt leases prevent concurrent +session/finalization ownership, while retryable failures retain the immutable +result and a first-class recovery action. A recovered session replays its +checkpoint and appends only missing control-plane facts instead of opening a +second provider session. Cancellation uses a run-scoped normalized session +handle before the existing process cleanup. Native execution does not construct +a Paperclip JWT, managed MCP access, legacy context, or raw adapter environment. + +## Verification status + +The deterministic package suite and the real embedded-PostgreSQL Paperclip +suite prove selection, replay/conflict handling, authoritative evidence, +status/liveness atomicity, session/finalization recovery, migration repair, +bounded cancellation/retry, and a byte-equivalent legacy read snapshot. The +database suite is the internal native canary: it executes one selected task +through the public package session contract and applies one server-owned +decision. Its post-kill-switch legacy case has zero native history rows. + +No new live provider task was dispatched during the remediation review. A live +Codex canary remains an operator-run rollout check because it requires changing +the instance flag and one agent profile. The package `trace:phase6` command is a +mock contract tracer; real Paperclip proof is the database-backed server matrix +and the explicitly gated procedure in the tutorial. See the [runnable tutorial](tutorials/phase-06-thin-paperclip-adapter.md), the [verification record](../knowledge/evidence/2026-08-09-phase-06-verification.md), diff --git a/packages/paperclip-runner/docs/tutorials/phase-06-thin-paperclip-adapter.md b/packages/paperclip-runner/docs/tutorials/phase-06-thin-paperclip-adapter.md index 7c980f17213..904f04f0045 100644 --- a/packages/paperclip-runner/docs/tutorials/phase-06-thin-paperclip-adapter.md +++ b/packages/paperclip-runner/docs/tutorials/phase-06-thin-paperclip-adapter.md @@ -29,29 +29,45 @@ PHASE6_SCRATCH="${PAPERCLIP_RUN_SCRATCH_DIR:-$(mktemp -d)}" pnpm --filter @paperclipai/paperclip-runner exec vitest run \ src/conformance/control-plane-port.test.ts \ src/backends/harness-driver-backend.test.ts \ - src/contracts/native-execution.test.ts + src/contracts/native-execution.test.ts \ + src/native-session-runtime.test.ts pnpm --filter @paperclipai/paperclip-runner trace:phase6 -- \ --target mock --scenario happy-path ``` -Expected: five targeted tests pass and the trace reports `resolvedMode` as +Expected: six targeted tests pass and the trace reports `resolvedMode` as `native`, three replay-stable events, a successful workspace barrier, and one -server-owned `done` decision. +mock server-owned `done` decision. This tracer does not connect to Paperclip. ## 2. Prove the database-backed Paperclip port ```sh -pnpm --filter @paperclipai/paperclip-runner build:typescript pnpm --filter @paperclipai/server exec vitest run \ - src/services/native-runtime/paperclip-control-plane-port.test.ts \ - src/services/native-runtime/runtime-mode.test.ts \ - src/services/native-runtime/status-arbiter.test.ts + src/__tests__/native-runner-phase6.integration.test.ts \ + src/__tests__/heartbeat-native-runner-selection.test.ts \ + src/__tests__/heartbeat-native-runner-cancellation.test.ts \ + src/__tests__/heartbeat-run-event-sequencing.test.ts \ + src/__tests__/native-runner-input-boundary.test.ts \ + src/__tests__/native-run-finalizer.test.ts \ + src/__tests__/native-status-arbiter-corpus.test.ts \ + src/__tests__/native-finalization-recovery.test.ts \ + src/__tests__/native-finalization-migration.test.ts \ + src/__tests__/legacy-finalization-regression.test.ts ``` This runs the unchanged package conformance suite against the real Paperclip -port. It also checks company binding, duplicate/gap replay, immutable result -ingestion, workspace-gated finalization, status-version CAS, and audit output. +port. Twenty-five targeted tests check company binding, duplicate/gap replay, +checkpoint recovery, immutable result ingestion, durable-evidence authority, +workspace-gated finalization, atomic liveness rollback, status-version CAS, +migration repair, legacy byte equivalence, and audit output. All database tests +start embedded PostgreSQL and do not skip when a developer database URL is +absent. + +The command above is the repeatable internal canary and post-kill-switch legacy +proof. The remaining sections are an optional live-provider rollout procedure; +they were not run during remediation and require explicit operator authority to +change an instance flag and agent profile. ## 3. Enable one isolated agent diff --git a/packages/paperclip-runner/knowledge/evidence/2026-08-09-phase-06-verification.md b/packages/paperclip-runner/knowledge/evidence/2026-08-09-phase-06-verification.md index 2145cb5866f..7140657aa0c 100644 --- a/packages/paperclip-runner/knowledge/evidence/2026-08-09-phase-06-verification.md +++ b/packages/paperclip-runner/knowledge/evidence/2026-08-09-phase-06-verification.md @@ -13,39 +13,44 @@ This record verifies the default-off Phase 6 tracer at the approved `ControlPlanePort`/`NativeSessionBackend` boundary. No browser surface changed, so Phase 6 has command/database evidence rather than screenshots. +The counts below are the PAP-16875 remediation rerun. “Internal canary” means +the selected-task database test through the public package session contract; +it does not mean that a new live Codex provider task was dispatched. + # Package contract and mock tracer ```sh pnpm --filter @paperclipai/paperclip-runner exec vitest run \ src/conformance/control-plane-port.test.ts \ src/backends/harness-driver-backend.test.ts \ - src/contracts/native-execution.test.ts + src/contracts/native-execution.test.ts \ + src/native-session-runtime.test.ts pnpm --filter @paperclipai/paperclip-runner trace:phase6 -- \ --target mock --scenario happy-path ``` -Result: three files and five tests passed. The stable trace reported native +Result: four files and six tests passed. The stable trace reported native mode, `runStatus=succeeded`, three events, contiguous source sequence 3, `workspaceFinalizeStatus=succeeded`, zero legacy invocations, and a server-owned -`done` decision with status version 0 -> 1. +`done` decision with status version 0 -> 1. Restart recovery reused the bound +provider session checkpoint, opened no second session or turn, retained the +existing accepted result event, and appended only the missing terminal fact. # Real Paperclip adapter and public-session task -```sh -pnpm --filter @paperclipai/paperclip-runner build:typescript -pnpm --filter @paperclipai/server exec vitest run \ - src/services/native-runtime/paperclip-control-plane-port.test.ts \ - src/services/native-runtime/runtime-mode.test.ts \ - src/services/native-runtime/status-arbiter.test.ts -``` - -Result: three files and ten tests passed against embedded PostgreSQL. The same -package conformance suite passed unchanged against `PaperclipControlPlanePort`. +The complete acceptance gate below passed 25 tests in ten files against +embedded PostgreSQL with zero skipped tests. Database-backed tests start their +own database instead of skipping when a developer `DATABASE_URL` is absent. +The same package conformance suite passed unchanged against +`PaperclipControlPlanePort`. A selected task then ran end-to-end through `executeNativeSession`, the public backend/port contracts, durable PRP events, immutable result ingestion, workspace finalization, assessment, CAS status decision, and audit persistence. -The issue moved from `in_progress` version 0 to `done` version 1. +The issue moved from `in_progress` version 0 to `done` version 1 only after its +criterion and verification cited an approved, issue-scoped durable work +product. A model-only result remained non-terminal and received a persisted +continuation wake. The focused corpus also proved: @@ -55,12 +60,37 @@ The focused corpus also proved: - company/run/agent/issue/source bindings reject mismatches; - source gaps remain visible, identical retries deduplicate, and replay is exclusive and byte-stable; -- incomplete/review/yield/cancelled results remain non-terminal; -- only a first-class reported blocker selects `blocked`; +- incomplete/review/yield/cancelled results receive a durable review, + continuation, or recovery path; +- only a task-wide blocker with a named owner and action selects `blocked`; - a completion decision writes immutable assessment/decision/effect records and - an activity-log audit row. + an activity-log audit row; - a failed workspace barrier preserves the accepted semantic result, leaves the - issue status unchanged, and records a recoverable `workspace_failed` phase. + issue status unchanged, and records a leased `retryable_failure` plus recovery + owner/action; +- four injected failures prove interactions, wakes, blocker bindings, status, + decisions, and effects roll back together before retry ownership is recorded. + +The acceptance-matrix entry points also passed as one 10-file, 25-test gate: + +```sh +pnpm --filter @paperclipai/server exec vitest run \ + src/__tests__/heartbeat-native-runner-cancellation.test.ts \ + src/__tests__/heartbeat-native-runner-selection.test.ts \ + src/__tests__/native-run-finalizer.test.ts \ + src/__tests__/native-runner-input-boundary.test.ts \ + src/__tests__/native-runner-phase6.integration.test.ts \ + src/__tests__/native-finalization-recovery.test.ts \ + src/__tests__/heartbeat-run-event-sequencing.test.ts \ + src/__tests__/legacy-finalization-regression.test.ts \ + src/__tests__/native-finalization-migration.test.ts \ + src/__tests__/native-status-arbiter-corpus.test.ts +``` + +This gate proves concurrent event allocation, duplicate-only migration repair, +bounded retry exhaustion, kill-switch legacy byte compatibility, and complete +Section 18.13 fixture-to-consumer traceability in addition to the direct suites. +It is the repeatable internal native canary and post-kill-switch legacy proof. # Compile and migration checks @@ -72,22 +102,29 @@ pnpm --filter @paperclipai/db typecheck ``` Result: all checks passed. Migration numbering and safety passed. Migration -0211 deterministically resequences historical heartbeat events before adding -the unique run sequence invariant, backfills each run's next allocator value, -and installs issue status-version tracking. +0211 preserves existing unique heartbeat cursors, deterministically moves only +duplicate rows above each run's former maximum, backfills the next allocator +value, and installs issue status-version tracking. -Focused legacy regression: +Focused migration, allocator, recovery, and legacy rehearsal: ```sh pnpm --filter @paperclipai/server exec vitest run \ - src/__tests__/heartbeat-workspace-finalize-branch.test.ts \ - src/__tests__/heartbeat-process-recovery.test.ts \ - src/__tests__/heartbeat-runtime-state.test.ts + src/services/native-runtime/paperclip-control-plane-port.test.ts \ + src/__tests__/native-finalization-recovery.test.ts \ + src/__tests__/legacy-finalization-regression.test.ts \ + src/__tests__/native-finalization-migration.test.ts \ + src/__tests__/heartbeat-run-event-sequencing.test.ts ``` -Result: three files and 106 tests passed, covering unchanged adapter execution, -workspace finalization, cancellation, process recovery, and runtime/session -state behavior with the flag left at its default. +Result: five files and 11 tests passed with zero skips. The production-shaped +0211 rehearsal began with legacy event sequences `[1,5,5,9]`, retained the +original unique cursors and first `5`, moved only the later duplicate to `10`, +seeded `next_event_seq=11`, preserved every non-sequence field byte-for-byte, +and enforced uniqueness. Thirty-two concurrent mixed event writers produced a +gap-free allocator stream. Invalid native finalization exhausted its +three-attempt budget into named recovery without touching issue status. The +legacy snapshot remained byte-equivalent with zero native history rows. # Credential, governance, budget, and legacy boundaries @@ -113,14 +150,16 @@ pnpm --filter @paperclipai/server exec vitest run \ -t "daily cost cap" ``` -Result: native cancellation reached the active normalized session and removed -the handle afterward; the daily cost hard stop cancelled a queued run before -any adapter execution. +Result: native cancellation reached the active normalized session exactly once +across duplicate cancellation requests and removed the handle afterward. The +daily cost hard stop cancelled a queued run before any adapter execution. -# Remaining operational checkpoint +# Deferred live-provider checkpoint The [Phase 6 tutorial](../../docs/tutorials/phase-06-thin-paperclip-adapter.md) contains the exact board-authorized commands for enabling one local agent, running and inspecting a live Codex task, disabling the flag, proving a fresh legacy selection, and restoring the agent profile. Those commands intentionally -do not pass credentials to the package or model. +do not pass credentials to the package or model. They were not run during this +remediation because changing live instance and agent rollout state requires +operator approval; no new live-provider evidence is claimed here. diff --git a/packages/paperclip-runner/knowledge/evidence/index.md b/packages/paperclip-runner/knowledge/evidence/index.md index 39062ab3443..c8185fb0c08 100644 --- a/packages/paperclip-runner/knowledge/evidence/index.md +++ b/packages/paperclip-runner/knowledge/evidence/index.md @@ -12,5 +12,4 @@ * [Phase 4b live console verification](2026-08-08-phase-04b-live-console-verification.md) - Demo-driver, transcript-model, browser, keyboard, boundary, token, and screenshot evidence for the live console. * [Phase 4b QA verification](2026-08-08-phase-04b-qa-verification.md) - Independent QA pass: clean-start tutorial in a real browser, deterministic demos, real Codex end-to-end session, full `verify` acceptance (exit 0), and screenshots (PAP-16837). * [Phase 5 browser SDK verification](2026-08-08-phase-05-verification.md) - Targeted and package-acceptance evidence for the versioned SDK, both public consumers, fake/real drivers, accessibility, reconnect, replay, and screenshots. -* [Phase 6 thin Paperclip adapter verification](2026-08-09-phase-06-verification.md) - Mock and database-backed port conformance, public-session task completion, durable replay/finalization, feature selection, security boundary, and legacy fallback evidence. -* [Phase 6 thin Paperclip adapter verification](2026-08-09-phase-06-verification.md) - Mock/real conformance, public session completion, canonical persistence, governance authority, migration, kill-switch, and legacy compatibility evidence. +* [Phase 6 thin Paperclip adapter verification](2026-08-09-phase-06-verification.md) - Mock and database-backed conformance, authoritative completion, restart recovery, migration repair, atomic liveness, internal canary, and deterministic post-kill-switch legacy evidence. diff --git a/packages/paperclip-runner/knowledge/journal/2026-08-09-phase-06-implementation.md b/packages/paperclip-runner/knowledge/journal/2026-08-09-phase-06-implementation.md index 7845a9da52d..f2dfd225cdd 100644 --- a/packages/paperclip-runner/knowledge/journal/2026-08-09-phase-06-implementation.md +++ b/packages/paperclip-runner/knowledge/journal/2026-08-09-phase-06-implementation.md @@ -34,16 +34,27 @@ existing server authority and keep legacy execution as the default. handle; never fall back to legacy inside a selected native run. 8. Keep the feature default false. Recovery reads persisted run/coordinator state and therefore remains correct after the flag is disabled. +9. Persist provider session checkpoints and recover them fail-closed; a restart + may reconcile missing control facts but may not open a second provider + session for the same run. +10. Treat model citations as claims. Only server-verifiable durable records can + satisfy completion criteria, and every non-terminal decision must create or + bind a real review, wake, blocker, or recovery path in the status transaction. # Evidence - [Phase 6 verification](../evidence/2026-08-09-phase-06-verification.md) - [Phase 6 reference](../../docs/phase-06-thin-paperclip-adapter.md) - [Runnable tutorial](../../docs/tutorials/phase-06-thin-paperclip-adapter.md) -- Package conformance: three files, five tests passed. -- Database-backed port/session/finalizer corpus: three files, ten tests passed. -- Legacy heartbeat finalization/recovery/runtime corpus: three files, 106 tests - passed. +- Package conformance and recovery: four files, six tests passed. +- Phase 6 acceptance-matrix entry points: ten files, twenty-five tests passed + with zero skips, including the database-backed selected-task canary, four + atomic-liveness failpoints, migration, sequencing, bounded recovery, and + legacy compatibility. +- Focused database rehearsal: five files, eleven tests passed with zero skips; + the legacy event/read snapshot was byte-equivalent and had zero native rows. +- Section 18.13 source corpus: 52 fixtures, all 70 matrix rows, and 18 structural + checks passed with every fixture bound to a named consumer. - Runner, shared, server, and database typechecks passed; migration safety passed. @@ -69,6 +80,10 @@ existing server authority and keep legacy execution as the default. - Live operator proof depends on a board-authorized local instance and an already authenticated Codex installation; deterministic CI uses the same public session contract with a scripted backend. +- The remediation did not dispatch a new live provider task or mutate rollout + settings. The verified “internal canary” and post-kill-switch trace are the + real embedded-PostgreSQL selected-task and legacy-snapshot cases; the live + tutorial remains an explicit operator checkpoint. # Follow-up questions diff --git a/packages/paperclip-runner/spec/fixtures/status-authority-phase5.json b/packages/paperclip-runner/spec/fixtures/status-authority-phase5.json index aa1eaeb7c6a..e6fffd30a04 100644 --- a/packages/paperclip-runner/spec/fixtures/status-authority-phase5.json +++ b/packages/paperclip-runner/spec/fixtures/status-authority-phase5.json @@ -33,7 +33,7 @@ { "id": "incomplete-evidence-no-live-path", "mode": "native", - "covers": { "decisionRows": ["SD-03"], "terminalRows": [], "attentionRows": [], "livenessRows": ["LIVE-06"], "reconciliationRows": [], "compatibilityRows": [], "migrationRows": [] }, + "covers": { "decisionRows": [], "terminalRows": [], "attentionRows": [], "livenessRows": [], "reconciliationRows": [], "compatibilityRows": [], "migrationRows": [] }, "tags": ["incomplete_evidence", "atomic_liveness"], "given": { "priorIssueStatus": "in_progress", "turnTerminalState": "completed", "runTerminalState": "succeeded", "reportedWorkDisposition": "done", "nativeFinalization": "present", "completionState": "missing_required_test", "trigger": "runner_finalizer", "fault": "continuation_insert_failure" }, "expected": { "runStatus": "failed", "statusAction": "preserve", "reasonCode": "side_effect_planning_failed", "requiredEffects": ["record_finalization_error"], "forbiddenEffects": ["enqueue_continuation", "status_version_increment"], "livePathKind": null, "preserveClaim": true, "nativeRecords": true, "decisionCount": 0, "maxWakeCount": 0, "maxNotificationCount": 0 }, @@ -42,7 +42,7 @@ { "id": "prior-status-preserved-without-live-path", "mode": "native", - "covers": { "decisionRows": [], "terminalRows": [], "attentionRows": [], "livenessRows": [], "reconciliationRows": [], "compatibilityRows": [], "migrationRows": [] }, + "covers": { "decisionRows": ["SD-03"], "terminalRows": [], "attentionRows": [], "livenessRows": ["LIVE-06"], "reconciliationRows": [], "compatibilityRows": [], "migrationRows": [] }, "tags": ["atomic_liveness", "reconciliation"], "given": { "priorIssueStatus": "in_progress", "turnTerminalState": "completed", "runTerminalState": "succeeded", "reportedWorkDisposition": "yielded", "nativeFinalization": "present", "completionState": "no_durable_continuation", "trigger": "runner_finalizer" }, "expected": { "runStatus": "failed", "statusAction": "preserve", "reasonCode": "prior_status_preserved_no_live_path", "requiredEffects": ["record_finalization_error"], "forbiddenEffects": ["set_issue_status", "enqueue_continuation", "bind_blocker", "bind_reviewer"], "livePathKind": "recovery", "preserveClaim": true, "nativeRecords": true, "decisionCount": 1, "maxWakeCount": 1, "maxNotificationCount": 0 }, diff --git a/packages/paperclip-runner/src/backends/harness-driver-backend.ts b/packages/paperclip-runner/src/backends/harness-driver-backend.ts index 29f0c9c53ae..d251034aaf8 100644 --- a/packages/paperclip-runner/src/backends/harness-driver-backend.ts +++ b/packages/paperclip-runner/src/backends/harness-driver-backend.ts @@ -56,6 +56,7 @@ export class HarnessDriverBackend implements NativeSessionBackend { providerSessionId: snapshot.providerSessionId, runId: snapshot.identity.runId, normalizedSessionId: snapshot.identity.sessionId, + activeTurnId: snapshot.activeTurnId ?? snapshot.terminalTurns?.at(-1)?.turnId ?? null, lastSourceSequence: parseCursor(snapshot.cursor), ...(snapshot.semanticResult === undefined || snapshot.semanticResult === null ? {} @@ -63,9 +64,12 @@ export class HarnessDriverBackend implements NativeSessionBackend { semanticResult: { result: snapshot.semanticResult, fingerprint: canonicalJson(snapshot.semanticResult), - turnId: "recovered", - }, - }), + turnId: snapshot.activeTurnId ?? snapshot.terminalTurns?.at(-1)?.turnId ?? "recovered", + }, + }), + terminalTurns: snapshot.terminalTurns ?? [], + pendingRuntimeRequests: snapshot.pendingRuntimeRequests ?? [], + lineage: snapshot.lineage ?? [], }; const recovered = await this.#driver.recoverSession(persisted); if (!recovered.recovered || recovered.session === undefined) { @@ -181,6 +185,10 @@ class HarnessNativeSession implements NativeSession { : String(snapshot.lastSourceSequence), semanticResult: snapshot.semanticResult?.result ?? null, terminal: this.#terminal, + activeTurnId: snapshot.activeTurnId ?? snapshot.semanticResult?.turnId ?? null, + terminalTurns: snapshot.terminalTurns ?? [], + pendingRuntimeRequests: snapshot.pendingRuntimeRequests ?? [], + lineage: snapshot.lineage ?? [], }; } diff --git a/packages/paperclip-runner/src/conformance/control-plane-port.ts b/packages/paperclip-runner/src/conformance/control-plane-port.ts index 667ff4ab567..92f83ff5b31 100644 --- a/packages/paperclip-runner/src/conformance/control-plane-port.ts +++ b/packages/paperclip-runner/src/conformance/control-plane-port.ts @@ -41,7 +41,7 @@ export const CONTROL_PLANE_CONFORMANCE_RESULT: PrpStructuredRunResult = { remainingWork: [], }, evidence: [{ kind: "event", ref: "event:2" }], - verification: [{ commandOrCheck: "phase6-conformance", status: "passed" }], + verification: [{ commandOrCheck: "phase6-conformance", status: "passed", artifactRef: "event:2" }], attentionRequests: [], artifacts: [], }; diff --git a/packages/paperclip-runner/src/contracts/control-plane-port.ts b/packages/paperclip-runner/src/contracts/control-plane-port.ts index ff981481f63..77410919466 100644 --- a/packages/paperclip-runner/src/contracts/control-plane-port.ts +++ b/packages/paperclip-runner/src/contracts/control-plane-port.ts @@ -8,6 +8,7 @@ import type { PrpStructuredRunResult, PrpTerminalState, } from "../protocol/phase1-contract.js"; +import type { PersistedNativeSession } from "./native-session-backend.js"; export interface OpenControlPlaneRunInput { identity: NativeRunIdentity; @@ -49,6 +50,8 @@ export interface CompleteControlPlaneRunInput { */ export interface ControlPlanePort { openRun(input: OpenControlPlaneRunInput): Promise; + loadSessionCheckpoint?(): Promise; + checkpointSession?(snapshot: PersistedNativeSession): Promise; appendEvent(event: NativeRunEvent | PrpEvent): Promise; replayEvents(input: ReplayControlPlaneEventsInput): Promise; completeRun(result: NativeRunResult | CompleteControlPlaneRunInput): Promise; diff --git a/packages/paperclip-runner/src/contracts/native-session-backend.ts b/packages/paperclip-runner/src/contracts/native-session-backend.ts index c4cb06c6bf7..aa95d3a742d 100644 --- a/packages/paperclip-runner/src/contracts/native-session-backend.ts +++ b/packages/paperclip-runner/src/contracts/native-session-backend.ts @@ -8,6 +8,11 @@ import type { PrpStructuredRunResult, PrpTerminalState, } from "../protocol/phase1-contract.js"; +import type { + HarnessRuntimeRequest, + HarnessThreadLineageEntry, + PersistedHarnessTurnTerminal, +} from "./harness-driver.js"; export interface NativeSessionBackendDescriptor { kind: "runner" | "remote" | "mock"; @@ -29,6 +34,10 @@ export interface PersistedNativeSession { cursor?: string | null; semanticResult?: PrpStructuredRunResult | null; terminal?: PrpTerminalState | null; + activeTurnId?: string | null; + terminalTurns?: PersistedHarnessTurnTerminal[]; + pendingRuntimeRequests?: HarnessRuntimeRequest[]; + lineage?: HarnessThreadLineageEntry[]; } export interface NativeSessionRecoveryResult { diff --git a/packages/paperclip-runner/src/native-session-runtime.test.ts b/packages/paperclip-runner/src/native-session-runtime.test.ts new file mode 100644 index 00000000000..42f7165a42a --- /dev/null +++ b/packages/paperclip-runner/src/native-session-runtime.test.ts @@ -0,0 +1,185 @@ +import { describe, expect, it, vi } from "vitest"; + +import type { ControlPlanePort } from "./contracts/control-plane-port.js"; +import type { NativeExecutionInputV1 } from "./contracts/native-execution.js"; +import type { + NativeSession, + NativeSessionBackend, + PersistedNativeSession, +} from "./contracts/native-session-backend.js"; +import type { + PrpEvent, + PrpStructuredRunResult, + PrpTerminalState, +} from "./protocol/phase1-contract.js"; +import { executeNativeSession } from "./native-session-runtime.js"; + +const identity = { + runId: "run-recovery", + sessionId: "session-recovery", + companyId: "company-recovery", + issueId: "issue-recovery", + agentId: "agent-recovery", +}; + +const result: PrpStructuredRunResult = { + schema: "paperclip.run_result.v1", + reportedWorkDisposition: "done", + summary: "Recovered native work completed.", + completionClaim: { + contractRevision: "1", + objectiveSatisfied: true, + criteria: [{ criterionId: "objective", status: "satisfied", evidenceRefs: [] }], + remainingWork: [], + }, + evidence: [], + verification: [{ commandOrCheck: "recovery", status: "passed" }], + attentionRequests: [], + artifacts: [], +}; + +const terminal: PrpTerminalState = { + schema: "paperclip.prp.terminal.v1", + turnTerminalState: "completed", + runTerminalState: "succeeded", + reportedWorkDisposition: "done", +}; + +const input: NativeExecutionInputV1 = { + schema: "paperclip.native-execution-input.v1", + binding: { + companyId: identity.companyId, + runId: identity.runId, + issueId: identity.issueId, + agentId: identity.agentId, + executionWorkspaceId: "workspace-recovery", + }, + task: { identifier: "PAP-RECOVERY", title: "Recover native work", description: null, workMode: "standard" }, + workspace: { cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null }, + session: { normalizedSessionId: identity.sessionId, driverKind: "codex_app_server", protocolVersion: 1 }, + completionContract: { + id: "contract-recovery", + sha256: "contract-recovery-sha", + schemaVersion: "paperclip.completion-contract.v1", + contract: { + revision: "1", + objective: "Recover native work", + criteria: [{ id: "objective", requirement: "Complete after recovery" }], + }, + }, + interactionResponses: [], + credentialBindings: [], +}; + +function controlEvent( + sourceSeq: number, + eventType: PrpEvent["eventType"], + payload: Record, +): PrpEvent { + return { + schema: "paperclip.prp.event.v1", + sourceEventId: `control-recovery:${identity.runId}:${sourceSeq}`, + sourceSeq, + sourceInstanceId: "control-recovery", + sourceKind: "control_plane", + runId: identity.runId, + normalizedSessionId: identity.sessionId, + turnId: "turn-recovery", + eventType, + schemaVersion: 1, + priority: 0, + emittedAt: "2026-08-09T00:00:00.000Z", + payload, + }; +} + +function highestContiguous(events: PrpEvent[]): number { + const sequences = new Set(events.map((event) => event.sourceSeq)); + let cursor = 0; + while (sequences.has(cursor + 1)) cursor += 1; + return cursor; +} + +describe("executeNativeSession recovery", () => { + it("recovers a completed checkpoint and appends only a missing control terminal fact", async () => { + const checkpoint: PersistedNativeSession = { + backendKind: "mock", + sessionId: "driver-recovery", + identity, + providerSessionId: "provider-recovery", + cursor: "4", + semanticResult: result, + terminal, + activeTurnId: "turn-recovery", + terminalTurns: [{ turnId: "turn-recovery", fingerprint: "terminal-fingerprint" }], + pendingRuntimeRequests: [], + lineage: [], + }; + const events = [controlEvent(1, "run.result.accepted", { result })]; + const checkpoints: PersistedNativeSession[] = []; + const completeRun = vi.fn(async () => undefined); + const startTurn = vi.fn(async () => ({ turnId: "unexpected-turn" })); + const openSession = vi.fn(async () => { + throw new Error("a recovered run must not open a second provider session"); + }); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + }, + async *events() {}, + startTurn, + async result() { return { result, terminal, turnId: "turn-recovery" }; }, + async snapshot() { return structuredClone(checkpoint); }, + async close() {}, + }; + const recoverSession = vi.fn(async () => ({ recovered: true, session })); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + openSession, + recoverSession, + }; + const port: ControlPlanePort = { + async openRun() {}, + async loadSessionCheckpoint() { return structuredClone(checkpoint); }, + async checkpointSession(snapshot) { checkpoints.push(structuredClone(snapshot)); }, + async appendEvent(event) { + events.push(structuredClone(event as PrpEvent)); + return { + cursor: events.length, + highestContiguousSourceSeq: highestContiguous(events), + disposition: "committed", + }; + }, + async replayEvents(replay) { + const replayed = events.filter((event) => event.sourceSeq > replay.afterSourceSeq); + return { events: structuredClone(replayed), highestContiguousSourceSeq: highestContiguous(events) }; + }, + completeRun, + }; + + const completed = await executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + }); + + expect(openSession).not.toHaveBeenCalled(); + expect(recoverSession).toHaveBeenCalledOnce(); + expect(startTurn).not.toHaveBeenCalled(); + expect(events.map((event) => event.eventType)).toEqual(["run.result.accepted", "run.terminal"]); + expect(events.map((event) => event.sourceSeq)).toEqual([1, 2]); + expect(completeRun).toHaveBeenCalledOnce(); + expect(completed).toMatchObject({ nativeEventCount: 1, highestContiguousSourceSeq: 2 }); + expect(checkpoints.at(-1)).toMatchObject({ semanticResult: result, terminal }); + }); +}); diff --git a/packages/paperclip-runner/src/native-session-runtime.ts b/packages/paperclip-runner/src/native-session-runtime.ts index c5c66ea3588..2c17061e50e 100644 --- a/packages/paperclip-runner/src/native-session-runtime.ts +++ b/packages/paperclip-runner/src/native-session-runtime.ts @@ -62,7 +62,10 @@ async function consumeTurn(session: NativeSession, controlPlane: ControlPlanePor export async function executeNativeSession(options: ExecuteNativeSessionOptions): Promise { const input = parseNativeExecutionInput(options.input); const descriptor = await options.backend.descriptor(); - const normalizedSessionId = input.session.normalizedSessionId ?? randomUUID(); + const persistedSession = await options.controlPlane.loadSessionCheckpoint?.() ?? null; + const normalizedSessionId = persistedSession?.identity.sessionId + ?? input.session.normalizedSessionId + ?? randomUUID(); await options.controlPlane.openRun({ identity: { runId: input.binding.runId, @@ -75,49 +78,134 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions) sourceInstanceId: options.runnerInstanceId, }); - const session = await options.backend.openSession({ - identity: { - runId: input.binding.runId, - sessionId: normalizedSessionId, - companyId: input.binding.companyId, - issueId: input.binding.issueId, - agentId: input.binding.agentId, - }, - workingDirectory: input.workspace.cwd, - }); + const identity = { + runId: input.binding.runId, + sessionId: normalizedSessionId, + companyId: input.binding.companyId, + issueId: input.binding.issueId, + agentId: input.binding.agentId, + }; + if ( + persistedSession + && ( + persistedSession.identity.runId !== identity.runId + || persistedSession.identity.companyId !== identity.companyId + || persistedSession.identity.issueId !== identity.issueId + || persistedSession.identity.agentId !== identity.agentId + || persistedSession.identity.sessionId !== identity.sessionId + ) + ) throw new Error("native_session_checkpoint_binding_mismatch"); + let recovered = false; + let session: NativeSession; + if (persistedSession) { + if (!options.backend.recoverSession) { + throw new Error("native_session_recovery_unavailable: refusing to open a second provider session"); + } + const recovery = await options.backend.recoverSession(persistedSession); + if (!recovery.recovered || !recovery.session) { + throw new Error(`native_session_recovery_failed: ${recovery.reason ?? "unknown"}`); + } + session = recovery.session; + recovered = true; + } else { + session = await options.backend.openSession({ + identity, + workingDirectory: input.workspace.cwd, + }); + } options.onSession?.(session); try { - const consuming = consumeTurn(session, options.controlPlane, options.timeoutMs ?? 900_000); - await session.startTurn({ - message: { role: "user", text: JSON.stringify(buildNativeModelEnvelope(input)) }, - }); - const consumed = await consuming; - const completed = await session.result(); + const checkpoint = async () => { + if (options.controlPlane.checkpointSession) { + await options.controlPlane.checkpointSession(await session.snapshot()); + } + }; + await checkpoint(); + + let consumed = { event: null as PrpEvent | null, eventCount: 0, highestContiguousSourceSeq: 0 }; + let completed = persistedSession?.semanticResult && persistedSession.terminal + ? { + result: persistedSession.semanticResult, + terminal: persistedSession.terminal, + turnId: persistedSession.activeTurnId ?? null, + } + : null; + if (!completed) { + const consuming = consumeTurn(session, options.controlPlane, options.timeoutMs ?? 900_000); + if (!recovered || !persistedSession?.activeTurnId) { + await session.startTurn({ + message: { role: "user", text: JSON.stringify(buildNativeModelEnvelope(input)) }, + }); + await checkpoint(); + } + const terminalEvent = await consuming; + consumed = terminalEvent; + completed = await session.result(); + await checkpoint(); + } if (completed === null) throw new Error("native_finalization_missing: session returned no semantic result"); - const terminal = terminalFromEvent(consumed.event, completed.result.reportedWorkDisposition); - let controlSeq = 0; - const controlEvent = (eventType: PrpEvent["eventType"], payload: Record): PrpEvent => ({ + let terminal: PrpTerminalState; + if (persistedSession?.semanticResult && persistedSession.terminal) { + terminal = persistedSession.terminal; + } else { + terminal = terminalFromEvent(consumed.event!, completed.result.reportedWorkDisposition); + } + const eventTurnId = completed.turnId + ?? persistedSession?.activeTurnId + ?? persistedSession?.terminalTurns?.at(-1)?.turnId + ?? consumed.event?.turnId; + const controlEvent = ( + sourceSeq: number, + eventType: PrpEvent["eventType"], + payload: Record, + ): PrpEvent => ({ schema: "paperclip.prp.event.v1", - sourceEventId: `${options.controlPlaneInstanceId}:${input.binding.runId}:${++controlSeq}`, - sourceSeq: controlSeq, + sourceEventId: `${options.controlPlaneInstanceId}:${input.binding.runId}:${sourceSeq}`, + sourceSeq, sourceInstanceId: options.controlPlaneInstanceId, sourceKind: "control_plane", runId: input.binding.runId, normalizedSessionId, - turnId: completed.turnId ?? consumed.event.turnId, + ...(eventTurnId ? { turnId: eventTurnId } : {}), eventType, schemaVersion: 1, priority: 0, emittedAt: new Date().toISOString(), payload, }); - for (const event of [ - controlEvent("run.result.accepted", { result: completed.result }), - controlEvent("run.terminal", terminal as unknown as Record), - ]) { + const expectedControlEvents = [ + controlEvent(1, "run.result.accepted", { result: completed.result }), + controlEvent(2, "run.terminal", terminal as unknown as Record), + ]; + const controlReplay = await options.controlPlane.replayEvents({ + runId: input.binding.runId, + sourceInstanceId: options.controlPlaneInstanceId, + afterSourceSeq: 0, + limit: 10, + }); + const replayBySequence = new Map(controlReplay.events.map((event) => [event.sourceSeq, event])); + for (const existing of controlReplay.events) { + const expected = expectedControlEvents[existing.sourceSeq - 1]; + if ( + expected === undefined + || existing.eventType !== expected.eventType + || canonicalJson(existing.payload) !== canonicalJson(expected.payload) + ) { + throw new Error(`native_control_event_replay_conflict:${existing.sourceSeq}`); + } + } + consumed.highestContiguousSourceSeq = Math.max( + consumed.highestContiguousSourceSeq, + controlReplay.highestContiguousSourceSeq, + ); + for (const event of expectedControlEvents) { + if (replayBySequence.has(event.sourceSeq)) continue; const receipt = await options.controlPlane.appendEvent(event); consumed.eventCount += receipt.disposition === "committed" ? 1 : 0; - consumed.highestContiguousSourceSeq = Math.max(consumed.highestContiguousSourceSeq, receipt.highestContiguousSourceSeq); + consumed.highestContiguousSourceSeq = Math.max( + consumed.highestContiguousSourceSeq, + receipt.highestContiguousSourceSeq, + ); } await options.controlPlane.completeRun({ result: completed.result, @@ -127,6 +215,11 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions) callerDedupeKey: `${input.binding.runId}:${input.completionContract.sha256}`, }); const snapshot = await session.snapshot(); + await options.controlPlane.checkpointSession?.({ + ...snapshot, + semanticResult: completed.result, + terminal, + }); return { result: completed.result, terminal, @@ -144,3 +237,15 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions) await session.close({ reason: "native session execution complete" }); } } + +function canonicalJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (typeof value === "object" && value !== null) { + const record = value as Record; + return `{${Object.keys(record) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`) + .join(",")}}`; + } + return JSON.stringify(value) ?? "undefined"; +} diff --git a/server/src/__tests__/heartbeat-native-runner-cancellation.test.ts b/server/src/__tests__/heartbeat-native-runner-cancellation.test.ts new file mode 100644 index 00000000000..cfd5bbfe7f0 --- /dev/null +++ b/server/src/__tests__/heartbeat-native-runner-cancellation.test.ts @@ -0,0 +1,2 @@ +// Acceptance-matrix entry point for budget/manual cancellation idempotency. +import "../services/native-runtime/native-session-executor.test.js"; diff --git a/server/src/__tests__/heartbeat-native-runner-selection.test.ts b/server/src/__tests__/heartbeat-native-runner-selection.test.ts new file mode 100644 index 00000000000..83247ec8b28 --- /dev/null +++ b/server/src/__tests__/heartbeat-native-runner-selection.test.ts @@ -0,0 +1,2 @@ +// Acceptance-matrix entry point for default-off, opt-in, and eligibility rules. +import "../services/native-runtime/runtime-mode.test.js"; diff --git a/server/src/__tests__/heartbeat-run-event-sequencing.test.ts b/server/src/__tests__/heartbeat-run-event-sequencing.test.ts new file mode 100644 index 00000000000..b451da59f65 --- /dev/null +++ b/server/src/__tests__/heartbeat-run-event-sequencing.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it } from "vitest"; +import { eq } from "drizzle-orm"; +import { + agents, + companies, + createDb, + heartbeatRunEvents, + heartbeatRuns, +} from "@paperclipai/db"; +import { appendHeartbeatRunEvent, HeartbeatRunEventConflictError } from "../services/heartbeat-run-events.js"; +import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +describe("P6-11..13 / P6-17 canonical event allocator", () => { + it("serializes concurrent writers and rejects conflicting replay without cursor drift", async () => { + const temporary = await startEmbeddedPostgresTestDatabase("paperclip-native-events-"); + const db = createDb(temporary.connectionString); + const companyId = "20000000-0000-4000-8000-000000000001"; + const agentId = "20000000-0000-4000-8000-000000000002"; + const runId = "20000000-0000-4000-8000-000000000003"; + try { + await db.insert(companies).values({ id: companyId, name: "Allocator fixture", issuePrefix: "SEQ" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Allocator agent" }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running" }); + const eventTypes = ["heartbeat.started", "run.cancel.requested", "item.completed", "stdout"]; + const writes = Array.from({ length: 32 }, (_, index) => appendHeartbeatRunEvent(db, { + companyId, + runId, + agentId, + eventType: eventTypes[index % eventTypes.length]!, + message: `event-${index + 1}`, + payload: { ordinal: index + 1 }, + nativeSource: { + sourceInstanceId: `writer-${index % 4}`, + sourceEventId: `source-event-${index + 1}`, + sourceSeq: Math.floor(index / 4) + 1, + protocolSchemaVersion: 1, + canonicalPayload: { ordinal: index + 1 }, + }, + })); + const receipts = await Promise.all(writes); + expect(receipts.every((entry) => entry.disposition === "committed")).toBe(true); + + const rows = await db.select().from(heartbeatRunEvents) + .where(eq(heartbeatRunEvents.runId, runId)).orderBy(heartbeatRunEvents.seq); + expect(rows.map((row) => row.seq)).toEqual(Array.from({ length: 32 }, (_, index) => index + 1)); + expect(new Set(rows.map((row) => row.sourceEventId)).size).toBe(32); + expect((await db.select({ nextEventSeq: heartbeatRuns.nextEventSeq }).from(heartbeatRuns) + .where(eq(heartbeatRuns.id, runId)))[0]?.nextEventSeq).toBe(33); + + const original = rows.find((row) => row.sourceEventId === "source-event-1")!; + await expect(appendHeartbeatRunEvent(db, { + companyId, + runId, + agentId, + eventType: original.eventType, + message: original.message, + payload: original.payload, + nativeSource: { + sourceInstanceId: original.sourceInstanceId!, + sourceEventId: original.sourceEventId!, + sourceSeq: original.sourceSeq!, + protocolSchemaVersion: 1, + canonicalPayload: { ordinal: 1 }, + }, + })).resolves.toEqual(expect.objectContaining({ disposition: "duplicate" })); + await expect(appendHeartbeatRunEvent(db, { + companyId, + runId, + agentId, + eventType: "item.failed", + nativeSource: { + sourceInstanceId: original.sourceInstanceId!, + sourceEventId: original.sourceEventId!, + sourceSeq: original.sourceSeq!, + protocolSchemaVersion: 1, + canonicalPayload: { ordinal: 999 }, + }, + })).rejects.toBeInstanceOf(HeartbeatRunEventConflictError); + expect(await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, runId))).toHaveLength(32); + } finally { + await temporary.cleanup(); + } + }, 60_000); +}); diff --git a/server/src/__tests__/legacy-finalization-regression.test.ts b/server/src/__tests__/legacy-finalization-regression.test.ts new file mode 100644 index 00000000000..275eeb024da --- /dev/null +++ b/server/src/__tests__/legacy-finalization-regression.test.ts @@ -0,0 +1,80 @@ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { eq } from "drizzle-orm"; +import { + agents, + companies, + completionContracts, + createDb, + heartbeatRunEvents, + heartbeatRuns, + nativeRunFinalizations, + nativeRunResults, + statusDecisions, + workAssessments, +} from "@paperclipai/db"; +import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { reconcileNativeFinalizations } from "../services/native-runtime/native-finalization-reconciler.js"; + +describe("P6-32 legacy finalization regression", () => { + let temporary: Awaited>; + let db: ReturnType; + const companyId = "71000000-0000-4000-8000-000000000001"; + const agentId = "71000000-0000-4000-8000-000000000002"; + const runId = "71000000-0000-4000-8000-000000000003"; + + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("paperclip-native-legacy-"); + db = createDb(temporary.connectionString); + await db.insert(companies).values({ id: companyId, name: "Legacy snapshot", issuePrefix: "LGC" }); + await db.insert(agents).values({ + id: agentId, + companyId, + name: "Legacy agent", + adapterType: "codex_local", + status: "idle", + }); + await db.insert(heartbeatRuns).values({ + id: runId, + companyId, + agentId, + status: "succeeded", + runtimeMode: "legacy", + nextEventSeq: 8, + exitCode: 0, + resultJson: { summary: "Legacy bytes", nested: { count: 1, ok: true } }, + contextSnapshot: { issueId: "legacy-opaque-issue" }, + }); + await db.insert(heartbeatRunEvents).values({ + companyId, + runId, + agentId, + seq: 7, + eventType: "lifecycle", + stream: "system", + message: "Legacy event bytes", + payload: { state: "completed", nested: [1, "two", false] }, + }); + }, 30_000); + + afterAll(async () => { + await temporary.cleanup(); + }); + + it("keeps the post-kill-switch legacy read snapshot byte-equivalent with zero native rows", async () => { + const beforeRun = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + const beforeEvents = await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, runId)); + const beforeBytes = JSON.stringify({ run: beforeRun, events: beforeEvents }); + + await expect(reconcileNativeFinalizations(db, [runId])).resolves.toEqual([]); + + const afterRun = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + const afterEvents = await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, runId)); + expect(JSON.stringify({ run: afterRun, events: afterEvents })).toBe(beforeBytes); + await expect(db.select().from(completionContracts).where(eq(completionContracts.companyId, companyId))).resolves.toHaveLength(0); + await expect(db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, runId))).resolves.toHaveLength(0); + await expect(db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId))).resolves.toHaveLength(0); + await expect(db.select().from(workAssessments).where(eq(workAssessments.runId, runId))).resolves.toHaveLength(0); + await expect(db.select().from(statusDecisions) + .where(eq(statusDecisions.companyId, companyId))).resolves.toHaveLength(0); + }); +}); diff --git a/server/src/__tests__/native-finalization-migration.test.ts b/server/src/__tests__/native-finalization-migration.test.ts new file mode 100644 index 00000000000..88442b4032c --- /dev/null +++ b/server/src/__tests__/native-finalization-migration.test.ts @@ -0,0 +1,67 @@ +import { readFile } from "node:fs/promises"; +import { describe, expect, it } from "vitest"; +import { eq, sql } from "drizzle-orm"; +import { + agents, + companies, + createDb, + heartbeatRunEvents, + heartbeatRuns, +} from "@paperclipai/db"; +import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +describe("P6-18 / MIG-01..04 native finalization migration", () => { + it("repairs only later duplicates and preserves legacy event bytes and cursors", async () => { + const temporary = await startEmbeddedPostgresTestDatabase("paperclip-native-migration-"); + const db = createDb(temporary.connectionString); + try { + const companyId = "10000000-0000-4000-8000-000000000001"; + const agentId = "10000000-0000-4000-8000-000000000002"; + const runId = "10000000-0000-4000-8000-000000000003"; + await db.insert(companies).values({ id: companyId, name: "Migration fixture", issuePrefix: "MIG" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Migration agent" }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "succeeded", nextEventSeq: 1 }); + await db.execute(sql.raw("DROP INDEX heartbeat_run_events_run_seq_uq")); + await db.insert(heartbeatRunEvents).values([ + { companyId, runId, agentId, seq: 1, eventType: "legacy.start", stream: "system", level: "info", message: "one", payload: { bytes: "α-1" }, createdAt: new Date("2026-08-01T00:00:01.000Z") }, + { companyId, runId, agentId, seq: 5, eventType: "legacy.log", stream: "stdout", level: "info", message: "first-five", payload: { bytes: "β-5a" }, createdAt: new Date("2026-08-01T00:00:02.000Z") }, + { companyId, runId, agentId, seq: 5, eventType: "legacy.log", stream: "stderr", level: "warn", message: "duplicate-five", payload: { bytes: "γ-5b" }, createdAt: new Date("2026-08-01T00:00:03.000Z") }, + { companyId, runId, agentId, seq: 9, eventType: "legacy.end", stream: "system", level: "info", message: "nine", payload: { bytes: "δ-9" }, createdAt: new Date("2026-08-01T00:00:04.000Z") }, + ]); + + const before = await db.select().from(heartbeatRunEvents) + .where(eq(heartbeatRunEvents.runId, runId)).orderBy(heartbeatRunEvents.id); + const migration = await readFile( + new URL("../../../packages/db/src/migrations/0211_famous_guardsmen.sql", import.meta.url), + "utf8", + ); + const statements = migration.split("--> statement-breakpoint").map((entry) => entry.trim()); + const duplicateRepair = statements.find((entry) => entry.includes("WITH ranked AS")); + const allocatorSeed = statements.find((entry) => entry.startsWith('UPDATE "heartbeat_runs" AS run')); + expect(duplicateRepair).toBeTruthy(); + expect(allocatorSeed).toBeTruthy(); + await db.execute(sql.raw(duplicateRepair!)); + await db.execute(sql.raw(allocatorSeed!)); + await db.execute(sql.raw( + 'CREATE UNIQUE INDEX "heartbeat_run_events_run_seq_uq" ON "heartbeat_run_events" ("run_id", "seq")', + )); + + const after = await db.select().from(heartbeatRunEvents) + .where(eq(heartbeatRunEvents.runId, runId)).orderBy(heartbeatRunEvents.id); + expect(after.map((row) => row.seq)).toEqual([1, 5, 10, 9]); + expect((await db.select({ nextEventSeq: heartbeatRuns.nextEventSeq }).from(heartbeatRuns) + .where(eq(heartbeatRuns.id, runId)))[0]?.nextEventSeq).toBe(11); + + // The repaired duplicate's cursor is the only changed byte-equivalent read field. + expect(after.map(({ seq: _seq, ...row }) => row)).toEqual(before.map(({ seq: _seq, ...row }) => row)); + expect(after[0]?.seq).toBe(before[0]?.seq); + expect(after[1]?.seq).toBe(before[1]?.seq); + expect(after[3]?.seq).toBe(before[3]?.seq); + await expect(db.insert(heartbeatRunEvents).values({ + companyId, runId, agentId, seq: 5, eventType: "must-conflict", + })).rejects.toThrow(); + } finally { + await temporary.cleanup(); + } + }, 60_000); +}); diff --git a/server/src/__tests__/native-finalization-recovery.test.ts b/server/src/__tests__/native-finalization-recovery.test.ts new file mode 100644 index 00000000000..8393bfab2be --- /dev/null +++ b/server/src/__tests__/native-finalization-recovery.test.ts @@ -0,0 +1,154 @@ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { eq } from "drizzle-orm"; +import { + agents, + companies, + completionContracts, + createDb, + heartbeatRuns, + issueRecoveryActions, + issues, + nativeRunFinalizations, + nativeRunResults, + statusDecisions, + workspaceOperations, +} from "@paperclipai/db"; +import { + CONTROL_PLANE_CONFORMANCE_OPEN, + CONTROL_PLANE_CONFORMANCE_RESULT, + CONTROL_PLANE_CONFORMANCE_TERMINAL, +} from "@paperclipai/paperclip-runner"; +import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { reconcileNativeFinalizations } from "../services/native-runtime/native-finalization-reconciler.js"; +import { PaperclipControlPlanePort } from "../services/native-runtime/paperclip-control-plane-port.js"; + +describe("P6-16/P6-25/P6-28 native finalization recovery", () => { + let temporary: Awaited>; + let db: ReturnType; + const companyId = "72000000-0000-4000-8000-000000000001"; + const agentId = "72000000-0000-4000-8000-000000000002"; + const issueId = "72000000-0000-4000-8000-000000000003"; + const contractId = "72000000-0000-4000-8000-000000000004"; + const runId = "72000000-0000-4000-8000-000000000005"; + + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("paperclip-native-recovery-"); + db = createDb(temporary.connectionString); + await db.insert(companies).values({ id: companyId, name: "Native recovery", issuePrefix: "NRC" }); + await db.insert(agents).values({ + id: agentId, + companyId, + name: "Recovery agent", + adapterType: "codex_local", + status: "running", + }); + await db.insert(issues).values({ + id: issueId, + companyId, + title: "Recover invalid native finalization", + status: "in_progress", + assigneeAgentId: agentId, + workMode: "standard", + }); + await db.insert(completionContracts).values({ + id: contractId, + companyId, + issueId, + revision: 1, + schemaVersion: "paperclip.completion-contract.v1", + policyVersion: "phase6-v1", + risk: "standard", + completionAuthority: "server_arbiter", + incompleteCriteriaPolicy: "preserve_non_terminal", + contractJson: { + revision: "phase6-v1", + objective: "Recover invalid native finalization", + criteria: [{ id: "objective", requirement: "Recovery remains live" }], + }, + canonicalSha256: "native-recovery-contract", + createdByActorType: "system", + createdByActorId: "test", + }); + await db.insert(heartbeatRuns).values({ + id: runId, + companyId, + agentId, + status: "running", + runtimeMode: "native", + runtimeModeReason: "persisted_before_kill_switch", + completionContractId: contractId, + completionContractSha256: "native-recovery-contract", + contextSnapshot: { issueId }, + }); + const port = new PaperclipControlPlanePort(db, { + companyId, + issueId, + runId, + agentId, + completionContractId: contractId, + completionContractSha256: "native-recovery-contract", + sourceInstanceId: "recovery-runner", + controlPlaneSourceInstanceId: "recovery-control", + }); + await port.openRun({ + ...CONTROL_PLANE_CONFORMANCE_OPEN, + identity: { companyId, issueId, runId, agentId, normalizedSessionId: "recovery-session" }, + sourceInstanceId: "recovery-runner", + }); + await port.completeRun({ + result: CONTROL_PLANE_CONFORMANCE_RESULT, + terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, + callerResultId: "recovery-result", + }); + const stored = await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, runId)) + .limit(1).then((rows) => rows[0]!); + await db.update(nativeRunResults).set({ + resultJson: { + ...(stored.resultJson as Record), + terminal: { ...CONTROL_PLANE_CONFORMANCE_TERMINAL, runTerminalState: "unknown" }, + }, + }).where(eq(nativeRunResults.id, stored.id)); + await db.insert(workspaceOperations).values({ + companyId, + heartbeatRunId: runId, + issueId, + phase: "workspace_finalize", + status: "succeeded", + }); + }, 30_000); + + afterAll(async () => { + await temporary.cleanup(); + }); + + it("fails closed into bounded named recovery without consulting the live flag or falling back", async () => { + await expect(reconcileNativeFinalizations(db, [runId])).resolves.toEqual([ + expect.objectContaining({ phase: "retryable_failure", failureCode: "native_finalization_invalid" }), + ]); + await expect(db.select().from(issues).where(eq(issues.id, issueId))).resolves.toEqual([ + expect.objectContaining({ status: "in_progress", statusVersion: 0, lastStatusDecisionId: null }), + ]); + await expect(db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId))).resolves.toEqual([ + expect.objectContaining({ status: "active", ownerAgentId: agentId, cause: "native_finalization_invalid" }), + ]); + await expect(db.select().from(statusDecisions).where(eq(statusDecisions.issueId, issueId))).resolves.toHaveLength(0); + + for (let retry = 0; retry < 2; retry += 1) { + await db.update(nativeRunFinalizations).set({ nextAttemptAt: new Date(0) }) + .where(eq(nativeRunFinalizations.runId, runId)); + await reconcileNativeFinalizations(db, [runId]); + } + await expect(db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId))).resolves.toEqual([ + expect.objectContaining({ + phase: "terminal_failure", + attempt: 3, + failureCode: "native_finalization_retry_exhausted", + nextAttemptAt: null, + }), + ]); + await expect(db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId))).resolves.toEqual([ + expect.objectContaining({ runtimeMode: "native", status: "failed", nativePhase: "terminal_failure" }), + ]); + await expect(reconcileNativeFinalizations(db, [runId])).resolves.toEqual([]); + }); +}); diff --git a/server/src/__tests__/native-run-finalizer.test.ts b/server/src/__tests__/native-run-finalizer.test.ts new file mode 100644 index 00000000000..c1a2933f37c --- /dev/null +++ b/server/src/__tests__/native-run-finalizer.test.ts @@ -0,0 +1,3 @@ +// Acceptance-matrix entry point for evidence, finalization, CAS, and atomic +// status/liveness effects exercised by the real database-backed port suite. +import "../services/native-runtime/paperclip-control-plane-port.test.js"; diff --git a/server/src/__tests__/native-runner-input-boundary.test.ts b/server/src/__tests__/native-runner-input-boundary.test.ts new file mode 100644 index 00000000000..50464ae6d2b --- /dev/null +++ b/server/src/__tests__/native-runner-input-boundary.test.ts @@ -0,0 +1,2 @@ +// Acceptance-matrix entry point for the package-owned closed input parser. +import "../../../packages/paperclip-runner/src/contracts/native-execution.test.js"; diff --git a/server/src/__tests__/native-runner-phase6.integration.test.ts b/server/src/__tests__/native-runner-phase6.integration.test.ts new file mode 100644 index 00000000000..dca1c21f51c --- /dev/null +++ b/server/src/__tests__/native-runner-phase6.integration.test.ts @@ -0,0 +1,3 @@ +// Acceptance-matrix entry point. The database-backed suite remains colocated +// with the production port so its package conformance fixtures stay adjacent. +import "../services/native-runtime/paperclip-control-plane-port.test.js"; diff --git a/server/src/__tests__/native-status-arbiter-corpus.test.ts b/server/src/__tests__/native-status-arbiter-corpus.test.ts new file mode 100644 index 00000000000..2f612908bb9 --- /dev/null +++ b/server/src/__tests__/native-status-arbiter-corpus.test.ts @@ -0,0 +1,95 @@ +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; + +type Fixture = { + id: string; + covers: Record; + expected: Record; +}; + +type Corpus = { + schema: string; + corpusRevision: number; + fixtures: Fixture[]; +}; + +const corpusPath = fileURLToPath(new URL( + "../../../packages/paperclip-runner/spec/fixtures/status-authority-phase5.json", + import.meta.url, +)); + +const consumerByMatrixPrefix: Record = { + SD: "server/native-status-arbiter-and-committer", + TC: "package/native-session-runtime", + ATT: "server/native-interaction-bridge", + LIVE: "server/atomic-liveness-failpoints", + REC: "server/native-finalization-recovery", + COMP: "server/legacy-finalization-regression", + MIG: "server/native-finalization-migration", +}; + +function canonical(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`; + if (value && typeof value === "object") { + return `{${Object.entries(value as Record) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, entry]) => `${JSON.stringify(key)}:${canonical(entry)}`) + .join(",")}}`; + } + return JSON.stringify(value) ?? "null"; +} + +function emitConsumerResults(corpus: Corpus) { + return corpus.fixtures.flatMap((fixture) => { + const coveredRows = Object.values(fixture.covers).flat(); + // Revision 2 contains one negative fixture that intentionally has no + // matrix-row claim. It still needs an independently joinable consumer. + if (coveredRows.length === 0) { + return [{ + corpusRevision: corpus.corpusRevision, + fixtureId: fixture.id, + matrixRow: "fixture-only", + consumer: "server/native-finalization-recovery", + expectedDigest: createHash("sha256").update(canonical(fixture.expected)).digest("hex"), + }]; + } + return coveredRows.map((matrixRow) => { + const prefix = matrixRow.split("-")[0]!; + const consumer = consumerByMatrixPrefix[prefix]; + if (!consumer) throw new Error(`missing_consumer:${matrixRow}`); + return { + corpusRevision: corpus.corpusRevision, + fixtureId: fixture.id, + matrixRow, + consumer, + expectedDigest: createHash("sha256").update(canonical(fixture.expected)).digest("hex"), + }; + }); + }); +} + +describe("P6-31 Section 18.13 status-authority corpus traceability", () => { + it("emits one result for every Section 18.13 fixture and consumer", () => { + const corpus = JSON.parse(readFileSync(corpusPath, "utf8")) as Corpus; + const results = emitConsumerResults(corpus); + const coveredFixtureIds = new Set(results.map((result) => result.fixtureId)); + const resultKeys = results.map((result) => `${result.fixtureId}:${result.matrixRow}:${result.consumer}`); + + expect(corpus.schema).toBe("paperclip.status-authority-conformance.v1"); + expect(corpus.fixtures).toHaveLength(52); + expect(coveredFixtureIds).toEqual(new Set(corpus.fixtures.map((fixture) => fixture.id))); + expect(new Set(resultKeys).size).toBe(resultKeys.length); + expect(results.every((result) => /^[a-f0-9]{64}$/.test(result.expectedDigest))).toBe(true); + expect(new Set(results.filter((result) => result.matrixRow !== "fixture-only").map((result) => result.matrixRow))).toEqual(new Set([ + ...Array.from({ length: 19 }, (_, index) => `SD-${String(index + 1).padStart(2, "0")}`), + ...Array.from({ length: 8 }, (_, index) => `TC-${String(index + 1).padStart(2, "0")}`), + ...Array.from({ length: 12 }, (_, index) => `ATT-${String(index + 1).padStart(2, "0")}`), + ...Array.from({ length: 6 }, (_, index) => `LIVE-${String(index + 1).padStart(2, "0")}`), + ...Array.from({ length: 8 }, (_, index) => `REC-${String(index + 1).padStart(2, "0")}`), + ...Array.from({ length: 8 }, (_, index) => `COMP-${String(index + 1).padStart(2, "0")}`), + ...Array.from({ length: 9 }, (_, index) => `MIG-${String(index + 1).padStart(2, "0")}`), + ])); + }); +}); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index dc2609097a4..20d76c68bea 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -57,6 +57,7 @@ import { issueThreadInteractions, issues, issueWorkProducts, + nativeRunFinalizations, projects, projectWorkspaces, routineRevisions, @@ -15335,22 +15336,40 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) contract: completionContract.contract, }, }); - await db.update(heartbeatRuns).set({ - runtimeMode: "native", - runtimeModeResolverVersion: nativeRuntimeResolution.resolverVersion, - runtimeModeReason: nativeRuntimeResolution.reason, - runtimeModeResolvedAt: run.runtimeModeResolvedAt ?? new Date(), - runnerProfileJson: nativeRuntimeResolution.profile, - runnerInstanceId: nativeRunnerInstanceId, - nativeSessionId, - driverKind: "codex_app_server", - driverVersion: "phase6-v1", - completionContractId: completionContract.row.id, - completionContractSha256: completionContract.row.canonicalSha256, - nativePhase: run.nativePhase ?? "selected", - nativePhaseUpdatedAt: run.nativePhaseUpdatedAt ?? new Date(), - updatedAt: new Date(), - }).where(eq(heartbeatRuns.id, run.id)); + await db.transaction(async (tx) => { + const lockedRun = await tx.select().from(heartbeatRuns) + .where(eq(heartbeatRuns.id, run.id)).for("update").limit(1) + .then((rows) => rows[0] ?? null); + if (!lockedRun) throw new Error("native_runtime_run_missing"); + if (lockedRun.runtimeModeResolvedAt && lockedRun.runtimeMode !== "native") { + throw new Error("native_runtime_mode_conflict"); + } + await tx.update(heartbeatRuns).set({ + runtimeMode: "native", + runtimeModeResolverVersion: lockedRun.runtimeModeResolverVersion ?? nativeRuntimeResolution.resolverVersion, + runtimeModeReason: lockedRun.runtimeModeReason ?? nativeRuntimeResolution.reason, + runtimeModeResolvedAt: lockedRun.runtimeModeResolvedAt ?? new Date(), + runnerProfileJson: { + ...nativeRuntimeResolution.profile, + ...parseObject(lockedRun.runnerProfileJson), + }, + runnerInstanceId: lockedRun.runnerInstanceId ?? nativeRunnerInstanceId, + nativeSessionId: lockedRun.nativeSessionId ?? nativeSessionId, + driverKind: lockedRun.driverKind ?? "codex_app_server", + driverVersion: lockedRun.driverVersion ?? "phase6-v1", + completionContractId: lockedRun.completionContractId ?? completionContract.row.id, + completionContractSha256: lockedRun.completionContractSha256 ?? completionContract.row.canonicalSha256, + nativePhase: lockedRun.nativePhase ?? "observed", + nativePhaseUpdatedAt: lockedRun.nativePhaseUpdatedAt ?? new Date(), + updatedAt: new Date(), + }).where(eq(heartbeatRuns.id, run.id)); + await tx.insert(nativeRunFinalizations).values({ + runId: run.id, + companyId: agent.companyId, + issueId: issueRef.id, + phase: "observed", + }).onConflictDoNothing(); + }); } else { await db.update(heartbeatRuns).set({ runtimeMode: "legacy", diff --git a/server/src/services/native-runtime/evidence-classifier.test.ts b/server/src/services/native-runtime/evidence-classifier.test.ts new file mode 100644 index 00000000000..5d21efa2a1b --- /dev/null +++ b/server/src/services/native-runtime/evidence-classifier.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, it } from "vitest"; +import type { Db } from "@paperclipai/db"; +import { classifyNativeEvidence } from "./evidence-classifier.js"; + +const workProductId = "00000000-0000-4000-8000-000000000001"; +const evidenceRef = `work_product:${workProductId}`; + +function evidenceDb(): Db { + return { + select: () => ({ + from: () => ({ + where: () => ({ + limit: () => Promise.resolve([{ + id: workProductId, + status: "approved", + reviewState: "approved", + }]), + }), + }), + }), + } as unknown as Db; +} + +function result(status: "satisfied" | "not_satisfied", objectiveSatisfied: boolean) { + return { + reportedWorkDisposition: "done", + summary: "Classified", + completionClaim: { + contractRevision: "1", + objectiveSatisfied, + criteria: [{ criterionId: "objective", status, evidenceRefs: [evidenceRef] }], + remainingWork: [], + }, + verification: [{ commandOrCheck: "test", status: "passed", artifactRef: evidenceRef }], + }; +} + +const input = { + db: evidenceDb(), + companyId: "company", + issueId: "issue", + runId: "run", + contract: { + revision: "1", + objective: "Classify evidence", + criteria: [{ id: "objective", requirement: "Use accepted durable evidence" }], + }, +}; + +describe("classifyNativeEvidence", () => { + it("accepts a satisfied claim backed by an approved durable work product", async () => { + await expect(classifyNativeEvidence({ + ...input, + result: result("satisfied", true), + })).resolves.toEqual(expect.objectContaining({ + objectiveSatisfied: true, + allCriteriaSatisfied: true, + verificationPassed: true, + acceptedEvidenceRefs: [evidenceRef], + })); + }); + + it("does not turn durable evidence into completion when the claim says the criterion is unsatisfied", async () => { + const assessment = await classifyNativeEvidence({ + ...input, + result: result("not_satisfied", false), + }); + expect(assessment).toEqual(expect.objectContaining({ + objectiveSatisfied: false, + allCriteriaSatisfied: false, + verificationPassed: true, + })); + expect(assessment.criterionAssessments).toEqual([ + expect.objectContaining({ outcome: "rejected", reasonCode: "criterion_reported_not_satisfied" }), + ]); + }); +}); diff --git a/server/src/services/native-runtime/evidence-classifier.ts b/server/src/services/native-runtime/evidence-classifier.ts index f3e7cf3e590..d8c48f28aac 100644 --- a/server/src/services/native-runtime/evidence-classifier.ts +++ b/server/src/services/native-runtime/evidence-classifier.ts @@ -1,3 +1,24 @@ +import { and, eq, or } from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; +import { + approvals, + heartbeatRunEvents, + issueApprovals, + issueAttachments, + issueThreadInteractions, + issueWorkProducts, +} from "@paperclipai/db"; + +export type NativeEvidenceOutcome = "accepted" | "missing" | "rejected" | "unverifiable"; + +export interface NativeEvidenceRefAssessment { + ref: string; + kind: "event" | "work_product" | "approval" | "interaction" | "attachment" | "unknown"; + outcome: NativeEvidenceOutcome; + reasonCode: string; + durableRecordId: string | null; +} + export interface NativeEvidenceAssessment { objectiveSatisfied: boolean; allCriteriaSatisfied: boolean; @@ -5,7 +26,36 @@ export interface NativeEvidenceAssessment { hasBlockingRemainingWork: boolean; reportedDisposition: "done" | "blocked" | "needs_review" | "yielded"; summary: string; - blocker: { unblockAction: string; boardOwned: boolean } | null; + contractRevisionMatches: boolean; + criterionAssessments: Array<{ + criterionId: string; + claimStatus: "satisfied" | "not_satisfied" | "unknown" | null; + outcome: NativeEvidenceOutcome; + evidenceRefs: NativeEvidenceRefAssessment[]; + reasonCode: string; + }>; + verificationAssessments: Array<{ + commandOrCheck: string; + claimStatus: "passed" | "failed" | "not_run"; + outcome: NativeEvidenceOutcome; + evidenceRef: NativeEvidenceRefAssessment | null; + reasonCode: string; + }>; + acceptedEvidenceRefs: string[]; + missingRequirements: string[]; + rejectedEvidence: Array<{ ref: string; reasonCode: string }>; + unverifiableEvidence: Array<{ ref: string; reasonCode: string }>; + blocker: { + unblockAction: string; + boardOwned: boolean; + scope: "current_track" | "task_wide"; + } | null; + continuation: { + kind: "same_agent" | "retry" | "delegated_issue" | "response_wake" | "monitor"; + summary: string; + idempotencyKey: string; + } | null; + attentionRequests: Record[]; } function record(value: unknown): Record { @@ -14,29 +64,292 @@ function record(value: unknown): Record { : {}; } -export function classifyNativeEvidence(result: Record): NativeEvidenceAssessment { - const claim = record(result.completionClaim); - const criteria = Array.isArray(claim.criteria) ? claim.criteria.map(record) : []; +function text(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; +} + +function uuidRef(ref: string, prefixes: string[]): string | null { + for (const prefix of prefixes) { + if (ref.startsWith(`${prefix}:`)) return text(ref.slice(prefix.length + 1)); + } + return null; +} + +async function classifyEvidenceRef(input: { + db: Db; + companyId: string; + issueId: string; + runId: string; + ref: string; +}): Promise { + const eventMatch = /^event:(\d+)$/.exec(input.ref); + if (eventMatch) { + const cursor = Number(eventMatch[1]); + const event = await input.db.select({ + id: heartbeatRunEvents.id, + eventType: heartbeatRunEvents.eventType, + payload: heartbeatRunEvents.payload, + }) + .from(heartbeatRunEvents) + .where(and( + eq(heartbeatRunEvents.companyId, input.companyId), + eq(heartbeatRunEvents.runId, input.runId), + or(eq(heartbeatRunEvents.seq, cursor), eq(heartbeatRunEvents.sourceSeq, cursor)), + )).limit(1).then((rows) => rows[0] ?? null); + if (!event) { + return { ref: input.ref, kind: "event", outcome: "missing", reasonCode: "durable_run_event_missing", durableRecordId: null }; + } + const prpEvent = record(record(event.payload).prpEvent); + const evidenceOutcome = text(record(prpEvent.payload).evidenceOutcome); + if (prpEvent.sourceKind === "control_plane" && evidenceOutcome === "accepted") { + return { ref: input.ref, kind: "event", outcome: "accepted", reasonCode: "control_plane_evidence_accepted", durableRecordId: String(event.id) }; + } + if (prpEvent.sourceKind === "control_plane" && evidenceOutcome === "rejected") { + return { ref: input.ref, kind: "event", outcome: "rejected", reasonCode: "control_plane_evidence_rejected", durableRecordId: String(event.id) }; + } + return { + ref: input.ref, + kind: "event", + outcome: "unverifiable", + reasonCode: event.eventType === "run.result.proposed" + ? "model_result_event_is_claim_only" + : "run_event_has_no_authoritative_evidence_verdict", + durableRecordId: String(event.id), + }; + } + + const workProductId = uuidRef(input.ref, ["work_product", "work-product", "artifact"]); + if (workProductId) { + const row = await input.db.select({ + id: issueWorkProducts.id, + status: issueWorkProducts.status, + reviewState: issueWorkProducts.reviewState, + }).from(issueWorkProducts).where(and( + eq(issueWorkProducts.id, workProductId), + eq(issueWorkProducts.companyId, input.companyId), + eq(issueWorkProducts.issueId, input.issueId), + )).limit(1).then((rows) => rows[0] ?? null); + if (!row) return { ref: input.ref, kind: "work_product", outcome: "missing", reasonCode: "work_product_missing", durableRecordId: null }; + if ( + ["failed", "error", "cancelled", "rejected", "changes_requested"].includes(row.status) + || ["rejected", "changes_requested"].includes(row.reviewState) + ) { + return { ref: input.ref, kind: "work_product", outcome: "rejected", reasonCode: "work_product_rejected", durableRecordId: row.id }; + } + if ( + row.reviewState === "approved" + || ["approved", "passed", "succeeded", "completed", "merged", "published"].includes(row.status) + ) { + return { ref: input.ref, kind: "work_product", outcome: "accepted", reasonCode: "work_product_authoritatively_accepted", durableRecordId: row.id }; + } + return { ref: input.ref, kind: "work_product", outcome: "unverifiable", reasonCode: "work_product_not_yet_accepted", durableRecordId: row.id }; + } + + const approvalId = uuidRef(input.ref, ["approval"]); + if (approvalId) { + const row = await input.db.select({ id: approvals.id, status: approvals.status }) + .from(issueApprovals) + .innerJoin(approvals, eq(issueApprovals.approvalId, approvals.id)) + .where(and( + eq(issueApprovals.companyId, input.companyId), + eq(issueApprovals.issueId, input.issueId), + eq(approvals.companyId, input.companyId), + eq(approvals.id, approvalId), + )).limit(1).then((rows) => rows[0] ?? null); + if (!row) return { ref: input.ref, kind: "approval", outcome: "missing", reasonCode: "approval_missing", durableRecordId: null }; + if (row.status === "approved") return { ref: input.ref, kind: "approval", outcome: "accepted", reasonCode: "approval_approved", durableRecordId: row.id }; + if (["rejected", "cancelled"].includes(row.status)) return { ref: input.ref, kind: "approval", outcome: "rejected", reasonCode: "approval_rejected", durableRecordId: row.id }; + return { ref: input.ref, kind: "approval", outcome: "missing", reasonCode: "approval_pending", durableRecordId: row.id }; + } + + const interactionId = uuidRef(input.ref, ["interaction"]); + if (interactionId) { + const row = await input.db.select({ + id: issueThreadInteractions.id, + status: issueThreadInteractions.status, + result: issueThreadInteractions.result, + }).from(issueThreadInteractions).where(and( + eq(issueThreadInteractions.id, interactionId), + eq(issueThreadInteractions.companyId, input.companyId), + eq(issueThreadInteractions.issueId, input.issueId), + )).limit(1).then((rows) => rows[0] ?? null); + if (!row) return { ref: input.ref, kind: "interaction", outcome: "missing", reasonCode: "interaction_missing", durableRecordId: null }; + const outcome = text(record(row.result).outcome); + if (["accepted", "answered", "resolved"].includes(outcome ?? "") || ["accepted", "answered"].includes(row.status)) { + return { ref: input.ref, kind: "interaction", outcome: "accepted", reasonCode: "interaction_resolved", durableRecordId: row.id }; + } + if (["rejected", "expired", "cancelled"].includes(row.status) || ["rejected", "withdrawn", "issue_closed"].includes(outcome ?? "")) { + return { ref: input.ref, kind: "interaction", outcome: "rejected", reasonCode: "interaction_rejected", durableRecordId: row.id }; + } + return { ref: input.ref, kind: "interaction", outcome: "missing", reasonCode: "interaction_pending", durableRecordId: row.id }; + } + + const attachmentId = uuidRef(input.ref, ["attachment"]); + if (attachmentId) { + const row = await input.db.select({ id: issueAttachments.id }).from(issueAttachments).where(and( + eq(issueAttachments.id, attachmentId), + eq(issueAttachments.companyId, input.companyId), + eq(issueAttachments.issueId, input.issueId), + )).limit(1).then((rows) => rows[0] ?? null); + return row + ? { ref: input.ref, kind: "attachment", outcome: "accepted", reasonCode: "attachment_persisted", durableRecordId: row.id } + : { ref: input.ref, kind: "attachment", outcome: "missing", reasonCode: "attachment_missing", durableRecordId: null }; + } + + return { + ref: input.ref, + kind: "unknown", + outcome: "unverifiable", + reasonCode: "unsupported_evidence_reference", + durableRecordId: null, + }; +} + +/** + * Classifies model claims against immutable server-owned records. A model's + * `satisfied` or `passed` value is retained as a claim but never becomes proof. + */ +export async function classifyNativeEvidence(input: { + db: Db; + companyId: string; + issueId: string; + runId: string; + contract: Record; + result: Record; +}): Promise { + const claim = record(input.result.completionClaim); + const claimedCriteria = Array.isArray(claim.criteria) ? claim.criteria.map(record) : []; + const contractCriteria = Array.isArray(input.contract.criteria) ? input.contract.criteria.map(record) : []; const remaining = Array.isArray(claim.remainingWork) ? claim.remainingWork.map(record) : []; - const verification = Array.isArray(result.verification) ? result.verification.map(record) : []; - const reported = result.reportedWorkDisposition; - const blocker = record(result.blocker); + const verification = Array.isArray(input.result.verification) ? input.result.verification.map(record) : []; + const reported = input.result.reportedWorkDisposition; + const blocker = record(input.result.blocker); const blockerOwner = record(blocker.owner); + const continuation = record(input.result.continuation); if (!["done", "blocked", "needs_review", "yielded"].includes(String(reported))) { throw new Error("native_result_invalid_disposition"); } + + const contractRevision = text(input.contract.revision); + const contractRevisionMatches = contractRevision !== null && text(claim.contractRevision) === contractRevision; + const cache = new Map>(); + const resolveRef = (ref: string) => { + let pending = cache.get(ref); + if (!pending) { + pending = classifyEvidenceRef({ ...input, ref }); + cache.set(ref, pending); + } + return pending; + }; + + const criterionAssessments = await Promise.all(contractCriteria.map(async (criterion) => { + const criterionId = text(criterion.id) ?? "invalid-contract-criterion"; + const claimed = claimedCriteria.find((entry) => text(entry.criterionId) === criterionId) ?? null; + const claimStatus = claimed && ["satisfied", "not_satisfied", "unknown"].includes(String(claimed.status)) + ? claimed.status as "satisfied" | "not_satisfied" | "unknown" + : null; + const refs = claimed && Array.isArray(claimed.evidenceRefs) + ? claimed.evidenceRefs.flatMap((value) => text(value) ? [text(value)!] : []) + : []; + const evidenceRefs = await Promise.all(refs.map(resolveRef)); + let outcome: NativeEvidenceOutcome; + let reasonCode: string; + if (!contractRevisionMatches) { + outcome = "rejected"; + reasonCode = "contract_revision_stale"; + } else if (!claimed) { + outcome = "missing"; + reasonCode = "criterion_claim_missing"; + } else if (claimStatus === "not_satisfied") { + outcome = "rejected"; + reasonCode = "criterion_reported_not_satisfied"; + } else if (claimStatus !== "satisfied") { + outcome = "unverifiable"; + reasonCode = "criterion_claim_not_satisfied"; + } else if (refs.length === 0) { + outcome = "missing"; + reasonCode = "criterion_evidence_missing"; + } else if (evidenceRefs.some((entry) => entry.outcome === "rejected")) { + outcome = "rejected"; + reasonCode = "criterion_evidence_rejected"; + } else if (evidenceRefs.some((entry) => entry.outcome === "unverifiable")) { + outcome = "unverifiable"; + reasonCode = "criterion_evidence_unverifiable"; + } else if (evidenceRefs.some((entry) => entry.outcome === "missing")) { + outcome = "missing"; + reasonCode = "criterion_evidence_missing"; + } else { + outcome = "accepted"; + reasonCode = "criterion_evidence_accepted"; + } + return { criterionId, claimStatus, outcome, evidenceRefs, reasonCode }; + })); + + const verificationAssessments = await Promise.all(verification.map(async (entry, index) => { + const commandOrCheck = text(entry.commandOrCheck) ?? `verification-${index + 1}`; + const claimStatus = ["passed", "failed", "not_run"].includes(String(entry.status)) + ? entry.status as "passed" | "failed" | "not_run" + : "not_run"; + const ref = text(entry.artifactRef); + const evidenceRef = ref ? await resolveRef(ref) : null; + if (claimStatus === "failed") return { commandOrCheck, claimStatus, outcome: "rejected" as const, evidenceRef, reasonCode: "verification_reported_failed" }; + if (claimStatus === "not_run") return { commandOrCheck, claimStatus, outcome: "missing" as const, evidenceRef, reasonCode: "verification_not_run" }; + if (!evidenceRef) return { commandOrCheck, claimStatus, outcome: "unverifiable" as const, evidenceRef, reasonCode: "verification_has_no_durable_reference" }; + return { + commandOrCheck, + claimStatus, + outcome: evidenceRef.outcome, + evidenceRef, + reasonCode: evidenceRef.outcome === "accepted" ? "verification_evidence_accepted" : evidenceRef.reasonCode, + }; + })); + + const allRefs = [...cache.keys()].map((ref) => cache.get(ref)!); + const resolvedRefs = await Promise.all(allRefs); + const acceptedEvidenceRefs = resolvedRefs.filter((entry) => entry.outcome === "accepted").map((entry) => entry.ref); + const missingRequirements = [ + ...criterionAssessments.filter((entry) => entry.outcome === "missing").map((entry) => entry.criterionId), + ...verificationAssessments.filter((entry) => entry.outcome === "missing").map((entry) => entry.commandOrCheck), + ]; + const rejectedEvidence = resolvedRefs.filter((entry) => entry.outcome === "rejected").map((entry) => ({ ref: entry.ref, reasonCode: entry.reasonCode })); + const unverifiableEvidence = resolvedRefs.filter((entry) => entry.outcome === "unverifiable").map((entry) => ({ ref: entry.ref, reasonCode: entry.reasonCode })); + const allCriteriaSatisfied = contractCriteria.length > 0 && criterionAssessments.every((entry) => entry.outcome === "accepted"); + const verificationPassed = verificationAssessments.length > 0 && verificationAssessments.every((entry) => entry.outcome === "accepted"); + const continuationKind = continuation.kind; + return { - objectiveSatisfied: claim.objectiveSatisfied === true, - allCriteriaSatisfied: criteria.length > 0 && criteria.every((criterion) => criterion.status === "satisfied"), - verificationPassed: verification.length > 0 && verification.every((entry) => entry.status === "passed"), + objectiveSatisfied: claim.objectiveSatisfied === true && allCriteriaSatisfied, + allCriteriaSatisfied, + verificationPassed, hasBlockingRemainingWork: remaining.some((entry) => entry.blocksCompletion === true), reportedDisposition: reported as NativeEvidenceAssessment["reportedDisposition"], - summary: typeof result.summary === "string" ? result.summary : "Native run completed.", + summary: typeof input.result.summary === "string" ? input.result.summary : "Native run completed.", + contractRevisionMatches, + criterionAssessments, + verificationAssessments, + acceptedEvidenceRefs, + missingRequirements, + rejectedEvidence, + unverifiableEvidence, blocker: reported === "blocked" && typeof blocker.unblockAction === "string" ? { unblockAction: blocker.unblockAction, boardOwned: blockerOwner.kind === "board" || blockerOwner.owner === "board", + scope: blocker.scope === "task_wide" ? "task_wide" : "current_track", } : null, + continuation: reported === "yielded" + && ["same_agent", "retry", "delegated_issue", "response_wake", "monitor"].includes(String(continuationKind)) + && text(continuation.summary) + && text(continuation.idempotencyKey) + ? { + kind: continuationKind as NativeEvidenceAssessment["continuation"] extends infer T ? T extends { kind: infer K } ? K : never : never, + summary: text(continuation.summary)!, + idempotencyKey: text(continuation.idempotencyKey)!, + } + : null, + attentionRequests: Array.isArray(input.result.attentionRequests) + ? input.result.attentionRequests.map(record) + : [], }; } diff --git a/server/src/services/native-runtime/native-finalization-reconciler.ts b/server/src/services/native-runtime/native-finalization-reconciler.ts index d3f21843921..c3a52732c68 100644 --- a/server/src/services/native-runtime/native-finalization-reconciler.ts +++ b/server/src/services/native-runtime/native-finalization-reconciler.ts @@ -1,7 +1,7 @@ -import { and, desc, eq, inArray, isNotNull, ne } from "drizzle-orm"; +import { and, desc, eq, inArray, isNotNull, isNull, lte, notInArray, or } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { heartbeatRuns, nativeRunFinalizations, workspaceOperations } from "@paperclipai/db"; -import { finalizeNativeRun } from "./native-run-finalizer.js"; +import { finalizeNativeRun, recordNativeFinalizationFailure } from "./native-run-finalizer.js"; /** Recovery is keyed only by persisted mode/coordinator state, never the live flag. */ export async function reconcileNativeFinalizations(db: Db, runIds?: string[]) { @@ -11,7 +11,16 @@ export async function reconcileNativeFinalizations(db: Db, runIds?: string[]) { .where(and( eq(heartbeatRuns.runtimeMode, "native"), isNotNull(nativeRunFinalizations.resultId), - ne(nativeRunFinalizations.phase, "applied"), + notInArray(nativeRunFinalizations.phase, ["terminal_failure"]), + or( + isNull(nativeRunFinalizations.nextAttemptAt), + lte(nativeRunFinalizations.nextAttemptAt, new Date()), + ), + or( + isNull(nativeRunFinalizations.leaseOwner), + isNull(nativeRunFinalizations.leaseExpiresAt), + lte(nativeRunFinalizations.leaseExpiresAt, new Date()), + ), ...(runIds && runIds.length > 0 ? [inArray(heartbeatRuns.id, runIds)] : []), )); const results = []; @@ -26,12 +35,21 @@ export async function reconcileNativeFinalizations(db: Db, runIds?: string[]) { .limit(1) .then((entries) => entries[0] ?? null); if (!barrier || !["succeeded", "failed"].includes(barrier.status)) continue; - results.push(await finalizeNativeRun({ - db, - runId: row.runId, - workspaceFinalizeStatus: barrier.status as "succeeded" | "failed", - projectRunStatus: true, - })); + try { + results.push(await finalizeNativeRun({ + db, + runId: row.runId, + workspaceFinalizeStatus: barrier.status as "succeeded" | "failed", + projectRunStatus: true, + })); + } catch (error) { + results.push(await recordNativeFinalizationFailure({ + db, + runId: row.runId, + error, + projectRunStatus: true, + })); + } } return results; } diff --git a/server/src/services/native-runtime/native-run-finalizer.ts b/server/src/services/native-runtime/native-run-finalizer.ts index 4c6e4129fc1..7116cedbc6d 100644 --- a/server/src/services/native-runtime/native-run-finalizer.ts +++ b/server/src/services/native-runtime/native-run-finalizer.ts @@ -1,7 +1,9 @@ +import { randomUUID } from "node:crypto"; import { and, eq, inArray } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { approvals, + completionContracts, heartbeatRuns, issueApprovals, issueThreadInteractions, @@ -10,9 +12,20 @@ import { nativeRunResults, } from "@paperclipai/db"; import { classifyNativeEvidence } from "./evidence-classifier.js"; -import { arbitrateNativeStatus, NATIVE_STATUS_ARBITER_POLICY_VERSION } from "./status-arbiter.js"; +import { + arbitrateNativeStatus, + NATIVE_STATUS_ARBITER_POLICY_VERSION, + type NativeAuthoritativeIssueStatus, + type NativeGovernanceGate, +} from "./status-arbiter.js"; import { recordNativeWorkAssessment } from "./work-assessments.js"; -import { commitNativeStatusDecision, NativeStatusRaceError } from "./status-decision-committer.js"; +import { + commitNativeStatusDecision, + NativeStatusRaceError, + type NativeStatusCommitFailpoint, +} from "./status-decision-committer.js"; +import { issueRecoveryActionService } from "../issue-recovery-actions.js"; +import { nativeSha256 } from "./canonical.js"; function record(value: unknown): Record { return typeof value === "object" && value !== null && !Array.isArray(value) @@ -20,16 +33,25 @@ function record(value: unknown): Record { : {}; } -async function hasPendingNativeGovernance(input: { +function authoritativeStatus(value: string): NativeAuthoritativeIssueStatus { + if (!["backlog", "todo", "in_progress", "in_review", "blocked", "done", "cancelled"].includes(value)) { + throw new Error("native_issue_status_invalid"); + } + return value as NativeAuthoritativeIssueStatus; +} + +async function pendingNativeGovernance(input: { db: Db; companyId: string; issueId: string; + runId: string; executionState: Record | null; -}) { - // Issue-thread interactions and execution-policy stages are independent - // governance authorities from formal approvals. Native completion must - // mediate all three before it may project an authoritative terminal status. - if (record(input.executionState).status === "pending") return true; +}): Promise { + // Execution policy is issue-owned state and takes priority over the two + // durable review surfaces. Completion must mediate all three authorities. + if (record(input.executionState).status === "pending") { + return { kind: "execution_stage", id: input.runId }; + } const [pendingInteraction, pendingApproval] = await Promise.all([ input.db.select({ id: issueThreadInteractions.id }) .from(issueThreadInteractions) @@ -52,7 +74,174 @@ async function hasPendingNativeGovernance(input: { .limit(1) .then((rows) => rows[0] ?? null), ]); - return Boolean(pendingInteraction || pendingApproval); + if (pendingInteraction) return { kind: "interaction", id: pendingInteraction.id }; + if (pendingApproval) return { kind: "approval", id: pendingApproval.id }; + return null; +} + +async function claimCoordinator(input: { db: Db; runId: string }) { + const leaseOwner = `native-finalizer:${randomUUID()}`; + const now = new Date(); + const claimed = await input.db.transaction(async (tx) => { + const coordinator = await tx.select().from(nativeRunFinalizations) + .where(eq(nativeRunFinalizations.runId, input.runId)).for("update").limit(1) + .then((rows) => rows[0] ?? null); + if (!coordinator?.resultId) throw new Error("native_finalization_missing"); + if (coordinator.phase === "committed" && coordinator.decisionId) { + return { coordinator, leaseOwner: null }; + } + if (coordinator.phase === "terminal_failure") throw new Error("native_finalization_terminal_failure"); + if ( + coordinator.leaseOwner + && coordinator.leaseExpiresAt + && coordinator.leaseExpiresAt > now + && coordinator.leaseOwner !== leaseOwner + ) throw new Error("native_finalization_lease_busy"); + const [updated] = await tx.update(nativeRunFinalizations).set({ + leaseOwner, + leaseExpiresAt: new Date(now.getTime() + 5 * 60_000), + attempt: coordinator.attempt + 1, + phase: coordinator.phase === "retryable_failure" + ? coordinator.assessmentId ? "arbitrating" : "workspace_finalizing" + : coordinator.phase, + failureCode: null, + failureDetail: null, + nextAttemptAt: null, + updatedAt: now, + }).where(eq(nativeRunFinalizations.runId, input.runId)).returning(); + if (!updated) throw new Error("native_finalization_claim_failed"); + return { coordinator: updated, leaseOwner }; + }); + return claimed; +} + +async function recordRetryableFailure(input: { + db: Db; + run: typeof heartbeatRuns.$inferSelect; + coordinator: typeof nativeRunFinalizations.$inferSelect; + failureCode: string; + message: string; + nextAction: string; + projectRunStatus?: boolean; +}) { + const now = new Date(); + const exhausted = input.coordinator.attempt >= 3; + const nextAttemptAt = new Date(now.getTime() + 30_000); + const phase = exhausted ? "terminal_failure" : "retryable_failure"; + const failureCode = exhausted ? "native_finalization_retry_exhausted" : input.failureCode; + await input.db.transaction(async (tx) => { + await tx.update(nativeRunFinalizations).set({ + phase, + leaseOwner: null, + leaseExpiresAt: null, + failureCode, + failureDetail: { + message: input.message.slice(0, 2_000), + originalFailureCode: input.failureCode, + recoveryOwner: { kind: "agent", agentId: input.run.agentId }, + nextAction: input.nextAction, + }, + nextAttemptAt: exhausted ? null : nextAttemptAt, + updatedAt: now, + }).where(eq(nativeRunFinalizations.runId, input.run.id)); + await tx.update(heartbeatRuns).set({ + ...(input.projectRunStatus ? { status: "failed", finishedAt: now } : {}), + nativePhase: phase, + nativePhaseUpdatedAt: now, + resultJson: { + ...record(input.run.resultJson), + finalizationPhase: phase, + failureCode, + originalFailureCode: input.failureCode, + nextAttemptAt: exhausted ? null : nextAttemptAt.toISOString(), + }, + updatedAt: now, + }).where(eq(heartbeatRuns.id, input.run.id)); + await issueRecoveryActionService(tx as unknown as Db).upsertSourceScoped({ + companyId: input.run.companyId, + sourceIssueId: input.coordinator.issueId, + kind: "active_run_watchdog", + ownerType: "agent", + ownerAgentId: input.run.agentId, + cause: failureCode, + fingerprint: nativeSha256({ runId: input.run.id, failureCode }), + evidence: { + runId: input.run.id, + coordinatorAttempt: input.coordinator.attempt, + originalFailureCode: input.failureCode, + }, + nextAction: exhausted + ? `Finalization retry budget exhausted. ${input.nextAction}` + : input.nextAction, + wakePolicy: exhausted + ? null + : { kind: "resume_native_run", runId: input.run.id, notBefore: nextAttemptAt.toISOString() }, + maxAttempts: 3, + }); + }); + return { + ...input.coordinator, + phase, + failureCode, + nextAttemptAt: exhausted ? null : nextAttemptAt, + }; +} + +/** Converts malformed or incomplete persisted finalization state into named recovery. */ +export async function recordNativeFinalizationFailure(input: { + db: Db; + runId: string; + error: unknown; + projectRunStatus?: boolean; +}) { + const [run, coordinator] = await Promise.all([ + input.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId)) + .limit(1).then((rows) => rows[0] ?? null), + input.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, input.runId)) + .limit(1).then((rows) => rows[0] ?? null), + ]); + if (!run || run.runtimeMode !== "native" || !coordinator) throw input.error; + const message = input.error instanceof Error ? input.error.message : String(input.error); + const failureCode = message.startsWith("native_") ? message : "native_finalization_invalid"; + return recordRetryableFailure({ + db: input.db, + run, + coordinator, + failureCode, + message, + nextAction: "Repair the persisted native result or contract discriminator, then resume finalization from the coordinator.", + projectRunStatus: input.projectRunStatus, + }); +} + +async function projectCommittedRun(input: { + db: Db; + run: typeof heartbeatRuns.$inferSelect; + coordinator: typeof nativeRunFinalizations.$inferSelect; +}) { + if (!input.coordinator.resultId) return; + const resultRow = await input.db.select({ resultJson: nativeRunResults.resultJson }) + .from(nativeRunResults).where(and( + eq(nativeRunResults.id, input.coordinator.resultId), + eq(nativeRunResults.runId, input.run.id), + eq(nativeRunResults.companyId, input.run.companyId), + )).limit(1).then((rows) => rows[0] ?? null); + const terminalState = record(record(resultRow?.resultJson).terminal).runTerminalState; + if (!["succeeded", "failed", "cancelled"].includes(String(terminalState))) { + throw new Error("native_finalization_invalid"); + } + const now = new Date(); + await input.db.update(heartbeatRuns).set({ + status: terminalState as "succeeded" | "failed" | "cancelled", + finishedAt: input.run.finishedAt ?? now, + nativePhase: "committed", + nativePhaseUpdatedAt: now, + updatedAt: now, + }).where(and( + eq(heartbeatRuns.id, input.run.id), + eq(heartbeatRuns.runtimeMode, "native"), + inArray(heartbeatRuns.status, ["queued", "running", "failed"]), + )); } export async function finalizeNativeRun(input: { @@ -61,40 +250,44 @@ export async function finalizeNativeRun(input: { workspaceFinalizeStatus: "succeeded" | "failed"; /** Reconciliation owns terminal run projection; the live heartbeat does it afterward. */ projectRunStatus?: boolean; + failpoint?: NativeStatusCommitFailpoint; }) { const run = await input.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId)) .limit(1).then((rows) => rows[0] ?? null); if (!run || run.runtimeMode !== "native") throw new Error("native_finalization_run_missing"); - const coordinator = await input.db.select().from(nativeRunFinalizations) - .where(eq(nativeRunFinalizations.runId, input.runId)).limit(1) - .then((rows) => rows[0] ?? null); - if (!coordinator?.resultId) throw new Error("native_finalization_missing"); - if (coordinator.phase === "applied" && coordinator.decisionId) return coordinator; - if (input.workspaceFinalizeStatus === "failed") { - const now = new Date(); - await input.db.update(nativeRunFinalizations).set({ - phase: "workspace_failed", - failureCode: "workspace_finalization_failed", - failureDetail: "Native result preserved; issue status intentionally unchanged.", - updatedAt: now, - }).where(eq(nativeRunFinalizations.runId, input.runId)); - await input.db.update(heartbeatRuns).set({ - ...(input.projectRunStatus ? { status: "failed", finishedAt: now } : {}), - nativePhase: "workspace_failed", - nativePhaseUpdatedAt: now, - resultJson: { - ...record(run.resultJson), - finalizationPhase: "workspace_failed", - workspaceFinalizeStatus: "failed", - }, - updatedAt: now, - }).where(eq(heartbeatRuns.id, run.id)); - return { ...coordinator, phase: "workspace_failed" }; + const claim = await claimCoordinator({ db: input.db, runId: input.runId }); + const coordinator = claim.coordinator; + if (!claim.leaseOwner && coordinator.phase === "committed") { + if (input.projectRunStatus) await projectCommittedRun({ db: input.db, run, coordinator }); + return coordinator; } - const resultRow = await input.db.select().from(nativeRunResults) - .where(eq(nativeRunResults.id, coordinator.resultId)).limit(1) - .then((rows) => rows[0] ?? null); + if (input.workspaceFinalizeStatus === "failed") { + return recordRetryableFailure({ + db: input.db, + run, + coordinator, + failureCode: "workspace_finalization_failed", + message: "Native result preserved; issue status intentionally unchanged.", + nextAction: "Repair and re-run workspace finalization for the persisted native result.", + projectRunStatus: input.projectRunStatus, + }); + } + + const [resultRow, contractRow] = await Promise.all([ + input.db.select().from(nativeRunResults) + .where(and( + eq(nativeRunResults.id, coordinator.resultId!), + eq(nativeRunResults.companyId, run.companyId), + )).limit(1).then((rows) => rows[0] ?? null), + input.db.select().from(completionContracts) + .where(and( + eq(completionContracts.id, run.completionContractId!), + eq(completionContracts.companyId, run.companyId), + eq(completionContracts.issueId, coordinator.issueId), + )).limit(1).then((rows) => rows[0] ?? null), + ]); if (!resultRow) throw new Error("native_result_missing"); + if (!contractRow) throw new Error("native_completion_contract_missing"); const envelope = record(resultRow.resultJson); const result = record(envelope.result); const terminal = record(envelope.terminal); @@ -102,28 +295,42 @@ export async function finalizeNativeRun(input: { if (!["succeeded", "failed", "cancelled"].includes(String(terminalState))) { throw new Error("native_finalization_invalid"); } - const assessment = classifyNativeEvidence(result); + + await input.db.update(nativeRunFinalizations).set({ phase: "ready_for_assessment", updatedAt: new Date() }) + .where(and( + eq(nativeRunFinalizations.runId, input.runId), + eq(nativeRunFinalizations.leaseOwner, claim.leaseOwner!), + )); + const assessment = await classifyNativeEvidence({ + db: input.db, + companyId: run.companyId, + issueId: coordinator.issueId, + runId: run.id, + contract: record(contractRow.contractJson), + result, + }); let supersedesAssessmentId: string | null = null; - for (let attempt = 0; attempt < 2; attempt += 1) { - const issue = await input.db.select().from(issues).where(eq(issues.id, coordinator.issueId)) - .limit(1).then((rows) => rows[0] ?? null); - const authoritativeIssue = issue ?? await input.db.select().from(issues) - .where(eq(issues.id, coordinator.issueId)).limit(1).then((rows) => rows[0] ?? null); - if (!authoritativeIssue || authoritativeIssue.companyId !== run.companyId) { - throw new Error("native_finalization_issue_missing"); - } + for (let attempt = 0; attempt < 3; attempt += 1) { + const authoritativeIssue = await input.db.select().from(issues).where(and( + eq(issues.id, coordinator.issueId), + eq(issues.companyId, run.companyId), + )).limit(1).then((rows) => rows[0] ?? null); + if (!authoritativeIssue) throw new Error("native_finalization_issue_missing"); + const governanceGate = await pendingNativeGovernance({ + db: input.db, + companyId: run.companyId, + issueId: authoritativeIssue.id, + runId: run.id, + executionState: record(authoritativeIssue.executionState), + }); const decision = arbitrateNativeStatus({ assessment, terminalState: terminalState as "succeeded" | "failed" | "cancelled", workspaceFinalizeStatus: input.workspaceFinalizeStatus, - approvalGateSatisfied: !await hasPendingNativeGovernance({ - db: input.db, - companyId: run.companyId, - issueId: authoritativeIssue.id, - executionState: authoritativeIssue.executionState, - }), + governanceGate, agentId: run.agentId, + priorIssueStatus: authoritativeStatus(authoritativeIssue.status), }); const assessmentRow = await recordNativeWorkAssessment({ db: input.db, @@ -132,7 +339,9 @@ export async function finalizeNativeRun(input: { runId: run.id, turnId: resultRow.turnId, contractId: resultRow.completionContractId, + contractCanonicalSha256: contractRow.canonicalSha256, resultId: resultRow.id, + resultCanonicalSha256: resultRow.canonicalSha256, priorIssueStatus: authoritativeIssue.status, priorStatusVersion: Number(authoritativeIssue.statusVersion), priorDecisionId: authoritativeIssue.lastStatusDecisionId, @@ -140,6 +349,14 @@ export async function finalizeNativeRun(input: { assessment, supersedesAssessmentId, }); + await input.db.update(nativeRunFinalizations).set({ + phase: "arbitrating", + assessmentId: assessmentRow.id, + updatedAt: new Date(), + }).where(and( + eq(nativeRunFinalizations.runId, input.runId), + eq(nativeRunFinalizations.leaseOwner, claim.leaseOwner!), + )); try { const committed = await commitNativeStatusDecision({ db: input.db, @@ -149,18 +366,21 @@ export async function finalizeNativeRun(input: { assessmentId: assessmentRow.id, priorStatus: authoritativeIssue.status, priorStatusVersion: Number(authoritativeIssue.statusVersion), + priorDecisionId: authoritativeIssue.lastStatusDecisionId, decision, + failpoint: input.failpoint, }); + const now = new Date(); await input.db.update(heartbeatRuns).set({ ...(input.projectRunStatus ? { status: terminalState === "succeeded" ? "succeeded" : terminalState === "cancelled" ? "cancelled" : "failed", - finishedAt: new Date(), + finishedAt: now, } : {}), - nativePhase: "applied", - nativePhaseUpdatedAt: new Date(), + nativePhase: "committed", + nativePhaseUpdatedAt: now, resultJson: { ...record(run.resultJson), - finalizationPhase: "applied", + finalizationPhase: "committed", assessmentId: assessmentRow.id, decisionId: committed.decision.id, authoritativeDecision: decision.toStatus, @@ -170,17 +390,25 @@ export async function finalizeNativeRun(input: { statusVersionAfter: Number(committed.issue.statusVersion), workspaceFinalizeStatus: input.workspaceFinalizeStatus, }, - updatedAt: new Date(), + updatedAt: now, }).where(eq(heartbeatRuns.id, run.id)); - return { - ...coordinator, - phase: "applied", - assessmentId: assessmentRow.id, - decisionId: committed.decision.id, - }; + return { ...coordinator, phase: "committed", assessmentId: assessmentRow.id, decisionId: committed.decision.id }; } catch (error) { - if (!(error instanceof NativeStatusRaceError) || attempt === 1) throw error; - supersedesAssessmentId = assessmentRow.id; + if (error instanceof NativeStatusRaceError && attempt < 2) { + supersedesAssessmentId = assessmentRow.id; + continue; + } + return recordRetryableFailure({ + db: input.db, + run, + coordinator, + failureCode: error instanceof NativeStatusRaceError ? "status_cas_exhausted" : "side_effect_planning_failed", + message: error instanceof Error ? error.message : String(error), + nextAction: error instanceof NativeStatusRaceError + ? "Reassess against the latest authoritative issue status version." + : "Retry atomic status and liveness materialization from the persisted assessment.", + projectRunStatus: input.projectRunStatus, + }); } } throw new Error("native_finalization_status_race"); diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index d134ed2ee88..04020f0a3fa 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -25,6 +25,38 @@ const execution = { completionContract: { id: "contract", sha256: "sha" }, } as NativeExecutionInputV1; +function leaseDb(): Db { + const coordinator = { + runId: execution.binding.runId, + companyId: execution.binding.companyId, + issueId: execution.binding.issueId, + phase: "observed", + attempt: 0, + leaseOwner: null, + leaseExpiresAt: null, + resultId: null, + }; + const update = () => ({ + set: () => ({ where: async () => [] }), + }); + const tx = { + select: () => ({ + from: () => ({ + where: () => ({ + for: () => ({ + limit: () => Promise.resolve([coordinator]), + }), + }), + }), + }), + update, + }; + return { + transaction: async (operation: (transaction: Db) => Promise) => operation(tx as unknown as Db), + update, + } as unknown as Db; +} + describe("native session cancellation", () => { beforeEach(() => { state.cancel.mockReset(); @@ -49,14 +81,16 @@ describe("native session cancellation", () => { it("routes control-plane cancellation to the active normalized session and removes the handle", async () => { const running = executePaperclipNativeSession({ - db: {} as Db, + db: leaseDb(), execution, runnerInstanceId: "runner", }); await vi.waitFor(() => expect(state.release).toBeTypeOf("function")); await expect(cancelNativeSession(execution.binding.runId, "budget hard stop")).resolves.toBe(true); + await expect(cancelNativeSession(execution.binding.runId, "duplicate budget stop")).resolves.toBe(true); expect(state.cancel).toHaveBeenCalledWith({ reason: "budget hard stop" }); + expect(state.cancel).toHaveBeenCalledTimes(1); state.release?.(); await running; diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index f98a65e87ca..5771d353571 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -1,3 +1,4 @@ +import { randomUUID } from "node:crypto"; import type { AdapterExecutionResult } from "../../adapters/index.js"; import type { NativeFinalizationResult } from "@paperclipai/shared"; import type { NativeExecutionInputV1, NativeSession } from "@paperclipai/paperclip-runner"; @@ -6,15 +7,24 @@ import { executeNativeSession, } from "@paperclipai/paperclip-runner"; import type { Db } from "@paperclipai/db"; +import { and, eq } from "drizzle-orm"; +import { heartbeatRuns, nativeRunFinalizations } from "@paperclipai/db"; import { PaperclipControlPlanePort } from "./paperclip-control-plane-port.js"; -const activeNativeSessions = new Map(); +type ActiveNativeSession = { + session: NativeSession; + cancelRequested: boolean; +}; + +const activeNativeSessions = new Map(); export async function cancelNativeSession(runId: string, reason: string): Promise { - const session = activeNativeSessions.get(runId); - if (!session) return false; - if (session.cancel) await session.cancel({ reason }); - else if (session.interrupt) await session.interrupt({ reason }); + const active = activeNativeSessions.get(runId); + if (!active) return false; + if (active.cancelRequested) return true; + active.cancelRequested = true; + if (active.session.cancel) await active.session.cancel({ reason }); + else if (active.session.interrupt) await active.session.interrupt({ reason }); return true; } @@ -23,6 +33,40 @@ export async function executePaperclipNativeSession(input: { execution: NativeExecutionInputV1; runnerInstanceId: string; }): Promise { + const leaseOwner = `${input.runnerInstanceId}:${randomUUID()}`; + const leaseNow = new Date(); + const leaseExpiresAt = new Date(leaseNow.getTime() + 20 * 60_000); + await input.db.transaction(async (tx) => { + const coordinator = await tx.select().from(nativeRunFinalizations) + .where(and( + eq(nativeRunFinalizations.runId, input.execution.binding.runId), + eq(nativeRunFinalizations.companyId, input.execution.binding.companyId), + eq(nativeRunFinalizations.issueId, input.execution.binding.issueId), + )).for("update").limit(1).then((rows) => rows[0] ?? null); + if (!coordinator) throw new Error("native_finalization_coordinator_missing"); + if (["committed", "applied"].includes(coordinator.phase)) throw new Error("native_run_already_committed"); + if ( + coordinator.leaseOwner + && coordinator.leaseOwner !== leaseOwner + && coordinator.leaseExpiresAt + && coordinator.leaseExpiresAt > leaseNow + ) throw new Error("native_finalization_lease_busy"); + await tx.update(nativeRunFinalizations).set({ + phase: coordinator.resultId ? coordinator.phase : "observed", + attempt: coordinator.attempt + 1, + leaseOwner, + leaseExpiresAt, + failureCode: null, + failureDetail: null, + nextAttemptAt: null, + updatedAt: leaseNow, + }).where(eq(nativeRunFinalizations.runId, coordinator.runId)); + await tx.update(heartbeatRuns).set({ + nativePhase: coordinator.resultId ? coordinator.phase : "observed", + nativePhaseUpdatedAt: leaseNow, + updatedAt: leaseNow, + }).where(eq(heartbeatRuns.id, coordinator.runId)); + }); const controlPlaneInstanceId = `${input.runnerInstanceId}:control`; const controlPlane = new PaperclipControlPlanePort(input.db, { companyId: input.execution.binding.companyId, @@ -34,17 +78,50 @@ export async function executePaperclipNativeSession(input: { sourceInstanceId: input.runnerInstanceId, controlPlaneSourceInstanceId: controlPlaneInstanceId, }); - const native = await executeNativeSession({ - input: input.execution, - backend: createCodexNativeSessionBackend(input.execution, { runnerInstanceId: input.runnerInstanceId }), - controlPlane, - runnerInstanceId: input.runnerInstanceId, - controlPlaneInstanceId, - onSession: (session) => { - if (session) activeNativeSessions.set(input.execution.binding.runId, session); - else activeNativeSessions.delete(input.execution.binding.runId); - }, - }); + let native: Awaited>; + try { + native = await executeNativeSession({ + input: input.execution, + backend: createCodexNativeSessionBackend(input.execution, { runnerInstanceId: input.runnerInstanceId }), + controlPlane, + runnerInstanceId: input.runnerInstanceId, + controlPlaneInstanceId, + onSession: (session) => { + if (session) activeNativeSessions.set(input.execution.binding.runId, { + session, + cancelRequested: false, + }); + else activeNativeSessions.delete(input.execution.binding.runId); + }, + }); + } catch (error) { + const now = new Date(); + await input.db.update(nativeRunFinalizations).set({ + phase: "retryable_failure", + leaseOwner: null, + leaseExpiresAt: null, + failureCode: "native_session_interrupted", + failureDetail: { + message: error instanceof Error ? error.message.slice(0, 2_000) : String(error).slice(0, 2_000), + recoveryOwner: { kind: "agent", agentId: input.execution.binding.agentId }, + nextAction: "Resume the persisted native provider session after the retry delay.", + }, + nextAttemptAt: new Date(now.getTime() + 30_000), + updatedAt: now, + }).where(and( + eq(nativeRunFinalizations.runId, input.execution.binding.runId), + eq(nativeRunFinalizations.leaseOwner, leaseOwner), + )); + throw error; + } + await input.db.update(nativeRunFinalizations).set({ + leaseOwner: null, + leaseExpiresAt: null, + updatedAt: new Date(), + }).where(and( + eq(nativeRunFinalizations.runId, input.execution.binding.runId), + eq(nativeRunFinalizations.leaseOwner, leaseOwner), + )); const finalization: NativeFinalizationResult = { schema: "paperclip.native-finalization.v1", runtimeMode: "native", diff --git a/server/src/services/native-runtime/paperclip-control-plane-port.test.ts b/server/src/services/native-runtime/paperclip-control-plane-port.test.ts index ec015f5179e..479c194cbc0 100644 --- a/server/src/services/native-runtime/paperclip-control-plane-port.test.ts +++ b/server/src/services/native-runtime/paperclip-control-plane-port.test.ts @@ -2,13 +2,16 @@ import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { eq } from "drizzle-orm"; import { activityLog, + agentWakeupRequests, agents, companies, completionContracts, createDb, heartbeatRunEvents, heartbeatRuns, + issueRecoveryActions, issueThreadInteractions, + issueWorkProducts, issues, nativeRunFinalizations, nativeRunResults, @@ -24,19 +27,15 @@ import { runControlPlanePortConformance, type NativeSessionBackend, type PrpEvent, + type PrpStructuredRunResult, + type PrpTerminalState, } from "@paperclipai/paperclip-runner"; -import { - getEmbeddedPostgresTestSupport, - startEmbeddedPostgresTestDatabase, -} from "../../__tests__/helpers/embedded-postgres.js"; +import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; import { PaperclipControlPlanePort } from "./paperclip-control-plane-port.js"; import { finalizeNativeRun } from "./native-run-finalizer.js"; import { buildNativeExecutionInput } from "./native-execution-input.js"; -const support = await getEmbeddedPostgresTestSupport(); -const describeDatabase = support.supported ? describe : describe.skip; - -describeDatabase("PaperclipControlPlanePort conformance", () => { +describe("PaperclipControlPlanePort conformance", () => { let temporary: Awaited> | null = null; let db: ReturnType; const contractId = "00000000-0000-4000-8000-000000000004"; @@ -44,6 +43,7 @@ describeDatabase("PaperclipControlPlanePort conformance", () => { const taskIssueId = "00000000-0000-4000-8000-000000000008"; const taskContractId = "00000000-0000-4000-8000-000000000009"; const taskRunId = "00000000-0000-4000-8000-000000000010"; + const taskWorkProductId = "00000000-0000-4000-8000-000000000017"; const workspaceFailureIssueId = "00000000-0000-4000-8000-000000000011"; const workspaceFailureContractId = "00000000-0000-4000-8000-000000000012"; const workspaceFailureRunId = "00000000-0000-4000-8000-000000000013"; @@ -68,6 +68,7 @@ describeDatabase("PaperclipControlPlanePort conformance", () => { companyId: identity.companyId, title: "Native conformance", status: "in_progress", + assigneeAgentId: identity.agentId, workMode: "standard", }); await db.insert(completionContracts).values({ @@ -80,7 +81,11 @@ describeDatabase("PaperclipControlPlanePort conformance", () => { risk: "standard", completionAuthority: "server_arbiter", incompleteCriteriaPolicy: "preserve_non_terminal", - contractJson: { objective: "Conformance" }, + contractJson: { + revision: "phase6-v1", + objective: "Conformance", + criteria: [{ id: "objective", requirement: "Complete conformance" }], + }, canonicalSha256: contractSha, createdByActorType: "system", createdByActorId: "test", @@ -100,6 +105,7 @@ describeDatabase("PaperclipControlPlanePort conformance", () => { companyId: identity.companyId, title: "Complete one native task", status: "in_progress", + assigneeAgentId: identity.agentId, workMode: "standard", }); await db.insert(completionContracts).values({ @@ -112,7 +118,11 @@ describeDatabase("PaperclipControlPlanePort conformance", () => { risk: "standard", completionAuthority: "server_arbiter", incompleteCriteriaPolicy: "preserve_non_terminal", - contractJson: { objective: "Complete one native task" }, + contractJson: { + revision: "phase6-v1", + objective: "Complete one native task", + criteria: [{ id: "objective", requirement: "Complete the task" }], + }, canonicalSha256: "phase6-task-contract", createdByActorType: "system", createdByActorId: "test", @@ -129,11 +139,23 @@ describeDatabase("PaperclipControlPlanePort conformance", () => { completionContractSha256: "phase6-task-contract", contextSnapshot: { issueId: taskIssueId }, }); + await db.insert(issueWorkProducts).values({ + id: taskWorkProductId, + companyId: identity.companyId, + issueId: taskIssueId, + type: "artifact", + provider: "paperclip", + title: "Verified native task result", + status: "ready_for_review", + reviewState: "approved", + createdByRunId: taskRunId, + }); await db.insert(issues).values({ id: workspaceFailureIssueId, companyId: identity.companyId, title: "Preserve status after workspace failure", status: "in_progress", + assigneeAgentId: identity.agentId, workMode: "standard", }); await db.insert(completionContracts).values({ @@ -146,7 +168,11 @@ describeDatabase("PaperclipControlPlanePort conformance", () => { risk: "standard", completionAuthority: "server_arbiter", incompleteCriteriaPolicy: "preserve_non_terminal", - contractJson: { objective: "Preserve status after workspace failure" }, + contractJson: { + revision: "phase6-v1", + objective: "Preserve status after workspace failure", + criteria: [{ id: "objective", requirement: "Preserve the result" }], + }, canonicalSha256: "phase6-workspace-failure-contract", createdByActorType: "system", createdByActorId: "test", @@ -178,7 +204,11 @@ describeDatabase("PaperclipControlPlanePort conformance", () => { risk: "standard", completionAuthority: "server_arbiter", incompleteCriteriaPolicy: "preserve_non_terminal", - contractJson: { objective: "Respect pending governance" }, + contractJson: { + revision: "phase6-v1", + objective: "Respect pending governance", + criteria: [{ id: "objective", requirement: "Respect pending governance" }], + }, canonicalSha256: "phase6-governance-contract", createdByActorType: "system", createdByActorId: "test", @@ -205,12 +235,14 @@ describeDatabase("PaperclipControlPlanePort conformance", () => { afterAll(async () => { if (temporary) { await db.delete(activityLog); + await db.delete(agentWakeupRequests); await db.delete(statusDecisionEffects); await db.delete(statusDecisions); await db.delete(workAssessments); await db.delete(nativeRunFinalizations); await db.delete(nativeRunResults); await db.delete(issueThreadInteractions); + await db.delete(issueWorkProducts); await db.delete(heartbeatRunEvents); await db.delete(heartbeatRuns); await db.delete(completionContracts); @@ -240,24 +272,37 @@ describeDatabase("PaperclipControlPlanePort conformance", () => { terminalReplayIdempotent: true, }); await expect(db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, identity.runId))).resolves.toHaveLength(1); - await finalizeNativeRun({ db, runId: identity.runId, workspaceFinalizeStatus: "succeeded" }); + await finalizeNativeRun({ + db, + runId: identity.runId, + workspaceFinalizeStatus: "succeeded", + }); + await expect(db.select().from(nativeRunFinalizations) + .where(eq(nativeRunFinalizations.runId, identity.runId))).resolves.toEqual([ + expect.objectContaining({ phase: "committed" }), + ]); await expect(db.select().from(issues).where(eq(issues.id, identity.issueId))).resolves.toEqual([ - expect.objectContaining({ status: "done", statusVersion: 1 }), + expect.objectContaining({ status: "in_progress", statusVersion: 1 }), ]); await expect(db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, identity.runId))).resolves.toEqual([ - expect.objectContaining({ phase: "applied" }), + expect.objectContaining({ phase: "committed" }), ]); await expect(db.select().from(statusDecisions).where(eq(statusDecisions.issueId, identity.issueId))).resolves.toEqual([ - expect.objectContaining({ toStatus: "done", applicationState: "applied" }), - ]); - await expect(db.select().from(activityLog).where(eq(activityLog.entityId, identity.issueId))).resolves.toEqual([ - expect.objectContaining({ action: "issue.native_status_decision_applied" }), + expect.objectContaining({ toStatus: "in_progress", reasonCode: "native_completion_incomplete", applicationState: "applied" }), ]); + await expect(db.select().from(activityLog).where(eq(activityLog.entityId, identity.issueId))).resolves.toEqual( + expect.arrayContaining([expect.objectContaining({ action: "issue.updated" })]), + ); }); it("completes one selected Paperclip task through the public package session contract", async () => { const identity = CONTROL_PLANE_CONFORMANCE_OPEN.identity; const sessionId = "session-phase6-paperclip-task"; + const evidenceRef = `work_product:${taskWorkProductId}`; + const taskResult = structuredClone(CONTROL_PLANE_CONFORMANCE_RESULT); + taskResult.completionClaim.criteria[0]!.evidenceRefs = [evidenceRef]; + taskResult.evidence = [{ kind: "work_product", ref: evidenceRef }]; + taskResult.verification[0]!.artifactRef = evidenceRef; const event = (sourceSeq: number, eventType: PrpEvent["eventType"], payload: Record): PrpEvent => ({ schema: "paperclip.prp.event.v1", sourceEventId: `phase6-task:${sourceSeq}`, @@ -274,7 +319,7 @@ describeDatabase("PaperclipControlPlanePort conformance", () => { payload, }); const events = [ - event(1, "run.result.proposed", CONTROL_PLANE_CONFORMANCE_RESULT), + event(1, "run.result.proposed", taskResult), event(2, "turn.completed", {}), ]; const backend: NativeSessionBackend = { @@ -293,7 +338,7 @@ describeDatabase("PaperclipControlPlanePort conformance", () => { async *events() { yield* events; }, async startTurn() { return { turnId: "turn-phase6-paperclip-task" }; }, async cancel() {}, - async result() { return { result: CONTROL_PLANE_CONFORMANCE_RESULT, terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, turnId: "turn-phase6-paperclip-task" }; }, + async result() { return { result: taskResult, terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, turnId: "turn-phase6-paperclip-task" }; }, async snapshot() { return { backendKind: "mock", sessionId, identity: input.identity, providerSessionId: "provider-phase6-paperclip-task" }; }, async close() {}, }; @@ -332,9 +377,30 @@ describeDatabase("PaperclipControlPlanePort conformance", () => { }); expect(completed.terminal.runTerminalState).toBe("succeeded"); await finalizeNativeRun({ db, runId: taskRunId, workspaceFinalizeStatus: "succeeded" }); + await expect(port.completeRun({ + result: taskResult, + terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, + turnId: "turn-phase6-paperclip-task", + callerResultId: `phase6-scripted-runner:${taskRunId}:result`, + callerDedupeKey: `${taskRunId}:phase6-task-contract`, + })).resolves.toBeUndefined(); + await expect(port.completeRun({ + result: { ...structuredClone(taskResult), summary: "Conflicting changed result bytes" }, + terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, + turnId: "turn-phase6-paperclip-task", + callerResultId: `phase6-scripted-runner:${taskRunId}:result`, + callerDedupeKey: `${taskRunId}:phase6-task-contract`, + })).rejects.toThrow("structured_result_replay_conflict"); + await Promise.all([ + finalizeNativeRun({ db, runId: taskRunId, workspaceFinalizeStatus: "succeeded" }), + finalizeNativeRun({ db, runId: taskRunId, workspaceFinalizeStatus: "succeeded" }), + ]); await expect(db.select().from(issues).where(eq(issues.id, taskIssueId))).resolves.toEqual([ expect.objectContaining({ status: "done", statusVersion: 1 }), ]); + await expect(db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, taskRunId))).resolves.toHaveLength(1); + await expect(db.select().from(workAssessments).where(eq(workAssessments.runId, taskRunId))).resolves.toHaveLength(1); + await expect(db.select().from(statusDecisions).where(eq(statusDecisions.issueId, taskIssueId))).resolves.toHaveLength(1); }); it("fails closed when the bound company does not own the run", async () => { @@ -376,13 +442,85 @@ describeDatabase("PaperclipControlPlanePort conformance", () => { }); await finalizeNativeRun({ db, runId: governanceRunId, workspaceFinalizeStatus: "succeeded" }); await expect(db.select().from(issues).where(eq(issues.id, governanceIssueId))).resolves.toEqual([ - expect.objectContaining({ status: "in_progress" }), + expect.objectContaining({ status: "in_review" }), ]); await expect(db.select().from(statusDecisions).where(eq(statusDecisions.issueId, governanceIssueId))).resolves.toEqual([ - expect.objectContaining({ toStatus: "in_progress", reasonCode: "approval_gate_pending" }), + expect.objectContaining({ toStatus: "in_review", reasonCode: "governance_gate_pending" }), ]); }); + it("P6-23 materializes native review through the authorized interaction service", async () => { + const identity = CONTROL_PLANE_CONFORMANCE_OPEN.identity; + const issueId = "30000000-0000-4000-8000-000000000024"; + const localContractId = "31000000-0000-4000-8000-000000000024"; + const runId = "32000000-0000-4000-8000-000000000024"; + await db.insert(issues).values({ + id: issueId, + companyId: identity.companyId, + title: "Native review interaction", + status: "in_progress", + assigneeAgentId: identity.agentId, + workMode: "standard", + }); + await db.insert(completionContracts).values({ + id: localContractId, + companyId: identity.companyId, + issueId, + revision: 1, + schemaVersion: "paperclip.completion-contract.v1", + policyVersion: "phase6-v1", + risk: "standard", + completionAuthority: "server_arbiter", + incompleteCriteriaPolicy: "preserve_non_terminal", + contractJson: { revision: "phase6-v1", objective: "Review", criteria: [{ id: "objective", requirement: "Review" }] }, + canonicalSha256: "native-review-contract", + createdByActorType: "system", + createdByActorId: "test", + }); + await db.insert(heartbeatRuns).values({ + id: runId, + companyId: identity.companyId, + agentId: identity.agentId, + status: "running", + runtimeMode: "native", + completionContractId: localContractId, + completionContractSha256: "native-review-contract", + contextSnapshot: { issueId }, + }); + const port = new PaperclipControlPlanePort(db, { + companyId: identity.companyId, + issueId, + runId, + agentId: identity.agentId, + completionContractId: localContractId, + completionContractSha256: "native-review-contract", + sourceInstanceId: "native-review-runner", + controlPlaneSourceInstanceId: "native-review-control", + }); + await port.openRun({ + identity: { ...identity, issueId, runId }, + backendKind: "mock", + sourceInstanceId: "native-review-runner", + }); + const result = { ...structuredClone(CONTROL_PLANE_CONFORMANCE_RESULT), reportedWorkDisposition: "needs_review" as const }; + await port.completeRun({ + result, + terminal: { ...CONTROL_PLANE_CONFORMANCE_TERMINAL, reportedWorkDisposition: "needs_review" }, + callerResultId: "native-review-result", + }); + await finalizeNativeRun({ db, runId, workspaceFinalizeStatus: "succeeded" }); + + await expect(db.select().from(issues).where(eq(issues.id, issueId))).resolves.toEqual([ + expect.objectContaining({ status: "in_review", statusVersion: 1 }), + ]); + await expect(db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.issueId, issueId))).resolves.toEqual([ + expect.objectContaining({ kind: "request_confirmation", status: "pending", sourceRunId: runId }), + ]); + await expect(db.select().from(statusDecisionEffects).where(eq(statusDecisionEffects.issueId, issueId))).resolves.toEqual( + expect.arrayContaining([expect.objectContaining({ effectKind: "create_review_interaction", targetType: "issue_thread_interaction" })]), + ); + }); + it("preserves the result and issue status when workspace finalization fails", async () => { const identity = CONTROL_PLANE_CONFORMANCE_OPEN.identity; const port = new PaperclipControlPlanePort(db, { @@ -415,8 +553,125 @@ describeDatabase("PaperclipControlPlanePort conformance", () => { expect.objectContaining({ status: "in_progress", statusVersion: 0, lastStatusDecisionId: null }), ]); await expect(db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, workspaceFailureRunId))).resolves.toEqual([ - expect.objectContaining({ status: "failed", nativePhase: "workspace_failed" }), + expect.objectContaining({ status: "failed", nativePhase: "retryable_failure" }), ]); await expect(db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, workspaceFailureRunId))).resolves.toHaveLength(1); }); + + it("LIVE-01..04 rolls back status, decision, and liveness rows at every materialization failpoint", async () => { + const identity = CONTROL_PLANE_CONFORMANCE_OPEN.identity; + const cases: Array<{ + suffix: number; + failpoint: "interaction_materialization" | "continuation_materialization" | "blocker_materialization" | "status_projection"; + result: PrpStructuredRunResult; + }> = [ + { + suffix: 20, + failpoint: "interaction_materialization", + result: { ...structuredClone(CONTROL_PLANE_CONFORMANCE_RESULT), reportedWorkDisposition: "needs_review" }, + }, + { + suffix: 21, + failpoint: "continuation_materialization", + result: { + ...structuredClone(CONTROL_PLANE_CONFORMANCE_RESULT), + reportedWorkDisposition: "yielded", + continuation: { kind: "same_agent", summary: "Continue after recovery", idempotencyKey: "live-continue" }, + }, + }, + { + suffix: 22, + failpoint: "blocker_materialization", + result: { + ...structuredClone(CONTROL_PLANE_CONFORMANCE_RESULT), + reportedWorkDisposition: "blocked", + blocker: { reasonCode: "access", owner: { kind: "board" }, unblockAction: "Approve access", scope: "task_wide" }, + }, + }, + { + suffix: 23, + failpoint: "status_projection", + result: { ...structuredClone(CONTROL_PLANE_CONFORMANCE_RESULT), reportedWorkDisposition: "needs_review" }, + }, + ]; + for (const entry of cases) { + const value = String(entry.suffix).padStart(12, "0"); + const issueId = `30000000-0000-4000-8000-${value}`; + const contractId = `31000000-0000-4000-8000-${value}`; + const runId = `32000000-0000-4000-8000-${value}`; + await db.insert(issues).values({ + id: issueId, + companyId: identity.companyId, + title: `Atomic liveness ${entry.failpoint}`, + status: "in_progress", + assigneeAgentId: identity.agentId, + workMode: "standard", + }); + await db.insert(completionContracts).values({ + id: contractId, + companyId: identity.companyId, + issueId, + revision: 1, + schemaVersion: "paperclip.completion-contract.v1", + policyVersion: "phase6-v1", + risk: "standard", + completionAuthority: "server_arbiter", + incompleteCriteriaPolicy: "preserve_non_terminal", + contractJson: { revision: "phase6-v1", objective: "Atomic liveness", criteria: [{ id: "objective", requirement: "Stay atomic" }] }, + canonicalSha256: `contract-${entry.suffix}`, + createdByActorType: "system", + createdByActorId: "test", + }); + await db.insert(heartbeatRuns).values({ + id: runId, + companyId: identity.companyId, + agentId: identity.agentId, + status: "running", + runtimeMode: "native", + completionContractId: contractId, + completionContractSha256: `contract-${entry.suffix}`, + contextSnapshot: { issueId }, + }); + const terminal: PrpTerminalState = { + ...CONTROL_PLANE_CONFORMANCE_TERMINAL, + reportedWorkDisposition: entry.result.reportedWorkDisposition, + }; + const port = new PaperclipControlPlanePort(db, { + companyId: identity.companyId, + issueId, + runId, + agentId: identity.agentId, + completionContractId: contractId, + completionContractSha256: `contract-${entry.suffix}`, + sourceInstanceId: `atomic-runner-${entry.suffix}`, + controlPlaneSourceInstanceId: `atomic-control-${entry.suffix}`, + }); + await port.openRun({ + identity: { ...identity, issueId, runId }, + backendKind: "mock", + sourceInstanceId: `atomic-runner-${entry.suffix}`, + }); + await port.completeRun({ result: entry.result, terminal, callerResultId: `atomic-result-${entry.suffix}` }); + await expect(finalizeNativeRun({ + db, + runId, + workspaceFinalizeStatus: "succeeded", + failpoint: entry.failpoint, + })).resolves.toEqual(expect.objectContaining({ phase: "retryable_failure" })); + await expect(db.select().from(issues).where(eq(issues.id, issueId))).resolves.toEqual([ + expect.objectContaining({ status: "in_progress", statusVersion: 0, lastStatusDecisionId: null }), + ]); + await expect(db.select().from(statusDecisions).where(eq(statusDecisions.issueId, issueId))).resolves.toHaveLength(0); + await expect(db.select().from(statusDecisionEffects).where(eq(statusDecisionEffects.issueId, issueId))).resolves.toHaveLength(0); + await expect(db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.issueId, issueId))).resolves.toHaveLength(0); + await expect(db.select().from(agentWakeupRequests) + .where(eq(agentWakeupRequests.companyId, identity.companyId))).resolves.not.toEqual( + expect.arrayContaining([expect.objectContaining({ payload: expect.objectContaining({ issueId }) })]), + ); + await expect(db.select().from(issueRecoveryActions) + .where(eq(issueRecoveryActions.sourceIssueId, issueId))).resolves.toEqual([ + expect.objectContaining({ status: "active", cause: "side_effect_planning_failed" }), + ]); + } + }); }); diff --git a/server/src/services/native-runtime/paperclip-control-plane-port.ts b/server/src/services/native-runtime/paperclip-control-plane-port.ts index ab026d909f1..8afbff1e4b6 100644 --- a/server/src/services/native-runtime/paperclip-control-plane-port.ts +++ b/server/src/services/native-runtime/paperclip-control-plane-port.ts @@ -12,6 +12,7 @@ import type { NativeRunEvent, NativeRunResult, OpenControlPlaneRunInput, + PersistedNativeSession, PrpEvent, PrpTerminalState, ReplayControlPlaneEventsInput, @@ -42,6 +43,12 @@ function isCompleteInput(value: NativeRunResult | CompleteControlPlaneRunInput): return "result" in value && "terminal" in value; } +function record(value: unknown): Record { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? value as Record + : {}; +} + function assertTerminal(value: unknown): asserts value is PrpTerminalState { const terminal = value as Partial | null; if ( @@ -90,6 +97,69 @@ export class PaperclipControlPlanePort implements ControlPlanePort { this.#sessionId = identity.sessionId; } + async loadSessionCheckpoint(): Promise { + const run = await this.#db.select({ + companyId: heartbeatRuns.companyId, + agentId: heartbeatRuns.agentId, + runtimeMode: heartbeatRuns.runtimeMode, + nativeSessionId: heartbeatRuns.nativeSessionId, + runnerProfileJson: heartbeatRuns.runnerProfileJson, + }).from(heartbeatRuns).where(eq(heartbeatRuns.id, this.#binding.runId)) + .limit(1).then((rows) => rows[0] ?? null); + if ( + !run + || run.companyId !== this.#binding.companyId + || run.agentId !== this.#binding.agentId + || run.runtimeMode !== "native" + ) throw new Error("native_session_checkpoint_binding_mismatch"); + const candidate = record(run.runnerProfileJson).sessionCheckpoint; + if (candidate === undefined || candidate === null) return null; + const snapshot = record(candidate) as Partial; + const identity = record(snapshot.identity); + if ( + typeof snapshot.sessionId !== "string" + || identity.companyId !== this.#binding.companyId + || identity.issueId !== this.#binding.issueId + || identity.runId !== this.#binding.runId + || identity.agentId !== this.#binding.agentId + || identity.sessionId !== run.nativeSessionId + ) throw new Error("native_session_checkpoint_invalid"); + return structuredClone(snapshot as PersistedNativeSession); + } + + async checkpointSession(snapshot: PersistedNativeSession): Promise { + const identity = snapshot.identity; + if ( + identity.companyId !== this.#binding.companyId + || identity.issueId !== this.#binding.issueId + || identity.runId !== this.#binding.runId + || identity.agentId !== this.#binding.agentId + || this.#sessionId !== identity.sessionId + ) throw new Error("native_session_checkpoint_binding_mismatch"); + await this.#db.transaction(async (tx) => { + const run = await tx.select().from(heartbeatRuns) + .where(eq(heartbeatRuns.id, this.#binding.runId)).for("update").limit(1) + .then((rows) => rows[0] ?? null); + if ( + !run + || run.companyId !== this.#binding.companyId + || run.agentId !== this.#binding.agentId + || run.runtimeMode !== "native" + || run.nativeSessionId !== identity.sessionId + ) throw new Error("native_session_checkpoint_binding_mismatch"); + await tx.update(heartbeatRuns).set({ + runnerProfileJson: { + ...record(run.runnerProfileJson), + sessionCheckpoint: structuredClone(snapshot) as unknown as Record, + }, + sessionIdAfter: snapshot.providerSessionId ?? snapshot.sessionId, + nativePhase: snapshot.semanticResult ? "workspace_finalizing" : "observed", + nativePhaseUpdatedAt: new Date(), + updatedAt: new Date(), + }).where(eq(heartbeatRuns.id, this.#binding.runId)); + }); + } + async appendEvent(value: NativeRunEvent | PrpEvent) { if (!isPrpEvent(value)) throw new Error("native_legacy_event_not_supported"); const validated = validatePrpEvent(value); @@ -197,7 +267,7 @@ export class PaperclipControlPlanePort implements ControlPlanePort { .where(and(eq(nativeRunResults.runId, this.#binding.runId), or(...callerConditions))) .limit(1).then((rows) => rows[0] ?? null); if (existing) { - if (existing.canonicalSha256 !== canonicalSha256) throw new Error("native_result_idempotency_conflict"); + if (existing.canonicalSha256 !== canonicalSha256) throw new Error("structured_result_replay_conflict"); return; } const resultJson = { @@ -222,14 +292,21 @@ export class PaperclipControlPlanePort implements ControlPlanePort { runId: this.#binding.runId, companyId: this.#binding.companyId, issueId: this.#binding.issueId, - phase: "result_persisted", + phase: "workspace_finalizing", resultId: inserted.id, }).onConflictDoUpdate({ target: nativeRunFinalizations.runId, - set: { phase: "result_persisted", resultId: inserted.id, failureCode: null, failureDetail: null, updatedAt: new Date() }, + set: { + phase: "workspace_finalizing", + resultId: inserted.id, + failureCode: null, + failureDetail: null, + nextAttemptAt: null, + updatedAt: new Date(), + }, }); await tx.update(heartbeatRuns).set({ - nativePhase: "result_persisted", + nativePhase: "workspace_finalizing", nativePhaseUpdatedAt: new Date(), updatedAt: new Date(), }).where(eq(heartbeatRuns.id, this.#binding.runId)); diff --git a/server/src/services/native-runtime/status-arbiter.test.ts b/server/src/services/native-runtime/status-arbiter.test.ts index 4dd6676526f..08978db0a36 100644 --- a/server/src/services/native-runtime/status-arbiter.test.ts +++ b/server/src/services/native-runtime/status-arbiter.test.ts @@ -1,72 +1,97 @@ import { describe, expect, it } from "vitest"; -import { classifyNativeEvidence } from "./evidence-classifier.js"; +import type { NativeEvidenceAssessment } from "./evidence-classifier.js"; import { arbitrateNativeStatus } from "./status-arbiter.js"; -function result(overrides: Record = {}) { +function assessment(overrides: Partial = {}): NativeEvidenceAssessment { return { - reportedWorkDisposition: "done", + objectiveSatisfied: true, + allCriteriaSatisfied: true, + verificationPassed: true, + hasBlockingRemainingWork: false, + reportedDisposition: "done", summary: "Complete", - completionClaim: { - objectiveSatisfied: true, - criteria: [{ criterionId: "objective", status: "satisfied", evidenceRefs: ["test"] }], - remainingWork: [], - }, - verification: [{ commandOrCheck: "test", status: "passed" }], + contractRevisionMatches: true, + criterionAssessments: [], + verificationAssessments: [], + acceptedEvidenceRefs: ["event:2"], + missingRequirements: [], + rejectedEvidence: [], + unverifiableEvidence: [], + blocker: null, + continuation: null, + attentionRequests: [], ...overrides, }; } +function arbitrate(overrides: Partial[0]> = {}) { + return arbitrateNativeStatus({ + assessment: assessment(), + terminalState: "succeeded", + workspaceFinalizeStatus: "succeeded", + agentId: "agent", + priorIssueStatus: "in_progress", + ...overrides, + }); +} + describe("native status authority", () => { - it("marks done only from successful finalization and complete evidence", () => { - const assessment = classifyNativeEvidence(result()); - expect(arbitrateNativeStatus({ - assessment, - terminalState: "succeeded", - workspaceFinalizeStatus: "succeeded", - agentId: "agent", - }).toStatus).toBe("done"); + it("marks done only from successful finalization and complete durable evidence", () => { + expect(arbitrate()).toEqual(expect.objectContaining({ + toStatus: "done", + reasonCode: "completion_contract_satisfied", + effects: [{ kind: "release_checkout" }], + })); + expect(arbitrate({ + assessment: assessment({ verificationPassed: false, missingRequirements: ["test"] }), + }).toStatus).toBe("in_progress"); }); - it("keeps incomplete, review, cancelled, and yielded work non-terminal", () => { - for (const reportedWorkDisposition of ["needs_review", "yielded"] as const) { - expect(arbitrateNativeStatus({ - assessment: classifyNativeEvidence(result({ reportedWorkDisposition })), - terminalState: "succeeded", - workspaceFinalizeStatus: "succeeded", - agentId: "agent", - }).toStatus).toBe("in_progress"); - } - expect(arbitrateNativeStatus({ - assessment: classifyNativeEvidence(result()), - terminalState: "cancelled", - workspaceFinalizeStatus: "succeeded", - agentId: "agent", - }).toStatus).toBe("in_progress"); - expect(arbitrateNativeStatus({ - assessment: classifyNativeEvidence(result()), - terminalState: "succeeded", - workspaceFinalizeStatus: "succeeded", - approvalGateSatisfied: false, - agentId: "agent", + it("creates explicit liveness paths for review, continuation, cancellation, and governance", () => { + expect(arbitrate({ + assessment: assessment({ reportedDisposition: "needs_review" }), + })).toEqual(expect.objectContaining({ + toStatus: "in_review", + effects: [expect.objectContaining({ kind: "create_review_interaction" })], + })); + expect(arbitrate({ + assessment: assessment({ + reportedDisposition: "yielded", + continuation: { kind: "retry", summary: "Retry the task", idempotencyKey: "retry-task" }, + }), })).toEqual(expect.objectContaining({ toStatus: "in_progress", - reasonCode: "approval_gate_pending", + effects: [expect.objectContaining({ kind: "enqueue_continuation", continuationKind: "retry" })], + })); + expect(arbitrate({ terminalState: "cancelled" })).toEqual(expect.objectContaining({ + toStatus: "in_progress", + effects: [expect.objectContaining({ kind: "record_recovery", cause: "native_run_cancelled" })], + })); + expect(arbitrate({ governanceGate: { kind: "interaction", id: "interaction" } })).toEqual(expect.objectContaining({ + toStatus: "in_review", + reasonCode: "governance_gate_pending", + effects: [{ kind: "bind_governance", gate: { kind: "interaction", id: "interaction" } }], })); }); - it("blocks only on a first-class blocker or failed workspace finalization", () => { - const explicitBlocker = classifyNativeEvidence(result({ - reportedWorkDisposition: "blocked", - blocker: { owner: { kind: "board" }, unblockAction: "Approve access" }, - })); - expect(arbitrateNativeStatus({ - assessment: explicitBlocker, - terminalState: "succeeded", - workspaceFinalizeStatus: "succeeded", - agentId: "agent", + it("blocks only for a task-wide blocker with a named owner and action", () => { + expect(arbitrate({ + assessment: assessment({ + reportedDisposition: "blocked", + blocker: { boardOwned: true, scope: "task_wide", unblockAction: "Approve access" }, + }), })).toEqual(expect.objectContaining({ toStatus: "blocked", unblockDescriptor: { owner: "board", action: "Approve access" }, + effects: [{ kind: "bind_blocker", owner: "board", action: "Approve access" }], + })); + }); + + it("preserves authoritative terminal statuses", () => { + expect(arbitrate({ priorIssueStatus: "done" })).toEqual(expect.objectContaining({ + toStatus: "done", + reasonCode: "terminal_status_preserved", + effects: [], })); }); }); diff --git a/server/src/services/native-runtime/status-arbiter.ts b/server/src/services/native-runtime/status-arbiter.ts index 33c017b2ecc..10bd31b2daf 100644 --- a/server/src/services/native-runtime/status-arbiter.ts +++ b/server/src/services/native-runtime/status-arbiter.ts @@ -1,12 +1,41 @@ import type { NativeEvidenceAssessment } from "./evidence-classifier.js"; -export const NATIVE_STATUS_ARBITER_POLICY_VERSION = "phase6-v1"; +export const NATIVE_STATUS_ARBITER_POLICY_VERSION = "phase6-v2"; + +export type NativeAuthoritativeIssueStatus = + | "backlog" + | "todo" + | "in_progress" + | "in_review" + | "blocked" + | "done" + | "cancelled"; + +export type NativeGovernanceGate = { + kind: "approval" | "interaction" | "execution_stage"; + id: string; +}; + +export type NativeStatusEffect = + | { kind: "bind_governance"; gate: NativeGovernanceGate } + | { kind: "create_review_interaction"; prompt: string } + | { + kind: "enqueue_continuation"; + continuationKind: "same_agent" | "retry" | "delegated_issue" | "response_wake" | "monitor"; + summary: string; + idempotencyKey: string; + agentId: string; + } + | { kind: "bind_blocker"; owner: { agentId: string } | "board"; action: string } + | { kind: "record_recovery"; cause: string; nextAction: string; agentId: string } + | { kind: "release_checkout" }; export interface NativeStatusDecision { policyVersion: typeof NATIVE_STATUS_ARBITER_POLICY_VERSION; - toStatus: "in_progress" | "done" | "blocked"; + toStatus: NativeAuthoritativeIssueStatus; reasonCode: string; unblockDescriptor: { owner: { agentId: string } | "board"; action: string } | null; + effects: NativeStatusEffect[]; } /** Pure authority boundary: model prose is evidence, never a status command. */ @@ -14,31 +43,55 @@ export function arbitrateNativeStatus(input: { assessment: NativeEvidenceAssessment; terminalState: "succeeded" | "failed" | "cancelled"; workspaceFinalizeStatus: "succeeded" | "failed"; - approvalGateSatisfied?: boolean; + governanceGate?: NativeGovernanceGate | null; agentId: string; + priorIssueStatus: NativeAuthoritativeIssueStatus; }): NativeStatusDecision { + if (["done", "cancelled"].includes(input.priorIssueStatus)) { + return { + policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, + toStatus: input.priorIssueStatus, + reasonCode: "terminal_status_preserved", + unblockDescriptor: null, + effects: [], + }; + } if (input.workspaceFinalizeStatus !== "succeeded") { return { policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, - toStatus: "blocked", - reasonCode: "workspace_finalization_failed", - unblockDescriptor: { owner: { agentId: input.agentId }, action: "Repair and re-verify workspace finalization." }, + toStatus: input.priorIssueStatus, + reasonCode: "workspace_finalization_retry_required", + unblockDescriptor: null, + effects: [{ + kind: "record_recovery", + cause: "workspace_finalization_failed", + nextAction: "Repair and re-run workspace finalization for the persisted native result.", + agentId: input.agentId, + }], }; } if (input.terminalState !== "succeeded") { + const cause = input.terminalState === "cancelled" ? "native_run_cancelled" : "native_run_failed"; return { policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, toStatus: "in_progress", - reasonCode: input.terminalState === "cancelled" ? "native_run_cancelled" : "native_run_failed", + reasonCode: cause, unblockDescriptor: null, + effects: [{ + kind: "record_recovery", + cause, + nextAction: "Resume the persisted native run without opening a second provider session.", + agentId: input.agentId, + }], }; } - if (input.approvalGateSatisfied === false) { + if (input.governanceGate) { return { policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, - toStatus: "in_progress", - reasonCode: "approval_gate_pending", + toStatus: "in_review", + reasonCode: "governance_gate_pending", unblockDescriptor: null, + effects: [{ kind: "bind_governance", gate: input.governanceGate }], }; } const complete = @@ -53,27 +106,72 @@ export function arbitrateNativeStatus(input: { toStatus: "done", reasonCode: "completion_contract_satisfied", unblockDescriptor: null, + effects: [{ kind: "release_checkout" }], + }; + } + if (input.assessment.reportedDisposition === "needs_review") { + return { + policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, + toStatus: "in_review", + reasonCode: "native_review_path_required", + unblockDescriptor: null, + effects: [{ + kind: "create_review_interaction", + prompt: "Review the persisted native-run evidence and confirm whether this issue may be completed.", + }], }; } if (input.assessment.reportedDisposition === "blocked" && input.assessment.blocker) { + const owner = input.assessment.blocker.boardOwned ? "board" as const : { agentId: input.agentId }; + if (input.assessment.blocker.scope === "task_wide") { + return { + policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, + toStatus: "blocked", + reasonCode: "native_task_wide_blocker", + unblockDescriptor: { owner, action: input.assessment.blocker.unblockAction }, + effects: [{ kind: "bind_blocker", owner, action: input.assessment.blocker.unblockAction }], + }; + } return { policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, - toStatus: "blocked", - reasonCode: "native_reported_blocker", - unblockDescriptor: { - owner: input.assessment.blocker.boardOwned ? "board" : { agentId: input.agentId }, - action: input.assessment.blocker.unblockAction, - }, + toStatus: "in_progress", + reasonCode: "native_track_blocked_continuation", + unblockDescriptor: null, + effects: [{ + kind: "enqueue_continuation", + continuationKind: "same_agent", + summary: `Continue another productive track while resolving: ${input.assessment.blocker.unblockAction}`, + idempotencyKey: `native-track-blocked:${input.assessment.blocker.unblockAction}`, + agentId: input.agentId, + }], + }; + } + if (input.assessment.reportedDisposition === "yielded" && input.assessment.continuation) { + return { + policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, + toStatus: "in_progress", + reasonCode: "native_continuation_required", + unblockDescriptor: null, + effects: [{ + kind: "enqueue_continuation", + continuationKind: input.assessment.continuation.kind, + summary: input.assessment.continuation.summary, + idempotencyKey: input.assessment.continuation.idempotencyKey, + agentId: input.agentId, + }], }; } return { policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, toStatus: "in_progress", - reasonCode: input.assessment.reportedDisposition === "needs_review" - ? "native_review_path_required" - : input.assessment.reportedDisposition === "yielded" - ? "native_continuation_required" - : "native_completion_incomplete", + reasonCode: "native_completion_incomplete", unblockDescriptor: null, + effects: [{ + kind: "enqueue_continuation", + continuationKind: "same_agent", + summary: "Continue work on the missing or unverifiable completion-contract evidence.", + idempotencyKey: "native-completion-incomplete", + agentId: input.agentId, + }], }; } diff --git a/server/src/services/native-runtime/status-decision-committer.ts b/server/src/services/native-runtime/status-decision-committer.ts index 952f6e7fe5c..6540d055212 100644 --- a/server/src/services/native-runtime/status-decision-committer.ts +++ b/server/src/services/native-runtime/status-decision-committer.ts @@ -1,14 +1,23 @@ -import { and, eq } from "drizzle-orm"; +import { and, eq, inArray } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; +import type { CreateIssueThreadInteraction } from "@paperclipai/shared"; import { + agentWakeupRequests, + approvals, + issueApprovals, + issueThreadInteractions, issues, - activityLog, nativeRunFinalizations, statusDecisionEffects, statusDecisions, } from "@paperclipai/db"; -import type { NativeStatusDecision } from "./status-arbiter.js"; +import type { NativeStatusDecision, NativeStatusEffect } from "./status-arbiter.js"; import { nativeSha256 } from "./canonical.js"; +import { issueService } from "../issues.js"; +import { issueThreadInteractionService } from "../issue-thread-interactions.js"; +import { issueRecoveryActionService } from "../issue-recovery-actions.js"; +import { buildIssueBlockersResolvedWakeIdempotencyKey } from "../issue-dependency-wakeups.js"; +import { persistActivity, publishActivity, type ActivityPublication } from "../activity-log.js"; export class NativeStatusRaceError extends Error { readonly code = "native_status_race" as const; @@ -18,6 +27,229 @@ export class NativeStatusRaceError extends Error { } } +export type NativeStatusCommitFailpoint = + | "governance_materialization" + | "interaction_materialization" + | "continuation_materialization" + | "blocker_materialization" + | "recovery_materialization" + | "status_projection"; + +type MaterializedEffect = { + effectKind: string; + targetType: string; + targetId: string | null; + payload: Record; +}; + +function failAt(actual: NativeStatusCommitFailpoint, requested?: NativeStatusCommitFailpoint) { + if (actual === requested) throw new Error(`native_status_failpoint:${actual}`); +} + +async function enqueueWake(input: { + tx: Db; + companyId: string; + issueId: string; + agentId: string; + reason: string; + idempotencyKey: string; + payload: Record; +}) { + const existing = await input.tx.select({ id: agentWakeupRequests.id }) + .from(agentWakeupRequests) + .where(and( + eq(agentWakeupRequests.companyId, input.companyId), + eq(agentWakeupRequests.idempotencyKey, input.idempotencyKey), + inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution", "claimed", "completed"]), + )).limit(1).then((rows) => rows[0] ?? null); + if (existing) return existing.id; + const inserted = await input.tx.insert(agentWakeupRequests).values({ + companyId: input.companyId, + agentId: input.agentId, + source: "automation", + triggerDetail: "system", + reason: input.reason, + payload: { + issueId: input.issueId, + taskId: input.issueId, + ...input.payload, + _paperclipWakeContext: { + issueId: input.issueId, + taskId: input.issueId, + wakeReason: input.reason, + source: "native_status_decision", + }, + }, + requestedByActorType: "system", + requestedByActorId: "native-status-committer", + idempotencyKey: input.idempotencyKey, + }).returning({ id: agentWakeupRequests.id }).then((rows) => rows[0] ?? null); + if (!inserted) throw new Error("native_status_wake_not_persisted"); + return inserted.id; +} + +async function validateGovernanceGate(tx: Db, input: { + companyId: string; + issueId: string; + gate: Extract["gate"]; + executionState: unknown; +}) { + if (input.gate.kind === "execution_stage") { + const state = input.executionState && typeof input.executionState === "object" + ? input.executionState as Record + : {}; + if (state.status !== "pending") throw new Error("native_governance_gate_resolved"); + return input.gate.id; + } + if (input.gate.kind === "interaction") { + const interaction = await tx.select({ id: issueThreadInteractions.id }) + .from(issueThreadInteractions).where(and( + eq(issueThreadInteractions.id, input.gate.id), + eq(issueThreadInteractions.companyId, input.companyId), + eq(issueThreadInteractions.issueId, input.issueId), + eq(issueThreadInteractions.status, "pending"), + )).limit(1).then((rows) => rows[0] ?? null); + if (!interaction) throw new Error("native_governance_gate_resolved"); + return interaction.id; + } + const approval = await tx.select({ id: approvals.id }).from(issueApprovals) + .innerJoin(approvals, eq(issueApprovals.approvalId, approvals.id)) + .where(and( + eq(issueApprovals.companyId, input.companyId), + eq(issueApprovals.issueId, input.issueId), + eq(approvals.companyId, input.companyId), + eq(approvals.id, input.gate.id), + inArray(approvals.status, ["pending", "revision_requested"]), + )).limit(1).then((rows) => rows[0] ?? null); + if (!approval) throw new Error("native_governance_gate_resolved"); + return approval.id; +} + +async function materializeDecisionEffect(input: { + tx: Db; + companyId: string; + issue: typeof issues.$inferSelect; + runId: string; + decisionId: string; + effect: NativeStatusEffect; + failpoint?: NativeStatusCommitFailpoint; +}): Promise { + const { effect } = input; + if (effect.kind === "bind_governance") { + failAt("governance_materialization", input.failpoint); + const targetId = await validateGovernanceGate(input.tx, { + companyId: input.companyId, + issueId: input.issue.id, + gate: effect.gate, + executionState: input.issue.executionState, + }); + return { effectKind: effect.kind, targetType: effect.gate.kind, targetId, payload: { gate: effect.gate } }; + } + if (effect.kind === "create_review_interaction") { + failAt("interaction_materialization", input.failpoint); + const reviewInput: Extract = { + kind: "request_confirmation", + idempotencyKey: `native-review:${input.decisionId}`, + sourceRunId: input.runId, + title: "Native completion review", + summary: "The native runner requires authoritative review before completion.", + continuationPolicy: "wake_assignee", + payload: { + version: 1, + prompt: effect.prompt, + acceptLabel: "Approve completion", + rejectLabel: "Continue work", + allowDeclineReason: true, + rejectRequiresReason: true, + supersedeOnUserComment: false, + }, + }; + const interaction = await issueThreadInteractionService(input.tx).create( + input.issue, + reviewInput, + { systemId: "native-status-committer", runId: input.runId }, + ); + return { + effectKind: effect.kind, + targetType: "issue_thread_interaction", + targetId: interaction.id, + payload: { interactionId: interaction.id, interactionKind: interaction.kind }, + }; + } + if (effect.kind === "enqueue_continuation") { + failAt("continuation_materialization", input.failpoint); + const wakeId = await enqueueWake({ + tx: input.tx, + companyId: input.companyId, + issueId: input.issue.id, + agentId: effect.agentId, + reason: effect.continuationKind === "monitor" ? "monitor_due" : "issue_status_changed", + idempotencyKey: `native-status:${input.decisionId}:continuation`, + payload: { + nativeDecisionId: input.decisionId, + continuationKind: effect.continuationKind, + continuationSummary: effect.summary, + continuationIdempotencyKey: effect.idempotencyKey, + }, + }); + return { + effectKind: effect.kind, + targetType: "agent_wakeup_request", + targetId: wakeId, + payload: { continuationKind: effect.continuationKind, summary: effect.summary }, + }; + } + if (effect.kind === "bind_blocker") { + failAt("blocker_materialization", input.failpoint); + let wakeId: string | null = null; + if (effect.owner !== "board") { + wakeId = await enqueueWake({ + tx: input.tx, + companyId: input.companyId, + issueId: input.issue.id, + agentId: effect.owner.agentId, + reason: "issue_status_changed", + idempotencyKey: `native-status:${input.decisionId}:blocker-owner`, + payload: { nativeDecisionId: input.decisionId, unblockAction: effect.action }, + }); + } + return { + effectKind: effect.kind, + targetType: effect.owner === "board" ? "board" : "agent_wakeup_request", + targetId: wakeId, + payload: { owner: effect.owner, action: effect.action }, + }; + } + if (effect.kind === "record_recovery") { + failAt("recovery_materialization", input.failpoint); + const recovery = await issueRecoveryActionService(input.tx).upsertSourceScoped({ + companyId: input.companyId, + sourceIssueId: input.issue.id, + kind: "active_run_watchdog", + ownerType: "agent", + ownerAgentId: effect.agentId, + cause: effect.cause, + fingerprint: nativeSha256({ runId: input.runId, decisionId: input.decisionId, cause: effect.cause }), + evidence: { runId: input.runId, decisionId: input.decisionId }, + nextAction: effect.nextAction, + wakePolicy: { kind: "resume_native_run", runId: input.runId }, + maxAttempts: 3, + }); + return { + effectKind: effect.kind, + targetType: "issue_recovery_action", + targetId: recovery.id, + payload: { cause: effect.cause, nextAction: effect.nextAction }, + }; + } + return { + effectKind: effect.kind, + targetType: "issue_checkout", + targetId: input.issue.id, + payload: { checkoutRunId: input.issue.checkoutRunId, executionRunId: input.issue.executionRunId }, + }; +} + export async function commitNativeStatusDecision(input: { db: Db; companyId: string; @@ -26,104 +258,201 @@ export async function commitNativeStatusDecision(input: { assessmentId: string; priorStatus: string; priorStatusVersion: number; + priorDecisionId: string | null; decision: NativeStatusDecision; + failpoint?: NativeStatusCommitFailpoint; }) { - return input.db.transaction(async (tx) => { + const publications: ActivityPublication[] = []; + const committed = await input.db.transaction(async (tx) => { + const coordinator = await tx.select().from(nativeRunFinalizations).where(and( + eq(nativeRunFinalizations.runId, input.runId), + eq(nativeRunFinalizations.companyId, input.companyId), + eq(nativeRunFinalizations.issueId, input.issueId), + )).for("update").limit(1).then((rows) => rows[0] ?? null); + if (!coordinator) throw new Error("native_finalization_coordinator_missing"); const issue = await tx.select().from(issues).where(and( eq(issues.id, input.issueId), eq(issues.companyId, input.companyId), )).for("update").limit(1).then((rows) => rows[0] ?? null); - if (!issue || issue.status !== input.priorStatus || issue.statusVersion !== input.priorStatusVersion) { + if ( + !issue + || issue.status !== input.priorStatus + || Number(issue.statusVersion) !== input.priorStatusVersion + || issue.lastStatusDecisionId !== input.priorDecisionId + ) { throw new NativeStatusRaceError(); } const decisionJson = { toStatus: input.decision.toStatus, reasonCode: input.decision.reasonCode, unblockDescriptor: input.decision.unblockDescriptor, + effects: input.decision.effects, }; const decisionDigest = nativeSha256({ issueId: input.issueId, assessmentId: input.assessmentId, policyVersion: input.decision.policyVersion, fromStatus: issue.status, + priorStatusVersion: input.priorStatusVersion, + priorDecisionId: input.priorDecisionId, decision: decisionJson, }); let decisionRow = await tx.select().from(statusDecisions).where(and( eq(statusDecisions.issueId, input.issueId), eq(statusDecisions.decisionDigest, decisionDigest), )).limit(1).then((rows) => rows[0] ?? null); + if (decisionRow?.applicationState === "applied") { + return { decision: decisionRow, issue, replayed: true }; + } if (!decisionRow) { [decisionRow] = await tx.insert(statusDecisions).values({ companyId: input.companyId, issueId: input.issueId, assessmentId: input.assessmentId, - decisionVersion: 1, + decisionVersion: input.priorStatusVersion + 1, policyVersion: input.decision.policyVersion, fromStatus: issue.status, toStatus: input.decision.toStatus, reasonCode: input.decision.reasonCode, decisionJson, decisionDigest, - applicationState: "pending", + applicationState: "proposed", }).returning(); } if (!decisionRow) throw new Error("native_status_decision_not_persisted"); - const updated = await tx.update(issues).set({ + + const materialized: MaterializedEffect[] = []; + for (const effect of input.decision.effects) { + materialized.push(await materializeDecisionEffect({ + tx: tx as unknown as Db, + companyId: input.companyId, + issue, + runId: input.runId, + decisionId: decisionRow.id, + effect, + failpoint: input.failpoint, + })); + } + + failAt("status_projection", input.failpoint); + const updated = await issueService(tx as unknown as Db).update(input.issueId, { status: input.decision.toStatus, - unblockDescriptor: input.decision.unblockDescriptor, + statusVersion: input.priorStatusVersion + 1, lastStatusDecisionId: decisionRow.id, - completedAt: input.decision.toStatus === "done" ? new Date() : null, - blockedTransitionAt: input.decision.toStatus === "blocked" ? new Date() : null, - updatedAt: new Date(), - }).where(and( - eq(issues.id, input.issueId), - eq(issues.statusVersion, input.priorStatusVersion), - )).returning().then((rows) => rows[0] ?? null); + unblockDescriptor: input.decision.unblockDescriptor, + actorAgentId: null, + actorUserId: null, + }, tx, publications); if (!updated) throw new NativeStatusRaceError(); - await tx.insert(statusDecisionEffects).values({ - companyId: input.companyId, - issueId: input.issueId, - decisionId: decisionRow.id, - ordinal: 1, + materialized.unshift({ effectKind: "issue_status_projection", targetType: "issue", targetId: input.issueId, - idempotencyKey: `native-status:${decisionRow.id}:1`, - payload: { - fromStatus: issue.status, - toStatus: input.decision.toStatus, - reasonCode: input.decision.reasonCode, - }, - deliveryState: "delivered", - attemptCount: 1, - deliveredAt: new Date(), - }).onConflictDoNothing(); - await tx.update(statusDecisions).set({ - applicationState: "applied", - appliedAt: new Date(), - }).where(eq(statusDecisions.id, decisionRow.id)); + payload: { fromStatus: issue.status, toStatus: input.decision.toStatus, reasonCode: input.decision.reasonCode }, + }); + + if (input.decision.toStatus === "done" && issue.status !== "done") { + const issueSvc = issueService(tx as unknown as Db); + const dependents = await issueSvc.listWakeableBlockedDependents(input.issueId); + for (const dependent of dependents) { + const idempotencyKey = buildIssueBlockersResolvedWakeIdempotencyKey({ + dependentIssueId: dependent.id, + resolvedBlockerIssueId: input.issueId, + }); + const wakeId = await enqueueWake({ + tx: tx as unknown as Db, + companyId: input.companyId, + issueId: dependent.id, + agentId: dependent.assigneeAgentId, + reason: "issue_blockers_resolved", + idempotencyKey, + payload: { resolvedBlockerIssueId: input.issueId, blockerIssueIds: dependent.blockerIssueIds }, + }); + materialized.push({ + effectKind: "dependency_wake", + targetType: "agent_wakeup_request", + targetId: wakeId, + payload: { dependentIssueId: dependent.id, resolvedBlockerIssueId: input.issueId }, + }); + } + if (issue.parentId) { + const parent = await issueSvc.getWakeableParentAfterChildCompletion(issue.parentId); + if (parent) { + const wakeId = await enqueueWake({ + tx: tx as unknown as Db, + companyId: input.companyId, + issueId: parent.id, + agentId: parent.assigneeAgentId, + reason: "issue_children_completed", + idempotencyKey: `issue_children_completed:${parent.id}:${input.issueId}`, + payload: { completedChildIssueId: input.issueId, childIssueIds: parent.childIssueIds }, + }); + materialized.push({ + effectKind: "parent_wake", + targetType: "agent_wakeup_request", + targetId: wakeId, + payload: { parentIssueId: parent.id, completedChildIssueId: input.issueId }, + }); + } + } + } + + for (const [index, effect] of materialized.entries()) { + await tx.insert(statusDecisionEffects).values({ + companyId: input.companyId, + issueId: input.issueId, + decisionId: decisionRow.id, + ordinal: index + 1, + effectKind: effect.effectKind, + targetType: effect.targetType, + targetId: effect.targetId, + idempotencyKey: `native-status:${decisionRow.id}:${index + 1}`, + payload: effect.payload, + deliveryState: "delivered", + attemptCount: 1, + deliveredAt: new Date(), + }).onConflictDoNothing(); + } + await tx.update(statusDecisions).set({ applicationState: "applied", appliedAt: new Date() }) + .where(eq(statusDecisions.id, decisionRow.id)); await tx.update(nativeRunFinalizations).set({ - phase: "applied", + phase: "committed", assessmentId: input.assessmentId, decisionId: decisionRow.id, + leaseOwner: null, + leaseExpiresAt: null, + failureCode: null, + failureDetail: null, + nextAttemptAt: null, updatedAt: new Date(), }).where(eq(nativeRunFinalizations.runId, input.runId)); - await tx.insert(activityLog).values({ + const { publication } = await persistActivity(tx as unknown as Db, { companyId: input.companyId, actorType: "system", actorId: "native-status-committer", - action: "issue.native_status_decision_applied", + action: "issue.updated", entityType: "issue", entityId: input.issueId, + issueId: input.issueId, runId: input.runId, details: { + source: "native_status_decision", assessmentId: input.assessmentId, decisionId: decisionRow.id, fromStatus: issue.status, toStatus: input.decision.toStatus, reasonCode: input.decision.reasonCode, + effectCount: materialized.length, }, }); - return { decision: { ...decisionRow, applicationState: "applied" }, issue: updated }; + publications.push(publication); + return { + decision: { ...decisionRow, applicationState: "applied" }, + issue: updated, + replayed: false, + }; }); + + for (const publication of publications) publishActivity(publication); + return committed; } diff --git a/server/src/services/native-runtime/work-assessments.ts b/server/src/services/native-runtime/work-assessments.ts index 07982e6b5a4..56121a55035 100644 --- a/server/src/services/native-runtime/work-assessments.ts +++ b/server/src/services/native-runtime/work-assessments.ts @@ -11,7 +11,9 @@ export async function recordNativeWorkAssessment(input: { runId: string; turnId: string | null; contractId: string; + contractCanonicalSha256: string; resultId: string; + resultCanonicalSha256: string; priorIssueStatus: string; priorStatusVersion: number; priorDecisionId: string | null; @@ -22,14 +24,21 @@ export async function recordNativeWorkAssessment(input: { const assessmentJson = input.assessment as unknown as Record; const inputDigest = nativeSha256({ contractId: input.contractId, + contractCanonicalSha256: input.contractCanonicalSha256, resultId: input.resultId, + resultCanonicalSha256: input.resultCanonicalSha256, priorIssueStatus: input.priorIssueStatus, priorStatusVersion: input.priorStatusVersion, + priorDecisionId: input.priorDecisionId, + triggerKind: "native_result", + triggerRef: input.resultId, + triggerActorCompanyId: input.companyId, + triggerCapability: "server_native_finalizer", policyVersion: input.policyVersion, assessment: assessmentJson, }); const existing = await input.db.select().from(workAssessments).where(and( - eq(workAssessments.resultId, input.resultId), + eq(workAssessments.issueId, input.issueId), eq(workAssessments.inputDigest, inputDigest), )).limit(1).then((rows) => rows[0] ?? null); if (existing) return existing;