fix(runner): make native finalization restart-safe

Persist bound session checkpoints, verify durable completion evidence, and commit status/liveness effects atomically. Preserve unique event cursors during migration and add bounded recovery plus legacy compatibility coverage.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-08-09 03:44:38 +00:00
1 parent 512c0647b0
commit 0ca23dd535
42 files changed
+2931 -386

No files matched your search

@@ -28,7 +28,7 @@ CREATE TABLE "native_run_finalizations" (
"assessment_id" uuid,
"decision_id" uuid,
"failure_code" text,
"failure_detail" text,
"failure_detail" jsonb,
"next_attempt_at" timestamp with time zone,
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
@@ -76,14 +76,14 @@ CREATE TABLE "status_decisions" (
"company_id" uuid NOT NULL,
"issue_id" uuid NOT NULL,
"assessment_id" uuid NOT NULL,
"decision_version" integer NOT NULL,
"decision_version" bigint NOT NULL,
"policy_version" text NOT NULL,
"from_status" text NOT NULL,
"to_status" text NOT NULL,
"reason_code" text NOT NULL,
"decision_json" jsonb NOT NULL,
"decision_digest" text NOT NULL,
"application_state" text DEFAULT 'pending' NOT NULL,
"application_state" text DEFAULT 'proposed' NOT NULL,
"supersedes_decision_id" uuid,
"applied_at" timestamp with time zone,
"created_at" timestamp with time zone DEFAULT now() NOT NULL
@@ -133,16 +133,35 @@ ALTER TABLE "heartbeat_runs" ADD COLUMN "native_phase" text;--> statement-breakp
ALTER TABLE "heartbeat_runs" ADD COLUMN "native_phase_updated_at" timestamp with time zone;--> statement-breakpoint
ALTER TABLE "issues" ADD COLUMN "status_version" bigint DEFAULT 0 NOT NULL;--> statement-breakpoint
ALTER TABLE "issues" ADD COLUMN "last_status_decision_id" uuid;--> statement-breakpoint
-- Historical event writers computed max(seq) outside their insert transaction. Repair any
-- collisions deterministically before enforcing the native runner's single-writer invariant.
WITH resequenced AS (
SELECT "id", row_number() OVER (PARTITION BY "run_id" ORDER BY "created_at", "id") AS "new_seq"
-- Historical event writers computed max(seq) outside their insert transaction. Preserve every
-- existing unique cursor and the first row at each duplicated cursor. Only later colliding rows
-- move above the run's pre-migration maximum, in deterministic legacy order.
WITH ranked AS (
SELECT
"id",
"run_id",
"seq",
"created_at",
row_number() OVER (
PARTITION BY "run_id", "seq"
ORDER BY "created_at", "id"
) AS "duplicate_ordinal",
max("seq") OVER (PARTITION BY "run_id") AS "old_run_max"
FROM "heartbeat_run_events"
), duplicate_repairs AS (
SELECT
"id",
"old_run_max" + row_number() OVER (
PARTITION BY "run_id"
ORDER BY "seq", "created_at", "id"
) AS "new_seq"
FROM ranked
WHERE "duplicate_ordinal" > 1
)
UPDATE "heartbeat_run_events" AS event
SET "seq" = resequenced."new_seq"
FROM resequenced
WHERE event."id" = resequenced."id";--> statement-breakpoint
SET "seq" = duplicate_repairs."new_seq"
FROM duplicate_repairs
WHERE event."id" = duplicate_repairs."id";--> statement-breakpoint
UPDATE "heartbeat_runs" AS run
SET "next_event_seq" = COALESCE((
SELECT max(event."seq") + 1
@@ -189,10 +208,11 @@ CREATE UNIQUE INDEX "native_run_results_run_fingerprint_uq" ON "native_run_resul
CREATE UNIQUE INDEX "native_run_results_run_caller_result_uq" ON "native_run_results" USING btree ("run_id","caller_result_id");--> statement-breakpoint
CREATE UNIQUE INDEX "native_run_results_run_caller_dedupe_uq" ON "native_run_results" USING btree ("run_id","caller_dedupe_key");--> statement-breakpoint
CREATE UNIQUE INDEX "status_decision_effects_decision_ordinal_uq" ON "status_decision_effects" USING btree ("decision_id","ordinal");--> statement-breakpoint
CREATE UNIQUE INDEX "status_decision_effects_idempotency_uq" ON "status_decision_effects" USING btree ("idempotency_key");--> statement-breakpoint
CREATE UNIQUE INDEX "status_decisions_assessment_version_uq" ON "status_decisions" USING btree ("assessment_id","decision_version");--> statement-breakpoint
CREATE UNIQUE INDEX "status_decisions_issue_digest_uq" ON "status_decisions" USING btree ("issue_id","decision_digest");--> statement-breakpoint
CREATE UNIQUE INDEX "work_assessments_result_input_uq" ON "work_assessments" USING btree ("result_id","input_digest");--> statement-breakpoint
CREATE UNIQUE INDEX "status_decision_effects_company_idempotency_uq" ON "status_decision_effects" USING btree ("company_id","idempotency_key");--> statement-breakpoint
CREATE UNIQUE INDEX "status_decisions_company_issue_version_uq" ON "status_decisions" USING btree ("company_id","issue_id","decision_version");--> statement-breakpoint
CREATE UNIQUE INDEX "status_decisions_company_assessment_uq" ON "status_decisions" USING btree ("company_id","assessment_id");--> statement-breakpoint
CREATE UNIQUE INDEX "status_decisions_company_issue_digest_uq" ON "status_decisions" USING btree ("company_id","issue_id","decision_digest");--> statement-breakpoint
CREATE UNIQUE INDEX "work_assessments_company_issue_input_uq" ON "work_assessments" USING btree ("company_id","issue_id","input_digest");--> statement-breakpoint
-- paperclip:migration-safety-ignore large-create-index-not-concurrently: duplicate rows are deterministically resequenced above before this invariant is enabled in the same transaction.
CREATE UNIQUE INDEX "heartbeat_run_events_run_seq_uq" ON "heartbeat_run_events" USING btree ("run_id","seq");--> statement-breakpoint
-- paperclip:migration-safety-ignore large-create-index-not-concurrently: nullable native source IDs mean historical rows need no backfill and the invariant must commit atomically with the new columns.
@@ -24318,7 +24318,7 @@
},
"failure_detail": {
"name": "failure_detail",
"type": "text",
"type": "jsonb",
"primaryKey": false,
"notNull": false
},
@@ -32244,9 +32244,15 @@
"method": "btree",
"with": {}
},
"status_decision_effects_idempotency_uq": {
"name": "status_decision_effects_idempotency_uq",
"status_decision_effects_company_idempotency_uq": {
"name": "status_decision_effects_company_idempotency_uq",
"columns": [
{
"expression": "company_id",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "idempotency_key",
"isExpression": false,
@@ -32338,7 +32344,7 @@
},
"decision_version": {
"name": "decision_version",
"type": "integer",
"type": "bigint",
"primaryKey": false,
"notNull": true
},
@@ -32383,7 +32389,7 @@
"type": "text",
"primaryKey": false,
"notNull": true,
"default": "'pending'"
"default": "'proposed'"
},
"supersedes_decision_id": {
"name": "supersedes_decision_id",
@@ -32406,11 +32412,17 @@
}
},
"indexes": {
"status_decisions_assessment_version_uq": {
"name": "status_decisions_assessment_version_uq",
"status_decisions_company_issue_version_uq": {
"name": "status_decisions_company_issue_version_uq",
"columns": [
{
"expression": "assessment_id",
"expression": "company_id",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "issue_id",
"isExpression": false,
"asc": true,
"nulls": "last"
@@ -32427,9 +32439,36 @@
"method": "btree",
"with": {}
},
"status_decisions_issue_digest_uq": {
"name": "status_decisions_issue_digest_uq",
"status_decisions_company_assessment_uq": {
"name": "status_decisions_company_assessment_uq",
"columns": [
{
"expression": "company_id",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "assessment_id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": true,
"concurrently": false,
"method": "btree",
"with": {}
},
"status_decisions_company_issue_digest_uq": {
"name": "status_decisions_company_issue_digest_uq",
"columns": [
{
"expression": "company_id",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "issue_id",
"isExpression": false,
@@ -39258,11 +39297,17 @@
}
},
"indexes": {
"work_assessments_result_input_uq": {
"name": "work_assessments_result_input_uq",
"work_assessments_company_issue_input_uq": {
"name": "work_assessments_company_issue_input_uq",
"columns": [
{
"expression": "result_id",
"expression": "company_id",
"isExpression": false,
"asc": true,
"nulls": "last"
},
{
"expression": "issue_id",
"isExpression": false,
"asc": true,
"nulls": "last"
@@ -40060,4 +40105,4 @@
"schemas": {},
"tables": {}
}
}
}
@@ -1,4 +1,4 @@
import { pgTable, uuid, text, timestamp, integer } from "drizzle-orm/pg-core";
import { pgTable, uuid, text, timestamp, integer, jsonb } from "drizzle-orm/pg-core";
import { companies } from "./companies.js";
import { issues } from "./issues.js";
import { heartbeatRuns } from "./heartbeat_runs.js";
@@ -16,7 +16,7 @@ export const nativeRunFinalizations = pgTable("native_run_finalizations", {
assessmentId: uuid("assessment_id"),
decisionId: uuid("decision_id"),
failureCode: text("failure_code"),
failureDetail: text("failure_detail"),
failureDetail: jsonb("failure_detail").$type<Record<string, unknown>>(),
nextAttemptAt: timestamp("next_attempt_at", { withTimezone: true }),
createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(),
@@ -23,5 +23,6 @@ export const statusDecisionEffects = pgTable("status_decision_effects", {
updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(),
}, (table) => ({
decisionOrdinalUq: uniqueIndex("status_decision_effects_decision_ordinal_uq").on(table.decisionId, table.ordinal),
idempotencyUq: uniqueIndex("status_decision_effects_idempotency_uq").on(table.idempotencyKey),
idempotencyUq: uniqueIndex("status_decision_effects_company_idempotency_uq")
.on(table.companyId, table.idempotencyKey),
}));
+8 -5
View File
@@ -1,4 +1,4 @@
import { pgTable, uuid, text, timestamp, integer, jsonb, uniqueIndex } from "drizzle-orm/pg-core";
import { pgTable, uuid, text, timestamp, bigint, jsonb, uniqueIndex } from "drizzle-orm/pg-core";
import { companies } from "./companies.js";
import { issues } from "./issues.js";
import { workAssessments } from "./work_assessments.js";
@@ -8,18 +8,21 @@ export const statusDecisions = pgTable("status_decisions", {
companyId: uuid("company_id").notNull().references(() => companies.id),
issueId: uuid("issue_id").notNull().references(() => issues.id),
assessmentId: uuid("assessment_id").notNull().references(() => workAssessments.id),
decisionVersion: integer("decision_version").notNull(),
decisionVersion: bigint("decision_version", { mode: "number" }).notNull(),
policyVersion: text("policy_version").notNull(),
fromStatus: text("from_status").notNull(),
toStatus: text("to_status").notNull(),
reasonCode: text("reason_code").notNull(),
decisionJson: jsonb("decision_json").$type<Record<string, unknown>>().notNull(),
decisionDigest: text("decision_digest").notNull(),
applicationState: text("application_state").notNull().default("pending"),
applicationState: text("application_state").notNull().default("proposed"),
supersedesDecisionId: uuid("supersedes_decision_id"),
appliedAt: timestamp("applied_at", { withTimezone: true }),
createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
}, (table) => ({
assessmentVersionUq: uniqueIndex("status_decisions_assessment_version_uq").on(table.assessmentId, table.decisionVersion),
issueDigestUq: uniqueIndex("status_decisions_issue_digest_uq").on(table.issueId, table.decisionDigest),
issueVersionUq: uniqueIndex("status_decisions_company_issue_version_uq")
.on(table.companyId, table.issueId, table.decisionVersion),
assessmentUq: uniqueIndex("status_decisions_company_assessment_uq").on(table.companyId, table.assessmentId),
issueDigestUq: uniqueIndex("status_decisions_company_issue_digest_uq")
.on(table.companyId, table.issueId, table.decisionDigest),
}));
+2 -1
View File
@@ -26,5 +26,6 @@ export const workAssessments = pgTable("work_assessments", {
supersedesAssessmentId: uuid("supersedes_assessment_id"),
createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
}, (table) => ({
resultInputUq: uniqueIndex("work_assessments_result_input_uq").on(table.resultId, table.inputDigest),
issueInputUq: uniqueIndex("work_assessments_company_issue_input_uq")
.on(table.companyId, table.issueId, table.inputDigest),
}));
@@ -695,6 +695,29 @@ allowlist amendment and CTO review. In particular, Phase 6 must not change a
concrete legacy adapter, approval authority, browser UI, workspace policy,
budget policy, or activity semantics merely to make the tracer pass.
### Remediation allowlist amendment (2026-08-09)
The implementation review found four seam-local paths whose names differed
from the provisional list above. They are added without expanding authority:
```text
packages/paperclip-runner/src/native-session-runtime.ts
packages/paperclip-runner/src/native-session-runtime.test.ts
server/src/services/native-runtime/native-session-executor.ts
server/src/services/native-runtime/native-session-executor.test.ts
server/src/services/native-runtime/paperclip-control-plane-port.test.ts
server/src/services/native-runtime/status-arbiter.test.ts
packages/paperclip-runner/docs/design/phase-6-thin-paperclip-adapter.md
```
The package runtime files own persisted provider-session recovery. The server
executor files own only the run-scoped cancellation handle and coordinator
lease around that package runtime. The colocated port and arbiter tests exercise
the same approved database and pure-policy seams exposed through the named
`server/src/__tests__` matrix entry points. This amendment does not authorize
changes to a concrete legacy adapter, approval authority, UI, workspace policy,
budget policy, or runner/provider/session behavior outside the package.
## Commands the implementation must make runnable
These commands are the acceptance contract for the implementation issue. They
@@ -32,6 +32,8 @@ contract are persisted before provider execution.
`PaperclipControlPlanePort` implements the package's `ControlPlanePort`:
- `openRun` verifies company/run/issue/agent/contract binding;
- `loadSessionCheckpoint` and `checkpointSession` bind restart state to the
same company/run/issue/agent/session identity;
- `appendEvent` validates PRP, commits through the shared per-run allocator,
deduplicates source identity by canonical digest, and acknowledges afterward;
- `replayEvents` uses an exclusive source cursor;
@@ -51,21 +53,41 @@ versions, native run metadata, and native source identity on heartbeat events.
All event writers allocate from the row-locked `heartbeat_runs.next_event_seq`.
Finalization runs only after result persistence and workspace finalization. The
pure arbiter marks `done` only when the terminal state succeeded, the objective
and every criterion are satisfied, verification passed, and no blocking work
remains. It marks `blocked` only for a first-class reported blocker. A failed
workspace barrier preserves the immutable native result, fails the run, and
leaves issue status unchanged. Review/yield/incomplete/failure/cancellation
remain non-terminal.
Status projection is a versioned CAS and emits an audit record.
server verifies cited evidence against company- and issue-scoped durable
events, work products, approvals, interactions, or attachments; model claims
alone cannot complete an issue. The pure arbiter marks `done` only when the
terminal state succeeded, every criterion and verification has accepted
durable evidence, and no blocking work remains. It marks `blocked` only for a
task-wide first-class blocker with a named owner and action. Review, retry,
continuation, blocker, and recovery effects materialize atomically with the
status-version CAS and audit record.
## Recovery and safety
The finalization reconciler selects persisted `runtime_mode=native` rows; it
does not consult the current flag. Cancellation uses a run-scoped normalized
session handle before the existing process cleanup. Native execution does not
construct a Paperclip JWT, managed MCP access, legacy context, or raw adapter
environment.
does not consult the current flag. Per-attempt leases prevent concurrent
session/finalization ownership, while retryable failures retain the immutable
result and a first-class recovery action. A recovered session replays its
checkpoint and appends only missing control-plane facts instead of opening a
second provider session. Cancellation uses a run-scoped normalized session
handle before the existing process cleanup. Native execution does not construct
a Paperclip JWT, managed MCP access, legacy context, or raw adapter environment.
## Verification status
The deterministic package suite and the real embedded-PostgreSQL Paperclip
suite prove selection, replay/conflict handling, authoritative evidence,
status/liveness atomicity, session/finalization recovery, migration repair,
bounded cancellation/retry, and a byte-equivalent legacy read snapshot. The
database suite is the internal native canary: it executes one selected task
through the public package session contract and applies one server-owned
decision. Its post-kill-switch legacy case has zero native history rows.
No new live provider task was dispatched during the remediation review. A live
Codex canary remains an operator-run rollout check because it requires changing
the instance flag and one agent profile. The package `trace:phase6` command is a
mock contract tracer; real Paperclip proof is the database-backed server matrix
and the explicitly gated procedure in the tutorial.
See the [runnable tutorial](tutorials/phase-06-thin-paperclip-adapter.md), the
[verification record](../knowledge/evidence/2026-08-09-phase-06-verification.md),
@@ -29,29 +29,45 @@ PHASE6_SCRATCH="${PAPERCLIP_RUN_SCRATCH_DIR:-$(mktemp -d)}"
pnpm --filter @paperclipai/paperclip-runner exec vitest run \
src/conformance/control-plane-port.test.ts \
src/backends/harness-driver-backend.test.ts \
src/contracts/native-execution.test.ts
src/contracts/native-execution.test.ts \
src/native-session-runtime.test.ts
pnpm --filter @paperclipai/paperclip-runner trace:phase6 -- \
--target mock --scenario happy-path
```
Expected: five targeted tests pass and the trace reports `resolvedMode` as
Expected: six targeted tests pass and the trace reports `resolvedMode` as
`native`, three replay-stable events, a successful workspace barrier, and one
server-owned `done` decision.
mock server-owned `done` decision. This tracer does not connect to Paperclip.
## 2. Prove the database-backed Paperclip port
```sh
pnpm --filter @paperclipai/paperclip-runner build:typescript
pnpm --filter @paperclipai/server exec vitest run \
src/services/native-runtime/paperclip-control-plane-port.test.ts \
src/services/native-runtime/runtime-mode.test.ts \
src/services/native-runtime/status-arbiter.test.ts
src/__tests__/native-runner-phase6.integration.test.ts \
src/__tests__/heartbeat-native-runner-selection.test.ts \
src/__tests__/heartbeat-native-runner-cancellation.test.ts \
src/__tests__/heartbeat-run-event-sequencing.test.ts \
src/__tests__/native-runner-input-boundary.test.ts \
src/__tests__/native-run-finalizer.test.ts \
src/__tests__/native-status-arbiter-corpus.test.ts \
src/__tests__/native-finalization-recovery.test.ts \
src/__tests__/native-finalization-migration.test.ts \
src/__tests__/legacy-finalization-regression.test.ts
```
This runs the unchanged package conformance suite against the real Paperclip
port. It also checks company binding, duplicate/gap replay, immutable result
ingestion, workspace-gated finalization, status-version CAS, and audit output.
port. Twenty-five targeted tests check company binding, duplicate/gap replay,
checkpoint recovery, immutable result ingestion, durable-evidence authority,
workspace-gated finalization, atomic liveness rollback, status-version CAS,
migration repair, legacy byte equivalence, and audit output. All database tests
start embedded PostgreSQL and do not skip when a developer database URL is
absent.
The command above is the repeatable internal canary and post-kill-switch legacy
proof. The remaining sections are an optional live-provider rollout procedure;
they were not run during remediation and require explicit operator authority to
change an instance flag and agent profile.
## 3. Enable one isolated agent
@@ -13,39 +13,44 @@ This record verifies the default-off Phase 6 tracer at the approved
`ControlPlanePort`/`NativeSessionBackend` boundary. No browser surface changed,
so Phase 6 has command/database evidence rather than screenshots.
The counts below are the PAP-16875 remediation rerun. “Internal canary” means
the selected-task database test through the public package session contract;
it does not mean that a new live Codex provider task was dispatched.
# Package contract and mock tracer
```sh
pnpm --filter @paperclipai/paperclip-runner exec vitest run \
src/conformance/control-plane-port.test.ts \
src/backends/harness-driver-backend.test.ts \
src/contracts/native-execution.test.ts
src/contracts/native-execution.test.ts \
src/native-session-runtime.test.ts
pnpm --filter @paperclipai/paperclip-runner trace:phase6 -- \
--target mock --scenario happy-path
```
Result: three files and five tests passed. The stable trace reported native
Result: four files and six tests passed. The stable trace reported native
mode, `runStatus=succeeded`, three events, contiguous source sequence 3,
`workspaceFinalizeStatus=succeeded`, zero legacy invocations, and a server-owned
`done` decision with status version 0 -> 1.
`done` decision with status version 0 -> 1. Restart recovery reused the bound
provider session checkpoint, opened no second session or turn, retained the
existing accepted result event, and appended only the missing terminal fact.
# Real Paperclip adapter and public-session task
```sh
pnpm --filter @paperclipai/paperclip-runner build:typescript
pnpm --filter @paperclipai/server exec vitest run \
src/services/native-runtime/paperclip-control-plane-port.test.ts \
src/services/native-runtime/runtime-mode.test.ts \
src/services/native-runtime/status-arbiter.test.ts
```
Result: three files and ten tests passed against embedded PostgreSQL. The same
package conformance suite passed unchanged against `PaperclipControlPlanePort`.
The complete acceptance gate below passed 25 tests in ten files against
embedded PostgreSQL with zero skipped tests. Database-backed tests start their
own database instead of skipping when a developer `DATABASE_URL` is absent.
The same package conformance suite passed unchanged against
`PaperclipControlPlanePort`.
A selected task then ran end-to-end through `executeNativeSession`, the public
backend/port contracts, durable PRP events, immutable result ingestion,
workspace finalization, assessment, CAS status decision, and audit persistence.
The issue moved from `in_progress` version 0 to `done` version 1.
The issue moved from `in_progress` version 0 to `done` version 1 only after its
criterion and verification cited an approved, issue-scoped durable work
product. A model-only result remained non-terminal and received a persisted
continuation wake.
The focused corpus also proved:
@@ -55,12 +60,37 @@ The focused corpus also proved:
- company/run/agent/issue/source bindings reject mismatches;
- source gaps remain visible, identical retries deduplicate, and replay is
exclusive and byte-stable;
- incomplete/review/yield/cancelled results remain non-terminal;
- only a first-class reported blocker selects `blocked`;
- incomplete/review/yield/cancelled results receive a durable review,
continuation, or recovery path;
- only a task-wide blocker with a named owner and action selects `blocked`;
- a completion decision writes immutable assessment/decision/effect records and
an activity-log audit row.
an activity-log audit row;
- a failed workspace barrier preserves the accepted semantic result, leaves the
issue status unchanged, and records a recoverable `workspace_failed` phase.
issue status unchanged, and records a leased `retryable_failure` plus recovery
owner/action;
- four injected failures prove interactions, wakes, blocker bindings, status,
decisions, and effects roll back together before retry ownership is recorded.
The acceptance-matrix entry points also passed as one 10-file, 25-test gate:
```sh
pnpm --filter @paperclipai/server exec vitest run \
src/__tests__/heartbeat-native-runner-cancellation.test.ts \
src/__tests__/heartbeat-native-runner-selection.test.ts \
src/__tests__/native-run-finalizer.test.ts \
src/__tests__/native-runner-input-boundary.test.ts \
src/__tests__/native-runner-phase6.integration.test.ts \
src/__tests__/native-finalization-recovery.test.ts \
src/__tests__/heartbeat-run-event-sequencing.test.ts \
src/__tests__/legacy-finalization-regression.test.ts \
src/__tests__/native-finalization-migration.test.ts \
src/__tests__/native-status-arbiter-corpus.test.ts
```
This gate proves concurrent event allocation, duplicate-only migration repair,
bounded retry exhaustion, kill-switch legacy byte compatibility, and complete
Section 18.13 fixture-to-consumer traceability in addition to the direct suites.
It is the repeatable internal native canary and post-kill-switch legacy proof.
# Compile and migration checks
@@ -72,22 +102,29 @@ pnpm --filter @paperclipai/db typecheck
```
Result: all checks passed. Migration numbering and safety passed. Migration
0211 deterministically resequences historical heartbeat events before adding
the unique run sequence invariant, backfills each run's next allocator value,
and installs issue status-version tracking.
0211 preserves existing unique heartbeat cursors, deterministically moves only
duplicate rows above each run's former maximum, backfills the next allocator
value, and installs issue status-version tracking.
Focused legacy regression:
Focused migration, allocator, recovery, and legacy rehearsal:
```sh
pnpm --filter @paperclipai/server exec vitest run \
src/__tests__/heartbeat-workspace-finalize-branch.test.ts \
src/__tests__/heartbeat-process-recovery.test.ts \
src/__tests__/heartbeat-runtime-state.test.ts
src/services/native-runtime/paperclip-control-plane-port.test.ts \
src/__tests__/native-finalization-recovery.test.ts \
src/__tests__/legacy-finalization-regression.test.ts \
src/__tests__/native-finalization-migration.test.ts \
src/__tests__/heartbeat-run-event-sequencing.test.ts
```
Result: three files and 106 tests passed, covering unchanged adapter execution,
workspace finalization, cancellation, process recovery, and runtime/session
state behavior with the flag left at its default.
Result: five files and 11 tests passed with zero skips. The production-shaped
0211 rehearsal began with legacy event sequences `[1,5,5,9]`, retained the
original unique cursors and first `5`, moved only the later duplicate to `10`,
seeded `next_event_seq=11`, preserved every non-sequence field byte-for-byte,
and enforced uniqueness. Thirty-two concurrent mixed event writers produced a
gap-free allocator stream. Invalid native finalization exhausted its
three-attempt budget into named recovery without touching issue status. The
legacy snapshot remained byte-equivalent with zero native history rows.
# Credential, governance, budget, and legacy boundaries
@@ -113,14 +150,16 @@ pnpm --filter @paperclipai/server exec vitest run \
-t "daily cost cap"
```
Result: native cancellation reached the active normalized session and removed
the handle afterward; the daily cost hard stop cancelled a queued run before
any adapter execution.
Result: native cancellation reached the active normalized session exactly once
across duplicate cancellation requests and removed the handle afterward. The
daily cost hard stop cancelled a queued run before any adapter execution.
# Remaining operational checkpoint
# Deferred live-provider checkpoint
The [Phase 6 tutorial](../../docs/tutorials/phase-06-thin-paperclip-adapter.md)
contains the exact board-authorized commands for enabling one local agent,
running and inspecting a live Codex task, disabling the flag, proving a fresh
legacy selection, and restoring the agent profile. Those commands intentionally
do not pass credentials to the package or model.
do not pass credentials to the package or model. They were not run during this
remediation because changing live instance and agent rollout state requires
operator approval; no new live-provider evidence is claimed here.
@@ -12,5 +12,4 @@
* [Phase 4b live console verification](2026-08-08-phase-04b-live-console-verification.md) - Demo-driver, transcript-model, browser, keyboard, boundary, token, and screenshot evidence for the live console.
* [Phase 4b QA verification](2026-08-08-phase-04b-qa-verification.md) - Independent QA pass: clean-start tutorial in a real browser, deterministic demos, real Codex end-to-end session, full `verify` acceptance (exit 0), and screenshots (PAP-16837).
* [Phase 5 browser SDK verification](2026-08-08-phase-05-verification.md) - Targeted and package-acceptance evidence for the versioned SDK, both public consumers, fake/real drivers, accessibility, reconnect, replay, and screenshots.
* [Phase 6 thin Paperclip adapter verification](2026-08-09-phase-06-verification.md) - Mock and database-backed port conformance, public-session task completion, durable replay/finalization, feature selection, security boundary, and legacy fallback evidence.
* [Phase 6 thin Paperclip adapter verification](2026-08-09-phase-06-verification.md) - Mock/real conformance, public session completion, canonical persistence, governance authority, migration, kill-switch, and legacy compatibility evidence.
* [Phase 6 thin Paperclip adapter verification](2026-08-09-phase-06-verification.md) - Mock and database-backed conformance, authoritative completion, restart recovery, migration repair, atomic liveness, internal canary, and deterministic post-kill-switch legacy evidence.
@@ -34,16 +34,27 @@ existing server authority and keep legacy execution as the default.
handle; never fall back to legacy inside a selected native run.
8. Keep the feature default false. Recovery reads persisted run/coordinator
state and therefore remains correct after the flag is disabled.
9. Persist provider session checkpoints and recover them fail-closed; a restart
may reconcile missing control facts but may not open a second provider
session for the same run.
10. Treat model citations as claims. Only server-verifiable durable records can
satisfy completion criteria, and every non-terminal decision must create or
bind a real review, wake, blocker, or recovery path in the status transaction.
# Evidence
- [Phase 6 verification](../evidence/2026-08-09-phase-06-verification.md)
- [Phase 6 reference](../../docs/phase-06-thin-paperclip-adapter.md)
- [Runnable tutorial](../../docs/tutorials/phase-06-thin-paperclip-adapter.md)
- Package conformance: three files, five tests passed.
- Database-backed port/session/finalizer corpus: three files, ten tests passed.
- Legacy heartbeat finalization/recovery/runtime corpus: three files, 106 tests
passed.
- Package conformance and recovery: four files, six tests passed.
- Phase 6 acceptance-matrix entry points: ten files, twenty-five tests passed
with zero skips, including the database-backed selected-task canary, four
atomic-liveness failpoints, migration, sequencing, bounded recovery, and
legacy compatibility.
- Focused database rehearsal: five files, eleven tests passed with zero skips;
the legacy event/read snapshot was byte-equivalent and had zero native rows.
- Section 18.13 source corpus: 52 fixtures, all 70 matrix rows, and 18 structural
checks passed with every fixture bound to a named consumer.
- Runner, shared, server, and database typechecks passed; migration safety
passed.
@@ -69,6 +80,10 @@ existing server authority and keep legacy execution as the default.
- Live operator proof depends on a board-authorized local instance and an
already authenticated Codex installation; deterministic CI uses the same
public session contract with a scripted backend.
- The remediation did not dispatch a new live provider task or mutate rollout
settings. The verified “internal canary” and post-kill-switch trace are the
real embedded-PostgreSQL selected-task and legacy-snapshot cases; the live
tutorial remains an explicit operator checkpoint.
# Follow-up questions
@@ -33,7 +33,7 @@
{
"id": "incomplete-evidence-no-live-path",
"mode": "native",
"covers": { "decisionRows": ["SD-03"], "terminalRows": [], "attentionRows": [], "livenessRows": ["LIVE-06"], "reconciliationRows": [], "compatibilityRows": [], "migrationRows": [] },
"covers": { "decisionRows": [], "terminalRows": [], "attentionRows": [], "livenessRows": [], "reconciliationRows": [], "compatibilityRows": [], "migrationRows": [] },
"tags": ["incomplete_evidence", "atomic_liveness"],
"given": { "priorIssueStatus": "in_progress", "turnTerminalState": "completed", "runTerminalState": "succeeded", "reportedWorkDisposition": "done", "nativeFinalization": "present", "completionState": "missing_required_test", "trigger": "runner_finalizer", "fault": "continuation_insert_failure" },
"expected": { "runStatus": "failed", "statusAction": "preserve", "reasonCode": "side_effect_planning_failed", "requiredEffects": ["record_finalization_error"], "forbiddenEffects": ["enqueue_continuation", "status_version_increment"], "livePathKind": null, "preserveClaim": true, "nativeRecords": true, "decisionCount": 0, "maxWakeCount": 0, "maxNotificationCount": 0 },
@@ -42,7 +42,7 @@
{
"id": "prior-status-preserved-without-live-path",
"mode": "native",
"covers": { "decisionRows": [], "terminalRows": [], "attentionRows": [], "livenessRows": [], "reconciliationRows": [], "compatibilityRows": [], "migrationRows": [] },
"covers": { "decisionRows": ["SD-03"], "terminalRows": [], "attentionRows": [], "livenessRows": ["LIVE-06"], "reconciliationRows": [], "compatibilityRows": [], "migrationRows": [] },
"tags": ["atomic_liveness", "reconciliation"],
"given": { "priorIssueStatus": "in_progress", "turnTerminalState": "completed", "runTerminalState": "succeeded", "reportedWorkDisposition": "yielded", "nativeFinalization": "present", "completionState": "no_durable_continuation", "trigger": "runner_finalizer" },
"expected": { "runStatus": "failed", "statusAction": "preserve", "reasonCode": "prior_status_preserved_no_live_path", "requiredEffects": ["record_finalization_error"], "forbiddenEffects": ["set_issue_status", "enqueue_continuation", "bind_blocker", "bind_reviewer"], "livePathKind": "recovery", "preserveClaim": true, "nativeRecords": true, "decisionCount": 1, "maxWakeCount": 1, "maxNotificationCount": 0 },
@@ -56,6 +56,7 @@ export class HarnessDriverBackend implements NativeSessionBackend {
providerSessionId: snapshot.providerSessionId,
runId: snapshot.identity.runId,
normalizedSessionId: snapshot.identity.sessionId,
activeTurnId: snapshot.activeTurnId ?? snapshot.terminalTurns?.at(-1)?.turnId ?? null,
lastSourceSequence: parseCursor(snapshot.cursor),
...(snapshot.semanticResult === undefined || snapshot.semanticResult === null
? {}
@@ -63,9 +64,12 @@ export class HarnessDriverBackend implements NativeSessionBackend {
semanticResult: {
result: snapshot.semanticResult,
fingerprint: canonicalJson(snapshot.semanticResult),
turnId: "recovered",
},
}),
turnId: snapshot.activeTurnId ?? snapshot.terminalTurns?.at(-1)?.turnId ?? "recovered",
},
}),
terminalTurns: snapshot.terminalTurns ?? [],
pendingRuntimeRequests: snapshot.pendingRuntimeRequests ?? [],
lineage: snapshot.lineage ?? [],
};
const recovered = await this.#driver.recoverSession(persisted);
if (!recovered.recovered || recovered.session === undefined) {
@@ -181,6 +185,10 @@ class HarnessNativeSession implements NativeSession {
: String(snapshot.lastSourceSequence),
semanticResult: snapshot.semanticResult?.result ?? null,
terminal: this.#terminal,
activeTurnId: snapshot.activeTurnId ?? snapshot.semanticResult?.turnId ?? null,
terminalTurns: snapshot.terminalTurns ?? [],
pendingRuntimeRequests: snapshot.pendingRuntimeRequests ?? [],
lineage: snapshot.lineage ?? [],
};
}
@@ -41,7 +41,7 @@ export const CONTROL_PLANE_CONFORMANCE_RESULT: PrpStructuredRunResult = {
remainingWork: [],
},
evidence: [{ kind: "event", ref: "event:2" }],
verification: [{ commandOrCheck: "phase6-conformance", status: "passed" }],
verification: [{ commandOrCheck: "phase6-conformance", status: "passed", artifactRef: "event:2" }],
attentionRequests: [],
artifacts: [],
};
@@ -8,6 +8,7 @@ import type {
PrpStructuredRunResult,
PrpTerminalState,
} from "../protocol/phase1-contract.js";
import type { PersistedNativeSession } from "./native-session-backend.js";
export interface OpenControlPlaneRunInput {
identity: NativeRunIdentity;
@@ -49,6 +50,8 @@ export interface CompleteControlPlaneRunInput {
*/
export interface ControlPlanePort {
openRun(input: OpenControlPlaneRunInput): Promise<void>;
loadSessionCheckpoint?(): Promise<PersistedNativeSession | null>;
checkpointSession?(snapshot: PersistedNativeSession): Promise<void>;
appendEvent(event: NativeRunEvent | PrpEvent): Promise<AppendedEventReceipt>;
replayEvents(input: ReplayControlPlaneEventsInput): Promise<ReplayedControlPlaneEvents>;
completeRun(result: NativeRunResult | CompleteControlPlaneRunInput): Promise<void>;
@@ -8,6 +8,11 @@ import type {
PrpStructuredRunResult,
PrpTerminalState,
} from "../protocol/phase1-contract.js";
import type {
HarnessRuntimeRequest,
HarnessThreadLineageEntry,
PersistedHarnessTurnTerminal,
} from "./harness-driver.js";
export interface NativeSessionBackendDescriptor {
kind: "runner" | "remote" | "mock";
@@ -29,6 +34,10 @@ export interface PersistedNativeSession {
cursor?: string | null;
semanticResult?: PrpStructuredRunResult | null;
terminal?: PrpTerminalState | null;
activeTurnId?: string | null;
terminalTurns?: PersistedHarnessTurnTerminal[];
pendingRuntimeRequests?: HarnessRuntimeRequest[];
lineage?: HarnessThreadLineageEntry[];
}
export interface NativeSessionRecoveryResult {
@@ -0,0 +1,185 @@
import { describe, expect, it, vi } from "vitest";
import type { ControlPlanePort } from "./contracts/control-plane-port.js";
import type { NativeExecutionInputV1 } from "./contracts/native-execution.js";
import type {
NativeSession,
NativeSessionBackend,
PersistedNativeSession,
} from "./contracts/native-session-backend.js";
import type {
PrpEvent,
PrpStructuredRunResult,
PrpTerminalState,
} from "./protocol/phase1-contract.js";
import { executeNativeSession } from "./native-session-runtime.js";
const identity = {
runId: "run-recovery",
sessionId: "session-recovery",
companyId: "company-recovery",
issueId: "issue-recovery",
agentId: "agent-recovery",
};
const result: PrpStructuredRunResult = {
schema: "paperclip.run_result.v1",
reportedWorkDisposition: "done",
summary: "Recovered native work completed.",
completionClaim: {
contractRevision: "1",
objectiveSatisfied: true,
criteria: [{ criterionId: "objective", status: "satisfied", evidenceRefs: [] }],
remainingWork: [],
},
evidence: [],
verification: [{ commandOrCheck: "recovery", status: "passed" }],
attentionRequests: [],
artifacts: [],
};
const terminal: PrpTerminalState = {
schema: "paperclip.prp.terminal.v1",
turnTerminalState: "completed",
runTerminalState: "succeeded",
reportedWorkDisposition: "done",
};
const input: NativeExecutionInputV1 = {
schema: "paperclip.native-execution-input.v1",
binding: {
companyId: identity.companyId,
runId: identity.runId,
issueId: identity.issueId,
agentId: identity.agentId,
executionWorkspaceId: "workspace-recovery",
},
task: { identifier: "PAP-RECOVERY", title: "Recover native work", description: null, workMode: "standard" },
workspace: { cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null },
session: { normalizedSessionId: identity.sessionId, driverKind: "codex_app_server", protocolVersion: 1 },
completionContract: {
id: "contract-recovery",
sha256: "contract-recovery-sha",
schemaVersion: "paperclip.completion-contract.v1",
contract: {
revision: "1",
objective: "Recover native work",
criteria: [{ id: "objective", requirement: "Complete after recovery" }],
},
},
interactionResponses: [],
credentialBindings: [],
};
function controlEvent(
sourceSeq: number,
eventType: PrpEvent["eventType"],
payload: Record<string, unknown>,
): PrpEvent {
return {
schema: "paperclip.prp.event.v1",
sourceEventId: `control-recovery:${identity.runId}:${sourceSeq}`,
sourceSeq,
sourceInstanceId: "control-recovery",
sourceKind: "control_plane",
runId: identity.runId,
normalizedSessionId: identity.sessionId,
turnId: "turn-recovery",
eventType,
schemaVersion: 1,
priority: 0,
emittedAt: "2026-08-09T00:00:00.000Z",
payload,
};
}
function highestContiguous(events: PrpEvent[]): number {
const sequences = new Set(events.map((event) => event.sourceSeq));
let cursor = 0;
while (sequences.has(cursor + 1)) cursor += 1;
return cursor;
}
describe("executeNativeSession recovery", () => {
it("recovers a completed checkpoint and appends only a missing control terminal fact", async () => {
const checkpoint: PersistedNativeSession = {
backendKind: "mock",
sessionId: "driver-recovery",
identity,
providerSessionId: "provider-recovery",
cursor: "4",
semanticResult: result,
terminal,
activeTurnId: "turn-recovery",
terminalTurns: [{ turnId: "turn-recovery", fingerprint: "terminal-fingerprint" }],
pendingRuntimeRequests: [],
lineage: [],
};
const events = [controlEvent(1, "run.result.accepted", { result })];
const checkpoints: PersistedNativeSession[] = [];
const completeRun = vi.fn(async () => undefined);
const startTurn = vi.fn(async () => ({ turnId: "unexpected-turn" }));
const openSession = vi.fn(async () => {
throw new Error("a recovered run must not open a second provider session");
});
const session: NativeSession = {
identity: () => identity,
async capabilities() {
return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };
},
async *events() {},
startTurn,
async result() { return { result, terminal, turnId: "turn-recovery" }; },
async snapshot() { return structuredClone(checkpoint); },
async close() {},
};
const recoverSession = vi.fn(async () => ({ recovered: true, session }));
const backend: NativeSessionBackend = {
async descriptor() {
return {
kind: "mock",
name: "recovery-backend",
version: "1",
capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },
};
},
openSession,
recoverSession,
};
const port: ControlPlanePort = {
async openRun() {},
async loadSessionCheckpoint() { return structuredClone(checkpoint); },
async checkpointSession(snapshot) { checkpoints.push(structuredClone(snapshot)); },
async appendEvent(event) {
events.push(structuredClone(event as PrpEvent));
return {
cursor: events.length,
highestContiguousSourceSeq: highestContiguous(events),
disposition: "committed",
};
},
async replayEvents(replay) {
const replayed = events.filter((event) => event.sourceSeq > replay.afterSourceSeq);
return { events: structuredClone(replayed), highestContiguousSourceSeq: highestContiguous(events) };
},
completeRun,
};
const completed = await executeNativeSession({
input,
backend,
controlPlane: port,
runnerInstanceId: "runner-recovery",
controlPlaneInstanceId: "control-recovery",
});
expect(openSession).not.toHaveBeenCalled();
expect(recoverSession).toHaveBeenCalledOnce();
expect(startTurn).not.toHaveBeenCalled();
expect(events.map((event) => event.eventType)).toEqual(["run.result.accepted", "run.terminal"]);
expect(events.map((event) => event.sourceSeq)).toEqual([1, 2]);
expect(completeRun).toHaveBeenCalledOnce();
expect(completed).toMatchObject({ nativeEventCount: 1, highestContiguousSourceSeq: 2 });
expect(checkpoints.at(-1)).toMatchObject({ semanticResult: result, terminal });
});
});
@@ -62,7 +62,10 @@ async function consumeTurn(session: NativeSession, controlPlane: ControlPlanePor
export async function executeNativeSession(options: ExecuteNativeSessionOptions): Promise<NativeSessionExecutionResult> {
const input = parseNativeExecutionInput(options.input);
const descriptor = await options.backend.descriptor();
const normalizedSessionId = input.session.normalizedSessionId ?? randomUUID();
const persistedSession = await options.controlPlane.loadSessionCheckpoint?.() ?? null;
const normalizedSessionId = persistedSession?.identity.sessionId
?? input.session.normalizedSessionId
?? randomUUID();
await options.controlPlane.openRun({
identity: {
runId: input.binding.runId,
@@ -75,49 +78,134 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)
sourceInstanceId: options.runnerInstanceId,
});
const session = await options.backend.openSession({
identity: {
runId: input.binding.runId,
sessionId: normalizedSessionId,
companyId: input.binding.companyId,
issueId: input.binding.issueId,
agentId: input.binding.agentId,
},
workingDirectory: input.workspace.cwd,
});
const identity = {
runId: input.binding.runId,
sessionId: normalizedSessionId,
companyId: input.binding.companyId,
issueId: input.binding.issueId,
agentId: input.binding.agentId,
};
if (
persistedSession
&& (
persistedSession.identity.runId !== identity.runId
|| persistedSession.identity.companyId !== identity.companyId
|| persistedSession.identity.issueId !== identity.issueId
|| persistedSession.identity.agentId !== identity.agentId
|| persistedSession.identity.sessionId !== identity.sessionId
)
) throw new Error("native_session_checkpoint_binding_mismatch");
let recovered = false;
let session: NativeSession;
if (persistedSession) {
if (!options.backend.recoverSession) {
throw new Error("native_session_recovery_unavailable: refusing to open a second provider session");
}
const recovery = await options.backend.recoverSession(persistedSession);
if (!recovery.recovered || !recovery.session) {
throw new Error(`native_session_recovery_failed: ${recovery.reason ?? "unknown"}`);
}
session = recovery.session;
recovered = true;
} else {
session = await options.backend.openSession({
identity,
workingDirectory: input.workspace.cwd,
});
}
options.onSession?.(session);
try {
const consuming = consumeTurn(session, options.controlPlane, options.timeoutMs ?? 900_000);
await session.startTurn({
message: { role: "user", text: JSON.stringify(buildNativeModelEnvelope(input)) },
});
const consumed = await consuming;
const completed = await session.result();
const checkpoint = async () => {
if (options.controlPlane.checkpointSession) {
await options.controlPlane.checkpointSession(await session.snapshot());
}
};
await checkpoint();
let consumed = { event: null as PrpEvent | null, eventCount: 0, highestContiguousSourceSeq: 0 };
let completed = persistedSession?.semanticResult && persistedSession.terminal
? {
result: persistedSession.semanticResult,
terminal: persistedSession.terminal,
turnId: persistedSession.activeTurnId ?? null,
}
: null;
if (!completed) {
const consuming = consumeTurn(session, options.controlPlane, options.timeoutMs ?? 900_000);
if (!recovered || !persistedSession?.activeTurnId) {
await session.startTurn({
message: { role: "user", text: JSON.stringify(buildNativeModelEnvelope(input)) },
});
await checkpoint();
}
const terminalEvent = await consuming;
consumed = terminalEvent;
completed = await session.result();
await checkpoint();
}
if (completed === null) throw new Error("native_finalization_missing: session returned no semantic result");
const terminal = terminalFromEvent(consumed.event, completed.result.reportedWorkDisposition);
let controlSeq = 0;
const controlEvent = (eventType: PrpEvent["eventType"], payload: Record<string, unknown>): PrpEvent => ({
let terminal: PrpTerminalState;
if (persistedSession?.semanticResult && persistedSession.terminal) {
terminal = persistedSession.terminal;
} else {
terminal = terminalFromEvent(consumed.event!, completed.result.reportedWorkDisposition);
}
const eventTurnId = completed.turnId
?? persistedSession?.activeTurnId
?? persistedSession?.terminalTurns?.at(-1)?.turnId
?? consumed.event?.turnId;
const controlEvent = (
sourceSeq: number,
eventType: PrpEvent["eventType"],
payload: Record<string, unknown>,
): PrpEvent => ({
schema: "paperclip.prp.event.v1",
sourceEventId: `${options.controlPlaneInstanceId}:${input.binding.runId}:${++controlSeq}`,
sourceSeq: controlSeq,
sourceEventId: `${options.controlPlaneInstanceId}:${input.binding.runId}:${sourceSeq}`,
sourceSeq,
sourceInstanceId: options.controlPlaneInstanceId,
sourceKind: "control_plane",
runId: input.binding.runId,
normalizedSessionId,
turnId: completed.turnId ?? consumed.event.turnId,
...(eventTurnId ? { turnId: eventTurnId } : {}),
eventType,
schemaVersion: 1,
priority: 0,
emittedAt: new Date().toISOString(),
payload,
});
for (const event of [
controlEvent("run.result.accepted", { result: completed.result }),
controlEvent("run.terminal", terminal as unknown as Record<string, unknown>),
]) {
const expectedControlEvents = [
controlEvent(1, "run.result.accepted", { result: completed.result }),
controlEvent(2, "run.terminal", terminal as unknown as Record<string, unknown>),
];
const controlReplay = await options.controlPlane.replayEvents({
runId: input.binding.runId,
sourceInstanceId: options.controlPlaneInstanceId,
afterSourceSeq: 0,
limit: 10,
});
const replayBySequence = new Map(controlReplay.events.map((event) => [event.sourceSeq, event]));
for (const existing of controlReplay.events) {
const expected = expectedControlEvents[existing.sourceSeq - 1];
if (
expected === undefined
|| existing.eventType !== expected.eventType
|| canonicalJson(existing.payload) !== canonicalJson(expected.payload)
) {
throw new Error(`native_control_event_replay_conflict:${existing.sourceSeq}`);
}
}
consumed.highestContiguousSourceSeq = Math.max(
consumed.highestContiguousSourceSeq,
controlReplay.highestContiguousSourceSeq,
);
for (const event of expectedControlEvents) {
if (replayBySequence.has(event.sourceSeq)) continue;
const receipt = await options.controlPlane.appendEvent(event);
consumed.eventCount += receipt.disposition === "committed" ? 1 : 0;
consumed.highestContiguousSourceSeq = Math.max(consumed.highestContiguousSourceSeq, receipt.highestContiguousSourceSeq);
consumed.highestContiguousSourceSeq = Math.max(
consumed.highestContiguousSourceSeq,
receipt.highestContiguousSourceSeq,
);
}
await options.controlPlane.completeRun({
result: completed.result,
@@ -127,6 +215,11 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)
callerDedupeKey: `${input.binding.runId}:${input.completionContract.sha256}`,
});
const snapshot = await session.snapshot();
await options.controlPlane.checkpointSession?.({
...snapshot,
semanticResult: completed.result,
terminal,
});
return {
result: completed.result,
terminal,
@@ -144,3 +237,15 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)
await session.close({ reason: "native session execution complete" });
}
}
function canonicalJson(value: unknown): string {
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
if (typeof value === "object" && value !== null) {
const record = value as Record<string, unknown>;
return `{${Object.keys(record)
.sort()
.map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`)
.join(",")}}`;
}
return JSON.stringify(value) ?? "undefined";
}
@@ -0,0 +1,2 @@
// Acceptance-matrix entry point for budget/manual cancellation idempotency.
import "../services/native-runtime/native-session-executor.test.js";
@@ -0,0 +1,2 @@
// Acceptance-matrix entry point for default-off, opt-in, and eligibility rules.
import "../services/native-runtime/runtime-mode.test.js";
@@ -0,0 +1,84 @@
import { describe, expect, it } from "vitest";
import { eq } from "drizzle-orm";
import {
agents,
companies,
createDb,
heartbeatRunEvents,
heartbeatRuns,
} from "@paperclipai/db";
import { appendHeartbeatRunEvent, HeartbeatRunEventConflictError } from "../services/heartbeat-run-events.js";
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
describe("P6-11..13 / P6-17 canonical event allocator", () => {
it("serializes concurrent writers and rejects conflicting replay without cursor drift", async () => {
const temporary = await startEmbeddedPostgresTestDatabase("paperclip-native-events-");
const db = createDb(temporary.connectionString);
const companyId = "20000000-0000-4000-8000-000000000001";
const agentId = "20000000-0000-4000-8000-000000000002";
const runId = "20000000-0000-4000-8000-000000000003";
try {
await db.insert(companies).values({ id: companyId, name: "Allocator fixture", issuePrefix: "SEQ" });
await db.insert(agents).values({ id: agentId, companyId, name: "Allocator agent" });
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running" });
const eventTypes = ["heartbeat.started", "run.cancel.requested", "item.completed", "stdout"];
const writes = Array.from({ length: 32 }, (_, index) => appendHeartbeatRunEvent(db, {
companyId,
runId,
agentId,
eventType: eventTypes[index % eventTypes.length]!,
message: `event-${index + 1}`,
payload: { ordinal: index + 1 },
nativeSource: {
sourceInstanceId: `writer-${index % 4}`,
sourceEventId: `source-event-${index + 1}`,
sourceSeq: Math.floor(index / 4) + 1,
protocolSchemaVersion: 1,
canonicalPayload: { ordinal: index + 1 },
},
}));
const receipts = await Promise.all(writes);
expect(receipts.every((entry) => entry.disposition === "committed")).toBe(true);
const rows = await db.select().from(heartbeatRunEvents)
.where(eq(heartbeatRunEvents.runId, runId)).orderBy(heartbeatRunEvents.seq);
expect(rows.map((row) => row.seq)).toEqual(Array.from({ length: 32 }, (_, index) => index + 1));
expect(new Set(rows.map((row) => row.sourceEventId)).size).toBe(32);
expect((await db.select({ nextEventSeq: heartbeatRuns.nextEventSeq }).from(heartbeatRuns)
.where(eq(heartbeatRuns.id, runId)))[0]?.nextEventSeq).toBe(33);
const original = rows.find((row) => row.sourceEventId === "source-event-1")!;
await expect(appendHeartbeatRunEvent(db, {
companyId,
runId,
agentId,
eventType: original.eventType,
message: original.message,
payload: original.payload,
nativeSource: {
sourceInstanceId: original.sourceInstanceId!,
sourceEventId: original.sourceEventId!,
sourceSeq: original.sourceSeq!,
protocolSchemaVersion: 1,
canonicalPayload: { ordinal: 1 },
},
})).resolves.toEqual(expect.objectContaining({ disposition: "duplicate" }));
await expect(appendHeartbeatRunEvent(db, {
companyId,
runId,
agentId,
eventType: "item.failed",
nativeSource: {
sourceInstanceId: original.sourceInstanceId!,
sourceEventId: original.sourceEventId!,
sourceSeq: original.sourceSeq!,
protocolSchemaVersion: 1,
canonicalPayload: { ordinal: 999 },
},
})).rejects.toBeInstanceOf(HeartbeatRunEventConflictError);
expect(await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, runId))).toHaveLength(32);
} finally {
await temporary.cleanup();
}
}, 60_000);
});
@@ -0,0 +1,80 @@
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { eq } from "drizzle-orm";
import {
agents,
companies,
completionContracts,
createDb,
heartbeatRunEvents,
heartbeatRuns,
nativeRunFinalizations,
nativeRunResults,
statusDecisions,
workAssessments,
} from "@paperclipai/db";
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { reconcileNativeFinalizations } from "../services/native-runtime/native-finalization-reconciler.js";
describe("P6-32 legacy finalization regression", () => {
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let db: ReturnType<typeof createDb>;
const companyId = "71000000-0000-4000-8000-000000000001";
const agentId = "71000000-0000-4000-8000-000000000002";
const runId = "71000000-0000-4000-8000-000000000003";
beforeAll(async () => {
temporary = await startEmbeddedPostgresTestDatabase("paperclip-native-legacy-");
db = createDb(temporary.connectionString);
await db.insert(companies).values({ id: companyId, name: "Legacy snapshot", issuePrefix: "LGC" });
await db.insert(agents).values({
id: agentId,
companyId,
name: "Legacy agent",
adapterType: "codex_local",
status: "idle",
});
await db.insert(heartbeatRuns).values({
id: runId,
companyId,
agentId,
status: "succeeded",
runtimeMode: "legacy",
nextEventSeq: 8,
exitCode: 0,
resultJson: { summary: "Legacy bytes", nested: { count: 1, ok: true } },
contextSnapshot: { issueId: "legacy-opaque-issue" },
});
await db.insert(heartbeatRunEvents).values({
companyId,
runId,
agentId,
seq: 7,
eventType: "lifecycle",
stream: "system",
message: "Legacy event bytes",
payload: { state: "completed", nested: [1, "two", false] },
});
}, 30_000);
afterAll(async () => {
await temporary.cleanup();
});
it("keeps the post-kill-switch legacy read snapshot byte-equivalent with zero native rows", async () => {
const beforeRun = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
const beforeEvents = await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, runId));
const beforeBytes = JSON.stringify({ run: beforeRun, events: beforeEvents });
await expect(reconcileNativeFinalizations(db, [runId])).resolves.toEqual([]);
const afterRun = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
const afterEvents = await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, runId));
expect(JSON.stringify({ run: afterRun, events: afterEvents })).toBe(beforeBytes);
await expect(db.select().from(completionContracts).where(eq(completionContracts.companyId, companyId))).resolves.toHaveLength(0);
await expect(db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, runId))).resolves.toHaveLength(0);
await expect(db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId))).resolves.toHaveLength(0);
await expect(db.select().from(workAssessments).where(eq(workAssessments.runId, runId))).resolves.toHaveLength(0);
await expect(db.select().from(statusDecisions)
.where(eq(statusDecisions.companyId, companyId))).resolves.toHaveLength(0);
});
});
@@ -0,0 +1,67 @@
import { readFile } from "node:fs/promises";
import { describe, expect, it } from "vitest";
import { eq, sql } from "drizzle-orm";
import {
agents,
companies,
createDb,
heartbeatRunEvents,
heartbeatRuns,
} from "@paperclipai/db";
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
describe("P6-18 / MIG-01..04 native finalization migration", () => {
it("repairs only later duplicates and preserves legacy event bytes and cursors", async () => {
const temporary = await startEmbeddedPostgresTestDatabase("paperclip-native-migration-");
const db = createDb(temporary.connectionString);
try {
const companyId = "10000000-0000-4000-8000-000000000001";
const agentId = "10000000-0000-4000-8000-000000000002";
const runId = "10000000-0000-4000-8000-000000000003";
await db.insert(companies).values({ id: companyId, name: "Migration fixture", issuePrefix: "MIG" });
await db.insert(agents).values({ id: agentId, companyId, name: "Migration agent" });
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "succeeded", nextEventSeq: 1 });
await db.execute(sql.raw("DROP INDEX heartbeat_run_events_run_seq_uq"));
await db.insert(heartbeatRunEvents).values([
{ companyId, runId, agentId, seq: 1, eventType: "legacy.start", stream: "system", level: "info", message: "one", payload: { bytes: "α-1" }, createdAt: new Date("2026-08-01T00:00:01.000Z") },
{ companyId, runId, agentId, seq: 5, eventType: "legacy.log", stream: "stdout", level: "info", message: "first-five", payload: { bytes: "β-5a" }, createdAt: new Date("2026-08-01T00:00:02.000Z") },
{ companyId, runId, agentId, seq: 5, eventType: "legacy.log", stream: "stderr", level: "warn", message: "duplicate-five", payload: { bytes: "γ-5b" }, createdAt: new Date("2026-08-01T00:00:03.000Z") },
{ companyId, runId, agentId, seq: 9, eventType: "legacy.end", stream: "system", level: "info", message: "nine", payload: { bytes: "δ-9" }, createdAt: new Date("2026-08-01T00:00:04.000Z") },
]);
const before = await db.select().from(heartbeatRunEvents)
.where(eq(heartbeatRunEvents.runId, runId)).orderBy(heartbeatRunEvents.id);
const migration = await readFile(
new URL("../../../packages/db/src/migrations/0211_famous_guardsmen.sql", import.meta.url),
"utf8",
);
const statements = migration.split("--> statement-breakpoint").map((entry) => entry.trim());
const duplicateRepair = statements.find((entry) => entry.includes("WITH ranked AS"));
const allocatorSeed = statements.find((entry) => entry.startsWith('UPDATE "heartbeat_runs" AS run'));
expect(duplicateRepair).toBeTruthy();
expect(allocatorSeed).toBeTruthy();
await db.execute(sql.raw(duplicateRepair!));
await db.execute(sql.raw(allocatorSeed!));
await db.execute(sql.raw(
'CREATE UNIQUE INDEX "heartbeat_run_events_run_seq_uq" ON "heartbeat_run_events" ("run_id", "seq")',
));
const after = await db.select().from(heartbeatRunEvents)
.where(eq(heartbeatRunEvents.runId, runId)).orderBy(heartbeatRunEvents.id);
expect(after.map((row) => row.seq)).toEqual([1, 5, 10, 9]);
expect((await db.select({ nextEventSeq: heartbeatRuns.nextEventSeq }).from(heartbeatRuns)
.where(eq(heartbeatRuns.id, runId)))[0]?.nextEventSeq).toBe(11);
// The repaired duplicate's cursor is the only changed byte-equivalent read field.
expect(after.map(({ seq: _seq, ...row }) => row)).toEqual(before.map(({ seq: _seq, ...row }) => row));
expect(after[0]?.seq).toBe(before[0]?.seq);
expect(after[1]?.seq).toBe(before[1]?.seq);
expect(after[3]?.seq).toBe(before[3]?.seq);
await expect(db.insert(heartbeatRunEvents).values({
companyId, runId, agentId, seq: 5, eventType: "must-conflict",
})).rejects.toThrow();
} finally {
await temporary.cleanup();
}
}, 60_000);
});
@@ -0,0 +1,154 @@
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { eq } from "drizzle-orm";
import {
agents,
companies,
completionContracts,
createDb,
heartbeatRuns,
issueRecoveryActions,
issues,
nativeRunFinalizations,
nativeRunResults,
statusDecisions,
workspaceOperations,
} from "@paperclipai/db";
import {
CONTROL_PLANE_CONFORMANCE_OPEN,
CONTROL_PLANE_CONFORMANCE_RESULT,
CONTROL_PLANE_CONFORMANCE_TERMINAL,
} from "@paperclipai/paperclip-runner";
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { reconcileNativeFinalizations } from "../services/native-runtime/native-finalization-reconciler.js";
import { PaperclipControlPlanePort } from "../services/native-runtime/paperclip-control-plane-port.js";
describe("P6-16/P6-25/P6-28 native finalization recovery", () => {
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let db: ReturnType<typeof createDb>;
const companyId = "72000000-0000-4000-8000-000000000001";
const agentId = "72000000-0000-4000-8000-000000000002";
const issueId = "72000000-0000-4000-8000-000000000003";
const contractId = "72000000-0000-4000-8000-000000000004";
const runId = "72000000-0000-4000-8000-000000000005";
beforeAll(async () => {
temporary = await startEmbeddedPostgresTestDatabase("paperclip-native-recovery-");
db = createDb(temporary.connectionString);
await db.insert(companies).values({ id: companyId, name: "Native recovery", issuePrefix: "NRC" });
await db.insert(agents).values({
id: agentId,
companyId,
name: "Recovery agent",
adapterType: "codex_local",
status: "running",
});
await db.insert(issues).values({
id: issueId,
companyId,
title: "Recover invalid native finalization",
status: "in_progress",
assigneeAgentId: agentId,
workMode: "standard",
});
await db.insert(completionContracts).values({
id: contractId,
companyId,
issueId,
revision: 1,
schemaVersion: "paperclip.completion-contract.v1",
policyVersion: "phase6-v1",
risk: "standard",
completionAuthority: "server_arbiter",
incompleteCriteriaPolicy: "preserve_non_terminal",
contractJson: {
revision: "phase6-v1",
objective: "Recover invalid native finalization",
criteria: [{ id: "objective", requirement: "Recovery remains live" }],
},
canonicalSha256: "native-recovery-contract",
createdByActorType: "system",
createdByActorId: "test",
});
await db.insert(heartbeatRuns).values({
id: runId,
companyId,
agentId,
status: "running",
runtimeMode: "native",
runtimeModeReason: "persisted_before_kill_switch",
completionContractId: contractId,
completionContractSha256: "native-recovery-contract",
contextSnapshot: { issueId },
});
const port = new PaperclipControlPlanePort(db, {
companyId,
issueId,
runId,
agentId,
completionContractId: contractId,
completionContractSha256: "native-recovery-contract",
sourceInstanceId: "recovery-runner",
controlPlaneSourceInstanceId: "recovery-control",
});
await port.openRun({
...CONTROL_PLANE_CONFORMANCE_OPEN,
identity: { companyId, issueId, runId, agentId, normalizedSessionId: "recovery-session" },
sourceInstanceId: "recovery-runner",
});
await port.completeRun({
result: CONTROL_PLANE_CONFORMANCE_RESULT,
terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL,
callerResultId: "recovery-result",
});
const stored = await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, runId))
.limit(1).then((rows) => rows[0]!);
await db.update(nativeRunResults).set({
resultJson: {
...(stored.resultJson as Record<string, unknown>),
terminal: { ...CONTROL_PLANE_CONFORMANCE_TERMINAL, runTerminalState: "unknown" },
},
}).where(eq(nativeRunResults.id, stored.id));
await db.insert(workspaceOperations).values({
companyId,
heartbeatRunId: runId,
issueId,
phase: "workspace_finalize",
status: "succeeded",
});
}, 30_000);
afterAll(async () => {
await temporary.cleanup();
});
it("fails closed into bounded named recovery without consulting the live flag or falling back", async () => {
await expect(reconcileNativeFinalizations(db, [runId])).resolves.toEqual([
expect.objectContaining({ phase: "retryable_failure", failureCode: "native_finalization_invalid" }),
]);
await expect(db.select().from(issues).where(eq(issues.id, issueId))).resolves.toEqual([
expect.objectContaining({ status: "in_progress", statusVersion: 0, lastStatusDecisionId: null }),
]);
await expect(db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId))).resolves.toEqual([
expect.objectContaining({ status: "active", ownerAgentId: agentId, cause: "native_finalization_invalid" }),
]);
await expect(db.select().from(statusDecisions).where(eq(statusDecisions.issueId, issueId))).resolves.toHaveLength(0);
for (let retry = 0; retry < 2; retry += 1) {
await db.update(nativeRunFinalizations).set({ nextAttemptAt: new Date(0) })
.where(eq(nativeRunFinalizations.runId, runId));
await reconcileNativeFinalizations(db, [runId]);
}
await expect(db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId))).resolves.toEqual([
expect.objectContaining({
phase: "terminal_failure",
attempt: 3,
failureCode: "native_finalization_retry_exhausted",
nextAttemptAt: null,
}),
]);
await expect(db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId))).resolves.toEqual([
expect.objectContaining({ runtimeMode: "native", status: "failed", nativePhase: "terminal_failure" }),
]);
await expect(reconcileNativeFinalizations(db, [runId])).resolves.toEqual([]);
});
});
@@ -0,0 +1,3 @@
// Acceptance-matrix entry point for evidence, finalization, CAS, and atomic
// status/liveness effects exercised by the real database-backed port suite.
import "../services/native-runtime/paperclip-control-plane-port.test.js";
@@ -0,0 +1,2 @@
// Acceptance-matrix entry point for the package-owned closed input parser.
import "../../../packages/paperclip-runner/src/contracts/native-execution.test.js";
@@ -0,0 +1,3 @@
// Acceptance-matrix entry point. The database-backed suite remains colocated
// with the production port so its package conformance fixtures stay adjacent.
import "../services/native-runtime/paperclip-control-plane-port.test.js";
@@ -0,0 +1,95 @@
import { createHash } from "node:crypto";
import { readFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
import { describe, expect, it } from "vitest";
type Fixture = {
id: string;
covers: Record<string, string[]>;
expected: Record<string, unknown>;
};
type Corpus = {
schema: string;
corpusRevision: number;
fixtures: Fixture[];
};
const corpusPath = fileURLToPath(new URL(
"../../../packages/paperclip-runner/spec/fixtures/status-authority-phase5.json",
import.meta.url,
));
const consumerByMatrixPrefix: Record<string, string> = {
SD: "server/native-status-arbiter-and-committer",
TC: "package/native-session-runtime",
ATT: "server/native-interaction-bridge",
LIVE: "server/atomic-liveness-failpoints",
REC: "server/native-finalization-recovery",
COMP: "server/legacy-finalization-regression",
MIG: "server/native-finalization-migration",
};
function canonical(value: unknown): string {
if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`;
if (value && typeof value === "object") {
return `{${Object.entries(value as Record<string, unknown>)
.sort(([left], [right]) => left.localeCompare(right))
.map(([key, entry]) => `${JSON.stringify(key)}:${canonical(entry)}`)
.join(",")}}`;
}
return JSON.stringify(value) ?? "null";
}
function emitConsumerResults(corpus: Corpus) {
return corpus.fixtures.flatMap((fixture) => {
const coveredRows = Object.values(fixture.covers).flat();
// Revision 2 contains one negative fixture that intentionally has no
// matrix-row claim. It still needs an independently joinable consumer.
if (coveredRows.length === 0) {
return [{
corpusRevision: corpus.corpusRevision,
fixtureId: fixture.id,
matrixRow: "fixture-only",
consumer: "server/native-finalization-recovery",
expectedDigest: createHash("sha256").update(canonical(fixture.expected)).digest("hex"),
}];
}
return coveredRows.map((matrixRow) => {
const prefix = matrixRow.split("-")[0]!;
const consumer = consumerByMatrixPrefix[prefix];
if (!consumer) throw new Error(`missing_consumer:${matrixRow}`);
return {
corpusRevision: corpus.corpusRevision,
fixtureId: fixture.id,
matrixRow,
consumer,
expectedDigest: createHash("sha256").update(canonical(fixture.expected)).digest("hex"),
};
});
});
}
describe("P6-31 Section 18.13 status-authority corpus traceability", () => {
it("emits one result for every Section 18.13 fixture and consumer", () => {
const corpus = JSON.parse(readFileSync(corpusPath, "utf8")) as Corpus;
const results = emitConsumerResults(corpus);
const coveredFixtureIds = new Set(results.map((result) => result.fixtureId));
const resultKeys = results.map((result) => `${result.fixtureId}:${result.matrixRow}:${result.consumer}`);
expect(corpus.schema).toBe("paperclip.status-authority-conformance.v1");
expect(corpus.fixtures).toHaveLength(52);
expect(coveredFixtureIds).toEqual(new Set(corpus.fixtures.map((fixture) => fixture.id)));
expect(new Set(resultKeys).size).toBe(resultKeys.length);
expect(results.every((result) => /^[a-f0-9]{64}$/.test(result.expectedDigest))).toBe(true);
expect(new Set(results.filter((result) => result.matrixRow !== "fixture-only").map((result) => result.matrixRow))).toEqual(new Set([
...Array.from({ length: 19 }, (_, index) => `SD-${String(index + 1).padStart(2, "0")}`),
...Array.from({ length: 8 }, (_, index) => `TC-${String(index + 1).padStart(2, "0")}`),
...Array.from({ length: 12 }, (_, index) => `ATT-${String(index + 1).padStart(2, "0")}`),
...Array.from({ length: 6 }, (_, index) => `LIVE-${String(index + 1).padStart(2, "0")}`),
...Array.from({ length: 8 }, (_, index) => `REC-${String(index + 1).padStart(2, "0")}`),
...Array.from({ length: 8 }, (_, index) => `COMP-${String(index + 1).padStart(2, "0")}`),
...Array.from({ length: 9 }, (_, index) => `MIG-${String(index + 1).padStart(2, "0")}`),
]));
});
});
+35 -16
View File
@@ -57,6 +57,7 @@ import {
issueThreadInteractions,
issues,
issueWorkProducts,
nativeRunFinalizations,
projects,
projectWorkspaces,
routineRevisions,
@@ -15335,22 +15336,40 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
contract: completionContract.contract,
},
});
await db.update(heartbeatRuns).set({
runtimeMode: "native",
runtimeModeResolverVersion: nativeRuntimeResolution.resolverVersion,
runtimeModeReason: nativeRuntimeResolution.reason,
runtimeModeResolvedAt: run.runtimeModeResolvedAt ?? new Date(),
runnerProfileJson: nativeRuntimeResolution.profile,
runnerInstanceId: nativeRunnerInstanceId,
nativeSessionId,
driverKind: "codex_app_server",
driverVersion: "phase6-v1",
completionContractId: completionContract.row.id,
completionContractSha256: completionContract.row.canonicalSha256,
nativePhase: run.nativePhase ?? "selected",
nativePhaseUpdatedAt: run.nativePhaseUpdatedAt ?? new Date(),
updatedAt: new Date(),
}).where(eq(heartbeatRuns.id, run.id));
await db.transaction(async (tx) => {
const lockedRun = await tx.select().from(heartbeatRuns)
.where(eq(heartbeatRuns.id, run.id)).for("update").limit(1)
.then((rows) => rows[0] ?? null);
if (!lockedRun) throw new Error("native_runtime_run_missing");
if (lockedRun.runtimeModeResolvedAt && lockedRun.runtimeMode !== "native") {
throw new Error("native_runtime_mode_conflict");
}
await tx.update(heartbeatRuns).set({
runtimeMode: "native",
runtimeModeResolverVersion: lockedRun.runtimeModeResolverVersion ?? nativeRuntimeResolution.resolverVersion,
runtimeModeReason: lockedRun.runtimeModeReason ?? nativeRuntimeResolution.reason,
runtimeModeResolvedAt: lockedRun.runtimeModeResolvedAt ?? new Date(),
runnerProfileJson: {
...nativeRuntimeResolution.profile,
...parseObject(lockedRun.runnerProfileJson),
},
runnerInstanceId: lockedRun.runnerInstanceId ?? nativeRunnerInstanceId,
nativeSessionId: lockedRun.nativeSessionId ?? nativeSessionId,
driverKind: lockedRun.driverKind ?? "codex_app_server",
driverVersion: lockedRun.driverVersion ?? "phase6-v1",
completionContractId: lockedRun.completionContractId ?? completionContract.row.id,
completionContractSha256: lockedRun.completionContractSha256 ?? completionContract.row.canonicalSha256,
nativePhase: lockedRun.nativePhase ?? "observed",
nativePhaseUpdatedAt: lockedRun.nativePhaseUpdatedAt ?? new Date(),
updatedAt: new Date(),
}).where(eq(heartbeatRuns.id, run.id));
await tx.insert(nativeRunFinalizations).values({
runId: run.id,
companyId: agent.companyId,
issueId: issueRef.id,
phase: "observed",
}).onConflictDoNothing();
});
} else {
await db.update(heartbeatRuns).set({
runtimeMode: "legacy",
@@ -0,0 +1,77 @@
import { describe, expect, it } from "vitest";
import type { Db } from "@paperclipai/db";
import { classifyNativeEvidence } from "./evidence-classifier.js";
const workProductId = "00000000-0000-4000-8000-000000000001";
const evidenceRef = `work_product:${workProductId}`;
function evidenceDb(): Db {
return {
select: () => ({
from: () => ({
where: () => ({
limit: () => Promise.resolve([{
id: workProductId,
status: "approved",
reviewState: "approved",
}]),
}),
}),
}),
} as unknown as Db;
}
function result(status: "satisfied" | "not_satisfied", objectiveSatisfied: boolean) {
return {
reportedWorkDisposition: "done",
summary: "Classified",
completionClaim: {
contractRevision: "1",
objectiveSatisfied,
criteria: [{ criterionId: "objective", status, evidenceRefs: [evidenceRef] }],
remainingWork: [],
},
verification: [{ commandOrCheck: "test", status: "passed", artifactRef: evidenceRef }],
};
}
const input = {
db: evidenceDb(),
companyId: "company",
issueId: "issue",
runId: "run",
contract: {
revision: "1",
objective: "Classify evidence",
criteria: [{ id: "objective", requirement: "Use accepted durable evidence" }],
},
};
describe("classifyNativeEvidence", () => {
it("accepts a satisfied claim backed by an approved durable work product", async () => {
await expect(classifyNativeEvidence({
...input,
result: result("satisfied", true),
})).resolves.toEqual(expect.objectContaining({
objectiveSatisfied: true,
allCriteriaSatisfied: true,
verificationPassed: true,
acceptedEvidenceRefs: [evidenceRef],
}));
});
it("does not turn durable evidence into completion when the claim says the criterion is unsatisfied", async () => {
const assessment = await classifyNativeEvidence({
...input,
result: result("not_satisfied", false),
});
expect(assessment).toEqual(expect.objectContaining({
objectiveSatisfied: false,
allCriteriaSatisfied: false,
verificationPassed: true,
}));
expect(assessment.criterionAssessments).toEqual([
expect.objectContaining({ outcome: "rejected", reasonCode: "criterion_reported_not_satisfied" }),
]);
});
});
@@ -1,3 +1,24 @@
import { and, eq, or } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import {
approvals,
heartbeatRunEvents,
issueApprovals,
issueAttachments,
issueThreadInteractions,
issueWorkProducts,
} from "@paperclipai/db";
export type NativeEvidenceOutcome = "accepted" | "missing" | "rejected" | "unverifiable";
export interface NativeEvidenceRefAssessment {
ref: string;
kind: "event" | "work_product" | "approval" | "interaction" | "attachment" | "unknown";
outcome: NativeEvidenceOutcome;
reasonCode: string;
durableRecordId: string | null;
}
export interface NativeEvidenceAssessment {
objectiveSatisfied: boolean;
allCriteriaSatisfied: boolean;
@@ -5,7 +26,36 @@ export interface NativeEvidenceAssessment {
hasBlockingRemainingWork: boolean;
reportedDisposition: "done" | "blocked" | "needs_review" | "yielded";
summary: string;
blocker: { unblockAction: string; boardOwned: boolean } | null;
contractRevisionMatches: boolean;
criterionAssessments: Array<{
criterionId: string;
claimStatus: "satisfied" | "not_satisfied" | "unknown" | null;
outcome: NativeEvidenceOutcome;
evidenceRefs: NativeEvidenceRefAssessment[];
reasonCode: string;
}>;
verificationAssessments: Array<{
commandOrCheck: string;
claimStatus: "passed" | "failed" | "not_run";
outcome: NativeEvidenceOutcome;
evidenceRef: NativeEvidenceRefAssessment | null;
reasonCode: string;
}>;
acceptedEvidenceRefs: string[];
missingRequirements: string[];
rejectedEvidence: Array<{ ref: string; reasonCode: string }>;
unverifiableEvidence: Array<{ ref: string; reasonCode: string }>;
blocker: {
unblockAction: string;
boardOwned: boolean;
scope: "current_track" | "task_wide";
} | null;
continuation: {
kind: "same_agent" | "retry" | "delegated_issue" | "response_wake" | "monitor";
summary: string;
idempotencyKey: string;
} | null;
attentionRequests: Record<string, unknown>[];
}
function record(value: unknown): Record<string, unknown> {
@@ -14,29 +64,292 @@ function record(value: unknown): Record<string, unknown> {
: {};
}
export function classifyNativeEvidence(result: Record<string, unknown>): NativeEvidenceAssessment {
const claim = record(result.completionClaim);
const criteria = Array.isArray(claim.criteria) ? claim.criteria.map(record) : [];
function text(value: unknown): string | null {
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
}
function uuidRef(ref: string, prefixes: string[]): string | null {
for (const prefix of prefixes) {
if (ref.startsWith(`${prefix}:`)) return text(ref.slice(prefix.length + 1));
}
return null;
}
async function classifyEvidenceRef(input: {
db: Db;
companyId: string;
issueId: string;
runId: string;
ref: string;
}): Promise<NativeEvidenceRefAssessment> {
const eventMatch = /^event:(\d+)$/.exec(input.ref);
if (eventMatch) {
const cursor = Number(eventMatch[1]);
const event = await input.db.select({
id: heartbeatRunEvents.id,
eventType: heartbeatRunEvents.eventType,
payload: heartbeatRunEvents.payload,
})
.from(heartbeatRunEvents)
.where(and(
eq(heartbeatRunEvents.companyId, input.companyId),
eq(heartbeatRunEvents.runId, input.runId),
or(eq(heartbeatRunEvents.seq, cursor), eq(heartbeatRunEvents.sourceSeq, cursor)),
)).limit(1).then((rows) => rows[0] ?? null);
if (!event) {
return { ref: input.ref, kind: "event", outcome: "missing", reasonCode: "durable_run_event_missing", durableRecordId: null };
}
const prpEvent = record(record(event.payload).prpEvent);
const evidenceOutcome = text(record(prpEvent.payload).evidenceOutcome);
if (prpEvent.sourceKind === "control_plane" && evidenceOutcome === "accepted") {
return { ref: input.ref, kind: "event", outcome: "accepted", reasonCode: "control_plane_evidence_accepted", durableRecordId: String(event.id) };
}
if (prpEvent.sourceKind === "control_plane" && evidenceOutcome === "rejected") {
return { ref: input.ref, kind: "event", outcome: "rejected", reasonCode: "control_plane_evidence_rejected", durableRecordId: String(event.id) };
}
return {
ref: input.ref,
kind: "event",
outcome: "unverifiable",
reasonCode: event.eventType === "run.result.proposed"
? "model_result_event_is_claim_only"
: "run_event_has_no_authoritative_evidence_verdict",
durableRecordId: String(event.id),
};
}
const workProductId = uuidRef(input.ref, ["work_product", "work-product", "artifact"]);
if (workProductId) {
const row = await input.db.select({
id: issueWorkProducts.id,
status: issueWorkProducts.status,
reviewState: issueWorkProducts.reviewState,
}).from(issueWorkProducts).where(and(
eq(issueWorkProducts.id, workProductId),
eq(issueWorkProducts.companyId, input.companyId),
eq(issueWorkProducts.issueId, input.issueId),
)).limit(1).then((rows) => rows[0] ?? null);
if (!row) return { ref: input.ref, kind: "work_product", outcome: "missing", reasonCode: "work_product_missing", durableRecordId: null };
if (
["failed", "error", "cancelled", "rejected", "changes_requested"].includes(row.status)
|| ["rejected", "changes_requested"].includes(row.reviewState)
) {
return { ref: input.ref, kind: "work_product", outcome: "rejected", reasonCode: "work_product_rejected", durableRecordId: row.id };
}
if (
row.reviewState === "approved"
|| ["approved", "passed", "succeeded", "completed", "merged", "published"].includes(row.status)
) {
return { ref: input.ref, kind: "work_product", outcome: "accepted", reasonCode: "work_product_authoritatively_accepted", durableRecordId: row.id };
}
return { ref: input.ref, kind: "work_product", outcome: "unverifiable", reasonCode: "work_product_not_yet_accepted", durableRecordId: row.id };
}
const approvalId = uuidRef(input.ref, ["approval"]);
if (approvalId) {
const row = await input.db.select({ id: approvals.id, status: approvals.status })
.from(issueApprovals)
.innerJoin(approvals, eq(issueApprovals.approvalId, approvals.id))
.where(and(
eq(issueApprovals.companyId, input.companyId),
eq(issueApprovals.issueId, input.issueId),
eq(approvals.companyId, input.companyId),
eq(approvals.id, approvalId),
)).limit(1).then((rows) => rows[0] ?? null);
if (!row) return { ref: input.ref, kind: "approval", outcome: "missing", reasonCode: "approval_missing", durableRecordId: null };
if (row.status === "approved") return { ref: input.ref, kind: "approval", outcome: "accepted", reasonCode: "approval_approved", durableRecordId: row.id };
if (["rejected", "cancelled"].includes(row.status)) return { ref: input.ref, kind: "approval", outcome: "rejected", reasonCode: "approval_rejected", durableRecordId: row.id };
return { ref: input.ref, kind: "approval", outcome: "missing", reasonCode: "approval_pending", durableRecordId: row.id };
}
const interactionId = uuidRef(input.ref, ["interaction"]);
if (interactionId) {
const row = await input.db.select({
id: issueThreadInteractions.id,
status: issueThreadInteractions.status,
result: issueThreadInteractions.result,
}).from(issueThreadInteractions).where(and(
eq(issueThreadInteractions.id, interactionId),
eq(issueThreadInteractions.companyId, input.companyId),
eq(issueThreadInteractions.issueId, input.issueId),
)).limit(1).then((rows) => rows[0] ?? null);
if (!row) return { ref: input.ref, kind: "interaction", outcome: "missing", reasonCode: "interaction_missing", durableRecordId: null };
const outcome = text(record(row.result).outcome);
if (["accepted", "answered", "resolved"].includes(outcome ?? "") || ["accepted", "answered"].includes(row.status)) {
return { ref: input.ref, kind: "interaction", outcome: "accepted", reasonCode: "interaction_resolved", durableRecordId: row.id };
}
if (["rejected", "expired", "cancelled"].includes(row.status) || ["rejected", "withdrawn", "issue_closed"].includes(outcome ?? "")) {
return { ref: input.ref, kind: "interaction", outcome: "rejected", reasonCode: "interaction_rejected", durableRecordId: row.id };
}
return { ref: input.ref, kind: "interaction", outcome: "missing", reasonCode: "interaction_pending", durableRecordId: row.id };
}
const attachmentId = uuidRef(input.ref, ["attachment"]);
if (attachmentId) {
const row = await input.db.select({ id: issueAttachments.id }).from(issueAttachments).where(and(
eq(issueAttachments.id, attachmentId),
eq(issueAttachments.companyId, input.companyId),
eq(issueAttachments.issueId, input.issueId),
)).limit(1).then((rows) => rows[0] ?? null);
return row
? { ref: input.ref, kind: "attachment", outcome: "accepted", reasonCode: "attachment_persisted", durableRecordId: row.id }
: { ref: input.ref, kind: "attachment", outcome: "missing", reasonCode: "attachment_missing", durableRecordId: null };
}
return {
ref: input.ref,
kind: "unknown",
outcome: "unverifiable",
reasonCode: "unsupported_evidence_reference",
durableRecordId: null,
};
}
/**
* Classifies model claims against immutable server-owned records. A model's
* `satisfied` or `passed` value is retained as a claim but never becomes proof.
*/
export async function classifyNativeEvidence(input: {
db: Db;
companyId: string;
issueId: string;
runId: string;
contract: Record<string, unknown>;
result: Record<string, unknown>;
}): Promise<NativeEvidenceAssessment> {
const claim = record(input.result.completionClaim);
const claimedCriteria = Array.isArray(claim.criteria) ? claim.criteria.map(record) : [];
const contractCriteria = Array.isArray(input.contract.criteria) ? input.contract.criteria.map(record) : [];
const remaining = Array.isArray(claim.remainingWork) ? claim.remainingWork.map(record) : [];
const verification = Array.isArray(result.verification) ? result.verification.map(record) : [];
const reported = result.reportedWorkDisposition;
const blocker = record(result.blocker);
const verification = Array.isArray(input.result.verification) ? input.result.verification.map(record) : [];
const reported = input.result.reportedWorkDisposition;
const blocker = record(input.result.blocker);
const blockerOwner = record(blocker.owner);
const continuation = record(input.result.continuation);
if (!["done", "blocked", "needs_review", "yielded"].includes(String(reported))) {
throw new Error("native_result_invalid_disposition");
}
const contractRevision = text(input.contract.revision);
const contractRevisionMatches = contractRevision !== null && text(claim.contractRevision) === contractRevision;
const cache = new Map<string, Promise<NativeEvidenceRefAssessment>>();
const resolveRef = (ref: string) => {
let pending = cache.get(ref);
if (!pending) {
pending = classifyEvidenceRef({ ...input, ref });
cache.set(ref, pending);
}
return pending;
};
const criterionAssessments = await Promise.all(contractCriteria.map(async (criterion) => {
const criterionId = text(criterion.id) ?? "invalid-contract-criterion";
const claimed = claimedCriteria.find((entry) => text(entry.criterionId) === criterionId) ?? null;
const claimStatus = claimed && ["satisfied", "not_satisfied", "unknown"].includes(String(claimed.status))
? claimed.status as "satisfied" | "not_satisfied" | "unknown"
: null;
const refs = claimed && Array.isArray(claimed.evidenceRefs)
? claimed.evidenceRefs.flatMap((value) => text(value) ? [text(value)!] : [])
: [];
const evidenceRefs = await Promise.all(refs.map(resolveRef));
let outcome: NativeEvidenceOutcome;
let reasonCode: string;
if (!contractRevisionMatches) {
outcome = "rejected";
reasonCode = "contract_revision_stale";
} else if (!claimed) {
outcome = "missing";
reasonCode = "criterion_claim_missing";
} else if (claimStatus === "not_satisfied") {
outcome = "rejected";
reasonCode = "criterion_reported_not_satisfied";
} else if (claimStatus !== "satisfied") {
outcome = "unverifiable";
reasonCode = "criterion_claim_not_satisfied";
} else if (refs.length === 0) {
outcome = "missing";
reasonCode = "criterion_evidence_missing";
} else if (evidenceRefs.some((entry) => entry.outcome === "rejected")) {
outcome = "rejected";
reasonCode = "criterion_evidence_rejected";
} else if (evidenceRefs.some((entry) => entry.outcome === "unverifiable")) {
outcome = "unverifiable";
reasonCode = "criterion_evidence_unverifiable";
} else if (evidenceRefs.some((entry) => entry.outcome === "missing")) {
outcome = "missing";
reasonCode = "criterion_evidence_missing";
} else {
outcome = "accepted";
reasonCode = "criterion_evidence_accepted";
}
return { criterionId, claimStatus, outcome, evidenceRefs, reasonCode };
}));
const verificationAssessments = await Promise.all(verification.map(async (entry, index) => {
const commandOrCheck = text(entry.commandOrCheck) ?? `verification-${index + 1}`;
const claimStatus = ["passed", "failed", "not_run"].includes(String(entry.status))
? entry.status as "passed" | "failed" | "not_run"
: "not_run";
const ref = text(entry.artifactRef);
const evidenceRef = ref ? await resolveRef(ref) : null;
if (claimStatus === "failed") return { commandOrCheck, claimStatus, outcome: "rejected" as const, evidenceRef, reasonCode: "verification_reported_failed" };
if (claimStatus === "not_run") return { commandOrCheck, claimStatus, outcome: "missing" as const, evidenceRef, reasonCode: "verification_not_run" };
if (!evidenceRef) return { commandOrCheck, claimStatus, outcome: "unverifiable" as const, evidenceRef, reasonCode: "verification_has_no_durable_reference" };
return {
commandOrCheck,
claimStatus,
outcome: evidenceRef.outcome,
evidenceRef,
reasonCode: evidenceRef.outcome === "accepted" ? "verification_evidence_accepted" : evidenceRef.reasonCode,
};
}));
const allRefs = [...cache.keys()].map((ref) => cache.get(ref)!);
const resolvedRefs = await Promise.all(allRefs);
const acceptedEvidenceRefs = resolvedRefs.filter((entry) => entry.outcome === "accepted").map((entry) => entry.ref);
const missingRequirements = [
...criterionAssessments.filter((entry) => entry.outcome === "missing").map((entry) => entry.criterionId),
...verificationAssessments.filter((entry) => entry.outcome === "missing").map((entry) => entry.commandOrCheck),
];
const rejectedEvidence = resolvedRefs.filter((entry) => entry.outcome === "rejected").map((entry) => ({ ref: entry.ref, reasonCode: entry.reasonCode }));
const unverifiableEvidence = resolvedRefs.filter((entry) => entry.outcome === "unverifiable").map((entry) => ({ ref: entry.ref, reasonCode: entry.reasonCode }));
const allCriteriaSatisfied = contractCriteria.length > 0 && criterionAssessments.every((entry) => entry.outcome === "accepted");
const verificationPassed = verificationAssessments.length > 0 && verificationAssessments.every((entry) => entry.outcome === "accepted");
const continuationKind = continuation.kind;
return {
objectiveSatisfied: claim.objectiveSatisfied === true,
allCriteriaSatisfied: criteria.length > 0 && criteria.every((criterion) => criterion.status === "satisfied"),
verificationPassed: verification.length > 0 && verification.every((entry) => entry.status === "passed"),
objectiveSatisfied: claim.objectiveSatisfied === true && allCriteriaSatisfied,
allCriteriaSatisfied,
verificationPassed,
hasBlockingRemainingWork: remaining.some((entry) => entry.blocksCompletion === true),
reportedDisposition: reported as NativeEvidenceAssessment["reportedDisposition"],
summary: typeof result.summary === "string" ? result.summary : "Native run completed.",
summary: typeof input.result.summary === "string" ? input.result.summary : "Native run completed.",
contractRevisionMatches,
criterionAssessments,
verificationAssessments,
acceptedEvidenceRefs,
missingRequirements,
rejectedEvidence,
unverifiableEvidence,
blocker: reported === "blocked" && typeof blocker.unblockAction === "string"
? {
unblockAction: blocker.unblockAction,
boardOwned: blockerOwner.kind === "board" || blockerOwner.owner === "board",
scope: blocker.scope === "task_wide" ? "task_wide" : "current_track",
}
: null,
continuation: reported === "yielded"
&& ["same_agent", "retry", "delegated_issue", "response_wake", "monitor"].includes(String(continuationKind))
&& text(continuation.summary)
&& text(continuation.idempotencyKey)
? {
kind: continuationKind as NativeEvidenceAssessment["continuation"] extends infer T ? T extends { kind: infer K } ? K : never : never,
summary: text(continuation.summary)!,
idempotencyKey: text(continuation.idempotencyKey)!,
}
: null,
attentionRequests: Array.isArray(input.result.attentionRequests)
? input.result.attentionRequests.map(record)
: [],
};
}
@@ -1,7 +1,7 @@
import { and, desc, eq, inArray, isNotNull, ne } from "drizzle-orm";
import { and, desc, eq, inArray, isNotNull, isNull, lte, notInArray, or } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import { heartbeatRuns, nativeRunFinalizations, workspaceOperations } from "@paperclipai/db";
import { finalizeNativeRun } from "./native-run-finalizer.js";
import { finalizeNativeRun, recordNativeFinalizationFailure } from "./native-run-finalizer.js";
/** Recovery is keyed only by persisted mode/coordinator state, never the live flag. */
export async function reconcileNativeFinalizations(db: Db, runIds?: string[]) {
@@ -11,7 +11,16 @@ export async function reconcileNativeFinalizations(db: Db, runIds?: string[]) {
.where(and(
eq(heartbeatRuns.runtimeMode, "native"),
isNotNull(nativeRunFinalizations.resultId),
ne(nativeRunFinalizations.phase, "applied"),
notInArray(nativeRunFinalizations.phase, ["terminal_failure"]),
or(
isNull(nativeRunFinalizations.nextAttemptAt),
lte(nativeRunFinalizations.nextAttemptAt, new Date()),
),
or(
isNull(nativeRunFinalizations.leaseOwner),
isNull(nativeRunFinalizations.leaseExpiresAt),
lte(nativeRunFinalizations.leaseExpiresAt, new Date()),
),
...(runIds && runIds.length > 0 ? [inArray(heartbeatRuns.id, runIds)] : []),
));
const results = [];
@@ -26,12 +35,21 @@ export async function reconcileNativeFinalizations(db: Db, runIds?: string[]) {
.limit(1)
.then((entries) => entries[0] ?? null);
if (!barrier || !["succeeded", "failed"].includes(barrier.status)) continue;
results.push(await finalizeNativeRun({
db,
runId: row.runId,
workspaceFinalizeStatus: barrier.status as "succeeded" | "failed",
projectRunStatus: true,
}));
try {
results.push(await finalizeNativeRun({
db,
runId: row.runId,
workspaceFinalizeStatus: barrier.status as "succeeded" | "failed",
projectRunStatus: true,
}));
} catch (error) {
results.push(await recordNativeFinalizationFailure({
db,
runId: row.runId,
error,
projectRunStatus: true,
}));
}
}
return results;
}
@@ -1,7 +1,9 @@
import { randomUUID } from "node:crypto";
import { and, eq, inArray } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import {
approvals,
completionContracts,
heartbeatRuns,
issueApprovals,
issueThreadInteractions,
@@ -10,9 +12,20 @@ import {
nativeRunResults,
} from "@paperclipai/db";
import { classifyNativeEvidence } from "./evidence-classifier.js";
import { arbitrateNativeStatus, NATIVE_STATUS_ARBITER_POLICY_VERSION } from "./status-arbiter.js";
import {
arbitrateNativeStatus,
NATIVE_STATUS_ARBITER_POLICY_VERSION,
type NativeAuthoritativeIssueStatus,
type NativeGovernanceGate,
} from "./status-arbiter.js";
import { recordNativeWorkAssessment } from "./work-assessments.js";
import { commitNativeStatusDecision, NativeStatusRaceError } from "./status-decision-committer.js";
import {
commitNativeStatusDecision,
NativeStatusRaceError,
type NativeStatusCommitFailpoint,
} from "./status-decision-committer.js";
import { issueRecoveryActionService } from "../issue-recovery-actions.js";
import { nativeSha256 } from "./canonical.js";
function record(value: unknown): Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value)
@@ -20,16 +33,25 @@ function record(value: unknown): Record<string, unknown> {
: {};
}
async function hasPendingNativeGovernance(input: {
function authoritativeStatus(value: string): NativeAuthoritativeIssueStatus {
if (!["backlog", "todo", "in_progress", "in_review", "blocked", "done", "cancelled"].includes(value)) {
throw new Error("native_issue_status_invalid");
}
return value as NativeAuthoritativeIssueStatus;
}
async function pendingNativeGovernance(input: {
db: Db;
companyId: string;
issueId: string;
runId: string;
executionState: Record<string, unknown> | null;
}) {
// Issue-thread interactions and execution-policy stages are independent
// governance authorities from formal approvals. Native completion must
// mediate all three before it may project an authoritative terminal status.
if (record(input.executionState).status === "pending") return true;
}): Promise<NativeGovernanceGate | null> {
// Execution policy is issue-owned state and takes priority over the two
// durable review surfaces. Completion must mediate all three authorities.
if (record(input.executionState).status === "pending") {
return { kind: "execution_stage", id: input.runId };
}
const [pendingInteraction, pendingApproval] = await Promise.all([
input.db.select({ id: issueThreadInteractions.id })
.from(issueThreadInteractions)
@@ -52,7 +74,174 @@ async function hasPendingNativeGovernance(input: {
.limit(1)
.then((rows) => rows[0] ?? null),
]);
return Boolean(pendingInteraction || pendingApproval);
if (pendingInteraction) return { kind: "interaction", id: pendingInteraction.id };
if (pendingApproval) return { kind: "approval", id: pendingApproval.id };
return null;
}
async function claimCoordinator(input: { db: Db; runId: string }) {
const leaseOwner = `native-finalizer:${randomUUID()}`;
const now = new Date();
const claimed = await input.db.transaction(async (tx) => {
const coordinator = await tx.select().from(nativeRunFinalizations)
.where(eq(nativeRunFinalizations.runId, input.runId)).for("update").limit(1)
.then((rows) => rows[0] ?? null);
if (!coordinator?.resultId) throw new Error("native_finalization_missing");
if (coordinator.phase === "committed" && coordinator.decisionId) {
return { coordinator, leaseOwner: null };
}
if (coordinator.phase === "terminal_failure") throw new Error("native_finalization_terminal_failure");
if (
coordinator.leaseOwner
&& coordinator.leaseExpiresAt
&& coordinator.leaseExpiresAt > now
&& coordinator.leaseOwner !== leaseOwner
) throw new Error("native_finalization_lease_busy");
const [updated] = await tx.update(nativeRunFinalizations).set({
leaseOwner,
leaseExpiresAt: new Date(now.getTime() + 5 * 60_000),
attempt: coordinator.attempt + 1,
phase: coordinator.phase === "retryable_failure"
? coordinator.assessmentId ? "arbitrating" : "workspace_finalizing"
: coordinator.phase,
failureCode: null,
failureDetail: null,
nextAttemptAt: null,
updatedAt: now,
}).where(eq(nativeRunFinalizations.runId, input.runId)).returning();
if (!updated) throw new Error("native_finalization_claim_failed");
return { coordinator: updated, leaseOwner };
});
return claimed;
}
async function recordRetryableFailure(input: {
db: Db;
run: typeof heartbeatRuns.$inferSelect;
coordinator: typeof nativeRunFinalizations.$inferSelect;
failureCode: string;
message: string;
nextAction: string;
projectRunStatus?: boolean;
}) {
const now = new Date();
const exhausted = input.coordinator.attempt >= 3;
const nextAttemptAt = new Date(now.getTime() + 30_000);
const phase = exhausted ? "terminal_failure" : "retryable_failure";
const failureCode = exhausted ? "native_finalization_retry_exhausted" : input.failureCode;
await input.db.transaction(async (tx) => {
await tx.update(nativeRunFinalizations).set({
phase,
leaseOwner: null,
leaseExpiresAt: null,
failureCode,
failureDetail: {
message: input.message.slice(0, 2_000),
originalFailureCode: input.failureCode,
recoveryOwner: { kind: "agent", agentId: input.run.agentId },
nextAction: input.nextAction,
},
nextAttemptAt: exhausted ? null : nextAttemptAt,
updatedAt: now,
}).where(eq(nativeRunFinalizations.runId, input.run.id));
await tx.update(heartbeatRuns).set({
...(input.projectRunStatus ? { status: "failed", finishedAt: now } : {}),
nativePhase: phase,
nativePhaseUpdatedAt: now,
resultJson: {
...record(input.run.resultJson),
finalizationPhase: phase,
failureCode,
originalFailureCode: input.failureCode,
nextAttemptAt: exhausted ? null : nextAttemptAt.toISOString(),
},
updatedAt: now,
}).where(eq(heartbeatRuns.id, input.run.id));
await issueRecoveryActionService(tx as unknown as Db).upsertSourceScoped({
companyId: input.run.companyId,
sourceIssueId: input.coordinator.issueId,
kind: "active_run_watchdog",
ownerType: "agent",
ownerAgentId: input.run.agentId,
cause: failureCode,
fingerprint: nativeSha256({ runId: input.run.id, failureCode }),
evidence: {
runId: input.run.id,
coordinatorAttempt: input.coordinator.attempt,
originalFailureCode: input.failureCode,
},
nextAction: exhausted
? `Finalization retry budget exhausted. ${input.nextAction}`
: input.nextAction,
wakePolicy: exhausted
? null
: { kind: "resume_native_run", runId: input.run.id, notBefore: nextAttemptAt.toISOString() },
maxAttempts: 3,
});
});
return {
...input.coordinator,
phase,
failureCode,
nextAttemptAt: exhausted ? null : nextAttemptAt,
};
}
/** Converts malformed or incomplete persisted finalization state into named recovery. */
export async function recordNativeFinalizationFailure(input: {
db: Db;
runId: string;
error: unknown;
projectRunStatus?: boolean;
}) {
const [run, coordinator] = await Promise.all([
input.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId))
.limit(1).then((rows) => rows[0] ?? null),
input.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, input.runId))
.limit(1).then((rows) => rows[0] ?? null),
]);
if (!run || run.runtimeMode !== "native" || !coordinator) throw input.error;
const message = input.error instanceof Error ? input.error.message : String(input.error);
const failureCode = message.startsWith("native_") ? message : "native_finalization_invalid";
return recordRetryableFailure({
db: input.db,
run,
coordinator,
failureCode,
message,
nextAction: "Repair the persisted native result or contract discriminator, then resume finalization from the coordinator.",
projectRunStatus: input.projectRunStatus,
});
}
async function projectCommittedRun(input: {
db: Db;
run: typeof heartbeatRuns.$inferSelect;
coordinator: typeof nativeRunFinalizations.$inferSelect;
}) {
if (!input.coordinator.resultId) return;
const resultRow = await input.db.select({ resultJson: nativeRunResults.resultJson })
.from(nativeRunResults).where(and(
eq(nativeRunResults.id, input.coordinator.resultId),
eq(nativeRunResults.runId, input.run.id),
eq(nativeRunResults.companyId, input.run.companyId),
)).limit(1).then((rows) => rows[0] ?? null);
const terminalState = record(record(resultRow?.resultJson).terminal).runTerminalState;
if (!["succeeded", "failed", "cancelled"].includes(String(terminalState))) {
throw new Error("native_finalization_invalid");
}
const now = new Date();
await input.db.update(heartbeatRuns).set({
status: terminalState as "succeeded" | "failed" | "cancelled",
finishedAt: input.run.finishedAt ?? now,
nativePhase: "committed",
nativePhaseUpdatedAt: now,
updatedAt: now,
}).where(and(
eq(heartbeatRuns.id, input.run.id),
eq(heartbeatRuns.runtimeMode, "native"),
inArray(heartbeatRuns.status, ["queued", "running", "failed"]),
));
}
export async function finalizeNativeRun(input: {
@@ -61,40 +250,44 @@ export async function finalizeNativeRun(input: {
workspaceFinalizeStatus: "succeeded" | "failed";
/** Reconciliation owns terminal run projection; the live heartbeat does it afterward. */
projectRunStatus?: boolean;
failpoint?: NativeStatusCommitFailpoint;
}) {
const run = await input.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId))
.limit(1).then((rows) => rows[0] ?? null);
if (!run || run.runtimeMode !== "native") throw new Error("native_finalization_run_missing");
const coordinator = await input.db.select().from(nativeRunFinalizations)
.where(eq(nativeRunFinalizations.runId, input.runId)).limit(1)
.then((rows) => rows[0] ?? null);
if (!coordinator?.resultId) throw new Error("native_finalization_missing");
if (coordinator.phase === "applied" && coordinator.decisionId) return coordinator;
if (input.workspaceFinalizeStatus === "failed") {
const now = new Date();
await input.db.update(nativeRunFinalizations).set({
phase: "workspace_failed",
failureCode: "workspace_finalization_failed",
failureDetail: "Native result preserved; issue status intentionally unchanged.",
updatedAt: now,
}).where(eq(nativeRunFinalizations.runId, input.runId));
await input.db.update(heartbeatRuns).set({
...(input.projectRunStatus ? { status: "failed", finishedAt: now } : {}),
nativePhase: "workspace_failed",
nativePhaseUpdatedAt: now,
resultJson: {
...record(run.resultJson),
finalizationPhase: "workspace_failed",
workspaceFinalizeStatus: "failed",
},
updatedAt: now,
}).where(eq(heartbeatRuns.id, run.id));
return { ...coordinator, phase: "workspace_failed" };
const claim = await claimCoordinator({ db: input.db, runId: input.runId });
const coordinator = claim.coordinator;
if (!claim.leaseOwner && coordinator.phase === "committed") {
if (input.projectRunStatus) await projectCommittedRun({ db: input.db, run, coordinator });
return coordinator;
}
const resultRow = await input.db.select().from(nativeRunResults)
.where(eq(nativeRunResults.id, coordinator.resultId)).limit(1)
.then((rows) => rows[0] ?? null);
if (input.workspaceFinalizeStatus === "failed") {
return recordRetryableFailure({
db: input.db,
run,
coordinator,
failureCode: "workspace_finalization_failed",
message: "Native result preserved; issue status intentionally unchanged.",
nextAction: "Repair and re-run workspace finalization for the persisted native result.",
projectRunStatus: input.projectRunStatus,
});
}
const [resultRow, contractRow] = await Promise.all([
input.db.select().from(nativeRunResults)
.where(and(
eq(nativeRunResults.id, coordinator.resultId!),
eq(nativeRunResults.companyId, run.companyId),
)).limit(1).then((rows) => rows[0] ?? null),
input.db.select().from(completionContracts)
.where(and(
eq(completionContracts.id, run.completionContractId!),
eq(completionContracts.companyId, run.companyId),
eq(completionContracts.issueId, coordinator.issueId),
)).limit(1).then((rows) => rows[0] ?? null),
]);
if (!resultRow) throw new Error("native_result_missing");
if (!contractRow) throw new Error("native_completion_contract_missing");
const envelope = record(resultRow.resultJson);
const result = record(envelope.result);
const terminal = record(envelope.terminal);
@@ -102,28 +295,42 @@ export async function finalizeNativeRun(input: {
if (!["succeeded", "failed", "cancelled"].includes(String(terminalState))) {
throw new Error("native_finalization_invalid");
}
const assessment = classifyNativeEvidence(result);
await input.db.update(nativeRunFinalizations).set({ phase: "ready_for_assessment", updatedAt: new Date() })
.where(and(
eq(nativeRunFinalizations.runId, input.runId),
eq(nativeRunFinalizations.leaseOwner, claim.leaseOwner!),
));
const assessment = await classifyNativeEvidence({
db: input.db,
companyId: run.companyId,
issueId: coordinator.issueId,
runId: run.id,
contract: record(contractRow.contractJson),
result,
});
let supersedesAssessmentId: string | null = null;
for (let attempt = 0; attempt < 2; attempt += 1) {
const issue = await input.db.select().from(issues).where(eq(issues.id, coordinator.issueId))
.limit(1).then((rows) => rows[0] ?? null);
const authoritativeIssue = issue ?? await input.db.select().from(issues)
.where(eq(issues.id, coordinator.issueId)).limit(1).then((rows) => rows[0] ?? null);
if (!authoritativeIssue || authoritativeIssue.companyId !== run.companyId) {
throw new Error("native_finalization_issue_missing");
}
for (let attempt = 0; attempt < 3; attempt += 1) {
const authoritativeIssue = await input.db.select().from(issues).where(and(
eq(issues.id, coordinator.issueId),
eq(issues.companyId, run.companyId),
)).limit(1).then((rows) => rows[0] ?? null);
if (!authoritativeIssue) throw new Error("native_finalization_issue_missing");
const governanceGate = await pendingNativeGovernance({
db: input.db,
companyId: run.companyId,
issueId: authoritativeIssue.id,
runId: run.id,
executionState: record(authoritativeIssue.executionState),
});
const decision = arbitrateNativeStatus({
assessment,
terminalState: terminalState as "succeeded" | "failed" | "cancelled",
workspaceFinalizeStatus: input.workspaceFinalizeStatus,
approvalGateSatisfied: !await hasPendingNativeGovernance({
db: input.db,
companyId: run.companyId,
issueId: authoritativeIssue.id,
executionState: authoritativeIssue.executionState,
}),
governanceGate,
agentId: run.agentId,
priorIssueStatus: authoritativeStatus(authoritativeIssue.status),
});
const assessmentRow = await recordNativeWorkAssessment({
db: input.db,
@@ -132,7 +339,9 @@ export async function finalizeNativeRun(input: {
runId: run.id,
turnId: resultRow.turnId,
contractId: resultRow.completionContractId,
contractCanonicalSha256: contractRow.canonicalSha256,
resultId: resultRow.id,
resultCanonicalSha256: resultRow.canonicalSha256,
priorIssueStatus: authoritativeIssue.status,
priorStatusVersion: Number(authoritativeIssue.statusVersion),
priorDecisionId: authoritativeIssue.lastStatusDecisionId,
@@ -140,6 +349,14 @@ export async function finalizeNativeRun(input: {
assessment,
supersedesAssessmentId,
});
await input.db.update(nativeRunFinalizations).set({
phase: "arbitrating",
assessmentId: assessmentRow.id,
updatedAt: new Date(),
}).where(and(
eq(nativeRunFinalizations.runId, input.runId),
eq(nativeRunFinalizations.leaseOwner, claim.leaseOwner!),
));
try {
const committed = await commitNativeStatusDecision({
db: input.db,
@@ -149,18 +366,21 @@ export async function finalizeNativeRun(input: {
assessmentId: assessmentRow.id,
priorStatus: authoritativeIssue.status,
priorStatusVersion: Number(authoritativeIssue.statusVersion),
priorDecisionId: authoritativeIssue.lastStatusDecisionId,
decision,
failpoint: input.failpoint,
});
const now = new Date();
await input.db.update(heartbeatRuns).set({
...(input.projectRunStatus ? {
status: terminalState === "succeeded" ? "succeeded" : terminalState === "cancelled" ? "cancelled" : "failed",
finishedAt: new Date(),
finishedAt: now,
} : {}),
nativePhase: "applied",
nativePhaseUpdatedAt: new Date(),
nativePhase: "committed",
nativePhaseUpdatedAt: now,
resultJson: {
...record(run.resultJson),
finalizationPhase: "applied",
finalizationPhase: "committed",
assessmentId: assessmentRow.id,
decisionId: committed.decision.id,
authoritativeDecision: decision.toStatus,
@@ -170,17 +390,25 @@ export async function finalizeNativeRun(input: {
statusVersionAfter: Number(committed.issue.statusVersion),
workspaceFinalizeStatus: input.workspaceFinalizeStatus,
},
updatedAt: new Date(),
updatedAt: now,
}).where(eq(heartbeatRuns.id, run.id));
return {
...coordinator,
phase: "applied",
assessmentId: assessmentRow.id,
decisionId: committed.decision.id,
};
return { ...coordinator, phase: "committed", assessmentId: assessmentRow.id, decisionId: committed.decision.id };
} catch (error) {
if (!(error instanceof NativeStatusRaceError) || attempt === 1) throw error;
supersedesAssessmentId = assessmentRow.id;
if (error instanceof NativeStatusRaceError && attempt < 2) {
supersedesAssessmentId = assessmentRow.id;
continue;
}
return recordRetryableFailure({
db: input.db,
run,
coordinator,
failureCode: error instanceof NativeStatusRaceError ? "status_cas_exhausted" : "side_effect_planning_failed",
message: error instanceof Error ? error.message : String(error),
nextAction: error instanceof NativeStatusRaceError
? "Reassess against the latest authoritative issue status version."
: "Retry atomic status and liveness materialization from the persisted assessment.",
projectRunStatus: input.projectRunStatus,
});
}
}
throw new Error("native_finalization_status_race");
@@ -25,6 +25,38 @@ const execution = {
completionContract: { id: "contract", sha256: "sha" },
} as NativeExecutionInputV1;
function leaseDb(): Db {
const coordinator = {
runId: execution.binding.runId,
companyId: execution.binding.companyId,
issueId: execution.binding.issueId,
phase: "observed",
attempt: 0,
leaseOwner: null,
leaseExpiresAt: null,
resultId: null,
};
const update = () => ({
set: () => ({ where: async () => [] }),
});
const tx = {
select: () => ({
from: () => ({
where: () => ({
for: () => ({
limit: () => Promise.resolve([coordinator]),
}),
}),
}),
}),
update,
};
return {
transaction: async (operation: (transaction: Db) => Promise<unknown>) => operation(tx as unknown as Db),
update,
} as unknown as Db;
}
describe("native session cancellation", () => {
beforeEach(() => {
state.cancel.mockReset();
@@ -49,14 +81,16 @@ describe("native session cancellation", () => {
it("routes control-plane cancellation to the active normalized session and removes the handle", async () => {
const running = executePaperclipNativeSession({
db: {} as Db,
db: leaseDb(),
execution,
runnerInstanceId: "runner",
});
await vi.waitFor(() => expect(state.release).toBeTypeOf("function"));
await expect(cancelNativeSession(execution.binding.runId, "budget hard stop")).resolves.toBe(true);
await expect(cancelNativeSession(execution.binding.runId, "duplicate budget stop")).resolves.toBe(true);
expect(state.cancel).toHaveBeenCalledWith({ reason: "budget hard stop" });
expect(state.cancel).toHaveBeenCalledTimes(1);
state.release?.();
await running;
@@ -1,3 +1,4 @@
import { randomUUID } from "node:crypto";
import type { AdapterExecutionResult } from "../../adapters/index.js";
import type { NativeFinalizationResult } from "@paperclipai/shared";
import type { NativeExecutionInputV1, NativeSession } from "@paperclipai/paperclip-runner";
@@ -6,15 +7,24 @@ import {
executeNativeSession,
} from "@paperclipai/paperclip-runner";
import type { Db } from "@paperclipai/db";
import { and, eq } from "drizzle-orm";
import { heartbeatRuns, nativeRunFinalizations } from "@paperclipai/db";
import { PaperclipControlPlanePort } from "./paperclip-control-plane-port.js";
const activeNativeSessions = new Map<string, NativeSession>();
type ActiveNativeSession = {
session: NativeSession;
cancelRequested: boolean;
};
const activeNativeSessions = new Map<string, ActiveNativeSession>();
export async function cancelNativeSession(runId: string, reason: string): Promise<boolean> {
const session = activeNativeSessions.get(runId);
if (!session) return false;
if (session.cancel) await session.cancel({ reason });
else if (session.interrupt) await session.interrupt({ reason });
const active = activeNativeSessions.get(runId);
if (!active) return false;
if (active.cancelRequested) return true;
active.cancelRequested = true;
if (active.session.cancel) await active.session.cancel({ reason });
else if (active.session.interrupt) await active.session.interrupt({ reason });
return true;
}
@@ -23,6 +33,40 @@ export async function executePaperclipNativeSession(input: {
execution: NativeExecutionInputV1;
runnerInstanceId: string;
}): Promise<AdapterExecutionResult> {
const leaseOwner = `${input.runnerInstanceId}:${randomUUID()}`;
const leaseNow = new Date();
const leaseExpiresAt = new Date(leaseNow.getTime() + 20 * 60_000);
await input.db.transaction(async (tx) => {
const coordinator = await tx.select().from(nativeRunFinalizations)
.where(and(
eq(nativeRunFinalizations.runId, input.execution.binding.runId),
eq(nativeRunFinalizations.companyId, input.execution.binding.companyId),
eq(nativeRunFinalizations.issueId, input.execution.binding.issueId),
)).for("update").limit(1).then((rows) => rows[0] ?? null);
if (!coordinator) throw new Error("native_finalization_coordinator_missing");
if (["committed", "applied"].includes(coordinator.phase)) throw new Error("native_run_already_committed");
if (
coordinator.leaseOwner
&& coordinator.leaseOwner !== leaseOwner
&& coordinator.leaseExpiresAt
&& coordinator.leaseExpiresAt > leaseNow
) throw new Error("native_finalization_lease_busy");
await tx.update(nativeRunFinalizations).set({
phase: coordinator.resultId ? coordinator.phase : "observed",
attempt: coordinator.attempt + 1,
leaseOwner,
leaseExpiresAt,
failureCode: null,
failureDetail: null,
nextAttemptAt: null,
updatedAt: leaseNow,
}).where(eq(nativeRunFinalizations.runId, coordinator.runId));
await tx.update(heartbeatRuns).set({
nativePhase: coordinator.resultId ? coordinator.phase : "observed",
nativePhaseUpdatedAt: leaseNow,
updatedAt: leaseNow,
}).where(eq(heartbeatRuns.id, coordinator.runId));
});
const controlPlaneInstanceId = `${input.runnerInstanceId}:control`;
const controlPlane = new PaperclipControlPlanePort(input.db, {
companyId: input.execution.binding.companyId,
@@ -34,17 +78,50 @@ export async function executePaperclipNativeSession(input: {
sourceInstanceId: input.runnerInstanceId,
controlPlaneSourceInstanceId: controlPlaneInstanceId,
});
const native = await executeNativeSession({
input: input.execution,
backend: createCodexNativeSessionBackend(input.execution, { runnerInstanceId: input.runnerInstanceId }),
controlPlane,
runnerInstanceId: input.runnerInstanceId,
controlPlaneInstanceId,
onSession: (session) => {
if (session) activeNativeSessions.set(input.execution.binding.runId, session);
else activeNativeSessions.delete(input.execution.binding.runId);
},
});
let native: Awaited<ReturnType<typeof executeNativeSession>>;
try {
native = await executeNativeSession({
input: input.execution,
backend: createCodexNativeSessionBackend(input.execution, { runnerInstanceId: input.runnerInstanceId }),
controlPlane,
runnerInstanceId: input.runnerInstanceId,
controlPlaneInstanceId,
onSession: (session) => {
if (session) activeNativeSessions.set(input.execution.binding.runId, {
session,
cancelRequested: false,
});
else activeNativeSessions.delete(input.execution.binding.runId);
},
});
} catch (error) {
const now = new Date();
await input.db.update(nativeRunFinalizations).set({
phase: "retryable_failure",
leaseOwner: null,
leaseExpiresAt: null,
failureCode: "native_session_interrupted",
failureDetail: {
message: error instanceof Error ? error.message.slice(0, 2_000) : String(error).slice(0, 2_000),
recoveryOwner: { kind: "agent", agentId: input.execution.binding.agentId },
nextAction: "Resume the persisted native provider session after the retry delay.",
},
nextAttemptAt: new Date(now.getTime() + 30_000),
updatedAt: now,
}).where(and(
eq(nativeRunFinalizations.runId, input.execution.binding.runId),
eq(nativeRunFinalizations.leaseOwner, leaseOwner),
));
throw error;
}
await input.db.update(nativeRunFinalizations).set({
leaseOwner: null,
leaseExpiresAt: null,
updatedAt: new Date(),
}).where(and(
eq(nativeRunFinalizations.runId, input.execution.binding.runId),
eq(nativeRunFinalizations.leaseOwner, leaseOwner),
));
const finalization: NativeFinalizationResult = {
schema: "paperclip.native-finalization.v1",
runtimeMode: "native",
@@ -2,13 +2,16 @@ import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { eq } from "drizzle-orm";
import {
activityLog,
agentWakeupRequests,
agents,
companies,
completionContracts,
createDb,
heartbeatRunEvents,
heartbeatRuns,
issueRecoveryActions,
issueThreadInteractions,
issueWorkProducts,
issues,
nativeRunFinalizations,
nativeRunResults,
@@ -24,19 +27,15 @@ import {
runControlPlanePortConformance,
type NativeSessionBackend,
type PrpEvent,
type PrpStructuredRunResult,
type PrpTerminalState,
} from "@paperclipai/paperclip-runner";
import {
getEmbeddedPostgresTestSupport,
startEmbeddedPostgresTestDatabase,
} from "../../__tests__/helpers/embedded-postgres.js";
import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
import { PaperclipControlPlanePort } from "./paperclip-control-plane-port.js";
import { finalizeNativeRun } from "./native-run-finalizer.js";
import { buildNativeExecutionInput } from "./native-execution-input.js";
const support = await getEmbeddedPostgresTestSupport();
const describeDatabase = support.supported ? describe : describe.skip;
describeDatabase("PaperclipControlPlanePort conformance", () => {
describe("PaperclipControlPlanePort conformance", () => {
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
let db: ReturnType<typeof createDb>;
const contractId = "00000000-0000-4000-8000-000000000004";
@@ -44,6 +43,7 @@ describeDatabase("PaperclipControlPlanePort conformance", () => {
const taskIssueId = "00000000-0000-4000-8000-000000000008";
const taskContractId = "00000000-0000-4000-8000-000000000009";
const taskRunId = "00000000-0000-4000-8000-000000000010";
const taskWorkProductId = "00000000-0000-4000-8000-000000000017";
const workspaceFailureIssueId = "00000000-0000-4000-8000-000000000011";
const workspaceFailureContractId = "00000000-0000-4000-8000-000000000012";
const workspaceFailureRunId = "00000000-0000-4000-8000-000000000013";
@@ -68,6 +68,7 @@ describeDatabase("PaperclipControlPlanePort conformance", () => {
companyId: identity.companyId,
title: "Native conformance",
status: "in_progress",
assigneeAgentId: identity.agentId,
workMode: "standard",
});
await db.insert(completionContracts).values({
@@ -80,7 +81,11 @@ describeDatabase("PaperclipControlPlanePort conformance", () => {
risk: "standard",
completionAuthority: "server_arbiter",
incompleteCriteriaPolicy: "preserve_non_terminal",
contractJson: { objective: "Conformance" },
contractJson: {
revision: "phase6-v1",
objective: "Conformance",
criteria: [{ id: "objective", requirement: "Complete conformance" }],
},
canonicalSha256: contractSha,
createdByActorType: "system",
createdByActorId: "test",
@@ -100,6 +105,7 @@ describeDatabase("PaperclipControlPlanePort conformance", () => {
companyId: identity.companyId,
title: "Complete one native task",
status: "in_progress",
assigneeAgentId: identity.agentId,
workMode: "standard",
});
await db.insert(completionContracts).values({
@@ -112,7 +118,11 @@ describeDatabase("PaperclipControlPlanePort conformance", () => {
risk: "standard",
completionAuthority: "server_arbiter",
incompleteCriteriaPolicy: "preserve_non_terminal",
contractJson: { objective: "Complete one native task" },
contractJson: {
revision: "phase6-v1",
objective: "Complete one native task",
criteria: [{ id: "objective", requirement: "Complete the task" }],
},
canonicalSha256: "phase6-task-contract",
createdByActorType: "system",
createdByActorId: "test",
@@ -129,11 +139,23 @@ describeDatabase("PaperclipControlPlanePort conformance", () => {
completionContractSha256: "phase6-task-contract",
contextSnapshot: { issueId: taskIssueId },
});
await db.insert(issueWorkProducts).values({
id: taskWorkProductId,
companyId: identity.companyId,
issueId: taskIssueId,
type: "artifact",
provider: "paperclip",
title: "Verified native task result",
status: "ready_for_review",
reviewState: "approved",
createdByRunId: taskRunId,
});
await db.insert(issues).values({
id: workspaceFailureIssueId,
companyId: identity.companyId,
title: "Preserve status after workspace failure",
status: "in_progress",
assigneeAgentId: identity.agentId,
workMode: "standard",
});
await db.insert(completionContracts).values({
@@ -146,7 +168,11 @@ describeDatabase("PaperclipControlPlanePort conformance", () => {
risk: "standard",
completionAuthority: "server_arbiter",
incompleteCriteriaPolicy: "preserve_non_terminal",
contractJson: { objective: "Preserve status after workspace failure" },
contractJson: {
revision: "phase6-v1",
objective: "Preserve status after workspace failure",
criteria: [{ id: "objective", requirement: "Preserve the result" }],
},
canonicalSha256: "phase6-workspace-failure-contract",
createdByActorType: "system",
createdByActorId: "test",
@@ -178,7 +204,11 @@ describeDatabase("PaperclipControlPlanePort conformance", () => {
risk: "standard",
completionAuthority: "server_arbiter",
incompleteCriteriaPolicy: "preserve_non_terminal",
contractJson: { objective: "Respect pending governance" },
contractJson: {
revision: "phase6-v1",
objective: "Respect pending governance",
criteria: [{ id: "objective", requirement: "Respect pending governance" }],
},
canonicalSha256: "phase6-governance-contract",
createdByActorType: "system",
createdByActorId: "test",
@@ -205,12 +235,14 @@ describeDatabase("PaperclipControlPlanePort conformance", () => {
afterAll(async () => {
if (temporary) {
await db.delete(activityLog);
await db.delete(agentWakeupRequests);
await db.delete(statusDecisionEffects);
await db.delete(statusDecisions);
await db.delete(workAssessments);
await db.delete(nativeRunFinalizations);
await db.delete(nativeRunResults);
await db.delete(issueThreadInteractions);
await db.delete(issueWorkProducts);
await db.delete(heartbeatRunEvents);
await db.delete(heartbeatRuns);
await db.delete(completionContracts);
@@ -240,24 +272,37 @@ describeDatabase("PaperclipControlPlanePort conformance", () => {
terminalReplayIdempotent: true,
});
await expect(db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, identity.runId))).resolves.toHaveLength(1);
await finalizeNativeRun({ db, runId: identity.runId, workspaceFinalizeStatus: "succeeded" });
await finalizeNativeRun({
db,
runId: identity.runId,
workspaceFinalizeStatus: "succeeded",
});
await expect(db.select().from(nativeRunFinalizations)
.where(eq(nativeRunFinalizations.runId, identity.runId))).resolves.toEqual([
expect.objectContaining({ phase: "committed" }),
]);
await expect(db.select().from(issues).where(eq(issues.id, identity.issueId))).resolves.toEqual([
expect.objectContaining({ status: "done", statusVersion: 1 }),
expect.objectContaining({ status: "in_progress", statusVersion: 1 }),
]);
await expect(db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, identity.runId))).resolves.toEqual([
expect.objectContaining({ phase: "applied" }),
expect.objectContaining({ phase: "committed" }),
]);
await expect(db.select().from(statusDecisions).where(eq(statusDecisions.issueId, identity.issueId))).resolves.toEqual([
expect.objectContaining({ toStatus: "done", applicationState: "applied" }),
]);
await expect(db.select().from(activityLog).where(eq(activityLog.entityId, identity.issueId))).resolves.toEqual([
expect.objectContaining({ action: "issue.native_status_decision_applied" }),
expect.objectContaining({ toStatus: "in_progress", reasonCode: "native_completion_incomplete", applicationState: "applied" }),
]);
await expect(db.select().from(activityLog).where(eq(activityLog.entityId, identity.issueId))).resolves.toEqual(
expect.arrayContaining([expect.objectContaining({ action: "issue.updated" })]),
);
});
it("completes one selected Paperclip task through the public package session contract", async () => {
const identity = CONTROL_PLANE_CONFORMANCE_OPEN.identity;
const sessionId = "session-phase6-paperclip-task";
const evidenceRef = `work_product:${taskWorkProductId}`;
const taskResult = structuredClone(CONTROL_PLANE_CONFORMANCE_RESULT);
taskResult.completionClaim.criteria[0]!.evidenceRefs = [evidenceRef];
taskResult.evidence = [{ kind: "work_product", ref: evidenceRef }];
taskResult.verification[0]!.artifactRef = evidenceRef;
const event = (sourceSeq: number, eventType: PrpEvent["eventType"], payload: Record<string, unknown>): PrpEvent => ({
schema: "paperclip.prp.event.v1",
sourceEventId: `phase6-task:${sourceSeq}`,
@@ -274,7 +319,7 @@ describeDatabase("PaperclipControlPlanePort conformance", () => {
payload,
});
const events = [
event(1, "run.result.proposed", CONTROL_PLANE_CONFORMANCE_RESULT),
event(1, "run.result.proposed", taskResult),
event(2, "turn.completed", {}),
];
const backend: NativeSessionBackend = {
@@ -293,7 +338,7 @@ describeDatabase("PaperclipControlPlanePort conformance", () => {
async *events() { yield* events; },
async startTurn() { return { turnId: "turn-phase6-paperclip-task" }; },
async cancel() {},
async result() { return { result: CONTROL_PLANE_CONFORMANCE_RESULT, terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, turnId: "turn-phase6-paperclip-task" }; },
async result() { return { result: taskResult, terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, turnId: "turn-phase6-paperclip-task" }; },
async snapshot() { return { backendKind: "mock", sessionId, identity: input.identity, providerSessionId: "provider-phase6-paperclip-task" }; },
async close() {},
};
@@ -332,9 +377,30 @@ describeDatabase("PaperclipControlPlanePort conformance", () => {
});
expect(completed.terminal.runTerminalState).toBe("succeeded");
await finalizeNativeRun({ db, runId: taskRunId, workspaceFinalizeStatus: "succeeded" });
await expect(port.completeRun({
result: taskResult,
terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL,
turnId: "turn-phase6-paperclip-task",
callerResultId: `phase6-scripted-runner:${taskRunId}:result`,
callerDedupeKey: `${taskRunId}:phase6-task-contract`,
})).resolves.toBeUndefined();
await expect(port.completeRun({
result: { ...structuredClone(taskResult), summary: "Conflicting changed result bytes" },
terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL,
turnId: "turn-phase6-paperclip-task",
callerResultId: `phase6-scripted-runner:${taskRunId}:result`,
callerDedupeKey: `${taskRunId}:phase6-task-contract`,
})).rejects.toThrow("structured_result_replay_conflict");
await Promise.all([
finalizeNativeRun({ db, runId: taskRunId, workspaceFinalizeStatus: "succeeded" }),
finalizeNativeRun({ db, runId: taskRunId, workspaceFinalizeStatus: "succeeded" }),
]);
await expect(db.select().from(issues).where(eq(issues.id, taskIssueId))).resolves.toEqual([
expect.objectContaining({ status: "done", statusVersion: 1 }),
]);
await expect(db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, taskRunId))).resolves.toHaveLength(1);
await expect(db.select().from(workAssessments).where(eq(workAssessments.runId, taskRunId))).resolves.toHaveLength(1);
await expect(db.select().from(statusDecisions).where(eq(statusDecisions.issueId, taskIssueId))).resolves.toHaveLength(1);
});
it("fails closed when the bound company does not own the run", async () => {
@@ -376,13 +442,85 @@ describeDatabase("PaperclipControlPlanePort conformance", () => {
});
await finalizeNativeRun({ db, runId: governanceRunId, workspaceFinalizeStatus: "succeeded" });
await expect(db.select().from(issues).where(eq(issues.id, governanceIssueId))).resolves.toEqual([
expect.objectContaining({ status: "in_progress" }),
expect.objectContaining({ status: "in_review" }),
]);
await expect(db.select().from(statusDecisions).where(eq(statusDecisions.issueId, governanceIssueId))).resolves.toEqual([
expect.objectContaining({ toStatus: "in_progress", reasonCode: "approval_gate_pending" }),
expect.objectContaining({ toStatus: "in_review", reasonCode: "governance_gate_pending" }),
]);
});
it("P6-23 materializes native review through the authorized interaction service", async () => {
const identity = CONTROL_PLANE_CONFORMANCE_OPEN.identity;
const issueId = "30000000-0000-4000-8000-000000000024";
const localContractId = "31000000-0000-4000-8000-000000000024";
const runId = "32000000-0000-4000-8000-000000000024";
await db.insert(issues).values({
id: issueId,
companyId: identity.companyId,
title: "Native review interaction",
status: "in_progress",
assigneeAgentId: identity.agentId,
workMode: "standard",
});
await db.insert(completionContracts).values({
id: localContractId,
companyId: identity.companyId,
issueId,
revision: 1,
schemaVersion: "paperclip.completion-contract.v1",
policyVersion: "phase6-v1",
risk: "standard",
completionAuthority: "server_arbiter",
incompleteCriteriaPolicy: "preserve_non_terminal",
contractJson: { revision: "phase6-v1", objective: "Review", criteria: [{ id: "objective", requirement: "Review" }] },
canonicalSha256: "native-review-contract",
createdByActorType: "system",
createdByActorId: "test",
});
await db.insert(heartbeatRuns).values({
id: runId,
companyId: identity.companyId,
agentId: identity.agentId,
status: "running",
runtimeMode: "native",
completionContractId: localContractId,
completionContractSha256: "native-review-contract",
contextSnapshot: { issueId },
});
const port = new PaperclipControlPlanePort(db, {
companyId: identity.companyId,
issueId,
runId,
agentId: identity.agentId,
completionContractId: localContractId,
completionContractSha256: "native-review-contract",
sourceInstanceId: "native-review-runner",
controlPlaneSourceInstanceId: "native-review-control",
});
await port.openRun({
identity: { ...identity, issueId, runId },
backendKind: "mock",
sourceInstanceId: "native-review-runner",
});
const result = { ...structuredClone(CONTROL_PLANE_CONFORMANCE_RESULT), reportedWorkDisposition: "needs_review" as const };
await port.completeRun({
result,
terminal: { ...CONTROL_PLANE_CONFORMANCE_TERMINAL, reportedWorkDisposition: "needs_review" },
callerResultId: "native-review-result",
});
await finalizeNativeRun({ db, runId, workspaceFinalizeStatus: "succeeded" });
await expect(db.select().from(issues).where(eq(issues.id, issueId))).resolves.toEqual([
expect.objectContaining({ status: "in_review", statusVersion: 1 }),
]);
await expect(db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.issueId, issueId))).resolves.toEqual([
expect.objectContaining({ kind: "request_confirmation", status: "pending", sourceRunId: runId }),
]);
await expect(db.select().from(statusDecisionEffects).where(eq(statusDecisionEffects.issueId, issueId))).resolves.toEqual(
expect.arrayContaining([expect.objectContaining({ effectKind: "create_review_interaction", targetType: "issue_thread_interaction" })]),
);
});
it("preserves the result and issue status when workspace finalization fails", async () => {
const identity = CONTROL_PLANE_CONFORMANCE_OPEN.identity;
const port = new PaperclipControlPlanePort(db, {
@@ -415,8 +553,125 @@ describeDatabase("PaperclipControlPlanePort conformance", () => {
expect.objectContaining({ status: "in_progress", statusVersion: 0, lastStatusDecisionId: null }),
]);
await expect(db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, workspaceFailureRunId))).resolves.toEqual([
expect.objectContaining({ status: "failed", nativePhase: "workspace_failed" }),
expect.objectContaining({ status: "failed", nativePhase: "retryable_failure" }),
]);
await expect(db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, workspaceFailureRunId))).resolves.toHaveLength(1);
});
it("LIVE-01..04 rolls back status, decision, and liveness rows at every materialization failpoint", async () => {
const identity = CONTROL_PLANE_CONFORMANCE_OPEN.identity;
const cases: Array<{
suffix: number;
failpoint: "interaction_materialization" | "continuation_materialization" | "blocker_materialization" | "status_projection";
result: PrpStructuredRunResult;
}> = [
{
suffix: 20,
failpoint: "interaction_materialization",
result: { ...structuredClone(CONTROL_PLANE_CONFORMANCE_RESULT), reportedWorkDisposition: "needs_review" },
},
{
suffix: 21,
failpoint: "continuation_materialization",
result: {
...structuredClone(CONTROL_PLANE_CONFORMANCE_RESULT),
reportedWorkDisposition: "yielded",
continuation: { kind: "same_agent", summary: "Continue after recovery", idempotencyKey: "live-continue" },
},
},
{
suffix: 22,
failpoint: "blocker_materialization",
result: {
...structuredClone(CONTROL_PLANE_CONFORMANCE_RESULT),
reportedWorkDisposition: "blocked",
blocker: { reasonCode: "access", owner: { kind: "board" }, unblockAction: "Approve access", scope: "task_wide" },
},
},
{
suffix: 23,
failpoint: "status_projection",
result: { ...structuredClone(CONTROL_PLANE_CONFORMANCE_RESULT), reportedWorkDisposition: "needs_review" },
},
];
for (const entry of cases) {
const value = String(entry.suffix).padStart(12, "0");
const issueId = `30000000-0000-4000-8000-${value}`;
const contractId = `31000000-0000-4000-8000-${value}`;
const runId = `32000000-0000-4000-8000-${value}`;
await db.insert(issues).values({
id: issueId,
companyId: identity.companyId,
title: `Atomic liveness ${entry.failpoint}`,
status: "in_progress",
assigneeAgentId: identity.agentId,
workMode: "standard",
});
await db.insert(completionContracts).values({
id: contractId,
companyId: identity.companyId,
issueId,
revision: 1,
schemaVersion: "paperclip.completion-contract.v1",
policyVersion: "phase6-v1",
risk: "standard",
completionAuthority: "server_arbiter",
incompleteCriteriaPolicy: "preserve_non_terminal",
contractJson: { revision: "phase6-v1", objective: "Atomic liveness", criteria: [{ id: "objective", requirement: "Stay atomic" }] },
canonicalSha256: `contract-${entry.suffix}`,
createdByActorType: "system",
createdByActorId: "test",
});
await db.insert(heartbeatRuns).values({
id: runId,
companyId: identity.companyId,
agentId: identity.agentId,
status: "running",
runtimeMode: "native",
completionContractId: contractId,
completionContractSha256: `contract-${entry.suffix}`,
contextSnapshot: { issueId },
});
const terminal: PrpTerminalState = {
...CONTROL_PLANE_CONFORMANCE_TERMINAL,
reportedWorkDisposition: entry.result.reportedWorkDisposition,
};
const port = new PaperclipControlPlanePort(db, {
companyId: identity.companyId,
issueId,
runId,
agentId: identity.agentId,
completionContractId: contractId,
completionContractSha256: `contract-${entry.suffix}`,
sourceInstanceId: `atomic-runner-${entry.suffix}`,
controlPlaneSourceInstanceId: `atomic-control-${entry.suffix}`,
});
await port.openRun({
identity: { ...identity, issueId, runId },
backendKind: "mock",
sourceInstanceId: `atomic-runner-${entry.suffix}`,
});
await port.completeRun({ result: entry.result, terminal, callerResultId: `atomic-result-${entry.suffix}` });
await expect(finalizeNativeRun({
db,
runId,
workspaceFinalizeStatus: "succeeded",
failpoint: entry.failpoint,
})).resolves.toEqual(expect.objectContaining({ phase: "retryable_failure" }));
await expect(db.select().from(issues).where(eq(issues.id, issueId))).resolves.toEqual([
expect.objectContaining({ status: "in_progress", statusVersion: 0, lastStatusDecisionId: null }),
]);
await expect(db.select().from(statusDecisions).where(eq(statusDecisions.issueId, issueId))).resolves.toHaveLength(0);
await expect(db.select().from(statusDecisionEffects).where(eq(statusDecisionEffects.issueId, issueId))).resolves.toHaveLength(0);
await expect(db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.issueId, issueId))).resolves.toHaveLength(0);
await expect(db.select().from(agentWakeupRequests)
.where(eq(agentWakeupRequests.companyId, identity.companyId))).resolves.not.toEqual(
expect.arrayContaining([expect.objectContaining({ payload: expect.objectContaining({ issueId }) })]),
);
await expect(db.select().from(issueRecoveryActions)
.where(eq(issueRecoveryActions.sourceIssueId, issueId))).resolves.toEqual([
expect.objectContaining({ status: "active", cause: "side_effect_planning_failed" }),
]);
}
});
});
@@ -12,6 +12,7 @@ import type {
NativeRunEvent,
NativeRunResult,
OpenControlPlaneRunInput,
PersistedNativeSession,
PrpEvent,
PrpTerminalState,
ReplayControlPlaneEventsInput,
@@ -42,6 +43,12 @@ function isCompleteInput(value: NativeRunResult | CompleteControlPlaneRunInput):
return "result" in value && "terminal" in value;
}
function record(value: unknown): Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value)
? value as Record<string, unknown>
: {};
}
function assertTerminal(value: unknown): asserts value is PrpTerminalState {
const terminal = value as Partial<PrpTerminalState> | null;
if (
@@ -90,6 +97,69 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
this.#sessionId = identity.sessionId;
}
async loadSessionCheckpoint(): Promise<PersistedNativeSession | null> {
const run = await this.#db.select({
companyId: heartbeatRuns.companyId,
agentId: heartbeatRuns.agentId,
runtimeMode: heartbeatRuns.runtimeMode,
nativeSessionId: heartbeatRuns.nativeSessionId,
runnerProfileJson: heartbeatRuns.runnerProfileJson,
}).from(heartbeatRuns).where(eq(heartbeatRuns.id, this.#binding.runId))
.limit(1).then((rows) => rows[0] ?? null);
if (
!run
|| run.companyId !== this.#binding.companyId
|| run.agentId !== this.#binding.agentId
|| run.runtimeMode !== "native"
) throw new Error("native_session_checkpoint_binding_mismatch");
const candidate = record(run.runnerProfileJson).sessionCheckpoint;
if (candidate === undefined || candidate === null) return null;
const snapshot = record(candidate) as Partial<PersistedNativeSession>;
const identity = record(snapshot.identity);
if (
typeof snapshot.sessionId !== "string"
|| identity.companyId !== this.#binding.companyId
|| identity.issueId !== this.#binding.issueId
|| identity.runId !== this.#binding.runId
|| identity.agentId !== this.#binding.agentId
|| identity.sessionId !== run.nativeSessionId
) throw new Error("native_session_checkpoint_invalid");
return structuredClone(snapshot as PersistedNativeSession);
}
async checkpointSession(snapshot: PersistedNativeSession): Promise<void> {
const identity = snapshot.identity;
if (
identity.companyId !== this.#binding.companyId
|| identity.issueId !== this.#binding.issueId
|| identity.runId !== this.#binding.runId
|| identity.agentId !== this.#binding.agentId
|| this.#sessionId !== identity.sessionId
) throw new Error("native_session_checkpoint_binding_mismatch");
await this.#db.transaction(async (tx) => {
const run = await tx.select().from(heartbeatRuns)
.where(eq(heartbeatRuns.id, this.#binding.runId)).for("update").limit(1)
.then((rows) => rows[0] ?? null);
if (
!run
|| run.companyId !== this.#binding.companyId
|| run.agentId !== this.#binding.agentId
|| run.runtimeMode !== "native"
|| run.nativeSessionId !== identity.sessionId
) throw new Error("native_session_checkpoint_binding_mismatch");
await tx.update(heartbeatRuns).set({
runnerProfileJson: {
...record(run.runnerProfileJson),
sessionCheckpoint: structuredClone(snapshot) as unknown as Record<string, unknown>,
},
sessionIdAfter: snapshot.providerSessionId ?? snapshot.sessionId,
nativePhase: snapshot.semanticResult ? "workspace_finalizing" : "observed",
nativePhaseUpdatedAt: new Date(),
updatedAt: new Date(),
}).where(eq(heartbeatRuns.id, this.#binding.runId));
});
}
async appendEvent(value: NativeRunEvent | PrpEvent) {
if (!isPrpEvent(value)) throw new Error("native_legacy_event_not_supported");
const validated = validatePrpEvent(value);
@@ -197,7 +267,7 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
.where(and(eq(nativeRunResults.runId, this.#binding.runId), or(...callerConditions)))
.limit(1).then((rows) => rows[0] ?? null);
if (existing) {
if (existing.canonicalSha256 !== canonicalSha256) throw new Error("native_result_idempotency_conflict");
if (existing.canonicalSha256 !== canonicalSha256) throw new Error("structured_result_replay_conflict");
return;
}
const resultJson = {
@@ -222,14 +292,21 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
runId: this.#binding.runId,
companyId: this.#binding.companyId,
issueId: this.#binding.issueId,
phase: "result_persisted",
phase: "workspace_finalizing",
resultId: inserted.id,
}).onConflictDoUpdate({
target: nativeRunFinalizations.runId,
set: { phase: "result_persisted", resultId: inserted.id, failureCode: null, failureDetail: null, updatedAt: new Date() },
set: {
phase: "workspace_finalizing",
resultId: inserted.id,
failureCode: null,
failureDetail: null,
nextAttemptAt: null,
updatedAt: new Date(),
},
});
await tx.update(heartbeatRuns).set({
nativePhase: "result_persisted",
nativePhase: "workspace_finalizing",
nativePhaseUpdatedAt: new Date(),
updatedAt: new Date(),
}).where(eq(heartbeatRuns.id, this.#binding.runId));
@@ -1,72 +1,97 @@
import { describe, expect, it } from "vitest";
import { classifyNativeEvidence } from "./evidence-classifier.js";
import type { NativeEvidenceAssessment } from "./evidence-classifier.js";
import { arbitrateNativeStatus } from "./status-arbiter.js";
function result(overrides: Record<string, unknown> = {}) {
function assessment(overrides: Partial<NativeEvidenceAssessment> = {}): NativeEvidenceAssessment {
return {
reportedWorkDisposition: "done",
objectiveSatisfied: true,
allCriteriaSatisfied: true,
verificationPassed: true,
hasBlockingRemainingWork: false,
reportedDisposition: "done",
summary: "Complete",
completionClaim: {
objectiveSatisfied: true,
criteria: [{ criterionId: "objective", status: "satisfied", evidenceRefs: ["test"] }],
remainingWork: [],
},
verification: [{ commandOrCheck: "test", status: "passed" }],
contractRevisionMatches: true,
criterionAssessments: [],
verificationAssessments: [],
acceptedEvidenceRefs: ["event:2"],
missingRequirements: [],
rejectedEvidence: [],
unverifiableEvidence: [],
blocker: null,
continuation: null,
attentionRequests: [],
...overrides,
};
}
function arbitrate(overrides: Partial<Parameters<typeof arbitrateNativeStatus>[0]> = {}) {
return arbitrateNativeStatus({
assessment: assessment(),
terminalState: "succeeded",
workspaceFinalizeStatus: "succeeded",
agentId: "agent",
priorIssueStatus: "in_progress",
...overrides,
});
}
describe("native status authority", () => {
it("marks done only from successful finalization and complete evidence", () => {
const assessment = classifyNativeEvidence(result());
expect(arbitrateNativeStatus({
assessment,
terminalState: "succeeded",
workspaceFinalizeStatus: "succeeded",
agentId: "agent",
}).toStatus).toBe("done");
it("marks done only from successful finalization and complete durable evidence", () => {
expect(arbitrate()).toEqual(expect.objectContaining({
toStatus: "done",
reasonCode: "completion_contract_satisfied",
effects: [{ kind: "release_checkout" }],
}));
expect(arbitrate({
assessment: assessment({ verificationPassed: false, missingRequirements: ["test"] }),
}).toStatus).toBe("in_progress");
});
it("keeps incomplete, review, cancelled, and yielded work non-terminal", () => {
for (const reportedWorkDisposition of ["needs_review", "yielded"] as const) {
expect(arbitrateNativeStatus({
assessment: classifyNativeEvidence(result({ reportedWorkDisposition })),
terminalState: "succeeded",
workspaceFinalizeStatus: "succeeded",
agentId: "agent",
}).toStatus).toBe("in_progress");
}
expect(arbitrateNativeStatus({
assessment: classifyNativeEvidence(result()),
terminalState: "cancelled",
workspaceFinalizeStatus: "succeeded",
agentId: "agent",
}).toStatus).toBe("in_progress");
expect(arbitrateNativeStatus({
assessment: classifyNativeEvidence(result()),
terminalState: "succeeded",
workspaceFinalizeStatus: "succeeded",
approvalGateSatisfied: false,
agentId: "agent",
it("creates explicit liveness paths for review, continuation, cancellation, and governance", () => {
expect(arbitrate({
assessment: assessment({ reportedDisposition: "needs_review" }),
})).toEqual(expect.objectContaining({
toStatus: "in_review",
effects: [expect.objectContaining({ kind: "create_review_interaction" })],
}));
expect(arbitrate({
assessment: assessment({
reportedDisposition: "yielded",
continuation: { kind: "retry", summary: "Retry the task", idempotencyKey: "retry-task" },
}),
})).toEqual(expect.objectContaining({
toStatus: "in_progress",
reasonCode: "approval_gate_pending",
effects: [expect.objectContaining({ kind: "enqueue_continuation", continuationKind: "retry" })],
}));
expect(arbitrate({ terminalState: "cancelled" })).toEqual(expect.objectContaining({
toStatus: "in_progress",
effects: [expect.objectContaining({ kind: "record_recovery", cause: "native_run_cancelled" })],
}));
expect(arbitrate({ governanceGate: { kind: "interaction", id: "interaction" } })).toEqual(expect.objectContaining({
toStatus: "in_review",
reasonCode: "governance_gate_pending",
effects: [{ kind: "bind_governance", gate: { kind: "interaction", id: "interaction" } }],
}));
});
it("blocks only on a first-class blocker or failed workspace finalization", () => {
const explicitBlocker = classifyNativeEvidence(result({
reportedWorkDisposition: "blocked",
blocker: { owner: { kind: "board" }, unblockAction: "Approve access" },
}));
expect(arbitrateNativeStatus({
assessment: explicitBlocker,
terminalState: "succeeded",
workspaceFinalizeStatus: "succeeded",
agentId: "agent",
it("blocks only for a task-wide blocker with a named owner and action", () => {
expect(arbitrate({
assessment: assessment({
reportedDisposition: "blocked",
blocker: { boardOwned: true, scope: "task_wide", unblockAction: "Approve access" },
}),
})).toEqual(expect.objectContaining({
toStatus: "blocked",
unblockDescriptor: { owner: "board", action: "Approve access" },
effects: [{ kind: "bind_blocker", owner: "board", action: "Approve access" }],
}));
});
it("preserves authoritative terminal statuses", () => {
expect(arbitrate({ priorIssueStatus: "done" })).toEqual(expect.objectContaining({
toStatus: "done",
reasonCode: "terminal_status_preserved",
effects: [],
}));
});
});
@@ -1,12 +1,41 @@
import type { NativeEvidenceAssessment } from "./evidence-classifier.js";
export const NATIVE_STATUS_ARBITER_POLICY_VERSION = "phase6-v1";
export const NATIVE_STATUS_ARBITER_POLICY_VERSION = "phase6-v2";
export type NativeAuthoritativeIssueStatus =
| "backlog"
| "todo"
| "in_progress"
| "in_review"
| "blocked"
| "done"
| "cancelled";
export type NativeGovernanceGate = {
kind: "approval" | "interaction" | "execution_stage";
id: string;
};
export type NativeStatusEffect =
| { kind: "bind_governance"; gate: NativeGovernanceGate }
| { kind: "create_review_interaction"; prompt: string }
| {
kind: "enqueue_continuation";
continuationKind: "same_agent" | "retry" | "delegated_issue" | "response_wake" | "monitor";
summary: string;
idempotencyKey: string;
agentId: string;
}
| { kind: "bind_blocker"; owner: { agentId: string } | "board"; action: string }
| { kind: "record_recovery"; cause: string; nextAction: string; agentId: string }
| { kind: "release_checkout" };
export interface NativeStatusDecision {
policyVersion: typeof NATIVE_STATUS_ARBITER_POLICY_VERSION;
toStatus: "in_progress" | "done" | "blocked";
toStatus: NativeAuthoritativeIssueStatus;
reasonCode: string;
unblockDescriptor: { owner: { agentId: string } | "board"; action: string } | null;
effects: NativeStatusEffect[];
}
/** Pure authority boundary: model prose is evidence, never a status command. */
@@ -14,31 +43,55 @@ export function arbitrateNativeStatus(input: {
assessment: NativeEvidenceAssessment;
terminalState: "succeeded" | "failed" | "cancelled";
workspaceFinalizeStatus: "succeeded" | "failed";
approvalGateSatisfied?: boolean;
governanceGate?: NativeGovernanceGate | null;
agentId: string;
priorIssueStatus: NativeAuthoritativeIssueStatus;
}): NativeStatusDecision {
if (["done", "cancelled"].includes(input.priorIssueStatus)) {
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
toStatus: input.priorIssueStatus,
reasonCode: "terminal_status_preserved",
unblockDescriptor: null,
effects: [],
};
}
if (input.workspaceFinalizeStatus !== "succeeded") {
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
toStatus: "blocked",
reasonCode: "workspace_finalization_failed",
unblockDescriptor: { owner: { agentId: input.agentId }, action: "Repair and re-verify workspace finalization." },
toStatus: input.priorIssueStatus,
reasonCode: "workspace_finalization_retry_required",
unblockDescriptor: null,
effects: [{
kind: "record_recovery",
cause: "workspace_finalization_failed",
nextAction: "Repair and re-run workspace finalization for the persisted native result.",
agentId: input.agentId,
}],
};
}
if (input.terminalState !== "succeeded") {
const cause = input.terminalState === "cancelled" ? "native_run_cancelled" : "native_run_failed";
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
toStatus: "in_progress",
reasonCode: input.terminalState === "cancelled" ? "native_run_cancelled" : "native_run_failed",
reasonCode: cause,
unblockDescriptor: null,
effects: [{
kind: "record_recovery",
cause,
nextAction: "Resume the persisted native run without opening a second provider session.",
agentId: input.agentId,
}],
};
}
if (input.approvalGateSatisfied === false) {
if (input.governanceGate) {
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
toStatus: "in_progress",
reasonCode: "approval_gate_pending",
toStatus: "in_review",
reasonCode: "governance_gate_pending",
unblockDescriptor: null,
effects: [{ kind: "bind_governance", gate: input.governanceGate }],
};
}
const complete =
@@ -53,27 +106,72 @@ export function arbitrateNativeStatus(input: {
toStatus: "done",
reasonCode: "completion_contract_satisfied",
unblockDescriptor: null,
effects: [{ kind: "release_checkout" }],
};
}
if (input.assessment.reportedDisposition === "needs_review") {
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
toStatus: "in_review",
reasonCode: "native_review_path_required",
unblockDescriptor: null,
effects: [{
kind: "create_review_interaction",
prompt: "Review the persisted native-run evidence and confirm whether this issue may be completed.",
}],
};
}
if (input.assessment.reportedDisposition === "blocked" && input.assessment.blocker) {
const owner = input.assessment.blocker.boardOwned ? "board" as const : { agentId: input.agentId };
if (input.assessment.blocker.scope === "task_wide") {
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
toStatus: "blocked",
reasonCode: "native_task_wide_blocker",
unblockDescriptor: { owner, action: input.assessment.blocker.unblockAction },
effects: [{ kind: "bind_blocker", owner, action: input.assessment.blocker.unblockAction }],
};
}
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
toStatus: "blocked",
reasonCode: "native_reported_blocker",
unblockDescriptor: {
owner: input.assessment.blocker.boardOwned ? "board" : { agentId: input.agentId },
action: input.assessment.blocker.unblockAction,
},
toStatus: "in_progress",
reasonCode: "native_track_blocked_continuation",
unblockDescriptor: null,
effects: [{
kind: "enqueue_continuation",
continuationKind: "same_agent",
summary: `Continue another productive track while resolving: ${input.assessment.blocker.unblockAction}`,
idempotencyKey: `native-track-blocked:${input.assessment.blocker.unblockAction}`,
agentId: input.agentId,
}],
};
}
if (input.assessment.reportedDisposition === "yielded" && input.assessment.continuation) {
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
toStatus: "in_progress",
reasonCode: "native_continuation_required",
unblockDescriptor: null,
effects: [{
kind: "enqueue_continuation",
continuationKind: input.assessment.continuation.kind,
summary: input.assessment.continuation.summary,
idempotencyKey: input.assessment.continuation.idempotencyKey,
agentId: input.agentId,
}],
};
}
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
toStatus: "in_progress",
reasonCode: input.assessment.reportedDisposition === "needs_review"
? "native_review_path_required"
: input.assessment.reportedDisposition === "yielded"
? "native_continuation_required"
: "native_completion_incomplete",
reasonCode: "native_completion_incomplete",
unblockDescriptor: null,
effects: [{
kind: "enqueue_continuation",
continuationKind: "same_agent",
summary: "Continue work on the missing or unverifiable completion-contract evidence.",
idempotencyKey: "native-completion-incomplete",
agentId: input.agentId,
}],
};
}
@@ -1,14 +1,23 @@
import { and, eq } from "drizzle-orm";
import { and, eq, inArray } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import type { CreateIssueThreadInteraction } from "@paperclipai/shared";
import {
agentWakeupRequests,
approvals,
issueApprovals,
issueThreadInteractions,
issues,
activityLog,
nativeRunFinalizations,
statusDecisionEffects,
statusDecisions,
} from "@paperclipai/db";
import type { NativeStatusDecision } from "./status-arbiter.js";
import type { NativeStatusDecision, NativeStatusEffect } from "./status-arbiter.js";
import { nativeSha256 } from "./canonical.js";
import { issueService } from "../issues.js";
import { issueThreadInteractionService } from "../issue-thread-interactions.js";
import { issueRecoveryActionService } from "../issue-recovery-actions.js";
import { buildIssueBlockersResolvedWakeIdempotencyKey } from "../issue-dependency-wakeups.js";
import { persistActivity, publishActivity, type ActivityPublication } from "../activity-log.js";
export class NativeStatusRaceError extends Error {
readonly code = "native_status_race" as const;
@@ -18,6 +27,229 @@ export class NativeStatusRaceError extends Error {
}
}
export type NativeStatusCommitFailpoint =
| "governance_materialization"
| "interaction_materialization"
| "continuation_materialization"
| "blocker_materialization"
| "recovery_materialization"
| "status_projection";
type MaterializedEffect = {
effectKind: string;
targetType: string;
targetId: string | null;
payload: Record<string, unknown>;
};
function failAt(actual: NativeStatusCommitFailpoint, requested?: NativeStatusCommitFailpoint) {
if (actual === requested) throw new Error(`native_status_failpoint:${actual}`);
}
async function enqueueWake(input: {
tx: Db;
companyId: string;
issueId: string;
agentId: string;
reason: string;
idempotencyKey: string;
payload: Record<string, unknown>;
}) {
const existing = await input.tx.select({ id: agentWakeupRequests.id })
.from(agentWakeupRequests)
.where(and(
eq(agentWakeupRequests.companyId, input.companyId),
eq(agentWakeupRequests.idempotencyKey, input.idempotencyKey),
inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution", "claimed", "completed"]),
)).limit(1).then((rows) => rows[0] ?? null);
if (existing) return existing.id;
const inserted = await input.tx.insert(agentWakeupRequests).values({
companyId: input.companyId,
agentId: input.agentId,
source: "automation",
triggerDetail: "system",
reason: input.reason,
payload: {
issueId: input.issueId,
taskId: input.issueId,
...input.payload,
_paperclipWakeContext: {
issueId: input.issueId,
taskId: input.issueId,
wakeReason: input.reason,
source: "native_status_decision",
},
},
requestedByActorType: "system",
requestedByActorId: "native-status-committer",
idempotencyKey: input.idempotencyKey,
}).returning({ id: agentWakeupRequests.id }).then((rows) => rows[0] ?? null);
if (!inserted) throw new Error("native_status_wake_not_persisted");
return inserted.id;
}
async function validateGovernanceGate(tx: Db, input: {
companyId: string;
issueId: string;
gate: Extract<NativeStatusEffect, { kind: "bind_governance" }>["gate"];
executionState: unknown;
}) {
if (input.gate.kind === "execution_stage") {
const state = input.executionState && typeof input.executionState === "object"
? input.executionState as Record<string, unknown>
: {};
if (state.status !== "pending") throw new Error("native_governance_gate_resolved");
return input.gate.id;
}
if (input.gate.kind === "interaction") {
const interaction = await tx.select({ id: issueThreadInteractions.id })
.from(issueThreadInteractions).where(and(
eq(issueThreadInteractions.id, input.gate.id),
eq(issueThreadInteractions.companyId, input.companyId),
eq(issueThreadInteractions.issueId, input.issueId),
eq(issueThreadInteractions.status, "pending"),
)).limit(1).then((rows) => rows[0] ?? null);
if (!interaction) throw new Error("native_governance_gate_resolved");
return interaction.id;
}
const approval = await tx.select({ id: approvals.id }).from(issueApprovals)
.innerJoin(approvals, eq(issueApprovals.approvalId, approvals.id))
.where(and(
eq(issueApprovals.companyId, input.companyId),
eq(issueApprovals.issueId, input.issueId),
eq(approvals.companyId, input.companyId),
eq(approvals.id, input.gate.id),
inArray(approvals.status, ["pending", "revision_requested"]),
)).limit(1).then((rows) => rows[0] ?? null);
if (!approval) throw new Error("native_governance_gate_resolved");
return approval.id;
}
async function materializeDecisionEffect(input: {
tx: Db;
companyId: string;
issue: typeof issues.$inferSelect;
runId: string;
decisionId: string;
effect: NativeStatusEffect;
failpoint?: NativeStatusCommitFailpoint;
}): Promise<MaterializedEffect> {
const { effect } = input;
if (effect.kind === "bind_governance") {
failAt("governance_materialization", input.failpoint);
const targetId = await validateGovernanceGate(input.tx, {
companyId: input.companyId,
issueId: input.issue.id,
gate: effect.gate,
executionState: input.issue.executionState,
});
return { effectKind: effect.kind, targetType: effect.gate.kind, targetId, payload: { gate: effect.gate } };
}
if (effect.kind === "create_review_interaction") {
failAt("interaction_materialization", input.failpoint);
const reviewInput: Extract<CreateIssueThreadInteraction, { kind: "request_confirmation" }> = {
kind: "request_confirmation",
idempotencyKey: `native-review:${input.decisionId}`,
sourceRunId: input.runId,
title: "Native completion review",
summary: "The native runner requires authoritative review before completion.",
continuationPolicy: "wake_assignee",
payload: {
version: 1,
prompt: effect.prompt,
acceptLabel: "Approve completion",
rejectLabel: "Continue work",
allowDeclineReason: true,
rejectRequiresReason: true,
supersedeOnUserComment: false,
},
};
const interaction = await issueThreadInteractionService(input.tx).create(
input.issue,
reviewInput,
{ systemId: "native-status-committer", runId: input.runId },
);
return {
effectKind: effect.kind,
targetType: "issue_thread_interaction",
targetId: interaction.id,
payload: { interactionId: interaction.id, interactionKind: interaction.kind },
};
}
if (effect.kind === "enqueue_continuation") {
failAt("continuation_materialization", input.failpoint);
const wakeId = await enqueueWake({
tx: input.tx,
companyId: input.companyId,
issueId: input.issue.id,
agentId: effect.agentId,
reason: effect.continuationKind === "monitor" ? "monitor_due" : "issue_status_changed",
idempotencyKey: `native-status:${input.decisionId}:continuation`,
payload: {
nativeDecisionId: input.decisionId,
continuationKind: effect.continuationKind,
continuationSummary: effect.summary,
continuationIdempotencyKey: effect.idempotencyKey,
},
});
return {
effectKind: effect.kind,
targetType: "agent_wakeup_request",
targetId: wakeId,
payload: { continuationKind: effect.continuationKind, summary: effect.summary },
};
}
if (effect.kind === "bind_blocker") {
failAt("blocker_materialization", input.failpoint);
let wakeId: string | null = null;
if (effect.owner !== "board") {
wakeId = await enqueueWake({
tx: input.tx,
companyId: input.companyId,
issueId: input.issue.id,
agentId: effect.owner.agentId,
reason: "issue_status_changed",
idempotencyKey: `native-status:${input.decisionId}:blocker-owner`,
payload: { nativeDecisionId: input.decisionId, unblockAction: effect.action },
});
}
return {
effectKind: effect.kind,
targetType: effect.owner === "board" ? "board" : "agent_wakeup_request",
targetId: wakeId,
payload: { owner: effect.owner, action: effect.action },
};
}
if (effect.kind === "record_recovery") {
failAt("recovery_materialization", input.failpoint);
const recovery = await issueRecoveryActionService(input.tx).upsertSourceScoped({
companyId: input.companyId,
sourceIssueId: input.issue.id,
kind: "active_run_watchdog",
ownerType: "agent",
ownerAgentId: effect.agentId,
cause: effect.cause,
fingerprint: nativeSha256({ runId: input.runId, decisionId: input.decisionId, cause: effect.cause }),
evidence: { runId: input.runId, decisionId: input.decisionId },
nextAction: effect.nextAction,
wakePolicy: { kind: "resume_native_run", runId: input.runId },
maxAttempts: 3,
});
return {
effectKind: effect.kind,
targetType: "issue_recovery_action",
targetId: recovery.id,
payload: { cause: effect.cause, nextAction: effect.nextAction },
};
}
return {
effectKind: effect.kind,
targetType: "issue_checkout",
targetId: input.issue.id,
payload: { checkoutRunId: input.issue.checkoutRunId, executionRunId: input.issue.executionRunId },
};
}
export async function commitNativeStatusDecision(input: {
db: Db;
companyId: string;
@@ -26,104 +258,201 @@ export async function commitNativeStatusDecision(input: {
assessmentId: string;
priorStatus: string;
priorStatusVersion: number;
priorDecisionId: string | null;
decision: NativeStatusDecision;
failpoint?: NativeStatusCommitFailpoint;
}) {
return input.db.transaction(async (tx) => {
const publications: ActivityPublication[] = [];
const committed = await input.db.transaction(async (tx) => {
const coordinator = await tx.select().from(nativeRunFinalizations).where(and(
eq(nativeRunFinalizations.runId, input.runId),
eq(nativeRunFinalizations.companyId, input.companyId),
eq(nativeRunFinalizations.issueId, input.issueId),
)).for("update").limit(1).then((rows) => rows[0] ?? null);
if (!coordinator) throw new Error("native_finalization_coordinator_missing");
const issue = await tx.select().from(issues).where(and(
eq(issues.id, input.issueId),
eq(issues.companyId, input.companyId),
)).for("update").limit(1).then((rows) => rows[0] ?? null);
if (!issue || issue.status !== input.priorStatus || issue.statusVersion !== input.priorStatusVersion) {
if (
!issue
|| issue.status !== input.priorStatus
|| Number(issue.statusVersion) !== input.priorStatusVersion
|| issue.lastStatusDecisionId !== input.priorDecisionId
) {
throw new NativeStatusRaceError();
}
const decisionJson = {
toStatus: input.decision.toStatus,
reasonCode: input.decision.reasonCode,
unblockDescriptor: input.decision.unblockDescriptor,
effects: input.decision.effects,
};
const decisionDigest = nativeSha256({
issueId: input.issueId,
assessmentId: input.assessmentId,
policyVersion: input.decision.policyVersion,
fromStatus: issue.status,
priorStatusVersion: input.priorStatusVersion,
priorDecisionId: input.priorDecisionId,
decision: decisionJson,
});
let decisionRow = await tx.select().from(statusDecisions).where(and(
eq(statusDecisions.issueId, input.issueId),
eq(statusDecisions.decisionDigest, decisionDigest),
)).limit(1).then((rows) => rows[0] ?? null);
if (decisionRow?.applicationState === "applied") {
return { decision: decisionRow, issue, replayed: true };
}
if (!decisionRow) {
[decisionRow] = await tx.insert(statusDecisions).values({
companyId: input.companyId,
issueId: input.issueId,
assessmentId: input.assessmentId,
decisionVersion: 1,
decisionVersion: input.priorStatusVersion + 1,
policyVersion: input.decision.policyVersion,
fromStatus: issue.status,
toStatus: input.decision.toStatus,
reasonCode: input.decision.reasonCode,
decisionJson,
decisionDigest,
applicationState: "pending",
applicationState: "proposed",
}).returning();
}
if (!decisionRow) throw new Error("native_status_decision_not_persisted");
const updated = await tx.update(issues).set({
const materialized: MaterializedEffect[] = [];
for (const effect of input.decision.effects) {
materialized.push(await materializeDecisionEffect({
tx: tx as unknown as Db,
companyId: input.companyId,
issue,
runId: input.runId,
decisionId: decisionRow.id,
effect,
failpoint: input.failpoint,
}));
}
failAt("status_projection", input.failpoint);
const updated = await issueService(tx as unknown as Db).update(input.issueId, {
status: input.decision.toStatus,
unblockDescriptor: input.decision.unblockDescriptor,
statusVersion: input.priorStatusVersion + 1,
lastStatusDecisionId: decisionRow.id,
completedAt: input.decision.toStatus === "done" ? new Date() : null,
blockedTransitionAt: input.decision.toStatus === "blocked" ? new Date() : null,
updatedAt: new Date(),
}).where(and(
eq(issues.id, input.issueId),
eq(issues.statusVersion, input.priorStatusVersion),
)).returning().then((rows) => rows[0] ?? null);
unblockDescriptor: input.decision.unblockDescriptor,
actorAgentId: null,
actorUserId: null,
}, tx, publications);
if (!updated) throw new NativeStatusRaceError();
await tx.insert(statusDecisionEffects).values({
companyId: input.companyId,
issueId: input.issueId,
decisionId: decisionRow.id,
ordinal: 1,
materialized.unshift({
effectKind: "issue_status_projection",
targetType: "issue",
targetId: input.issueId,
idempotencyKey: `native-status:${decisionRow.id}:1`,
payload: {
fromStatus: issue.status,
toStatus: input.decision.toStatus,
reasonCode: input.decision.reasonCode,
},
deliveryState: "delivered",
attemptCount: 1,
deliveredAt: new Date(),
}).onConflictDoNothing();
await tx.update(statusDecisions).set({
applicationState: "applied",
appliedAt: new Date(),
}).where(eq(statusDecisions.id, decisionRow.id));
payload: { fromStatus: issue.status, toStatus: input.decision.toStatus, reasonCode: input.decision.reasonCode },
});
if (input.decision.toStatus === "done" && issue.status !== "done") {
const issueSvc = issueService(tx as unknown as Db);
const dependents = await issueSvc.listWakeableBlockedDependents(input.issueId);
for (const dependent of dependents) {
const idempotencyKey = buildIssueBlockersResolvedWakeIdempotencyKey({
dependentIssueId: dependent.id,
resolvedBlockerIssueId: input.issueId,
});
const wakeId = await enqueueWake({
tx: tx as unknown as Db,
companyId: input.companyId,
issueId: dependent.id,
agentId: dependent.assigneeAgentId,
reason: "issue_blockers_resolved",
idempotencyKey,
payload: { resolvedBlockerIssueId: input.issueId, blockerIssueIds: dependent.blockerIssueIds },
});
materialized.push({
effectKind: "dependency_wake",
targetType: "agent_wakeup_request",
targetId: wakeId,
payload: { dependentIssueId: dependent.id, resolvedBlockerIssueId: input.issueId },
});
}
if (issue.parentId) {
const parent = await issueSvc.getWakeableParentAfterChildCompletion(issue.parentId);
if (parent) {
const wakeId = await enqueueWake({
tx: tx as unknown as Db,
companyId: input.companyId,
issueId: parent.id,
agentId: parent.assigneeAgentId,
reason: "issue_children_completed",
idempotencyKey: `issue_children_completed:${parent.id}:${input.issueId}`,
payload: { completedChildIssueId: input.issueId, childIssueIds: parent.childIssueIds },
});
materialized.push({
effectKind: "parent_wake",
targetType: "agent_wakeup_request",
targetId: wakeId,
payload: { parentIssueId: parent.id, completedChildIssueId: input.issueId },
});
}
}
}
for (const [index, effect] of materialized.entries()) {
await tx.insert(statusDecisionEffects).values({
companyId: input.companyId,
issueId: input.issueId,
decisionId: decisionRow.id,
ordinal: index + 1,
effectKind: effect.effectKind,
targetType: effect.targetType,
targetId: effect.targetId,
idempotencyKey: `native-status:${decisionRow.id}:${index + 1}`,
payload: effect.payload,
deliveryState: "delivered",
attemptCount: 1,
deliveredAt: new Date(),
}).onConflictDoNothing();
}
await tx.update(statusDecisions).set({ applicationState: "applied", appliedAt: new Date() })
.where(eq(statusDecisions.id, decisionRow.id));
await tx.update(nativeRunFinalizations).set({
phase: "applied",
phase: "committed",
assessmentId: input.assessmentId,
decisionId: decisionRow.id,
leaseOwner: null,
leaseExpiresAt: null,
failureCode: null,
failureDetail: null,
nextAttemptAt: null,
updatedAt: new Date(),
}).where(eq(nativeRunFinalizations.runId, input.runId));
await tx.insert(activityLog).values({
const { publication } = await persistActivity(tx as unknown as Db, {
companyId: input.companyId,
actorType: "system",
actorId: "native-status-committer",
action: "issue.native_status_decision_applied",
action: "issue.updated",
entityType: "issue",
entityId: input.issueId,
issueId: input.issueId,
runId: input.runId,
details: {
source: "native_status_decision",
assessmentId: input.assessmentId,
decisionId: decisionRow.id,
fromStatus: issue.status,
toStatus: input.decision.toStatus,
reasonCode: input.decision.reasonCode,
effectCount: materialized.length,
},
});
return { decision: { ...decisionRow, applicationState: "applied" }, issue: updated };
publications.push(publication);
return {
decision: { ...decisionRow, applicationState: "applied" },
issue: updated,
replayed: false,
};
});
for (const publication of publications) publishActivity(publication);
return committed;
}
@@ -11,7 +11,9 @@ export async function recordNativeWorkAssessment(input: {
runId: string;
turnId: string | null;
contractId: string;
contractCanonicalSha256: string;
resultId: string;
resultCanonicalSha256: string;
priorIssueStatus: string;
priorStatusVersion: number;
priorDecisionId: string | null;
@@ -22,14 +24,21 @@ export async function recordNativeWorkAssessment(input: {
const assessmentJson = input.assessment as unknown as Record<string, unknown>;
const inputDigest = nativeSha256({
contractId: input.contractId,
contractCanonicalSha256: input.contractCanonicalSha256,
resultId: input.resultId,
resultCanonicalSha256: input.resultCanonicalSha256,
priorIssueStatus: input.priorIssueStatus,
priorStatusVersion: input.priorStatusVersion,
priorDecisionId: input.priorDecisionId,
triggerKind: "native_result",
triggerRef: input.resultId,
triggerActorCompanyId: input.companyId,
triggerCapability: "server_native_finalizer",
policyVersion: input.policyVersion,
assessment: assessmentJson,
});
const existing = await input.db.select().from(workAssessments).where(and(
eq(workAssessments.resultId, input.resultId),
eq(workAssessments.issueId, input.issueId),
eq(workAssessments.inputDigest, inputDigest),
)).limit(1).then((rows) => rows[0] ?? null);
if (existing) return existing;