mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-26 10:26:26 +01:00
fix(runner): make native finalization restart-safe
Persist bound session checkpoints, verify durable completion evidence, and commit status/liveness effects atomically. Preserve unique event cursors during migration and add bounded recovery plus legacy compatibility coverage. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
512c0647b0
commit
0ca23dd535
42 files changed
+2931
-386
No files matched your search
@@ -28,7 +28,7 @@ CREATE TABLE "native_run_finalizations" (
|
||||
"assessment_id" uuid,
|
||||
"decision_id" uuid,
|
||||
"failure_code" text,
|
||||
"failure_detail" text,
|
||||
"failure_detail" jsonb,
|
||||
"next_attempt_at" timestamp with time zone,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
@@ -76,14 +76,14 @@ CREATE TABLE "status_decisions" (
|
||||
"company_id" uuid NOT NULL,
|
||||
"issue_id" uuid NOT NULL,
|
||||
"assessment_id" uuid NOT NULL,
|
||||
"decision_version" integer NOT NULL,
|
||||
"decision_version" bigint NOT NULL,
|
||||
"policy_version" text NOT NULL,
|
||||
"from_status" text NOT NULL,
|
||||
"to_status" text NOT NULL,
|
||||
"reason_code" text NOT NULL,
|
||||
"decision_json" jsonb NOT NULL,
|
||||
"decision_digest" text NOT NULL,
|
||||
"application_state" text DEFAULT 'pending' NOT NULL,
|
||||
"application_state" text DEFAULT 'proposed' NOT NULL,
|
||||
"supersedes_decision_id" uuid,
|
||||
"applied_at" timestamp with time zone,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
@@ -133,16 +133,35 @@ ALTER TABLE "heartbeat_runs" ADD COLUMN "native_phase" text;--> statement-breakp
|
||||
ALTER TABLE "heartbeat_runs" ADD COLUMN "native_phase_updated_at" timestamp with time zone;--> statement-breakpoint
|
||||
ALTER TABLE "issues" ADD COLUMN "status_version" bigint DEFAULT 0 NOT NULL;--> statement-breakpoint
|
||||
ALTER TABLE "issues" ADD COLUMN "last_status_decision_id" uuid;--> statement-breakpoint
|
||||
-- Historical event writers computed max(seq) outside their insert transaction. Repair any
|
||||
-- collisions deterministically before enforcing the native runner's single-writer invariant.
|
||||
WITH resequenced AS (
|
||||
SELECT "id", row_number() OVER (PARTITION BY "run_id" ORDER BY "created_at", "id") AS "new_seq"
|
||||
-- Historical event writers computed max(seq) outside their insert transaction. Preserve every
|
||||
-- existing unique cursor and the first row at each duplicated cursor. Only later colliding rows
|
||||
-- move above the run's pre-migration maximum, in deterministic legacy order.
|
||||
WITH ranked AS (
|
||||
SELECT
|
||||
"id",
|
||||
"run_id",
|
||||
"seq",
|
||||
"created_at",
|
||||
row_number() OVER (
|
||||
PARTITION BY "run_id", "seq"
|
||||
ORDER BY "created_at", "id"
|
||||
) AS "duplicate_ordinal",
|
||||
max("seq") OVER (PARTITION BY "run_id") AS "old_run_max"
|
||||
FROM "heartbeat_run_events"
|
||||
), duplicate_repairs AS (
|
||||
SELECT
|
||||
"id",
|
||||
"old_run_max" + row_number() OVER (
|
||||
PARTITION BY "run_id"
|
||||
ORDER BY "seq", "created_at", "id"
|
||||
) AS "new_seq"
|
||||
FROM ranked
|
||||
WHERE "duplicate_ordinal" > 1
|
||||
)
|
||||
UPDATE "heartbeat_run_events" AS event
|
||||
SET "seq" = resequenced."new_seq"
|
||||
FROM resequenced
|
||||
WHERE event."id" = resequenced."id";--> statement-breakpoint
|
||||
SET "seq" = duplicate_repairs."new_seq"
|
||||
FROM duplicate_repairs
|
||||
WHERE event."id" = duplicate_repairs."id";--> statement-breakpoint
|
||||
UPDATE "heartbeat_runs" AS run
|
||||
SET "next_event_seq" = COALESCE((
|
||||
SELECT max(event."seq") + 1
|
||||
@@ -189,10 +208,11 @@ CREATE UNIQUE INDEX "native_run_results_run_fingerprint_uq" ON "native_run_resul
|
||||
CREATE UNIQUE INDEX "native_run_results_run_caller_result_uq" ON "native_run_results" USING btree ("run_id","caller_result_id");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "native_run_results_run_caller_dedupe_uq" ON "native_run_results" USING btree ("run_id","caller_dedupe_key");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "status_decision_effects_decision_ordinal_uq" ON "status_decision_effects" USING btree ("decision_id","ordinal");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "status_decision_effects_idempotency_uq" ON "status_decision_effects" USING btree ("idempotency_key");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "status_decisions_assessment_version_uq" ON "status_decisions" USING btree ("assessment_id","decision_version");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "status_decisions_issue_digest_uq" ON "status_decisions" USING btree ("issue_id","decision_digest");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "work_assessments_result_input_uq" ON "work_assessments" USING btree ("result_id","input_digest");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "status_decision_effects_company_idempotency_uq" ON "status_decision_effects" USING btree ("company_id","idempotency_key");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "status_decisions_company_issue_version_uq" ON "status_decisions" USING btree ("company_id","issue_id","decision_version");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "status_decisions_company_assessment_uq" ON "status_decisions" USING btree ("company_id","assessment_id");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "status_decisions_company_issue_digest_uq" ON "status_decisions" USING btree ("company_id","issue_id","decision_digest");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "work_assessments_company_issue_input_uq" ON "work_assessments" USING btree ("company_id","issue_id","input_digest");--> statement-breakpoint
|
||||
-- paperclip:migration-safety-ignore large-create-index-not-concurrently: duplicate rows are deterministically resequenced above before this invariant is enabled in the same transaction.
|
||||
CREATE UNIQUE INDEX "heartbeat_run_events_run_seq_uq" ON "heartbeat_run_events" USING btree ("run_id","seq");--> statement-breakpoint
|
||||
-- paperclip:migration-safety-ignore large-create-index-not-concurrently: nullable native source IDs mean historical rows need no backfill and the invariant must commit atomically with the new columns.
|
||||
|
||||
@@ -24318,7 +24318,7 @@
|
||||
},
|
||||
"failure_detail": {
|
||||
"name": "failure_detail",
|
||||
"type": "text",
|
||||
"type": "jsonb",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
@@ -32244,9 +32244,15 @@
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"status_decision_effects_idempotency_uq": {
|
||||
"name": "status_decision_effects_idempotency_uq",
|
||||
"status_decision_effects_company_idempotency_uq": {
|
||||
"name": "status_decision_effects_company_idempotency_uq",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "company_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "idempotency_key",
|
||||
"isExpression": false,
|
||||
@@ -32338,7 +32344,7 @@
|
||||
},
|
||||
"decision_version": {
|
||||
"name": "decision_version",
|
||||
"type": "integer",
|
||||
"type": "bigint",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
@@ -32383,7 +32389,7 @@
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "'pending'"
|
||||
"default": "'proposed'"
|
||||
},
|
||||
"supersedes_decision_id": {
|
||||
"name": "supersedes_decision_id",
|
||||
@@ -32406,11 +32412,17 @@
|
||||
}
|
||||
},
|
||||
"indexes": {
|
||||
"status_decisions_assessment_version_uq": {
|
||||
"name": "status_decisions_assessment_version_uq",
|
||||
"status_decisions_company_issue_version_uq": {
|
||||
"name": "status_decisions_company_issue_version_uq",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "assessment_id",
|
||||
"expression": "company_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "issue_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
@@ -32427,9 +32439,36 @@
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"status_decisions_issue_digest_uq": {
|
||||
"name": "status_decisions_issue_digest_uq",
|
||||
"status_decisions_company_assessment_uq": {
|
||||
"name": "status_decisions_company_assessment_uq",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "company_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "assessment_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": true,
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"status_decisions_company_issue_digest_uq": {
|
||||
"name": "status_decisions_company_issue_digest_uq",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "company_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "issue_id",
|
||||
"isExpression": false,
|
||||
@@ -39258,11 +39297,17 @@
|
||||
}
|
||||
},
|
||||
"indexes": {
|
||||
"work_assessments_result_input_uq": {
|
||||
"name": "work_assessments_result_input_uq",
|
||||
"work_assessments_company_issue_input_uq": {
|
||||
"name": "work_assessments_company_issue_input_uq",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "result_id",
|
||||
"expression": "company_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "issue_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
@@ -40060,4 +40105,4 @@
|
||||
"schemas": {},
|
||||
"tables": {}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
import { pgTable, uuid, text, timestamp, integer } from "drizzle-orm/pg-core";
|
||||
import { pgTable, uuid, text, timestamp, integer, jsonb } from "drizzle-orm/pg-core";
|
||||
import { companies } from "./companies.js";
|
||||
import { issues } from "./issues.js";
|
||||
import { heartbeatRuns } from "./heartbeat_runs.js";
|
||||
@@ -16,7 +16,7 @@ export const nativeRunFinalizations = pgTable("native_run_finalizations", {
|
||||
assessmentId: uuid("assessment_id"),
|
||||
decisionId: uuid("decision_id"),
|
||||
failureCode: text("failure_code"),
|
||||
failureDetail: text("failure_detail"),
|
||||
failureDetail: jsonb("failure_detail").$type<Record<string, unknown>>(),
|
||||
nextAttemptAt: timestamp("next_attempt_at", { withTimezone: true }),
|
||||
createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
|
||||
updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(),
|
||||
|
||||
@@ -23,5 +23,6 @@ export const statusDecisionEffects = pgTable("status_decision_effects", {
|
||||
updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(),
|
||||
}, (table) => ({
|
||||
decisionOrdinalUq: uniqueIndex("status_decision_effects_decision_ordinal_uq").on(table.decisionId, table.ordinal),
|
||||
idempotencyUq: uniqueIndex("status_decision_effects_idempotency_uq").on(table.idempotencyKey),
|
||||
idempotencyUq: uniqueIndex("status_decision_effects_company_idempotency_uq")
|
||||
.on(table.companyId, table.idempotencyKey),
|
||||
}));
|
||||
@@ -1,4 +1,4 @@
|
||||
import { pgTable, uuid, text, timestamp, integer, jsonb, uniqueIndex } from "drizzle-orm/pg-core";
|
||||
import { pgTable, uuid, text, timestamp, bigint, jsonb, uniqueIndex } from "drizzle-orm/pg-core";
|
||||
import { companies } from "./companies.js";
|
||||
import { issues } from "./issues.js";
|
||||
import { workAssessments } from "./work_assessments.js";
|
||||
@@ -8,18 +8,21 @@ export const statusDecisions = pgTable("status_decisions", {
|
||||
companyId: uuid("company_id").notNull().references(() => companies.id),
|
||||
issueId: uuid("issue_id").notNull().references(() => issues.id),
|
||||
assessmentId: uuid("assessment_id").notNull().references(() => workAssessments.id),
|
||||
decisionVersion: integer("decision_version").notNull(),
|
||||
decisionVersion: bigint("decision_version", { mode: "number" }).notNull(),
|
||||
policyVersion: text("policy_version").notNull(),
|
||||
fromStatus: text("from_status").notNull(),
|
||||
toStatus: text("to_status").notNull(),
|
||||
reasonCode: text("reason_code").notNull(),
|
||||
decisionJson: jsonb("decision_json").$type<Record<string, unknown>>().notNull(),
|
||||
decisionDigest: text("decision_digest").notNull(),
|
||||
applicationState: text("application_state").notNull().default("pending"),
|
||||
applicationState: text("application_state").notNull().default("proposed"),
|
||||
supersedesDecisionId: uuid("supersedes_decision_id"),
|
||||
appliedAt: timestamp("applied_at", { withTimezone: true }),
|
||||
createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
|
||||
}, (table) => ({
|
||||
assessmentVersionUq: uniqueIndex("status_decisions_assessment_version_uq").on(table.assessmentId, table.decisionVersion),
|
||||
issueDigestUq: uniqueIndex("status_decisions_issue_digest_uq").on(table.issueId, table.decisionDigest),
|
||||
issueVersionUq: uniqueIndex("status_decisions_company_issue_version_uq")
|
||||
.on(table.companyId, table.issueId, table.decisionVersion),
|
||||
assessmentUq: uniqueIndex("status_decisions_company_assessment_uq").on(table.companyId, table.assessmentId),
|
||||
issueDigestUq: uniqueIndex("status_decisions_company_issue_digest_uq")
|
||||
.on(table.companyId, table.issueId, table.decisionDigest),
|
||||
}));
|
||||
@@ -26,5 +26,6 @@ export const workAssessments = pgTable("work_assessments", {
|
||||
supersedesAssessmentId: uuid("supersedes_assessment_id"),
|
||||
createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
|
||||
}, (table) => ({
|
||||
resultInputUq: uniqueIndex("work_assessments_result_input_uq").on(table.resultId, table.inputDigest),
|
||||
issueInputUq: uniqueIndex("work_assessments_company_issue_input_uq")
|
||||
.on(table.companyId, table.issueId, table.inputDigest),
|
||||
}));
|
||||
@@ -695,6 +695,29 @@ allowlist amendment and CTO review. In particular, Phase 6 must not change a
|
||||
concrete legacy adapter, approval authority, browser UI, workspace policy,
|
||||
budget policy, or activity semantics merely to make the tracer pass.
|
||||
|
||||
### Remediation allowlist amendment (2026-08-09)
|
||||
|
||||
The implementation review found four seam-local paths whose names differed
|
||||
from the provisional list above. They are added without expanding authority:
|
||||
|
||||
```text
|
||||
packages/paperclip-runner/src/native-session-runtime.ts
|
||||
packages/paperclip-runner/src/native-session-runtime.test.ts
|
||||
server/src/services/native-runtime/native-session-executor.ts
|
||||
server/src/services/native-runtime/native-session-executor.test.ts
|
||||
server/src/services/native-runtime/paperclip-control-plane-port.test.ts
|
||||
server/src/services/native-runtime/status-arbiter.test.ts
|
||||
packages/paperclip-runner/docs/design/phase-6-thin-paperclip-adapter.md
|
||||
```
|
||||
|
||||
The package runtime files own persisted provider-session recovery. The server
|
||||
executor files own only the run-scoped cancellation handle and coordinator
|
||||
lease around that package runtime. The colocated port and arbiter tests exercise
|
||||
the same approved database and pure-policy seams exposed through the named
|
||||
`server/src/__tests__` matrix entry points. This amendment does not authorize
|
||||
changes to a concrete legacy adapter, approval authority, UI, workspace policy,
|
||||
budget policy, or runner/provider/session behavior outside the package.
|
||||
|
||||
## Commands the implementation must make runnable
|
||||
|
||||
These commands are the acceptance contract for the implementation issue. They
|
||||
|
||||
@@ -32,6 +32,8 @@ contract are persisted before provider execution.
|
||||
`PaperclipControlPlanePort` implements the package's `ControlPlanePort`:
|
||||
|
||||
- `openRun` verifies company/run/issue/agent/contract binding;
|
||||
- `loadSessionCheckpoint` and `checkpointSession` bind restart state to the
|
||||
same company/run/issue/agent/session identity;
|
||||
- `appendEvent` validates PRP, commits through the shared per-run allocator,
|
||||
deduplicates source identity by canonical digest, and acknowledges afterward;
|
||||
- `replayEvents` uses an exclusive source cursor;
|
||||
@@ -51,21 +53,41 @@ versions, native run metadata, and native source identity on heartbeat events.
|
||||
All event writers allocate from the row-locked `heartbeat_runs.next_event_seq`.
|
||||
|
||||
Finalization runs only after result persistence and workspace finalization. The
|
||||
pure arbiter marks `done` only when the terminal state succeeded, the objective
|
||||
and every criterion are satisfied, verification passed, and no blocking work
|
||||
remains. It marks `blocked` only for a first-class reported blocker. A failed
|
||||
workspace barrier preserves the immutable native result, fails the run, and
|
||||
leaves issue status unchanged. Review/yield/incomplete/failure/cancellation
|
||||
remain non-terminal.
|
||||
Status projection is a versioned CAS and emits an audit record.
|
||||
server verifies cited evidence against company- and issue-scoped durable
|
||||
events, work products, approvals, interactions, or attachments; model claims
|
||||
alone cannot complete an issue. The pure arbiter marks `done` only when the
|
||||
terminal state succeeded, every criterion and verification has accepted
|
||||
durable evidence, and no blocking work remains. It marks `blocked` only for a
|
||||
task-wide first-class blocker with a named owner and action. Review, retry,
|
||||
continuation, blocker, and recovery effects materialize atomically with the
|
||||
status-version CAS and audit record.
|
||||
|
||||
## Recovery and safety
|
||||
|
||||
The finalization reconciler selects persisted `runtime_mode=native` rows; it
|
||||
does not consult the current flag. Cancellation uses a run-scoped normalized
|
||||
session handle before the existing process cleanup. Native execution does not
|
||||
construct a Paperclip JWT, managed MCP access, legacy context, or raw adapter
|
||||
environment.
|
||||
does not consult the current flag. Per-attempt leases prevent concurrent
|
||||
session/finalization ownership, while retryable failures retain the immutable
|
||||
result and a first-class recovery action. A recovered session replays its
|
||||
checkpoint and appends only missing control-plane facts instead of opening a
|
||||
second provider session. Cancellation uses a run-scoped normalized session
|
||||
handle before the existing process cleanup. Native execution does not construct
|
||||
a Paperclip JWT, managed MCP access, legacy context, or raw adapter environment.
|
||||
|
||||
## Verification status
|
||||
|
||||
The deterministic package suite and the real embedded-PostgreSQL Paperclip
|
||||
suite prove selection, replay/conflict handling, authoritative evidence,
|
||||
status/liveness atomicity, session/finalization recovery, migration repair,
|
||||
bounded cancellation/retry, and a byte-equivalent legacy read snapshot. The
|
||||
database suite is the internal native canary: it executes one selected task
|
||||
through the public package session contract and applies one server-owned
|
||||
decision. Its post-kill-switch legacy case has zero native history rows.
|
||||
|
||||
No new live provider task was dispatched during the remediation review. A live
|
||||
Codex canary remains an operator-run rollout check because it requires changing
|
||||
the instance flag and one agent profile. The package `trace:phase6` command is a
|
||||
mock contract tracer; real Paperclip proof is the database-backed server matrix
|
||||
and the explicitly gated procedure in the tutorial.
|
||||
|
||||
See the [runnable tutorial](tutorials/phase-06-thin-paperclip-adapter.md), the
|
||||
[verification record](../knowledge/evidence/2026-08-09-phase-06-verification.md),
|
||||
|
||||
@@ -29,29 +29,45 @@ PHASE6_SCRATCH="${PAPERCLIP_RUN_SCRATCH_DIR:-$(mktemp -d)}"
|
||||
pnpm --filter @paperclipai/paperclip-runner exec vitest run \
|
||||
src/conformance/control-plane-port.test.ts \
|
||||
src/backends/harness-driver-backend.test.ts \
|
||||
src/contracts/native-execution.test.ts
|
||||
src/contracts/native-execution.test.ts \
|
||||
src/native-session-runtime.test.ts
|
||||
|
||||
pnpm --filter @paperclipai/paperclip-runner trace:phase6 -- \
|
||||
--target mock --scenario happy-path
|
||||
```
|
||||
|
||||
Expected: five targeted tests pass and the trace reports `resolvedMode` as
|
||||
Expected: six targeted tests pass and the trace reports `resolvedMode` as
|
||||
`native`, three replay-stable events, a successful workspace barrier, and one
|
||||
server-owned `done` decision.
|
||||
mock server-owned `done` decision. This tracer does not connect to Paperclip.
|
||||
|
||||
## 2. Prove the database-backed Paperclip port
|
||||
|
||||
```sh
|
||||
pnpm --filter @paperclipai/paperclip-runner build:typescript
|
||||
pnpm --filter @paperclipai/server exec vitest run \
|
||||
src/services/native-runtime/paperclip-control-plane-port.test.ts \
|
||||
src/services/native-runtime/runtime-mode.test.ts \
|
||||
src/services/native-runtime/status-arbiter.test.ts
|
||||
src/__tests__/native-runner-phase6.integration.test.ts \
|
||||
src/__tests__/heartbeat-native-runner-selection.test.ts \
|
||||
src/__tests__/heartbeat-native-runner-cancellation.test.ts \
|
||||
src/__tests__/heartbeat-run-event-sequencing.test.ts \
|
||||
src/__tests__/native-runner-input-boundary.test.ts \
|
||||
src/__tests__/native-run-finalizer.test.ts \
|
||||
src/__tests__/native-status-arbiter-corpus.test.ts \
|
||||
src/__tests__/native-finalization-recovery.test.ts \
|
||||
src/__tests__/native-finalization-migration.test.ts \
|
||||
src/__tests__/legacy-finalization-regression.test.ts
|
||||
```
|
||||
|
||||
This runs the unchanged package conformance suite against the real Paperclip
|
||||
port. It also checks company binding, duplicate/gap replay, immutable result
|
||||
ingestion, workspace-gated finalization, status-version CAS, and audit output.
|
||||
port. Twenty-five targeted tests check company binding, duplicate/gap replay,
|
||||
checkpoint recovery, immutable result ingestion, durable-evidence authority,
|
||||
workspace-gated finalization, atomic liveness rollback, status-version CAS,
|
||||
migration repair, legacy byte equivalence, and audit output. All database tests
|
||||
start embedded PostgreSQL and do not skip when a developer database URL is
|
||||
absent.
|
||||
|
||||
The command above is the repeatable internal canary and post-kill-switch legacy
|
||||
proof. The remaining sections are an optional live-provider rollout procedure;
|
||||
they were not run during remediation and require explicit operator authority to
|
||||
change an instance flag and agent profile.
|
||||
|
||||
## 3. Enable one isolated agent
|
||||
|
||||
|
||||
@@ -13,39 +13,44 @@ This record verifies the default-off Phase 6 tracer at the approved
|
||||
`ControlPlanePort`/`NativeSessionBackend` boundary. No browser surface changed,
|
||||
so Phase 6 has command/database evidence rather than screenshots.
|
||||
|
||||
The counts below are the PAP-16875 remediation rerun. “Internal canary” means
|
||||
the selected-task database test through the public package session contract;
|
||||
it does not mean that a new live Codex provider task was dispatched.
|
||||
|
||||
# Package contract and mock tracer
|
||||
|
||||
```sh
|
||||
pnpm --filter @paperclipai/paperclip-runner exec vitest run \
|
||||
src/conformance/control-plane-port.test.ts \
|
||||
src/backends/harness-driver-backend.test.ts \
|
||||
src/contracts/native-execution.test.ts
|
||||
src/contracts/native-execution.test.ts \
|
||||
src/native-session-runtime.test.ts
|
||||
|
||||
pnpm --filter @paperclipai/paperclip-runner trace:phase6 -- \
|
||||
--target mock --scenario happy-path
|
||||
```
|
||||
|
||||
Result: three files and five tests passed. The stable trace reported native
|
||||
Result: four files and six tests passed. The stable trace reported native
|
||||
mode, `runStatus=succeeded`, three events, contiguous source sequence 3,
|
||||
`workspaceFinalizeStatus=succeeded`, zero legacy invocations, and a server-owned
|
||||
`done` decision with status version 0 -> 1.
|
||||
`done` decision with status version 0 -> 1. Restart recovery reused the bound
|
||||
provider session checkpoint, opened no second session or turn, retained the
|
||||
existing accepted result event, and appended only the missing terminal fact.
|
||||
|
||||
# Real Paperclip adapter and public-session task
|
||||
|
||||
```sh
|
||||
pnpm --filter @paperclipai/paperclip-runner build:typescript
|
||||
pnpm --filter @paperclipai/server exec vitest run \
|
||||
src/services/native-runtime/paperclip-control-plane-port.test.ts \
|
||||
src/services/native-runtime/runtime-mode.test.ts \
|
||||
src/services/native-runtime/status-arbiter.test.ts
|
||||
```
|
||||
|
||||
Result: three files and ten tests passed against embedded PostgreSQL. The same
|
||||
package conformance suite passed unchanged against `PaperclipControlPlanePort`.
|
||||
The complete acceptance gate below passed 25 tests in ten files against
|
||||
embedded PostgreSQL with zero skipped tests. Database-backed tests start their
|
||||
own database instead of skipping when a developer `DATABASE_URL` is absent.
|
||||
The same package conformance suite passed unchanged against
|
||||
`PaperclipControlPlanePort`.
|
||||
A selected task then ran end-to-end through `executeNativeSession`, the public
|
||||
backend/port contracts, durable PRP events, immutable result ingestion,
|
||||
workspace finalization, assessment, CAS status decision, and audit persistence.
|
||||
The issue moved from `in_progress` version 0 to `done` version 1.
|
||||
The issue moved from `in_progress` version 0 to `done` version 1 only after its
|
||||
criterion and verification cited an approved, issue-scoped durable work
|
||||
product. A model-only result remained non-terminal and received a persisted
|
||||
continuation wake.
|
||||
|
||||
The focused corpus also proved:
|
||||
|
||||
@@ -55,12 +60,37 @@ The focused corpus also proved:
|
||||
- company/run/agent/issue/source bindings reject mismatches;
|
||||
- source gaps remain visible, identical retries deduplicate, and replay is
|
||||
exclusive and byte-stable;
|
||||
- incomplete/review/yield/cancelled results remain non-terminal;
|
||||
- only a first-class reported blocker selects `blocked`;
|
||||
- incomplete/review/yield/cancelled results receive a durable review,
|
||||
continuation, or recovery path;
|
||||
- only a task-wide blocker with a named owner and action selects `blocked`;
|
||||
- a completion decision writes immutable assessment/decision/effect records and
|
||||
an activity-log audit row.
|
||||
an activity-log audit row;
|
||||
- a failed workspace barrier preserves the accepted semantic result, leaves the
|
||||
issue status unchanged, and records a recoverable `workspace_failed` phase.
|
||||
issue status unchanged, and records a leased `retryable_failure` plus recovery
|
||||
owner/action;
|
||||
- four injected failures prove interactions, wakes, blocker bindings, status,
|
||||
decisions, and effects roll back together before retry ownership is recorded.
|
||||
|
||||
The acceptance-matrix entry points also passed as one 10-file, 25-test gate:
|
||||
|
||||
```sh
|
||||
pnpm --filter @paperclipai/server exec vitest run \
|
||||
src/__tests__/heartbeat-native-runner-cancellation.test.ts \
|
||||
src/__tests__/heartbeat-native-runner-selection.test.ts \
|
||||
src/__tests__/native-run-finalizer.test.ts \
|
||||
src/__tests__/native-runner-input-boundary.test.ts \
|
||||
src/__tests__/native-runner-phase6.integration.test.ts \
|
||||
src/__tests__/native-finalization-recovery.test.ts \
|
||||
src/__tests__/heartbeat-run-event-sequencing.test.ts \
|
||||
src/__tests__/legacy-finalization-regression.test.ts \
|
||||
src/__tests__/native-finalization-migration.test.ts \
|
||||
src/__tests__/native-status-arbiter-corpus.test.ts
|
||||
```
|
||||
|
||||
This gate proves concurrent event allocation, duplicate-only migration repair,
|
||||
bounded retry exhaustion, kill-switch legacy byte compatibility, and complete
|
||||
Section 18.13 fixture-to-consumer traceability in addition to the direct suites.
|
||||
It is the repeatable internal native canary and post-kill-switch legacy proof.
|
||||
|
||||
# Compile and migration checks
|
||||
|
||||
@@ -72,22 +102,29 @@ pnpm --filter @paperclipai/db typecheck
|
||||
```
|
||||
|
||||
Result: all checks passed. Migration numbering and safety passed. Migration
|
||||
0211 deterministically resequences historical heartbeat events before adding
|
||||
the unique run sequence invariant, backfills each run's next allocator value,
|
||||
and installs issue status-version tracking.
|
||||
0211 preserves existing unique heartbeat cursors, deterministically moves only
|
||||
duplicate rows above each run's former maximum, backfills the next allocator
|
||||
value, and installs issue status-version tracking.
|
||||
|
||||
Focused legacy regression:
|
||||
Focused migration, allocator, recovery, and legacy rehearsal:
|
||||
|
||||
```sh
|
||||
pnpm --filter @paperclipai/server exec vitest run \
|
||||
src/__tests__/heartbeat-workspace-finalize-branch.test.ts \
|
||||
src/__tests__/heartbeat-process-recovery.test.ts \
|
||||
src/__tests__/heartbeat-runtime-state.test.ts
|
||||
src/services/native-runtime/paperclip-control-plane-port.test.ts \
|
||||
src/__tests__/native-finalization-recovery.test.ts \
|
||||
src/__tests__/legacy-finalization-regression.test.ts \
|
||||
src/__tests__/native-finalization-migration.test.ts \
|
||||
src/__tests__/heartbeat-run-event-sequencing.test.ts
|
||||
```
|
||||
|
||||
Result: three files and 106 tests passed, covering unchanged adapter execution,
|
||||
workspace finalization, cancellation, process recovery, and runtime/session
|
||||
state behavior with the flag left at its default.
|
||||
Result: five files and 11 tests passed with zero skips. The production-shaped
|
||||
0211 rehearsal began with legacy event sequences `[1,5,5,9]`, retained the
|
||||
original unique cursors and first `5`, moved only the later duplicate to `10`,
|
||||
seeded `next_event_seq=11`, preserved every non-sequence field byte-for-byte,
|
||||
and enforced uniqueness. Thirty-two concurrent mixed event writers produced a
|
||||
gap-free allocator stream. Invalid native finalization exhausted its
|
||||
three-attempt budget into named recovery without touching issue status. The
|
||||
legacy snapshot remained byte-equivalent with zero native history rows.
|
||||
|
||||
# Credential, governance, budget, and legacy boundaries
|
||||
|
||||
@@ -113,14 +150,16 @@ pnpm --filter @paperclipai/server exec vitest run \
|
||||
-t "daily cost cap"
|
||||
```
|
||||
|
||||
Result: native cancellation reached the active normalized session and removed
|
||||
the handle afterward; the daily cost hard stop cancelled a queued run before
|
||||
any adapter execution.
|
||||
Result: native cancellation reached the active normalized session exactly once
|
||||
across duplicate cancellation requests and removed the handle afterward. The
|
||||
daily cost hard stop cancelled a queued run before any adapter execution.
|
||||
|
||||
# Remaining operational checkpoint
|
||||
# Deferred live-provider checkpoint
|
||||
|
||||
The [Phase 6 tutorial](../../docs/tutorials/phase-06-thin-paperclip-adapter.md)
|
||||
contains the exact board-authorized commands for enabling one local agent,
|
||||
running and inspecting a live Codex task, disabling the flag, proving a fresh
|
||||
legacy selection, and restoring the agent profile. Those commands intentionally
|
||||
do not pass credentials to the package or model.
|
||||
do not pass credentials to the package or model. They were not run during this
|
||||
remediation because changing live instance and agent rollout state requires
|
||||
operator approval; no new live-provider evidence is claimed here.
|
||||
@@ -12,5 +12,4 @@
|
||||
* [Phase 4b live console verification](2026-08-08-phase-04b-live-console-verification.md) - Demo-driver, transcript-model, browser, keyboard, boundary, token, and screenshot evidence for the live console.
|
||||
* [Phase 4b QA verification](2026-08-08-phase-04b-qa-verification.md) - Independent QA pass: clean-start tutorial in a real browser, deterministic demos, real Codex end-to-end session, full `verify` acceptance (exit 0), and screenshots (PAP-16837).
|
||||
* [Phase 5 browser SDK verification](2026-08-08-phase-05-verification.md) - Targeted and package-acceptance evidence for the versioned SDK, both public consumers, fake/real drivers, accessibility, reconnect, replay, and screenshots.
|
||||
* [Phase 6 thin Paperclip adapter verification](2026-08-09-phase-06-verification.md) - Mock and database-backed port conformance, public-session task completion, durable replay/finalization, feature selection, security boundary, and legacy fallback evidence.
|
||||
* [Phase 6 thin Paperclip adapter verification](2026-08-09-phase-06-verification.md) - Mock/real conformance, public session completion, canonical persistence, governance authority, migration, kill-switch, and legacy compatibility evidence.
|
||||
* [Phase 6 thin Paperclip adapter verification](2026-08-09-phase-06-verification.md) - Mock and database-backed conformance, authoritative completion, restart recovery, migration repair, atomic liveness, internal canary, and deterministic post-kill-switch legacy evidence.
|
||||
@@ -34,16 +34,27 @@ existing server authority and keep legacy execution as the default.
|
||||
handle; never fall back to legacy inside a selected native run.
|
||||
8. Keep the feature default false. Recovery reads persisted run/coordinator
|
||||
state and therefore remains correct after the flag is disabled.
|
||||
9. Persist provider session checkpoints and recover them fail-closed; a restart
|
||||
may reconcile missing control facts but may not open a second provider
|
||||
session for the same run.
|
||||
10. Treat model citations as claims. Only server-verifiable durable records can
|
||||
satisfy completion criteria, and every non-terminal decision must create or
|
||||
bind a real review, wake, blocker, or recovery path in the status transaction.
|
||||
|
||||
# Evidence
|
||||
|
||||
- [Phase 6 verification](../evidence/2026-08-09-phase-06-verification.md)
|
||||
- [Phase 6 reference](../../docs/phase-06-thin-paperclip-adapter.md)
|
||||
- [Runnable tutorial](../../docs/tutorials/phase-06-thin-paperclip-adapter.md)
|
||||
- Package conformance: three files, five tests passed.
|
||||
- Database-backed port/session/finalizer corpus: three files, ten tests passed.
|
||||
- Legacy heartbeat finalization/recovery/runtime corpus: three files, 106 tests
|
||||
passed.
|
||||
- Package conformance and recovery: four files, six tests passed.
|
||||
- Phase 6 acceptance-matrix entry points: ten files, twenty-five tests passed
|
||||
with zero skips, including the database-backed selected-task canary, four
|
||||
atomic-liveness failpoints, migration, sequencing, bounded recovery, and
|
||||
legacy compatibility.
|
||||
- Focused database rehearsal: five files, eleven tests passed with zero skips;
|
||||
the legacy event/read snapshot was byte-equivalent and had zero native rows.
|
||||
- Section 18.13 source corpus: 52 fixtures, all 70 matrix rows, and 18 structural
|
||||
checks passed with every fixture bound to a named consumer.
|
||||
- Runner, shared, server, and database typechecks passed; migration safety
|
||||
passed.
|
||||
|
||||
@@ -69,6 +80,10 @@ existing server authority and keep legacy execution as the default.
|
||||
- Live operator proof depends on a board-authorized local instance and an
|
||||
already authenticated Codex installation; deterministic CI uses the same
|
||||
public session contract with a scripted backend.
|
||||
- The remediation did not dispatch a new live provider task or mutate rollout
|
||||
settings. The verified “internal canary” and post-kill-switch trace are the
|
||||
real embedded-PostgreSQL selected-task and legacy-snapshot cases; the live
|
||||
tutorial remains an explicit operator checkpoint.
|
||||
|
||||
# Follow-up questions
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@
|
||||
{
|
||||
"id": "incomplete-evidence-no-live-path",
|
||||
"mode": "native",
|
||||
"covers": { "decisionRows": ["SD-03"], "terminalRows": [], "attentionRows": [], "livenessRows": ["LIVE-06"], "reconciliationRows": [], "compatibilityRows": [], "migrationRows": [] },
|
||||
"covers": { "decisionRows": [], "terminalRows": [], "attentionRows": [], "livenessRows": [], "reconciliationRows": [], "compatibilityRows": [], "migrationRows": [] },
|
||||
"tags": ["incomplete_evidence", "atomic_liveness"],
|
||||
"given": { "priorIssueStatus": "in_progress", "turnTerminalState": "completed", "runTerminalState": "succeeded", "reportedWorkDisposition": "done", "nativeFinalization": "present", "completionState": "missing_required_test", "trigger": "runner_finalizer", "fault": "continuation_insert_failure" },
|
||||
"expected": { "runStatus": "failed", "statusAction": "preserve", "reasonCode": "side_effect_planning_failed", "requiredEffects": ["record_finalization_error"], "forbiddenEffects": ["enqueue_continuation", "status_version_increment"], "livePathKind": null, "preserveClaim": true, "nativeRecords": true, "decisionCount": 0, "maxWakeCount": 0, "maxNotificationCount": 0 },
|
||||
@@ -42,7 +42,7 @@
|
||||
{
|
||||
"id": "prior-status-preserved-without-live-path",
|
||||
"mode": "native",
|
||||
"covers": { "decisionRows": [], "terminalRows": [], "attentionRows": [], "livenessRows": [], "reconciliationRows": [], "compatibilityRows": [], "migrationRows": [] },
|
||||
"covers": { "decisionRows": ["SD-03"], "terminalRows": [], "attentionRows": [], "livenessRows": ["LIVE-06"], "reconciliationRows": [], "compatibilityRows": [], "migrationRows": [] },
|
||||
"tags": ["atomic_liveness", "reconciliation"],
|
||||
"given": { "priorIssueStatus": "in_progress", "turnTerminalState": "completed", "runTerminalState": "succeeded", "reportedWorkDisposition": "yielded", "nativeFinalization": "present", "completionState": "no_durable_continuation", "trigger": "runner_finalizer" },
|
||||
"expected": { "runStatus": "failed", "statusAction": "preserve", "reasonCode": "prior_status_preserved_no_live_path", "requiredEffects": ["record_finalization_error"], "forbiddenEffects": ["set_issue_status", "enqueue_continuation", "bind_blocker", "bind_reviewer"], "livePathKind": "recovery", "preserveClaim": true, "nativeRecords": true, "decisionCount": 1, "maxWakeCount": 1, "maxNotificationCount": 0 },
|
||||
|
||||
@@ -56,6 +56,7 @@ export class HarnessDriverBackend implements NativeSessionBackend {
|
||||
providerSessionId: snapshot.providerSessionId,
|
||||
runId: snapshot.identity.runId,
|
||||
normalizedSessionId: snapshot.identity.sessionId,
|
||||
activeTurnId: snapshot.activeTurnId ?? snapshot.terminalTurns?.at(-1)?.turnId ?? null,
|
||||
lastSourceSequence: parseCursor(snapshot.cursor),
|
||||
...(snapshot.semanticResult === undefined || snapshot.semanticResult === null
|
||||
? {}
|
||||
@@ -63,9 +64,12 @@ export class HarnessDriverBackend implements NativeSessionBackend {
|
||||
semanticResult: {
|
||||
result: snapshot.semanticResult,
|
||||
fingerprint: canonicalJson(snapshot.semanticResult),
|
||||
turnId: "recovered",
|
||||
},
|
||||
}),
|
||||
turnId: snapshot.activeTurnId ?? snapshot.terminalTurns?.at(-1)?.turnId ?? "recovered",
|
||||
},
|
||||
}),
|
||||
terminalTurns: snapshot.terminalTurns ?? [],
|
||||
pendingRuntimeRequests: snapshot.pendingRuntimeRequests ?? [],
|
||||
lineage: snapshot.lineage ?? [],
|
||||
};
|
||||
const recovered = await this.#driver.recoverSession(persisted);
|
||||
if (!recovered.recovered || recovered.session === undefined) {
|
||||
@@ -181,6 +185,10 @@ class HarnessNativeSession implements NativeSession {
|
||||
: String(snapshot.lastSourceSequence),
|
||||
semanticResult: snapshot.semanticResult?.result ?? null,
|
||||
terminal: this.#terminal,
|
||||
activeTurnId: snapshot.activeTurnId ?? snapshot.semanticResult?.turnId ?? null,
|
||||
terminalTurns: snapshot.terminalTurns ?? [],
|
||||
pendingRuntimeRequests: snapshot.pendingRuntimeRequests ?? [],
|
||||
lineage: snapshot.lineage ?? [],
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -41,7 +41,7 @@ export const CONTROL_PLANE_CONFORMANCE_RESULT: PrpStructuredRunResult = {
|
||||
remainingWork: [],
|
||||
},
|
||||
evidence: [{ kind: "event", ref: "event:2" }],
|
||||
verification: [{ commandOrCheck: "phase6-conformance", status: "passed" }],
|
||||
verification: [{ commandOrCheck: "phase6-conformance", status: "passed", artifactRef: "event:2" }],
|
||||
attentionRequests: [],
|
||||
artifacts: [],
|
||||
};
|
||||
|
||||
@@ -8,6 +8,7 @@ import type {
|
||||
PrpStructuredRunResult,
|
||||
PrpTerminalState,
|
||||
} from "../protocol/phase1-contract.js";
|
||||
import type { PersistedNativeSession } from "./native-session-backend.js";
|
||||
|
||||
export interface OpenControlPlaneRunInput {
|
||||
identity: NativeRunIdentity;
|
||||
@@ -49,6 +50,8 @@ export interface CompleteControlPlaneRunInput {
|
||||
*/
|
||||
export interface ControlPlanePort {
|
||||
openRun(input: OpenControlPlaneRunInput): Promise<void>;
|
||||
loadSessionCheckpoint?(): Promise<PersistedNativeSession | null>;
|
||||
checkpointSession?(snapshot: PersistedNativeSession): Promise<void>;
|
||||
appendEvent(event: NativeRunEvent | PrpEvent): Promise<AppendedEventReceipt>;
|
||||
replayEvents(input: ReplayControlPlaneEventsInput): Promise<ReplayedControlPlaneEvents>;
|
||||
completeRun(result: NativeRunResult | CompleteControlPlaneRunInput): Promise<void>;
|
||||
|
||||
@@ -8,6 +8,11 @@ import type {
|
||||
PrpStructuredRunResult,
|
||||
PrpTerminalState,
|
||||
} from "../protocol/phase1-contract.js";
|
||||
import type {
|
||||
HarnessRuntimeRequest,
|
||||
HarnessThreadLineageEntry,
|
||||
PersistedHarnessTurnTerminal,
|
||||
} from "./harness-driver.js";
|
||||
|
||||
export interface NativeSessionBackendDescriptor {
|
||||
kind: "runner" | "remote" | "mock";
|
||||
@@ -29,6 +34,10 @@ export interface PersistedNativeSession {
|
||||
cursor?: string | null;
|
||||
semanticResult?: PrpStructuredRunResult | null;
|
||||
terminal?: PrpTerminalState | null;
|
||||
activeTurnId?: string | null;
|
||||
terminalTurns?: PersistedHarnessTurnTerminal[];
|
||||
pendingRuntimeRequests?: HarnessRuntimeRequest[];
|
||||
lineage?: HarnessThreadLineageEntry[];
|
||||
}
|
||||
|
||||
export interface NativeSessionRecoveryResult {
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import type { ControlPlanePort } from "./contracts/control-plane-port.js";
|
||||
import type { NativeExecutionInputV1 } from "./contracts/native-execution.js";
|
||||
import type {
|
||||
NativeSession,
|
||||
NativeSessionBackend,
|
||||
PersistedNativeSession,
|
||||
} from "./contracts/native-session-backend.js";
|
||||
import type {
|
||||
PrpEvent,
|
||||
PrpStructuredRunResult,
|
||||
PrpTerminalState,
|
||||
} from "./protocol/phase1-contract.js";
|
||||
import { executeNativeSession } from "./native-session-runtime.js";
|
||||
|
||||
const identity = {
|
||||
runId: "run-recovery",
|
||||
sessionId: "session-recovery",
|
||||
companyId: "company-recovery",
|
||||
issueId: "issue-recovery",
|
||||
agentId: "agent-recovery",
|
||||
};
|
||||
|
||||
const result: PrpStructuredRunResult = {
|
||||
schema: "paperclip.run_result.v1",
|
||||
reportedWorkDisposition: "done",
|
||||
summary: "Recovered native work completed.",
|
||||
completionClaim: {
|
||||
contractRevision: "1",
|
||||
objectiveSatisfied: true,
|
||||
criteria: [{ criterionId: "objective", status: "satisfied", evidenceRefs: [] }],
|
||||
remainingWork: [],
|
||||
},
|
||||
evidence: [],
|
||||
verification: [{ commandOrCheck: "recovery", status: "passed" }],
|
||||
attentionRequests: [],
|
||||
artifacts: [],
|
||||
};
|
||||
|
||||
const terminal: PrpTerminalState = {
|
||||
schema: "paperclip.prp.terminal.v1",
|
||||
turnTerminalState: "completed",
|
||||
runTerminalState: "succeeded",
|
||||
reportedWorkDisposition: "done",
|
||||
};
|
||||
|
||||
const input: NativeExecutionInputV1 = {
|
||||
schema: "paperclip.native-execution-input.v1",
|
||||
binding: {
|
||||
companyId: identity.companyId,
|
||||
runId: identity.runId,
|
||||
issueId: identity.issueId,
|
||||
agentId: identity.agentId,
|
||||
executionWorkspaceId: "workspace-recovery",
|
||||
},
|
||||
task: { identifier: "PAP-RECOVERY", title: "Recover native work", description: null, workMode: "standard" },
|
||||
workspace: { cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null },
|
||||
session: { normalizedSessionId: identity.sessionId, driverKind: "codex_app_server", protocolVersion: 1 },
|
||||
completionContract: {
|
||||
id: "contract-recovery",
|
||||
sha256: "contract-recovery-sha",
|
||||
schemaVersion: "paperclip.completion-contract.v1",
|
||||
contract: {
|
||||
revision: "1",
|
||||
objective: "Recover native work",
|
||||
criteria: [{ id: "objective", requirement: "Complete after recovery" }],
|
||||
},
|
||||
},
|
||||
interactionResponses: [],
|
||||
credentialBindings: [],
|
||||
};
|
||||
|
||||
function controlEvent(
|
||||
sourceSeq: number,
|
||||
eventType: PrpEvent["eventType"],
|
||||
payload: Record<string, unknown>,
|
||||
): PrpEvent {
|
||||
return {
|
||||
schema: "paperclip.prp.event.v1",
|
||||
sourceEventId: `control-recovery:${identity.runId}:${sourceSeq}`,
|
||||
sourceSeq,
|
||||
sourceInstanceId: "control-recovery",
|
||||
sourceKind: "control_plane",
|
||||
runId: identity.runId,
|
||||
normalizedSessionId: identity.sessionId,
|
||||
turnId: "turn-recovery",
|
||||
eventType,
|
||||
schemaVersion: 1,
|
||||
priority: 0,
|
||||
emittedAt: "2026-08-09T00:00:00.000Z",
|
||||
payload,
|
||||
};
|
||||
}
|
||||
|
||||
function highestContiguous(events: PrpEvent[]): number {
|
||||
const sequences = new Set(events.map((event) => event.sourceSeq));
|
||||
let cursor = 0;
|
||||
while (sequences.has(cursor + 1)) cursor += 1;
|
||||
return cursor;
|
||||
}
|
||||
|
||||
describe("executeNativeSession recovery", () => {
|
||||
it("recovers a completed checkpoint and appends only a missing control terminal fact", async () => {
|
||||
const checkpoint: PersistedNativeSession = {
|
||||
backendKind: "mock",
|
||||
sessionId: "driver-recovery",
|
||||
identity,
|
||||
providerSessionId: "provider-recovery",
|
||||
cursor: "4",
|
||||
semanticResult: result,
|
||||
terminal,
|
||||
activeTurnId: "turn-recovery",
|
||||
terminalTurns: [{ turnId: "turn-recovery", fingerprint: "terminal-fingerprint" }],
|
||||
pendingRuntimeRequests: [],
|
||||
lineage: [],
|
||||
};
|
||||
const events = [controlEvent(1, "run.result.accepted", { result })];
|
||||
const checkpoints: PersistedNativeSession[] = [];
|
||||
const completeRun = vi.fn(async () => undefined);
|
||||
const startTurn = vi.fn(async () => ({ turnId: "unexpected-turn" }));
|
||||
const openSession = vi.fn(async () => {
|
||||
throw new Error("a recovered run must not open a second provider session");
|
||||
});
|
||||
const session: NativeSession = {
|
||||
identity: () => identity,
|
||||
async capabilities() {
|
||||
return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };
|
||||
},
|
||||
async *events() {},
|
||||
startTurn,
|
||||
async result() { return { result, terminal, turnId: "turn-recovery" }; },
|
||||
async snapshot() { return structuredClone(checkpoint); },
|
||||
async close() {},
|
||||
};
|
||||
const recoverSession = vi.fn(async () => ({ recovered: true, session }));
|
||||
const backend: NativeSessionBackend = {
|
||||
async descriptor() {
|
||||
return {
|
||||
kind: "mock",
|
||||
name: "recovery-backend",
|
||||
version: "1",
|
||||
capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },
|
||||
};
|
||||
},
|
||||
openSession,
|
||||
recoverSession,
|
||||
};
|
||||
const port: ControlPlanePort = {
|
||||
async openRun() {},
|
||||
async loadSessionCheckpoint() { return structuredClone(checkpoint); },
|
||||
async checkpointSession(snapshot) { checkpoints.push(structuredClone(snapshot)); },
|
||||
async appendEvent(event) {
|
||||
events.push(structuredClone(event as PrpEvent));
|
||||
return {
|
||||
cursor: events.length,
|
||||
highestContiguousSourceSeq: highestContiguous(events),
|
||||
disposition: "committed",
|
||||
};
|
||||
},
|
||||
async replayEvents(replay) {
|
||||
const replayed = events.filter((event) => event.sourceSeq > replay.afterSourceSeq);
|
||||
return { events: structuredClone(replayed), highestContiguousSourceSeq: highestContiguous(events) };
|
||||
},
|
||||
completeRun,
|
||||
};
|
||||
|
||||
const completed = await executeNativeSession({
|
||||
input,
|
||||
backend,
|
||||
controlPlane: port,
|
||||
runnerInstanceId: "runner-recovery",
|
||||
controlPlaneInstanceId: "control-recovery",
|
||||
});
|
||||
|
||||
expect(openSession).not.toHaveBeenCalled();
|
||||
expect(recoverSession).toHaveBeenCalledOnce();
|
||||
expect(startTurn).not.toHaveBeenCalled();
|
||||
expect(events.map((event) => event.eventType)).toEqual(["run.result.accepted", "run.terminal"]);
|
||||
expect(events.map((event) => event.sourceSeq)).toEqual([1, 2]);
|
||||
expect(completeRun).toHaveBeenCalledOnce();
|
||||
expect(completed).toMatchObject({ nativeEventCount: 1, highestContiguousSourceSeq: 2 });
|
||||
expect(checkpoints.at(-1)).toMatchObject({ semanticResult: result, terminal });
|
||||
});
|
||||
});
|
||||
@@ -62,7 +62,10 @@ async function consumeTurn(session: NativeSession, controlPlane: ControlPlanePor
|
||||
export async function executeNativeSession(options: ExecuteNativeSessionOptions): Promise<NativeSessionExecutionResult> {
|
||||
const input = parseNativeExecutionInput(options.input);
|
||||
const descriptor = await options.backend.descriptor();
|
||||
const normalizedSessionId = input.session.normalizedSessionId ?? randomUUID();
|
||||
const persistedSession = await options.controlPlane.loadSessionCheckpoint?.() ?? null;
|
||||
const normalizedSessionId = persistedSession?.identity.sessionId
|
||||
?? input.session.normalizedSessionId
|
||||
?? randomUUID();
|
||||
await options.controlPlane.openRun({
|
||||
identity: {
|
||||
runId: input.binding.runId,
|
||||
@@ -75,49 +78,134 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)
|
||||
sourceInstanceId: options.runnerInstanceId,
|
||||
});
|
||||
|
||||
const session = await options.backend.openSession({
|
||||
identity: {
|
||||
runId: input.binding.runId,
|
||||
sessionId: normalizedSessionId,
|
||||
companyId: input.binding.companyId,
|
||||
issueId: input.binding.issueId,
|
||||
agentId: input.binding.agentId,
|
||||
},
|
||||
workingDirectory: input.workspace.cwd,
|
||||
});
|
||||
const identity = {
|
||||
runId: input.binding.runId,
|
||||
sessionId: normalizedSessionId,
|
||||
companyId: input.binding.companyId,
|
||||
issueId: input.binding.issueId,
|
||||
agentId: input.binding.agentId,
|
||||
};
|
||||
if (
|
||||
persistedSession
|
||||
&& (
|
||||
persistedSession.identity.runId !== identity.runId
|
||||
|| persistedSession.identity.companyId !== identity.companyId
|
||||
|| persistedSession.identity.issueId !== identity.issueId
|
||||
|| persistedSession.identity.agentId !== identity.agentId
|
||||
|| persistedSession.identity.sessionId !== identity.sessionId
|
||||
)
|
||||
) throw new Error("native_session_checkpoint_binding_mismatch");
|
||||
let recovered = false;
|
||||
let session: NativeSession;
|
||||
if (persistedSession) {
|
||||
if (!options.backend.recoverSession) {
|
||||
throw new Error("native_session_recovery_unavailable: refusing to open a second provider session");
|
||||
}
|
||||
const recovery = await options.backend.recoverSession(persistedSession);
|
||||
if (!recovery.recovered || !recovery.session) {
|
||||
throw new Error(`native_session_recovery_failed: ${recovery.reason ?? "unknown"}`);
|
||||
}
|
||||
session = recovery.session;
|
||||
recovered = true;
|
||||
} else {
|
||||
session = await options.backend.openSession({
|
||||
identity,
|
||||
workingDirectory: input.workspace.cwd,
|
||||
});
|
||||
}
|
||||
options.onSession?.(session);
|
||||
try {
|
||||
const consuming = consumeTurn(session, options.controlPlane, options.timeoutMs ?? 900_000);
|
||||
await session.startTurn({
|
||||
message: { role: "user", text: JSON.stringify(buildNativeModelEnvelope(input)) },
|
||||
});
|
||||
const consumed = await consuming;
|
||||
const completed = await session.result();
|
||||
const checkpoint = async () => {
|
||||
if (options.controlPlane.checkpointSession) {
|
||||
await options.controlPlane.checkpointSession(await session.snapshot());
|
||||
}
|
||||
};
|
||||
await checkpoint();
|
||||
|
||||
let consumed = { event: null as PrpEvent | null, eventCount: 0, highestContiguousSourceSeq: 0 };
|
||||
let completed = persistedSession?.semanticResult && persistedSession.terminal
|
||||
? {
|
||||
result: persistedSession.semanticResult,
|
||||
terminal: persistedSession.terminal,
|
||||
turnId: persistedSession.activeTurnId ?? null,
|
||||
}
|
||||
: null;
|
||||
if (!completed) {
|
||||
const consuming = consumeTurn(session, options.controlPlane, options.timeoutMs ?? 900_000);
|
||||
if (!recovered || !persistedSession?.activeTurnId) {
|
||||
await session.startTurn({
|
||||
message: { role: "user", text: JSON.stringify(buildNativeModelEnvelope(input)) },
|
||||
});
|
||||
await checkpoint();
|
||||
}
|
||||
const terminalEvent = await consuming;
|
||||
consumed = terminalEvent;
|
||||
completed = await session.result();
|
||||
await checkpoint();
|
||||
}
|
||||
if (completed === null) throw new Error("native_finalization_missing: session returned no semantic result");
|
||||
const terminal = terminalFromEvent(consumed.event, completed.result.reportedWorkDisposition);
|
||||
let controlSeq = 0;
|
||||
const controlEvent = (eventType: PrpEvent["eventType"], payload: Record<string, unknown>): PrpEvent => ({
|
||||
let terminal: PrpTerminalState;
|
||||
if (persistedSession?.semanticResult && persistedSession.terminal) {
|
||||
terminal = persistedSession.terminal;
|
||||
} else {
|
||||
terminal = terminalFromEvent(consumed.event!, completed.result.reportedWorkDisposition);
|
||||
}
|
||||
const eventTurnId = completed.turnId
|
||||
?? persistedSession?.activeTurnId
|
||||
?? persistedSession?.terminalTurns?.at(-1)?.turnId
|
||||
?? consumed.event?.turnId;
|
||||
const controlEvent = (
|
||||
sourceSeq: number,
|
||||
eventType: PrpEvent["eventType"],
|
||||
payload: Record<string, unknown>,
|
||||
): PrpEvent => ({
|
||||
schema: "paperclip.prp.event.v1",
|
||||
sourceEventId: `${options.controlPlaneInstanceId}:${input.binding.runId}:${++controlSeq}`,
|
||||
sourceSeq: controlSeq,
|
||||
sourceEventId: `${options.controlPlaneInstanceId}:${input.binding.runId}:${sourceSeq}`,
|
||||
sourceSeq,
|
||||
sourceInstanceId: options.controlPlaneInstanceId,
|
||||
sourceKind: "control_plane",
|
||||
runId: input.binding.runId,
|
||||
normalizedSessionId,
|
||||
turnId: completed.turnId ?? consumed.event.turnId,
|
||||
...(eventTurnId ? { turnId: eventTurnId } : {}),
|
||||
eventType,
|
||||
schemaVersion: 1,
|
||||
priority: 0,
|
||||
emittedAt: new Date().toISOString(),
|
||||
payload,
|
||||
});
|
||||
for (const event of [
|
||||
controlEvent("run.result.accepted", { result: completed.result }),
|
||||
controlEvent("run.terminal", terminal as unknown as Record<string, unknown>),
|
||||
]) {
|
||||
const expectedControlEvents = [
|
||||
controlEvent(1, "run.result.accepted", { result: completed.result }),
|
||||
controlEvent(2, "run.terminal", terminal as unknown as Record<string, unknown>),
|
||||
];
|
||||
const controlReplay = await options.controlPlane.replayEvents({
|
||||
runId: input.binding.runId,
|
||||
sourceInstanceId: options.controlPlaneInstanceId,
|
||||
afterSourceSeq: 0,
|
||||
limit: 10,
|
||||
});
|
||||
const replayBySequence = new Map(controlReplay.events.map((event) => [event.sourceSeq, event]));
|
||||
for (const existing of controlReplay.events) {
|
||||
const expected = expectedControlEvents[existing.sourceSeq - 1];
|
||||
if (
|
||||
expected === undefined
|
||||
|| existing.eventType !== expected.eventType
|
||||
|| canonicalJson(existing.payload) !== canonicalJson(expected.payload)
|
||||
) {
|
||||
throw new Error(`native_control_event_replay_conflict:${existing.sourceSeq}`);
|
||||
}
|
||||
}
|
||||
consumed.highestContiguousSourceSeq = Math.max(
|
||||
consumed.highestContiguousSourceSeq,
|
||||
controlReplay.highestContiguousSourceSeq,
|
||||
);
|
||||
for (const event of expectedControlEvents) {
|
||||
if (replayBySequence.has(event.sourceSeq)) continue;
|
||||
const receipt = await options.controlPlane.appendEvent(event);
|
||||
consumed.eventCount += receipt.disposition === "committed" ? 1 : 0;
|
||||
consumed.highestContiguousSourceSeq = Math.max(consumed.highestContiguousSourceSeq, receipt.highestContiguousSourceSeq);
|
||||
consumed.highestContiguousSourceSeq = Math.max(
|
||||
consumed.highestContiguousSourceSeq,
|
||||
receipt.highestContiguousSourceSeq,
|
||||
);
|
||||
}
|
||||
await options.controlPlane.completeRun({
|
||||
result: completed.result,
|
||||
@@ -127,6 +215,11 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)
|
||||
callerDedupeKey: `${input.binding.runId}:${input.completionContract.sha256}`,
|
||||
});
|
||||
const snapshot = await session.snapshot();
|
||||
await options.controlPlane.checkpointSession?.({
|
||||
...snapshot,
|
||||
semanticResult: completed.result,
|
||||
terminal,
|
||||
});
|
||||
return {
|
||||
result: completed.result,
|
||||
terminal,
|
||||
@@ -144,3 +237,15 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)
|
||||
await session.close({ reason: "native session execution complete" });
|
||||
}
|
||||
}
|
||||
|
||||
function canonicalJson(value: unknown): string {
|
||||
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
|
||||
if (typeof value === "object" && value !== null) {
|
||||
const record = value as Record<string, unknown>;
|
||||
return `{${Object.keys(record)
|
||||
.sort()
|
||||
.map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`)
|
||||
.join(",")}}`;
|
||||
}
|
||||
return JSON.stringify(value) ?? "undefined";
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
// Acceptance-matrix entry point for budget/manual cancellation idempotency.
|
||||
import "../services/native-runtime/native-session-executor.test.js";
|
||||
@@ -0,0 +1,2 @@
|
||||
// Acceptance-matrix entry point for default-off, opt-in, and eligibility rules.
|
||||
import "../services/native-runtime/runtime-mode.test.js";
|
||||
@@ -0,0 +1,84 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { eq } from "drizzle-orm";
|
||||
import {
|
||||
agents,
|
||||
companies,
|
||||
createDb,
|
||||
heartbeatRunEvents,
|
||||
heartbeatRuns,
|
||||
} from "@paperclipai/db";
|
||||
import { appendHeartbeatRunEvent, HeartbeatRunEventConflictError } from "../services/heartbeat-run-events.js";
|
||||
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
|
||||
describe("P6-11..13 / P6-17 canonical event allocator", () => {
|
||||
it("serializes concurrent writers and rejects conflicting replay without cursor drift", async () => {
|
||||
const temporary = await startEmbeddedPostgresTestDatabase("paperclip-native-events-");
|
||||
const db = createDb(temporary.connectionString);
|
||||
const companyId = "20000000-0000-4000-8000-000000000001";
|
||||
const agentId = "20000000-0000-4000-8000-000000000002";
|
||||
const runId = "20000000-0000-4000-8000-000000000003";
|
||||
try {
|
||||
await db.insert(companies).values({ id: companyId, name: "Allocator fixture", issuePrefix: "SEQ" });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Allocator agent" });
|
||||
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running" });
|
||||
const eventTypes = ["heartbeat.started", "run.cancel.requested", "item.completed", "stdout"];
|
||||
const writes = Array.from({ length: 32 }, (_, index) => appendHeartbeatRunEvent(db, {
|
||||
companyId,
|
||||
runId,
|
||||
agentId,
|
||||
eventType: eventTypes[index % eventTypes.length]!,
|
||||
message: `event-${index + 1}`,
|
||||
payload: { ordinal: index + 1 },
|
||||
nativeSource: {
|
||||
sourceInstanceId: `writer-${index % 4}`,
|
||||
sourceEventId: `source-event-${index + 1}`,
|
||||
sourceSeq: Math.floor(index / 4) + 1,
|
||||
protocolSchemaVersion: 1,
|
||||
canonicalPayload: { ordinal: index + 1 },
|
||||
},
|
||||
}));
|
||||
const receipts = await Promise.all(writes);
|
||||
expect(receipts.every((entry) => entry.disposition === "committed")).toBe(true);
|
||||
|
||||
const rows = await db.select().from(heartbeatRunEvents)
|
||||
.where(eq(heartbeatRunEvents.runId, runId)).orderBy(heartbeatRunEvents.seq);
|
||||
expect(rows.map((row) => row.seq)).toEqual(Array.from({ length: 32 }, (_, index) => index + 1));
|
||||
expect(new Set(rows.map((row) => row.sourceEventId)).size).toBe(32);
|
||||
expect((await db.select({ nextEventSeq: heartbeatRuns.nextEventSeq }).from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, runId)))[0]?.nextEventSeq).toBe(33);
|
||||
|
||||
const original = rows.find((row) => row.sourceEventId === "source-event-1")!;
|
||||
await expect(appendHeartbeatRunEvent(db, {
|
||||
companyId,
|
||||
runId,
|
||||
agentId,
|
||||
eventType: original.eventType,
|
||||
message: original.message,
|
||||
payload: original.payload,
|
||||
nativeSource: {
|
||||
sourceInstanceId: original.sourceInstanceId!,
|
||||
sourceEventId: original.sourceEventId!,
|
||||
sourceSeq: original.sourceSeq!,
|
||||
protocolSchemaVersion: 1,
|
||||
canonicalPayload: { ordinal: 1 },
|
||||
},
|
||||
})).resolves.toEqual(expect.objectContaining({ disposition: "duplicate" }));
|
||||
await expect(appendHeartbeatRunEvent(db, {
|
||||
companyId,
|
||||
runId,
|
||||
agentId,
|
||||
eventType: "item.failed",
|
||||
nativeSource: {
|
||||
sourceInstanceId: original.sourceInstanceId!,
|
||||
sourceEventId: original.sourceEventId!,
|
||||
sourceSeq: original.sourceSeq!,
|
||||
protocolSchemaVersion: 1,
|
||||
canonicalPayload: { ordinal: 999 },
|
||||
},
|
||||
})).rejects.toBeInstanceOf(HeartbeatRunEventConflictError);
|
||||
expect(await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, runId))).toHaveLength(32);
|
||||
} finally {
|
||||
await temporary.cleanup();
|
||||
}
|
||||
}, 60_000);
|
||||
});
|
||||
@@ -0,0 +1,80 @@
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { eq } from "drizzle-orm";
|
||||
import {
|
||||
agents,
|
||||
companies,
|
||||
completionContracts,
|
||||
createDb,
|
||||
heartbeatRunEvents,
|
||||
heartbeatRuns,
|
||||
nativeRunFinalizations,
|
||||
nativeRunResults,
|
||||
statusDecisions,
|
||||
workAssessments,
|
||||
} from "@paperclipai/db";
|
||||
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { reconcileNativeFinalizations } from "../services/native-runtime/native-finalization-reconciler.js";
|
||||
|
||||
describe("P6-32 legacy finalization regression", () => {
|
||||
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
const companyId = "71000000-0000-4000-8000-000000000001";
|
||||
const agentId = "71000000-0000-4000-8000-000000000002";
|
||||
const runId = "71000000-0000-4000-8000-000000000003";
|
||||
|
||||
beforeAll(async () => {
|
||||
temporary = await startEmbeddedPostgresTestDatabase("paperclip-native-legacy-");
|
||||
db = createDb(temporary.connectionString);
|
||||
await db.insert(companies).values({ id: companyId, name: "Legacy snapshot", issuePrefix: "LGC" });
|
||||
await db.insert(agents).values({
|
||||
id: agentId,
|
||||
companyId,
|
||||
name: "Legacy agent",
|
||||
adapterType: "codex_local",
|
||||
status: "idle",
|
||||
});
|
||||
await db.insert(heartbeatRuns).values({
|
||||
id: runId,
|
||||
companyId,
|
||||
agentId,
|
||||
status: "succeeded",
|
||||
runtimeMode: "legacy",
|
||||
nextEventSeq: 8,
|
||||
exitCode: 0,
|
||||
resultJson: { summary: "Legacy bytes", nested: { count: 1, ok: true } },
|
||||
contextSnapshot: { issueId: "legacy-opaque-issue" },
|
||||
});
|
||||
await db.insert(heartbeatRunEvents).values({
|
||||
companyId,
|
||||
runId,
|
||||
agentId,
|
||||
seq: 7,
|
||||
eventType: "lifecycle",
|
||||
stream: "system",
|
||||
message: "Legacy event bytes",
|
||||
payload: { state: "completed", nested: [1, "two", false] },
|
||||
});
|
||||
}, 30_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await temporary.cleanup();
|
||||
});
|
||||
|
||||
it("keeps the post-kill-switch legacy read snapshot byte-equivalent with zero native rows", async () => {
|
||||
const beforeRun = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
|
||||
const beforeEvents = await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, runId));
|
||||
const beforeBytes = JSON.stringify({ run: beforeRun, events: beforeEvents });
|
||||
|
||||
await expect(reconcileNativeFinalizations(db, [runId])).resolves.toEqual([]);
|
||||
|
||||
const afterRun = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
|
||||
const afterEvents = await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, runId));
|
||||
expect(JSON.stringify({ run: afterRun, events: afterEvents })).toBe(beforeBytes);
|
||||
await expect(db.select().from(completionContracts).where(eq(completionContracts.companyId, companyId))).resolves.toHaveLength(0);
|
||||
await expect(db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, runId))).resolves.toHaveLength(0);
|
||||
await expect(db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId))).resolves.toHaveLength(0);
|
||||
await expect(db.select().from(workAssessments).where(eq(workAssessments.runId, runId))).resolves.toHaveLength(0);
|
||||
await expect(db.select().from(statusDecisions)
|
||||
.where(eq(statusDecisions.companyId, companyId))).resolves.toHaveLength(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,67 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import {
|
||||
agents,
|
||||
companies,
|
||||
createDb,
|
||||
heartbeatRunEvents,
|
||||
heartbeatRuns,
|
||||
} from "@paperclipai/db";
|
||||
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
|
||||
describe("P6-18 / MIG-01..04 native finalization migration", () => {
|
||||
it("repairs only later duplicates and preserves legacy event bytes and cursors", async () => {
|
||||
const temporary = await startEmbeddedPostgresTestDatabase("paperclip-native-migration-");
|
||||
const db = createDb(temporary.connectionString);
|
||||
try {
|
||||
const companyId = "10000000-0000-4000-8000-000000000001";
|
||||
const agentId = "10000000-0000-4000-8000-000000000002";
|
||||
const runId = "10000000-0000-4000-8000-000000000003";
|
||||
await db.insert(companies).values({ id: companyId, name: "Migration fixture", issuePrefix: "MIG" });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Migration agent" });
|
||||
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "succeeded", nextEventSeq: 1 });
|
||||
await db.execute(sql.raw("DROP INDEX heartbeat_run_events_run_seq_uq"));
|
||||
await db.insert(heartbeatRunEvents).values([
|
||||
{ companyId, runId, agentId, seq: 1, eventType: "legacy.start", stream: "system", level: "info", message: "one", payload: { bytes: "α-1" }, createdAt: new Date("2026-08-01T00:00:01.000Z") },
|
||||
{ companyId, runId, agentId, seq: 5, eventType: "legacy.log", stream: "stdout", level: "info", message: "first-five", payload: { bytes: "β-5a" }, createdAt: new Date("2026-08-01T00:00:02.000Z") },
|
||||
{ companyId, runId, agentId, seq: 5, eventType: "legacy.log", stream: "stderr", level: "warn", message: "duplicate-five", payload: { bytes: "γ-5b" }, createdAt: new Date("2026-08-01T00:00:03.000Z") },
|
||||
{ companyId, runId, agentId, seq: 9, eventType: "legacy.end", stream: "system", level: "info", message: "nine", payload: { bytes: "δ-9" }, createdAt: new Date("2026-08-01T00:00:04.000Z") },
|
||||
]);
|
||||
|
||||
const before = await db.select().from(heartbeatRunEvents)
|
||||
.where(eq(heartbeatRunEvents.runId, runId)).orderBy(heartbeatRunEvents.id);
|
||||
const migration = await readFile(
|
||||
new URL("../../../packages/db/src/migrations/0211_famous_guardsmen.sql", import.meta.url),
|
||||
"utf8",
|
||||
);
|
||||
const statements = migration.split("--> statement-breakpoint").map((entry) => entry.trim());
|
||||
const duplicateRepair = statements.find((entry) => entry.includes("WITH ranked AS"));
|
||||
const allocatorSeed = statements.find((entry) => entry.startsWith('UPDATE "heartbeat_runs" AS run'));
|
||||
expect(duplicateRepair).toBeTruthy();
|
||||
expect(allocatorSeed).toBeTruthy();
|
||||
await db.execute(sql.raw(duplicateRepair!));
|
||||
await db.execute(sql.raw(allocatorSeed!));
|
||||
await db.execute(sql.raw(
|
||||
'CREATE UNIQUE INDEX "heartbeat_run_events_run_seq_uq" ON "heartbeat_run_events" ("run_id", "seq")',
|
||||
));
|
||||
|
||||
const after = await db.select().from(heartbeatRunEvents)
|
||||
.where(eq(heartbeatRunEvents.runId, runId)).orderBy(heartbeatRunEvents.id);
|
||||
expect(after.map((row) => row.seq)).toEqual([1, 5, 10, 9]);
|
||||
expect((await db.select({ nextEventSeq: heartbeatRuns.nextEventSeq }).from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, runId)))[0]?.nextEventSeq).toBe(11);
|
||||
|
||||
// The repaired duplicate's cursor is the only changed byte-equivalent read field.
|
||||
expect(after.map(({ seq: _seq, ...row }) => row)).toEqual(before.map(({ seq: _seq, ...row }) => row));
|
||||
expect(after[0]?.seq).toBe(before[0]?.seq);
|
||||
expect(after[1]?.seq).toBe(before[1]?.seq);
|
||||
expect(after[3]?.seq).toBe(before[3]?.seq);
|
||||
await expect(db.insert(heartbeatRunEvents).values({
|
||||
companyId, runId, agentId, seq: 5, eventType: "must-conflict",
|
||||
})).rejects.toThrow();
|
||||
} finally {
|
||||
await temporary.cleanup();
|
||||
}
|
||||
}, 60_000);
|
||||
});
|
||||
@@ -0,0 +1,154 @@
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { eq } from "drizzle-orm";
|
||||
import {
|
||||
agents,
|
||||
companies,
|
||||
completionContracts,
|
||||
createDb,
|
||||
heartbeatRuns,
|
||||
issueRecoveryActions,
|
||||
issues,
|
||||
nativeRunFinalizations,
|
||||
nativeRunResults,
|
||||
statusDecisions,
|
||||
workspaceOperations,
|
||||
} from "@paperclipai/db";
|
||||
import {
|
||||
CONTROL_PLANE_CONFORMANCE_OPEN,
|
||||
CONTROL_PLANE_CONFORMANCE_RESULT,
|
||||
CONTROL_PLANE_CONFORMANCE_TERMINAL,
|
||||
} from "@paperclipai/paperclip-runner";
|
||||
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { reconcileNativeFinalizations } from "../services/native-runtime/native-finalization-reconciler.js";
|
||||
import { PaperclipControlPlanePort } from "../services/native-runtime/paperclip-control-plane-port.js";
|
||||
|
||||
describe("P6-16/P6-25/P6-28 native finalization recovery", () => {
|
||||
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
const companyId = "72000000-0000-4000-8000-000000000001";
|
||||
const agentId = "72000000-0000-4000-8000-000000000002";
|
||||
const issueId = "72000000-0000-4000-8000-000000000003";
|
||||
const contractId = "72000000-0000-4000-8000-000000000004";
|
||||
const runId = "72000000-0000-4000-8000-000000000005";
|
||||
|
||||
beforeAll(async () => {
|
||||
temporary = await startEmbeddedPostgresTestDatabase("paperclip-native-recovery-");
|
||||
db = createDb(temporary.connectionString);
|
||||
await db.insert(companies).values({ id: companyId, name: "Native recovery", issuePrefix: "NRC" });
|
||||
await db.insert(agents).values({
|
||||
id: agentId,
|
||||
companyId,
|
||||
name: "Recovery agent",
|
||||
adapterType: "codex_local",
|
||||
status: "running",
|
||||
});
|
||||
await db.insert(issues).values({
|
||||
id: issueId,
|
||||
companyId,
|
||||
title: "Recover invalid native finalization",
|
||||
status: "in_progress",
|
||||
assigneeAgentId: agentId,
|
||||
workMode: "standard",
|
||||
});
|
||||
await db.insert(completionContracts).values({
|
||||
id: contractId,
|
||||
companyId,
|
||||
issueId,
|
||||
revision: 1,
|
||||
schemaVersion: "paperclip.completion-contract.v1",
|
||||
policyVersion: "phase6-v1",
|
||||
risk: "standard",
|
||||
completionAuthority: "server_arbiter",
|
||||
incompleteCriteriaPolicy: "preserve_non_terminal",
|
||||
contractJson: {
|
||||
revision: "phase6-v1",
|
||||
objective: "Recover invalid native finalization",
|
||||
criteria: [{ id: "objective", requirement: "Recovery remains live" }],
|
||||
},
|
||||
canonicalSha256: "native-recovery-contract",
|
||||
createdByActorType: "system",
|
||||
createdByActorId: "test",
|
||||
});
|
||||
await db.insert(heartbeatRuns).values({
|
||||
id: runId,
|
||||
companyId,
|
||||
agentId,
|
||||
status: "running",
|
||||
runtimeMode: "native",
|
||||
runtimeModeReason: "persisted_before_kill_switch",
|
||||
completionContractId: contractId,
|
||||
completionContractSha256: "native-recovery-contract",
|
||||
contextSnapshot: { issueId },
|
||||
});
|
||||
const port = new PaperclipControlPlanePort(db, {
|
||||
companyId,
|
||||
issueId,
|
||||
runId,
|
||||
agentId,
|
||||
completionContractId: contractId,
|
||||
completionContractSha256: "native-recovery-contract",
|
||||
sourceInstanceId: "recovery-runner",
|
||||
controlPlaneSourceInstanceId: "recovery-control",
|
||||
});
|
||||
await port.openRun({
|
||||
...CONTROL_PLANE_CONFORMANCE_OPEN,
|
||||
identity: { companyId, issueId, runId, agentId, normalizedSessionId: "recovery-session" },
|
||||
sourceInstanceId: "recovery-runner",
|
||||
});
|
||||
await port.completeRun({
|
||||
result: CONTROL_PLANE_CONFORMANCE_RESULT,
|
||||
terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL,
|
||||
callerResultId: "recovery-result",
|
||||
});
|
||||
const stored = await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, runId))
|
||||
.limit(1).then((rows) => rows[0]!);
|
||||
await db.update(nativeRunResults).set({
|
||||
resultJson: {
|
||||
...(stored.resultJson as Record<string, unknown>),
|
||||
terminal: { ...CONTROL_PLANE_CONFORMANCE_TERMINAL, runTerminalState: "unknown" },
|
||||
},
|
||||
}).where(eq(nativeRunResults.id, stored.id));
|
||||
await db.insert(workspaceOperations).values({
|
||||
companyId,
|
||||
heartbeatRunId: runId,
|
||||
issueId,
|
||||
phase: "workspace_finalize",
|
||||
status: "succeeded",
|
||||
});
|
||||
}, 30_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await temporary.cleanup();
|
||||
});
|
||||
|
||||
it("fails closed into bounded named recovery without consulting the live flag or falling back", async () => {
|
||||
await expect(reconcileNativeFinalizations(db, [runId])).resolves.toEqual([
|
||||
expect.objectContaining({ phase: "retryable_failure", failureCode: "native_finalization_invalid" }),
|
||||
]);
|
||||
await expect(db.select().from(issues).where(eq(issues.id, issueId))).resolves.toEqual([
|
||||
expect.objectContaining({ status: "in_progress", statusVersion: 0, lastStatusDecisionId: null }),
|
||||
]);
|
||||
await expect(db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId))).resolves.toEqual([
|
||||
expect.objectContaining({ status: "active", ownerAgentId: agentId, cause: "native_finalization_invalid" }),
|
||||
]);
|
||||
await expect(db.select().from(statusDecisions).where(eq(statusDecisions.issueId, issueId))).resolves.toHaveLength(0);
|
||||
|
||||
for (let retry = 0; retry < 2; retry += 1) {
|
||||
await db.update(nativeRunFinalizations).set({ nextAttemptAt: new Date(0) })
|
||||
.where(eq(nativeRunFinalizations.runId, runId));
|
||||
await reconcileNativeFinalizations(db, [runId]);
|
||||
}
|
||||
await expect(db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId))).resolves.toEqual([
|
||||
expect.objectContaining({
|
||||
phase: "terminal_failure",
|
||||
attempt: 3,
|
||||
failureCode: "native_finalization_retry_exhausted",
|
||||
nextAttemptAt: null,
|
||||
}),
|
||||
]);
|
||||
await expect(db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId))).resolves.toEqual([
|
||||
expect.objectContaining({ runtimeMode: "native", status: "failed", nativePhase: "terminal_failure" }),
|
||||
]);
|
||||
await expect(reconcileNativeFinalizations(db, [runId])).resolves.toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,3 @@
|
||||
// Acceptance-matrix entry point for evidence, finalization, CAS, and atomic
|
||||
// status/liveness effects exercised by the real database-backed port suite.
|
||||
import "../services/native-runtime/paperclip-control-plane-port.test.js";
|
||||
@@ -0,0 +1,2 @@
|
||||
// Acceptance-matrix entry point for the package-owned closed input parser.
|
||||
import "../../../packages/paperclip-runner/src/contracts/native-execution.test.js";
|
||||
@@ -0,0 +1,3 @@
|
||||
// Acceptance-matrix entry point. The database-backed suite remains colocated
|
||||
// with the production port so its package conformance fixtures stay adjacent.
|
||||
import "../services/native-runtime/paperclip-control-plane-port.test.js";
|
||||
@@ -0,0 +1,95 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
type Fixture = {
|
||||
id: string;
|
||||
covers: Record<string, string[]>;
|
||||
expected: Record<string, unknown>;
|
||||
};
|
||||
|
||||
type Corpus = {
|
||||
schema: string;
|
||||
corpusRevision: number;
|
||||
fixtures: Fixture[];
|
||||
};
|
||||
|
||||
const corpusPath = fileURLToPath(new URL(
|
||||
"../../../packages/paperclip-runner/spec/fixtures/status-authority-phase5.json",
|
||||
import.meta.url,
|
||||
));
|
||||
|
||||
const consumerByMatrixPrefix: Record<string, string> = {
|
||||
SD: "server/native-status-arbiter-and-committer",
|
||||
TC: "package/native-session-runtime",
|
||||
ATT: "server/native-interaction-bridge",
|
||||
LIVE: "server/atomic-liveness-failpoints",
|
||||
REC: "server/native-finalization-recovery",
|
||||
COMP: "server/legacy-finalization-regression",
|
||||
MIG: "server/native-finalization-migration",
|
||||
};
|
||||
|
||||
function canonical(value: unknown): string {
|
||||
if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`;
|
||||
if (value && typeof value === "object") {
|
||||
return `{${Object.entries(value as Record<string, unknown>)
|
||||
.sort(([left], [right]) => left.localeCompare(right))
|
||||
.map(([key, entry]) => `${JSON.stringify(key)}:${canonical(entry)}`)
|
||||
.join(",")}}`;
|
||||
}
|
||||
return JSON.stringify(value) ?? "null";
|
||||
}
|
||||
|
||||
function emitConsumerResults(corpus: Corpus) {
|
||||
return corpus.fixtures.flatMap((fixture) => {
|
||||
const coveredRows = Object.values(fixture.covers).flat();
|
||||
// Revision 2 contains one negative fixture that intentionally has no
|
||||
// matrix-row claim. It still needs an independently joinable consumer.
|
||||
if (coveredRows.length === 0) {
|
||||
return [{
|
||||
corpusRevision: corpus.corpusRevision,
|
||||
fixtureId: fixture.id,
|
||||
matrixRow: "fixture-only",
|
||||
consumer: "server/native-finalization-recovery",
|
||||
expectedDigest: createHash("sha256").update(canonical(fixture.expected)).digest("hex"),
|
||||
}];
|
||||
}
|
||||
return coveredRows.map((matrixRow) => {
|
||||
const prefix = matrixRow.split("-")[0]!;
|
||||
const consumer = consumerByMatrixPrefix[prefix];
|
||||
if (!consumer) throw new Error(`missing_consumer:${matrixRow}`);
|
||||
return {
|
||||
corpusRevision: corpus.corpusRevision,
|
||||
fixtureId: fixture.id,
|
||||
matrixRow,
|
||||
consumer,
|
||||
expectedDigest: createHash("sha256").update(canonical(fixture.expected)).digest("hex"),
|
||||
};
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
describe("P6-31 Section 18.13 status-authority corpus traceability", () => {
|
||||
it("emits one result for every Section 18.13 fixture and consumer", () => {
|
||||
const corpus = JSON.parse(readFileSync(corpusPath, "utf8")) as Corpus;
|
||||
const results = emitConsumerResults(corpus);
|
||||
const coveredFixtureIds = new Set(results.map((result) => result.fixtureId));
|
||||
const resultKeys = results.map((result) => `${result.fixtureId}:${result.matrixRow}:${result.consumer}`);
|
||||
|
||||
expect(corpus.schema).toBe("paperclip.status-authority-conformance.v1");
|
||||
expect(corpus.fixtures).toHaveLength(52);
|
||||
expect(coveredFixtureIds).toEqual(new Set(corpus.fixtures.map((fixture) => fixture.id)));
|
||||
expect(new Set(resultKeys).size).toBe(resultKeys.length);
|
||||
expect(results.every((result) => /^[a-f0-9]{64}$/.test(result.expectedDigest))).toBe(true);
|
||||
expect(new Set(results.filter((result) => result.matrixRow !== "fixture-only").map((result) => result.matrixRow))).toEqual(new Set([
|
||||
...Array.from({ length: 19 }, (_, index) => `SD-${String(index + 1).padStart(2, "0")}`),
|
||||
...Array.from({ length: 8 }, (_, index) => `TC-${String(index + 1).padStart(2, "0")}`),
|
||||
...Array.from({ length: 12 }, (_, index) => `ATT-${String(index + 1).padStart(2, "0")}`),
|
||||
...Array.from({ length: 6 }, (_, index) => `LIVE-${String(index + 1).padStart(2, "0")}`),
|
||||
...Array.from({ length: 8 }, (_, index) => `REC-${String(index + 1).padStart(2, "0")}`),
|
||||
...Array.from({ length: 8 }, (_, index) => `COMP-${String(index + 1).padStart(2, "0")}`),
|
||||
...Array.from({ length: 9 }, (_, index) => `MIG-${String(index + 1).padStart(2, "0")}`),
|
||||
]));
|
||||
});
|
||||
});
|
||||
@@ -57,6 +57,7 @@ import {
|
||||
issueThreadInteractions,
|
||||
issues,
|
||||
issueWorkProducts,
|
||||
nativeRunFinalizations,
|
||||
projects,
|
||||
projectWorkspaces,
|
||||
routineRevisions,
|
||||
@@ -15335,22 +15336,40 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
|
||||
contract: completionContract.contract,
|
||||
},
|
||||
});
|
||||
await db.update(heartbeatRuns).set({
|
||||
runtimeMode: "native",
|
||||
runtimeModeResolverVersion: nativeRuntimeResolution.resolverVersion,
|
||||
runtimeModeReason: nativeRuntimeResolution.reason,
|
||||
runtimeModeResolvedAt: run.runtimeModeResolvedAt ?? new Date(),
|
||||
runnerProfileJson: nativeRuntimeResolution.profile,
|
||||
runnerInstanceId: nativeRunnerInstanceId,
|
||||
nativeSessionId,
|
||||
driverKind: "codex_app_server",
|
||||
driverVersion: "phase6-v1",
|
||||
completionContractId: completionContract.row.id,
|
||||
completionContractSha256: completionContract.row.canonicalSha256,
|
||||
nativePhase: run.nativePhase ?? "selected",
|
||||
nativePhaseUpdatedAt: run.nativePhaseUpdatedAt ?? new Date(),
|
||||
updatedAt: new Date(),
|
||||
}).where(eq(heartbeatRuns.id, run.id));
|
||||
await db.transaction(async (tx) => {
|
||||
const lockedRun = await tx.select().from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, run.id)).for("update").limit(1)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
if (!lockedRun) throw new Error("native_runtime_run_missing");
|
||||
if (lockedRun.runtimeModeResolvedAt && lockedRun.runtimeMode !== "native") {
|
||||
throw new Error("native_runtime_mode_conflict");
|
||||
}
|
||||
await tx.update(heartbeatRuns).set({
|
||||
runtimeMode: "native",
|
||||
runtimeModeResolverVersion: lockedRun.runtimeModeResolverVersion ?? nativeRuntimeResolution.resolverVersion,
|
||||
runtimeModeReason: lockedRun.runtimeModeReason ?? nativeRuntimeResolution.reason,
|
||||
runtimeModeResolvedAt: lockedRun.runtimeModeResolvedAt ?? new Date(),
|
||||
runnerProfileJson: {
|
||||
...nativeRuntimeResolution.profile,
|
||||
...parseObject(lockedRun.runnerProfileJson),
|
||||
},
|
||||
runnerInstanceId: lockedRun.runnerInstanceId ?? nativeRunnerInstanceId,
|
||||
nativeSessionId: lockedRun.nativeSessionId ?? nativeSessionId,
|
||||
driverKind: lockedRun.driverKind ?? "codex_app_server",
|
||||
driverVersion: lockedRun.driverVersion ?? "phase6-v1",
|
||||
completionContractId: lockedRun.completionContractId ?? completionContract.row.id,
|
||||
completionContractSha256: lockedRun.completionContractSha256 ?? completionContract.row.canonicalSha256,
|
||||
nativePhase: lockedRun.nativePhase ?? "observed",
|
||||
nativePhaseUpdatedAt: lockedRun.nativePhaseUpdatedAt ?? new Date(),
|
||||
updatedAt: new Date(),
|
||||
}).where(eq(heartbeatRuns.id, run.id));
|
||||
await tx.insert(nativeRunFinalizations).values({
|
||||
runId: run.id,
|
||||
companyId: agent.companyId,
|
||||
issueId: issueRef.id,
|
||||
phase: "observed",
|
||||
}).onConflictDoNothing();
|
||||
});
|
||||
} else {
|
||||
await db.update(heartbeatRuns).set({
|
||||
runtimeMode: "legacy",
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { classifyNativeEvidence } from "./evidence-classifier.js";
|
||||
|
||||
const workProductId = "00000000-0000-4000-8000-000000000001";
|
||||
const evidenceRef = `work_product:${workProductId}`;
|
||||
|
||||
function evidenceDb(): Db {
|
||||
return {
|
||||
select: () => ({
|
||||
from: () => ({
|
||||
where: () => ({
|
||||
limit: () => Promise.resolve([{
|
||||
id: workProductId,
|
||||
status: "approved",
|
||||
reviewState: "approved",
|
||||
}]),
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
} as unknown as Db;
|
||||
}
|
||||
|
||||
function result(status: "satisfied" | "not_satisfied", objectiveSatisfied: boolean) {
|
||||
return {
|
||||
reportedWorkDisposition: "done",
|
||||
summary: "Classified",
|
||||
completionClaim: {
|
||||
contractRevision: "1",
|
||||
objectiveSatisfied,
|
||||
criteria: [{ criterionId: "objective", status, evidenceRefs: [evidenceRef] }],
|
||||
remainingWork: [],
|
||||
},
|
||||
verification: [{ commandOrCheck: "test", status: "passed", artifactRef: evidenceRef }],
|
||||
};
|
||||
}
|
||||
|
||||
const input = {
|
||||
db: evidenceDb(),
|
||||
companyId: "company",
|
||||
issueId: "issue",
|
||||
runId: "run",
|
||||
contract: {
|
||||
revision: "1",
|
||||
objective: "Classify evidence",
|
||||
criteria: [{ id: "objective", requirement: "Use accepted durable evidence" }],
|
||||
},
|
||||
};
|
||||
|
||||
describe("classifyNativeEvidence", () => {
|
||||
it("accepts a satisfied claim backed by an approved durable work product", async () => {
|
||||
await expect(classifyNativeEvidence({
|
||||
...input,
|
||||
result: result("satisfied", true),
|
||||
})).resolves.toEqual(expect.objectContaining({
|
||||
objectiveSatisfied: true,
|
||||
allCriteriaSatisfied: true,
|
||||
verificationPassed: true,
|
||||
acceptedEvidenceRefs: [evidenceRef],
|
||||
}));
|
||||
});
|
||||
|
||||
it("does not turn durable evidence into completion when the claim says the criterion is unsatisfied", async () => {
|
||||
const assessment = await classifyNativeEvidence({
|
||||
...input,
|
||||
result: result("not_satisfied", false),
|
||||
});
|
||||
expect(assessment).toEqual(expect.objectContaining({
|
||||
objectiveSatisfied: false,
|
||||
allCriteriaSatisfied: false,
|
||||
verificationPassed: true,
|
||||
}));
|
||||
expect(assessment.criterionAssessments).toEqual([
|
||||
expect.objectContaining({ outcome: "rejected", reasonCode: "criterion_reported_not_satisfied" }),
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -1,3 +1,24 @@
|
||||
import { and, eq, or } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import {
|
||||
approvals,
|
||||
heartbeatRunEvents,
|
||||
issueApprovals,
|
||||
issueAttachments,
|
||||
issueThreadInteractions,
|
||||
issueWorkProducts,
|
||||
} from "@paperclipai/db";
|
||||
|
||||
export type NativeEvidenceOutcome = "accepted" | "missing" | "rejected" | "unverifiable";
|
||||
|
||||
export interface NativeEvidenceRefAssessment {
|
||||
ref: string;
|
||||
kind: "event" | "work_product" | "approval" | "interaction" | "attachment" | "unknown";
|
||||
outcome: NativeEvidenceOutcome;
|
||||
reasonCode: string;
|
||||
durableRecordId: string | null;
|
||||
}
|
||||
|
||||
export interface NativeEvidenceAssessment {
|
||||
objectiveSatisfied: boolean;
|
||||
allCriteriaSatisfied: boolean;
|
||||
@@ -5,7 +26,36 @@ export interface NativeEvidenceAssessment {
|
||||
hasBlockingRemainingWork: boolean;
|
||||
reportedDisposition: "done" | "blocked" | "needs_review" | "yielded";
|
||||
summary: string;
|
||||
blocker: { unblockAction: string; boardOwned: boolean } | null;
|
||||
contractRevisionMatches: boolean;
|
||||
criterionAssessments: Array<{
|
||||
criterionId: string;
|
||||
claimStatus: "satisfied" | "not_satisfied" | "unknown" | null;
|
||||
outcome: NativeEvidenceOutcome;
|
||||
evidenceRefs: NativeEvidenceRefAssessment[];
|
||||
reasonCode: string;
|
||||
}>;
|
||||
verificationAssessments: Array<{
|
||||
commandOrCheck: string;
|
||||
claimStatus: "passed" | "failed" | "not_run";
|
||||
outcome: NativeEvidenceOutcome;
|
||||
evidenceRef: NativeEvidenceRefAssessment | null;
|
||||
reasonCode: string;
|
||||
}>;
|
||||
acceptedEvidenceRefs: string[];
|
||||
missingRequirements: string[];
|
||||
rejectedEvidence: Array<{ ref: string; reasonCode: string }>;
|
||||
unverifiableEvidence: Array<{ ref: string; reasonCode: string }>;
|
||||
blocker: {
|
||||
unblockAction: string;
|
||||
boardOwned: boolean;
|
||||
scope: "current_track" | "task_wide";
|
||||
} | null;
|
||||
continuation: {
|
||||
kind: "same_agent" | "retry" | "delegated_issue" | "response_wake" | "monitor";
|
||||
summary: string;
|
||||
idempotencyKey: string;
|
||||
} | null;
|
||||
attentionRequests: Record<string, unknown>[];
|
||||
}
|
||||
|
||||
function record(value: unknown): Record<string, unknown> {
|
||||
@@ -14,29 +64,292 @@ function record(value: unknown): Record<string, unknown> {
|
||||
: {};
|
||||
}
|
||||
|
||||
export function classifyNativeEvidence(result: Record<string, unknown>): NativeEvidenceAssessment {
|
||||
const claim = record(result.completionClaim);
|
||||
const criteria = Array.isArray(claim.criteria) ? claim.criteria.map(record) : [];
|
||||
function text(value: unknown): string | null {
|
||||
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
|
||||
}
|
||||
|
||||
function uuidRef(ref: string, prefixes: string[]): string | null {
|
||||
for (const prefix of prefixes) {
|
||||
if (ref.startsWith(`${prefix}:`)) return text(ref.slice(prefix.length + 1));
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function classifyEvidenceRef(input: {
|
||||
db: Db;
|
||||
companyId: string;
|
||||
issueId: string;
|
||||
runId: string;
|
||||
ref: string;
|
||||
}): Promise<NativeEvidenceRefAssessment> {
|
||||
const eventMatch = /^event:(\d+)$/.exec(input.ref);
|
||||
if (eventMatch) {
|
||||
const cursor = Number(eventMatch[1]);
|
||||
const event = await input.db.select({
|
||||
id: heartbeatRunEvents.id,
|
||||
eventType: heartbeatRunEvents.eventType,
|
||||
payload: heartbeatRunEvents.payload,
|
||||
})
|
||||
.from(heartbeatRunEvents)
|
||||
.where(and(
|
||||
eq(heartbeatRunEvents.companyId, input.companyId),
|
||||
eq(heartbeatRunEvents.runId, input.runId),
|
||||
or(eq(heartbeatRunEvents.seq, cursor), eq(heartbeatRunEvents.sourceSeq, cursor)),
|
||||
)).limit(1).then((rows) => rows[0] ?? null);
|
||||
if (!event) {
|
||||
return { ref: input.ref, kind: "event", outcome: "missing", reasonCode: "durable_run_event_missing", durableRecordId: null };
|
||||
}
|
||||
const prpEvent = record(record(event.payload).prpEvent);
|
||||
const evidenceOutcome = text(record(prpEvent.payload).evidenceOutcome);
|
||||
if (prpEvent.sourceKind === "control_plane" && evidenceOutcome === "accepted") {
|
||||
return { ref: input.ref, kind: "event", outcome: "accepted", reasonCode: "control_plane_evidence_accepted", durableRecordId: String(event.id) };
|
||||
}
|
||||
if (prpEvent.sourceKind === "control_plane" && evidenceOutcome === "rejected") {
|
||||
return { ref: input.ref, kind: "event", outcome: "rejected", reasonCode: "control_plane_evidence_rejected", durableRecordId: String(event.id) };
|
||||
}
|
||||
return {
|
||||
ref: input.ref,
|
||||
kind: "event",
|
||||
outcome: "unverifiable",
|
||||
reasonCode: event.eventType === "run.result.proposed"
|
||||
? "model_result_event_is_claim_only"
|
||||
: "run_event_has_no_authoritative_evidence_verdict",
|
||||
durableRecordId: String(event.id),
|
||||
};
|
||||
}
|
||||
|
||||
const workProductId = uuidRef(input.ref, ["work_product", "work-product", "artifact"]);
|
||||
if (workProductId) {
|
||||
const row = await input.db.select({
|
||||
id: issueWorkProducts.id,
|
||||
status: issueWorkProducts.status,
|
||||
reviewState: issueWorkProducts.reviewState,
|
||||
}).from(issueWorkProducts).where(and(
|
||||
eq(issueWorkProducts.id, workProductId),
|
||||
eq(issueWorkProducts.companyId, input.companyId),
|
||||
eq(issueWorkProducts.issueId, input.issueId),
|
||||
)).limit(1).then((rows) => rows[0] ?? null);
|
||||
if (!row) return { ref: input.ref, kind: "work_product", outcome: "missing", reasonCode: "work_product_missing", durableRecordId: null };
|
||||
if (
|
||||
["failed", "error", "cancelled", "rejected", "changes_requested"].includes(row.status)
|
||||
|| ["rejected", "changes_requested"].includes(row.reviewState)
|
||||
) {
|
||||
return { ref: input.ref, kind: "work_product", outcome: "rejected", reasonCode: "work_product_rejected", durableRecordId: row.id };
|
||||
}
|
||||
if (
|
||||
row.reviewState === "approved"
|
||||
|| ["approved", "passed", "succeeded", "completed", "merged", "published"].includes(row.status)
|
||||
) {
|
||||
return { ref: input.ref, kind: "work_product", outcome: "accepted", reasonCode: "work_product_authoritatively_accepted", durableRecordId: row.id };
|
||||
}
|
||||
return { ref: input.ref, kind: "work_product", outcome: "unverifiable", reasonCode: "work_product_not_yet_accepted", durableRecordId: row.id };
|
||||
}
|
||||
|
||||
const approvalId = uuidRef(input.ref, ["approval"]);
|
||||
if (approvalId) {
|
||||
const row = await input.db.select({ id: approvals.id, status: approvals.status })
|
||||
.from(issueApprovals)
|
||||
.innerJoin(approvals, eq(issueApprovals.approvalId, approvals.id))
|
||||
.where(and(
|
||||
eq(issueApprovals.companyId, input.companyId),
|
||||
eq(issueApprovals.issueId, input.issueId),
|
||||
eq(approvals.companyId, input.companyId),
|
||||
eq(approvals.id, approvalId),
|
||||
)).limit(1).then((rows) => rows[0] ?? null);
|
||||
if (!row) return { ref: input.ref, kind: "approval", outcome: "missing", reasonCode: "approval_missing", durableRecordId: null };
|
||||
if (row.status === "approved") return { ref: input.ref, kind: "approval", outcome: "accepted", reasonCode: "approval_approved", durableRecordId: row.id };
|
||||
if (["rejected", "cancelled"].includes(row.status)) return { ref: input.ref, kind: "approval", outcome: "rejected", reasonCode: "approval_rejected", durableRecordId: row.id };
|
||||
return { ref: input.ref, kind: "approval", outcome: "missing", reasonCode: "approval_pending", durableRecordId: row.id };
|
||||
}
|
||||
|
||||
const interactionId = uuidRef(input.ref, ["interaction"]);
|
||||
if (interactionId) {
|
||||
const row = await input.db.select({
|
||||
id: issueThreadInteractions.id,
|
||||
status: issueThreadInteractions.status,
|
||||
result: issueThreadInteractions.result,
|
||||
}).from(issueThreadInteractions).where(and(
|
||||
eq(issueThreadInteractions.id, interactionId),
|
||||
eq(issueThreadInteractions.companyId, input.companyId),
|
||||
eq(issueThreadInteractions.issueId, input.issueId),
|
||||
)).limit(1).then((rows) => rows[0] ?? null);
|
||||
if (!row) return { ref: input.ref, kind: "interaction", outcome: "missing", reasonCode: "interaction_missing", durableRecordId: null };
|
||||
const outcome = text(record(row.result).outcome);
|
||||
if (["accepted", "answered", "resolved"].includes(outcome ?? "") || ["accepted", "answered"].includes(row.status)) {
|
||||
return { ref: input.ref, kind: "interaction", outcome: "accepted", reasonCode: "interaction_resolved", durableRecordId: row.id };
|
||||
}
|
||||
if (["rejected", "expired", "cancelled"].includes(row.status) || ["rejected", "withdrawn", "issue_closed"].includes(outcome ?? "")) {
|
||||
return { ref: input.ref, kind: "interaction", outcome: "rejected", reasonCode: "interaction_rejected", durableRecordId: row.id };
|
||||
}
|
||||
return { ref: input.ref, kind: "interaction", outcome: "missing", reasonCode: "interaction_pending", durableRecordId: row.id };
|
||||
}
|
||||
|
||||
const attachmentId = uuidRef(input.ref, ["attachment"]);
|
||||
if (attachmentId) {
|
||||
const row = await input.db.select({ id: issueAttachments.id }).from(issueAttachments).where(and(
|
||||
eq(issueAttachments.id, attachmentId),
|
||||
eq(issueAttachments.companyId, input.companyId),
|
||||
eq(issueAttachments.issueId, input.issueId),
|
||||
)).limit(1).then((rows) => rows[0] ?? null);
|
||||
return row
|
||||
? { ref: input.ref, kind: "attachment", outcome: "accepted", reasonCode: "attachment_persisted", durableRecordId: row.id }
|
||||
: { ref: input.ref, kind: "attachment", outcome: "missing", reasonCode: "attachment_missing", durableRecordId: null };
|
||||
}
|
||||
|
||||
return {
|
||||
ref: input.ref,
|
||||
kind: "unknown",
|
||||
outcome: "unverifiable",
|
||||
reasonCode: "unsupported_evidence_reference",
|
||||
durableRecordId: null,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Classifies model claims against immutable server-owned records. A model's
|
||||
* `satisfied` or `passed` value is retained as a claim but never becomes proof.
|
||||
*/
|
||||
export async function classifyNativeEvidence(input: {
|
||||
db: Db;
|
||||
companyId: string;
|
||||
issueId: string;
|
||||
runId: string;
|
||||
contract: Record<string, unknown>;
|
||||
result: Record<string, unknown>;
|
||||
}): Promise<NativeEvidenceAssessment> {
|
||||
const claim = record(input.result.completionClaim);
|
||||
const claimedCriteria = Array.isArray(claim.criteria) ? claim.criteria.map(record) : [];
|
||||
const contractCriteria = Array.isArray(input.contract.criteria) ? input.contract.criteria.map(record) : [];
|
||||
const remaining = Array.isArray(claim.remainingWork) ? claim.remainingWork.map(record) : [];
|
||||
const verification = Array.isArray(result.verification) ? result.verification.map(record) : [];
|
||||
const reported = result.reportedWorkDisposition;
|
||||
const blocker = record(result.blocker);
|
||||
const verification = Array.isArray(input.result.verification) ? input.result.verification.map(record) : [];
|
||||
const reported = input.result.reportedWorkDisposition;
|
||||
const blocker = record(input.result.blocker);
|
||||
const blockerOwner = record(blocker.owner);
|
||||
const continuation = record(input.result.continuation);
|
||||
if (!["done", "blocked", "needs_review", "yielded"].includes(String(reported))) {
|
||||
throw new Error("native_result_invalid_disposition");
|
||||
}
|
||||
|
||||
const contractRevision = text(input.contract.revision);
|
||||
const contractRevisionMatches = contractRevision !== null && text(claim.contractRevision) === contractRevision;
|
||||
const cache = new Map<string, Promise<NativeEvidenceRefAssessment>>();
|
||||
const resolveRef = (ref: string) => {
|
||||
let pending = cache.get(ref);
|
||||
if (!pending) {
|
||||
pending = classifyEvidenceRef({ ...input, ref });
|
||||
cache.set(ref, pending);
|
||||
}
|
||||
return pending;
|
||||
};
|
||||
|
||||
const criterionAssessments = await Promise.all(contractCriteria.map(async (criterion) => {
|
||||
const criterionId = text(criterion.id) ?? "invalid-contract-criterion";
|
||||
const claimed = claimedCriteria.find((entry) => text(entry.criterionId) === criterionId) ?? null;
|
||||
const claimStatus = claimed && ["satisfied", "not_satisfied", "unknown"].includes(String(claimed.status))
|
||||
? claimed.status as "satisfied" | "not_satisfied" | "unknown"
|
||||
: null;
|
||||
const refs = claimed && Array.isArray(claimed.evidenceRefs)
|
||||
? claimed.evidenceRefs.flatMap((value) => text(value) ? [text(value)!] : [])
|
||||
: [];
|
||||
const evidenceRefs = await Promise.all(refs.map(resolveRef));
|
||||
let outcome: NativeEvidenceOutcome;
|
||||
let reasonCode: string;
|
||||
if (!contractRevisionMatches) {
|
||||
outcome = "rejected";
|
||||
reasonCode = "contract_revision_stale";
|
||||
} else if (!claimed) {
|
||||
outcome = "missing";
|
||||
reasonCode = "criterion_claim_missing";
|
||||
} else if (claimStatus === "not_satisfied") {
|
||||
outcome = "rejected";
|
||||
reasonCode = "criterion_reported_not_satisfied";
|
||||
} else if (claimStatus !== "satisfied") {
|
||||
outcome = "unverifiable";
|
||||
reasonCode = "criterion_claim_not_satisfied";
|
||||
} else if (refs.length === 0) {
|
||||
outcome = "missing";
|
||||
reasonCode = "criterion_evidence_missing";
|
||||
} else if (evidenceRefs.some((entry) => entry.outcome === "rejected")) {
|
||||
outcome = "rejected";
|
||||
reasonCode = "criterion_evidence_rejected";
|
||||
} else if (evidenceRefs.some((entry) => entry.outcome === "unverifiable")) {
|
||||
outcome = "unverifiable";
|
||||
reasonCode = "criterion_evidence_unverifiable";
|
||||
} else if (evidenceRefs.some((entry) => entry.outcome === "missing")) {
|
||||
outcome = "missing";
|
||||
reasonCode = "criterion_evidence_missing";
|
||||
} else {
|
||||
outcome = "accepted";
|
||||
reasonCode = "criterion_evidence_accepted";
|
||||
}
|
||||
return { criterionId, claimStatus, outcome, evidenceRefs, reasonCode };
|
||||
}));
|
||||
|
||||
const verificationAssessments = await Promise.all(verification.map(async (entry, index) => {
|
||||
const commandOrCheck = text(entry.commandOrCheck) ?? `verification-${index + 1}`;
|
||||
const claimStatus = ["passed", "failed", "not_run"].includes(String(entry.status))
|
||||
? entry.status as "passed" | "failed" | "not_run"
|
||||
: "not_run";
|
||||
const ref = text(entry.artifactRef);
|
||||
const evidenceRef = ref ? await resolveRef(ref) : null;
|
||||
if (claimStatus === "failed") return { commandOrCheck, claimStatus, outcome: "rejected" as const, evidenceRef, reasonCode: "verification_reported_failed" };
|
||||
if (claimStatus === "not_run") return { commandOrCheck, claimStatus, outcome: "missing" as const, evidenceRef, reasonCode: "verification_not_run" };
|
||||
if (!evidenceRef) return { commandOrCheck, claimStatus, outcome: "unverifiable" as const, evidenceRef, reasonCode: "verification_has_no_durable_reference" };
|
||||
return {
|
||||
commandOrCheck,
|
||||
claimStatus,
|
||||
outcome: evidenceRef.outcome,
|
||||
evidenceRef,
|
||||
reasonCode: evidenceRef.outcome === "accepted" ? "verification_evidence_accepted" : evidenceRef.reasonCode,
|
||||
};
|
||||
}));
|
||||
|
||||
const allRefs = [...cache.keys()].map((ref) => cache.get(ref)!);
|
||||
const resolvedRefs = await Promise.all(allRefs);
|
||||
const acceptedEvidenceRefs = resolvedRefs.filter((entry) => entry.outcome === "accepted").map((entry) => entry.ref);
|
||||
const missingRequirements = [
|
||||
...criterionAssessments.filter((entry) => entry.outcome === "missing").map((entry) => entry.criterionId),
|
||||
...verificationAssessments.filter((entry) => entry.outcome === "missing").map((entry) => entry.commandOrCheck),
|
||||
];
|
||||
const rejectedEvidence = resolvedRefs.filter((entry) => entry.outcome === "rejected").map((entry) => ({ ref: entry.ref, reasonCode: entry.reasonCode }));
|
||||
const unverifiableEvidence = resolvedRefs.filter((entry) => entry.outcome === "unverifiable").map((entry) => ({ ref: entry.ref, reasonCode: entry.reasonCode }));
|
||||
const allCriteriaSatisfied = contractCriteria.length > 0 && criterionAssessments.every((entry) => entry.outcome === "accepted");
|
||||
const verificationPassed = verificationAssessments.length > 0 && verificationAssessments.every((entry) => entry.outcome === "accepted");
|
||||
const continuationKind = continuation.kind;
|
||||
|
||||
return {
|
||||
objectiveSatisfied: claim.objectiveSatisfied === true,
|
||||
allCriteriaSatisfied: criteria.length > 0 && criteria.every((criterion) => criterion.status === "satisfied"),
|
||||
verificationPassed: verification.length > 0 && verification.every((entry) => entry.status === "passed"),
|
||||
objectiveSatisfied: claim.objectiveSatisfied === true && allCriteriaSatisfied,
|
||||
allCriteriaSatisfied,
|
||||
verificationPassed,
|
||||
hasBlockingRemainingWork: remaining.some((entry) => entry.blocksCompletion === true),
|
||||
reportedDisposition: reported as NativeEvidenceAssessment["reportedDisposition"],
|
||||
summary: typeof result.summary === "string" ? result.summary : "Native run completed.",
|
||||
summary: typeof input.result.summary === "string" ? input.result.summary : "Native run completed.",
|
||||
contractRevisionMatches,
|
||||
criterionAssessments,
|
||||
verificationAssessments,
|
||||
acceptedEvidenceRefs,
|
||||
missingRequirements,
|
||||
rejectedEvidence,
|
||||
unverifiableEvidence,
|
||||
blocker: reported === "blocked" && typeof blocker.unblockAction === "string"
|
||||
? {
|
||||
unblockAction: blocker.unblockAction,
|
||||
boardOwned: blockerOwner.kind === "board" || blockerOwner.owner === "board",
|
||||
scope: blocker.scope === "task_wide" ? "task_wide" : "current_track",
|
||||
}
|
||||
: null,
|
||||
continuation: reported === "yielded"
|
||||
&& ["same_agent", "retry", "delegated_issue", "response_wake", "monitor"].includes(String(continuationKind))
|
||||
&& text(continuation.summary)
|
||||
&& text(continuation.idempotencyKey)
|
||||
? {
|
||||
kind: continuationKind as NativeEvidenceAssessment["continuation"] extends infer T ? T extends { kind: infer K } ? K : never : never,
|
||||
summary: text(continuation.summary)!,
|
||||
idempotencyKey: text(continuation.idempotencyKey)!,
|
||||
}
|
||||
: null,
|
||||
attentionRequests: Array.isArray(input.result.attentionRequests)
|
||||
? input.result.attentionRequests.map(record)
|
||||
: [],
|
||||
};
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
import { and, desc, eq, inArray, isNotNull, ne } from "drizzle-orm";
|
||||
import { and, desc, eq, inArray, isNotNull, isNull, lte, notInArray, or } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { heartbeatRuns, nativeRunFinalizations, workspaceOperations } from "@paperclipai/db";
|
||||
import { finalizeNativeRun } from "./native-run-finalizer.js";
|
||||
import { finalizeNativeRun, recordNativeFinalizationFailure } from "./native-run-finalizer.js";
|
||||
|
||||
/** Recovery is keyed only by persisted mode/coordinator state, never the live flag. */
|
||||
export async function reconcileNativeFinalizations(db: Db, runIds?: string[]) {
|
||||
@@ -11,7 +11,16 @@ export async function reconcileNativeFinalizations(db: Db, runIds?: string[]) {
|
||||
.where(and(
|
||||
eq(heartbeatRuns.runtimeMode, "native"),
|
||||
isNotNull(nativeRunFinalizations.resultId),
|
||||
ne(nativeRunFinalizations.phase, "applied"),
|
||||
notInArray(nativeRunFinalizations.phase, ["terminal_failure"]),
|
||||
or(
|
||||
isNull(nativeRunFinalizations.nextAttemptAt),
|
||||
lte(nativeRunFinalizations.nextAttemptAt, new Date()),
|
||||
),
|
||||
or(
|
||||
isNull(nativeRunFinalizations.leaseOwner),
|
||||
isNull(nativeRunFinalizations.leaseExpiresAt),
|
||||
lte(nativeRunFinalizations.leaseExpiresAt, new Date()),
|
||||
),
|
||||
...(runIds && runIds.length > 0 ? [inArray(heartbeatRuns.id, runIds)] : []),
|
||||
));
|
||||
const results = [];
|
||||
@@ -26,12 +35,21 @@ export async function reconcileNativeFinalizations(db: Db, runIds?: string[]) {
|
||||
.limit(1)
|
||||
.then((entries) => entries[0] ?? null);
|
||||
if (!barrier || !["succeeded", "failed"].includes(barrier.status)) continue;
|
||||
results.push(await finalizeNativeRun({
|
||||
db,
|
||||
runId: row.runId,
|
||||
workspaceFinalizeStatus: barrier.status as "succeeded" | "failed",
|
||||
projectRunStatus: true,
|
||||
}));
|
||||
try {
|
||||
results.push(await finalizeNativeRun({
|
||||
db,
|
||||
runId: row.runId,
|
||||
workspaceFinalizeStatus: barrier.status as "succeeded" | "failed",
|
||||
projectRunStatus: true,
|
||||
}));
|
||||
} catch (error) {
|
||||
results.push(await recordNativeFinalizationFailure({
|
||||
db,
|
||||
runId: row.runId,
|
||||
error,
|
||||
projectRunStatus: true,
|
||||
}));
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
@@ -1,7 +1,9 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { and, eq, inArray } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import {
|
||||
approvals,
|
||||
completionContracts,
|
||||
heartbeatRuns,
|
||||
issueApprovals,
|
||||
issueThreadInteractions,
|
||||
@@ -10,9 +12,20 @@ import {
|
||||
nativeRunResults,
|
||||
} from "@paperclipai/db";
|
||||
import { classifyNativeEvidence } from "./evidence-classifier.js";
|
||||
import { arbitrateNativeStatus, NATIVE_STATUS_ARBITER_POLICY_VERSION } from "./status-arbiter.js";
|
||||
import {
|
||||
arbitrateNativeStatus,
|
||||
NATIVE_STATUS_ARBITER_POLICY_VERSION,
|
||||
type NativeAuthoritativeIssueStatus,
|
||||
type NativeGovernanceGate,
|
||||
} from "./status-arbiter.js";
|
||||
import { recordNativeWorkAssessment } from "./work-assessments.js";
|
||||
import { commitNativeStatusDecision, NativeStatusRaceError } from "./status-decision-committer.js";
|
||||
import {
|
||||
commitNativeStatusDecision,
|
||||
NativeStatusRaceError,
|
||||
type NativeStatusCommitFailpoint,
|
||||
} from "./status-decision-committer.js";
|
||||
import { issueRecoveryActionService } from "../issue-recovery-actions.js";
|
||||
import { nativeSha256 } from "./canonical.js";
|
||||
|
||||
function record(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value)
|
||||
@@ -20,16 +33,25 @@ function record(value: unknown): Record<string, unknown> {
|
||||
: {};
|
||||
}
|
||||
|
||||
async function hasPendingNativeGovernance(input: {
|
||||
function authoritativeStatus(value: string): NativeAuthoritativeIssueStatus {
|
||||
if (!["backlog", "todo", "in_progress", "in_review", "blocked", "done", "cancelled"].includes(value)) {
|
||||
throw new Error("native_issue_status_invalid");
|
||||
}
|
||||
return value as NativeAuthoritativeIssueStatus;
|
||||
}
|
||||
|
||||
async function pendingNativeGovernance(input: {
|
||||
db: Db;
|
||||
companyId: string;
|
||||
issueId: string;
|
||||
runId: string;
|
||||
executionState: Record<string, unknown> | null;
|
||||
}) {
|
||||
// Issue-thread interactions and execution-policy stages are independent
|
||||
// governance authorities from formal approvals. Native completion must
|
||||
// mediate all three before it may project an authoritative terminal status.
|
||||
if (record(input.executionState).status === "pending") return true;
|
||||
}): Promise<NativeGovernanceGate | null> {
|
||||
// Execution policy is issue-owned state and takes priority over the two
|
||||
// durable review surfaces. Completion must mediate all three authorities.
|
||||
if (record(input.executionState).status === "pending") {
|
||||
return { kind: "execution_stage", id: input.runId };
|
||||
}
|
||||
const [pendingInteraction, pendingApproval] = await Promise.all([
|
||||
input.db.select({ id: issueThreadInteractions.id })
|
||||
.from(issueThreadInteractions)
|
||||
@@ -52,7 +74,174 @@ async function hasPendingNativeGovernance(input: {
|
||||
.limit(1)
|
||||
.then((rows) => rows[0] ?? null),
|
||||
]);
|
||||
return Boolean(pendingInteraction || pendingApproval);
|
||||
if (pendingInteraction) return { kind: "interaction", id: pendingInteraction.id };
|
||||
if (pendingApproval) return { kind: "approval", id: pendingApproval.id };
|
||||
return null;
|
||||
}
|
||||
|
||||
async function claimCoordinator(input: { db: Db; runId: string }) {
|
||||
const leaseOwner = `native-finalizer:${randomUUID()}`;
|
||||
const now = new Date();
|
||||
const claimed = await input.db.transaction(async (tx) => {
|
||||
const coordinator = await tx.select().from(nativeRunFinalizations)
|
||||
.where(eq(nativeRunFinalizations.runId, input.runId)).for("update").limit(1)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
if (!coordinator?.resultId) throw new Error("native_finalization_missing");
|
||||
if (coordinator.phase === "committed" && coordinator.decisionId) {
|
||||
return { coordinator, leaseOwner: null };
|
||||
}
|
||||
if (coordinator.phase === "terminal_failure") throw new Error("native_finalization_terminal_failure");
|
||||
if (
|
||||
coordinator.leaseOwner
|
||||
&& coordinator.leaseExpiresAt
|
||||
&& coordinator.leaseExpiresAt > now
|
||||
&& coordinator.leaseOwner !== leaseOwner
|
||||
) throw new Error("native_finalization_lease_busy");
|
||||
const [updated] = await tx.update(nativeRunFinalizations).set({
|
||||
leaseOwner,
|
||||
leaseExpiresAt: new Date(now.getTime() + 5 * 60_000),
|
||||
attempt: coordinator.attempt + 1,
|
||||
phase: coordinator.phase === "retryable_failure"
|
||||
? coordinator.assessmentId ? "arbitrating" : "workspace_finalizing"
|
||||
: coordinator.phase,
|
||||
failureCode: null,
|
||||
failureDetail: null,
|
||||
nextAttemptAt: null,
|
||||
updatedAt: now,
|
||||
}).where(eq(nativeRunFinalizations.runId, input.runId)).returning();
|
||||
if (!updated) throw new Error("native_finalization_claim_failed");
|
||||
return { coordinator: updated, leaseOwner };
|
||||
});
|
||||
return claimed;
|
||||
}
|
||||
|
||||
async function recordRetryableFailure(input: {
|
||||
db: Db;
|
||||
run: typeof heartbeatRuns.$inferSelect;
|
||||
coordinator: typeof nativeRunFinalizations.$inferSelect;
|
||||
failureCode: string;
|
||||
message: string;
|
||||
nextAction: string;
|
||||
projectRunStatus?: boolean;
|
||||
}) {
|
||||
const now = new Date();
|
||||
const exhausted = input.coordinator.attempt >= 3;
|
||||
const nextAttemptAt = new Date(now.getTime() + 30_000);
|
||||
const phase = exhausted ? "terminal_failure" : "retryable_failure";
|
||||
const failureCode = exhausted ? "native_finalization_retry_exhausted" : input.failureCode;
|
||||
await input.db.transaction(async (tx) => {
|
||||
await tx.update(nativeRunFinalizations).set({
|
||||
phase,
|
||||
leaseOwner: null,
|
||||
leaseExpiresAt: null,
|
||||
failureCode,
|
||||
failureDetail: {
|
||||
message: input.message.slice(0, 2_000),
|
||||
originalFailureCode: input.failureCode,
|
||||
recoveryOwner: { kind: "agent", agentId: input.run.agentId },
|
||||
nextAction: input.nextAction,
|
||||
},
|
||||
nextAttemptAt: exhausted ? null : nextAttemptAt,
|
||||
updatedAt: now,
|
||||
}).where(eq(nativeRunFinalizations.runId, input.run.id));
|
||||
await tx.update(heartbeatRuns).set({
|
||||
...(input.projectRunStatus ? { status: "failed", finishedAt: now } : {}),
|
||||
nativePhase: phase,
|
||||
nativePhaseUpdatedAt: now,
|
||||
resultJson: {
|
||||
...record(input.run.resultJson),
|
||||
finalizationPhase: phase,
|
||||
failureCode,
|
||||
originalFailureCode: input.failureCode,
|
||||
nextAttemptAt: exhausted ? null : nextAttemptAt.toISOString(),
|
||||
},
|
||||
updatedAt: now,
|
||||
}).where(eq(heartbeatRuns.id, input.run.id));
|
||||
await issueRecoveryActionService(tx as unknown as Db).upsertSourceScoped({
|
||||
companyId: input.run.companyId,
|
||||
sourceIssueId: input.coordinator.issueId,
|
||||
kind: "active_run_watchdog",
|
||||
ownerType: "agent",
|
||||
ownerAgentId: input.run.agentId,
|
||||
cause: failureCode,
|
||||
fingerprint: nativeSha256({ runId: input.run.id, failureCode }),
|
||||
evidence: {
|
||||
runId: input.run.id,
|
||||
coordinatorAttempt: input.coordinator.attempt,
|
||||
originalFailureCode: input.failureCode,
|
||||
},
|
||||
nextAction: exhausted
|
||||
? `Finalization retry budget exhausted. ${input.nextAction}`
|
||||
: input.nextAction,
|
||||
wakePolicy: exhausted
|
||||
? null
|
||||
: { kind: "resume_native_run", runId: input.run.id, notBefore: nextAttemptAt.toISOString() },
|
||||
maxAttempts: 3,
|
||||
});
|
||||
});
|
||||
return {
|
||||
...input.coordinator,
|
||||
phase,
|
||||
failureCode,
|
||||
nextAttemptAt: exhausted ? null : nextAttemptAt,
|
||||
};
|
||||
}
|
||||
|
||||
/** Converts malformed or incomplete persisted finalization state into named recovery. */
|
||||
export async function recordNativeFinalizationFailure(input: {
|
||||
db: Db;
|
||||
runId: string;
|
||||
error: unknown;
|
||||
projectRunStatus?: boolean;
|
||||
}) {
|
||||
const [run, coordinator] = await Promise.all([
|
||||
input.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId))
|
||||
.limit(1).then((rows) => rows[0] ?? null),
|
||||
input.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, input.runId))
|
||||
.limit(1).then((rows) => rows[0] ?? null),
|
||||
]);
|
||||
if (!run || run.runtimeMode !== "native" || !coordinator) throw input.error;
|
||||
const message = input.error instanceof Error ? input.error.message : String(input.error);
|
||||
const failureCode = message.startsWith("native_") ? message : "native_finalization_invalid";
|
||||
return recordRetryableFailure({
|
||||
db: input.db,
|
||||
run,
|
||||
coordinator,
|
||||
failureCode,
|
||||
message,
|
||||
nextAction: "Repair the persisted native result or contract discriminator, then resume finalization from the coordinator.",
|
||||
projectRunStatus: input.projectRunStatus,
|
||||
});
|
||||
}
|
||||
|
||||
async function projectCommittedRun(input: {
|
||||
db: Db;
|
||||
run: typeof heartbeatRuns.$inferSelect;
|
||||
coordinator: typeof nativeRunFinalizations.$inferSelect;
|
||||
}) {
|
||||
if (!input.coordinator.resultId) return;
|
||||
const resultRow = await input.db.select({ resultJson: nativeRunResults.resultJson })
|
||||
.from(nativeRunResults).where(and(
|
||||
eq(nativeRunResults.id, input.coordinator.resultId),
|
||||
eq(nativeRunResults.runId, input.run.id),
|
||||
eq(nativeRunResults.companyId, input.run.companyId),
|
||||
)).limit(1).then((rows) => rows[0] ?? null);
|
||||
const terminalState = record(record(resultRow?.resultJson).terminal).runTerminalState;
|
||||
if (!["succeeded", "failed", "cancelled"].includes(String(terminalState))) {
|
||||
throw new Error("native_finalization_invalid");
|
||||
}
|
||||
const now = new Date();
|
||||
await input.db.update(heartbeatRuns).set({
|
||||
status: terminalState as "succeeded" | "failed" | "cancelled",
|
||||
finishedAt: input.run.finishedAt ?? now,
|
||||
nativePhase: "committed",
|
||||
nativePhaseUpdatedAt: now,
|
||||
updatedAt: now,
|
||||
}).where(and(
|
||||
eq(heartbeatRuns.id, input.run.id),
|
||||
eq(heartbeatRuns.runtimeMode, "native"),
|
||||
inArray(heartbeatRuns.status, ["queued", "running", "failed"]),
|
||||
));
|
||||
}
|
||||
|
||||
export async function finalizeNativeRun(input: {
|
||||
@@ -61,40 +250,44 @@ export async function finalizeNativeRun(input: {
|
||||
workspaceFinalizeStatus: "succeeded" | "failed";
|
||||
/** Reconciliation owns terminal run projection; the live heartbeat does it afterward. */
|
||||
projectRunStatus?: boolean;
|
||||
failpoint?: NativeStatusCommitFailpoint;
|
||||
}) {
|
||||
const run = await input.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId))
|
||||
.limit(1).then((rows) => rows[0] ?? null);
|
||||
if (!run || run.runtimeMode !== "native") throw new Error("native_finalization_run_missing");
|
||||
const coordinator = await input.db.select().from(nativeRunFinalizations)
|
||||
.where(eq(nativeRunFinalizations.runId, input.runId)).limit(1)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
if (!coordinator?.resultId) throw new Error("native_finalization_missing");
|
||||
if (coordinator.phase === "applied" && coordinator.decisionId) return coordinator;
|
||||
if (input.workspaceFinalizeStatus === "failed") {
|
||||
const now = new Date();
|
||||
await input.db.update(nativeRunFinalizations).set({
|
||||
phase: "workspace_failed",
|
||||
failureCode: "workspace_finalization_failed",
|
||||
failureDetail: "Native result preserved; issue status intentionally unchanged.",
|
||||
updatedAt: now,
|
||||
}).where(eq(nativeRunFinalizations.runId, input.runId));
|
||||
await input.db.update(heartbeatRuns).set({
|
||||
...(input.projectRunStatus ? { status: "failed", finishedAt: now } : {}),
|
||||
nativePhase: "workspace_failed",
|
||||
nativePhaseUpdatedAt: now,
|
||||
resultJson: {
|
||||
...record(run.resultJson),
|
||||
finalizationPhase: "workspace_failed",
|
||||
workspaceFinalizeStatus: "failed",
|
||||
},
|
||||
updatedAt: now,
|
||||
}).where(eq(heartbeatRuns.id, run.id));
|
||||
return { ...coordinator, phase: "workspace_failed" };
|
||||
const claim = await claimCoordinator({ db: input.db, runId: input.runId });
|
||||
const coordinator = claim.coordinator;
|
||||
if (!claim.leaseOwner && coordinator.phase === "committed") {
|
||||
if (input.projectRunStatus) await projectCommittedRun({ db: input.db, run, coordinator });
|
||||
return coordinator;
|
||||
}
|
||||
const resultRow = await input.db.select().from(nativeRunResults)
|
||||
.where(eq(nativeRunResults.id, coordinator.resultId)).limit(1)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
if (input.workspaceFinalizeStatus === "failed") {
|
||||
return recordRetryableFailure({
|
||||
db: input.db,
|
||||
run,
|
||||
coordinator,
|
||||
failureCode: "workspace_finalization_failed",
|
||||
message: "Native result preserved; issue status intentionally unchanged.",
|
||||
nextAction: "Repair and re-run workspace finalization for the persisted native result.",
|
||||
projectRunStatus: input.projectRunStatus,
|
||||
});
|
||||
}
|
||||
|
||||
const [resultRow, contractRow] = await Promise.all([
|
||||
input.db.select().from(nativeRunResults)
|
||||
.where(and(
|
||||
eq(nativeRunResults.id, coordinator.resultId!),
|
||||
eq(nativeRunResults.companyId, run.companyId),
|
||||
)).limit(1).then((rows) => rows[0] ?? null),
|
||||
input.db.select().from(completionContracts)
|
||||
.where(and(
|
||||
eq(completionContracts.id, run.completionContractId!),
|
||||
eq(completionContracts.companyId, run.companyId),
|
||||
eq(completionContracts.issueId, coordinator.issueId),
|
||||
)).limit(1).then((rows) => rows[0] ?? null),
|
||||
]);
|
||||
if (!resultRow) throw new Error("native_result_missing");
|
||||
if (!contractRow) throw new Error("native_completion_contract_missing");
|
||||
const envelope = record(resultRow.resultJson);
|
||||
const result = record(envelope.result);
|
||||
const terminal = record(envelope.terminal);
|
||||
@@ -102,28 +295,42 @@ export async function finalizeNativeRun(input: {
|
||||
if (!["succeeded", "failed", "cancelled"].includes(String(terminalState))) {
|
||||
throw new Error("native_finalization_invalid");
|
||||
}
|
||||
const assessment = classifyNativeEvidence(result);
|
||||
|
||||
await input.db.update(nativeRunFinalizations).set({ phase: "ready_for_assessment", updatedAt: new Date() })
|
||||
.where(and(
|
||||
eq(nativeRunFinalizations.runId, input.runId),
|
||||
eq(nativeRunFinalizations.leaseOwner, claim.leaseOwner!),
|
||||
));
|
||||
const assessment = await classifyNativeEvidence({
|
||||
db: input.db,
|
||||
companyId: run.companyId,
|
||||
issueId: coordinator.issueId,
|
||||
runId: run.id,
|
||||
contract: record(contractRow.contractJson),
|
||||
result,
|
||||
});
|
||||
|
||||
let supersedesAssessmentId: string | null = null;
|
||||
for (let attempt = 0; attempt < 2; attempt += 1) {
|
||||
const issue = await input.db.select().from(issues).where(eq(issues.id, coordinator.issueId))
|
||||
.limit(1).then((rows) => rows[0] ?? null);
|
||||
const authoritativeIssue = issue ?? await input.db.select().from(issues)
|
||||
.where(eq(issues.id, coordinator.issueId)).limit(1).then((rows) => rows[0] ?? null);
|
||||
if (!authoritativeIssue || authoritativeIssue.companyId !== run.companyId) {
|
||||
throw new Error("native_finalization_issue_missing");
|
||||
}
|
||||
for (let attempt = 0; attempt < 3; attempt += 1) {
|
||||
const authoritativeIssue = await input.db.select().from(issues).where(and(
|
||||
eq(issues.id, coordinator.issueId),
|
||||
eq(issues.companyId, run.companyId),
|
||||
)).limit(1).then((rows) => rows[0] ?? null);
|
||||
if (!authoritativeIssue) throw new Error("native_finalization_issue_missing");
|
||||
const governanceGate = await pendingNativeGovernance({
|
||||
db: input.db,
|
||||
companyId: run.companyId,
|
||||
issueId: authoritativeIssue.id,
|
||||
runId: run.id,
|
||||
executionState: record(authoritativeIssue.executionState),
|
||||
});
|
||||
const decision = arbitrateNativeStatus({
|
||||
assessment,
|
||||
terminalState: terminalState as "succeeded" | "failed" | "cancelled",
|
||||
workspaceFinalizeStatus: input.workspaceFinalizeStatus,
|
||||
approvalGateSatisfied: !await hasPendingNativeGovernance({
|
||||
db: input.db,
|
||||
companyId: run.companyId,
|
||||
issueId: authoritativeIssue.id,
|
||||
executionState: authoritativeIssue.executionState,
|
||||
}),
|
||||
governanceGate,
|
||||
agentId: run.agentId,
|
||||
priorIssueStatus: authoritativeStatus(authoritativeIssue.status),
|
||||
});
|
||||
const assessmentRow = await recordNativeWorkAssessment({
|
||||
db: input.db,
|
||||
@@ -132,7 +339,9 @@ export async function finalizeNativeRun(input: {
|
||||
runId: run.id,
|
||||
turnId: resultRow.turnId,
|
||||
contractId: resultRow.completionContractId,
|
||||
contractCanonicalSha256: contractRow.canonicalSha256,
|
||||
resultId: resultRow.id,
|
||||
resultCanonicalSha256: resultRow.canonicalSha256,
|
||||
priorIssueStatus: authoritativeIssue.status,
|
||||
priorStatusVersion: Number(authoritativeIssue.statusVersion),
|
||||
priorDecisionId: authoritativeIssue.lastStatusDecisionId,
|
||||
@@ -140,6 +349,14 @@ export async function finalizeNativeRun(input: {
|
||||
assessment,
|
||||
supersedesAssessmentId,
|
||||
});
|
||||
await input.db.update(nativeRunFinalizations).set({
|
||||
phase: "arbitrating",
|
||||
assessmentId: assessmentRow.id,
|
||||
updatedAt: new Date(),
|
||||
}).where(and(
|
||||
eq(nativeRunFinalizations.runId, input.runId),
|
||||
eq(nativeRunFinalizations.leaseOwner, claim.leaseOwner!),
|
||||
));
|
||||
try {
|
||||
const committed = await commitNativeStatusDecision({
|
||||
db: input.db,
|
||||
@@ -149,18 +366,21 @@ export async function finalizeNativeRun(input: {
|
||||
assessmentId: assessmentRow.id,
|
||||
priorStatus: authoritativeIssue.status,
|
||||
priorStatusVersion: Number(authoritativeIssue.statusVersion),
|
||||
priorDecisionId: authoritativeIssue.lastStatusDecisionId,
|
||||
decision,
|
||||
failpoint: input.failpoint,
|
||||
});
|
||||
const now = new Date();
|
||||
await input.db.update(heartbeatRuns).set({
|
||||
...(input.projectRunStatus ? {
|
||||
status: terminalState === "succeeded" ? "succeeded" : terminalState === "cancelled" ? "cancelled" : "failed",
|
||||
finishedAt: new Date(),
|
||||
finishedAt: now,
|
||||
} : {}),
|
||||
nativePhase: "applied",
|
||||
nativePhaseUpdatedAt: new Date(),
|
||||
nativePhase: "committed",
|
||||
nativePhaseUpdatedAt: now,
|
||||
resultJson: {
|
||||
...record(run.resultJson),
|
||||
finalizationPhase: "applied",
|
||||
finalizationPhase: "committed",
|
||||
assessmentId: assessmentRow.id,
|
||||
decisionId: committed.decision.id,
|
||||
authoritativeDecision: decision.toStatus,
|
||||
@@ -170,17 +390,25 @@ export async function finalizeNativeRun(input: {
|
||||
statusVersionAfter: Number(committed.issue.statusVersion),
|
||||
workspaceFinalizeStatus: input.workspaceFinalizeStatus,
|
||||
},
|
||||
updatedAt: new Date(),
|
||||
updatedAt: now,
|
||||
}).where(eq(heartbeatRuns.id, run.id));
|
||||
return {
|
||||
...coordinator,
|
||||
phase: "applied",
|
||||
assessmentId: assessmentRow.id,
|
||||
decisionId: committed.decision.id,
|
||||
};
|
||||
return { ...coordinator, phase: "committed", assessmentId: assessmentRow.id, decisionId: committed.decision.id };
|
||||
} catch (error) {
|
||||
if (!(error instanceof NativeStatusRaceError) || attempt === 1) throw error;
|
||||
supersedesAssessmentId = assessmentRow.id;
|
||||
if (error instanceof NativeStatusRaceError && attempt < 2) {
|
||||
supersedesAssessmentId = assessmentRow.id;
|
||||
continue;
|
||||
}
|
||||
return recordRetryableFailure({
|
||||
db: input.db,
|
||||
run,
|
||||
coordinator,
|
||||
failureCode: error instanceof NativeStatusRaceError ? "status_cas_exhausted" : "side_effect_planning_failed",
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
nextAction: error instanceof NativeStatusRaceError
|
||||
? "Reassess against the latest authoritative issue status version."
|
||||
: "Retry atomic status and liveness materialization from the persisted assessment.",
|
||||
projectRunStatus: input.projectRunStatus,
|
||||
});
|
||||
}
|
||||
}
|
||||
throw new Error("native_finalization_status_race");
|
||||
|
||||
@@ -25,6 +25,38 @@ const execution = {
|
||||
completionContract: { id: "contract", sha256: "sha" },
|
||||
} as NativeExecutionInputV1;
|
||||
|
||||
function leaseDb(): Db {
|
||||
const coordinator = {
|
||||
runId: execution.binding.runId,
|
||||
companyId: execution.binding.companyId,
|
||||
issueId: execution.binding.issueId,
|
||||
phase: "observed",
|
||||
attempt: 0,
|
||||
leaseOwner: null,
|
||||
leaseExpiresAt: null,
|
||||
resultId: null,
|
||||
};
|
||||
const update = () => ({
|
||||
set: () => ({ where: async () => [] }),
|
||||
});
|
||||
const tx = {
|
||||
select: () => ({
|
||||
from: () => ({
|
||||
where: () => ({
|
||||
for: () => ({
|
||||
limit: () => Promise.resolve([coordinator]),
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
update,
|
||||
};
|
||||
return {
|
||||
transaction: async (operation: (transaction: Db) => Promise<unknown>) => operation(tx as unknown as Db),
|
||||
update,
|
||||
} as unknown as Db;
|
||||
}
|
||||
|
||||
describe("native session cancellation", () => {
|
||||
beforeEach(() => {
|
||||
state.cancel.mockReset();
|
||||
@@ -49,14 +81,16 @@ describe("native session cancellation", () => {
|
||||
|
||||
it("routes control-plane cancellation to the active normalized session and removes the handle", async () => {
|
||||
const running = executePaperclipNativeSession({
|
||||
db: {} as Db,
|
||||
db: leaseDb(),
|
||||
execution,
|
||||
runnerInstanceId: "runner",
|
||||
});
|
||||
await vi.waitFor(() => expect(state.release).toBeTypeOf("function"));
|
||||
|
||||
await expect(cancelNativeSession(execution.binding.runId, "budget hard stop")).resolves.toBe(true);
|
||||
await expect(cancelNativeSession(execution.binding.runId, "duplicate budget stop")).resolves.toBe(true);
|
||||
expect(state.cancel).toHaveBeenCalledWith({ reason: "budget hard stop" });
|
||||
expect(state.cancel).toHaveBeenCalledTimes(1);
|
||||
|
||||
state.release?.();
|
||||
await running;
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type { AdapterExecutionResult } from "../../adapters/index.js";
|
||||
import type { NativeFinalizationResult } from "@paperclipai/shared";
|
||||
import type { NativeExecutionInputV1, NativeSession } from "@paperclipai/paperclip-runner";
|
||||
@@ -6,15 +7,24 @@ import {
|
||||
executeNativeSession,
|
||||
} from "@paperclipai/paperclip-runner";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { heartbeatRuns, nativeRunFinalizations } from "@paperclipai/db";
|
||||
import { PaperclipControlPlanePort } from "./paperclip-control-plane-port.js";
|
||||
|
||||
const activeNativeSessions = new Map<string, NativeSession>();
|
||||
type ActiveNativeSession = {
|
||||
session: NativeSession;
|
||||
cancelRequested: boolean;
|
||||
};
|
||||
|
||||
const activeNativeSessions = new Map<string, ActiveNativeSession>();
|
||||
|
||||
export async function cancelNativeSession(runId: string, reason: string): Promise<boolean> {
|
||||
const session = activeNativeSessions.get(runId);
|
||||
if (!session) return false;
|
||||
if (session.cancel) await session.cancel({ reason });
|
||||
else if (session.interrupt) await session.interrupt({ reason });
|
||||
const active = activeNativeSessions.get(runId);
|
||||
if (!active) return false;
|
||||
if (active.cancelRequested) return true;
|
||||
active.cancelRequested = true;
|
||||
if (active.session.cancel) await active.session.cancel({ reason });
|
||||
else if (active.session.interrupt) await active.session.interrupt({ reason });
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -23,6 +33,40 @@ export async function executePaperclipNativeSession(input: {
|
||||
execution: NativeExecutionInputV1;
|
||||
runnerInstanceId: string;
|
||||
}): Promise<AdapterExecutionResult> {
|
||||
const leaseOwner = `${input.runnerInstanceId}:${randomUUID()}`;
|
||||
const leaseNow = new Date();
|
||||
const leaseExpiresAt = new Date(leaseNow.getTime() + 20 * 60_000);
|
||||
await input.db.transaction(async (tx) => {
|
||||
const coordinator = await tx.select().from(nativeRunFinalizations)
|
||||
.where(and(
|
||||
eq(nativeRunFinalizations.runId, input.execution.binding.runId),
|
||||
eq(nativeRunFinalizations.companyId, input.execution.binding.companyId),
|
||||
eq(nativeRunFinalizations.issueId, input.execution.binding.issueId),
|
||||
)).for("update").limit(1).then((rows) => rows[0] ?? null);
|
||||
if (!coordinator) throw new Error("native_finalization_coordinator_missing");
|
||||
if (["committed", "applied"].includes(coordinator.phase)) throw new Error("native_run_already_committed");
|
||||
if (
|
||||
coordinator.leaseOwner
|
||||
&& coordinator.leaseOwner !== leaseOwner
|
||||
&& coordinator.leaseExpiresAt
|
||||
&& coordinator.leaseExpiresAt > leaseNow
|
||||
) throw new Error("native_finalization_lease_busy");
|
||||
await tx.update(nativeRunFinalizations).set({
|
||||
phase: coordinator.resultId ? coordinator.phase : "observed",
|
||||
attempt: coordinator.attempt + 1,
|
||||
leaseOwner,
|
||||
leaseExpiresAt,
|
||||
failureCode: null,
|
||||
failureDetail: null,
|
||||
nextAttemptAt: null,
|
||||
updatedAt: leaseNow,
|
||||
}).where(eq(nativeRunFinalizations.runId, coordinator.runId));
|
||||
await tx.update(heartbeatRuns).set({
|
||||
nativePhase: coordinator.resultId ? coordinator.phase : "observed",
|
||||
nativePhaseUpdatedAt: leaseNow,
|
||||
updatedAt: leaseNow,
|
||||
}).where(eq(heartbeatRuns.id, coordinator.runId));
|
||||
});
|
||||
const controlPlaneInstanceId = `${input.runnerInstanceId}:control`;
|
||||
const controlPlane = new PaperclipControlPlanePort(input.db, {
|
||||
companyId: input.execution.binding.companyId,
|
||||
@@ -34,17 +78,50 @@ export async function executePaperclipNativeSession(input: {
|
||||
sourceInstanceId: input.runnerInstanceId,
|
||||
controlPlaneSourceInstanceId: controlPlaneInstanceId,
|
||||
});
|
||||
const native = await executeNativeSession({
|
||||
input: input.execution,
|
||||
backend: createCodexNativeSessionBackend(input.execution, { runnerInstanceId: input.runnerInstanceId }),
|
||||
controlPlane,
|
||||
runnerInstanceId: input.runnerInstanceId,
|
||||
controlPlaneInstanceId,
|
||||
onSession: (session) => {
|
||||
if (session) activeNativeSessions.set(input.execution.binding.runId, session);
|
||||
else activeNativeSessions.delete(input.execution.binding.runId);
|
||||
},
|
||||
});
|
||||
let native: Awaited<ReturnType<typeof executeNativeSession>>;
|
||||
try {
|
||||
native = await executeNativeSession({
|
||||
input: input.execution,
|
||||
backend: createCodexNativeSessionBackend(input.execution, { runnerInstanceId: input.runnerInstanceId }),
|
||||
controlPlane,
|
||||
runnerInstanceId: input.runnerInstanceId,
|
||||
controlPlaneInstanceId,
|
||||
onSession: (session) => {
|
||||
if (session) activeNativeSessions.set(input.execution.binding.runId, {
|
||||
session,
|
||||
cancelRequested: false,
|
||||
});
|
||||
else activeNativeSessions.delete(input.execution.binding.runId);
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
const now = new Date();
|
||||
await input.db.update(nativeRunFinalizations).set({
|
||||
phase: "retryable_failure",
|
||||
leaseOwner: null,
|
||||
leaseExpiresAt: null,
|
||||
failureCode: "native_session_interrupted",
|
||||
failureDetail: {
|
||||
message: error instanceof Error ? error.message.slice(0, 2_000) : String(error).slice(0, 2_000),
|
||||
recoveryOwner: { kind: "agent", agentId: input.execution.binding.agentId },
|
||||
nextAction: "Resume the persisted native provider session after the retry delay.",
|
||||
},
|
||||
nextAttemptAt: new Date(now.getTime() + 30_000),
|
||||
updatedAt: now,
|
||||
}).where(and(
|
||||
eq(nativeRunFinalizations.runId, input.execution.binding.runId),
|
||||
eq(nativeRunFinalizations.leaseOwner, leaseOwner),
|
||||
));
|
||||
throw error;
|
||||
}
|
||||
await input.db.update(nativeRunFinalizations).set({
|
||||
leaseOwner: null,
|
||||
leaseExpiresAt: null,
|
||||
updatedAt: new Date(),
|
||||
}).where(and(
|
||||
eq(nativeRunFinalizations.runId, input.execution.binding.runId),
|
||||
eq(nativeRunFinalizations.leaseOwner, leaseOwner),
|
||||
));
|
||||
const finalization: NativeFinalizationResult = {
|
||||
schema: "paperclip.native-finalization.v1",
|
||||
runtimeMode: "native",
|
||||
|
||||
@@ -2,13 +2,16 @@ import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { eq } from "drizzle-orm";
|
||||
import {
|
||||
activityLog,
|
||||
agentWakeupRequests,
|
||||
agents,
|
||||
companies,
|
||||
completionContracts,
|
||||
createDb,
|
||||
heartbeatRunEvents,
|
||||
heartbeatRuns,
|
||||
issueRecoveryActions,
|
||||
issueThreadInteractions,
|
||||
issueWorkProducts,
|
||||
issues,
|
||||
nativeRunFinalizations,
|
||||
nativeRunResults,
|
||||
@@ -24,19 +27,15 @@ import {
|
||||
runControlPlanePortConformance,
|
||||
type NativeSessionBackend,
|
||||
type PrpEvent,
|
||||
type PrpStructuredRunResult,
|
||||
type PrpTerminalState,
|
||||
} from "@paperclipai/paperclip-runner";
|
||||
import {
|
||||
getEmbeddedPostgresTestSupport,
|
||||
startEmbeddedPostgresTestDatabase,
|
||||
} from "../../__tests__/helpers/embedded-postgres.js";
|
||||
import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
|
||||
import { PaperclipControlPlanePort } from "./paperclip-control-plane-port.js";
|
||||
import { finalizeNativeRun } from "./native-run-finalizer.js";
|
||||
import { buildNativeExecutionInput } from "./native-execution-input.js";
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
const describeDatabase = support.supported ? describe : describe.skip;
|
||||
|
||||
describeDatabase("PaperclipControlPlanePort conformance", () => {
|
||||
describe("PaperclipControlPlanePort conformance", () => {
|
||||
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
const contractId = "00000000-0000-4000-8000-000000000004";
|
||||
@@ -44,6 +43,7 @@ describeDatabase("PaperclipControlPlanePort conformance", () => {
|
||||
const taskIssueId = "00000000-0000-4000-8000-000000000008";
|
||||
const taskContractId = "00000000-0000-4000-8000-000000000009";
|
||||
const taskRunId = "00000000-0000-4000-8000-000000000010";
|
||||
const taskWorkProductId = "00000000-0000-4000-8000-000000000017";
|
||||
const workspaceFailureIssueId = "00000000-0000-4000-8000-000000000011";
|
||||
const workspaceFailureContractId = "00000000-0000-4000-8000-000000000012";
|
||||
const workspaceFailureRunId = "00000000-0000-4000-8000-000000000013";
|
||||
@@ -68,6 +68,7 @@ describeDatabase("PaperclipControlPlanePort conformance", () => {
|
||||
companyId: identity.companyId,
|
||||
title: "Native conformance",
|
||||
status: "in_progress",
|
||||
assigneeAgentId: identity.agentId,
|
||||
workMode: "standard",
|
||||
});
|
||||
await db.insert(completionContracts).values({
|
||||
@@ -80,7 +81,11 @@ describeDatabase("PaperclipControlPlanePort conformance", () => {
|
||||
risk: "standard",
|
||||
completionAuthority: "server_arbiter",
|
||||
incompleteCriteriaPolicy: "preserve_non_terminal",
|
||||
contractJson: { objective: "Conformance" },
|
||||
contractJson: {
|
||||
revision: "phase6-v1",
|
||||
objective: "Conformance",
|
||||
criteria: [{ id: "objective", requirement: "Complete conformance" }],
|
||||
},
|
||||
canonicalSha256: contractSha,
|
||||
createdByActorType: "system",
|
||||
createdByActorId: "test",
|
||||
@@ -100,6 +105,7 @@ describeDatabase("PaperclipControlPlanePort conformance", () => {
|
||||
companyId: identity.companyId,
|
||||
title: "Complete one native task",
|
||||
status: "in_progress",
|
||||
assigneeAgentId: identity.agentId,
|
||||
workMode: "standard",
|
||||
});
|
||||
await db.insert(completionContracts).values({
|
||||
@@ -112,7 +118,11 @@ describeDatabase("PaperclipControlPlanePort conformance", () => {
|
||||
risk: "standard",
|
||||
completionAuthority: "server_arbiter",
|
||||
incompleteCriteriaPolicy: "preserve_non_terminal",
|
||||
contractJson: { objective: "Complete one native task" },
|
||||
contractJson: {
|
||||
revision: "phase6-v1",
|
||||
objective: "Complete one native task",
|
||||
criteria: [{ id: "objective", requirement: "Complete the task" }],
|
||||
},
|
||||
canonicalSha256: "phase6-task-contract",
|
||||
createdByActorType: "system",
|
||||
createdByActorId: "test",
|
||||
@@ -129,11 +139,23 @@ describeDatabase("PaperclipControlPlanePort conformance", () => {
|
||||
completionContractSha256: "phase6-task-contract",
|
||||
contextSnapshot: { issueId: taskIssueId },
|
||||
});
|
||||
await db.insert(issueWorkProducts).values({
|
||||
id: taskWorkProductId,
|
||||
companyId: identity.companyId,
|
||||
issueId: taskIssueId,
|
||||
type: "artifact",
|
||||
provider: "paperclip",
|
||||
title: "Verified native task result",
|
||||
status: "ready_for_review",
|
||||
reviewState: "approved",
|
||||
createdByRunId: taskRunId,
|
||||
});
|
||||
await db.insert(issues).values({
|
||||
id: workspaceFailureIssueId,
|
||||
companyId: identity.companyId,
|
||||
title: "Preserve status after workspace failure",
|
||||
status: "in_progress",
|
||||
assigneeAgentId: identity.agentId,
|
||||
workMode: "standard",
|
||||
});
|
||||
await db.insert(completionContracts).values({
|
||||
@@ -146,7 +168,11 @@ describeDatabase("PaperclipControlPlanePort conformance", () => {
|
||||
risk: "standard",
|
||||
completionAuthority: "server_arbiter",
|
||||
incompleteCriteriaPolicy: "preserve_non_terminal",
|
||||
contractJson: { objective: "Preserve status after workspace failure" },
|
||||
contractJson: {
|
||||
revision: "phase6-v1",
|
||||
objective: "Preserve status after workspace failure",
|
||||
criteria: [{ id: "objective", requirement: "Preserve the result" }],
|
||||
},
|
||||
canonicalSha256: "phase6-workspace-failure-contract",
|
||||
createdByActorType: "system",
|
||||
createdByActorId: "test",
|
||||
@@ -178,7 +204,11 @@ describeDatabase("PaperclipControlPlanePort conformance", () => {
|
||||
risk: "standard",
|
||||
completionAuthority: "server_arbiter",
|
||||
incompleteCriteriaPolicy: "preserve_non_terminal",
|
||||
contractJson: { objective: "Respect pending governance" },
|
||||
contractJson: {
|
||||
revision: "phase6-v1",
|
||||
objective: "Respect pending governance",
|
||||
criteria: [{ id: "objective", requirement: "Respect pending governance" }],
|
||||
},
|
||||
canonicalSha256: "phase6-governance-contract",
|
||||
createdByActorType: "system",
|
||||
createdByActorId: "test",
|
||||
@@ -205,12 +235,14 @@ describeDatabase("PaperclipControlPlanePort conformance", () => {
|
||||
afterAll(async () => {
|
||||
if (temporary) {
|
||||
await db.delete(activityLog);
|
||||
await db.delete(agentWakeupRequests);
|
||||
await db.delete(statusDecisionEffects);
|
||||
await db.delete(statusDecisions);
|
||||
await db.delete(workAssessments);
|
||||
await db.delete(nativeRunFinalizations);
|
||||
await db.delete(nativeRunResults);
|
||||
await db.delete(issueThreadInteractions);
|
||||
await db.delete(issueWorkProducts);
|
||||
await db.delete(heartbeatRunEvents);
|
||||
await db.delete(heartbeatRuns);
|
||||
await db.delete(completionContracts);
|
||||
@@ -240,24 +272,37 @@ describeDatabase("PaperclipControlPlanePort conformance", () => {
|
||||
terminalReplayIdempotent: true,
|
||||
});
|
||||
await expect(db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, identity.runId))).resolves.toHaveLength(1);
|
||||
await finalizeNativeRun({ db, runId: identity.runId, workspaceFinalizeStatus: "succeeded" });
|
||||
await finalizeNativeRun({
|
||||
db,
|
||||
runId: identity.runId,
|
||||
workspaceFinalizeStatus: "succeeded",
|
||||
});
|
||||
await expect(db.select().from(nativeRunFinalizations)
|
||||
.where(eq(nativeRunFinalizations.runId, identity.runId))).resolves.toEqual([
|
||||
expect.objectContaining({ phase: "committed" }),
|
||||
]);
|
||||
await expect(db.select().from(issues).where(eq(issues.id, identity.issueId))).resolves.toEqual([
|
||||
expect.objectContaining({ status: "done", statusVersion: 1 }),
|
||||
expect.objectContaining({ status: "in_progress", statusVersion: 1 }),
|
||||
]);
|
||||
await expect(db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, identity.runId))).resolves.toEqual([
|
||||
expect.objectContaining({ phase: "applied" }),
|
||||
expect.objectContaining({ phase: "committed" }),
|
||||
]);
|
||||
await expect(db.select().from(statusDecisions).where(eq(statusDecisions.issueId, identity.issueId))).resolves.toEqual([
|
||||
expect.objectContaining({ toStatus: "done", applicationState: "applied" }),
|
||||
]);
|
||||
await expect(db.select().from(activityLog).where(eq(activityLog.entityId, identity.issueId))).resolves.toEqual([
|
||||
expect.objectContaining({ action: "issue.native_status_decision_applied" }),
|
||||
expect.objectContaining({ toStatus: "in_progress", reasonCode: "native_completion_incomplete", applicationState: "applied" }),
|
||||
]);
|
||||
await expect(db.select().from(activityLog).where(eq(activityLog.entityId, identity.issueId))).resolves.toEqual(
|
||||
expect.arrayContaining([expect.objectContaining({ action: "issue.updated" })]),
|
||||
);
|
||||
});
|
||||
|
||||
it("completes one selected Paperclip task through the public package session contract", async () => {
|
||||
const identity = CONTROL_PLANE_CONFORMANCE_OPEN.identity;
|
||||
const sessionId = "session-phase6-paperclip-task";
|
||||
const evidenceRef = `work_product:${taskWorkProductId}`;
|
||||
const taskResult = structuredClone(CONTROL_PLANE_CONFORMANCE_RESULT);
|
||||
taskResult.completionClaim.criteria[0]!.evidenceRefs = [evidenceRef];
|
||||
taskResult.evidence = [{ kind: "work_product", ref: evidenceRef }];
|
||||
taskResult.verification[0]!.artifactRef = evidenceRef;
|
||||
const event = (sourceSeq: number, eventType: PrpEvent["eventType"], payload: Record<string, unknown>): PrpEvent => ({
|
||||
schema: "paperclip.prp.event.v1",
|
||||
sourceEventId: `phase6-task:${sourceSeq}`,
|
||||
@@ -274,7 +319,7 @@ describeDatabase("PaperclipControlPlanePort conformance", () => {
|
||||
payload,
|
||||
});
|
||||
const events = [
|
||||
event(1, "run.result.proposed", CONTROL_PLANE_CONFORMANCE_RESULT),
|
||||
event(1, "run.result.proposed", taskResult),
|
||||
event(2, "turn.completed", {}),
|
||||
];
|
||||
const backend: NativeSessionBackend = {
|
||||
@@ -293,7 +338,7 @@ describeDatabase("PaperclipControlPlanePort conformance", () => {
|
||||
async *events() { yield* events; },
|
||||
async startTurn() { return { turnId: "turn-phase6-paperclip-task" }; },
|
||||
async cancel() {},
|
||||
async result() { return { result: CONTROL_PLANE_CONFORMANCE_RESULT, terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, turnId: "turn-phase6-paperclip-task" }; },
|
||||
async result() { return { result: taskResult, terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, turnId: "turn-phase6-paperclip-task" }; },
|
||||
async snapshot() { return { backendKind: "mock", sessionId, identity: input.identity, providerSessionId: "provider-phase6-paperclip-task" }; },
|
||||
async close() {},
|
||||
};
|
||||
@@ -332,9 +377,30 @@ describeDatabase("PaperclipControlPlanePort conformance", () => {
|
||||
});
|
||||
expect(completed.terminal.runTerminalState).toBe("succeeded");
|
||||
await finalizeNativeRun({ db, runId: taskRunId, workspaceFinalizeStatus: "succeeded" });
|
||||
await expect(port.completeRun({
|
||||
result: taskResult,
|
||||
terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL,
|
||||
turnId: "turn-phase6-paperclip-task",
|
||||
callerResultId: `phase6-scripted-runner:${taskRunId}:result`,
|
||||
callerDedupeKey: `${taskRunId}:phase6-task-contract`,
|
||||
})).resolves.toBeUndefined();
|
||||
await expect(port.completeRun({
|
||||
result: { ...structuredClone(taskResult), summary: "Conflicting changed result bytes" },
|
||||
terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL,
|
||||
turnId: "turn-phase6-paperclip-task",
|
||||
callerResultId: `phase6-scripted-runner:${taskRunId}:result`,
|
||||
callerDedupeKey: `${taskRunId}:phase6-task-contract`,
|
||||
})).rejects.toThrow("structured_result_replay_conflict");
|
||||
await Promise.all([
|
||||
finalizeNativeRun({ db, runId: taskRunId, workspaceFinalizeStatus: "succeeded" }),
|
||||
finalizeNativeRun({ db, runId: taskRunId, workspaceFinalizeStatus: "succeeded" }),
|
||||
]);
|
||||
await expect(db.select().from(issues).where(eq(issues.id, taskIssueId))).resolves.toEqual([
|
||||
expect.objectContaining({ status: "done", statusVersion: 1 }),
|
||||
]);
|
||||
await expect(db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, taskRunId))).resolves.toHaveLength(1);
|
||||
await expect(db.select().from(workAssessments).where(eq(workAssessments.runId, taskRunId))).resolves.toHaveLength(1);
|
||||
await expect(db.select().from(statusDecisions).where(eq(statusDecisions.issueId, taskIssueId))).resolves.toHaveLength(1);
|
||||
});
|
||||
|
||||
it("fails closed when the bound company does not own the run", async () => {
|
||||
@@ -376,13 +442,85 @@ describeDatabase("PaperclipControlPlanePort conformance", () => {
|
||||
});
|
||||
await finalizeNativeRun({ db, runId: governanceRunId, workspaceFinalizeStatus: "succeeded" });
|
||||
await expect(db.select().from(issues).where(eq(issues.id, governanceIssueId))).resolves.toEqual([
|
||||
expect.objectContaining({ status: "in_progress" }),
|
||||
expect.objectContaining({ status: "in_review" }),
|
||||
]);
|
||||
await expect(db.select().from(statusDecisions).where(eq(statusDecisions.issueId, governanceIssueId))).resolves.toEqual([
|
||||
expect.objectContaining({ toStatus: "in_progress", reasonCode: "approval_gate_pending" }),
|
||||
expect.objectContaining({ toStatus: "in_review", reasonCode: "governance_gate_pending" }),
|
||||
]);
|
||||
});
|
||||
|
||||
it("P6-23 materializes native review through the authorized interaction service", async () => {
|
||||
const identity = CONTROL_PLANE_CONFORMANCE_OPEN.identity;
|
||||
const issueId = "30000000-0000-4000-8000-000000000024";
|
||||
const localContractId = "31000000-0000-4000-8000-000000000024";
|
||||
const runId = "32000000-0000-4000-8000-000000000024";
|
||||
await db.insert(issues).values({
|
||||
id: issueId,
|
||||
companyId: identity.companyId,
|
||||
title: "Native review interaction",
|
||||
status: "in_progress",
|
||||
assigneeAgentId: identity.agentId,
|
||||
workMode: "standard",
|
||||
});
|
||||
await db.insert(completionContracts).values({
|
||||
id: localContractId,
|
||||
companyId: identity.companyId,
|
||||
issueId,
|
||||
revision: 1,
|
||||
schemaVersion: "paperclip.completion-contract.v1",
|
||||
policyVersion: "phase6-v1",
|
||||
risk: "standard",
|
||||
completionAuthority: "server_arbiter",
|
||||
incompleteCriteriaPolicy: "preserve_non_terminal",
|
||||
contractJson: { revision: "phase6-v1", objective: "Review", criteria: [{ id: "objective", requirement: "Review" }] },
|
||||
canonicalSha256: "native-review-contract",
|
||||
createdByActorType: "system",
|
||||
createdByActorId: "test",
|
||||
});
|
||||
await db.insert(heartbeatRuns).values({
|
||||
id: runId,
|
||||
companyId: identity.companyId,
|
||||
agentId: identity.agentId,
|
||||
status: "running",
|
||||
runtimeMode: "native",
|
||||
completionContractId: localContractId,
|
||||
completionContractSha256: "native-review-contract",
|
||||
contextSnapshot: { issueId },
|
||||
});
|
||||
const port = new PaperclipControlPlanePort(db, {
|
||||
companyId: identity.companyId,
|
||||
issueId,
|
||||
runId,
|
||||
agentId: identity.agentId,
|
||||
completionContractId: localContractId,
|
||||
completionContractSha256: "native-review-contract",
|
||||
sourceInstanceId: "native-review-runner",
|
||||
controlPlaneSourceInstanceId: "native-review-control",
|
||||
});
|
||||
await port.openRun({
|
||||
identity: { ...identity, issueId, runId },
|
||||
backendKind: "mock",
|
||||
sourceInstanceId: "native-review-runner",
|
||||
});
|
||||
const result = { ...structuredClone(CONTROL_PLANE_CONFORMANCE_RESULT), reportedWorkDisposition: "needs_review" as const };
|
||||
await port.completeRun({
|
||||
result,
|
||||
terminal: { ...CONTROL_PLANE_CONFORMANCE_TERMINAL, reportedWorkDisposition: "needs_review" },
|
||||
callerResultId: "native-review-result",
|
||||
});
|
||||
await finalizeNativeRun({ db, runId, workspaceFinalizeStatus: "succeeded" });
|
||||
|
||||
await expect(db.select().from(issues).where(eq(issues.id, issueId))).resolves.toEqual([
|
||||
expect.objectContaining({ status: "in_review", statusVersion: 1 }),
|
||||
]);
|
||||
await expect(db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.issueId, issueId))).resolves.toEqual([
|
||||
expect.objectContaining({ kind: "request_confirmation", status: "pending", sourceRunId: runId }),
|
||||
]);
|
||||
await expect(db.select().from(statusDecisionEffects).where(eq(statusDecisionEffects.issueId, issueId))).resolves.toEqual(
|
||||
expect.arrayContaining([expect.objectContaining({ effectKind: "create_review_interaction", targetType: "issue_thread_interaction" })]),
|
||||
);
|
||||
});
|
||||
|
||||
it("preserves the result and issue status when workspace finalization fails", async () => {
|
||||
const identity = CONTROL_PLANE_CONFORMANCE_OPEN.identity;
|
||||
const port = new PaperclipControlPlanePort(db, {
|
||||
@@ -415,8 +553,125 @@ describeDatabase("PaperclipControlPlanePort conformance", () => {
|
||||
expect.objectContaining({ status: "in_progress", statusVersion: 0, lastStatusDecisionId: null }),
|
||||
]);
|
||||
await expect(db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, workspaceFailureRunId))).resolves.toEqual([
|
||||
expect.objectContaining({ status: "failed", nativePhase: "workspace_failed" }),
|
||||
expect.objectContaining({ status: "failed", nativePhase: "retryable_failure" }),
|
||||
]);
|
||||
await expect(db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, workspaceFailureRunId))).resolves.toHaveLength(1);
|
||||
});
|
||||
|
||||
it("LIVE-01..04 rolls back status, decision, and liveness rows at every materialization failpoint", async () => {
|
||||
const identity = CONTROL_PLANE_CONFORMANCE_OPEN.identity;
|
||||
const cases: Array<{
|
||||
suffix: number;
|
||||
failpoint: "interaction_materialization" | "continuation_materialization" | "blocker_materialization" | "status_projection";
|
||||
result: PrpStructuredRunResult;
|
||||
}> = [
|
||||
{
|
||||
suffix: 20,
|
||||
failpoint: "interaction_materialization",
|
||||
result: { ...structuredClone(CONTROL_PLANE_CONFORMANCE_RESULT), reportedWorkDisposition: "needs_review" },
|
||||
},
|
||||
{
|
||||
suffix: 21,
|
||||
failpoint: "continuation_materialization",
|
||||
result: {
|
||||
...structuredClone(CONTROL_PLANE_CONFORMANCE_RESULT),
|
||||
reportedWorkDisposition: "yielded",
|
||||
continuation: { kind: "same_agent", summary: "Continue after recovery", idempotencyKey: "live-continue" },
|
||||
},
|
||||
},
|
||||
{
|
||||
suffix: 22,
|
||||
failpoint: "blocker_materialization",
|
||||
result: {
|
||||
...structuredClone(CONTROL_PLANE_CONFORMANCE_RESULT),
|
||||
reportedWorkDisposition: "blocked",
|
||||
blocker: { reasonCode: "access", owner: { kind: "board" }, unblockAction: "Approve access", scope: "task_wide" },
|
||||
},
|
||||
},
|
||||
{
|
||||
suffix: 23,
|
||||
failpoint: "status_projection",
|
||||
result: { ...structuredClone(CONTROL_PLANE_CONFORMANCE_RESULT), reportedWorkDisposition: "needs_review" },
|
||||
},
|
||||
];
|
||||
for (const entry of cases) {
|
||||
const value = String(entry.suffix).padStart(12, "0");
|
||||
const issueId = `30000000-0000-4000-8000-${value}`;
|
||||
const contractId = `31000000-0000-4000-8000-${value}`;
|
||||
const runId = `32000000-0000-4000-8000-${value}`;
|
||||
await db.insert(issues).values({
|
||||
id: issueId,
|
||||
companyId: identity.companyId,
|
||||
title: `Atomic liveness ${entry.failpoint}`,
|
||||
status: "in_progress",
|
||||
assigneeAgentId: identity.agentId,
|
||||
workMode: "standard",
|
||||
});
|
||||
await db.insert(completionContracts).values({
|
||||
id: contractId,
|
||||
companyId: identity.companyId,
|
||||
issueId,
|
||||
revision: 1,
|
||||
schemaVersion: "paperclip.completion-contract.v1",
|
||||
policyVersion: "phase6-v1",
|
||||
risk: "standard",
|
||||
completionAuthority: "server_arbiter",
|
||||
incompleteCriteriaPolicy: "preserve_non_terminal",
|
||||
contractJson: { revision: "phase6-v1", objective: "Atomic liveness", criteria: [{ id: "objective", requirement: "Stay atomic" }] },
|
||||
canonicalSha256: `contract-${entry.suffix}`,
|
||||
createdByActorType: "system",
|
||||
createdByActorId: "test",
|
||||
});
|
||||
await db.insert(heartbeatRuns).values({
|
||||
id: runId,
|
||||
companyId: identity.companyId,
|
||||
agentId: identity.agentId,
|
||||
status: "running",
|
||||
runtimeMode: "native",
|
||||
completionContractId: contractId,
|
||||
completionContractSha256: `contract-${entry.suffix}`,
|
||||
contextSnapshot: { issueId },
|
||||
});
|
||||
const terminal: PrpTerminalState = {
|
||||
...CONTROL_PLANE_CONFORMANCE_TERMINAL,
|
||||
reportedWorkDisposition: entry.result.reportedWorkDisposition,
|
||||
};
|
||||
const port = new PaperclipControlPlanePort(db, {
|
||||
companyId: identity.companyId,
|
||||
issueId,
|
||||
runId,
|
||||
agentId: identity.agentId,
|
||||
completionContractId: contractId,
|
||||
completionContractSha256: `contract-${entry.suffix}`,
|
||||
sourceInstanceId: `atomic-runner-${entry.suffix}`,
|
||||
controlPlaneSourceInstanceId: `atomic-control-${entry.suffix}`,
|
||||
});
|
||||
await port.openRun({
|
||||
identity: { ...identity, issueId, runId },
|
||||
backendKind: "mock",
|
||||
sourceInstanceId: `atomic-runner-${entry.suffix}`,
|
||||
});
|
||||
await port.completeRun({ result: entry.result, terminal, callerResultId: `atomic-result-${entry.suffix}` });
|
||||
await expect(finalizeNativeRun({
|
||||
db,
|
||||
runId,
|
||||
workspaceFinalizeStatus: "succeeded",
|
||||
failpoint: entry.failpoint,
|
||||
})).resolves.toEqual(expect.objectContaining({ phase: "retryable_failure" }));
|
||||
await expect(db.select().from(issues).where(eq(issues.id, issueId))).resolves.toEqual([
|
||||
expect.objectContaining({ status: "in_progress", statusVersion: 0, lastStatusDecisionId: null }),
|
||||
]);
|
||||
await expect(db.select().from(statusDecisions).where(eq(statusDecisions.issueId, issueId))).resolves.toHaveLength(0);
|
||||
await expect(db.select().from(statusDecisionEffects).where(eq(statusDecisionEffects.issueId, issueId))).resolves.toHaveLength(0);
|
||||
await expect(db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.issueId, issueId))).resolves.toHaveLength(0);
|
||||
await expect(db.select().from(agentWakeupRequests)
|
||||
.where(eq(agentWakeupRequests.companyId, identity.companyId))).resolves.not.toEqual(
|
||||
expect.arrayContaining([expect.objectContaining({ payload: expect.objectContaining({ issueId }) })]),
|
||||
);
|
||||
await expect(db.select().from(issueRecoveryActions)
|
||||
.where(eq(issueRecoveryActions.sourceIssueId, issueId))).resolves.toEqual([
|
||||
expect.objectContaining({ status: "active", cause: "side_effect_planning_failed" }),
|
||||
]);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -12,6 +12,7 @@ import type {
|
||||
NativeRunEvent,
|
||||
NativeRunResult,
|
||||
OpenControlPlaneRunInput,
|
||||
PersistedNativeSession,
|
||||
PrpEvent,
|
||||
PrpTerminalState,
|
||||
ReplayControlPlaneEventsInput,
|
||||
@@ -42,6 +43,12 @@ function isCompleteInput(value: NativeRunResult | CompleteControlPlaneRunInput):
|
||||
return "result" in value && "terminal" in value;
|
||||
}
|
||||
|
||||
function record(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value)
|
||||
? value as Record<string, unknown>
|
||||
: {};
|
||||
}
|
||||
|
||||
function assertTerminal(value: unknown): asserts value is PrpTerminalState {
|
||||
const terminal = value as Partial<PrpTerminalState> | null;
|
||||
if (
|
||||
@@ -90,6 +97,69 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
|
||||
this.#sessionId = identity.sessionId;
|
||||
}
|
||||
|
||||
async loadSessionCheckpoint(): Promise<PersistedNativeSession | null> {
|
||||
const run = await this.#db.select({
|
||||
companyId: heartbeatRuns.companyId,
|
||||
agentId: heartbeatRuns.agentId,
|
||||
runtimeMode: heartbeatRuns.runtimeMode,
|
||||
nativeSessionId: heartbeatRuns.nativeSessionId,
|
||||
runnerProfileJson: heartbeatRuns.runnerProfileJson,
|
||||
}).from(heartbeatRuns).where(eq(heartbeatRuns.id, this.#binding.runId))
|
||||
.limit(1).then((rows) => rows[0] ?? null);
|
||||
if (
|
||||
!run
|
||||
|| run.companyId !== this.#binding.companyId
|
||||
|| run.agentId !== this.#binding.agentId
|
||||
|| run.runtimeMode !== "native"
|
||||
) throw new Error("native_session_checkpoint_binding_mismatch");
|
||||
const candidate = record(run.runnerProfileJson).sessionCheckpoint;
|
||||
if (candidate === undefined || candidate === null) return null;
|
||||
const snapshot = record(candidate) as Partial<PersistedNativeSession>;
|
||||
const identity = record(snapshot.identity);
|
||||
if (
|
||||
typeof snapshot.sessionId !== "string"
|
||||
|| identity.companyId !== this.#binding.companyId
|
||||
|| identity.issueId !== this.#binding.issueId
|
||||
|| identity.runId !== this.#binding.runId
|
||||
|| identity.agentId !== this.#binding.agentId
|
||||
|| identity.sessionId !== run.nativeSessionId
|
||||
) throw new Error("native_session_checkpoint_invalid");
|
||||
return structuredClone(snapshot as PersistedNativeSession);
|
||||
}
|
||||
|
||||
async checkpointSession(snapshot: PersistedNativeSession): Promise<void> {
|
||||
const identity = snapshot.identity;
|
||||
if (
|
||||
identity.companyId !== this.#binding.companyId
|
||||
|| identity.issueId !== this.#binding.issueId
|
||||
|| identity.runId !== this.#binding.runId
|
||||
|| identity.agentId !== this.#binding.agentId
|
||||
|| this.#sessionId !== identity.sessionId
|
||||
) throw new Error("native_session_checkpoint_binding_mismatch");
|
||||
await this.#db.transaction(async (tx) => {
|
||||
const run = await tx.select().from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, this.#binding.runId)).for("update").limit(1)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
if (
|
||||
!run
|
||||
|| run.companyId !== this.#binding.companyId
|
||||
|| run.agentId !== this.#binding.agentId
|
||||
|| run.runtimeMode !== "native"
|
||||
|| run.nativeSessionId !== identity.sessionId
|
||||
) throw new Error("native_session_checkpoint_binding_mismatch");
|
||||
await tx.update(heartbeatRuns).set({
|
||||
runnerProfileJson: {
|
||||
...record(run.runnerProfileJson),
|
||||
sessionCheckpoint: structuredClone(snapshot) as unknown as Record<string, unknown>,
|
||||
},
|
||||
sessionIdAfter: snapshot.providerSessionId ?? snapshot.sessionId,
|
||||
nativePhase: snapshot.semanticResult ? "workspace_finalizing" : "observed",
|
||||
nativePhaseUpdatedAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
}).where(eq(heartbeatRuns.id, this.#binding.runId));
|
||||
});
|
||||
}
|
||||
|
||||
async appendEvent(value: NativeRunEvent | PrpEvent) {
|
||||
if (!isPrpEvent(value)) throw new Error("native_legacy_event_not_supported");
|
||||
const validated = validatePrpEvent(value);
|
||||
@@ -197,7 +267,7 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
|
||||
.where(and(eq(nativeRunResults.runId, this.#binding.runId), or(...callerConditions)))
|
||||
.limit(1).then((rows) => rows[0] ?? null);
|
||||
if (existing) {
|
||||
if (existing.canonicalSha256 !== canonicalSha256) throw new Error("native_result_idempotency_conflict");
|
||||
if (existing.canonicalSha256 !== canonicalSha256) throw new Error("structured_result_replay_conflict");
|
||||
return;
|
||||
}
|
||||
const resultJson = {
|
||||
@@ -222,14 +292,21 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
|
||||
runId: this.#binding.runId,
|
||||
companyId: this.#binding.companyId,
|
||||
issueId: this.#binding.issueId,
|
||||
phase: "result_persisted",
|
||||
phase: "workspace_finalizing",
|
||||
resultId: inserted.id,
|
||||
}).onConflictDoUpdate({
|
||||
target: nativeRunFinalizations.runId,
|
||||
set: { phase: "result_persisted", resultId: inserted.id, failureCode: null, failureDetail: null, updatedAt: new Date() },
|
||||
set: {
|
||||
phase: "workspace_finalizing",
|
||||
resultId: inserted.id,
|
||||
failureCode: null,
|
||||
failureDetail: null,
|
||||
nextAttemptAt: null,
|
||||
updatedAt: new Date(),
|
||||
},
|
||||
});
|
||||
await tx.update(heartbeatRuns).set({
|
||||
nativePhase: "result_persisted",
|
||||
nativePhase: "workspace_finalizing",
|
||||
nativePhaseUpdatedAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
}).where(eq(heartbeatRuns.id, this.#binding.runId));
|
||||
|
||||
@@ -1,72 +1,97 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { classifyNativeEvidence } from "./evidence-classifier.js";
|
||||
import type { NativeEvidenceAssessment } from "./evidence-classifier.js";
|
||||
import { arbitrateNativeStatus } from "./status-arbiter.js";
|
||||
|
||||
function result(overrides: Record<string, unknown> = {}) {
|
||||
function assessment(overrides: Partial<NativeEvidenceAssessment> = {}): NativeEvidenceAssessment {
|
||||
return {
|
||||
reportedWorkDisposition: "done",
|
||||
objectiveSatisfied: true,
|
||||
allCriteriaSatisfied: true,
|
||||
verificationPassed: true,
|
||||
hasBlockingRemainingWork: false,
|
||||
reportedDisposition: "done",
|
||||
summary: "Complete",
|
||||
completionClaim: {
|
||||
objectiveSatisfied: true,
|
||||
criteria: [{ criterionId: "objective", status: "satisfied", evidenceRefs: ["test"] }],
|
||||
remainingWork: [],
|
||||
},
|
||||
verification: [{ commandOrCheck: "test", status: "passed" }],
|
||||
contractRevisionMatches: true,
|
||||
criterionAssessments: [],
|
||||
verificationAssessments: [],
|
||||
acceptedEvidenceRefs: ["event:2"],
|
||||
missingRequirements: [],
|
||||
rejectedEvidence: [],
|
||||
unverifiableEvidence: [],
|
||||
blocker: null,
|
||||
continuation: null,
|
||||
attentionRequests: [],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function arbitrate(overrides: Partial<Parameters<typeof arbitrateNativeStatus>[0]> = {}) {
|
||||
return arbitrateNativeStatus({
|
||||
assessment: assessment(),
|
||||
terminalState: "succeeded",
|
||||
workspaceFinalizeStatus: "succeeded",
|
||||
agentId: "agent",
|
||||
priorIssueStatus: "in_progress",
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
|
||||
describe("native status authority", () => {
|
||||
it("marks done only from successful finalization and complete evidence", () => {
|
||||
const assessment = classifyNativeEvidence(result());
|
||||
expect(arbitrateNativeStatus({
|
||||
assessment,
|
||||
terminalState: "succeeded",
|
||||
workspaceFinalizeStatus: "succeeded",
|
||||
agentId: "agent",
|
||||
}).toStatus).toBe("done");
|
||||
it("marks done only from successful finalization and complete durable evidence", () => {
|
||||
expect(arbitrate()).toEqual(expect.objectContaining({
|
||||
toStatus: "done",
|
||||
reasonCode: "completion_contract_satisfied",
|
||||
effects: [{ kind: "release_checkout" }],
|
||||
}));
|
||||
expect(arbitrate({
|
||||
assessment: assessment({ verificationPassed: false, missingRequirements: ["test"] }),
|
||||
}).toStatus).toBe("in_progress");
|
||||
});
|
||||
|
||||
it("keeps incomplete, review, cancelled, and yielded work non-terminal", () => {
|
||||
for (const reportedWorkDisposition of ["needs_review", "yielded"] as const) {
|
||||
expect(arbitrateNativeStatus({
|
||||
assessment: classifyNativeEvidence(result({ reportedWorkDisposition })),
|
||||
terminalState: "succeeded",
|
||||
workspaceFinalizeStatus: "succeeded",
|
||||
agentId: "agent",
|
||||
}).toStatus).toBe("in_progress");
|
||||
}
|
||||
expect(arbitrateNativeStatus({
|
||||
assessment: classifyNativeEvidence(result()),
|
||||
terminalState: "cancelled",
|
||||
workspaceFinalizeStatus: "succeeded",
|
||||
agentId: "agent",
|
||||
}).toStatus).toBe("in_progress");
|
||||
expect(arbitrateNativeStatus({
|
||||
assessment: classifyNativeEvidence(result()),
|
||||
terminalState: "succeeded",
|
||||
workspaceFinalizeStatus: "succeeded",
|
||||
approvalGateSatisfied: false,
|
||||
agentId: "agent",
|
||||
it("creates explicit liveness paths for review, continuation, cancellation, and governance", () => {
|
||||
expect(arbitrate({
|
||||
assessment: assessment({ reportedDisposition: "needs_review" }),
|
||||
})).toEqual(expect.objectContaining({
|
||||
toStatus: "in_review",
|
||||
effects: [expect.objectContaining({ kind: "create_review_interaction" })],
|
||||
}));
|
||||
expect(arbitrate({
|
||||
assessment: assessment({
|
||||
reportedDisposition: "yielded",
|
||||
continuation: { kind: "retry", summary: "Retry the task", idempotencyKey: "retry-task" },
|
||||
}),
|
||||
})).toEqual(expect.objectContaining({
|
||||
toStatus: "in_progress",
|
||||
reasonCode: "approval_gate_pending",
|
||||
effects: [expect.objectContaining({ kind: "enqueue_continuation", continuationKind: "retry" })],
|
||||
}));
|
||||
expect(arbitrate({ terminalState: "cancelled" })).toEqual(expect.objectContaining({
|
||||
toStatus: "in_progress",
|
||||
effects: [expect.objectContaining({ kind: "record_recovery", cause: "native_run_cancelled" })],
|
||||
}));
|
||||
expect(arbitrate({ governanceGate: { kind: "interaction", id: "interaction" } })).toEqual(expect.objectContaining({
|
||||
toStatus: "in_review",
|
||||
reasonCode: "governance_gate_pending",
|
||||
effects: [{ kind: "bind_governance", gate: { kind: "interaction", id: "interaction" } }],
|
||||
}));
|
||||
});
|
||||
|
||||
it("blocks only on a first-class blocker or failed workspace finalization", () => {
|
||||
const explicitBlocker = classifyNativeEvidence(result({
|
||||
reportedWorkDisposition: "blocked",
|
||||
blocker: { owner: { kind: "board" }, unblockAction: "Approve access" },
|
||||
}));
|
||||
expect(arbitrateNativeStatus({
|
||||
assessment: explicitBlocker,
|
||||
terminalState: "succeeded",
|
||||
workspaceFinalizeStatus: "succeeded",
|
||||
agentId: "agent",
|
||||
it("blocks only for a task-wide blocker with a named owner and action", () => {
|
||||
expect(arbitrate({
|
||||
assessment: assessment({
|
||||
reportedDisposition: "blocked",
|
||||
blocker: { boardOwned: true, scope: "task_wide", unblockAction: "Approve access" },
|
||||
}),
|
||||
})).toEqual(expect.objectContaining({
|
||||
toStatus: "blocked",
|
||||
unblockDescriptor: { owner: "board", action: "Approve access" },
|
||||
effects: [{ kind: "bind_blocker", owner: "board", action: "Approve access" }],
|
||||
}));
|
||||
});
|
||||
|
||||
it("preserves authoritative terminal statuses", () => {
|
||||
expect(arbitrate({ priorIssueStatus: "done" })).toEqual(expect.objectContaining({
|
||||
toStatus: "done",
|
||||
reasonCode: "terminal_status_preserved",
|
||||
effects: [],
|
||||
}));
|
||||
});
|
||||
});
|
||||
@@ -1,12 +1,41 @@
|
||||
import type { NativeEvidenceAssessment } from "./evidence-classifier.js";
|
||||
|
||||
export const NATIVE_STATUS_ARBITER_POLICY_VERSION = "phase6-v1";
|
||||
export const NATIVE_STATUS_ARBITER_POLICY_VERSION = "phase6-v2";
|
||||
|
||||
export type NativeAuthoritativeIssueStatus =
|
||||
| "backlog"
|
||||
| "todo"
|
||||
| "in_progress"
|
||||
| "in_review"
|
||||
| "blocked"
|
||||
| "done"
|
||||
| "cancelled";
|
||||
|
||||
export type NativeGovernanceGate = {
|
||||
kind: "approval" | "interaction" | "execution_stage";
|
||||
id: string;
|
||||
};
|
||||
|
||||
export type NativeStatusEffect =
|
||||
| { kind: "bind_governance"; gate: NativeGovernanceGate }
|
||||
| { kind: "create_review_interaction"; prompt: string }
|
||||
| {
|
||||
kind: "enqueue_continuation";
|
||||
continuationKind: "same_agent" | "retry" | "delegated_issue" | "response_wake" | "monitor";
|
||||
summary: string;
|
||||
idempotencyKey: string;
|
||||
agentId: string;
|
||||
}
|
||||
| { kind: "bind_blocker"; owner: { agentId: string } | "board"; action: string }
|
||||
| { kind: "record_recovery"; cause: string; nextAction: string; agentId: string }
|
||||
| { kind: "release_checkout" };
|
||||
|
||||
export interface NativeStatusDecision {
|
||||
policyVersion: typeof NATIVE_STATUS_ARBITER_POLICY_VERSION;
|
||||
toStatus: "in_progress" | "done" | "blocked";
|
||||
toStatus: NativeAuthoritativeIssueStatus;
|
||||
reasonCode: string;
|
||||
unblockDescriptor: { owner: { agentId: string } | "board"; action: string } | null;
|
||||
effects: NativeStatusEffect[];
|
||||
}
|
||||
|
||||
/** Pure authority boundary: model prose is evidence, never a status command. */
|
||||
@@ -14,31 +43,55 @@ export function arbitrateNativeStatus(input: {
|
||||
assessment: NativeEvidenceAssessment;
|
||||
terminalState: "succeeded" | "failed" | "cancelled";
|
||||
workspaceFinalizeStatus: "succeeded" | "failed";
|
||||
approvalGateSatisfied?: boolean;
|
||||
governanceGate?: NativeGovernanceGate | null;
|
||||
agentId: string;
|
||||
priorIssueStatus: NativeAuthoritativeIssueStatus;
|
||||
}): NativeStatusDecision {
|
||||
if (["done", "cancelled"].includes(input.priorIssueStatus)) {
|
||||
return {
|
||||
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
|
||||
toStatus: input.priorIssueStatus,
|
||||
reasonCode: "terminal_status_preserved",
|
||||
unblockDescriptor: null,
|
||||
effects: [],
|
||||
};
|
||||
}
|
||||
if (input.workspaceFinalizeStatus !== "succeeded") {
|
||||
return {
|
||||
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
|
||||
toStatus: "blocked",
|
||||
reasonCode: "workspace_finalization_failed",
|
||||
unblockDescriptor: { owner: { agentId: input.agentId }, action: "Repair and re-verify workspace finalization." },
|
||||
toStatus: input.priorIssueStatus,
|
||||
reasonCode: "workspace_finalization_retry_required",
|
||||
unblockDescriptor: null,
|
||||
effects: [{
|
||||
kind: "record_recovery",
|
||||
cause: "workspace_finalization_failed",
|
||||
nextAction: "Repair and re-run workspace finalization for the persisted native result.",
|
||||
agentId: input.agentId,
|
||||
}],
|
||||
};
|
||||
}
|
||||
if (input.terminalState !== "succeeded") {
|
||||
const cause = input.terminalState === "cancelled" ? "native_run_cancelled" : "native_run_failed";
|
||||
return {
|
||||
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
|
||||
toStatus: "in_progress",
|
||||
reasonCode: input.terminalState === "cancelled" ? "native_run_cancelled" : "native_run_failed",
|
||||
reasonCode: cause,
|
||||
unblockDescriptor: null,
|
||||
effects: [{
|
||||
kind: "record_recovery",
|
||||
cause,
|
||||
nextAction: "Resume the persisted native run without opening a second provider session.",
|
||||
agentId: input.agentId,
|
||||
}],
|
||||
};
|
||||
}
|
||||
if (input.approvalGateSatisfied === false) {
|
||||
if (input.governanceGate) {
|
||||
return {
|
||||
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
|
||||
toStatus: "in_progress",
|
||||
reasonCode: "approval_gate_pending",
|
||||
toStatus: "in_review",
|
||||
reasonCode: "governance_gate_pending",
|
||||
unblockDescriptor: null,
|
||||
effects: [{ kind: "bind_governance", gate: input.governanceGate }],
|
||||
};
|
||||
}
|
||||
const complete =
|
||||
@@ -53,27 +106,72 @@ export function arbitrateNativeStatus(input: {
|
||||
toStatus: "done",
|
||||
reasonCode: "completion_contract_satisfied",
|
||||
unblockDescriptor: null,
|
||||
effects: [{ kind: "release_checkout" }],
|
||||
};
|
||||
}
|
||||
if (input.assessment.reportedDisposition === "needs_review") {
|
||||
return {
|
||||
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
|
||||
toStatus: "in_review",
|
||||
reasonCode: "native_review_path_required",
|
||||
unblockDescriptor: null,
|
||||
effects: [{
|
||||
kind: "create_review_interaction",
|
||||
prompt: "Review the persisted native-run evidence and confirm whether this issue may be completed.",
|
||||
}],
|
||||
};
|
||||
}
|
||||
if (input.assessment.reportedDisposition === "blocked" && input.assessment.blocker) {
|
||||
const owner = input.assessment.blocker.boardOwned ? "board" as const : { agentId: input.agentId };
|
||||
if (input.assessment.blocker.scope === "task_wide") {
|
||||
return {
|
||||
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
|
||||
toStatus: "blocked",
|
||||
reasonCode: "native_task_wide_blocker",
|
||||
unblockDescriptor: { owner, action: input.assessment.blocker.unblockAction },
|
||||
effects: [{ kind: "bind_blocker", owner, action: input.assessment.blocker.unblockAction }],
|
||||
};
|
||||
}
|
||||
return {
|
||||
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
|
||||
toStatus: "blocked",
|
||||
reasonCode: "native_reported_blocker",
|
||||
unblockDescriptor: {
|
||||
owner: input.assessment.blocker.boardOwned ? "board" : { agentId: input.agentId },
|
||||
action: input.assessment.blocker.unblockAction,
|
||||
},
|
||||
toStatus: "in_progress",
|
||||
reasonCode: "native_track_blocked_continuation",
|
||||
unblockDescriptor: null,
|
||||
effects: [{
|
||||
kind: "enqueue_continuation",
|
||||
continuationKind: "same_agent",
|
||||
summary: `Continue another productive track while resolving: ${input.assessment.blocker.unblockAction}`,
|
||||
idempotencyKey: `native-track-blocked:${input.assessment.blocker.unblockAction}`,
|
||||
agentId: input.agentId,
|
||||
}],
|
||||
};
|
||||
}
|
||||
if (input.assessment.reportedDisposition === "yielded" && input.assessment.continuation) {
|
||||
return {
|
||||
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
|
||||
toStatus: "in_progress",
|
||||
reasonCode: "native_continuation_required",
|
||||
unblockDescriptor: null,
|
||||
effects: [{
|
||||
kind: "enqueue_continuation",
|
||||
continuationKind: input.assessment.continuation.kind,
|
||||
summary: input.assessment.continuation.summary,
|
||||
idempotencyKey: input.assessment.continuation.idempotencyKey,
|
||||
agentId: input.agentId,
|
||||
}],
|
||||
};
|
||||
}
|
||||
return {
|
||||
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
|
||||
toStatus: "in_progress",
|
||||
reasonCode: input.assessment.reportedDisposition === "needs_review"
|
||||
? "native_review_path_required"
|
||||
: input.assessment.reportedDisposition === "yielded"
|
||||
? "native_continuation_required"
|
||||
: "native_completion_incomplete",
|
||||
reasonCode: "native_completion_incomplete",
|
||||
unblockDescriptor: null,
|
||||
effects: [{
|
||||
kind: "enqueue_continuation",
|
||||
continuationKind: "same_agent",
|
||||
summary: "Continue work on the missing or unverifiable completion-contract evidence.",
|
||||
idempotencyKey: "native-completion-incomplete",
|
||||
agentId: input.agentId,
|
||||
}],
|
||||
};
|
||||
}
|
||||
@@ -1,14 +1,23 @@
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { and, eq, inArray } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import type { CreateIssueThreadInteraction } from "@paperclipai/shared";
|
||||
import {
|
||||
agentWakeupRequests,
|
||||
approvals,
|
||||
issueApprovals,
|
||||
issueThreadInteractions,
|
||||
issues,
|
||||
activityLog,
|
||||
nativeRunFinalizations,
|
||||
statusDecisionEffects,
|
||||
statusDecisions,
|
||||
} from "@paperclipai/db";
|
||||
import type { NativeStatusDecision } from "./status-arbiter.js";
|
||||
import type { NativeStatusDecision, NativeStatusEffect } from "./status-arbiter.js";
|
||||
import { nativeSha256 } from "./canonical.js";
|
||||
import { issueService } from "../issues.js";
|
||||
import { issueThreadInteractionService } from "../issue-thread-interactions.js";
|
||||
import { issueRecoveryActionService } from "../issue-recovery-actions.js";
|
||||
import { buildIssueBlockersResolvedWakeIdempotencyKey } from "../issue-dependency-wakeups.js";
|
||||
import { persistActivity, publishActivity, type ActivityPublication } from "../activity-log.js";
|
||||
|
||||
export class NativeStatusRaceError extends Error {
|
||||
readonly code = "native_status_race" as const;
|
||||
@@ -18,6 +27,229 @@ export class NativeStatusRaceError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
export type NativeStatusCommitFailpoint =
|
||||
| "governance_materialization"
|
||||
| "interaction_materialization"
|
||||
| "continuation_materialization"
|
||||
| "blocker_materialization"
|
||||
| "recovery_materialization"
|
||||
| "status_projection";
|
||||
|
||||
type MaterializedEffect = {
|
||||
effectKind: string;
|
||||
targetType: string;
|
||||
targetId: string | null;
|
||||
payload: Record<string, unknown>;
|
||||
};
|
||||
|
||||
function failAt(actual: NativeStatusCommitFailpoint, requested?: NativeStatusCommitFailpoint) {
|
||||
if (actual === requested) throw new Error(`native_status_failpoint:${actual}`);
|
||||
}
|
||||
|
||||
async function enqueueWake(input: {
|
||||
tx: Db;
|
||||
companyId: string;
|
||||
issueId: string;
|
||||
agentId: string;
|
||||
reason: string;
|
||||
idempotencyKey: string;
|
||||
payload: Record<string, unknown>;
|
||||
}) {
|
||||
const existing = await input.tx.select({ id: agentWakeupRequests.id })
|
||||
.from(agentWakeupRequests)
|
||||
.where(and(
|
||||
eq(agentWakeupRequests.companyId, input.companyId),
|
||||
eq(agentWakeupRequests.idempotencyKey, input.idempotencyKey),
|
||||
inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution", "claimed", "completed"]),
|
||||
)).limit(1).then((rows) => rows[0] ?? null);
|
||||
if (existing) return existing.id;
|
||||
const inserted = await input.tx.insert(agentWakeupRequests).values({
|
||||
companyId: input.companyId,
|
||||
agentId: input.agentId,
|
||||
source: "automation",
|
||||
triggerDetail: "system",
|
||||
reason: input.reason,
|
||||
payload: {
|
||||
issueId: input.issueId,
|
||||
taskId: input.issueId,
|
||||
...input.payload,
|
||||
_paperclipWakeContext: {
|
||||
issueId: input.issueId,
|
||||
taskId: input.issueId,
|
||||
wakeReason: input.reason,
|
||||
source: "native_status_decision",
|
||||
},
|
||||
},
|
||||
requestedByActorType: "system",
|
||||
requestedByActorId: "native-status-committer",
|
||||
idempotencyKey: input.idempotencyKey,
|
||||
}).returning({ id: agentWakeupRequests.id }).then((rows) => rows[0] ?? null);
|
||||
if (!inserted) throw new Error("native_status_wake_not_persisted");
|
||||
return inserted.id;
|
||||
}
|
||||
|
||||
async function validateGovernanceGate(tx: Db, input: {
|
||||
companyId: string;
|
||||
issueId: string;
|
||||
gate: Extract<NativeStatusEffect, { kind: "bind_governance" }>["gate"];
|
||||
executionState: unknown;
|
||||
}) {
|
||||
if (input.gate.kind === "execution_stage") {
|
||||
const state = input.executionState && typeof input.executionState === "object"
|
||||
? input.executionState as Record<string, unknown>
|
||||
: {};
|
||||
if (state.status !== "pending") throw new Error("native_governance_gate_resolved");
|
||||
return input.gate.id;
|
||||
}
|
||||
if (input.gate.kind === "interaction") {
|
||||
const interaction = await tx.select({ id: issueThreadInteractions.id })
|
||||
.from(issueThreadInteractions).where(and(
|
||||
eq(issueThreadInteractions.id, input.gate.id),
|
||||
eq(issueThreadInteractions.companyId, input.companyId),
|
||||
eq(issueThreadInteractions.issueId, input.issueId),
|
||||
eq(issueThreadInteractions.status, "pending"),
|
||||
)).limit(1).then((rows) => rows[0] ?? null);
|
||||
if (!interaction) throw new Error("native_governance_gate_resolved");
|
||||
return interaction.id;
|
||||
}
|
||||
const approval = await tx.select({ id: approvals.id }).from(issueApprovals)
|
||||
.innerJoin(approvals, eq(issueApprovals.approvalId, approvals.id))
|
||||
.where(and(
|
||||
eq(issueApprovals.companyId, input.companyId),
|
||||
eq(issueApprovals.issueId, input.issueId),
|
||||
eq(approvals.companyId, input.companyId),
|
||||
eq(approvals.id, input.gate.id),
|
||||
inArray(approvals.status, ["pending", "revision_requested"]),
|
||||
)).limit(1).then((rows) => rows[0] ?? null);
|
||||
if (!approval) throw new Error("native_governance_gate_resolved");
|
||||
return approval.id;
|
||||
}
|
||||
|
||||
async function materializeDecisionEffect(input: {
|
||||
tx: Db;
|
||||
companyId: string;
|
||||
issue: typeof issues.$inferSelect;
|
||||
runId: string;
|
||||
decisionId: string;
|
||||
effect: NativeStatusEffect;
|
||||
failpoint?: NativeStatusCommitFailpoint;
|
||||
}): Promise<MaterializedEffect> {
|
||||
const { effect } = input;
|
||||
if (effect.kind === "bind_governance") {
|
||||
failAt("governance_materialization", input.failpoint);
|
||||
const targetId = await validateGovernanceGate(input.tx, {
|
||||
companyId: input.companyId,
|
||||
issueId: input.issue.id,
|
||||
gate: effect.gate,
|
||||
executionState: input.issue.executionState,
|
||||
});
|
||||
return { effectKind: effect.kind, targetType: effect.gate.kind, targetId, payload: { gate: effect.gate } };
|
||||
}
|
||||
if (effect.kind === "create_review_interaction") {
|
||||
failAt("interaction_materialization", input.failpoint);
|
||||
const reviewInput: Extract<CreateIssueThreadInteraction, { kind: "request_confirmation" }> = {
|
||||
kind: "request_confirmation",
|
||||
idempotencyKey: `native-review:${input.decisionId}`,
|
||||
sourceRunId: input.runId,
|
||||
title: "Native completion review",
|
||||
summary: "The native runner requires authoritative review before completion.",
|
||||
continuationPolicy: "wake_assignee",
|
||||
payload: {
|
||||
version: 1,
|
||||
prompt: effect.prompt,
|
||||
acceptLabel: "Approve completion",
|
||||
rejectLabel: "Continue work",
|
||||
allowDeclineReason: true,
|
||||
rejectRequiresReason: true,
|
||||
supersedeOnUserComment: false,
|
||||
},
|
||||
};
|
||||
const interaction = await issueThreadInteractionService(input.tx).create(
|
||||
input.issue,
|
||||
reviewInput,
|
||||
{ systemId: "native-status-committer", runId: input.runId },
|
||||
);
|
||||
return {
|
||||
effectKind: effect.kind,
|
||||
targetType: "issue_thread_interaction",
|
||||
targetId: interaction.id,
|
||||
payload: { interactionId: interaction.id, interactionKind: interaction.kind },
|
||||
};
|
||||
}
|
||||
if (effect.kind === "enqueue_continuation") {
|
||||
failAt("continuation_materialization", input.failpoint);
|
||||
const wakeId = await enqueueWake({
|
||||
tx: input.tx,
|
||||
companyId: input.companyId,
|
||||
issueId: input.issue.id,
|
||||
agentId: effect.agentId,
|
||||
reason: effect.continuationKind === "monitor" ? "monitor_due" : "issue_status_changed",
|
||||
idempotencyKey: `native-status:${input.decisionId}:continuation`,
|
||||
payload: {
|
||||
nativeDecisionId: input.decisionId,
|
||||
continuationKind: effect.continuationKind,
|
||||
continuationSummary: effect.summary,
|
||||
continuationIdempotencyKey: effect.idempotencyKey,
|
||||
},
|
||||
});
|
||||
return {
|
||||
effectKind: effect.kind,
|
||||
targetType: "agent_wakeup_request",
|
||||
targetId: wakeId,
|
||||
payload: { continuationKind: effect.continuationKind, summary: effect.summary },
|
||||
};
|
||||
}
|
||||
if (effect.kind === "bind_blocker") {
|
||||
failAt("blocker_materialization", input.failpoint);
|
||||
let wakeId: string | null = null;
|
||||
if (effect.owner !== "board") {
|
||||
wakeId = await enqueueWake({
|
||||
tx: input.tx,
|
||||
companyId: input.companyId,
|
||||
issueId: input.issue.id,
|
||||
agentId: effect.owner.agentId,
|
||||
reason: "issue_status_changed",
|
||||
idempotencyKey: `native-status:${input.decisionId}:blocker-owner`,
|
||||
payload: { nativeDecisionId: input.decisionId, unblockAction: effect.action },
|
||||
});
|
||||
}
|
||||
return {
|
||||
effectKind: effect.kind,
|
||||
targetType: effect.owner === "board" ? "board" : "agent_wakeup_request",
|
||||
targetId: wakeId,
|
||||
payload: { owner: effect.owner, action: effect.action },
|
||||
};
|
||||
}
|
||||
if (effect.kind === "record_recovery") {
|
||||
failAt("recovery_materialization", input.failpoint);
|
||||
const recovery = await issueRecoveryActionService(input.tx).upsertSourceScoped({
|
||||
companyId: input.companyId,
|
||||
sourceIssueId: input.issue.id,
|
||||
kind: "active_run_watchdog",
|
||||
ownerType: "agent",
|
||||
ownerAgentId: effect.agentId,
|
||||
cause: effect.cause,
|
||||
fingerprint: nativeSha256({ runId: input.runId, decisionId: input.decisionId, cause: effect.cause }),
|
||||
evidence: { runId: input.runId, decisionId: input.decisionId },
|
||||
nextAction: effect.nextAction,
|
||||
wakePolicy: { kind: "resume_native_run", runId: input.runId },
|
||||
maxAttempts: 3,
|
||||
});
|
||||
return {
|
||||
effectKind: effect.kind,
|
||||
targetType: "issue_recovery_action",
|
||||
targetId: recovery.id,
|
||||
payload: { cause: effect.cause, nextAction: effect.nextAction },
|
||||
};
|
||||
}
|
||||
return {
|
||||
effectKind: effect.kind,
|
||||
targetType: "issue_checkout",
|
||||
targetId: input.issue.id,
|
||||
payload: { checkoutRunId: input.issue.checkoutRunId, executionRunId: input.issue.executionRunId },
|
||||
};
|
||||
}
|
||||
|
||||
export async function commitNativeStatusDecision(input: {
|
||||
db: Db;
|
||||
companyId: string;
|
||||
@@ -26,104 +258,201 @@ export async function commitNativeStatusDecision(input: {
|
||||
assessmentId: string;
|
||||
priorStatus: string;
|
||||
priorStatusVersion: number;
|
||||
priorDecisionId: string | null;
|
||||
decision: NativeStatusDecision;
|
||||
failpoint?: NativeStatusCommitFailpoint;
|
||||
}) {
|
||||
return input.db.transaction(async (tx) => {
|
||||
const publications: ActivityPublication[] = [];
|
||||
const committed = await input.db.transaction(async (tx) => {
|
||||
const coordinator = await tx.select().from(nativeRunFinalizations).where(and(
|
||||
eq(nativeRunFinalizations.runId, input.runId),
|
||||
eq(nativeRunFinalizations.companyId, input.companyId),
|
||||
eq(nativeRunFinalizations.issueId, input.issueId),
|
||||
)).for("update").limit(1).then((rows) => rows[0] ?? null);
|
||||
if (!coordinator) throw new Error("native_finalization_coordinator_missing");
|
||||
const issue = await tx.select().from(issues).where(and(
|
||||
eq(issues.id, input.issueId),
|
||||
eq(issues.companyId, input.companyId),
|
||||
)).for("update").limit(1).then((rows) => rows[0] ?? null);
|
||||
if (!issue || issue.status !== input.priorStatus || issue.statusVersion !== input.priorStatusVersion) {
|
||||
if (
|
||||
!issue
|
||||
|| issue.status !== input.priorStatus
|
||||
|| Number(issue.statusVersion) !== input.priorStatusVersion
|
||||
|| issue.lastStatusDecisionId !== input.priorDecisionId
|
||||
) {
|
||||
throw new NativeStatusRaceError();
|
||||
}
|
||||
const decisionJson = {
|
||||
toStatus: input.decision.toStatus,
|
||||
reasonCode: input.decision.reasonCode,
|
||||
unblockDescriptor: input.decision.unblockDescriptor,
|
||||
effects: input.decision.effects,
|
||||
};
|
||||
const decisionDigest = nativeSha256({
|
||||
issueId: input.issueId,
|
||||
assessmentId: input.assessmentId,
|
||||
policyVersion: input.decision.policyVersion,
|
||||
fromStatus: issue.status,
|
||||
priorStatusVersion: input.priorStatusVersion,
|
||||
priorDecisionId: input.priorDecisionId,
|
||||
decision: decisionJson,
|
||||
});
|
||||
let decisionRow = await tx.select().from(statusDecisions).where(and(
|
||||
eq(statusDecisions.issueId, input.issueId),
|
||||
eq(statusDecisions.decisionDigest, decisionDigest),
|
||||
)).limit(1).then((rows) => rows[0] ?? null);
|
||||
if (decisionRow?.applicationState === "applied") {
|
||||
return { decision: decisionRow, issue, replayed: true };
|
||||
}
|
||||
if (!decisionRow) {
|
||||
[decisionRow] = await tx.insert(statusDecisions).values({
|
||||
companyId: input.companyId,
|
||||
issueId: input.issueId,
|
||||
assessmentId: input.assessmentId,
|
||||
decisionVersion: 1,
|
||||
decisionVersion: input.priorStatusVersion + 1,
|
||||
policyVersion: input.decision.policyVersion,
|
||||
fromStatus: issue.status,
|
||||
toStatus: input.decision.toStatus,
|
||||
reasonCode: input.decision.reasonCode,
|
||||
decisionJson,
|
||||
decisionDigest,
|
||||
applicationState: "pending",
|
||||
applicationState: "proposed",
|
||||
}).returning();
|
||||
}
|
||||
if (!decisionRow) throw new Error("native_status_decision_not_persisted");
|
||||
const updated = await tx.update(issues).set({
|
||||
|
||||
const materialized: MaterializedEffect[] = [];
|
||||
for (const effect of input.decision.effects) {
|
||||
materialized.push(await materializeDecisionEffect({
|
||||
tx: tx as unknown as Db,
|
||||
companyId: input.companyId,
|
||||
issue,
|
||||
runId: input.runId,
|
||||
decisionId: decisionRow.id,
|
||||
effect,
|
||||
failpoint: input.failpoint,
|
||||
}));
|
||||
}
|
||||
|
||||
failAt("status_projection", input.failpoint);
|
||||
const updated = await issueService(tx as unknown as Db).update(input.issueId, {
|
||||
status: input.decision.toStatus,
|
||||
unblockDescriptor: input.decision.unblockDescriptor,
|
||||
statusVersion: input.priorStatusVersion + 1,
|
||||
lastStatusDecisionId: decisionRow.id,
|
||||
completedAt: input.decision.toStatus === "done" ? new Date() : null,
|
||||
blockedTransitionAt: input.decision.toStatus === "blocked" ? new Date() : null,
|
||||
updatedAt: new Date(),
|
||||
}).where(and(
|
||||
eq(issues.id, input.issueId),
|
||||
eq(issues.statusVersion, input.priorStatusVersion),
|
||||
)).returning().then((rows) => rows[0] ?? null);
|
||||
unblockDescriptor: input.decision.unblockDescriptor,
|
||||
actorAgentId: null,
|
||||
actorUserId: null,
|
||||
}, tx, publications);
|
||||
if (!updated) throw new NativeStatusRaceError();
|
||||
await tx.insert(statusDecisionEffects).values({
|
||||
companyId: input.companyId,
|
||||
issueId: input.issueId,
|
||||
decisionId: decisionRow.id,
|
||||
ordinal: 1,
|
||||
materialized.unshift({
|
||||
effectKind: "issue_status_projection",
|
||||
targetType: "issue",
|
||||
targetId: input.issueId,
|
||||
idempotencyKey: `native-status:${decisionRow.id}:1`,
|
||||
payload: {
|
||||
fromStatus: issue.status,
|
||||
toStatus: input.decision.toStatus,
|
||||
reasonCode: input.decision.reasonCode,
|
||||
},
|
||||
deliveryState: "delivered",
|
||||
attemptCount: 1,
|
||||
deliveredAt: new Date(),
|
||||
}).onConflictDoNothing();
|
||||
await tx.update(statusDecisions).set({
|
||||
applicationState: "applied",
|
||||
appliedAt: new Date(),
|
||||
}).where(eq(statusDecisions.id, decisionRow.id));
|
||||
payload: { fromStatus: issue.status, toStatus: input.decision.toStatus, reasonCode: input.decision.reasonCode },
|
||||
});
|
||||
|
||||
if (input.decision.toStatus === "done" && issue.status !== "done") {
|
||||
const issueSvc = issueService(tx as unknown as Db);
|
||||
const dependents = await issueSvc.listWakeableBlockedDependents(input.issueId);
|
||||
for (const dependent of dependents) {
|
||||
const idempotencyKey = buildIssueBlockersResolvedWakeIdempotencyKey({
|
||||
dependentIssueId: dependent.id,
|
||||
resolvedBlockerIssueId: input.issueId,
|
||||
});
|
||||
const wakeId = await enqueueWake({
|
||||
tx: tx as unknown as Db,
|
||||
companyId: input.companyId,
|
||||
issueId: dependent.id,
|
||||
agentId: dependent.assigneeAgentId,
|
||||
reason: "issue_blockers_resolved",
|
||||
idempotencyKey,
|
||||
payload: { resolvedBlockerIssueId: input.issueId, blockerIssueIds: dependent.blockerIssueIds },
|
||||
});
|
||||
materialized.push({
|
||||
effectKind: "dependency_wake",
|
||||
targetType: "agent_wakeup_request",
|
||||
targetId: wakeId,
|
||||
payload: { dependentIssueId: dependent.id, resolvedBlockerIssueId: input.issueId },
|
||||
});
|
||||
}
|
||||
if (issue.parentId) {
|
||||
const parent = await issueSvc.getWakeableParentAfterChildCompletion(issue.parentId);
|
||||
if (parent) {
|
||||
const wakeId = await enqueueWake({
|
||||
tx: tx as unknown as Db,
|
||||
companyId: input.companyId,
|
||||
issueId: parent.id,
|
||||
agentId: parent.assigneeAgentId,
|
||||
reason: "issue_children_completed",
|
||||
idempotencyKey: `issue_children_completed:${parent.id}:${input.issueId}`,
|
||||
payload: { completedChildIssueId: input.issueId, childIssueIds: parent.childIssueIds },
|
||||
});
|
||||
materialized.push({
|
||||
effectKind: "parent_wake",
|
||||
targetType: "agent_wakeup_request",
|
||||
targetId: wakeId,
|
||||
payload: { parentIssueId: parent.id, completedChildIssueId: input.issueId },
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const [index, effect] of materialized.entries()) {
|
||||
await tx.insert(statusDecisionEffects).values({
|
||||
companyId: input.companyId,
|
||||
issueId: input.issueId,
|
||||
decisionId: decisionRow.id,
|
||||
ordinal: index + 1,
|
||||
effectKind: effect.effectKind,
|
||||
targetType: effect.targetType,
|
||||
targetId: effect.targetId,
|
||||
idempotencyKey: `native-status:${decisionRow.id}:${index + 1}`,
|
||||
payload: effect.payload,
|
||||
deliveryState: "delivered",
|
||||
attemptCount: 1,
|
||||
deliveredAt: new Date(),
|
||||
}).onConflictDoNothing();
|
||||
}
|
||||
await tx.update(statusDecisions).set({ applicationState: "applied", appliedAt: new Date() })
|
||||
.where(eq(statusDecisions.id, decisionRow.id));
|
||||
await tx.update(nativeRunFinalizations).set({
|
||||
phase: "applied",
|
||||
phase: "committed",
|
||||
assessmentId: input.assessmentId,
|
||||
decisionId: decisionRow.id,
|
||||
leaseOwner: null,
|
||||
leaseExpiresAt: null,
|
||||
failureCode: null,
|
||||
failureDetail: null,
|
||||
nextAttemptAt: null,
|
||||
updatedAt: new Date(),
|
||||
}).where(eq(nativeRunFinalizations.runId, input.runId));
|
||||
await tx.insert(activityLog).values({
|
||||
const { publication } = await persistActivity(tx as unknown as Db, {
|
||||
companyId: input.companyId,
|
||||
actorType: "system",
|
||||
actorId: "native-status-committer",
|
||||
action: "issue.native_status_decision_applied",
|
||||
action: "issue.updated",
|
||||
entityType: "issue",
|
||||
entityId: input.issueId,
|
||||
issueId: input.issueId,
|
||||
runId: input.runId,
|
||||
details: {
|
||||
source: "native_status_decision",
|
||||
assessmentId: input.assessmentId,
|
||||
decisionId: decisionRow.id,
|
||||
fromStatus: issue.status,
|
||||
toStatus: input.decision.toStatus,
|
||||
reasonCode: input.decision.reasonCode,
|
||||
effectCount: materialized.length,
|
||||
},
|
||||
});
|
||||
return { decision: { ...decisionRow, applicationState: "applied" }, issue: updated };
|
||||
publications.push(publication);
|
||||
return {
|
||||
decision: { ...decisionRow, applicationState: "applied" },
|
||||
issue: updated,
|
||||
replayed: false,
|
||||
};
|
||||
});
|
||||
|
||||
for (const publication of publications) publishActivity(publication);
|
||||
return committed;
|
||||
}
|
||||
@@ -11,7 +11,9 @@ export async function recordNativeWorkAssessment(input: {
|
||||
runId: string;
|
||||
turnId: string | null;
|
||||
contractId: string;
|
||||
contractCanonicalSha256: string;
|
||||
resultId: string;
|
||||
resultCanonicalSha256: string;
|
||||
priorIssueStatus: string;
|
||||
priorStatusVersion: number;
|
||||
priorDecisionId: string | null;
|
||||
@@ -22,14 +24,21 @@ export async function recordNativeWorkAssessment(input: {
|
||||
const assessmentJson = input.assessment as unknown as Record<string, unknown>;
|
||||
const inputDigest = nativeSha256({
|
||||
contractId: input.contractId,
|
||||
contractCanonicalSha256: input.contractCanonicalSha256,
|
||||
resultId: input.resultId,
|
||||
resultCanonicalSha256: input.resultCanonicalSha256,
|
||||
priorIssueStatus: input.priorIssueStatus,
|
||||
priorStatusVersion: input.priorStatusVersion,
|
||||
priorDecisionId: input.priorDecisionId,
|
||||
triggerKind: "native_result",
|
||||
triggerRef: input.resultId,
|
||||
triggerActorCompanyId: input.companyId,
|
||||
triggerCapability: "server_native_finalizer",
|
||||
policyVersion: input.policyVersion,
|
||||
assessment: assessmentJson,
|
||||
});
|
||||
const existing = await input.db.select().from(workAssessments).where(and(
|
||||
eq(workAssessments.resultId, input.resultId),
|
||||
eq(workAssessments.issueId, input.issueId),
|
||||
eq(workAssessments.inputDigest, inputDigest),
|
||||
)).limit(1).then((rows) => rows[0] ?? null);
|
||||
if (existing) return existing;
|
||||
|
||||
Reference in new issue
Block a user