Merge remote-tracking branch 'origin/master' into codex/provider-routing-connections

* origin/master:
  fix(tasks): surface Codex ChatGPT model rejection (#15299)
  fix(runner): continue restart-interrupted Codex turns (#15297)
  fix(agents): grant configuration access by default (#15283)
  Guard routine UUID lookups without narrowing valid inputs (#15313)
  fix(ssh): transport project repositories as their own git checkouts (#14782)
  feat(exe-dev): copy a source VM with exe.dev cp (#14975)
  Diagnose native model rejections and preserve repairable reviews (#15304)
  Verify native semantic input against its raw wire digest (#15301)
  Retry pooled execution projection reads after disconnects (#15300)
  Fix browser polling for unsaved agent chats (#15298)
  test(ui): keep initial reasoning with the saved reply (#15107)
  docs: add a product feature map (#15288)
This commit is contained in:
Dotta committed 2026-10-06 08:42:29 -05:00
commit 7cbd61349a
126 files changed
+5961 -150

No files matched your search

@@ -7,6 +7,19 @@ const run = (overrides: Partial<Run> = {}) => ({ resultJson: null, ...overrides
const collect = (error: unknown) => collectRunFailureDiagnostics(run(), { error });
describe("run failure diagnostics", () => {
it("exports only the closed native model/auth rejection vocabulary", () => {
const diagnostic = { provider: "codex", category: "model_auth_incompatible", status: 400, authMode: "chatgpt" };
const result = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics(run({ resultJson: {
nativeProviderFailure: { ...diagnostic, model: "private-model", response: "private-provider-response", prompt: "private-prompt" },
} }), {}));
expect(result.provider).toEqual(diagnostic);
expect(JSON.stringify(result)).not.toContain("private");
for (const value of [null, [], { ...diagnostic, category: "private-category" },
Object.defineProperty({}, "provider", { get() { throw new Error("private"); } })]) {
expect(collectRunFailureDiagnostics(run({ resultJson: { nativeProviderFailure: value } }), {}).provider).toEqual({});
}
});
it("selects bounded lock-owner evidence from a caught timeout cause", () => {
const error = new Error("outer", { cause: Object.assign(new Error("lock timeout"), {
code: "ERR_WORKSPACE_RESTORE_LOCK_TIMEOUT",
+4 -1
View File
@@ -403,6 +403,9 @@ export function activityService(db: Db) {
invocationSource: heartbeatRuns.invocationSource,
responsibleUserId: heartbeatRuns.responsibleUserId,
errorCode: heartbeatRuns.errorCode,
error: sql<string | null>`case when ${heartbeatRuns.status} = 'failed'
and ${heartbeatRuns.errorCode} = 'native_provider_model_rejected'
then left(${heartbeatRuns.error}, 2000) else null end`,
usageJson: summarizedUsageJson,
resultJson: summarizedResultJson,
logBytes: heartbeatRuns.logBytes,
@@ -497,7 +500,7 @@ export function activityService(db: Db) {
const [exhaustionRows, leaseRows, executionByRunId, [savedPlan]] = await Promise.all([
exhaustionRowsQuery,
leaseRowsQuery,
executionProjectionsForRuns(db, companyId, runIds),
executionProjectionsForRuns(db, companyId, runIds, new Date(), { retryDatabaseReads: true }),
savedPlanQuery,
]);
const retryExhaustedReasonByRunId = new Map<string, string>();
+28 -1
View File
@@ -1,4 +1,31 @@
import { LOW_TRUST_REVIEW_PRESET } from "@paperclipai/shared";
import { LOW_TRUST_REVIEW_PRESET, type PermissionKey } from "@paperclipai/shared";
/** Direct grants for newly activated standard agents. Existing agents are not backfilled. */
export const NEW_STANDARD_AGENT_DEFAULT_GRANT_KEYS = [
"agents:configure",
"agents:suggest-changes",
"skills:create",
"skills:suggest-changes",
"tools:manage_connections",
"tools:manage_profiles",
"tools:view_audit",
"audit:view_agent_actions",
"tools:use",
"tools:manage_runtime",
"inbox:manage",
"tasks:assign",
"tasks:assign_scope",
"tasks:manage_active_checkouts",
] as const satisfies readonly PermissionKey[];
export function newStandardAgentGrantScope(
permissionKey: PermissionKey,
agentId: string,
): Record<string, unknown> | null {
if (permissionKey === "tasks:assign_scope") return { subtreeRootAgentId: agentId };
if (permissionKey === "inbox:manage") return { responsibleUserOnly: true };
return null;
}
export type NormalizedAgentPermissions = Record<string, unknown> & {
canCreateAgents: boolean;
+32 -1
View File
@@ -17,6 +17,7 @@ import {
issueExecutionDecisions,
issues,
issueComments,
principalPermissionGrants,
} from "@paperclipai/db";
import {
AGENT_DEFAULT_MAX_CONCURRENT_RUNS,
@@ -38,8 +39,8 @@ import {
syncAgentAdapterEnvBindings,
} from "./agent-secret-bindings.js";
import { logActivity } from "./activity-log.js";
import { NEW_STANDARD_AGENT_DEFAULT_GRANT_KEYS, newStandardAgentGrantScope, normalizeAgentPermissions, permissionsImplyLowTrust } from "./agent-permissions.js";
import { recordAgentStatusEvent, recordResourceCreationEvent } from "./resource-lifecycle-events.js";
import { normalizeAgentPermissions } from "./agent-permissions.js";
import { REDACTED_EVENT_VALUE, sanitizeRecord } from "../redaction.js";
import {
assertClaudeOAuthBindingInvariant,
@@ -944,6 +945,20 @@ export function agentService(db: Db) {
})
.returning()
.then((rows) => rows[0]);
// New standard agents receive the standard direct grants at activation.
// Low-trust and bundled agents keep their explicit, narrower grants.
if (created.status !== "pending_approval" && !permissionsImplyLowTrust(normalizedPermissions) &&
!readBuiltInAgentMarker(created.metadata)) {
await tx.insert(principalPermissionGrants).values(
NEW_STANDARD_AGENT_DEFAULT_GRANT_KEYS.map((permissionKey) => ({
companyId,
principalType: "agent" as const,
principalId: created.id,
permissionKey,
scope: newStandardAgentGrantScope(permissionKey, created.id),
})),
).onConflictDoNothing();
}
if (options?.aiConnectionInstall) {
const install = options.aiConnectionInstall;
const connectionIds = [...new Set([install.connectionId, ...(install.memberConnectionIds ?? [])])];
@@ -1084,6 +1099,11 @@ export function agentService(db: Db) {
await tx.delete(agentWakeupRequests).where(eq(agentWakeupRequests.agentId, id));
await tx.delete(agentApiKeys).where(eq(agentApiKeys.agentId, id));
await tx.delete(agentRuntimeState).where(eq(agentRuntimeState.agentId, id));
await tx.delete(principalPermissionGrants).where(and(
eq(principalPermissionGrants.companyId, existing.companyId),
eq(principalPermissionGrants.principalType, "agent"),
eq(principalPermissionGrants.principalId, id),
));
const deleted = await tx
.delete(agents)
.where(eq(agents.id, id))
@@ -1151,6 +1171,17 @@ export function agentService(db: Db) {
});
}
await syncAgentSecretBindings(updated, txDb, existing.adapterConfig);
if (!permissionsImplyLowTrust(updated.permissions) && !readBuiltInAgentMarker(updated.metadata)) {
await tx.insert(principalPermissionGrants).values(
NEW_STANDARD_AGENT_DEFAULT_GRANT_KEYS.map((permissionKey) => ({
companyId: updated.companyId,
principalType: "agent" as const,
principalId: updated.id,
permissionKey,
scope: newStandardAgentGrantScope(permissionKey, updated.id),
})),
).onConflictDoNothing();
}
await recordResourceCreationEvent(txDb, existing.companyId, "agent", updated.id);
const agent = await agentService(txDb).getById(updated.id);
if (!agent) {
+2 -1
View File
@@ -2060,7 +2060,8 @@ export function authorizationService(db: Db | DbTransaction) {
// implicit default-open policy remains responsible-user-only so an
// absent row never becomes a company-wide cross-user grant.
const grant = await findGrant(companyId, "agent", actorAgentId, "inbox:manage");
if (grant && (await scopeAllows(db, companyId, grant.scope, { userId: targetUserId }))) {
if (grant && grant.scope?.responsibleUserOnly !== true &&
(await scopeAllows(db, companyId, grant.scope, { userId: targetUserId }))) {
return allow({
action: input.action,
reason: "allow_explicit_grant",
@@ -0,0 +1,150 @@
import { getTableName, type SQL } from "drizzle-orm";
import { PgDialect } from "drizzle-orm/pg-core";
import type { Db } from "@paperclipai/db";
import { afterEach, describe, expect, it, vi } from "vitest";
import { activityService } from "./activity.js";
import { executionProjectionsForRuns } from "./execution-projection.js";
import { logger } from "../middleware/logger.js";
const companyId = "00000000-0000-4000-8000-000000000001";
const issueId = "00000000-0000-4000-8000-000000000002";
const runId = "00000000-0000-4000-8000-000000000003";
const now = new Date("2026-01-01T00:00:00Z");
const tables = ["heartbeat_runs", "native_run_finalizations", "issue_thread_interactions", "issue_recovery_actions"];
const dialect = new PgDialect();
function disconnected(code = "CONNECTION_CLOSED") {
return new Error("Failed query", { cause: Object.assign(new Error("Connection failed"), { code }) });
}
function fixture(options: {
failures?: Record<string, Error[]>;
missing?: boolean;
noIssue?: boolean;
backfill?: boolean;
writeFailure?: Error;
} = {}) {
const attempts: Array<{ key: string; params: unknown[]; order: string[] }> = [];
const row = {
id: runId, companyId, runId, status: "succeeded", runtimeMode: "native", errorCode: null,
nativeIssueId: options.noIssue ? null : issueId, contextSnapshot: {},
finishedAt: now, startedAt: now, createdAt: now, resultJson: null, usageJson: null,
scheduledRetryAt: null, scheduledRetryAttempt: 0, retryOfRunId: null, nextAction: null,
};
const select = vi.fn((fields: Record<string, unknown> = {}) => ({
from(table: Parameters<typeof getTableName>[0]) {
const name = getTableName(table);
const key = name === "heartbeat_runs" && "runId" in fields ? "run_list"
: name === "heartbeat_runs" && "stdoutExcerpt" in fields ? "liveness_backfill"
: name;
let condition: SQL;
let order: SQL[] = [];
let result: Promise<unknown[]> | undefined;
const query = {
where(value: SQL) { condition = value; return query; },
innerJoin() { return query; },
orderBy(...values: SQL[]) { order = values; return query; },
limit() { return query; },
then(resolve: (value: unknown[]) => unknown, reject: (error: unknown) => unknown) {
// Re-awaiting the same failed builder cannot recover the connection.
result ??= Promise.resolve().then(() => {
const attempt = attempts.filter((read) => read.key === key).length;
attempts.push({ key, params: dialect.sqlToQuery(condition).params,
order: order.map((value) => dialect.sqlToQuery(value).sql) });
const error = options.failures?.[key]?.[attempt];
if (error) throw error;
if (options.missing) return [];
if (key === "heartbeat_runs" || key === "run_list") return [row];
if (key === "liveness_backfill") return options.backfill ? [row] : [];
if (key === "issues") return [{ status: "in_progress", title: "Fixture", workMode: "standard" }];
if (key === "issue_thread_interactions") return [{ issueId, kind: "ask_user_questions" }];
return [];
});
return result.then(resolve, reject);
},
};
return query;
},
}));
const updateWhere = vi.fn(async () => { if (options.writeFailure) throw options.writeFailure; });
const update = vi.fn(() => ({ set: () => ({ where: updateWhere }) }));
const transaction = vi.fn(() => { throw new Error("Must not start or replay a transaction"); });
const db = { select, update, transaction } as unknown as Db;
return { db, attempts, select, update, updateWhere, transaction,
count: (key: string) => attempts.filter((read) => read.key === key).length };
}
afterEach(() => vi.restoreAllMocks());
describe("execution projection read retries", () => {
it.each(tables)("rebuilds only the failed %s SELECT within the existing retry budget", async (table) => {
const test = fixture({ failures: { [table]: [disconnected(), disconnected("CONNECTION_ENDED")] } });
const result = await executionProjectionsForRuns(test.db, companyId, [runId], now, { retryDatabaseReads: true });
expect(result.get(runId)).toMatchObject({ phase: "waiting_for_answer", label: "Waiting for answer" });
for (const name of tables) expect(test.count(name)).toBe(name === table ? 3 : 1);
expect(test.select).toHaveBeenCalledTimes(6);
const reads = test.attempts.filter((read) => read.key === table);
expect(reads[1]).toEqual(reads[0]);
expect(reads[2]).toEqual(reads[0]);
expect(reads[0].params).toContain(companyId);
expect(reads[0].params).toContain(table === "heartbeat_runs" || table === "native_run_finalizations" ? runId : issueId);
if (table === "issue_thread_interactions") expect(reads[0].params).toContain("pending");
if (table === "issue_recovery_actions") expect(reads[0].order).toEqual(['"issue_recovery_actions"."updated_at" desc']);
expect(test.update).not.toHaveBeenCalled();
expect(test.transaction).not.toHaveBeenCalled();
});
it.each(tables)("preserves the last %s error after three attempts", async (table) => {
const errors = [disconnected(), disconnected(), disconnected()];
const test = fixture({ failures: { [table]: errors } });
await expect(executionProjectionsForRuns(test.db, companyId, [runId], now, { retryDatabaseReads: true })).rejects.toBe(errors[2]);
for (const [index, name] of tables.entries()) {
expect(test.count(name)).toBe(index < tables.indexOf(table) ? 1 : name === table ? 3 : 0);
}
});
it.each(tables)("keeps the default transaction-compatible %s path single-attempt", async (table) => {
const error = disconnected();
const test = fixture({ failures: { [table]: [error] } });
await expect(executionProjectionsForRuns(test.db, companyId, [runId], now)).rejects.toBe(error);
expect(test.count(table)).toBe(1);
expect(test.transaction).not.toHaveBeenCalled();
});
it.each([new Error("CONNECTION_CLOSED"), disconnected("28P01"), disconnected("42501"), disconnected("22P02"), disconnected("25P02")])(
"does not retry an unclassified or non-transient error", async (error) => {
const test = fixture({ failures: { issue_thread_interactions: [error] } });
await expect(executionProjectionsForRuns(test.db, companyId, [runId], now, { retryDatabaseReads: true })).rejects.toBe(error);
expect(test.count("issue_thread_interactions")).toBe(1);
expect(test.count("issue_recovery_actions")).toBe(0);
},
);
it("preserves empty and taskless projection behavior without unnecessary reads", async () => {
const empty = fixture();
expect(await executionProjectionsForRuns(empty.db, companyId, [], now, { retryDatabaseReads: true })).toEqual(new Map());
expect(empty.select).not.toHaveBeenCalled();
for (const options of [{ missing: true }, { noIssue: true }]) {
const test = fixture(options);
const result = await executionProjectionsForRuns(test.db, companyId, [runId], now, { retryDatabaseReads: true });
expect(test.select).toHaveBeenCalledTimes(2);
expect(options.missing ? result.size : result.get(runId)?.phase).toBe(options.missing ? 0 : "completed");
}
});
it.each([false, true])("the run list retries projection reads without replaying other reads or backfill writes (write fails: %s)", async (writeFails) => {
vi.spyOn(logger, "warn").mockImplementation(() => undefined);
const test = fixture({ backfill: true, writeFailure: writeFails ? disconnected() : undefined,
failures: { issue_thread_interactions: [disconnected(), disconnected()] } });
const result = await activityService(test.db).runsForIssue(companyId, issueId);
expect(result).toMatchObject([{ runId, execution: { phase: "waiting_for_answer" } }]);
expect(test.count("issue_thread_interactions")).toBe(3);
for (const key of ["run_list", "liveness_backfill", "heartbeat_runs", "native_run_finalizations", "environment_leases", "issue_recovery_actions"]) {
expect(test.count(key)).toBe(1);
}
// One retry-exhaustion lookup plus the independent backfill's event count.
expect(test.count("heartbeat_run_events")).toBe(2);
expect(test.update).toHaveBeenCalledTimes(1);
expect(test.updateWhere).toHaveBeenCalledTimes(1);
expect(test.transaction).not.toHaveBeenCalled();
});
});
+15 -8
View File
@@ -9,6 +9,7 @@ import {
import type { ExecutionProjection } from "@paperclipai/shared";
import { EXECUTION_CONTROL_DEADLINE_MS } from "./execution-control-deadline.js";
import { executionFailureRetryCount } from "./execution-recovery-attempt.js";
import { retryIdempotentDatabaseOperation } from "../database-retry.js";
const text = (v: unknown) => (typeof v === "string" ? v : null);
const executionRunColumns = {
id: heartbeatRuns.id,
@@ -48,10 +49,16 @@ export async function executionProjectionsForRuns(
companyId: string,
runIds: string[],
now = new Date(),
options: { retryDatabaseReads?: boolean } = {},
) {
const projections = new Map<string, ExecutionProjection>();
if (!runIds.length) return projections;
const runs = await db
// Opt in only with a pooled Db outside a transaction. Issue enrichment also
// calls this helper with a transaction, whose failed reads must not replay.
const read = <T>(query: () => Promise<T>) => options.retryDatabaseReads
? retryIdempotentDatabaseOperation(query)
: query();
const runs = await read(async () => db
.select(executionRunColumns)
.from(heartbeatRuns)
.where(
@@ -59,8 +66,8 @@ export async function executionProjectionsForRuns(
eq(heartbeatRuns.companyId, companyId),
inArray(heartbeatRuns.id, runIds),
),
);
const coordinators = await db
));
const coordinators = await read(async () => db
.select()
.from(nativeRunFinalizations)
.where(
@@ -68,7 +75,7 @@ export async function executionProjectionsForRuns(
eq(nativeRunFinalizations.companyId, companyId),
inArray(nativeRunFinalizations.runId, runIds),
),
);
));
const issueIds = [
...new Set(
runs
@@ -77,7 +84,7 @@ export async function executionProjectionsForRuns(
),
];
const pending = issueIds.length
? await db
? await read(async () => db
.select({
issueId: issueThreadInteractions.issueId,
kind: issueThreadInteractions.kind,
@@ -89,10 +96,10 @@ export async function executionProjectionsForRuns(
inArray(issueThreadInteractions.issueId, issueIds),
eq(issueThreadInteractions.status, "pending"),
),
)
))
: [];
const recovery = issueIds.length
? await db
? await read(async () => db
.select({
issueId: issueRecoveryActions.sourceIssueId,
cause: issueRecoveryActions.cause,
@@ -112,7 +119,7 @@ export async function executionProjectionsForRuns(
]),
),
)
.orderBy(desc(issueRecoveryActions.updatedAt))
.orderBy(desc(issueRecoveryActions.updatedAt)))
: [];
const coordinatorByRun = new Map(coordinators.map((row) => [row.runId, row]));
for (const run of runs) {
+10 -10
View File
@@ -1,5 +1,6 @@
import { PERMISSION_KEYS } from "@paperclipai/shared";
import type { HumanCompanyMembershipRole } from "@paperclipai/shared";
import { NEW_STANDARD_AGENT_DEFAULT_GRANT_KEYS, newStandardAgentGrantScope } from "./agent-permissions.js";
import { grantsForHumanRole } from "./company-member-roles.js";
export function grantsFromDefaults(
@@ -38,22 +39,21 @@ export function grantsFromDefaults(
}
export function agentJoinGrantsFromDefaults(
defaultsPayload: Record<string, unknown> | null | undefined
defaultsPayload: Record<string, unknown> | null | undefined,
agentId: string,
): Array<{
permissionKey: (typeof PERMISSION_KEYS)[number];
scope: Record<string, unknown> | null;
}> {
const grants = grantsFromDefaults(defaultsPayload, "agent");
if (grants.some((grant) => grant.permissionKey === "tasks:assign")) {
return grants;
// Invitation approval replaces the whole grant set after agent creation.
// Retain new-agent defaults, but never widen an explicitly scoped grant.
for (const permissionKey of NEW_STANDARD_AGENT_DEFAULT_GRANT_KEYS) {
if (!grants.some((grant) => grant.permissionKey === permissionKey)) {
grants.push({ permissionKey, scope: newStandardAgentGrantScope(permissionKey, agentId) });
}
}
return [
...grants,
{
permissionKey: "tasks:assign",
scope: null,
},
];
return grants;
}
export function humanJoinGrantsFromDefaults(
@@ -0,0 +1,45 @@
import { and, desc, eq, sql } from "drizzle-orm";
import { heartbeatRunEvents, heartbeatRuns, type Db } from "@paperclipai/db";
import { parseNativeExecutionInput, validatePrpEvent, type PrpTerminalState } from "../../vendor/paperclip-runner/index.js";
import { nativeSha256 } from "./canonical.js";
import { createNativeProviderFailureObservation } from "./native-provider-failure.js";
function record(value: unknown): Record<string, unknown> {
return value !== null && typeof value === "object" && !Array.isArray(value)
? value as Record<string, unknown> : {};
}
/** Re-derive authority from immutable run-bound evidence, including after restart. */
export async function readPersistedNativeModelRejection(
db: Db,
run: typeof heartbeatRuns.$inferSelect,
turnId: string | null,
terminal: PrpTerminalState,
) {
if (run.runtimeMode !== "native" || !turnId || !run.nativeSessionId || !run.runnerInstanceId ||
terminal.turnTerminalState !== "failed" || terminal.runTerminalState !== "failed") return null;
let execution;
try { execution = parseNativeExecutionInput(record(run.runnerProfileJson).nativeExecutionInput); } catch { return null; }
if (execution.provider.kind !== "codex" || !execution.provider.model ||
execution.binding.runId !== run.id || execution.binding.companyId !== run.companyId ||
execution.binding.agentId !== run.agentId || execution.binding.issueId !== run.nativeIssueId ||
execution.session.normalizedSessionId !== run.nativeSessionId ||
execution.completionContract.id !== run.completionContractId ||
execution.completionContract.sha256 !== run.completionContractSha256) return null;
const [row] = await db.select().from(heartbeatRunEvents).where(and(
eq(heartbeatRunEvents.companyId, run.companyId), eq(heartbeatRunEvents.runId, run.id),
eq(heartbeatRunEvents.sourceInstanceId, run.runnerInstanceId), eq(heartbeatRunEvents.eventType, "turn.failed"),
sql`${heartbeatRunEvents.payload}->'prpEvent'->>'turnId' = ${turnId}`,
)).orderBy(desc(heartbeatRunEvents.seq)).limit(1);
if (!row) return null;
const parsed = validatePrpEvent(record(row.payload).prpEvent);
if (!parsed.ok || parsed.event.runId !== run.id || parsed.event.normalizedSessionId !== run.nativeSessionId ||
parsed.event.sourceKind !== "runner" || parsed.event.sourceInstanceId !== run.runnerInstanceId ||
parsed.event.sourceEventId !== row.sourceEventId || parsed.event.sourceSeq !== row.sourceSeq ||
parsed.event.eventType !== row.eventType || parsed.event.turnId !== turnId ||
row.sourcePayloadSha256 !== nativeSha256(parsed.event)) return null;
const observed = createNativeProviderFailureObservation(execution.provider);
observed.observe(parsed.event);
return observed.forTerminal(turnId, terminal);
}
@@ -0,0 +1,272 @@
import { randomUUID } from "node:crypto";
import { eq } from "drizzle-orm";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { agents, agentWakeupRequests, companies, completionContracts, createDb, heartbeatRunEvents, heartbeatRuns, issueRecoveryActions, issueThreadInteractions, issues, nativeRunFinalizations, nativeRunResults, statusDecisions, workAssessments } from "@paperclipai/db";
import type { ControlPlanePort, NativeExecutionInputV1, PrpEvent, PrpTerminalState } from "@paperclipai/paperclip-runner";
import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
import { CONTROL_PLANE_CONFORMANCE_RESULT } from "../../vendor/paperclip-runner/testing.js";
const provider = vi.hoisted(() => ({ execute: vi.fn() }));
vi.mock("../../vendor/paperclip-runner/index.js", async (importOriginal) => ({
...(await importOriginal<typeof import("../../vendor/paperclip-runner/index.js")>()),
executeNativeSession: provider.execute,
}));
import { executePaperclipNativeSession } from "./native-session-executor.js";
import { buildNativeCompletionContract } from "./completion-contracts.js";
import { PaperclipControlPlanePort } from "./paperclip-control-plane-port.js";
import { prepareNativeHeartbeatRun } from "./prepare-native-run.js";
import { NATIVE_MODEL_REJECTION_DIAGNOSTIC, NATIVE_MODEL_REJECTION_MESSAGE } from "./native-provider-failure.js";
import { collectRunFailureDiagnostics } from "../run-failure-diagnostics.js";
import { finalizeNativeRun } from "./native-run-finalizer.js";
import { readPersistedNativeModelRejection } from "./native-provider-failure-evidence.js";
import { heartbeatService } from "../heartbeat.js";
import { getNativeReviewAssignment, readNativeReviewAssignmentContext } from "./native-review-participant.js";
import { claimQueuedNativeReviewRun } from "./native-review-dispatch.js";
import { commitNativeStatusDecision, NativeStatusRaceError } from "./status-decision-committer.js";
import { NATIVE_STATUS_ARBITER_POLICY_VERSION } from "./status-arbiter.js";
import { issueThreadInteractionService } from "../issue-thread-interactions.js";
import { createReleaseIssueExecution } from "../../modules/wake-queue/application/use-cases.js";
import { createPostgresWakeQueueAdapter } from "../../modules/wake-queue/adapters/postgres.js";
describe("native completed failure diagnostics", () => {
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let db: ReturnType<typeof createDb>;
const companyId = randomUUID();
const agentId = randomUUID();
const workerId = randomUUID();
const model = "test-model";
beforeAll(async () => {
temporary = await startEmbeddedPostgresTestDatabase("native-model-failure-");
db = createDb(temporary.connectionString);
await db.insert(companies).values({ id: companyId, name: "Native failure regression", issuePrefix: "NMF" });
await db.insert(agents).values({ id: agentId, companyId, name: "Native failure agent", status: "active", adapterType: "paperclip_runner", adapterConfig: { provider: "codex", model } });
await db.insert(agents).values({ id: workerId, companyId, name: "Review source worker", status: "active", adapterType: "paperclip_runner" });
});
afterAll(async () => { await temporary?.cleanup(); });
it.each(["committed", "duplicate", "pending-review", "superseded-review", "resolved-review", "reassigned", "successor-race", "different-turn", "unknown-error", "untrusted-source", "untrusted-run", "untrusted-session", "succeeded", "cancelled"] as const)(
"preserves accepted results and only labels the matching provider rejection (%s)", async (scenario) => {
const issueId = randomUUID();
const runId = randomUUID();
const [issue] = await db.insert(issues).values({ id: issueId, companyId, title: "Explain a provider rejection", status: "in_progress", workMode: "standard", assigneeAgentId: agentId }).returning();
const [run] = await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running", runtimeMode: "native", nativeIssueId: issueId, invocationSource: "assignment", triggerDetail: "system", contextSnapshot: { issueId } }).returning();
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId));
const native = await prepareNativeHeartbeatRun({ db, run, issue, environmentLeaseId: randomUUID() });
const [contract] = await db.select().from(completionContracts).where(eq(completionContracts.issueId, issueId));
const completionInput = buildNativeCompletionContract(issue, { revision: contract.revision });
await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId, phase: "observed" });
const execution: NativeExecutionInputV1 = {
schema: "paperclip.native-execution-input.v1",
binding: { companyId, runId, issueId, agentId, executionWorkspaceId: runId },
provider: { kind: "codex", model },
task: { identifier: issueId, title: issue.title, description: null, prompt: "Reply to the current message.", workMode: "standard" },
workspace: { cwd: "/tmp", repoUrl: null, repoRef: null, branchName: null },
session: { normalizedSessionId: native.normalizedSessionId, driverKind: "codex_app_server", protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } },
completionContract: { id: contract.id, sha256: contract.canonicalSha256, schemaVersion: "paperclip.completion-contract.v1", contract: completionInput },
interactionResponses: [], credentialBindings: [],
};
await db.update(heartbeatRuns).set({ runnerProfileJson: { nativeExecutionInput: execution } }).where(eq(heartbeatRuns.id, runId));
if (scenario.endsWith("-review")) {
const sourceRunId = randomUUID(), sourceResultId = randomUUID(), assessmentId = randomUUID();
const decisionId = randomUUID(), interactionId = randomUUID();
await db.insert(heartbeatRuns).values({ id: sourceRunId, companyId, agentId: workerId, nativeIssueId: issueId, runtimeMode: "native", status: "succeeded", completionContractId: contract.id, completionContractSha256: contract.canonicalSha256 });
await db.insert(nativeRunResults).values({ id: sourceResultId, companyId, issueId, runId: sourceRunId, completionContractId: contract.id, serverFingerprint: sourceResultId, schemaStatus: "accepted", resultJson: {}, canonicalSha256: sourceResultId });
await db.insert(workAssessments).values({ id: assessmentId, companyId, issueId, runId: sourceRunId, contractId: contract.id, resultId: sourceResultId, triggerKind: "native_result", triggerActorCompanyId: companyId, priorIssueStatus: "in_progress", priorStatusVersion: 0, policyVersion: "fixture", assessmentJson: {}, inputDigest: assessmentId });
const [reviewIssue] = await db.update(issues).set({ status: "in_review", assigneeAgentId: workerId }).where(eq(issues.id, issueId)).returning();
await db.insert(statusDecisions).values({ id: decisionId, companyId, issueId, runId: sourceRunId, assessmentId, decisionVersion: 1, policyVersion: "fixture", fromStatus: "in_progress", toStatus: "in_review", reasonCode: "explicit_review", decisionJson: { projectedStatusVersion: reviewIssue.statusVersion }, decisionDigest: decisionId, applicationState: "applied" });
await db.update(issues).set({ lastStatusDecisionId: decisionId }).where(eq(issues.id, issueId));
await db.insert(issueThreadInteractions).values({ id: interactionId, companyId, issueId, kind: "request_confirmation", sourceRunId, addresseeAgentId: agentId, createdByAgentId: workerId,
continuationPolicy: "wake_assignee",
status: scenario === "resolved-review" ? "accepted" : "pending",
...(scenario === "resolved-review" ? { resolvedByRunId: runId, resolvedByAgentId: agentId } : {}),
payload: { version: 1, prompt: "Review the completed work.", target: { type: "custom", key: "native_completion_review", revisionId: decisionId } },
});
const contextSnapshot = { issueId, wakeReason: "native_completion_review", nativeReviewInteractionId: interactionId, nativeReviewDecisionId: decisionId };
await db.update(heartbeatRuns).set({ contextSnapshot }).where(eq(heartbeatRuns.id, runId));
if (scenario === "superseded-review") await db.update(issues).set({ lastStatusDecisionId: null }).where(eq(issues.id, issueId));
const review = await getNativeReviewAssignment(db, { companyId, issueId, agentId, contextSnapshot, allowResolvedByRunId: runId });
if (scenario === "superseded-review") expect(review).toBeNull();
else expect(review?.interaction.status).toBe(scenario === "resolved-review" ? "accepted" : "pending");
}
const event: PrpEvent = {
schema: "paperclip.prp.event.v1", runId, normalizedSessionId: native.normalizedSessionId,
turnId: native.turnId, sourceInstanceId: native.runnerInstanceId, sourceEventId: "model-terminal", sourceSeq: 1,
sourceKind: "runner", eventType: "turn.failed", schemaVersion: 1, priority: 0, emittedAt: new Date().toISOString(),
payload: { status: "failed", error: { codexErrorInfo: "other", message: JSON.stringify({
type: "error", status: 400, error: { type: "invalid_request_error", message: scenario === "unknown-error" ? "Private unknown failure" : `The '${model}' model is not supported when using Codex with a ChatGPT account.` },
}) } },
};
const openInput = { identity: { companyId, issueId, runId, agentId, sessionId: native.normalizedSessionId }, backendKind: "mock" as const, sourceInstanceId: native.runnerInstanceId };
if (scenario === "duplicate") {
const prior = new PaperclipControlPlanePort(db, { companyId, issueId, runId, agentId, sessionId: native.normalizedSessionId, sourceInstanceId: native.runnerInstanceId, controlPlaneSourceInstanceId: "prior-controller", completionContractId: contract.id, completionContractSha256: contract.canonicalSha256 });
await prior.openRun(openInput);
expect((await prior.appendEvent(event)).disposition).toBe("committed");
}
const turnId = scenario === "different-turn" ? "later-turn" : native.turnId;
const terminal: PrpTerminalState = {
schema: "paperclip.prp.terminal.v1", turnTerminalState: scenario === "succeeded" ? "completed" : scenario === "cancelled" ? "cancelled" : "failed",
runTerminalState: scenario === "succeeded" ? "succeeded" : scenario === "cancelled" ? "cancelled" : "failed", reportedWorkDisposition: "blocked",
};
const result = { ...CONTROL_PLANE_CONFORMANCE_RESULT, reportedWorkDisposition: "blocked" as const,
blocker: { reasonCode: "provider_rejected", owner: { kind: "user" as const, name: "Task owner" }, unblockAction: "Select a compatible connection.", scope: "task_wide" as const },
completionClaim: { ...CONTROL_PLANE_CONFORMANCE_RESULT.completionClaim, contractRevision: completionInput.revision } };
provider.execute.mockReset().mockImplementation(async (options: { controlPlane: ControlPlanePort }) => {
await options.controlPlane.openRun(openInput);
if (scenario === "untrusted-source") {
await expect(options.controlPlane.appendEvent({ ...event, sourceInstanceId: "wrong-runner" })).rejects.toThrow("native_event_source_binding_mismatch");
} else if (scenario === "untrusted-run" || scenario === "untrusted-session") {
await expect(options.controlPlane.appendEvent({ ...event,
...(scenario === "untrusted-run" ? { runId: randomUUID() } : { normalizedSessionId: "wrong-session" }),
})).rejects.toThrow("native_event_binding_mismatch");
} else {
expect((await options.controlPlane.appendEvent(event)).disposition).toBe(scenario === "duplicate" ? "duplicate" : "committed");
}
await options.controlPlane.completeRun({ result, terminal, turnId });
return { result, terminal, turnId, normalizedSessionId: native.normalizedSessionId, providerSessionId: "fixture-provider-session", driverKind: "codex_app_server", driverVersion: "fixture", nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null };
});
const adapter = await executePaperclipNativeSession({ db, execution, runnerInstanceId: native.runnerInstanceId, backend: {
descriptor: async () => { throw new Error("Unexpected provider call"); },
openSession: async () => { throw new Error("Unexpected provider call"); },
} });
const classified = scenario === "committed" || scenario === "duplicate" || scenario.endsWith("-review") || scenario === "reassigned" || scenario === "successor-race";
const reviewAuthorityEnded = scenario === "superseded-review" || scenario === "resolved-review";
const ownerChanged = scenario === "reassigned" || scenario === "successor-race";
expect(adapter.errorCode).toBe(classified ? "native_provider_model_rejected" : undefined);
expect(adapter.errorMessage).toBe(classified ? NATIVE_MODEL_REJECTION_MESSAGE : scenario === "succeeded" ? null : `Native session ${terminal.runTerminalState}`);
expect(adapter.resultJson?.nativeProviderFailure).toEqual(classified ? NATIVE_MODEL_REJECTION_DIAGNOSTIC : undefined);
expect(adapter.nativeFinalization).toMatchObject({ result, terminal, turnId, workspaceFinalizeStatus: "pending" });
expect(provider.execute).toHaveBeenCalledTimes(1);
expect(execution.provider).toEqual({ kind: "codex", model });
const [coordinator] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId));
expect(coordinator).toMatchObject({ phase: "workspace_finalizing", leaseOwner: null, attempt: 1, nextAttemptAt: null });
expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, runId))).toHaveLength(1);
expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId))).toHaveLength(0);
expect(await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, runId))).toHaveLength(scenario.startsWith("untrusted-") ? 0 : 1);
const diagnostics = collectRunFailureDiagnostics({ ...run, resultJson: adapter.resultJson ?? null }, {});
expect(diagnostics.provider).toEqual(classified ? NATIVE_MODEL_REJECTION_DIAGNOSTIC : {});
expect(JSON.stringify(diagnostics)).not.toContain(model);
expect(JSON.stringify(diagnostics)).not.toContain("Private");
const [persistedRun] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
expect(persistedRun.errorCode).toBeNull(); // No adapter metadata has been saved yet.
expect(await readPersistedNativeModelRejection(db, persistedRun, turnId, terminal)).toEqual(classified ? {
errorCode: "native_provider_model_rejected", errorMessage: NATIVE_MODEL_REJECTION_MESSAGE, diagnostic: NATIVE_MODEL_REJECTION_DIAGNOSTIC,
} : null);
if (classified) {
if (scenario === "committed") {
// Neither saved display diagnostics nor a different execution binding
// can confer the right to change the finalizer's retry decision.
for (const changed of [
{ ...persistedRun, runnerProfileJson: null },
{ ...persistedRun, nativeSessionId: "foreign-session" },
{ ...persistedRun, runnerInstanceId: "foreign-source" },
{ ...persistedRun, companyId: randomUUID() },
{ ...persistedRun, completionContractSha256: "foreign-contract" },
{ ...persistedRun, runnerProfileJson: { nativeExecutionInput: { ...execution, provider: { kind: "codex", model: "different-model" } } } },
{ ...persistedRun, runnerProfileJson: { nativeExecutionInput: { ...execution, binding: { ...execution.binding, runId: randomUUID() } } } },
]) expect(await readPersistedNativeModelRejection(db, changed, turnId, terminal)).toBeNull();
const [stored] = await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, runId));
await db.update(heartbeatRunEvents).set({ sourcePayloadSha256: "invalid-hash" }).where(eq(heartbeatRunEvents.id, stored.id));
expect(await readPersistedNativeModelRejection(db, persistedRun, turnId, terminal)).toBeNull();
await db.update(heartbeatRunEvents).set({ sourcePayloadSha256: stored.sourcePayloadSha256 }).where(eq(heartbeatRunEvents.id, stored.id));
}
// Simulate a restart after native result acceptance, before heartbeat's
// adapter update. Finalization must use durable provider evidence.
const acceptedBefore = await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, runId));
if (scenario === "reassigned") {
await db.update(issues).set({ assigneeAgentId: workerId, executionRunId: null }).where(eq(issues.id, issueId));
} else if (scenario === "successor-race") {
const [snapshot] = await db.select().from(issues).where(eq(issues.id, issueId));
const newerRunId = randomUUID(), raceAssessmentId = randomUUID();
await db.insert(heartbeatRuns).values({ id: newerRunId, companyId, agentId, status: "running", nativeIssueId: issueId });
await db.update(issues).set({ executionRunId: newerRunId }).where(eq(issues.id, issueId));
const [changed] = await db.select().from(issues).where(eq(issues.id, issueId));
expect(changed.statusVersion).toBe(snapshot.statusVersion);
await db.insert(workAssessments).values({ id: raceAssessmentId, companyId, issueId, runId, contractId: contract.id, resultId: acceptedBefore[0].id, triggerKind: "native_result", triggerActorCompanyId: companyId, priorIssueStatus: snapshot.status, priorStatusVersion: snapshot.statusVersion, policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, assessmentJson: {}, inputDigest: raceAssessmentId });
await expect(commitNativeStatusDecision({ db, companyId, issueId, runId, assessmentId: raceAssessmentId,
priorStatus: snapshot.status, priorStatusVersion: snapshot.statusVersion, priorDecisionId: snapshot.lastStatusDecisionId,
requireModelRejectionOwner: { agentId, reviewContext: null },
decision: { policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, statusAction: "blocked", toStatus: "blocked", reasonCode: "native_provider_model_rejected", unblockDescriptor: { owner: "board", action: NATIVE_MODEL_REJECTION_MESSAGE }, effects: [{ kind: "bind_blocker", owner: "board", action: NATIVE_MODEL_REJECTION_MESSAGE }] },
})).rejects.toBeInstanceOf(NativeStatusRaceError);
expect(await db.select().from(statusDecisions).where(eq(statusDecisions.runId, runId))).toHaveLength(0);
}
for (let retry = 0; retry < 2; retry++) {
await finalizeNativeRun({ db, runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
}
expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, runId))).toEqual(acceptedBefore);
const [finalRun] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
expect(finalRun).toMatchObject({ status: "failed", errorCode: "native_provider_model_rejected", error: NATIVE_MODEL_REJECTION_MESSAGE });
const [finalIssue] = await db.select().from(issues).where(eq(issues.id, issueId));
expect(finalIssue.status).toBe(scenario.endsWith("-review") ? "in_review" : ownerChanged ? "in_progress" : "blocked");
const decisions = await db.select().from(statusDecisions).where(eq(statusDecisions.runId, runId));
expect(decisions).toHaveLength(1);
expect(decisions[0].reasonCode).toBe(reviewAuthorityEnded ? "native_review_action_finished" : ownerChanged ? "run_failed_partial_evidence_preserved" : "native_provider_model_rejected");
const wakes = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId));
expect(wakes.filter(wake => wake.payload?.issueId === issueId)).toHaveLength(ownerChanged ? 1 : 0);
if (!ownerChanged) expect(await heartbeatService(db).dispatchPendingNativeStatusWakeups({ companyId })).toEqual({ scanned: 0, dispatched: 0, recovered: 0, deferred: 0 });
expect(provider.execute).toHaveBeenCalledTimes(1);
if (scenario === "pending-review") {
const reviewContext = readNativeReviewAssignmentContext(finalRun.contextSnapshot)!;
expect(finalIssue).toMatchObject({ lastStatusDecisionId: reviewContext.nativeReviewDecisionId, executionRunId: null, checkoutRunId: null });
const originalReview = await getNativeReviewAssignment(db, { companyId, issueId, agentId, contextSnapshot: reviewContext });
expect(originalReview?.interaction.status).toBe("pending");
expect(finalIssue.statusVersion).toBe(originalReview!.sourceDecision.decisionJson.projectedStatusVersion);
expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId))).toHaveLength(0);
// Exercise the real release and periodic recovery lanes, rather than
// assuming that absence of a finalizer wake prevents later retries.
const unexpected = vi.fn(async () => { throw new Error("Unexpected automatic recovery"); });
const release = createReleaseIssueExecution({
issueLock: createPostgresWakeQueueAdapter(db, { resolveResponsibleUserId: unexpected, getRoutineEnv: unexpected, resolveSessionBeforeForWakeup: unexpected }),
recovery: { escalateStrandedAssignedIssue: unexpected, escalateStrandedRecoveryIssueInPlace: unexpected },
});
expect((await release({ companyId, runId, now: new Date() })).outcome.kind).toBe("released");
await heartbeatService(db).reconcileStrandedAssignedIssues();
expect(unexpected).not.toHaveBeenCalled();
expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId))).toHaveLength(0);
expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId))).filter(wake => wake.payload?.issueId === issueId)).toHaveLength(0);
const [afterRecovery] = await db.select().from(issues).where(eq(issues.id, issueId));
expect(afterRecovery).toMatchObject({ status: "in_review", statusVersion: finalIssue.statusVersion, lastStatusDecisionId: finalIssue.lastStatusDecisionId });
// The operator repairs configuration and explicitly retries the
// assigned reviewer. No review rebinding or status reset is needed.
await db.update(agents).set({ adapterConfig: { provider: "codex", model: "repaired-model" }, runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 } } }).where(eq(agents.id, agentId));
// Occupy the execution slot to exercise real wake admission without
// starting a provider. The normal atomic review claim follows below.
const occupiedRunId = randomUUID();
await db.insert(heartbeatRuns).values({ id: occupiedRunId, companyId, agentId, status: "running", startedAt: new Date() });
const retryRun = await heartbeatService(db, { runtimeEnv: {} }).wakeup(agentId, {
failedRunId: runId, source: "on_demand", triggerDetail: "manual", reason: "retry_failed_run",
requestedByActorType: "user", requestedByActorId: "review-repair-operator",
payload: { issueId }, contextSnapshot: { ...reviewContext },
});
expect(retryRun).toMatchObject({ status: "queued", retryOfRunId: runId, contextSnapshot: { issueId, ...reviewContext } });
const retryRunId = retryRun!.id;
await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date() }).where(eq(heartbeatRuns.id, occupiedRunId));
expect(await claimQueuedNativeReviewRun(db, { run: retryRun!, agentNameKey: "reviewer", claimedAt: new Date(), claimValues: { nativeIssueId: issueId, runtimeMode: "native" } }))
.toMatchObject({ id: retryRunId, status: "running" });
expect(await getNativeReviewAssignment(db, { companyId, issueId, agentId, contextSnapshot: reviewContext, actingRunId: retryRunId, issueExecutionRunId: retryRunId })).not.toBeNull();
await issueThreadInteractionService(db).acceptInteraction(
{ id: issueId, companyId, projectId: null, goalId: null, status: "in_review" },
reviewContext.nativeReviewInteractionId, {}, { agentId, runId: retryRunId },
);
const [resolved] = await db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.id, reviewContext.nativeReviewInteractionId));
expect(resolved).toMatchObject({ status: "accepted", resolvedByAgentId: agentId, resolvedByRunId: retryRunId });
expect(await db.select().from(issues).where(eq(issues.id, issueId))).toMatchObject([{ status: "done", assigneeAgentId: workerId }]);
expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, runId))).toEqual(acceptedBefore);
expect(provider.execute).toHaveBeenCalledTimes(1);
await db.update(agents).set({ adapterConfig: { provider: "codex", model } }).where(eq(agents.id, agentId));
}
} else if (scenario === "unknown-error") {
// A lookalike diagnostic flag does not alter unknown failed-result policy.
await db.update(heartbeatRuns).set({ resultJson: { nativeProviderFailure: NATIVE_MODEL_REJECTION_DIAGNOSTIC } }).where(eq(heartbeatRuns.id, runId));
await finalizeNativeRun({ db, runId, workspaceFinalizeStatus: "succeeded" });
const [decision] = await db.select().from(statusDecisions).where(eq(statusDecisions.runId, runId));
expect(decision.reasonCode).toBe("run_failed_partial_evidence_preserved");
const wakes = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId));
expect(wakes.filter(wake => wake.payload?.issueId === issueId)).toHaveLength(1);
}
},
);
});
@@ -0,0 +1,73 @@
import { describe, expect, it } from "vitest";
import type { PrpEvent, PrpTerminalState } from "../../vendor/paperclip-runner/index.js";
import { createNativeProviderFailureObservation, NATIVE_MODEL_REJECTION_DIAGNOSTIC, NATIVE_MODEL_REJECTION_MESSAGE } from "./native-provider-failure.js";
const model = "test-model";
const response = {
type: "error", status: 400,
error: { type: "invalid_request_error", message: `The '${model}' model is not supported when using Codex with a ChatGPT account.` },
};
const event: PrpEvent = {
schema: "paperclip.prp.event.v1", runId: "run", normalizedSessionId: "session", turnId: "turn",
sourceInstanceId: "runner", sourceEventId: "failure", sourceSeq: 1, sourceKind: "runner",
eventType: "turn.failed", schemaVersion: 1, priority: 0, emittedAt: "2026-01-01T00:00:00.000Z",
payload: { status: "failed", error: { message: JSON.stringify(response), codexErrorInfo: "other" } },
};
const terminal: PrpTerminalState = {
schema: "paperclip.prp.terminal.v1", turnTerminalState: "failed", runTerminalState: "failed", reportedWorkDisposition: "blocked",
};
const observation = () => createNativeProviderFailureObservation({ kind: "codex", model });
describe("native provider failure observation", () => {
it("binds the structured rejection to the failed turn and emits no response or model text", () => {
const observed = observation();
observed.observe(event);
observed.observe(event); // A byte-identical committed replay remains idempotent.
const result = observed.forTerminal("turn", terminal);
expect(result).toEqual({ errorCode: "native_provider_model_rejected", errorMessage: NATIVE_MODEL_REJECTION_MESSAGE, diagnostic: NATIVE_MODEL_REJECTION_DIAGNOSTIC });
expect(JSON.stringify(result)).not.toContain(model);
expect(JSON.stringify(result)).not.toContain("invalid_request_error");
expect(observed.forTerminal("later-turn", terminal)).toBeNull();
expect(observed.forTerminal(null, terminal)).toBeNull();
expect(observed.forTerminal("turn", { ...terminal, runTerminalState: "succeeded" })).toBeNull();
expect(observed.forTerminal("turn", { ...terminal, turnTerminalState: "cancelled" })).toBeNull();
});
it.each([
{ ...event, sourceKind: "control_plane" as const },
{ ...event, eventType: "item.completed" as const },
{ ...event, eventType: "harness.diagnostic" as const },
{ ...event, turnId: undefined },
{ ...event, payload: { ...event.payload, status: "completed" } },
{ ...event, payload: { ...event.payload, recoverable: true } },
{ ...event, payload: { status: "failed", error: { message: JSON.stringify(response), recoverable: true } } },
])("ignores non-authoritative or retryable evidence %#", (unrelated) => {
const observed = observation();
observed.observe(unrelated);
expect(observed.forTerminal("turn", terminal)).toBeNull();
});
it.each([
response.error.message, "malformed JSON", " ".repeat(4_097),
JSON.stringify({ ...response, status: 401 }),
JSON.stringify({ ...response, status: "400" }),
JSON.stringify({ ...response, type: "notice" }),
JSON.stringify({ ...response, error: { ...response.error, type: "server_error" } }),
JSON.stringify({ ...response, error: { ...response.error, message: "Private arbitrary provider text" } }),
JSON.stringify({ ...response, error: { ...response.error, message: response.error.message + " Private suffix" } }),
JSON.stringify({ ...response, error: { ...response.error, message: response.error.message.replace(model, "another-model") } }),
JSON.stringify([response]), "null",
])("leaves unknown failures unchanged %#", (message) => {
const observed = observation();
observed.observe({ ...event, payload: { status: "failed", error: { message } } });
expect(observed.forTerminal("turn", terminal)).toBeNull();
});
it.each([{ kind: "claude", model }, { kind: "codex" }, { kind: "codex", model: "other" }])(
"does not apply a rejection to another selected provider/model (%j)", (provider) => {
const observed = createNativeProviderFailureObservation(provider);
observed.observe(event);
expect(observed.forTerminal("turn", terminal)).toBeNull();
},
);
});
@@ -0,0 +1,60 @@
import type { PrpEvent, PrpTerminalState } from "../../vendor/paperclip-runner/index.js";
export const NATIVE_MODEL_REJECTION_MESSAGE =
"The selected model is not supported by the current ChatGPT connection. Choose a supported model or a compatible AI connection.";
export const NATIVE_MODEL_REJECTION_DIAGNOSTIC = {
provider: "codex",
category: "model_auth_incompatible",
status: 400,
authMode: "chatgpt",
} as const;
function record(value: unknown): Record<string, unknown> {
return value !== null && typeof value === "object" && !Array.isArray(value)
? value as Record<string, unknown>
: {};
}
/** Reconstruct only the closed diagnostic vocabulary, never provider text. */
export function readNativeModelRejectionDiagnostic(value: unknown) {
try {
const entry = record(value);
return Object.entries(NATIVE_MODEL_REJECTION_DIAGNOSTIC).every(([key, expected]) => entry[key] === expected)
? { ...NATIVE_MODEL_REJECTION_DIAGNOSTIC }
: null;
} catch { return null; }
}
/** Observe only events accepted by the bound control-plane persistence port. */
export function createNativeProviderFailureObservation(provider: { kind: string; model?: string | null }) {
let rejectedTurnId: string | null = null;
return {
observe(event: PrpEvent) {
if (provider.kind !== "codex" || !provider.model ||
event.sourceKind !== "runner" || event.eventType !== "turn.failed" || !event.turnId) return;
const payload = record(event.payload);
const failure = record(payload.error);
if (payload.status !== "failed" || payload.recoverable === true || failure.recoverable === true ||
typeof failure.message !== "string" || failure.message.length > 4_096) return;
let response: Record<string, unknown>;
try { response = record(JSON.parse(failure.message)); } catch { return; }
const error = record(response.error);
// This is a provider terminal envelope, not text emitted by a tool or an
// agent. Bind the exact rejection to the configured model without storing
// its name or the response. Unknown provider errors retain their behavior.
if (response.type !== "error" || response.status !== 400 || error.type !== "invalid_request_error" ||
error.message !== `The '${provider.model}' model is not supported when using Codex with a ChatGPT account.`) return;
rejectedTurnId = event.turnId;
},
forTerminal(turnId: string | null | undefined, terminal: PrpTerminalState) {
if (!rejectedTurnId || turnId !== rejectedTurnId ||
terminal.turnTerminalState !== "failed" || terminal.runTerminalState !== "failed") return null;
return {
errorCode: "native_provider_model_rejected" as const,
errorMessage: NATIVE_MODEL_REJECTION_MESSAGE,
diagnostic: { ...NATIVE_MODEL_REJECTION_DIAGNOSTIC },
};
},
};
}
@@ -16,6 +16,7 @@ import { redactSensitiveText } from "../../redaction.js";
import { issueRecoveryActionService } from "../issue-recovery-actions.js";
import { reportRunFailure } from "../run-failure-report.js";
import { isNativeRunnerOwnershipHeld } from "./native-runner-ownership.js";
import { nativeRestartInterruptedTurnId, type PersistedNativeSession } from "../../vendor/paperclip-runner/index.js";
export type NativeControllerIdentity = {
bootId: string;
@@ -764,7 +765,8 @@ export async function claimNativeRestartRecoveries(input: {
hasProviderEvidence,
checkpointFailed:
(checkpointRecord.terminal as Record<string, unknown> | undefined)
?.runTerminalState === "failed",
?.runTerminalState === "failed" &&
nativeRestartInterruptedTurnId(checkpointRecord as unknown as PersistedNativeSession) === null,
providerAttempt: row.coordinator.attempt,
});
const ownershipChanged =
@@ -1,4 +1,5 @@
import { settleSlackConversation } from "../slack-conversation-lifecycle.js";
import { readPersistedNativeModelRejection } from "./native-provider-failure-evidence.js";
import { dismissAutomaticCompletionReviews } from "./automatic-completion-reviews.js";
import { getNativeReviewAssignment, readNativeReviewAssignmentContext } from "./native-review-participant.js";
import { conversationNativeDecision, isConversation } from "../agent-conversations.js";
@@ -24,7 +25,7 @@ import {
workspaceOperations,
} from "@paperclipai/db";
import { classifyNativeEvidence } from "./evidence-classifier.js";
import type { PrpIgnoredAttentionRequest } from "@paperclipai/paperclip-runner";
import type { PrpIgnoredAttentionRequest, PrpTerminalState } from "@paperclipai/paperclip-runner";
import {
arbitrateNativeStatus,
NATIVE_STATUS_ARBITER_POLICY_VERSION,
@@ -1275,7 +1276,15 @@ export async function finalizeNativeRun(input: {
companyId: run.companyId, issueId: authoritativeIssue.id, agentId: run.agentId,
contextSnapshot: reviewContext, allowResolvedByRunId: run.id,
}) : null;
const providerModelRejection = await readPersistedNativeModelRejection(
input.db, run, resultRow.turnId, envelope.terminal as PrpTerminalState,
);
const ownsModelRejectionDecision =
(!authoritativeIssue.executionRunId || authoritativeIssue.executionRunId === run.id) &&
(reviewContext ? nativeReview?.interaction.status === "pending"
: authoritativeIssue.assigneeAgentId === run.agentId && !authoritativeIssue.assigneeUserId);
const proposedDecision = resolveNativeFinalizerStatus({
providerModelRejected: providerModelRejection !== null && ownsModelRejectionDecision,
...(reviewContext ? { nativeReviewOutcome: nativeReview
? nativeReview.interaction.status === "pending" ? "pending" as const : "resolved" as const
: "stale" as const } : {}),
@@ -1358,6 +1367,9 @@ export async function finalizeNativeRun(input: {
priorStatusVersion: Number(authoritativeIssue.statusVersion),
priorDecisionId: authoritativeIssue.lastStatusDecisionId,
decision,
requireModelRejectionOwner: decision.reasonCode === "native_provider_model_rejected"
? { agentId: run.agentId, reviewContext }
: undefined,
requireBoardResponseWaitSource:
decision.reasonCode === "board_response_waiting" || repairBoardResponseWait
? boardResponseWait?.source
@@ -1400,6 +1412,7 @@ export async function finalizeNativeRun(input: {
);
const clearExecutionFailure = input.projectRunStatus && finalizationPhase === "committed" && terminalState === "succeeded";
const finalizationMetadata = {
...(providerModelRejection ? { nativeProviderFailure: providerModelRejection.diagnostic } : {}),
finalizationPhase,
...(finalizationPhase === "committed" ? { failureCode: null, originalFailureCode: null, nextAttemptAt: null } : {}),
assessmentId: assessmentRow.id,
@@ -1450,6 +1463,12 @@ export async function finalizeNativeRun(input: {
nativePhase: finalizationPhase,
nativePhaseUpdatedAt: now,
...(clearExecutionFailure ? { error: null, errorCode: null } : {}),
...(providerModelRejection ? {
// Recovery can finalize before heartbeat saves the adapter result.
// Retain any independently recorded execution/cleanup failure.
error: sql`coalesce(${heartbeatRuns.error}, ${providerModelRejection.errorMessage})`,
errorCode: sql`coalesce(${heartbeatRuns.errorCode}, ${providerModelRejection.errorCode})`,
} : {}),
resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb)
|| ${JSON.stringify(finalizationMetadata)}::jsonb
|| ${clearExecutionFailure ? recoveredExecutionFailureMetadata() : sql`'{}'::jsonb`}`,
@@ -1018,6 +1018,30 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
]);
});
it.each([2, 3])("recovers a checkpointed restart interruption within provider budget %s", async (attempt) => {
const fixture = await seedRun(`INTERRUPTED-${attempt}`);
await fixture.db.update(heartbeatRuns).set({ runnerProfileJson: { sessionCheckpoint: {
driverKind: "codex_app_server",
sessionId: fixture.native.normalizedSessionId,
identity: { companyId, issueId: fixture.issueId, runId: fixture.runId, agentId, sessionId: fixture.native.normalizedSessionId },
providerSessionId: "interrupted-provider",
activeTurnId: null,
semanticResult: null,
terminal: { runTerminalState: "failed", turnTerminalState: "failed" },
terminalTurns: [{ turnId: "interrupted-turn", fingerprint: JSON.stringify({
terminalState: "failed", result: null,
error: { code: "provider_turn_lost_on_restore", recoverable: true },
}) }],
} } }).where(eq(heartbeatRuns.id, fixture.runId));
await fixture.db.update(nativeRunFinalizations).set({ attempt }).where(eq(nativeRunFinalizations.runId, fixture.runId));
const dispositions = await claimNativeRestartRecoveries({
db: fixture.db, controller: successor, restartKind: "hot", runIds: [fixture.runId],
});
expect(dispositions).toEqual([expect.objectContaining(attempt < 3
? { kind: "resume_dead_runner", runId: fixture.runId, providerAttempt: attempt }
: { kind: "blocked", runId: fixture.runId, reason: "execution_recovery_budget_exhausted" })]);
});
it("terminalizes the recorded failed-checkpoint incident atomically instead of resuming it on upgrade", async () => {
const fixture = await seedRun("FAILED-CHECKPOINT");
await fixture.db.update(heartbeatRuns).set({ runnerProfileJson: { sessionCheckpoint: {
@@ -9,6 +9,7 @@ import {
REMOTE_CODEX_SUPPORTED_RANGE,
} from "./codex-runtime-compatibility.js";
import { createNativeToolTrace, type NativeToolTrace } from "./native-tool-trace.js";
import { createNativeProviderFailureObservation } from "./native-provider-failure.js";
import { createNativeGitHubAccess, type NativeGitHubAccess } from "./native-github-access.js";
import { resolveGitHubOperationCredentials } from "../github-operation-credentials.js";
import { bindManagedNativeCredentialTurn, completeManagedNativeCredentialTurn } from "./managed-native-credentials.js";
@@ -7805,6 +7806,7 @@ async function executePaperclipNativeSessionWithinScope(
let turnStartedAtMs: number | null = null;
let firstAgentEventRecorded = false;
let providerUsageLimitObserved = false;
const providerFailureObservation = createNativeProviderFailureObservation(input.execution.provider);
let turnCompletedAtMs: number | null = null;
let runnerSessionStartupScope: NativeRunSpanScope | null = null;
let agentTurnScope: NativeRunSpanScope | null = null;
@@ -7861,6 +7863,7 @@ async function executePaperclipNativeSessionWithinScope(
await liveQuestions.observe(event);
await projectSessionGoalEvent(event);
providerUsageLimitObserved ||= nativeProviderUsageLimitFromEvent(event);
providerFailureObservation.observe(event);
const eventAtMs = Date.parse(event.emittedAt);
const milestoneAtMs = Number.isFinite(eventAtMs)
? eventAtMs
@@ -8064,6 +8067,7 @@ async function executePaperclipNativeSessionWithinScope(
await liveQuestions.observe(event);
await projectSessionGoalEvent(event);
providerUsageLimitObserved ||= nativeProviderUsageLimitFromEvent(event);
providerFailureObservation.observe(event);
const questionFallback = await materializeRuntimeQuestionFallback({
db: input.db,
binding: input.execution.binding,
@@ -8334,6 +8338,7 @@ async function executePaperclipNativeSessionWithinScope(
trace.activate(runnerSessionStartupScope);
const result = await trace.run(runnerSessionStartupScope, () =>
executeNativeSession({
resumeInterruptedTurn: input.restartRecovery?.kind === "resume_dead_runner",
getFreshSessionHandoff: input.getFreshSessionHandoff,
onSessionAdmission: async () => {
// Invalidate prior stop evidence before a backend can spawn.
@@ -9203,6 +9208,7 @@ async function executePaperclipNativeSessionWithinScope(
);
if (collectInstructions && !collectedByOwner) await instructionCopy!.collectStopped();
}
const providerFailure = providerFailureObservation.forTerminal(native.turnId, native.terminal);
const adapterResult: AdapterExecutionResult = {
exitCode: native.terminal.runTerminalState === "succeeded" ? 0 : 1,
signal: null,
@@ -9210,10 +9216,12 @@ async function executePaperclipNativeSessionWithinScope(
errorMessage:
native.terminal.runTerminalState === "succeeded"
? null
: `Native session ${native.terminal.runTerminalState}`,
: providerFailure?.errorMessage ?? `Native session ${native.terminal.runTerminalState}`,
...(providerFailure ? { errorCode: providerFailure.errorCode } : {}),
resultJson: {
nativeResult: native.result as unknown as Record<string, unknown>,
nativeTerminal: native.terminal as unknown as Record<string, unknown>,
...(providerFailure ? { nativeProviderFailure: providerFailure.diagnostic } : {}),
...(native.goalRolloverRequired ? { goalRolloverRequired: true } : {}),
planSynchronizations,
},
@@ -44,6 +44,23 @@ function arbitrate(
}
describe("native status authority", () => {
it.each(["in_progress", "in_review"] as const)("blocks a current worker's proven model rejection without a retry (%s)", (priorIssueStatus) => {
const decision = arbitrate({ priorIssueStatus, terminalState: "failed", providerModelRejected: true });
expect(decision).toMatchObject({ statusAction: "blocked", toStatus: "blocked", reasonCode: "native_provider_model_rejected", unblockDescriptor: { owner: "board" } });
expect(decision.effects).toEqual([{ kind: "bind_blocker", owner: "board", action: expect.any(String) }]);
expect(arbitrate({ terminalState: "failed", providerModelRejected: false }).effects).toContainEqual(expect.objectContaining({ kind: "schedule_retry" }));
expect(arbitrate({ terminalState: "succeeded", providerModelRejected: true }).reasonCode).not.toBe("native_provider_model_rejected");
expect(arbitrate({ terminalState: "failed", providerModelRejected: true, workspaceFinalizeStatus: "failed" }).reasonCode).toBe("finalization_failed_claim_preserved");
expect(arbitrate({ terminalState: "failed", providerModelRejected: true, priorIssueStatus: "done" }).toStatus).toBe("done");
for (const nativeReviewOutcome of ["stale", "resolved"] as const) {
expect(arbitrate({ terminalState: "failed", providerModelRejected: true, priorIssueStatus, nativeReviewOutcome }))
.toMatchObject({ statusAction: "preserve", toStatus: priorIssueStatus, reasonCode: "native_review_action_finished" });
}
});
it("preserves pending review authority without automatic recovery after model rejection", () => {
expect(arbitrate({ priorIssueStatus: "in_review", terminalState: "failed", providerModelRejected: true, nativeReviewOutcome: "pending" }))
.toMatchObject({ statusAction: "preserve", toStatus: "in_review", reasonCode: "native_provider_model_rejected", unblockDescriptor: null, effects: [{ kind: "release_checkout" }] });
});
it("a reviewer finishes its decision without completing rejected or still-reviewed work", () => {
for (const priorIssueStatus of ["in_progress", "in_review"] as const) {
const decision = arbitrate({ priorIssueStatus, nativeReviewOutcome: "resolved" });
@@ -511,7 +528,7 @@ describe("native status authority", () => {
expect.objectContaining({
statusAction: "blocked",
toStatus: "blocked",
policyVersion: "phase6-v7",
policyVersion: "phase6-v8",
reasonCode: "current_track_blocker_waiting",
unblockDescriptor: {
owner: "board",
@@ -1,6 +1,7 @@
import type { NativeEvidenceAssessment } from "./evidence-classifier.js";
import { NATIVE_MODEL_REJECTION_MESSAGE } from "./native-provider-failure.js";
export const NATIVE_STATUS_ARBITER_POLICY_VERSION = "phase6-v7";
export const NATIVE_STATUS_ARBITER_POLICY_VERSION = "phase6-v8";
export type NativeAuthoritativeIssueStatus =
| "backlog"
@@ -115,6 +116,8 @@ export function arbitrateNativeStatus(input: {
reviewOwnerUserId?: string | null;
/** Review decisions own task state; a reviewer's finish report cannot override them. */
nativeReviewOutcome?: "resolved" | "pending" | "stale";
/** Re-derived from the committed runner terminal and pinned execution identity. */
providerModelRejected?: boolean;
agentId: string;
priorIssueStatus: NativeAuthoritativeIssueStatus;
}): NativeStatusDecision {
@@ -146,6 +149,31 @@ export function arbitrateNativeStatus(input: {
],
};
}
if (input.providerModelRejected && input.terminalState === "failed" &&
input.nativeReviewOutcome !== "stale" && input.nativeReviewOutcome !== "resolved") {
if (input.nativeReviewOutcome === "pending") {
// The pending review is bound to the worker's status decision and
// version. Preserve that authority so an explicit retry after a
// configuration repair can resolve the same review. In particular,
// do not create the normal unresolved-review recovery action.
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "preserve",
toStatus: input.priorIssueStatus,
reasonCode: "native_provider_model_rejected",
unblockDescriptor: null,
effects: [{ kind: "release_checkout" }],
};
}
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "blocked",
toStatus: "blocked",
reasonCode: "native_provider_model_rejected",
unblockDescriptor: { owner: "board", action: NATIVE_MODEL_REJECTION_MESSAGE },
effects: [{ kind: "bind_blocker", owner: "board", action: NATIVE_MODEL_REJECTION_MESSAGE }],
};
}
if (input.nativeReviewOutcome) {
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
@@ -24,6 +24,7 @@ import {
type NativeStatusEffect,
} from "./status-arbiter.js";
import { nativeSha256 } from "./canonical.js";
import { getNativeReviewAssignment, type NativeReviewAssignmentContext } from "./native-review-participant.js";
import {
readNativeBoardResponseWaitSource,
readNativeBoardResponseWaitOrigin,
@@ -1535,6 +1536,7 @@ export async function commitNativeStatusDecision(input: {
preMaterializedEffects?: NativeMaterializedStatusEffect[];
supersedesCommittedDecisionId?: string;
requireExternalChatResponseWaitAuthorization?: { agentId: string };
requireModelRejectionOwner?: { agentId: string; reviewContext: NativeReviewAssignmentContext | null };
requireBoardResponseWaitSource?: NativeBoardResponseWaitSource;
requireBoardResponseWaitOrigin?: NativeBoardResponseWaitOrigin;
reviewResponsePresentation?: {
@@ -1547,6 +1549,9 @@ export async function commitNativeStatusDecision(input: {
throw new Error("native_status_reason_code_required");
}
const reasonCode = input.decision.reasonCode;
if (reasonCode === "native_provider_model_rejected" && !input.requireModelRejectionOwner) {
throw new Error("native_model_rejection_owner_required");
}
const publications: ActivityPublication[] = [];
const terminalRunsToEmit: (typeof heartbeatRuns.$inferSelect)[] = [];
const committed = await input.db.transaction(async (tx) => {
@@ -1616,6 +1621,20 @@ export async function commitNativeStatusDecision(input: {
) {
throw new NativeStatusRaceError();
}
if (input.requireModelRejectionOwner) {
const owner = input.requireModelRejectionOwner;
// A successor can claim execution without changing statusVersion. Keep
// this new blocking authority behind the same locked issue snapshot.
if (issue.executionRunId && issue.executionRunId !== input.runId) throw new NativeStatusRaceError();
if (owner.reviewContext) {
const review = await getNativeReviewAssignment(tx as unknown as Db, {
companyId: input.companyId, issueId: input.issueId, agentId: owner.agentId, contextSnapshot: owner.reviewContext,
});
if (review?.interaction.status !== "pending") throw new NativeStatusRaceError();
} else if (issue.assigneeAgentId !== owner.agentId || issue.assigneeUserId) {
throw new NativeStatusRaceError();
}
}
if (input.requireExternalChatResponseWaitAuthorization) {
let authorization;
try {
@@ -7,8 +7,20 @@ import {
classifyContinuationFailure,
} from "./service.js";
import { legacyExecutionNeedsReconciliation } from "../legacy-execution-recovery.js";
import { isConfigurationIncompleteFailedRun } from "../../modules/wake-queue/domain/values.js";
describe("classifyAdapterFailureForRecovery", () => {
it("waits for model configuration changes instead of retrying a typed native model rejection", () => {
const run = { errorCode: "native_provider_model_rejected", error: "Fixed model configuration message", resultJson: null };
expect(classifyAdapterFailureForRecovery(run)).toEqual({ kind: "configuration_incomplete" });
expect(classifyContinuationFailure(run as never)).toMatchObject({ kind: "non_retryable", maxAttempts: 0 });
expect(isConfigurationIncompleteFailedRun(run)).toBe(true);
expect(isAiAuthenticationBlocked(run as never)).toBe(false);
const unknown = { ...run, errorCode: "adapter_failed", error: "native_provider_model_rejected" };
expect(classifyAdapterFailureForRecovery(unknown)).toBeNull();
expect(classifyContinuationFailure(unknown as never).kind).toBe("transient_infra");
expect(isConfigurationIncompleteFailedRun(unknown)).toBe(false);
});
it.each(["acpx_auth_required", "claude_auth_required", "codex_auth_required", "adapter_auth_missing", "refresh_token_reused", "authentication_required"])("does not automatically retry provider authentication failure %s", (errorCode) => {
const classification = classifyRunLiveness({ runStatus: "failed", issue: null, errorCode, authenticationRepairRequested: true });
const run = { errorCode, ...classification };
+2 -1
View File
@@ -491,6 +491,7 @@ const TRANSIENT_INFRA_CONTINUATION_ERROR_CODES = new Set<string>([
]);
const NON_RETRYABLE_CONTINUATION_ERROR_CODES = new Set<string>([
"native_provider_model_rejected",
"provider_tool_definition_invalid",
"adapter_engine_unavailable",
"agent_not_invokable",
@@ -626,7 +627,7 @@ export function classifyAdapterFailureForRecovery(
): AdapterFailureRecoveryClassification {
// An engine prerequisite cannot be repaired by asking the same unavailable
// engine to retry. Use the existing configuration-blocker path.
if (latestRun.errorCode === "adapter_engine_unavailable" || latestRun.errorCode === "provider_tool_definition_invalid") {
if (latestRun.errorCode === "adapter_engine_unavailable" || latestRun.errorCode === "provider_tool_definition_invalid" || latestRun.errorCode === "native_provider_model_rejected") {
return { kind: "configuration_incomplete" };
}
if (
+10
View File
@@ -665,6 +665,14 @@ function mapRoutineDescriptionDocument(row: {
};
}
// Match PostgreSQL UUID input, including compact/braced forms and any version.
// Do not trim or resolve prefixes: the exact input must be safe for a UUID query.
function isRoutineUuidInput(id: string): boolean {
if (/\s/.test(id)) return false;
const value = id.startsWith("{") && id.endsWith("}") ? id.slice(1, -1) : id;
return /^(?:[0-9a-f]{4}-?){7}[0-9a-f]{4}$/i.test(value);
}
export function routineService(
db: Db,
deps: {
@@ -682,6 +690,7 @@ export function routineService(
});
async function getRoutineById(id: string) {
if (!isRoutineUuidInput(id)) return null;
return db
.select()
.from(routines)
@@ -766,6 +775,7 @@ export function routineService(
}
async function getTriggerById(id: string) {
if (!isRoutineUuidInput(id)) return null;
return db
.select()
.from(routineTriggers)
@@ -5,6 +5,7 @@ import { redactDiagnosticText } from "@paperclipai/adapter-utils/command-redacti
import { sanitizeWorkspaceRestoreDiagnostic } from "@paperclipai/adapter-utils/workspace-restore-diagnostics";
import { redactCurrentUserText } from "../log-redaction.js";
import { redactSensitiveText, REDACTED_EVENT_VALUE } from "../redaction.js";
import { readNativeModelRejectionDiagnostic } from "./native-runtime/native-provider-failure.js";
type Run = typeof heartbeatRuns.$inferSelect;
type Context = Record<string, string | number | boolean>;
@@ -184,6 +185,8 @@ export function collectRunFailureDiagnostics(run: Run, options: RunFailureReport
"stackPreview", "status", "statusCode", "requestId",
]);
const provider = scalars(read(result, "terminalSessionFailure"), ["category", "title", "details"]);
const nativeProviderFailure = readNativeModelRejectionDiagnostic(read(result, "nativeProviderFailure"));
if (nativeProviderFailure) Object.assign(provider, nativeProviderFailure);
const truncatedFields: string[] = [];
const providerTruncation = read(read(result, "terminalSessionFailure"), "truncatedFields");
if (Array.isArray(providerTruncation)) {