mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-14 17:40:37 +02:00
Merge remote-tracking branch 'origin/master' into codex/provider-routing-connections
* origin/master: fix(tasks): surface Codex ChatGPT model rejection (#15299) fix(runner): continue restart-interrupted Codex turns (#15297) fix(agents): grant configuration access by default (#15283) Guard routine UUID lookups without narrowing valid inputs (#15313) fix(ssh): transport project repositories as their own git checkouts (#14782) feat(exe-dev): copy a source VM with exe.dev cp (#14975) Diagnose native model rejections and preserve repairable reviews (#15304) Verify native semantic input against its raw wire digest (#15301) Retry pooled execution projection reads after disconnects (#15300) Fix browser polling for unsaved agent chats (#15298) test(ui): keep initial reasoning with the saved reply (#15107) docs: add a product feature map (#15288)
This commit is contained in:
commit
7cbd61349a
126 files changed
+5961
-150
No files matched your search
@@ -7,6 +7,19 @@ const run = (overrides: Partial<Run> = {}) => ({ resultJson: null, ...overrides
|
||||
const collect = (error: unknown) => collectRunFailureDiagnostics(run(), { error });
|
||||
|
||||
describe("run failure diagnostics", () => {
|
||||
it("exports only the closed native model/auth rejection vocabulary", () => {
|
||||
const diagnostic = { provider: "codex", category: "model_auth_incompatible", status: 400, authMode: "chatgpt" };
|
||||
const result = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics(run({ resultJson: {
|
||||
nativeProviderFailure: { ...diagnostic, model: "private-model", response: "private-provider-response", prompt: "private-prompt" },
|
||||
} }), {}));
|
||||
expect(result.provider).toEqual(diagnostic);
|
||||
expect(JSON.stringify(result)).not.toContain("private");
|
||||
for (const value of [null, [], { ...diagnostic, category: "private-category" },
|
||||
Object.defineProperty({}, "provider", { get() { throw new Error("private"); } })]) {
|
||||
expect(collectRunFailureDiagnostics(run({ resultJson: { nativeProviderFailure: value } }), {}).provider).toEqual({});
|
||||
}
|
||||
});
|
||||
|
||||
it("selects bounded lock-owner evidence from a caught timeout cause", () => {
|
||||
const error = new Error("outer", { cause: Object.assign(new Error("lock timeout"), {
|
||||
code: "ERR_WORKSPACE_RESTORE_LOCK_TIMEOUT",
|
||||
|
||||
@@ -403,6 +403,9 @@ export function activityService(db: Db) {
|
||||
invocationSource: heartbeatRuns.invocationSource,
|
||||
responsibleUserId: heartbeatRuns.responsibleUserId,
|
||||
errorCode: heartbeatRuns.errorCode,
|
||||
error: sql<string | null>`case when ${heartbeatRuns.status} = 'failed'
|
||||
and ${heartbeatRuns.errorCode} = 'native_provider_model_rejected'
|
||||
then left(${heartbeatRuns.error}, 2000) else null end`,
|
||||
usageJson: summarizedUsageJson,
|
||||
resultJson: summarizedResultJson,
|
||||
logBytes: heartbeatRuns.logBytes,
|
||||
@@ -497,7 +500,7 @@ export function activityService(db: Db) {
|
||||
const [exhaustionRows, leaseRows, executionByRunId, [savedPlan]] = await Promise.all([
|
||||
exhaustionRowsQuery,
|
||||
leaseRowsQuery,
|
||||
executionProjectionsForRuns(db, companyId, runIds),
|
||||
executionProjectionsForRuns(db, companyId, runIds, new Date(), { retryDatabaseReads: true }),
|
||||
savedPlanQuery,
|
||||
]);
|
||||
const retryExhaustedReasonByRunId = new Map<string, string>();
|
||||
|
||||
@@ -1,4 +1,31 @@
|
||||
import { LOW_TRUST_REVIEW_PRESET } from "@paperclipai/shared";
|
||||
import { LOW_TRUST_REVIEW_PRESET, type PermissionKey } from "@paperclipai/shared";
|
||||
|
||||
/** Direct grants for newly activated standard agents. Existing agents are not backfilled. */
|
||||
export const NEW_STANDARD_AGENT_DEFAULT_GRANT_KEYS = [
|
||||
"agents:configure",
|
||||
"agents:suggest-changes",
|
||||
"skills:create",
|
||||
"skills:suggest-changes",
|
||||
"tools:manage_connections",
|
||||
"tools:manage_profiles",
|
||||
"tools:view_audit",
|
||||
"audit:view_agent_actions",
|
||||
"tools:use",
|
||||
"tools:manage_runtime",
|
||||
"inbox:manage",
|
||||
"tasks:assign",
|
||||
"tasks:assign_scope",
|
||||
"tasks:manage_active_checkouts",
|
||||
] as const satisfies readonly PermissionKey[];
|
||||
|
||||
export function newStandardAgentGrantScope(
|
||||
permissionKey: PermissionKey,
|
||||
agentId: string,
|
||||
): Record<string, unknown> | null {
|
||||
if (permissionKey === "tasks:assign_scope") return { subtreeRootAgentId: agentId };
|
||||
if (permissionKey === "inbox:manage") return { responsibleUserOnly: true };
|
||||
return null;
|
||||
}
|
||||
|
||||
export type NormalizedAgentPermissions = Record<string, unknown> & {
|
||||
canCreateAgents: boolean;
|
||||
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
issueExecutionDecisions,
|
||||
issues,
|
||||
issueComments,
|
||||
principalPermissionGrants,
|
||||
} from "@paperclipai/db";
|
||||
import {
|
||||
AGENT_DEFAULT_MAX_CONCURRENT_RUNS,
|
||||
@@ -38,8 +39,8 @@ import {
|
||||
syncAgentAdapterEnvBindings,
|
||||
} from "./agent-secret-bindings.js";
|
||||
import { logActivity } from "./activity-log.js";
|
||||
import { NEW_STANDARD_AGENT_DEFAULT_GRANT_KEYS, newStandardAgentGrantScope, normalizeAgentPermissions, permissionsImplyLowTrust } from "./agent-permissions.js";
|
||||
import { recordAgentStatusEvent, recordResourceCreationEvent } from "./resource-lifecycle-events.js";
|
||||
import { normalizeAgentPermissions } from "./agent-permissions.js";
|
||||
import { REDACTED_EVENT_VALUE, sanitizeRecord } from "../redaction.js";
|
||||
import {
|
||||
assertClaudeOAuthBindingInvariant,
|
||||
@@ -944,6 +945,20 @@ export function agentService(db: Db) {
|
||||
})
|
||||
.returning()
|
||||
.then((rows) => rows[0]);
|
||||
// New standard agents receive the standard direct grants at activation.
|
||||
// Low-trust and bundled agents keep their explicit, narrower grants.
|
||||
if (created.status !== "pending_approval" && !permissionsImplyLowTrust(normalizedPermissions) &&
|
||||
!readBuiltInAgentMarker(created.metadata)) {
|
||||
await tx.insert(principalPermissionGrants).values(
|
||||
NEW_STANDARD_AGENT_DEFAULT_GRANT_KEYS.map((permissionKey) => ({
|
||||
companyId,
|
||||
principalType: "agent" as const,
|
||||
principalId: created.id,
|
||||
permissionKey,
|
||||
scope: newStandardAgentGrantScope(permissionKey, created.id),
|
||||
})),
|
||||
).onConflictDoNothing();
|
||||
}
|
||||
if (options?.aiConnectionInstall) {
|
||||
const install = options.aiConnectionInstall;
|
||||
const connectionIds = [...new Set([install.connectionId, ...(install.memberConnectionIds ?? [])])];
|
||||
@@ -1084,6 +1099,11 @@ export function agentService(db: Db) {
|
||||
await tx.delete(agentWakeupRequests).where(eq(agentWakeupRequests.agentId, id));
|
||||
await tx.delete(agentApiKeys).where(eq(agentApiKeys.agentId, id));
|
||||
await tx.delete(agentRuntimeState).where(eq(agentRuntimeState.agentId, id));
|
||||
await tx.delete(principalPermissionGrants).where(and(
|
||||
eq(principalPermissionGrants.companyId, existing.companyId),
|
||||
eq(principalPermissionGrants.principalType, "agent"),
|
||||
eq(principalPermissionGrants.principalId, id),
|
||||
));
|
||||
const deleted = await tx
|
||||
.delete(agents)
|
||||
.where(eq(agents.id, id))
|
||||
@@ -1151,6 +1171,17 @@ export function agentService(db: Db) {
|
||||
});
|
||||
}
|
||||
await syncAgentSecretBindings(updated, txDb, existing.adapterConfig);
|
||||
if (!permissionsImplyLowTrust(updated.permissions) && !readBuiltInAgentMarker(updated.metadata)) {
|
||||
await tx.insert(principalPermissionGrants).values(
|
||||
NEW_STANDARD_AGENT_DEFAULT_GRANT_KEYS.map((permissionKey) => ({
|
||||
companyId: updated.companyId,
|
||||
principalType: "agent" as const,
|
||||
principalId: updated.id,
|
||||
permissionKey,
|
||||
scope: newStandardAgentGrantScope(permissionKey, updated.id),
|
||||
})),
|
||||
).onConflictDoNothing();
|
||||
}
|
||||
await recordResourceCreationEvent(txDb, existing.companyId, "agent", updated.id);
|
||||
const agent = await agentService(txDb).getById(updated.id);
|
||||
if (!agent) {
|
||||
|
||||
@@ -2060,7 +2060,8 @@ export function authorizationService(db: Db | DbTransaction) {
|
||||
// implicit default-open policy remains responsible-user-only so an
|
||||
// absent row never becomes a company-wide cross-user grant.
|
||||
const grant = await findGrant(companyId, "agent", actorAgentId, "inbox:manage");
|
||||
if (grant && (await scopeAllows(db, companyId, grant.scope, { userId: targetUserId }))) {
|
||||
if (grant && grant.scope?.responsibleUserOnly !== true &&
|
||||
(await scopeAllows(db, companyId, grant.scope, { userId: targetUserId }))) {
|
||||
return allow({
|
||||
action: input.action,
|
||||
reason: "allow_explicit_grant",
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
import { getTableName, type SQL } from "drizzle-orm";
|
||||
import { PgDialect } from "drizzle-orm/pg-core";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { activityService } from "./activity.js";
|
||||
import { executionProjectionsForRuns } from "./execution-projection.js";
|
||||
import { logger } from "../middleware/logger.js";
|
||||
|
||||
const companyId = "00000000-0000-4000-8000-000000000001";
|
||||
const issueId = "00000000-0000-4000-8000-000000000002";
|
||||
const runId = "00000000-0000-4000-8000-000000000003";
|
||||
const now = new Date("2026-01-01T00:00:00Z");
|
||||
const tables = ["heartbeat_runs", "native_run_finalizations", "issue_thread_interactions", "issue_recovery_actions"];
|
||||
const dialect = new PgDialect();
|
||||
function disconnected(code = "CONNECTION_CLOSED") {
|
||||
return new Error("Failed query", { cause: Object.assign(new Error("Connection failed"), { code }) });
|
||||
}
|
||||
|
||||
function fixture(options: {
|
||||
failures?: Record<string, Error[]>;
|
||||
missing?: boolean;
|
||||
noIssue?: boolean;
|
||||
backfill?: boolean;
|
||||
writeFailure?: Error;
|
||||
} = {}) {
|
||||
const attempts: Array<{ key: string; params: unknown[]; order: string[] }> = [];
|
||||
const row = {
|
||||
id: runId, companyId, runId, status: "succeeded", runtimeMode: "native", errorCode: null,
|
||||
nativeIssueId: options.noIssue ? null : issueId, contextSnapshot: {},
|
||||
finishedAt: now, startedAt: now, createdAt: now, resultJson: null, usageJson: null,
|
||||
scheduledRetryAt: null, scheduledRetryAttempt: 0, retryOfRunId: null, nextAction: null,
|
||||
};
|
||||
const select = vi.fn((fields: Record<string, unknown> = {}) => ({
|
||||
from(table: Parameters<typeof getTableName>[0]) {
|
||||
const name = getTableName(table);
|
||||
const key = name === "heartbeat_runs" && "runId" in fields ? "run_list"
|
||||
: name === "heartbeat_runs" && "stdoutExcerpt" in fields ? "liveness_backfill"
|
||||
: name;
|
||||
let condition: SQL;
|
||||
let order: SQL[] = [];
|
||||
let result: Promise<unknown[]> | undefined;
|
||||
const query = {
|
||||
where(value: SQL) { condition = value; return query; },
|
||||
innerJoin() { return query; },
|
||||
orderBy(...values: SQL[]) { order = values; return query; },
|
||||
limit() { return query; },
|
||||
then(resolve: (value: unknown[]) => unknown, reject: (error: unknown) => unknown) {
|
||||
// Re-awaiting the same failed builder cannot recover the connection.
|
||||
result ??= Promise.resolve().then(() => {
|
||||
const attempt = attempts.filter((read) => read.key === key).length;
|
||||
attempts.push({ key, params: dialect.sqlToQuery(condition).params,
|
||||
order: order.map((value) => dialect.sqlToQuery(value).sql) });
|
||||
const error = options.failures?.[key]?.[attempt];
|
||||
if (error) throw error;
|
||||
if (options.missing) return [];
|
||||
if (key === "heartbeat_runs" || key === "run_list") return [row];
|
||||
if (key === "liveness_backfill") return options.backfill ? [row] : [];
|
||||
if (key === "issues") return [{ status: "in_progress", title: "Fixture", workMode: "standard" }];
|
||||
if (key === "issue_thread_interactions") return [{ issueId, kind: "ask_user_questions" }];
|
||||
return [];
|
||||
});
|
||||
return result.then(resolve, reject);
|
||||
},
|
||||
};
|
||||
return query;
|
||||
},
|
||||
}));
|
||||
const updateWhere = vi.fn(async () => { if (options.writeFailure) throw options.writeFailure; });
|
||||
const update = vi.fn(() => ({ set: () => ({ where: updateWhere }) }));
|
||||
const transaction = vi.fn(() => { throw new Error("Must not start or replay a transaction"); });
|
||||
const db = { select, update, transaction } as unknown as Db;
|
||||
return { db, attempts, select, update, updateWhere, transaction,
|
||||
count: (key: string) => attempts.filter((read) => read.key === key).length };
|
||||
}
|
||||
|
||||
afterEach(() => vi.restoreAllMocks());
|
||||
|
||||
describe("execution projection read retries", () => {
|
||||
it.each(tables)("rebuilds only the failed %s SELECT within the existing retry budget", async (table) => {
|
||||
const test = fixture({ failures: { [table]: [disconnected(), disconnected("CONNECTION_ENDED")] } });
|
||||
const result = await executionProjectionsForRuns(test.db, companyId, [runId], now, { retryDatabaseReads: true });
|
||||
expect(result.get(runId)).toMatchObject({ phase: "waiting_for_answer", label: "Waiting for answer" });
|
||||
for (const name of tables) expect(test.count(name)).toBe(name === table ? 3 : 1);
|
||||
expect(test.select).toHaveBeenCalledTimes(6);
|
||||
const reads = test.attempts.filter((read) => read.key === table);
|
||||
expect(reads[1]).toEqual(reads[0]);
|
||||
expect(reads[2]).toEqual(reads[0]);
|
||||
expect(reads[0].params).toContain(companyId);
|
||||
expect(reads[0].params).toContain(table === "heartbeat_runs" || table === "native_run_finalizations" ? runId : issueId);
|
||||
if (table === "issue_thread_interactions") expect(reads[0].params).toContain("pending");
|
||||
if (table === "issue_recovery_actions") expect(reads[0].order).toEqual(['"issue_recovery_actions"."updated_at" desc']);
|
||||
expect(test.update).not.toHaveBeenCalled();
|
||||
expect(test.transaction).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(tables)("preserves the last %s error after three attempts", async (table) => {
|
||||
const errors = [disconnected(), disconnected(), disconnected()];
|
||||
const test = fixture({ failures: { [table]: errors } });
|
||||
await expect(executionProjectionsForRuns(test.db, companyId, [runId], now, { retryDatabaseReads: true })).rejects.toBe(errors[2]);
|
||||
for (const [index, name] of tables.entries()) {
|
||||
expect(test.count(name)).toBe(index < tables.indexOf(table) ? 1 : name === table ? 3 : 0);
|
||||
}
|
||||
});
|
||||
|
||||
it.each(tables)("keeps the default transaction-compatible %s path single-attempt", async (table) => {
|
||||
const error = disconnected();
|
||||
const test = fixture({ failures: { [table]: [error] } });
|
||||
await expect(executionProjectionsForRuns(test.db, companyId, [runId], now)).rejects.toBe(error);
|
||||
expect(test.count(table)).toBe(1);
|
||||
expect(test.transaction).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([new Error("CONNECTION_CLOSED"), disconnected("28P01"), disconnected("42501"), disconnected("22P02"), disconnected("25P02")])(
|
||||
"does not retry an unclassified or non-transient error", async (error) => {
|
||||
const test = fixture({ failures: { issue_thread_interactions: [error] } });
|
||||
await expect(executionProjectionsForRuns(test.db, companyId, [runId], now, { retryDatabaseReads: true })).rejects.toBe(error);
|
||||
expect(test.count("issue_thread_interactions")).toBe(1);
|
||||
expect(test.count("issue_recovery_actions")).toBe(0);
|
||||
},
|
||||
);
|
||||
|
||||
it("preserves empty and taskless projection behavior without unnecessary reads", async () => {
|
||||
const empty = fixture();
|
||||
expect(await executionProjectionsForRuns(empty.db, companyId, [], now, { retryDatabaseReads: true })).toEqual(new Map());
|
||||
expect(empty.select).not.toHaveBeenCalled();
|
||||
for (const options of [{ missing: true }, { noIssue: true }]) {
|
||||
const test = fixture(options);
|
||||
const result = await executionProjectionsForRuns(test.db, companyId, [runId], now, { retryDatabaseReads: true });
|
||||
expect(test.select).toHaveBeenCalledTimes(2);
|
||||
expect(options.missing ? result.size : result.get(runId)?.phase).toBe(options.missing ? 0 : "completed");
|
||||
}
|
||||
});
|
||||
|
||||
it.each([false, true])("the run list retries projection reads without replaying other reads or backfill writes (write fails: %s)", async (writeFails) => {
|
||||
vi.spyOn(logger, "warn").mockImplementation(() => undefined);
|
||||
const test = fixture({ backfill: true, writeFailure: writeFails ? disconnected() : undefined,
|
||||
failures: { issue_thread_interactions: [disconnected(), disconnected()] } });
|
||||
const result = await activityService(test.db).runsForIssue(companyId, issueId);
|
||||
expect(result).toMatchObject([{ runId, execution: { phase: "waiting_for_answer" } }]);
|
||||
expect(test.count("issue_thread_interactions")).toBe(3);
|
||||
for (const key of ["run_list", "liveness_backfill", "heartbeat_runs", "native_run_finalizations", "environment_leases", "issue_recovery_actions"]) {
|
||||
expect(test.count(key)).toBe(1);
|
||||
}
|
||||
// One retry-exhaustion lookup plus the independent backfill's event count.
|
||||
expect(test.count("heartbeat_run_events")).toBe(2);
|
||||
expect(test.update).toHaveBeenCalledTimes(1);
|
||||
expect(test.updateWhere).toHaveBeenCalledTimes(1);
|
||||
expect(test.transaction).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
import type { ExecutionProjection } from "@paperclipai/shared";
|
||||
import { EXECUTION_CONTROL_DEADLINE_MS } from "./execution-control-deadline.js";
|
||||
import { executionFailureRetryCount } from "./execution-recovery-attempt.js";
|
||||
import { retryIdempotentDatabaseOperation } from "../database-retry.js";
|
||||
const text = (v: unknown) => (typeof v === "string" ? v : null);
|
||||
const executionRunColumns = {
|
||||
id: heartbeatRuns.id,
|
||||
@@ -48,10 +49,16 @@ export async function executionProjectionsForRuns(
|
||||
companyId: string,
|
||||
runIds: string[],
|
||||
now = new Date(),
|
||||
options: { retryDatabaseReads?: boolean } = {},
|
||||
) {
|
||||
const projections = new Map<string, ExecutionProjection>();
|
||||
if (!runIds.length) return projections;
|
||||
const runs = await db
|
||||
// Opt in only with a pooled Db outside a transaction. Issue enrichment also
|
||||
// calls this helper with a transaction, whose failed reads must not replay.
|
||||
const read = <T>(query: () => Promise<T>) => options.retryDatabaseReads
|
||||
? retryIdempotentDatabaseOperation(query)
|
||||
: query();
|
||||
const runs = await read(async () => db
|
||||
.select(executionRunColumns)
|
||||
.from(heartbeatRuns)
|
||||
.where(
|
||||
@@ -59,8 +66,8 @@ export async function executionProjectionsForRuns(
|
||||
eq(heartbeatRuns.companyId, companyId),
|
||||
inArray(heartbeatRuns.id, runIds),
|
||||
),
|
||||
);
|
||||
const coordinators = await db
|
||||
));
|
||||
const coordinators = await read(async () => db
|
||||
.select()
|
||||
.from(nativeRunFinalizations)
|
||||
.where(
|
||||
@@ -68,7 +75,7 @@ export async function executionProjectionsForRuns(
|
||||
eq(nativeRunFinalizations.companyId, companyId),
|
||||
inArray(nativeRunFinalizations.runId, runIds),
|
||||
),
|
||||
);
|
||||
));
|
||||
const issueIds = [
|
||||
...new Set(
|
||||
runs
|
||||
@@ -77,7 +84,7 @@ export async function executionProjectionsForRuns(
|
||||
),
|
||||
];
|
||||
const pending = issueIds.length
|
||||
? await db
|
||||
? await read(async () => db
|
||||
.select({
|
||||
issueId: issueThreadInteractions.issueId,
|
||||
kind: issueThreadInteractions.kind,
|
||||
@@ -89,10 +96,10 @@ export async function executionProjectionsForRuns(
|
||||
inArray(issueThreadInteractions.issueId, issueIds),
|
||||
eq(issueThreadInteractions.status, "pending"),
|
||||
),
|
||||
)
|
||||
))
|
||||
: [];
|
||||
const recovery = issueIds.length
|
||||
? await db
|
||||
? await read(async () => db
|
||||
.select({
|
||||
issueId: issueRecoveryActions.sourceIssueId,
|
||||
cause: issueRecoveryActions.cause,
|
||||
@@ -112,7 +119,7 @@ export async function executionProjectionsForRuns(
|
||||
]),
|
||||
),
|
||||
)
|
||||
.orderBy(desc(issueRecoveryActions.updatedAt))
|
||||
.orderBy(desc(issueRecoveryActions.updatedAt)))
|
||||
: [];
|
||||
const coordinatorByRun = new Map(coordinators.map((row) => [row.runId, row]));
|
||||
for (const run of runs) {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { PERMISSION_KEYS } from "@paperclipai/shared";
|
||||
import type { HumanCompanyMembershipRole } from "@paperclipai/shared";
|
||||
import { NEW_STANDARD_AGENT_DEFAULT_GRANT_KEYS, newStandardAgentGrantScope } from "./agent-permissions.js";
|
||||
import { grantsForHumanRole } from "./company-member-roles.js";
|
||||
|
||||
export function grantsFromDefaults(
|
||||
@@ -38,22 +39,21 @@ export function grantsFromDefaults(
|
||||
}
|
||||
|
||||
export function agentJoinGrantsFromDefaults(
|
||||
defaultsPayload: Record<string, unknown> | null | undefined
|
||||
defaultsPayload: Record<string, unknown> | null | undefined,
|
||||
agentId: string,
|
||||
): Array<{
|
||||
permissionKey: (typeof PERMISSION_KEYS)[number];
|
||||
scope: Record<string, unknown> | null;
|
||||
}> {
|
||||
const grants = grantsFromDefaults(defaultsPayload, "agent");
|
||||
if (grants.some((grant) => grant.permissionKey === "tasks:assign")) {
|
||||
return grants;
|
||||
// Invitation approval replaces the whole grant set after agent creation.
|
||||
// Retain new-agent defaults, but never widen an explicitly scoped grant.
|
||||
for (const permissionKey of NEW_STANDARD_AGENT_DEFAULT_GRANT_KEYS) {
|
||||
if (!grants.some((grant) => grant.permissionKey === permissionKey)) {
|
||||
grants.push({ permissionKey, scope: newStandardAgentGrantScope(permissionKey, agentId) });
|
||||
}
|
||||
}
|
||||
return [
|
||||
...grants,
|
||||
{
|
||||
permissionKey: "tasks:assign",
|
||||
scope: null,
|
||||
},
|
||||
];
|
||||
return grants;
|
||||
}
|
||||
|
||||
export function humanJoinGrantsFromDefaults(
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
import { and, desc, eq, sql } from "drizzle-orm";
|
||||
import { heartbeatRunEvents, heartbeatRuns, type Db } from "@paperclipai/db";
|
||||
import { parseNativeExecutionInput, validatePrpEvent, type PrpTerminalState } from "../../vendor/paperclip-runner/index.js";
|
||||
import { nativeSha256 } from "./canonical.js";
|
||||
import { createNativeProviderFailureObservation } from "./native-provider-failure.js";
|
||||
|
||||
function record(value: unknown): Record<string, unknown> {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value)
|
||||
? value as Record<string, unknown> : {};
|
||||
}
|
||||
|
||||
/** Re-derive authority from immutable run-bound evidence, including after restart. */
|
||||
export async function readPersistedNativeModelRejection(
|
||||
db: Db,
|
||||
run: typeof heartbeatRuns.$inferSelect,
|
||||
turnId: string | null,
|
||||
terminal: PrpTerminalState,
|
||||
) {
|
||||
if (run.runtimeMode !== "native" || !turnId || !run.nativeSessionId || !run.runnerInstanceId ||
|
||||
terminal.turnTerminalState !== "failed" || terminal.runTerminalState !== "failed") return null;
|
||||
let execution;
|
||||
try { execution = parseNativeExecutionInput(record(run.runnerProfileJson).nativeExecutionInput); } catch { return null; }
|
||||
if (execution.provider.kind !== "codex" || !execution.provider.model ||
|
||||
execution.binding.runId !== run.id || execution.binding.companyId !== run.companyId ||
|
||||
execution.binding.agentId !== run.agentId || execution.binding.issueId !== run.nativeIssueId ||
|
||||
execution.session.normalizedSessionId !== run.nativeSessionId ||
|
||||
execution.completionContract.id !== run.completionContractId ||
|
||||
execution.completionContract.sha256 !== run.completionContractSha256) return null;
|
||||
|
||||
const [row] = await db.select().from(heartbeatRunEvents).where(and(
|
||||
eq(heartbeatRunEvents.companyId, run.companyId), eq(heartbeatRunEvents.runId, run.id),
|
||||
eq(heartbeatRunEvents.sourceInstanceId, run.runnerInstanceId), eq(heartbeatRunEvents.eventType, "turn.failed"),
|
||||
sql`${heartbeatRunEvents.payload}->'prpEvent'->>'turnId' = ${turnId}`,
|
||||
)).orderBy(desc(heartbeatRunEvents.seq)).limit(1);
|
||||
if (!row) return null;
|
||||
const parsed = validatePrpEvent(record(row.payload).prpEvent);
|
||||
if (!parsed.ok || parsed.event.runId !== run.id || parsed.event.normalizedSessionId !== run.nativeSessionId ||
|
||||
parsed.event.sourceKind !== "runner" || parsed.event.sourceInstanceId !== run.runnerInstanceId ||
|
||||
parsed.event.sourceEventId !== row.sourceEventId || parsed.event.sourceSeq !== row.sourceSeq ||
|
||||
parsed.event.eventType !== row.eventType || parsed.event.turnId !== turnId ||
|
||||
row.sourcePayloadSha256 !== nativeSha256(parsed.event)) return null;
|
||||
const observed = createNativeProviderFailureObservation(execution.provider);
|
||||
observed.observe(parsed.event);
|
||||
return observed.forTerminal(turnId, terminal);
|
||||
}
|
||||
@@ -0,0 +1,272 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { agents, agentWakeupRequests, companies, completionContracts, createDb, heartbeatRunEvents, heartbeatRuns, issueRecoveryActions, issueThreadInteractions, issues, nativeRunFinalizations, nativeRunResults, statusDecisions, workAssessments } from "@paperclipai/db";
|
||||
import type { ControlPlanePort, NativeExecutionInputV1, PrpEvent, PrpTerminalState } from "@paperclipai/paperclip-runner";
|
||||
import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
|
||||
import { CONTROL_PLANE_CONFORMANCE_RESULT } from "../../vendor/paperclip-runner/testing.js";
|
||||
|
||||
const provider = vi.hoisted(() => ({ execute: vi.fn() }));
|
||||
vi.mock("../../vendor/paperclip-runner/index.js", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("../../vendor/paperclip-runner/index.js")>()),
|
||||
executeNativeSession: provider.execute,
|
||||
}));
|
||||
|
||||
import { executePaperclipNativeSession } from "./native-session-executor.js";
|
||||
import { buildNativeCompletionContract } from "./completion-contracts.js";
|
||||
import { PaperclipControlPlanePort } from "./paperclip-control-plane-port.js";
|
||||
import { prepareNativeHeartbeatRun } from "./prepare-native-run.js";
|
||||
import { NATIVE_MODEL_REJECTION_DIAGNOSTIC, NATIVE_MODEL_REJECTION_MESSAGE } from "./native-provider-failure.js";
|
||||
import { collectRunFailureDiagnostics } from "../run-failure-diagnostics.js";
|
||||
import { finalizeNativeRun } from "./native-run-finalizer.js";
|
||||
import { readPersistedNativeModelRejection } from "./native-provider-failure-evidence.js";
|
||||
import { heartbeatService } from "../heartbeat.js";
|
||||
import { getNativeReviewAssignment, readNativeReviewAssignmentContext } from "./native-review-participant.js";
|
||||
import { claimQueuedNativeReviewRun } from "./native-review-dispatch.js";
|
||||
import { commitNativeStatusDecision, NativeStatusRaceError } from "./status-decision-committer.js";
|
||||
import { NATIVE_STATUS_ARBITER_POLICY_VERSION } from "./status-arbiter.js";
|
||||
import { issueThreadInteractionService } from "../issue-thread-interactions.js";
|
||||
import { createReleaseIssueExecution } from "../../modules/wake-queue/application/use-cases.js";
|
||||
import { createPostgresWakeQueueAdapter } from "../../modules/wake-queue/adapters/postgres.js";
|
||||
|
||||
describe("native completed failure diagnostics", () => {
|
||||
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
const companyId = randomUUID();
|
||||
const agentId = randomUUID();
|
||||
const workerId = randomUUID();
|
||||
const model = "test-model";
|
||||
|
||||
beforeAll(async () => {
|
||||
temporary = await startEmbeddedPostgresTestDatabase("native-model-failure-");
|
||||
db = createDb(temporary.connectionString);
|
||||
await db.insert(companies).values({ id: companyId, name: "Native failure regression", issuePrefix: "NMF" });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Native failure agent", status: "active", adapterType: "paperclip_runner", adapterConfig: { provider: "codex", model } });
|
||||
await db.insert(agents).values({ id: workerId, companyId, name: "Review source worker", status: "active", adapterType: "paperclip_runner" });
|
||||
});
|
||||
afterAll(async () => { await temporary?.cleanup(); });
|
||||
|
||||
it.each(["committed", "duplicate", "pending-review", "superseded-review", "resolved-review", "reassigned", "successor-race", "different-turn", "unknown-error", "untrusted-source", "untrusted-run", "untrusted-session", "succeeded", "cancelled"] as const)(
|
||||
"preserves accepted results and only labels the matching provider rejection (%s)", async (scenario) => {
|
||||
const issueId = randomUUID();
|
||||
const runId = randomUUID();
|
||||
const [issue] = await db.insert(issues).values({ id: issueId, companyId, title: "Explain a provider rejection", status: "in_progress", workMode: "standard", assigneeAgentId: agentId }).returning();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running", runtimeMode: "native", nativeIssueId: issueId, invocationSource: "assignment", triggerDetail: "system", contextSnapshot: { issueId } }).returning();
|
||||
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId));
|
||||
const native = await prepareNativeHeartbeatRun({ db, run, issue, environmentLeaseId: randomUUID() });
|
||||
const [contract] = await db.select().from(completionContracts).where(eq(completionContracts.issueId, issueId));
|
||||
const completionInput = buildNativeCompletionContract(issue, { revision: contract.revision });
|
||||
await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId, phase: "observed" });
|
||||
const execution: NativeExecutionInputV1 = {
|
||||
schema: "paperclip.native-execution-input.v1",
|
||||
binding: { companyId, runId, issueId, agentId, executionWorkspaceId: runId },
|
||||
provider: { kind: "codex", model },
|
||||
task: { identifier: issueId, title: issue.title, description: null, prompt: "Reply to the current message.", workMode: "standard" },
|
||||
workspace: { cwd: "/tmp", repoUrl: null, repoRef: null, branchName: null },
|
||||
session: { normalizedSessionId: native.normalizedSessionId, driverKind: "codex_app_server", protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } },
|
||||
completionContract: { id: contract.id, sha256: contract.canonicalSha256, schemaVersion: "paperclip.completion-contract.v1", contract: completionInput },
|
||||
interactionResponses: [], credentialBindings: [],
|
||||
};
|
||||
await db.update(heartbeatRuns).set({ runnerProfileJson: { nativeExecutionInput: execution } }).where(eq(heartbeatRuns.id, runId));
|
||||
if (scenario.endsWith("-review")) {
|
||||
const sourceRunId = randomUUID(), sourceResultId = randomUUID(), assessmentId = randomUUID();
|
||||
const decisionId = randomUUID(), interactionId = randomUUID();
|
||||
await db.insert(heartbeatRuns).values({ id: sourceRunId, companyId, agentId: workerId, nativeIssueId: issueId, runtimeMode: "native", status: "succeeded", completionContractId: contract.id, completionContractSha256: contract.canonicalSha256 });
|
||||
await db.insert(nativeRunResults).values({ id: sourceResultId, companyId, issueId, runId: sourceRunId, completionContractId: contract.id, serverFingerprint: sourceResultId, schemaStatus: "accepted", resultJson: {}, canonicalSha256: sourceResultId });
|
||||
await db.insert(workAssessments).values({ id: assessmentId, companyId, issueId, runId: sourceRunId, contractId: contract.id, resultId: sourceResultId, triggerKind: "native_result", triggerActorCompanyId: companyId, priorIssueStatus: "in_progress", priorStatusVersion: 0, policyVersion: "fixture", assessmentJson: {}, inputDigest: assessmentId });
|
||||
const [reviewIssue] = await db.update(issues).set({ status: "in_review", assigneeAgentId: workerId }).where(eq(issues.id, issueId)).returning();
|
||||
await db.insert(statusDecisions).values({ id: decisionId, companyId, issueId, runId: sourceRunId, assessmentId, decisionVersion: 1, policyVersion: "fixture", fromStatus: "in_progress", toStatus: "in_review", reasonCode: "explicit_review", decisionJson: { projectedStatusVersion: reviewIssue.statusVersion }, decisionDigest: decisionId, applicationState: "applied" });
|
||||
await db.update(issues).set({ lastStatusDecisionId: decisionId }).where(eq(issues.id, issueId));
|
||||
await db.insert(issueThreadInteractions).values({ id: interactionId, companyId, issueId, kind: "request_confirmation", sourceRunId, addresseeAgentId: agentId, createdByAgentId: workerId,
|
||||
continuationPolicy: "wake_assignee",
|
||||
status: scenario === "resolved-review" ? "accepted" : "pending",
|
||||
...(scenario === "resolved-review" ? { resolvedByRunId: runId, resolvedByAgentId: agentId } : {}),
|
||||
payload: { version: 1, prompt: "Review the completed work.", target: { type: "custom", key: "native_completion_review", revisionId: decisionId } },
|
||||
});
|
||||
const contextSnapshot = { issueId, wakeReason: "native_completion_review", nativeReviewInteractionId: interactionId, nativeReviewDecisionId: decisionId };
|
||||
await db.update(heartbeatRuns).set({ contextSnapshot }).where(eq(heartbeatRuns.id, runId));
|
||||
if (scenario === "superseded-review") await db.update(issues).set({ lastStatusDecisionId: null }).where(eq(issues.id, issueId));
|
||||
const review = await getNativeReviewAssignment(db, { companyId, issueId, agentId, contextSnapshot, allowResolvedByRunId: runId });
|
||||
if (scenario === "superseded-review") expect(review).toBeNull();
|
||||
else expect(review?.interaction.status).toBe(scenario === "resolved-review" ? "accepted" : "pending");
|
||||
}
|
||||
const event: PrpEvent = {
|
||||
schema: "paperclip.prp.event.v1", runId, normalizedSessionId: native.normalizedSessionId,
|
||||
turnId: native.turnId, sourceInstanceId: native.runnerInstanceId, sourceEventId: "model-terminal", sourceSeq: 1,
|
||||
sourceKind: "runner", eventType: "turn.failed", schemaVersion: 1, priority: 0, emittedAt: new Date().toISOString(),
|
||||
payload: { status: "failed", error: { codexErrorInfo: "other", message: JSON.stringify({
|
||||
type: "error", status: 400, error: { type: "invalid_request_error", message: scenario === "unknown-error" ? "Private unknown failure" : `The '${model}' model is not supported when using Codex with a ChatGPT account.` },
|
||||
}) } },
|
||||
};
|
||||
const openInput = { identity: { companyId, issueId, runId, agentId, sessionId: native.normalizedSessionId }, backendKind: "mock" as const, sourceInstanceId: native.runnerInstanceId };
|
||||
if (scenario === "duplicate") {
|
||||
const prior = new PaperclipControlPlanePort(db, { companyId, issueId, runId, agentId, sessionId: native.normalizedSessionId, sourceInstanceId: native.runnerInstanceId, controlPlaneSourceInstanceId: "prior-controller", completionContractId: contract.id, completionContractSha256: contract.canonicalSha256 });
|
||||
await prior.openRun(openInput);
|
||||
expect((await prior.appendEvent(event)).disposition).toBe("committed");
|
||||
}
|
||||
const turnId = scenario === "different-turn" ? "later-turn" : native.turnId;
|
||||
const terminal: PrpTerminalState = {
|
||||
schema: "paperclip.prp.terminal.v1", turnTerminalState: scenario === "succeeded" ? "completed" : scenario === "cancelled" ? "cancelled" : "failed",
|
||||
runTerminalState: scenario === "succeeded" ? "succeeded" : scenario === "cancelled" ? "cancelled" : "failed", reportedWorkDisposition: "blocked",
|
||||
};
|
||||
const result = { ...CONTROL_PLANE_CONFORMANCE_RESULT, reportedWorkDisposition: "blocked" as const,
|
||||
blocker: { reasonCode: "provider_rejected", owner: { kind: "user" as const, name: "Task owner" }, unblockAction: "Select a compatible connection.", scope: "task_wide" as const },
|
||||
completionClaim: { ...CONTROL_PLANE_CONFORMANCE_RESULT.completionClaim, contractRevision: completionInput.revision } };
|
||||
provider.execute.mockReset().mockImplementation(async (options: { controlPlane: ControlPlanePort }) => {
|
||||
await options.controlPlane.openRun(openInput);
|
||||
if (scenario === "untrusted-source") {
|
||||
await expect(options.controlPlane.appendEvent({ ...event, sourceInstanceId: "wrong-runner" })).rejects.toThrow("native_event_source_binding_mismatch");
|
||||
} else if (scenario === "untrusted-run" || scenario === "untrusted-session") {
|
||||
await expect(options.controlPlane.appendEvent({ ...event,
|
||||
...(scenario === "untrusted-run" ? { runId: randomUUID() } : { normalizedSessionId: "wrong-session" }),
|
||||
})).rejects.toThrow("native_event_binding_mismatch");
|
||||
} else {
|
||||
expect((await options.controlPlane.appendEvent(event)).disposition).toBe(scenario === "duplicate" ? "duplicate" : "committed");
|
||||
}
|
||||
await options.controlPlane.completeRun({ result, terminal, turnId });
|
||||
return { result, terminal, turnId, normalizedSessionId: native.normalizedSessionId, providerSessionId: "fixture-provider-session", driverKind: "codex_app_server", driverVersion: "fixture", nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null };
|
||||
});
|
||||
const adapter = await executePaperclipNativeSession({ db, execution, runnerInstanceId: native.runnerInstanceId, backend: {
|
||||
descriptor: async () => { throw new Error("Unexpected provider call"); },
|
||||
openSession: async () => { throw new Error("Unexpected provider call"); },
|
||||
} });
|
||||
const classified = scenario === "committed" || scenario === "duplicate" || scenario.endsWith("-review") || scenario === "reassigned" || scenario === "successor-race";
|
||||
const reviewAuthorityEnded = scenario === "superseded-review" || scenario === "resolved-review";
|
||||
const ownerChanged = scenario === "reassigned" || scenario === "successor-race";
|
||||
expect(adapter.errorCode).toBe(classified ? "native_provider_model_rejected" : undefined);
|
||||
expect(adapter.errorMessage).toBe(classified ? NATIVE_MODEL_REJECTION_MESSAGE : scenario === "succeeded" ? null : `Native session ${terminal.runTerminalState}`);
|
||||
expect(adapter.resultJson?.nativeProviderFailure).toEqual(classified ? NATIVE_MODEL_REJECTION_DIAGNOSTIC : undefined);
|
||||
expect(adapter.nativeFinalization).toMatchObject({ result, terminal, turnId, workspaceFinalizeStatus: "pending" });
|
||||
expect(provider.execute).toHaveBeenCalledTimes(1);
|
||||
expect(execution.provider).toEqual({ kind: "codex", model });
|
||||
const [coordinator] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId));
|
||||
expect(coordinator).toMatchObject({ phase: "workspace_finalizing", leaseOwner: null, attempt: 1, nextAttemptAt: null });
|
||||
expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, runId))).toHaveLength(1);
|
||||
expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId))).toHaveLength(0);
|
||||
expect(await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, runId))).toHaveLength(scenario.startsWith("untrusted-") ? 0 : 1);
|
||||
const diagnostics = collectRunFailureDiagnostics({ ...run, resultJson: adapter.resultJson ?? null }, {});
|
||||
expect(diagnostics.provider).toEqual(classified ? NATIVE_MODEL_REJECTION_DIAGNOSTIC : {});
|
||||
expect(JSON.stringify(diagnostics)).not.toContain(model);
|
||||
expect(JSON.stringify(diagnostics)).not.toContain("Private");
|
||||
const [persistedRun] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
|
||||
expect(persistedRun.errorCode).toBeNull(); // No adapter metadata has been saved yet.
|
||||
expect(await readPersistedNativeModelRejection(db, persistedRun, turnId, terminal)).toEqual(classified ? {
|
||||
errorCode: "native_provider_model_rejected", errorMessage: NATIVE_MODEL_REJECTION_MESSAGE, diagnostic: NATIVE_MODEL_REJECTION_DIAGNOSTIC,
|
||||
} : null);
|
||||
if (classified) {
|
||||
if (scenario === "committed") {
|
||||
// Neither saved display diagnostics nor a different execution binding
|
||||
// can confer the right to change the finalizer's retry decision.
|
||||
for (const changed of [
|
||||
{ ...persistedRun, runnerProfileJson: null },
|
||||
{ ...persistedRun, nativeSessionId: "foreign-session" },
|
||||
{ ...persistedRun, runnerInstanceId: "foreign-source" },
|
||||
{ ...persistedRun, companyId: randomUUID() },
|
||||
{ ...persistedRun, completionContractSha256: "foreign-contract" },
|
||||
{ ...persistedRun, runnerProfileJson: { nativeExecutionInput: { ...execution, provider: { kind: "codex", model: "different-model" } } } },
|
||||
{ ...persistedRun, runnerProfileJson: { nativeExecutionInput: { ...execution, binding: { ...execution.binding, runId: randomUUID() } } } },
|
||||
]) expect(await readPersistedNativeModelRejection(db, changed, turnId, terminal)).toBeNull();
|
||||
const [stored] = await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, runId));
|
||||
await db.update(heartbeatRunEvents).set({ sourcePayloadSha256: "invalid-hash" }).where(eq(heartbeatRunEvents.id, stored.id));
|
||||
expect(await readPersistedNativeModelRejection(db, persistedRun, turnId, terminal)).toBeNull();
|
||||
await db.update(heartbeatRunEvents).set({ sourcePayloadSha256: stored.sourcePayloadSha256 }).where(eq(heartbeatRunEvents.id, stored.id));
|
||||
}
|
||||
// Simulate a restart after native result acceptance, before heartbeat's
|
||||
// adapter update. Finalization must use durable provider evidence.
|
||||
const acceptedBefore = await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, runId));
|
||||
if (scenario === "reassigned") {
|
||||
await db.update(issues).set({ assigneeAgentId: workerId, executionRunId: null }).where(eq(issues.id, issueId));
|
||||
} else if (scenario === "successor-race") {
|
||||
const [snapshot] = await db.select().from(issues).where(eq(issues.id, issueId));
|
||||
const newerRunId = randomUUID(), raceAssessmentId = randomUUID();
|
||||
await db.insert(heartbeatRuns).values({ id: newerRunId, companyId, agentId, status: "running", nativeIssueId: issueId });
|
||||
await db.update(issues).set({ executionRunId: newerRunId }).where(eq(issues.id, issueId));
|
||||
const [changed] = await db.select().from(issues).where(eq(issues.id, issueId));
|
||||
expect(changed.statusVersion).toBe(snapshot.statusVersion);
|
||||
await db.insert(workAssessments).values({ id: raceAssessmentId, companyId, issueId, runId, contractId: contract.id, resultId: acceptedBefore[0].id, triggerKind: "native_result", triggerActorCompanyId: companyId, priorIssueStatus: snapshot.status, priorStatusVersion: snapshot.statusVersion, policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, assessmentJson: {}, inputDigest: raceAssessmentId });
|
||||
await expect(commitNativeStatusDecision({ db, companyId, issueId, runId, assessmentId: raceAssessmentId,
|
||||
priorStatus: snapshot.status, priorStatusVersion: snapshot.statusVersion, priorDecisionId: snapshot.lastStatusDecisionId,
|
||||
requireModelRejectionOwner: { agentId, reviewContext: null },
|
||||
decision: { policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, statusAction: "blocked", toStatus: "blocked", reasonCode: "native_provider_model_rejected", unblockDescriptor: { owner: "board", action: NATIVE_MODEL_REJECTION_MESSAGE }, effects: [{ kind: "bind_blocker", owner: "board", action: NATIVE_MODEL_REJECTION_MESSAGE }] },
|
||||
})).rejects.toBeInstanceOf(NativeStatusRaceError);
|
||||
expect(await db.select().from(statusDecisions).where(eq(statusDecisions.runId, runId))).toHaveLength(0);
|
||||
}
|
||||
for (let retry = 0; retry < 2; retry++) {
|
||||
await finalizeNativeRun({ db, runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
|
||||
}
|
||||
expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, runId))).toEqual(acceptedBefore);
|
||||
const [finalRun] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
|
||||
expect(finalRun).toMatchObject({ status: "failed", errorCode: "native_provider_model_rejected", error: NATIVE_MODEL_REJECTION_MESSAGE });
|
||||
const [finalIssue] = await db.select().from(issues).where(eq(issues.id, issueId));
|
||||
expect(finalIssue.status).toBe(scenario.endsWith("-review") ? "in_review" : ownerChanged ? "in_progress" : "blocked");
|
||||
const decisions = await db.select().from(statusDecisions).where(eq(statusDecisions.runId, runId));
|
||||
expect(decisions).toHaveLength(1);
|
||||
expect(decisions[0].reasonCode).toBe(reviewAuthorityEnded ? "native_review_action_finished" : ownerChanged ? "run_failed_partial_evidence_preserved" : "native_provider_model_rejected");
|
||||
const wakes = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId));
|
||||
expect(wakes.filter(wake => wake.payload?.issueId === issueId)).toHaveLength(ownerChanged ? 1 : 0);
|
||||
if (!ownerChanged) expect(await heartbeatService(db).dispatchPendingNativeStatusWakeups({ companyId })).toEqual({ scanned: 0, dispatched: 0, recovered: 0, deferred: 0 });
|
||||
expect(provider.execute).toHaveBeenCalledTimes(1);
|
||||
if (scenario === "pending-review") {
|
||||
const reviewContext = readNativeReviewAssignmentContext(finalRun.contextSnapshot)!;
|
||||
expect(finalIssue).toMatchObject({ lastStatusDecisionId: reviewContext.nativeReviewDecisionId, executionRunId: null, checkoutRunId: null });
|
||||
const originalReview = await getNativeReviewAssignment(db, { companyId, issueId, agentId, contextSnapshot: reviewContext });
|
||||
expect(originalReview?.interaction.status).toBe("pending");
|
||||
expect(finalIssue.statusVersion).toBe(originalReview!.sourceDecision.decisionJson.projectedStatusVersion);
|
||||
expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId))).toHaveLength(0);
|
||||
|
||||
// Exercise the real release and periodic recovery lanes, rather than
|
||||
// assuming that absence of a finalizer wake prevents later retries.
|
||||
const unexpected = vi.fn(async () => { throw new Error("Unexpected automatic recovery"); });
|
||||
const release = createReleaseIssueExecution({
|
||||
issueLock: createPostgresWakeQueueAdapter(db, { resolveResponsibleUserId: unexpected, getRoutineEnv: unexpected, resolveSessionBeforeForWakeup: unexpected }),
|
||||
recovery: { escalateStrandedAssignedIssue: unexpected, escalateStrandedRecoveryIssueInPlace: unexpected },
|
||||
});
|
||||
expect((await release({ companyId, runId, now: new Date() })).outcome.kind).toBe("released");
|
||||
await heartbeatService(db).reconcileStrandedAssignedIssues();
|
||||
expect(unexpected).not.toHaveBeenCalled();
|
||||
expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId))).toHaveLength(0);
|
||||
expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId))).filter(wake => wake.payload?.issueId === issueId)).toHaveLength(0);
|
||||
const [afterRecovery] = await db.select().from(issues).where(eq(issues.id, issueId));
|
||||
expect(afterRecovery).toMatchObject({ status: "in_review", statusVersion: finalIssue.statusVersion, lastStatusDecisionId: finalIssue.lastStatusDecisionId });
|
||||
|
||||
// The operator repairs configuration and explicitly retries the
|
||||
// assigned reviewer. No review rebinding or status reset is needed.
|
||||
await db.update(agents).set({ adapterConfig: { provider: "codex", model: "repaired-model" }, runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 } } }).where(eq(agents.id, agentId));
|
||||
// Occupy the execution slot to exercise real wake admission without
|
||||
// starting a provider. The normal atomic review claim follows below.
|
||||
const occupiedRunId = randomUUID();
|
||||
await db.insert(heartbeatRuns).values({ id: occupiedRunId, companyId, agentId, status: "running", startedAt: new Date() });
|
||||
const retryRun = await heartbeatService(db, { runtimeEnv: {} }).wakeup(agentId, {
|
||||
failedRunId: runId, source: "on_demand", triggerDetail: "manual", reason: "retry_failed_run",
|
||||
requestedByActorType: "user", requestedByActorId: "review-repair-operator",
|
||||
payload: { issueId }, contextSnapshot: { ...reviewContext },
|
||||
});
|
||||
expect(retryRun).toMatchObject({ status: "queued", retryOfRunId: runId, contextSnapshot: { issueId, ...reviewContext } });
|
||||
const retryRunId = retryRun!.id;
|
||||
await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date() }).where(eq(heartbeatRuns.id, occupiedRunId));
|
||||
expect(await claimQueuedNativeReviewRun(db, { run: retryRun!, agentNameKey: "reviewer", claimedAt: new Date(), claimValues: { nativeIssueId: issueId, runtimeMode: "native" } }))
|
||||
.toMatchObject({ id: retryRunId, status: "running" });
|
||||
expect(await getNativeReviewAssignment(db, { companyId, issueId, agentId, contextSnapshot: reviewContext, actingRunId: retryRunId, issueExecutionRunId: retryRunId })).not.toBeNull();
|
||||
await issueThreadInteractionService(db).acceptInteraction(
|
||||
{ id: issueId, companyId, projectId: null, goalId: null, status: "in_review" },
|
||||
reviewContext.nativeReviewInteractionId, {}, { agentId, runId: retryRunId },
|
||||
);
|
||||
const [resolved] = await db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.id, reviewContext.nativeReviewInteractionId));
|
||||
expect(resolved).toMatchObject({ status: "accepted", resolvedByAgentId: agentId, resolvedByRunId: retryRunId });
|
||||
expect(await db.select().from(issues).where(eq(issues.id, issueId))).toMatchObject([{ status: "done", assigneeAgentId: workerId }]);
|
||||
expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, runId))).toEqual(acceptedBefore);
|
||||
expect(provider.execute).toHaveBeenCalledTimes(1);
|
||||
await db.update(agents).set({ adapterConfig: { provider: "codex", model } }).where(eq(agents.id, agentId));
|
||||
}
|
||||
} else if (scenario === "unknown-error") {
|
||||
// A lookalike diagnostic flag does not alter unknown failed-result policy.
|
||||
await db.update(heartbeatRuns).set({ resultJson: { nativeProviderFailure: NATIVE_MODEL_REJECTION_DIAGNOSTIC } }).where(eq(heartbeatRuns.id, runId));
|
||||
await finalizeNativeRun({ db, runId, workspaceFinalizeStatus: "succeeded" });
|
||||
const [decision] = await db.select().from(statusDecisions).where(eq(statusDecisions.runId, runId));
|
||||
expect(decision.reasonCode).toBe("run_failed_partial_evidence_preserved");
|
||||
const wakes = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId));
|
||||
expect(wakes.filter(wake => wake.payload?.issueId === issueId)).toHaveLength(1);
|
||||
}
|
||||
},
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,73 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { PrpEvent, PrpTerminalState } from "../../vendor/paperclip-runner/index.js";
|
||||
import { createNativeProviderFailureObservation, NATIVE_MODEL_REJECTION_DIAGNOSTIC, NATIVE_MODEL_REJECTION_MESSAGE } from "./native-provider-failure.js";
|
||||
|
||||
const model = "test-model";
|
||||
const response = {
|
||||
type: "error", status: 400,
|
||||
error: { type: "invalid_request_error", message: `The '${model}' model is not supported when using Codex with a ChatGPT account.` },
|
||||
};
|
||||
const event: PrpEvent = {
|
||||
schema: "paperclip.prp.event.v1", runId: "run", normalizedSessionId: "session", turnId: "turn",
|
||||
sourceInstanceId: "runner", sourceEventId: "failure", sourceSeq: 1, sourceKind: "runner",
|
||||
eventType: "turn.failed", schemaVersion: 1, priority: 0, emittedAt: "2026-01-01T00:00:00.000Z",
|
||||
payload: { status: "failed", error: { message: JSON.stringify(response), codexErrorInfo: "other" } },
|
||||
};
|
||||
const terminal: PrpTerminalState = {
|
||||
schema: "paperclip.prp.terminal.v1", turnTerminalState: "failed", runTerminalState: "failed", reportedWorkDisposition: "blocked",
|
||||
};
|
||||
const observation = () => createNativeProviderFailureObservation({ kind: "codex", model });
|
||||
|
||||
describe("native provider failure observation", () => {
|
||||
it("binds the structured rejection to the failed turn and emits no response or model text", () => {
|
||||
const observed = observation();
|
||||
observed.observe(event);
|
||||
observed.observe(event); // A byte-identical committed replay remains idempotent.
|
||||
const result = observed.forTerminal("turn", terminal);
|
||||
expect(result).toEqual({ errorCode: "native_provider_model_rejected", errorMessage: NATIVE_MODEL_REJECTION_MESSAGE, diagnostic: NATIVE_MODEL_REJECTION_DIAGNOSTIC });
|
||||
expect(JSON.stringify(result)).not.toContain(model);
|
||||
expect(JSON.stringify(result)).not.toContain("invalid_request_error");
|
||||
expect(observed.forTerminal("later-turn", terminal)).toBeNull();
|
||||
expect(observed.forTerminal(null, terminal)).toBeNull();
|
||||
expect(observed.forTerminal("turn", { ...terminal, runTerminalState: "succeeded" })).toBeNull();
|
||||
expect(observed.forTerminal("turn", { ...terminal, turnTerminalState: "cancelled" })).toBeNull();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ ...event, sourceKind: "control_plane" as const },
|
||||
{ ...event, eventType: "item.completed" as const },
|
||||
{ ...event, eventType: "harness.diagnostic" as const },
|
||||
{ ...event, turnId: undefined },
|
||||
{ ...event, payload: { ...event.payload, status: "completed" } },
|
||||
{ ...event, payload: { ...event.payload, recoverable: true } },
|
||||
{ ...event, payload: { status: "failed", error: { message: JSON.stringify(response), recoverable: true } } },
|
||||
])("ignores non-authoritative or retryable evidence %#", (unrelated) => {
|
||||
const observed = observation();
|
||||
observed.observe(unrelated);
|
||||
expect(observed.forTerminal("turn", terminal)).toBeNull();
|
||||
});
|
||||
|
||||
it.each([
|
||||
response.error.message, "malformed JSON", " ".repeat(4_097),
|
||||
JSON.stringify({ ...response, status: 401 }),
|
||||
JSON.stringify({ ...response, status: "400" }),
|
||||
JSON.stringify({ ...response, type: "notice" }),
|
||||
JSON.stringify({ ...response, error: { ...response.error, type: "server_error" } }),
|
||||
JSON.stringify({ ...response, error: { ...response.error, message: "Private arbitrary provider text" } }),
|
||||
JSON.stringify({ ...response, error: { ...response.error, message: response.error.message + " Private suffix" } }),
|
||||
JSON.stringify({ ...response, error: { ...response.error, message: response.error.message.replace(model, "another-model") } }),
|
||||
JSON.stringify([response]), "null",
|
||||
])("leaves unknown failures unchanged %#", (message) => {
|
||||
const observed = observation();
|
||||
observed.observe({ ...event, payload: { status: "failed", error: { message } } });
|
||||
expect(observed.forTerminal("turn", terminal)).toBeNull();
|
||||
});
|
||||
|
||||
it.each([{ kind: "claude", model }, { kind: "codex" }, { kind: "codex", model: "other" }])(
|
||||
"does not apply a rejection to another selected provider/model (%j)", (provider) => {
|
||||
const observed = createNativeProviderFailureObservation(provider);
|
||||
observed.observe(event);
|
||||
expect(observed.forTerminal("turn", terminal)).toBeNull();
|
||||
},
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,60 @@
|
||||
import type { PrpEvent, PrpTerminalState } from "../../vendor/paperclip-runner/index.js";
|
||||
|
||||
export const NATIVE_MODEL_REJECTION_MESSAGE =
|
||||
"The selected model is not supported by the current ChatGPT connection. Choose a supported model or a compatible AI connection.";
|
||||
|
||||
export const NATIVE_MODEL_REJECTION_DIAGNOSTIC = {
|
||||
provider: "codex",
|
||||
category: "model_auth_incompatible",
|
||||
status: 400,
|
||||
authMode: "chatgpt",
|
||||
} as const;
|
||||
|
||||
function record(value: unknown): Record<string, unknown> {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value)
|
||||
? value as Record<string, unknown>
|
||||
: {};
|
||||
}
|
||||
|
||||
/** Reconstruct only the closed diagnostic vocabulary, never provider text. */
|
||||
export function readNativeModelRejectionDiagnostic(value: unknown) {
|
||||
try {
|
||||
const entry = record(value);
|
||||
return Object.entries(NATIVE_MODEL_REJECTION_DIAGNOSTIC).every(([key, expected]) => entry[key] === expected)
|
||||
? { ...NATIVE_MODEL_REJECTION_DIAGNOSTIC }
|
||||
: null;
|
||||
} catch { return null; }
|
||||
}
|
||||
|
||||
/** Observe only events accepted by the bound control-plane persistence port. */
|
||||
export function createNativeProviderFailureObservation(provider: { kind: string; model?: string | null }) {
|
||||
let rejectedTurnId: string | null = null;
|
||||
return {
|
||||
observe(event: PrpEvent) {
|
||||
if (provider.kind !== "codex" || !provider.model ||
|
||||
event.sourceKind !== "runner" || event.eventType !== "turn.failed" || !event.turnId) return;
|
||||
const payload = record(event.payload);
|
||||
const failure = record(payload.error);
|
||||
if (payload.status !== "failed" || payload.recoverable === true || failure.recoverable === true ||
|
||||
typeof failure.message !== "string" || failure.message.length > 4_096) return;
|
||||
let response: Record<string, unknown>;
|
||||
try { response = record(JSON.parse(failure.message)); } catch { return; }
|
||||
const error = record(response.error);
|
||||
// This is a provider terminal envelope, not text emitted by a tool or an
|
||||
// agent. Bind the exact rejection to the configured model without storing
|
||||
// its name or the response. Unknown provider errors retain their behavior.
|
||||
if (response.type !== "error" || response.status !== 400 || error.type !== "invalid_request_error" ||
|
||||
error.message !== `The '${provider.model}' model is not supported when using Codex with a ChatGPT account.`) return;
|
||||
rejectedTurnId = event.turnId;
|
||||
},
|
||||
forTerminal(turnId: string | null | undefined, terminal: PrpTerminalState) {
|
||||
if (!rejectedTurnId || turnId !== rejectedTurnId ||
|
||||
terminal.turnTerminalState !== "failed" || terminal.runTerminalState !== "failed") return null;
|
||||
return {
|
||||
errorCode: "native_provider_model_rejected" as const,
|
||||
errorMessage: NATIVE_MODEL_REJECTION_MESSAGE,
|
||||
diagnostic: { ...NATIVE_MODEL_REJECTION_DIAGNOSTIC },
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -16,6 +16,7 @@ import { redactSensitiveText } from "../../redaction.js";
|
||||
import { issueRecoveryActionService } from "../issue-recovery-actions.js";
|
||||
import { reportRunFailure } from "../run-failure-report.js";
|
||||
import { isNativeRunnerOwnershipHeld } from "./native-runner-ownership.js";
|
||||
import { nativeRestartInterruptedTurnId, type PersistedNativeSession } from "../../vendor/paperclip-runner/index.js";
|
||||
|
||||
export type NativeControllerIdentity = {
|
||||
bootId: string;
|
||||
@@ -764,7 +765,8 @@ export async function claimNativeRestartRecoveries(input: {
|
||||
hasProviderEvidence,
|
||||
checkpointFailed:
|
||||
(checkpointRecord.terminal as Record<string, unknown> | undefined)
|
||||
?.runTerminalState === "failed",
|
||||
?.runTerminalState === "failed" &&
|
||||
nativeRestartInterruptedTurnId(checkpointRecord as unknown as PersistedNativeSession) === null,
|
||||
providerAttempt: row.coordinator.attempt,
|
||||
});
|
||||
const ownershipChanged =
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { settleSlackConversation } from "../slack-conversation-lifecycle.js";
|
||||
import { readPersistedNativeModelRejection } from "./native-provider-failure-evidence.js";
|
||||
import { dismissAutomaticCompletionReviews } from "./automatic-completion-reviews.js";
|
||||
import { getNativeReviewAssignment, readNativeReviewAssignmentContext } from "./native-review-participant.js";
|
||||
import { conversationNativeDecision, isConversation } from "../agent-conversations.js";
|
||||
@@ -24,7 +25,7 @@ import {
|
||||
workspaceOperations,
|
||||
} from "@paperclipai/db";
|
||||
import { classifyNativeEvidence } from "./evidence-classifier.js";
|
||||
import type { PrpIgnoredAttentionRequest } from "@paperclipai/paperclip-runner";
|
||||
import type { PrpIgnoredAttentionRequest, PrpTerminalState } from "@paperclipai/paperclip-runner";
|
||||
import {
|
||||
arbitrateNativeStatus,
|
||||
NATIVE_STATUS_ARBITER_POLICY_VERSION,
|
||||
@@ -1275,7 +1276,15 @@ export async function finalizeNativeRun(input: {
|
||||
companyId: run.companyId, issueId: authoritativeIssue.id, agentId: run.agentId,
|
||||
contextSnapshot: reviewContext, allowResolvedByRunId: run.id,
|
||||
}) : null;
|
||||
const providerModelRejection = await readPersistedNativeModelRejection(
|
||||
input.db, run, resultRow.turnId, envelope.terminal as PrpTerminalState,
|
||||
);
|
||||
const ownsModelRejectionDecision =
|
||||
(!authoritativeIssue.executionRunId || authoritativeIssue.executionRunId === run.id) &&
|
||||
(reviewContext ? nativeReview?.interaction.status === "pending"
|
||||
: authoritativeIssue.assigneeAgentId === run.agentId && !authoritativeIssue.assigneeUserId);
|
||||
const proposedDecision = resolveNativeFinalizerStatus({
|
||||
providerModelRejected: providerModelRejection !== null && ownsModelRejectionDecision,
|
||||
...(reviewContext ? { nativeReviewOutcome: nativeReview
|
||||
? nativeReview.interaction.status === "pending" ? "pending" as const : "resolved" as const
|
||||
: "stale" as const } : {}),
|
||||
@@ -1358,6 +1367,9 @@ export async function finalizeNativeRun(input: {
|
||||
priorStatusVersion: Number(authoritativeIssue.statusVersion),
|
||||
priorDecisionId: authoritativeIssue.lastStatusDecisionId,
|
||||
decision,
|
||||
requireModelRejectionOwner: decision.reasonCode === "native_provider_model_rejected"
|
||||
? { agentId: run.agentId, reviewContext }
|
||||
: undefined,
|
||||
requireBoardResponseWaitSource:
|
||||
decision.reasonCode === "board_response_waiting" || repairBoardResponseWait
|
||||
? boardResponseWait?.source
|
||||
@@ -1400,6 +1412,7 @@ export async function finalizeNativeRun(input: {
|
||||
);
|
||||
const clearExecutionFailure = input.projectRunStatus && finalizationPhase === "committed" && terminalState === "succeeded";
|
||||
const finalizationMetadata = {
|
||||
...(providerModelRejection ? { nativeProviderFailure: providerModelRejection.diagnostic } : {}),
|
||||
finalizationPhase,
|
||||
...(finalizationPhase === "committed" ? { failureCode: null, originalFailureCode: null, nextAttemptAt: null } : {}),
|
||||
assessmentId: assessmentRow.id,
|
||||
@@ -1450,6 +1463,12 @@ export async function finalizeNativeRun(input: {
|
||||
nativePhase: finalizationPhase,
|
||||
nativePhaseUpdatedAt: now,
|
||||
...(clearExecutionFailure ? { error: null, errorCode: null } : {}),
|
||||
...(providerModelRejection ? {
|
||||
// Recovery can finalize before heartbeat saves the adapter result.
|
||||
// Retain any independently recorded execution/cleanup failure.
|
||||
error: sql`coalesce(${heartbeatRuns.error}, ${providerModelRejection.errorMessage})`,
|
||||
errorCode: sql`coalesce(${heartbeatRuns.errorCode}, ${providerModelRejection.errorCode})`,
|
||||
} : {}),
|
||||
resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb)
|
||||
|| ${JSON.stringify(finalizationMetadata)}::jsonb
|
||||
|| ${clearExecutionFailure ? recoveredExecutionFailureMetadata() : sql`'{}'::jsonb`}`,
|
||||
|
||||
@@ -1018,6 +1018,30 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
]);
|
||||
});
|
||||
|
||||
it.each([2, 3])("recovers a checkpointed restart interruption within provider budget %s", async (attempt) => {
|
||||
const fixture = await seedRun(`INTERRUPTED-${attempt}`);
|
||||
await fixture.db.update(heartbeatRuns).set({ runnerProfileJson: { sessionCheckpoint: {
|
||||
driverKind: "codex_app_server",
|
||||
sessionId: fixture.native.normalizedSessionId,
|
||||
identity: { companyId, issueId: fixture.issueId, runId: fixture.runId, agentId, sessionId: fixture.native.normalizedSessionId },
|
||||
providerSessionId: "interrupted-provider",
|
||||
activeTurnId: null,
|
||||
semanticResult: null,
|
||||
terminal: { runTerminalState: "failed", turnTerminalState: "failed" },
|
||||
terminalTurns: [{ turnId: "interrupted-turn", fingerprint: JSON.stringify({
|
||||
terminalState: "failed", result: null,
|
||||
error: { code: "provider_turn_lost_on_restore", recoverable: true },
|
||||
}) }],
|
||||
} } }).where(eq(heartbeatRuns.id, fixture.runId));
|
||||
await fixture.db.update(nativeRunFinalizations).set({ attempt }).where(eq(nativeRunFinalizations.runId, fixture.runId));
|
||||
const dispositions = await claimNativeRestartRecoveries({
|
||||
db: fixture.db, controller: successor, restartKind: "hot", runIds: [fixture.runId],
|
||||
});
|
||||
expect(dispositions).toEqual([expect.objectContaining(attempt < 3
|
||||
? { kind: "resume_dead_runner", runId: fixture.runId, providerAttempt: attempt }
|
||||
: { kind: "blocked", runId: fixture.runId, reason: "execution_recovery_budget_exhausted" })]);
|
||||
});
|
||||
|
||||
it("terminalizes the recorded failed-checkpoint incident atomically instead of resuming it on upgrade", async () => {
|
||||
const fixture = await seedRun("FAILED-CHECKPOINT");
|
||||
await fixture.db.update(heartbeatRuns).set({ runnerProfileJson: { sessionCheckpoint: {
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
REMOTE_CODEX_SUPPORTED_RANGE,
|
||||
} from "./codex-runtime-compatibility.js";
|
||||
import { createNativeToolTrace, type NativeToolTrace } from "./native-tool-trace.js";
|
||||
import { createNativeProviderFailureObservation } from "./native-provider-failure.js";
|
||||
import { createNativeGitHubAccess, type NativeGitHubAccess } from "./native-github-access.js";
|
||||
import { resolveGitHubOperationCredentials } from "../github-operation-credentials.js";
|
||||
import { bindManagedNativeCredentialTurn, completeManagedNativeCredentialTurn } from "./managed-native-credentials.js";
|
||||
@@ -7805,6 +7806,7 @@ async function executePaperclipNativeSessionWithinScope(
|
||||
let turnStartedAtMs: number | null = null;
|
||||
let firstAgentEventRecorded = false;
|
||||
let providerUsageLimitObserved = false;
|
||||
const providerFailureObservation = createNativeProviderFailureObservation(input.execution.provider);
|
||||
let turnCompletedAtMs: number | null = null;
|
||||
let runnerSessionStartupScope: NativeRunSpanScope | null = null;
|
||||
let agentTurnScope: NativeRunSpanScope | null = null;
|
||||
@@ -7861,6 +7863,7 @@ async function executePaperclipNativeSessionWithinScope(
|
||||
await liveQuestions.observe(event);
|
||||
await projectSessionGoalEvent(event);
|
||||
providerUsageLimitObserved ||= nativeProviderUsageLimitFromEvent(event);
|
||||
providerFailureObservation.observe(event);
|
||||
const eventAtMs = Date.parse(event.emittedAt);
|
||||
const milestoneAtMs = Number.isFinite(eventAtMs)
|
||||
? eventAtMs
|
||||
@@ -8064,6 +8067,7 @@ async function executePaperclipNativeSessionWithinScope(
|
||||
await liveQuestions.observe(event);
|
||||
await projectSessionGoalEvent(event);
|
||||
providerUsageLimitObserved ||= nativeProviderUsageLimitFromEvent(event);
|
||||
providerFailureObservation.observe(event);
|
||||
const questionFallback = await materializeRuntimeQuestionFallback({
|
||||
db: input.db,
|
||||
binding: input.execution.binding,
|
||||
@@ -8334,6 +8338,7 @@ async function executePaperclipNativeSessionWithinScope(
|
||||
trace.activate(runnerSessionStartupScope);
|
||||
const result = await trace.run(runnerSessionStartupScope, () =>
|
||||
executeNativeSession({
|
||||
resumeInterruptedTurn: input.restartRecovery?.kind === "resume_dead_runner",
|
||||
getFreshSessionHandoff: input.getFreshSessionHandoff,
|
||||
onSessionAdmission: async () => {
|
||||
// Invalidate prior stop evidence before a backend can spawn.
|
||||
@@ -9203,6 +9208,7 @@ async function executePaperclipNativeSessionWithinScope(
|
||||
);
|
||||
if (collectInstructions && !collectedByOwner) await instructionCopy!.collectStopped();
|
||||
}
|
||||
const providerFailure = providerFailureObservation.forTerminal(native.turnId, native.terminal);
|
||||
const adapterResult: AdapterExecutionResult = {
|
||||
exitCode: native.terminal.runTerminalState === "succeeded" ? 0 : 1,
|
||||
signal: null,
|
||||
@@ -9210,10 +9216,12 @@ async function executePaperclipNativeSessionWithinScope(
|
||||
errorMessage:
|
||||
native.terminal.runTerminalState === "succeeded"
|
||||
? null
|
||||
: `Native session ${native.terminal.runTerminalState}`,
|
||||
: providerFailure?.errorMessage ?? `Native session ${native.terminal.runTerminalState}`,
|
||||
...(providerFailure ? { errorCode: providerFailure.errorCode } : {}),
|
||||
resultJson: {
|
||||
nativeResult: native.result as unknown as Record<string, unknown>,
|
||||
nativeTerminal: native.terminal as unknown as Record<string, unknown>,
|
||||
...(providerFailure ? { nativeProviderFailure: providerFailure.diagnostic } : {}),
|
||||
...(native.goalRolloverRequired ? { goalRolloverRequired: true } : {}),
|
||||
planSynchronizations,
|
||||
},
|
||||
|
||||
@@ -44,6 +44,23 @@ function arbitrate(
|
||||
}
|
||||
|
||||
describe("native status authority", () => {
|
||||
it.each(["in_progress", "in_review"] as const)("blocks a current worker's proven model rejection without a retry (%s)", (priorIssueStatus) => {
|
||||
const decision = arbitrate({ priorIssueStatus, terminalState: "failed", providerModelRejected: true });
|
||||
expect(decision).toMatchObject({ statusAction: "blocked", toStatus: "blocked", reasonCode: "native_provider_model_rejected", unblockDescriptor: { owner: "board" } });
|
||||
expect(decision.effects).toEqual([{ kind: "bind_blocker", owner: "board", action: expect.any(String) }]);
|
||||
expect(arbitrate({ terminalState: "failed", providerModelRejected: false }).effects).toContainEqual(expect.objectContaining({ kind: "schedule_retry" }));
|
||||
expect(arbitrate({ terminalState: "succeeded", providerModelRejected: true }).reasonCode).not.toBe("native_provider_model_rejected");
|
||||
expect(arbitrate({ terminalState: "failed", providerModelRejected: true, workspaceFinalizeStatus: "failed" }).reasonCode).toBe("finalization_failed_claim_preserved");
|
||||
expect(arbitrate({ terminalState: "failed", providerModelRejected: true, priorIssueStatus: "done" }).toStatus).toBe("done");
|
||||
for (const nativeReviewOutcome of ["stale", "resolved"] as const) {
|
||||
expect(arbitrate({ terminalState: "failed", providerModelRejected: true, priorIssueStatus, nativeReviewOutcome }))
|
||||
.toMatchObject({ statusAction: "preserve", toStatus: priorIssueStatus, reasonCode: "native_review_action_finished" });
|
||||
}
|
||||
});
|
||||
it("preserves pending review authority without automatic recovery after model rejection", () => {
|
||||
expect(arbitrate({ priorIssueStatus: "in_review", terminalState: "failed", providerModelRejected: true, nativeReviewOutcome: "pending" }))
|
||||
.toMatchObject({ statusAction: "preserve", toStatus: "in_review", reasonCode: "native_provider_model_rejected", unblockDescriptor: null, effects: [{ kind: "release_checkout" }] });
|
||||
});
|
||||
it("a reviewer finishes its decision without completing rejected or still-reviewed work", () => {
|
||||
for (const priorIssueStatus of ["in_progress", "in_review"] as const) {
|
||||
const decision = arbitrate({ priorIssueStatus, nativeReviewOutcome: "resolved" });
|
||||
@@ -511,7 +528,7 @@ describe("native status authority", () => {
|
||||
expect.objectContaining({
|
||||
statusAction: "blocked",
|
||||
toStatus: "blocked",
|
||||
policyVersion: "phase6-v7",
|
||||
policyVersion: "phase6-v8",
|
||||
reasonCode: "current_track_blocker_waiting",
|
||||
unblockDescriptor: {
|
||||
owner: "board",
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { NativeEvidenceAssessment } from "./evidence-classifier.js";
|
||||
import { NATIVE_MODEL_REJECTION_MESSAGE } from "./native-provider-failure.js";
|
||||
|
||||
export const NATIVE_STATUS_ARBITER_POLICY_VERSION = "phase6-v7";
|
||||
export const NATIVE_STATUS_ARBITER_POLICY_VERSION = "phase6-v8";
|
||||
|
||||
export type NativeAuthoritativeIssueStatus =
|
||||
| "backlog"
|
||||
@@ -115,6 +116,8 @@ export function arbitrateNativeStatus(input: {
|
||||
reviewOwnerUserId?: string | null;
|
||||
/** Review decisions own task state; a reviewer's finish report cannot override them. */
|
||||
nativeReviewOutcome?: "resolved" | "pending" | "stale";
|
||||
/** Re-derived from the committed runner terminal and pinned execution identity. */
|
||||
providerModelRejected?: boolean;
|
||||
agentId: string;
|
||||
priorIssueStatus: NativeAuthoritativeIssueStatus;
|
||||
}): NativeStatusDecision {
|
||||
@@ -146,6 +149,31 @@ export function arbitrateNativeStatus(input: {
|
||||
],
|
||||
};
|
||||
}
|
||||
if (input.providerModelRejected && input.terminalState === "failed" &&
|
||||
input.nativeReviewOutcome !== "stale" && input.nativeReviewOutcome !== "resolved") {
|
||||
if (input.nativeReviewOutcome === "pending") {
|
||||
// The pending review is bound to the worker's status decision and
|
||||
// version. Preserve that authority so an explicit retry after a
|
||||
// configuration repair can resolve the same review. In particular,
|
||||
// do not create the normal unresolved-review recovery action.
|
||||
return {
|
||||
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
|
||||
statusAction: "preserve",
|
||||
toStatus: input.priorIssueStatus,
|
||||
reasonCode: "native_provider_model_rejected",
|
||||
unblockDescriptor: null,
|
||||
effects: [{ kind: "release_checkout" }],
|
||||
};
|
||||
}
|
||||
return {
|
||||
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
|
||||
statusAction: "blocked",
|
||||
toStatus: "blocked",
|
||||
reasonCode: "native_provider_model_rejected",
|
||||
unblockDescriptor: { owner: "board", action: NATIVE_MODEL_REJECTION_MESSAGE },
|
||||
effects: [{ kind: "bind_blocker", owner: "board", action: NATIVE_MODEL_REJECTION_MESSAGE }],
|
||||
};
|
||||
}
|
||||
if (input.nativeReviewOutcome) {
|
||||
return {
|
||||
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
|
||||
|
||||
@@ -24,6 +24,7 @@ import {
|
||||
type NativeStatusEffect,
|
||||
} from "./status-arbiter.js";
|
||||
import { nativeSha256 } from "./canonical.js";
|
||||
import { getNativeReviewAssignment, type NativeReviewAssignmentContext } from "./native-review-participant.js";
|
||||
import {
|
||||
readNativeBoardResponseWaitSource,
|
||||
readNativeBoardResponseWaitOrigin,
|
||||
@@ -1535,6 +1536,7 @@ export async function commitNativeStatusDecision(input: {
|
||||
preMaterializedEffects?: NativeMaterializedStatusEffect[];
|
||||
supersedesCommittedDecisionId?: string;
|
||||
requireExternalChatResponseWaitAuthorization?: { agentId: string };
|
||||
requireModelRejectionOwner?: { agentId: string; reviewContext: NativeReviewAssignmentContext | null };
|
||||
requireBoardResponseWaitSource?: NativeBoardResponseWaitSource;
|
||||
requireBoardResponseWaitOrigin?: NativeBoardResponseWaitOrigin;
|
||||
reviewResponsePresentation?: {
|
||||
@@ -1547,6 +1549,9 @@ export async function commitNativeStatusDecision(input: {
|
||||
throw new Error("native_status_reason_code_required");
|
||||
}
|
||||
const reasonCode = input.decision.reasonCode;
|
||||
if (reasonCode === "native_provider_model_rejected" && !input.requireModelRejectionOwner) {
|
||||
throw new Error("native_model_rejection_owner_required");
|
||||
}
|
||||
const publications: ActivityPublication[] = [];
|
||||
const terminalRunsToEmit: (typeof heartbeatRuns.$inferSelect)[] = [];
|
||||
const committed = await input.db.transaction(async (tx) => {
|
||||
@@ -1616,6 +1621,20 @@ export async function commitNativeStatusDecision(input: {
|
||||
) {
|
||||
throw new NativeStatusRaceError();
|
||||
}
|
||||
if (input.requireModelRejectionOwner) {
|
||||
const owner = input.requireModelRejectionOwner;
|
||||
// A successor can claim execution without changing statusVersion. Keep
|
||||
// this new blocking authority behind the same locked issue snapshot.
|
||||
if (issue.executionRunId && issue.executionRunId !== input.runId) throw new NativeStatusRaceError();
|
||||
if (owner.reviewContext) {
|
||||
const review = await getNativeReviewAssignment(tx as unknown as Db, {
|
||||
companyId: input.companyId, issueId: input.issueId, agentId: owner.agentId, contextSnapshot: owner.reviewContext,
|
||||
});
|
||||
if (review?.interaction.status !== "pending") throw new NativeStatusRaceError();
|
||||
} else if (issue.assigneeAgentId !== owner.agentId || issue.assigneeUserId) {
|
||||
throw new NativeStatusRaceError();
|
||||
}
|
||||
}
|
||||
if (input.requireExternalChatResponseWaitAuthorization) {
|
||||
let authorization;
|
||||
try {
|
||||
|
||||
@@ -7,8 +7,20 @@ import {
|
||||
classifyContinuationFailure,
|
||||
} from "./service.js";
|
||||
import { legacyExecutionNeedsReconciliation } from "../legacy-execution-recovery.js";
|
||||
import { isConfigurationIncompleteFailedRun } from "../../modules/wake-queue/domain/values.js";
|
||||
|
||||
describe("classifyAdapterFailureForRecovery", () => {
|
||||
it("waits for model configuration changes instead of retrying a typed native model rejection", () => {
|
||||
const run = { errorCode: "native_provider_model_rejected", error: "Fixed model configuration message", resultJson: null };
|
||||
expect(classifyAdapterFailureForRecovery(run)).toEqual({ kind: "configuration_incomplete" });
|
||||
expect(classifyContinuationFailure(run as never)).toMatchObject({ kind: "non_retryable", maxAttempts: 0 });
|
||||
expect(isConfigurationIncompleteFailedRun(run)).toBe(true);
|
||||
expect(isAiAuthenticationBlocked(run as never)).toBe(false);
|
||||
const unknown = { ...run, errorCode: "adapter_failed", error: "native_provider_model_rejected" };
|
||||
expect(classifyAdapterFailureForRecovery(unknown)).toBeNull();
|
||||
expect(classifyContinuationFailure(unknown as never).kind).toBe("transient_infra");
|
||||
expect(isConfigurationIncompleteFailedRun(unknown)).toBe(false);
|
||||
});
|
||||
it.each(["acpx_auth_required", "claude_auth_required", "codex_auth_required", "adapter_auth_missing", "refresh_token_reused", "authentication_required"])("does not automatically retry provider authentication failure %s", (errorCode) => {
|
||||
const classification = classifyRunLiveness({ runStatus: "failed", issue: null, errorCode, authenticationRepairRequested: true });
|
||||
const run = { errorCode, ...classification };
|
||||
|
||||
@@ -491,6 +491,7 @@ const TRANSIENT_INFRA_CONTINUATION_ERROR_CODES = new Set<string>([
|
||||
]);
|
||||
|
||||
const NON_RETRYABLE_CONTINUATION_ERROR_CODES = new Set<string>([
|
||||
"native_provider_model_rejected",
|
||||
"provider_tool_definition_invalid",
|
||||
"adapter_engine_unavailable",
|
||||
"agent_not_invokable",
|
||||
@@ -626,7 +627,7 @@ export function classifyAdapterFailureForRecovery(
|
||||
): AdapterFailureRecoveryClassification {
|
||||
// An engine prerequisite cannot be repaired by asking the same unavailable
|
||||
// engine to retry. Use the existing configuration-blocker path.
|
||||
if (latestRun.errorCode === "adapter_engine_unavailable" || latestRun.errorCode === "provider_tool_definition_invalid") {
|
||||
if (latestRun.errorCode === "adapter_engine_unavailable" || latestRun.errorCode === "provider_tool_definition_invalid" || latestRun.errorCode === "native_provider_model_rejected") {
|
||||
return { kind: "configuration_incomplete" };
|
||||
}
|
||||
if (
|
||||
|
||||
@@ -665,6 +665,14 @@ function mapRoutineDescriptionDocument(row: {
|
||||
};
|
||||
}
|
||||
|
||||
// Match PostgreSQL UUID input, including compact/braced forms and any version.
|
||||
// Do not trim or resolve prefixes: the exact input must be safe for a UUID query.
|
||||
function isRoutineUuidInput(id: string): boolean {
|
||||
if (/\s/.test(id)) return false;
|
||||
const value = id.startsWith("{") && id.endsWith("}") ? id.slice(1, -1) : id;
|
||||
return /^(?:[0-9a-f]{4}-?){7}[0-9a-f]{4}$/i.test(value);
|
||||
}
|
||||
|
||||
export function routineService(
|
||||
db: Db,
|
||||
deps: {
|
||||
@@ -682,6 +690,7 @@ export function routineService(
|
||||
});
|
||||
|
||||
async function getRoutineById(id: string) {
|
||||
if (!isRoutineUuidInput(id)) return null;
|
||||
return db
|
||||
.select()
|
||||
.from(routines)
|
||||
@@ -766,6 +775,7 @@ export function routineService(
|
||||
}
|
||||
|
||||
async function getTriggerById(id: string) {
|
||||
if (!isRoutineUuidInput(id)) return null;
|
||||
return db
|
||||
.select()
|
||||
.from(routineTriggers)
|
||||
|
||||
@@ -5,6 +5,7 @@ import { redactDiagnosticText } from "@paperclipai/adapter-utils/command-redacti
|
||||
import { sanitizeWorkspaceRestoreDiagnostic } from "@paperclipai/adapter-utils/workspace-restore-diagnostics";
|
||||
import { redactCurrentUserText } from "../log-redaction.js";
|
||||
import { redactSensitiveText, REDACTED_EVENT_VALUE } from "../redaction.js";
|
||||
import { readNativeModelRejectionDiagnostic } from "./native-runtime/native-provider-failure.js";
|
||||
|
||||
type Run = typeof heartbeatRuns.$inferSelect;
|
||||
type Context = Record<string, string | number | boolean>;
|
||||
@@ -184,6 +185,8 @@ export function collectRunFailureDiagnostics(run: Run, options: RunFailureReport
|
||||
"stackPreview", "status", "statusCode", "requestId",
|
||||
]);
|
||||
const provider = scalars(read(result, "terminalSessionFailure"), ["category", "title", "details"]);
|
||||
const nativeProviderFailure = readNativeModelRejectionDiagnostic(read(result, "nativeProviderFailure"));
|
||||
if (nativeProviderFailure) Object.assign(provider, nativeProviderFailure);
|
||||
const truncatedFields: string[] = [];
|
||||
const providerTruncation = read(read(result, "terminalSessionFailure"), "truncatedFields");
|
||||
if (Array.isArray(providerTruncation)) {
|
||||
|
||||
Reference in new issue
Block a user