Merge master into SSH agent account support

Keep the SSH base branch current so the dependent Tailscale migration can
be regenerated after the newly landed upstream migration. Preserve the
managed SSH account selection and remote session cleanup changes.

Verified the focused SSH, routing, and account probe tests, server
typecheck, and adapter-utils, Codex, and Claude package typechecks.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Devin FoleyandPaperclip committed 2026-11-12 19:04:45 -08:00
commit d59f9bbcee
191 files changed
+15683 -4374

No files matched your search

@@ -70,6 +70,12 @@ export async function resolveAssignedConnectionInstructionsForRun(db: Db, bindin
return gateway ? gateway.resolveConnectionInstructionsForRun(binding) : null;
}
/** MCP Skills: approved skills of the agent's usable connections; none without a configured gateway. */
export async function resolveMcpConnectionSkillsForRun(db: Db, binding: { companyId: string; agentId: string; runId: string }) {
const gateway = assignedMcpGateways.get(db);
return gateway ? gateway.resolveConnectionSkillsForRun(binding) : null;
}
function assignedToolName(name: string): string {
const sanitize = (value: string) => value.replace(/[^a-zA-Z0-9_]+/g, "_").replace(/^_+|_+$/g, "");
const separator = name.lastIndexOf(":");
@@ -1,5 +1,6 @@
import { and, eq, inArray, isNull, ne, or, sql } from "drizzle-orm";
import { agentWakeupRequests, heartbeatRuns, type Db } from "@paperclipai/db";
import { agentWakeupRequests, heartbeatRunEvents, heartbeatRuns, statusDecisions, type Db } from "@paperclipai/db";
import { appendHeartbeatRunEvent } from "../heartbeat-run-events.js";
export interface NativeChildCompletionRecipient {
companyId: string;
@@ -9,8 +10,25 @@ export interface NativeChildCompletionRecipient {
sourceIntentId: string | null;
}
export const CHILD_TASK_READ_EVENT = "child_task.read";
export async function recordChildTaskRead(db: Db, input: {
companyId: string; agentId: string; runId: string; parentIssueId: string;
children: ReadonlyArray<{ id: string; parentId: string | null; status: string; updatedAt: Date; lastStatusDecisionId?: string | null }>;
}): Promise<void> {
const completed = input.children.filter(child => child.parentId === input.parentIssueId && child.status === "done");
if (completed.length === 0) return;
await appendHeartbeatRunEvent(db, { companyId: input.companyId, runId: input.runId, agentId: input.agentId,
eventType: CHILD_TASK_READ_EVENT, stream: "system", level: "info",
payload: { parentIssueId: input.parentIssueId,
children: Object.fromEntries(completed.map(child => [child.id, { status: child.status, updatedAt: child.updatedAt.toISOString(),
statusDecisionId: child.lastStatusDecisionId ?? null }])) } });
}
/** New child results belong to a fresh parent turn, including across a Done claim. */
export async function hasPendingNativeChildCompletion(db: Db, input: NativeChildCompletionRecipient) {
const completedChildId = sql`coalesce(${agentWakeupRequests.payload}->>'completedChildIssueId',
${agentWakeupRequests.payload}->'_paperclipWakeContext'->>'completedChildIssueId')`;
const [pending] = await db.select({ id: agentWakeupRequests.id }).from(agentWakeupRequests).where(and(
eq(agentWakeupRequests.companyId, input.companyId),
eq(agentWakeupRequests.agentId, input.agentId),
@@ -26,6 +44,18 @@ export async function hasPendingNativeChildCompletion(db: Db, input: NativeChild
or(isNull(agentWakeupRequests.runId), ne(agentWakeupRequests.runId, input.runId)),
sql`${agentWakeupRequests.createdAt} >= coalesce((select ${heartbeatRuns.startedAt} from ${heartbeatRuns}
where ${heartbeatRuns.id} = ${input.runId} and ${heartbeatRuns.companyId} = ${input.companyId}), '-infinity'::timestamptz)`,
sql`not exists (select 1 from ${heartbeatRunEvents} child_read
where child_read.company_id = ${input.companyId} and child_read.run_id = ${input.runId}
and child_read.event_type = ${CHILD_TASK_READ_EVENT}
and child_read.payload->>'parentIssueId' = ${input.issueId}
and child_read.payload->'children'->(${completedChildId})->>'status' = 'done'
and (child_read.payload->'children'->(${completedChildId})->>'statusDecisionId'
= ${agentWakeupRequests.payload}->'_paperclipWakeContext'->>'nativeChildCompletionDecisionId'
or (child_read.payload->'children'->(${completedChildId})->>'updatedAt')::timestamptz
>= date_trunc('milliseconds', coalesce((select ${statusDecisions.createdAt} from ${statusDecisions}
where ${statusDecisions.companyId} = ${input.companyId}
and ${statusDecisions.id}::text = ${agentWakeupRequests.payload}->'_paperclipWakeContext'->>'nativeChildCompletionDecisionId'),
${agentWakeupRequests.createdAt}))))`,
input.sourceIntentId ? ne(agentWakeupRequests.id, input.sourceIntentId) : undefined,
)).limit(1);
return Boolean(pending);
@@ -0,0 +1,40 @@
import { describe, expect, it } from "vitest";
import { redactNativeFailureError } from "./native-failure-redaction.js";
const redact = (text: string) => text.replaceAll("agent-secret", "[redacted]");
describe("redactNativeFailureError", () => {
it("keeps a timed-out transfer's own cause when its message cannot be reassigned", async () => {
const signal = AbortSignal.timeout(1);
await new Promise(resolve => setTimeout(resolve, 20));
const timeout = signal.reason as Error;
const redacted = redactNativeFailureError(timeout, redact);
expect(redacted).toBeInstanceOf(Error);
expect(redacted.name).toBe("TimeoutError");
expect(redacted.message).toBe("The operation was aborted due to timeout");
});
it("redacts a read-only message and stack without throwing", () => {
const failure = new DOMException("upload for agent-secret stalled", "AbortError");
const redacted = redactNativeFailureError(failure, redact);
expect(redacted.name).toBe("AbortError");
expect(redacted.message).toBe("upload for [redacted] stalled");
expect(redacted.stack ?? "").not.toContain("agent-secret");
});
it("redacts an ordinary error in place and keeps its type", () => {
class TypedFailure extends Error {}
const failure = new TypedFailure("provider rejected agent-secret");
const redacted = redactNativeFailureError(failure, redact);
expect(redacted).toBe(failure);
expect(redacted).toBeInstanceOf(TypedFailure);
expect(redacted.message).toBe("provider rejected [redacted]");
expect(redacted.stack).not.toContain("agent-secret");
});
});
@@ -0,0 +1,17 @@
export function redactNativeFailureError(error: Error, redact: (text: string) => string): Error {
const originalMessage = error.message;
const message = redact(originalMessage);
const stack = error.stack
? redact(originalMessage ? error.stack.split(originalMessage).join(message) : error.stack)
: undefined;
try {
error.message = message;
if (stack !== undefined) error.stack = stack;
return error;
} catch {
const replacement = new Error(message);
replacement.name = error.name;
replacement.stack = stack ?? `${error.name}: ${message}`;
return replacement;
}
}
@@ -0,0 +1,84 @@
import { randomUUID } from "node:crypto";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { agents, companies, createDb, heartbeatRuns } from "@paperclipai/db";
import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
import { agentProviderLimitCooldown } from "./native-startup-retry.js";
const HOUR_MS = 60 * 60_000;
const DAY_MS = 24 * HOUR_MS;
/** Claude ACP wording for an exhausted weekly allowance, such as "resets Nov 10, 3pm (UTC)". */
function weeklyLimitMessage(at: Date) {
const month = at.toLocaleString("en-US", { month: "short", timeZone: "UTC" });
const hour = at.getUTCHours();
return `ACP agent reported a terminal limit failure.\nYou've hit your weekly limit · resets ${month} ${at.getUTCDate()}, ${
hour % 12 || 12}${hour < 12 ? "am" : "pm"} (UTC)`;
}
describe("native provider limit cooldown", () => {
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let db: ReturnType<typeof createDb>;
const companyId = randomUUID();
beforeAll(async () => {
temporary = await startEmbeddedPostgresTestDatabase("native-provider-limit-cooldown-");
db = createDb(temporary.connectionString);
await db.insert(companies).values({ id: companyId, name: "Native provider limit", issuePrefix: "NPL" });
});
afterAll(async () => {
await temporary?.cleanup();
});
/** An agent whose latest native run failed on an exhausted weekly allowance. */
async function limitedAgent(resetAt: Date) {
const agentId = randomUUID();
await db.insert(agents).values({
id: agentId,
companyId,
name: `Limited agent ${agentId.slice(0, 8)}`,
status: "active",
adapterType: "paperclip_runner",
adapterConfig: { provider: "acpx", agent: "claude" },
});
const limitedAt = new Date(Date.now() - 5 * 60_000);
await db.insert(heartbeatRuns).values({
id: randomUUID(),
companyId,
agentId,
runtimeMode: "native",
invocationSource: "assignment",
triggerDetail: "system",
status: "failed",
startedAt: new Date(limitedAt.getTime() - 60_000),
finishedAt: limitedAt,
errorCode: "native_provider_usage_limit",
resultJson: {
providerFailure: {
schema: "paperclip.provider-failure.v1",
provider: "anthropic",
phase: "turn",
category: "quota_exhausted",
message: weeklyLimitMessage(resetAt),
retryability: "requires_action",
},
},
});
return agentId;
}
it("holds an agent until a dated weekly-limit reset instead of the one-hour default", async () => {
const resetAt = new Date(Math.ceil((Date.now() + 20 * HOUR_MS) / HOUR_MS) * HOUR_MS);
const agentId = await limitedAgent(resetAt);
expect(await agentProviderLimitCooldown(db, { companyId, agentId, now: new Date() }))
.toMatchObject({ resetAt, resetReported: true, category: "quota_exhausted" });
});
it("re-checks a dated reset days away within one day", async () => {
const agentId = await limitedAgent(new Date(Math.ceil((Date.now() + 3 * DAY_MS) / HOUR_MS) * HOUR_MS));
const now = new Date();
const cooldown = await agentProviderLimitCooldown(db, { companyId, agentId, now });
expect(cooldown).toMatchObject({ resetReported: true });
expect(cooldown!.resetAt.getTime()).toBe(now.getTime() + DAY_MS);
});
});
@@ -1433,11 +1433,29 @@ export async function claimNativeRestartRecoveries(input: {
// old run's writes revoked and a new workspace root.
// The same holds when the previous controller's host was never recorded
// or cannot be proven from here: confirming its stop is optional.
// The run's own server boot is proven stopped or fenced (controller
// instance record): its lease is no sign of life, however long it was
// granted. The claim lease is 20 minutes for a local controller.
const ownerControllerStopped = evaluatedTakeover.allowed
&& ["controller_stopped", "controller_fenced"].includes(evaluatedTakeover.reason);
const leaseLapsedAt = row.coordinator.leaseExpiresAt === null ? null
: row.coordinator.leaseExpiresAt <= now ? row.coordinator.leaseExpiresAt.getTime() : null;
const unverifiableLapsedController = !takeover.allowed && !nonExecutionOwner
&& (row.coordinator.leaseExpiresAt === null || row.coordinator.leaseExpiresAt <= now)
&& (row.coordinator.leaseExpiresAt === null || leaseLapsedAt !== null || ownerControllerStopped)
&& (namespaceAttentionNeeded || ["controller_host_unrecorded", "controller_host_unprovable"].includes(takeover.reason));
let unverifiedWaitEndsAt: number | null = null;
if (unverifiableLapsedController) {
const waitingSince = namespaceWaitStartedAt(row.coordinator.recoveryHistory, takeover.reason);
// The bounded wait starts when the run's controller went away (its lease
// lapsed), not when a later server first looked: a successor that boots
// minutes later, or re-records the wait, must not restart the clock.
const observedSince = namespaceWaitStartedAt(row.coordinator.recoveryHistory, takeover.reason);
const starts = [observedSince, leaseLapsedAt].filter((at): at is number => at !== null);
// A controller boot proven stopped or fenced already outlived its lease
// (90 s) and the fence grace (30 s): waiting longer proves nothing more
// about a Runner in another process space, so the run ends now.
const waitingSince = ownerControllerStopped ? now.getTime() - NATIVE_NAMESPACE_UNVERIFIED_WAIT_MS
: starts.length ? Math.min(...starts) : null;
unverifiedWaitEndsAt = (waitingSince ?? now.getTime()) + NATIVE_NAMESPACE_UNVERIFIED_WAIT_MS;
if (waitingSince !== null && now.getTime() - waitingSince >= NATIVE_NAMESPACE_UNVERIFIED_WAIT_MS) {
await endNamespaceUnverifiedRun(tx as unknown as Db, row, now, takeover.reason, historyEntry({
now, restartKind: input.restartKind, controller, generation: row.coordinator.controllerGeneration,
@@ -1480,8 +1498,12 @@ export async function claimNativeRestartRecoveries(input: {
// Repeated scans of one ownership wait record it once, so the
// bounded history keeps the drain record the next claim reads.
...(waitAlreadyRecorded ? {} : { recoveryHistory: appendBoundedRecoveryHistory(event) }),
updatedAt: waitAlreadyRecorded && row.coordinator.recoveryState === "awaiting_evidence"
&& row.coordinator.recoveryRequestId === (input.recoveryRequestId ?? null) ? row.coordinator.updatedAt : now,
// updated_at is the owner's last sign of life: the ownership prover
// fences a dead boot only once it has been quiet past that write plus
// the lease and grace, and the fallback reads it as the owner's
// progress. A successor recording its own wait never moves it.
updatedAt: !ownControllerProcess || (waitAlreadyRecorded && row.coordinator.recoveryState === "awaiting_evidence"
&& row.coordinator.recoveryRequestId === (input.recoveryRequestId ?? null)) ? row.coordinator.updatedAt : now,
})
.where(
and(
@@ -1502,7 +1524,8 @@ export async function claimNativeRestartRecoveries(input: {
: ["controller_lease_held", "controller_lease_grace"].includes(takeover.reason)
? { retryAt: new Date(now.getTime() + CONTROLLER_LEASE_RENEW_MS) }
// The bounded wait for an unverifiable controller is checked again when it ends.
: unverifiableLapsedController ? { retryAt: new Date(now.getTime() + NATIVE_NAMESPACE_UNVERIFIED_WAIT_MS) } : {}),
: unverifiableLapsedController ? { retryAt: new Date(Math.max(now.getTime() + 1_000,
unverifiedWaitEndsAt ?? now.getTime() + NATIVE_NAMESPACE_UNVERIFIED_WAIT_MS)) } : {}),
} as const;
}
@@ -2478,18 +2478,23 @@ process.on("SIGTERM", () => child.kill("SIGTERM"));
const [run] = await fixture.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId));
const stopped = await hasNativeLocalProcessStop(fixture.db, companyId, fixture.runId);
expect(probes).toEqual([]);
// The namespace wait is bounded: it is checked again when the wait ends.
expect(dispositions).toEqual([{ kind: "awaiting_evidence", runId: fixture.runId,
reason: "controller_process_space_unverified", retryAt: expect.any(Date) }]);
// A stopped or fenced controller boot already outlived its lease and the
// fence grace, so the namespace wait ends on the first look: the run ends
// as unverified (never as a proven stop) and the task continues in a fresh
// run. Nothing local is probed, claimed or recorded as stopped.
expect(dispositions).toEqual([{ kind: "blocked", runId: fixture.runId,
reason: "controller_process_space_unverified" }]);
expect(claimed).not.toHaveBeenCalled();
expect(stamped).not.toHaveBeenCalled();
expect(stopped).toBe(false);
expect(after).toMatchObject({ phase: before.phase, attempt: before.attempt,
expect(after).toMatchObject({ phase: "terminal_failure", attempt: before.attempt,
controllerGeneration: before.controllerGeneration, controllerBootId: before.controllerBootId,
controllerPid: before.controllerPid, controllerHostHash: before.controllerHostHash,
leaseOwner: before.leaseOwner, leaseExpiresAt: before.leaseExpiresAt, resultId: null });
expect(run).toMatchObject({ status: "running", processPid: runnerPid, processGroupId: runnerGroup,
processStartedAt: startedAt, processHostHash: foreignSpace });
failureCode: "native_restart_recovery_blocked",
failureDetail: expect.objectContaining({ reason: "controller_process_space_unverified", executorStopEvidence: "unverified" }),
resultId: null });
expect(run).toMatchObject({ status: "failed", errorCode: "native_restart_recovery_blocked",
processPid: runnerPid, processGroupId: runnerGroup, processStartedAt: startedAt, processHostHash: foreignSpace });
expect(run.runnerProfileJson).toEqual({ ...fixture.preparedProfile, sessionCheckpoint: {
identity: { companyId, issueId: fixture.issueId, runId: fixture.runId, agentId,
sessionId: fixture.native.normalizedSessionId },
@@ -100,6 +100,7 @@ import { copyBackGrokAuth } from "@paperclipai/adapter-grok-local/server";
import { inferOpenAiCompatibleBiller, type AdapterUsageCheckpoint } from "@paperclipai/adapter-utils";
import { createAgentIdentityRedactor } from "../agent-identity-redaction.js";
import { redactNativeFailureError } from "./native-failure-redaction.js";
import { parseCodexCliVersion } from "./codex-runtime-compatibility.js";
import { createNativeToolTrace, type NativeToolTrace } from "./native-tool-trace.js";
import { createNativeProviderFailureObservation } from "./native-provider-failure.js";
@@ -12729,10 +12730,7 @@ async function executePaperclipNativeSessionAttempt(
// Preserve typed failure semantics while removing credentials before any
// persistence, reporting, logging, or rethrow at this boundary.
if (error instanceof Error) {
const originalMessage = error.message;
error.message = redactIdentityText(originalMessage);
if (error.stack) error.stack = redactIdentityText(originalMessage
? error.stack.split(originalMessage).join(error.message) : error.stack);
error = redactNativeFailureError(error, redactIdentityText);
} else {
error = identityRedactor.redact(error);
}
@@ -432,6 +432,33 @@ describe("native provider startup retry", () => {
}
});
it("records a timed-out step's own cause and settles the run through the failure classifier", async () => {
const seeded = await seedRun(await createAgent());
const expired = AbortSignal.timeout(1);
await new Promise(resolve => setTimeout(resolve, 20));
provider.execute.mockReset().mockImplementation(async () => { throw expired.reason; });
await expect(executePaperclipNativeSession({ db, execution: seeded.execution,
runnerInstanceId: seeded.runnerInstanceId,
backend: {
descriptor: async () => { throw new Error("Unexpected provider descriptor call"); },
openSession: async () => { throw new Error("Unexpected provider session call"); },
} })).rejects.toMatchObject({ name: "TimeoutError", message: "The operation was aborted due to timeout" });
const state = await read(seeded);
expect(state.coordinator).toMatchObject({
attempt: 1,
leaseOwner: null,
failureDetail: expect.objectContaining({
message: "The operation was aborted due to timeout",
originalFailureCode: "native_session_interrupted",
providerSessionEstablished: false,
providerEventsExist: false,
}),
});
expect(state.run).toMatchObject({ status: "failed", errorCode: state.coordinator.failureCode });
expect(state.run.errorCode).not.toBe("native_continuation_requires_reconciliation");
});
it("retries a rejected session.open with growing backoff instead of blocking the task", async () => {
const seeded = await seedRun(await createAgent());
@@ -139,6 +139,7 @@ import {
type NativeChatAttachmentReadScope,
} from "./chat-attachment-read.js";
import { recordTaskHistoryRead } from "../queued-comment-delivery.js";
import { recordChildTaskRead } from "./native-child-completion-delivery.js";
import { logger } from "../../middleware/logger.js";
const EXTERNAL_WORKSPACE_OPERATIONS = new Set(["get_workspace", "list_workspaces", "select_workspace", "prepare_repository", "register_deliverable"]);
@@ -947,6 +948,9 @@ export class PaperclipRunnerToolAuthority {
const readableChildren = this.binding.museRuntime
? await this.#readableMuseTaskIds(context.run, childTasks.map(task => task.id)) : null;
const visibleChildTasks = readableChildren ? childTasks.filter(task => readableChildren.has(task.id)) : childTasks;
await recordChildTaskRead(this.db, { companyId: this.binding.companyId, agentId: this.binding.agentId,
runId: this.binding.runId, parentIssueId: this.binding.issueId, children: visibleChildTasks.slice(0, 100) }).catch(error =>
logger.warn({ err: error, runId: this.binding.runId }, "could not record the child tasks a run read"));
return {
company: { id: this.binding.companyId },
actor: redactedActor(context.actor),