Merge qualified backup expiration parent into GitHub connection work

Co-Authored-By: Paperclip <noreply@paperclip.ing>

* commit 'aae32f2f493848199f6cb8c0a9996af3ee0061b1': (30 commits)
  Scope status-card refresh tasks to their claim revision
  test(chat): retain completion identity before link metadata loads
  test(chat): locate completion through its rendered task link
  fix(chat): recover committed native Agent Chat replies (#18465)
  perf(heartbeat): drive retry promotion from durable deadlines (#18387)
  fix(ui): make the Coordinator composer footer opaque (#18455)
  fix(testops): verify unexpanded result job runs (#18462)
  test(server): observe restored Retry before draining unrelated work
  test(runner): own listener cleanup across setup failures
  test(runner): buffer upgrade frames until the wire subscribes
  fix(runner): preserve OpenCode written-answer defaults (#18456)
  fix(native): fence retries when replaying an audited Stop (#18457)
  test: await async outcomes and drive saved deadlines deterministically (#18416)
  fix(ui): retain startup autosave through session recovery (#18412)
  perf(server): recover independent native replies concurrently (#18421)
  perf(chat): scope task questions and batch inbox queue checks (#18422)
  fix(native): recover controllers across webserver replacement (#18343)
  fix(ui): restore task document authoring and add persistence journey (#18356)
  fix(tools): stabilize tied profile entry order (#18427)
  fix(adapter-utils): tolerate entries removed during workspace snapshots (#18426)
  ...
This commit is contained in:
DottaandPaperclip committed 2026-10-31 09:41:51 -05:00
commit f522ab07e2
484 files changed
+33834 -1620

No files matched your search

@@ -229,7 +229,9 @@ describe("run failure diagnostics", () => {
const result = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics(run({ errorCode: "adapter_failed" }), { error: thrown }));
expect(result.execution).toEqual({ remoteRunnerRecoveryStage: "process_verification", remoteRunnerRecoveryReason: "process_not_running" });
expect(result.exceptions).toMatchObject([{ code: "runner_remote_process_identity_unavailable" }]);
expect(JSON.stringify(result)).not.toContain("private");
expect(JSON.stringify(result)).not.toContain("private-path");
expect(result.exceptions[0]).not.toHaveProperty("path");
expect(result.exceptions[0]).not.toHaveProperty("pid");
const transport = attachRemoteRunnerRecoveryDiagnostic(new Error("socket closed"),
{ stage: "identity_marker", reason: "probe_transport_failed" });
+4
View File
@@ -480,6 +480,10 @@ export function activityService(db: Db) {
.where(
and(
eq(heartbeatRuns.companyId, companyId),
sql`(not exists (select 1 from ${issues} where ${issues.id} = ${issueId}
and ${issues.companyId} = ${companyId} and ${issues.conversationAgentId} is not null)
or coalesce(${heartbeatRuns.nativeIssueId}::text, ${heartbeatRuns.issueId}::text,
${heartbeatRuns.contextSnapshot} ->> 'issueId', ${heartbeatRuns.contextSnapshot} ->> 'taskId') = ${issueId})`,
...(selectedRunIds ? [inArray(heartbeatRuns.id, selectedRunIds)] : []),
or(
eq(heartbeatRuns.issueId, issueId),
+193
View File
@@ -0,0 +1,193 @@
import { createHash } from "node:crypto";
import { and, eq, isNull, sql } from "drizzle-orm";
import { agentApiKeys, agents, companies, companyMemberships, heartbeatRuns, issueComments, issues, nativeRunResults, type Db } from "@paperclipai/db";
import { agentApiKeyScopeSchema, isUuidLike, sendChatMessageSchema, type AgentApiKeyScope } from "@paperclipai/shared";
import { badRequest, conflict, forbidden, notFound, tooManyRequests } from "../errors.js";
import { agentRunWritesRevoked, assertAgentRunWriteAllowed } from "../agent-run-cancellation.js";
import { authorizationService, canActorReadIssuePrivacy, type AuthorizationActor } from "./authorization.js";
import { persistActivity, publishActivity, type ActivityPublication } from "./activity-log.js";
import { instanceSettingsService } from "./instance-settings.js";
import { issueService } from "./issues.js";
import { issueTreeControlService } from "./issue-tree-control.js";
import { resolveActorSourceTrustForIssue } from "./source-trust.js";
import { resolveTaskWatchdogMutationScope } from "./task-watchdog-scope.js";
import { readNativeReviewAssignmentContext } from "./native-runtime/native-review-participant.js";
import { nativeRunKeyScope } from "./native-runtime/native-skill-test-scope.js";
import { crossIssueInfluenceLimitError, observeCrossIssueInfluence } from "./cross-issue-influence-limit.js";
type Actor = { companyId: string; agentId: string; runId: string; agentApiKeyId?: string | null;
keyScope?: AgentApiKeyScope | null; onBehalfOfUserId?: string | null;
authorizingUserId?: string | null; source?: AuthorizationActor["source"] };
export interface ChatMessageReceipt {
commentId: string;
conversationId: string;
chatUrl: string;
sourceTaskId: string;
replayed: boolean;
}
function runTask(run: typeof heartbeatRuns.$inferSelect) {
return run.nativeIssueId ?? run.issueId ?? run.contextSnapshot?.issueId ?? run.contextSnapshot?.taskId;
}
export async function sendAgentChatMessage(
db: Db,
actor: Actor,
value: unknown,
options: { authorizeNative?: (tx: Db) => Promise<unknown>; receiptOnly?: boolean } = {},
): Promise<ChatMessageReceipt | null> {
const parsed = sendChatMessageSchema.safeParse(value);
if (!parsed.success) throw badRequest("Chat messages require body and idempotencyKey only", parsed.error.issues);
if (![actor.companyId, actor.agentId, actor.runId].every(isUuidLike)) throw forbidden("An authenticated task run is required");
const credentialScope = agentApiKeyScopeSchema.safeParse(actor.keyScope ?? { kind: "standard" });
if (!credentialScope.success) throw forbidden("The agent credential scope is invalid");
if (credentialScope.data.kind !== "standard") throw forbidden("Scoped test and task bridge credentials cannot publish agent chat messages");
const input = parsed.data;
const contentHash = createHash("sha256").update(input.body).digest("hex");
const publications: ActivityPublication[] = [];
const result = await db.transaction(async transaction => {
const tx = transaction as unknown as Db;
await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${`issue-privacy-tree:${actor.companyId}`}, 0))`);
const runWhere = and(eq(heartbeatRuns.id, actor.runId), eq(heartbeatRuns.companyId, actor.companyId), eq(heartbeatRuns.agentId, actor.agentId));
const [initial] = await tx.select().from(heartbeatRuns).where(runWhere);
const sourceId = initial && runTask(initial);
if (typeof sourceId !== "string" || !isUuidLike(sourceId)) throw forbidden("Chat delivery requires a task-bound run");
const sourceQuery = () => tx.select().from(issues).where(and(eq(issues.id, sourceId), eq(issues.companyId, actor.companyId)));
const [initialSource] = await sourceQuery();
if (!initialSource) throw forbidden("This run no longer owns the source task");
let source = initialSource;
const contextQuery = () => tx.select({ run: heartbeatRuns, agent: agents }).from(heartbeatRuns)
.innerJoin(agents, and(eq(agents.id, heartbeatRuns.agentId), eq(agents.companyId, heartbeatRuns.companyId)))
.where(runWhere);
const validateContext = (context: { run: typeof heartbeatRuns.$inferSelect; agent: typeof agents.$inferSelect } | undefined) => {
if (!context || context.run.status !== "running" || agentRunWritesRevoked(context.run)
|| runTask(context.run) !== sourceId || source.assigneeAgentId !== actor.agentId
|| source.executionRunId !== actor.runId || (source.checkoutRunId && source.checkoutRunId !== actor.runId)
|| ["paused", "terminated", "pending_approval", "error"].includes(context.agent.status)) {
throw forbidden("This run no longer owns the source task");
}
if (source.conversationAgentId) throw forbidden("Reply normally in this conversation; send_chat_message is for execution tasks");
if (nativeRunKeyScope({ contextSnapshot: context.run.contextSnapshot, issueId: source.id, workMode: source.workMode }).kind === "skill_test") {
throw forbidden("Skill-test runs cannot publish agent chat messages");
}
if (source.workMode !== "standard" || ["done", "cancelled"].includes(source.status)
|| readNativeReviewAssignmentContext(context.run.contextSnapshot)) {
throw forbidden("Chat delivery requires an ordinary task in standard execution mode");
}
return context;
};
const [initialContext] = await contextQuery();
const context = validateContext(initialContext);
if (!(await instanceSettingsService(tx).getExperimental()).enableAgentChat) throw notFound("Agent Chat is disabled");
const userId = context.run.responsibleUserId;
if (!userId) throw forbidden("This task has no responsible human to receive the message");
if (actor.onBehalfOfUserId !== undefined && actor.onBehalfOfUserId !== userId) throw forbidden("The authenticated responsible human no longer matches this run");
const assertCurrentKey = async (lock: boolean) => {
if (!actor.agentApiKeyId) return;
const query = tx.select().from(agentApiKeys).where(and(eq(agentApiKeys.id, actor.agentApiKeyId),
eq(agentApiKeys.companyId, actor.companyId), eq(agentApiKeys.agentId, actor.agentId), isNull(agentApiKeys.revokedAt)));
const [key] = lock ? await query.for("share") : await query;
const currentScope = key && agentApiKeyScopeSchema.safeParse(key.scopeConfig ?? { kind: "standard" });
if (!key || !currentScope?.success || JSON.stringify(currentScope.data) !== JSON.stringify(credentialScope.data)
|| (key.responsibleUserId?.trim() || null) !== (actor.onBehalfOfUserId ?? userId)) {
throw forbidden("The agent key was revoked or its authority changed");
}
};
await assertCurrentKey(false);
const [membership] = await tx.select({ id: companyMemberships.id }).from(companyMemberships).where(and(
eq(companyMemberships.companyId, actor.companyId), eq(companyMemberships.principalType, "user"),
eq(companyMemberships.principalId, userId), eq(companyMemberships.status, "active"),
)).for("share");
if (!membership) throw forbidden("The responsible human is no longer an active company member");
const authActor: AuthorizationActor = { type: "agent", ...actor, keyId: actor.agentApiKeyId, keyScope: credentialScope.data,
source: actor.source ?? "agent_jwt", onBehalfOfUserId: actor.onBehalfOfUserId ?? userId };
const authorize = async (issueId: string, action: "issue:read" | "issue:comment") => {
const decision = await authorizationService(tx).decide({ actor: authActor, action,
resource: { type: "issue", companyId: actor.companyId, issueId } });
if (!decision.allowed) throw forbidden(decision.explanation);
};
await authorize(source.id, "issue:read");
if (!(await canActorReadIssuePrivacy(tx, { type: "none" }, source))) {
throw forbidden("Restricted task content cannot be sent to a company-visible chat");
}
const svc = issueService(tx);
let conversation = await svc.getConversation(actor.companyId, actor.agentId, userId);
if (!conversation && !options.receiptOnly) {
conversation = await svc.create(actor.companyId, {
title: `Chat with ${context.agent.name}`, assigneeAgentId: actor.agentId,
conversationAgentId: actor.agentId, conversationUserId: userId,
conversationState: "waiting", status: "in_review", createdByAgentId: actor.agentId,
responsibleUserId: userId, trustExplicitResponsibleUserId: true,
});
const opened = await persistActivity(tx, { companyId: actor.companyId, actorType: "agent", actorId: actor.agentId,
agentId: actor.agentId, runId: actor.runId, action: "issue.conversation_opened", entityType: "issue", entityId: conversation.id,
details: { agentId: actor.agentId, conversationUserId: userId } });
publications.push(opened.publication);
}
if (conversation) await tx.select({ id: issues.id }).from(issues).where(and(
eq(issues.id, conversation.id), eq(issues.companyId, actor.companyId),
)).for("update");
const [lockedSource] = await sourceQuery().for("update");
if (!lockedSource) throw forbidden("This run no longer owns the source task");
source = lockedSource;
if (options.authorizeNative) await options.authorizeNative(tx);
const [lockedContext] = await contextQuery().for("update");
const current = validateContext(lockedContext);
if (current.run.responsibleUserId !== userId) throw forbidden("The responsible human changed before this message was saved");
await assertCurrentKey(true);
if ((await resolveTaskWatchdogMutationScope(tx, { ...actor, type: "agent" })).kind !== "none") {
throw forbidden("Chat delivery requires an ordinary task in standard execution mode");
}
await assertAgentRunWriteAllowed(tx, actor.companyId, actor);
if (await issueTreeControlService(tx).getActivePauseHoldGate(actor.companyId, source.id)) throw forbidden("This task is paused");
const [finished] = await tx.select({ id: nativeRunResults.id }).from(nativeRunResults).where(and(
eq(nativeRunResults.companyId, actor.companyId), eq(nativeRunResults.runId, actor.runId), eq(nativeRunResults.schemaStatus, "accepted"),
)).limit(1);
if (finished) throw forbidden("Send the message before finishing the run");
if (!(await instanceSettingsService(tx).getExperimental()).enableAgentChat) throw notFound("Agent Chat is disabled");
await authorize(source.id, "issue:read");
if (!conversation) return null;
await authorize(conversation.id, "issue:comment");
const recipientDecision = await authorizationService(tx).decide({ actor: { type: "board", userId, source: "session", ignoreInstanceAdmin: true },
action: "issue:read", resource: { type: "issue", companyId: actor.companyId, issueId: conversation.id } });
if (!recipientDecision.allowed) throw forbidden("The responsible human cannot access this chat");
const requestId = `chat:${source.id}:${createHash("sha256").update(input.idempotencyKey).digest("hex")}`;
const [prior] = await tx.select().from(issueComments).where(and(eq(issueComments.companyId, actor.companyId),
eq(issueComments.issueId, conversation.id), eq(issueComments.authorAgentId, actor.agentId), eq(issueComments.clientRequestId, requestId)));
const [company] = await tx.select({ prefix: companies.issuePrefix }).from(companies).where(eq(companies.id, actor.companyId));
const receipt = (commentId: string, replayed: boolean): ChatMessageReceipt => ({ commentId, conversationId: conversation!.id,
chatUrl: `/${company.prefix}/chats/${actor.agentId}`, sourceTaskId: source.id, replayed });
if (prior) {
if (prior.metadata?.chatMessage?.contentHash !== contentHash) {
if (options.receiptOnly) return null;
throw conflict("Chat message retry key was already used with different content");
}
return receipt(prior.id, true);
}
if (options.receiptOnly) return null;
const influence = await observeCrossIssueInfluence(tx, { companyId: actor.companyId, agentId: actor.agentId,
runId: actor.runId, responsibleUserId: userId, targetIssueId: conversation.id,
targetIssueIdentifier: conversation.identifier, kind: "comment" });
if (influence && !influence.allowed) {
const denial = crossIssueInfluenceLimitError(influence, { actorLabel: current.agent.name,
assigneeLabel: current.agent.name, issueIdentifier: conversation.identifier });
throw tooManyRequests(denial.error, denial.details);
}
const sourceTrust = await resolveActorSourceTrustForIssue({ db: tx, issue: source,
actor: { actorType: "agent", actorId: actor.agentId, agentId: actor.agentId, runId: actor.runId } });
const comment = await svc.addComment(conversation.id, input.body, { agentId: actor.agentId, runId: actor.runId }, {
clientRequestId: requestId, sourceTrust, proactiveChatMessage: { sourceTaskId: source.id, userId },
metadata: { version: 1, chatMessage: { sourceTaskId: source.id, sourceTaskIdentifier: source.identifier!, sourceRunId: actor.runId, contentHash },
sections: [{ rows: [{ type: "issue_link", issueId: source.id, identifier: source.identifier, label: "Source task" }] }] },
}, tx);
const activity = await persistActivity(tx, { companyId: actor.companyId, actorType: "agent", actorId: actor.agentId,
agentId: actor.agentId, runId: actor.runId, agentApiKeyId: actor.agentApiKeyId,
action: "issue.comment_added", entityType: "issue", entityId: conversation.id,
details: { commentId: comment.id, identifier: conversation.identifier, conversationUserId: userId,
conversationAgentId: actor.agentId, sourceTaskId: source.id } });
publications.push(activity.publication);
return receipt(comment.id, false);
});
for (const publication of publications) publishActivity(publication);
return result;
}
@@ -0,0 +1,21 @@
import { and, eq, inArray, sql } from "drizzle-orm";
import { issueComments, issueReadStates, type Db } from "@paperclipai/db";
export async function withConversationReadState<T extends { id: string }>(db: Db, companyId: string, userId: string, conversations: T[]) {
if (!conversations.length) return [] as Array<T & { isUnreadForMe: boolean; lastExternalCommentAt: Date | null }>;
const rows = await db.select({
issueId: issueComments.issueId,
latest: sql<Date>`max(${issueComments.createdAt})`,
unread: sql<boolean>`bool_or(${issueComments.createdAt} > coalesce(${issueReadStates.lastReadAt}, to_timestamp(0)))`,
}).from(issueComments).leftJoin(issueReadStates, and(
eq(issueReadStates.companyId, companyId), eq(issueReadStates.issueId, issueComments.issueId), eq(issueReadStates.userId, userId),
)).where(and(eq(issueComments.companyId, companyId), inArray(issueComments.issueId, conversations.map(chat => chat.id)),
sql`${issueComments.deletedAt} is null`, sql`${issueComments.authorAgentId} is not null`,
sql`${issueComments.authorType} is distinct from 'system'`,
sql`${issueComments.presentation} ->> 'kind' is distinct from 'system_notice'`,
eq(issueComments.origin, "comment"),
)).groupBy(issueComments.issueId);
const byId = new Map(rows.map(row => [row.issueId, row]));
return conversations.map(chat => ({ ...chat, isUnreadForMe: byId.get(chat.id)?.unread ?? false,
lastExternalCommentAt: byId.get(chat.id)?.latest ?? null }));
}
@@ -530,6 +530,9 @@ export async function undeliveredConversationComments(
eq(issueComments.companyId, companyId),
eq(issueComments.issueId, issueId),
isNull(issueComments.deletedAt),
sql`${issueComments.authorUserId} is not null`,
isNull(issueComments.authorAgentId),
eq(issueComments.origin, "comment"),
sql`${issueComments.clientRequestId} is not null`,
sql`not exists (select 1 from ${agentWakeupRequests} where ${agentWakeupRequests.companyId} = ${companyId}
and ${agentWakeupRequests.idempotencyKey} = 'conversation-comment:' || ${issueComments.id}::text)`,
@@ -1,6 +1,7 @@
import { notifyInstructionCollection } from "./instruction-collection-notifications.js";
import fs from "node:fs/promises";
import path from "node:path";
import { isDeepStrictEqual } from "node:util";
import { and, eq, inArray, isNull, sql } from "drizzle-orm";
import { agents, environmentLeases, environments, agentInstructionWorkingCopies as copies, type Db } from "@paperclipai/db";
import { syncDirectoryToSsh, restoreWorkspaceFromSshExecution } from "@paperclipai/adapter-utils/ssh";
@@ -20,6 +21,11 @@ import { cachedAgentFileManifest, captureAgentFileCheckpoint, checkpointBaseline
import { logger } from "../middleware/logger.js";
type Copy = typeof copies.$inferSelect;
function adoptionAuthority(row: Copy) {
const { updatedAt: _updatedAt, nextAttemptAt: _nextAttemptAt, ...authority } = row;
return authority;
}
export class AgentDirectoryReuseInvalidatedError extends Error {}
const completed = new Set(["saved", "unchanged", "resolved", "unavailable"]);
const transports = new Map<string, PreparedAdapterExecutionTargetRuntime>();
@@ -291,10 +297,22 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
// adopt_remote constructs a collector on the successor target without
// uploading the stale host mirror over the retained remote directory.
const replacement = remote ? await transport({ ...prior, runId: input.runId }, remote, true) : undefined;
const expectedPrior = prior;
let adopted: Copy | null;
try { adopted = await store.locked(input.companyId, input.agentId, bound, false, async (tx, _agent, canonical) => {
if (remote && !validLeases(await tx.select().from(environmentLeases).where(and(
eq(environmentLeases.companyId, input.companyId), inArray(environmentLeases.id, leaseIds))).for("update"))) return null;
const [lockedPrior] = await tx.select().from(copies).where(and(eq(copies.companyId, input.companyId),
eq(copies.runId, expectedPrior.runId), eq(copies.agentId, input.agentId))).for("update");
if (!lockedPrior || !isDeepStrictEqual(adoptionAuthority(lockedPrior), adoptionAuthority(expectedPrior))) {
const fields = ["state", "receipt", "baseHash", "localRoot", "executionRoot", "location", "processStoppedAt", "responsibleUserId", "entryFile", "attempts", "candidateHash", "errorCode", "errorMessage", "baseRevisionId", "candidateBase64", "createdAt"] as const;
const diagnostic = { stage: "locked_adoption_authority", predecessorPresent: Boolean(lockedPrior), observedAfterInitialRead: true,
changedFields: lockedPrior ? fields.filter(field => !isDeepStrictEqual(lockedPrior[field], expectedPrior[field])) : [] };
logger.warn({ companyId: input.companyId, agentId: input.agentId, runId: input.runId, previousRunId: input.previousRunId,
adoptionConflict: diagnostic }, "Agent directory adoption conflict");
throw conflict("Agent directory ownership changed", diagnostic);
}
prior = lockedPrior;
const current = await inspectAgentDirectory(canonical);
const canonicalChanged = agentDirectoryBaselineDigest(current.snapshot) !== agentDirectoryBaselineDigest(baseline(prior));
if (canonicalChanged && !input.allowRetirementHandoff) return null;
@@ -304,7 +322,18 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
.where(and(eq(copies.companyId, input.companyId), eq(copies.runId, prior.runId), eq(copies.agentId, input.agentId), eq(copies.state, "unchanged_turn"),
eq(copies.updatedAt, prior.updatedAt), eq(copies.baseHash, prior.baseHash), eq(copies.localRoot, prior.localRoot),
isNull(copies.processStoppedAt), sql`${copies.receipt}->>'retainedByRunId' IS NULL`)).returning();
if (!claimed) throw conflict("Agent directory ownership changed");
if (!claimed) {
const [conditions] = await tx.select({ unchangedTurn: eq(copies.state, "unchanged_turn"),
timestampMatches: eq(copies.updatedAt, prior.updatedAt), baselineMatches: eq(copies.baseHash, prior.baseHash),
localRootMatches: eq(copies.localRoot, prior.localRoot), unstopped: isNull(copies.processStoppedAt),
successorAbsent: sql<boolean>`${copies.receipt}->>'retainedByRunId' IS NULL`,
receiptMatches: sql<boolean>`${copies.receipt} IS NOT DISTINCT FROM ${JSON.stringify(prior.receipt)}::jsonb`,
}).from(copies).where(and(eq(copies.companyId, input.companyId), eq(copies.runId, prior.runId), eq(copies.agentId, input.agentId)));
const diagnostic = { stage: "adoption_claim", observedAfterClaim: true, predecessorPresent: Boolean(conditions), conditions: conditions ?? null };
logger.warn({ companyId: input.companyId, agentId: input.agentId, runId: input.runId, previousRunId: input.previousRunId,
adoptionConflict: diagnostic }, "Agent directory adoption conflict");
throw conflict("Agent directory ownership changed", diagnostic);
}
const { createdAt: _createdAt, updatedAt: _updatedAt, ...values } = prior;
const [next] = await tx.insert(copies).values({ ...values, runId: input.runId, responsibleUserId: bound.onBehalfOfUserId!, state: "prepared",
receipt: { ...prior.receipt, materializationRunId: prior.receipt?.materializationRunId ?? prior.runId,
+16 -8
View File
@@ -1080,8 +1080,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
Math.max(Math.trunc(serviceOptions.openDecisionLimit ?? OPEN_DECISION_DEFAULT_LIMIT), 1),
OPEN_DECISION_MAX_LIMIT,
);
const service = {
list: async (companyId: string, options: AttentionListOptions = {}): Promise<AttentionFeed> => {
async function list(companyId: string, options: AttentionListOptions = {}, questionIssueIds?: readonly string[]): Promise<AttentionFeed> {
if (options.all && !options.queue && !options.allowUnscopedAll) {
throw badRequest("all requires a queue filter");
}
@@ -1105,7 +1104,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
collected.push({ ...item, dismissal });
};
if (!options.excludeSources?.includes("approval")) {
if (!questionIssueIds && !options.excludeSources?.includes("approval")) {
const pendingApprovals = await db
.select({
id: approvals.id,
@@ -1196,6 +1195,8 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
.where(and(
eq(issueThreadInteractions.companyId, companyId),
inArray(issueThreadInteractions.status, [...PENDING_INTERACTION_STATUSES]),
questionIssueIds ? inArray(issueThreadInteractions.issueId, [...questionIssueIds]) : undefined,
questionIssueIds ? eq(issueThreadInteractions.kind, "ask_user_questions") : undefined,
))
.orderBy(desc(issueThreadInteractions.updatedAt), desc(issueThreadInteractions.id));
// Addressee invokability needs the org graph; the audience line also needs
@@ -1285,6 +1286,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
}));
}
if (!questionIssueIds) {
const openDecisionQuery = db.select({
id: decisions.id,
bundleId: decisions.bundleId,
@@ -1865,6 +1867,8 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
}));
}
}
const deduped = new Map<string, AttentionItem>();
for (const item of collected) {
const current = deduped.get(item.dedupKey);
@@ -1972,18 +1976,22 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
}
}
return { ...summary, nextCursor, items };
},
};
}
return {
...service,
list: (companyId: string, options: AttentionListOptions = {}) => list(companyId, options),
taskQuestions: async (companyId: string, issueIds: readonly string[], options: { actor: AuthorizationActor; userId: string }): Promise<AttentionItem[]> => {
if (issueIds.length === 0) return [];
const feed = await list(companyId, { ...options, all: true, allowUnscopedAll: true }, issueIds);
return feed.items;
},
summary: async (companyId: string, options: Pick<AttentionListOptions, "userId" | "actor"> = {}): Promise<AttentionSummary> => {
const { items: _items, nextCursor: _cursor, ...summary } = await service.list(companyId, {
const { items: _items, nextCursor: _cursor, ...summary } = await list(companyId, {
...options, all: true, allowUnscopedAll: true, summaryOnly: true,
});
return summary;
},
taskIndex: async (companyId: string, options: Pick<AttentionListOptions, "userId" | "actor"> = {}): Promise<AttentionTaskIndex> => {
const { items, nextCursor: _cursor, ...summary } = await service.list(companyId, {
const { items, nextCursor: _cursor, ...summary } = await list(companyId, {
...options, all: true, allowUnscopedAll: true, summaryOnly: true,
});
return { ...summary, taskIds: [...new Set(items.flatMap(item => item.relatedIssue
+7 -2
View File
@@ -6150,9 +6150,14 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo
if (!versionId || (selectedKeys && !selectedKeys.has(key))) continue;
const [version] = await db.select({ skillId: companySkillVersions.companySkillId }).from(companySkillVersions)
.where(and(eq(companySkillVersions.companyId, companyId), eq(companySkillVersions.id, versionId)));
skills = skills.filter(skill => skill.key !== key);
// A saved JSONB selection map has a different key order from the fresh
// name/key query. Replace existing rows in place so pinning versions does
// not reorder the server-produced preference (and its session fingerprint).
const index = skills.findIndex(skill => skill.key === key);
if (index !== -1) skills.splice(index, 1);
const owner = version ? await getById(companyId, version.skillId) : null;
if (owner) skills.push({ ...owner, key, slug: owner.key === key ? owner.slug : key.split("/").at(-1)! });
if (owner) skills.splice(index === -1 ? skills.length : index, 0,
{ ...owner, key, slug: owner.key === key ? owner.slug : key.split("/").at(-1)! });
}
await packages.prefetch(companyId, skills.flatMap(skill => {
const id = options.versionSelections?.get(skill.key) ?? skill.currentVersionId;
@@ -56,17 +56,7 @@ export function conversationDecisionReceiptsService(db: Db) {
const taskIds = new Set([...handoffIds, ...directIds].filter(id => id !== conversationId));
if (taskIds.size === 0) return [];
// One on-demand canonical company snapshot, not one query per task. The
// attention owner applies current audience, privacy, dismissal, snooze and
// archive policy, including its existing queue/retention bookkeeping.
// Cost grows with the company feed, not only the returned task questions;
// there is currently no task-scoped attention API. Do not poll this read.
const feed = await attentionService(db).list(companyId, {
all: true, allowUnscopedAll: true, actor: options.actor, userId: options.userId,
});
return feed.items.filter(item => item.sourceKind === "issue_thread_interaction"
&& item.detail?.kind === "questions"
&& item.relatedIssue !== null && taskIds.has(item.relatedIssue.id));
return attentionService(db).taskQuestions(companyId, [...taskIds], options);
},
async listCreatedTasks(companyId: string, conversationId: string, options: {
actor: AuthorizationActor; userId: string; cursor?: string; limit?: number;
+22 -20
View File
@@ -456,34 +456,36 @@ export function decisionQueueService(db: Db) {
});
}
let insertedAnyItem = false;
for (const item of matchingItems) {
const inserted = await txDb.insert(decisionQueueItems).values({
for (let offset = 0; offset < matchingItems.length; offset += 500) {
const inserted = await txDb.insert(decisionQueueItems).values(matchingItems.slice(offset, offset + 500).map(item => ({
companyId,
queueId: queue.id,
sourceKind: item.sourceKind,
sourceId: item.sourceId,
addedByType: "system",
}).onConflictDoNothing({
}))).onConflictDoNothing({
target: [decisionQueueItems.queueId, decisionQueueItems.sourceKind, decisionQueueItems.sourceId],
}).returning();
if (!inserted[0]) continue;
if (inserted.length === 0) continue;
insertedAnyItem = true;
await txDb.insert(decisionTriageEvents).values({
companyId,
queueId: queue.id,
sourceKind: item.sourceKind,
sourceId: item.sourceId,
action: "queue_item.seeded",
...eventActorColumns(SYSTEM_ACTOR),
details: { seedKey: seed.key },
});
await recordActivity(txDb, SYSTEM_ACTOR, {
companyId,
action: "decision_queue_item.seeded",
entityType: "decision_queue",
entityId: queue.id,
details: { sourceKind: item.sourceKind, sourceId: item.sourceId, seedKey: seed.key },
});
for (const item of inserted) {
await txDb.insert(decisionTriageEvents).values({
companyId,
queueId: queue.id,
sourceKind: item.sourceKind,
sourceId: item.sourceId,
action: "queue_item.seeded",
...eventActorColumns(SYSTEM_ACTOR),
details: { seedKey: seed.key },
});
await recordActivity(txDb, SYSTEM_ACTOR, {
companyId,
action: "decision_queue_item.seeded",
entityType: "decision_queue",
entityId: queue.id,
details: { sourceKind: item.sourceKind, sourceId: item.sourceId, seedKey: seed.key },
});
}
}
if (insertedAnyItem) {
await txDb.update(decisionQueues).set({ updatedAt: new Date() })
+25 -2
View File
@@ -2,6 +2,7 @@ import { registerDeferredReceiptWork } from "./heartbeat/deferred-receipt-work.j
import { reconcileTaskChatRetryReceipts } from "./task-retry.js";
import { registerIssueRecoveryWork } from "./recovery/work.js";
import { registerRunDispatchWork } from "./heartbeat/dispatch-work.js";
import { NATIVE_RETAINED_OWNER_WAIT } from "./native-runtime/native-controller-recovery.js";
import { externalConversationSettlementWork } from "./external-conversation-settlement-work.js";
import { nativeRecoveryNoticeWork } from "./native-runtime/native-recovery-notice-work.js";
import { nativeRunProjectionWork } from "./native-runtime/native-run-projection-work.js";
@@ -610,6 +611,9 @@ const nativeSessionResumeDispatchTimers = new Map<
string,
ReturnType<typeof setTimeout>
>();
// Routes and the scheduler can drain/resume through different service facades.
// Rearm through the original owner so its database and dispatch mode survive.
const nativeRecoveryDeadlines = new Map<string, { rearm: () => void }>();
// Shared with HTTP admission so an idle hold fences work before inspection.
export { computeTaskDrain, applyTaskDrain, startTaskDrain, stopTaskDrain } from "./task-admission.js";
@@ -1339,6 +1343,9 @@ export function heartbeatService(
}
function resumeQueuedRunsAfterDrain(...args: Parameters<typeof heartbeatQueue.resumeQueuedRunsAfterDrain>) {
for (const pending of nativeRecoveryDeadlines.values()) {
pending.rearm();
}
const promise = heartbeatQueue.resumeQueuedRunsAfterDrain(...args);
activeWakeupPromises.add(promise);
void promise.then(
@@ -1708,10 +1715,15 @@ export function heartbeatService(
}
}
// Native ownership waits survive a temporary drain without adding a polling
// loop. A process restart rediscovers the same deadlines from coordinator rows.
function scheduleNativeSessionResumeDispatch(
runId: string,
nextAttemptAt: Date,
mode?: "restart",
) {
const pending = { rearm: () => scheduleNativeSessionResumeDispatch(runId, nextAttemptAt, mode) };
nativeRecoveryDeadlines.set(runId, pending);
const prior = nativeSessionResumeDispatchTimers.get(runId);
if (prior) clearTimeout(prior);
const delayMs = Math.max(0, nextAttemptAt.getTime() - Date.now());
@@ -1720,6 +1732,12 @@ export function heartbeatService(
nativeSessionResumeDispatchTimers.delete(runId);
void (async () => {
if ((await getSchedulingSuppression()).suppressed) return;
if (nativeRecoveryDeadlines.get(runId) !== pending) return;
nativeRecoveryDeadlines.delete(runId);
if (mode === "restart") {
await recoverNativeRunsAfterRestart(new Date(), [runId]);
return;
}
await dispatchNativeSessionResumptions({
db,
runnerInstanceId:
@@ -1741,6 +1759,9 @@ export function heartbeatService(
},
});
})().catch((error) => {
if (!nativeRecoveryDeadlines.has(runId)) {
scheduleNativeSessionResumeDispatch(runId, new Date(Date.now() + 30_000), mode);
}
logger.error(
{ err: error, runId },
"failed to dispatch scheduled native session resume",
@@ -3531,7 +3552,8 @@ export function heartbeatService(
err.original instanceof Error
? err.original.message
: String(err.original ?? "");
const retryReasonCode = /native_finalization_missing/i.test(
const waitingForRetainedOwner = retryMessage === NATIVE_RETAINED_OWNER_WAIT;
const retryReasonCode = waitingForRetainedOwner ? NATIVE_RETAINED_OWNER_WAIT : /native_finalization_missing/i.test(
retryMessage,
)
? "semantic_result_missing"
@@ -3551,7 +3573,8 @@ export function heartbeatService(
stream: "system",
level: "warn",
message:
retryReasonCode === "semantic_result_missing"
waitingForRetainedOwner ? "native continuation waiting for the retained controller lease"
: retryReasonCode === "semantic_result_missing"
? "provider turn completed without a semantic result; same-run disposition recovery persisted"
: "native session transport interrupted; same-run resume persisted",
payload: {
@@ -49,7 +49,11 @@ export function bindHeartbeatRunnerHomeMode(config: Record<string, unknown>, per
const saved = parseObject(persistedExecution);
if (Object.keys(saved).length === 0) return config;
// Execution schemas before v7 always used isolated homes.
return { ...config, homeMode: saved.homeMode ?? "isolated" };
const homeMode = saved.homeMode ?? "isolated";
// Omission already means isolated. Recovery must not add a fingerprinted
// configuration property merely to restate that unchanged default.
if (homeMode === "isolated" && config.homeMode === undefined) return config;
return { ...config, homeMode };
}
export interface HeartbeatExecutionConfigurationInput {
@@ -0,0 +1,102 @@
import type { Db } from "@paperclipai/db";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { heartbeatService, startTaskDrain, stopTaskDrain } from "../heartbeat.js";
import type { HeartbeatRecoveryDependencies } from "./recovery.js";
const hooks = vi.hoisted(() => ({
schedule: undefined as HeartbeatRecoveryDependencies["scheduleNativeSessionResumeDispatch"] | undefined,
restart: vi.fn(async () => []),
resume: vi.fn(async () => []),
queue: vi.fn(async () => undefined),
}));
vi.mock("./recovery.js", async original => {
const module = await original<typeof import("./recovery.js")>();
return { ...module, createHeartbeatRecovery: (db: Db, dependencies: HeartbeatRecoveryDependencies) => {
hooks.schedule = dependencies.scheduleNativeSessionResumeDispatch;
return { ...module.createHeartbeatRecovery(db, dependencies), recoverNativeRunsAfterRestart: hooks.restart };
} };
});
vi.mock("./queue.js", async original => {
const module = await original<typeof import("./queue.js")>();
return { ...module, createHeartbeatQueue: (...args: Parameters<typeof module.createHeartbeatQueue>) => ({
...module.createHeartbeatQueue(...args), resumeQueuedRunsAfterDrain: hooks.queue,
}) };
});
vi.mock("../native-runtime/index.js", async original => ({
...await original<typeof import("../native-runtime/index.js")>(),
dispatchNativeSessionResumptions: hooks.resume,
}));
describe("native deadline drain and resume", () => {
let heartbeat: ReturnType<typeof heartbeatService>;
beforeEach(() => {
vi.clearAllMocks();
vi.useFakeTimers();
stopTaskDrain();
const db = new Proxy({}, { get: () => { throw new Error("Unexpected deadline database access"); } }) as Db;
heartbeat = heartbeatService(db, { runtimeEnv: {} });
});
afterEach(async () => {
stopTaskDrain();
await heartbeat?.resumeQueuedRunsAfterDrain();
await vi.runOnlyPendingTimersAsync();
await heartbeat?.drainActiveRunExecutions();
vi.useRealTimers();
});
it.each([undefined, "restart"] as const)("retains an overdue %s wait through drain with its original dispatch mode", async mode => {
const deadline = new Date(Date.now() + 100);
startTaskDrain({});
hooks.schedule!("waiting-run", deadline, mode);
await vi.advanceTimersByTimeAsync(100);
expect(hooks.restart).not.toHaveBeenCalled();
expect(hooks.resume).not.toHaveBeenCalled();
stopTaskDrain();
await heartbeat.resumeQueuedRunsAfterDrain();
await vi.advanceTimersByTimeAsync(0);
const called = mode === "restart" ? hooks.restart : hooks.resume;
const other = mode === "restart" ? hooks.resume : hooks.restart;
expect(called).toHaveBeenCalledOnce();
expect(other).not.toHaveBeenCalled();
await heartbeat.resumeQueuedRunsAfterDrain();
await vi.advanceTimersByTimeAsync(60_000);
expect(called).toHaveBeenCalledOnce();
expect(vi.getTimerCount()).toBe(0);
});
it("preserves a future follow-up deadline when draining cancels its timer", async () => {
const deadline = new Date(Date.now() + 100);
hooks.schedule!("future-run", deadline);
await heartbeat.drainActiveRunExecutions();
await heartbeat.resumeQueuedRunsAfterDrain();
await vi.advanceTimersByTimeAsync(99);
expect(hooks.resume).not.toHaveBeenCalled();
await vi.advanceTimersByTimeAsync(1);
expect(hooks.resume).toHaveBeenCalledOnce();
expect(hooks.restart).not.toHaveBeenCalled();
});
it("rearms the original owner when another service facade ends the drain", async () => {
hooks.schedule!("other-facade-run", new Date(Date.now() + 100));
const db = new Proxy({}, { get: () => { throw new Error("Unexpected replacement facade database access"); } }) as Db;
const other = heartbeatService(db, { runtimeEnv: {} });
await other.drainActiveRunExecutions();
await vi.advanceTimersByTimeAsync(100);
await other.resumeQueuedRunsAfterDrain();
await vi.advanceTimersByTimeAsync(0);
expect(hooks.resume).toHaveBeenCalledOnce();
expect(hooks.restart).not.toHaveBeenCalled();
});
it("retries a failed follow-up obligation without changing its dispatch mode", async () => {
hooks.resume.mockRejectedValueOnce(new Error("temporary database failure"));
hooks.schedule!("failed-follow-up", new Date());
await vi.advanceTimersByTimeAsync(0);
expect(hooks.resume).toHaveBeenCalledOnce();
await vi.advanceTimersByTimeAsync(30_000);
expect(hooks.resume).toHaveBeenCalledTimes(2);
expect(hooks.restart).not.toHaveBeenCalled();
expect(vi.getTimerCount()).toBe(0);
});
});
+317 -5
View File
@@ -1,6 +1,8 @@
import { randomUUID } from "node:crypto";
import { spawn } from "node:child_process";
import { once } from "node:events";
import type { Db } from "@paperclipai/db";
import { activityLog, agents, applyPendingMigrations, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, environments, environmentLeases, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, nativeRunnerCheckpoints } from "@paperclipai/db";
import { activityLog, agents, applyPendingMigrations, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, environments, environmentLeases, heartbeatRunEvents, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, nativeRunnerCheckpoints } from "@paperclipai/db";
import { eq } from "drizzle-orm";
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
@@ -19,7 +21,10 @@ import { logger } from "../../middleware/logger.js";
import { listExpiredNativeRunnerAuthorities } from "../native-runtime/native-runner-checkpoint.js";
import { DELIVERY_QUEUES, subscribeDeliveryWork } from "../delivery-work-notifications.js";
import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js";
import { hasRemoteTerminationReceipt } from "../remote-execution-termination.js";
import { hasRemoteTerminationReceipt, remoteTerminationReceipt } from "../remote-execution-termination.js";
import { persistHeartbeatRunProcessMetadata } from "./run-lifecycle.js";
import { appendHeartbeatRunEvent } from "../heartbeat-run-events.js";
import { hasNativeLocalProcessStop, PROCESS_START_REQUESTED, releaseSettledNativeRunProcess } from "../native-local-process-stop.js";
type Run = typeof heartbeatRuns.$inferSelect;
@@ -202,6 +207,302 @@ describePostgres("heartbeat recovery module ownership", () => {
return (await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, issueId, invocationSource: "automation", status: "running", runtimeMode: "legacy" }).returning())[0];
}
async function ownedProcess() {
const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)", `paperclip-recovery-${randomUUID()}`], {
stdio: "ignore", detached: true, env: { PATH: process.env.PATH },
});
await once(child, "spawn");
return {
pid: child.pid!,
async stop() {
if (child.exitCode !== null || child.signalCode !== null) return;
const exited = once(child, "exit");
child.kill("SIGTERM");
await exited;
},
alive() { return process.kill(child.pid!, 0); },
};
}
async function nativeProcessFixture(pid: number, targetKind: "local" | "remote", status: "running" | "failed") {
const run = await seedRun();
await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: run.issueId, status,
nativePhase: status === "failed" ? "retryable_failure" : "running",
errorCode: status === "failed" ? "native_session_interrupted" : null,
error: status === "failed" ? "native session timed out after 120000ms" : null,
finishedAt: status === "failed" ? new Date() : null,
}).where(eq(heartbeatRuns.id, run.id));
await db.insert(nativeRunFinalizations).values({ runId: run.id, companyId: run.companyId, issueId: run.issueId!,
phase: status === "failed" ? "retryable_failure" : "running", nextAttemptAt: new Date(Date.now() - 1) });
await persistHeartbeatRunProcessMetadata(db, run.id, { pid, processGroupId: pid,
startedAt: new Date().toISOString(), targetKind });
const deps = callbacks(db);
deps.setRunStatusFromLive.mockImplementation(async (id, nextStatus, fromStatuses, patch) => {
const [current] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id));
if (!current || !fromStatuses.includes(current.status)) return { updated: false, run: current ?? null };
const [updated] = await db.update(heartbeatRuns).set({ ...patch, status: nextStatus })
.where(eq(heartbeatRuns.id, id)).returning();
return { updated: true, run: updated };
});
return {
run, deps,
current: async () => (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0],
async cleanup() {
await Promise.all([...deps.activeRunExecutionPromises]);
await db.delete(environmentLeases).where(eq(environmentLeases.heartbeatRunId, run.id));
await db.delete(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id));
await db.delete(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, run.id));
await db.delete(heartbeatRuns).where(eq(heartbeatRuns.id, run.id));
},
};
}
it.each(["failed", "running"] as const)("does not attribute a remote %s run's PID to a live host process", async status => {
const child = await ownedProcess();
const f = await nativeProcessFixture(child.pid, "remote", status);
try {
await db.update(agents).set({ adapterType: "process" }).where(eq(agents.id, f.run.agentId));
const before = await f.current();
expect(await createHeartbeatRecovery(db, f.deps).reapOrphanedRuns()).toEqual({ reaped: 0, runIds: [] });
expect(await f.current()).toMatchObject({ status, errorCode: before.errorCode, error: before.error,
processPid: child.pid, processGroupId: child.pid, processStartedAt: before.processStartedAt });
expect(f.deps.setRunStatusFromLive).not.toHaveBeenCalled();
expect(f.deps.executeRun).not.toHaveBeenCalled();
expect(f.deps.terminateHeartbeatRunProcess).not.toHaveBeenCalled();
expect(await hasNativeLocalProcessStop(db, f.run.companyId, f.run.id)).toBe(false);
expect(child.alive()).toBe(true);
} finally { try { await f.cleanup(); } finally { await child.stop(); } }
});
async function remoteRetryFixture(pid: number, target: "workspace" | "plugin_task" = "workspace") {
const fixture = await nativeProcessFixture(pid, "remote", "failed");
const sessionId = randomUUID();
const [lease] = await db.insert(environmentLeases).values({ companyId: fixture.run.companyId,
heartbeatRunId: fixture.run.id, issueId: fixture.run.issueId, status: "active", provider: "daytona",
providerLeaseId: randomUUID(), metadata: target === "plugin_task" ? { runnerTaskLease: true } : {},
}).returning();
await db.update(agents).set({ adapterType: "paperclip_runner" }).where(eq(agents.id, fixture.run.agentId));
await db.update(issues).set({ executionRunId: fixture.run.id }).where(eq(issues.id, fixture.run.issueId!));
await db.update(heartbeatRuns).set({ nativeSessionId: sessionId, runnerProfileJson: {
sessionCheckpoint: { identity: { runId: fixture.run.id, companyId: fixture.run.companyId,
issueId: fixture.run.issueId, agentId: fixture.run.agentId, sessionId }, providerSessionId: "existing-provider" },
...(target === "workspace" ? { nativeWorkspaceSync: { schema: "paperclip.native-workspace-sync/v3", state: "prepared",
workspaceId: randomUUID(), leaseId: lease.id, providerLeaseId: lease.providerLeaseId, remoteCwd: "/work",
descriptorSha256: "a".repeat(64), baselineSha256: "b".repeat(64), finalHostSha256: null,
resourceDisposition: "keep_running" } } : {}),
} }).where(eq(heartbeatRuns.id, fixture.run.id));
await db.insert(nativeRunnerCheckpoints).values({ runId: fixture.run.id, companyId: fixture.run.companyId,
issueId: fixture.run.issueId!, part: "opaque-checkpoint-record", material: { fixture: true } });
fixture.deps.getRun.mockImplementation(async id => (await db.select().from(heartbeatRuns)
.where(eq(heartbeatRuns.id, id)))[0] ?? null);
native.claims.mockImplementation(claimNativeRestartRecoveries);
return { ...fixture, lease };
}
it.each(["failed", "running"] as const)("routes a due remote %s retry through verified recovery without local PID inspection", async status => {
const child = await ownedProcess();
const f = await remoteRetryFixture(child.pid);
const originalKill = process.kill.bind(process);
const kill = vi.spyOn(process, "kill").mockImplementation(originalKill);
try {
await db.update(heartbeatRuns).set({ status }).where(eq(heartbeatRuns.id, f.run.id));
await db.update(nativeRunFinalizations).set({ nextAttemptAt: null }).where(eq(nativeRunFinalizations.runId, f.run.id));
const recovery = createHeartbeatRecovery(db, f.deps);
await recovery.reapOrphanedRuns();
await Promise.all([...f.deps.activeRunExecutionPromises]);
expect(f.deps.executeRun).toHaveBeenCalledExactlyOnceWith(f.run.id, expect.objectContaining({
nativeRestartRecovery: expect.objectContaining({ kind: "reattach_remote_runner", runId: f.run.id,
remote: { providerLeaseId: f.lease.providerLeaseId, remoteCwd: "/work" } }),
}));
expect(native.claims).toHaveBeenCalledWith(expect.objectContaining({ runIds: [f.run.id] }));
expect(await f.current()).toMatchObject({ status: "running", processPid: child.pid, processGroupId: child.pid });
expect(kill.mock.calls.filter(([pid]) => Math.abs(pid as number) === child.pid)).toEqual([]);
expect(await hasNativeLocalProcessStop(db, f.run.companyId, f.run.id)).toBe(false);
await recovery.reapOrphanedRuns();
expect(f.deps.executeRun).toHaveBeenCalledOnce();
expect(f.deps.terminateHeartbeatRunProcess).not.toHaveBeenCalled();
kill.mockRestore();
expect(child.alive()).toBe(true);
} finally { kill.mockRestore(); try { await f.cleanup(); } finally { await child.stop(); } }
});
it("routes a due plugin-task retry through its existing remote lease", async () => {
const child = await ownedProcess();
const f = await remoteRetryFixture(child.pid, "plugin_task");
try {
await createHeartbeatRecovery(db, f.deps).reapOrphanedRuns();
await Promise.all([...f.deps.activeRunExecutionPromises]);
expect(f.deps.executeRun).toHaveBeenCalledExactlyOnceWith(f.run.id, expect.objectContaining({
nativeRestartRecovery: expect.objectContaining({ kind: "reattach_remote_runner", runId: f.run.id,
remote: { kind: "plugin_task", leaseId: f.lease.id } }),
}));
expect(await f.current()).toMatchObject({ processPid: child.pid, processGroupId: child.pid });
expect(await hasNativeLocalProcessStop(db, f.run.companyId, f.run.id)).toBe(false);
expect(child.alive()).toBe(true);
} finally { try { await f.cleanup(); } finally { await child.stop(); } }
});
it.each(["future_retry", "live_owner", "missing_checkpoint", "active_execution"] as const)(
"does not execute a remote retry without current authority: %s", async condition => {
const child = await ownedProcess();
const f = await remoteRetryFixture(child.pid);
try {
if (condition === "future_retry") await db.update(nativeRunFinalizations)
.set({ nextAttemptAt: new Date(Date.now() + 60_000) }).where(eq(nativeRunFinalizations.runId, f.run.id));
if (condition === "live_owner") await db.update(nativeRunFinalizations)
.set({ leaseOwner: "other-controller", leaseExpiresAt: new Date(Date.now() + 60_000) })
.where(eq(nativeRunFinalizations.runId, f.run.id));
if (condition === "missing_checkpoint") {
const run = await f.current();
await db.update(heartbeatRuns).set({ runnerProfileJson: { ...run.runnerProfileJson, sessionCheckpoint: null } })
.where(eq(heartbeatRuns.id, f.run.id));
}
if (condition === "active_execution") f.deps.activeRunExecutions.add(f.run.id);
await createHeartbeatRecovery(db, f.deps).reapOrphanedRuns();
expect(f.deps.executeRun).not.toHaveBeenCalled();
expect(f.deps.terminateHeartbeatRunProcess).not.toHaveBeenCalled();
expect(await f.current()).toMatchObject({ processPid: child.pid, processGroupId: child.pid });
if (condition === "future_retry") expect(native.claims).not.toHaveBeenCalled();
if (condition === "missing_checkpoint") {
const [coordinator] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.run.id));
expect(coordinator).toMatchObject({ phase: "terminal_failure", recoveryState: "blocked",
failureCode: "native_restart_recovery_blocked" });
}
expect(child.alive()).toBe(true);
} finally { f.deps.activeRunExecutions.delete(f.run.id); try { await f.cleanup(); } finally { await child.stop(); } }
},
);
it.each(["unknown", "remote"] as const)("exposes expired observed %s ownership without attributing host liveness", async target => {
const child = await ownedProcess();
const f = await nativeProcessFixture(child.pid, "remote", "running");
const originalKill = process.kill.bind(process);
let restoreKill: (() => void) | undefined;
try {
if (target === "unknown") await db.delete(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, f.run.id));
await db.update(heartbeatRuns).set({ nativePhase: null })
.where(eq(heartbeatRuns.id, f.run.id));
await db.update(nativeRunFinalizations).set({ phase: "observed", attempt: 2,
leaseOwner: "prior-native-owner", leaseExpiresAt: new Date(0), nextAttemptAt: null })
.where(eq(nativeRunFinalizations.runId, f.run.id));
const before = await f.current();
const coordinatorBefore = await db.select().from(nativeRunFinalizations)
.where(eq(nativeRunFinalizations.runId, f.run.id));
const kill = vi.spyOn(process, "kill").mockImplementation(originalKill);
restoreKill = () => kill.mockRestore();
expect(await createHeartbeatRecovery(db, f.deps).reapOrphanedRuns()).toEqual({ reaped: 0, runIds: [] });
expect(kill.mock.calls.filter(([pid]) => Math.abs(pid as number) === child.pid)).toEqual([]);
expect(await f.current()).toMatchObject({ status: "running", nativePhase: null,
processPid: child.pid, processGroupId: child.pid, processStartedAt: before.processStartedAt,
errorCode: NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE });
expect(f.deps.appendRunEvent).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({
payload: { reason: "observed_owner_unverified" },
}));
expect(await db.select().from(nativeRunFinalizations)
.where(eq(nativeRunFinalizations.runId, f.run.id))).toEqual(coordinatorBefore);
expect(f.deps.executeRun).not.toHaveBeenCalled();
expect(f.deps.terminateHeartbeatRunProcess).not.toHaveBeenCalled();
kill.mockRestore(); restoreKill = undefined;
expect(child.alive()).toBe(true);
} finally { restoreKill?.(); try { await f.cleanup(); } finally { await child.stop(); } }
});
it.each(["failed", "running"] as const)("protects a verified local %s run's live process", async status => {
const child = await ownedProcess();
const f = await nativeProcessFixture(child.pid, "local", status);
try {
await db.update(agents).set({ adapterType: "paperclip_runner", adapterConfig: { driver: "daytona" } })
.where(eq(agents.id, f.run.agentId));
await createHeartbeatRecovery(db, f.deps).reapOrphanedRuns();
expect(await f.current()).toMatchObject({ status, processPid: child.pid, processGroupId: child.pid,
errorCode: NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE });
expect(f.deps.appendRunEvent).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({
payload: { reason: "live_process_identifier", processPidAlive: true, processGroupAlive: true },
}));
expect(f.deps.executeRun).not.toHaveBeenCalled();
expect(f.deps.terminateHeartbeatRunProcess).not.toHaveBeenCalled();
expect(child.alive()).toBe(true);
} finally { try { await f.cleanup(); } finally { await child.stop(); } }
});
it.each(["missing", "new_launch", "wrong_birth", "foreign_host", "wrong_agent", "wrong_pid", "wrong_group", "native_source", "remote_workspace", "remote_lease"] as const)(
"holds a native PID without local authority: %s", async evidence => {
const child = await ownedProcess();
await child.stop();
const f = await nativeProcessFixture(child.pid, "local", "failed");
try {
if (evidence === "missing") await db.delete(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, f.run.id));
if (evidence === "new_launch") await appendHeartbeatRunEvent(db, { companyId: f.run.companyId,
runId: f.run.id, agentId: f.run.agentId, eventType: PROCESS_START_REQUESTED });
if (["wrong_birth", "foreign_host", "wrong_agent", "wrong_pid", "wrong_group", "native_source"].includes(evidence)) {
const [event] = await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, f.run.id));
const foreignAgentId = randomUUID();
if (evidence === "wrong_agent") await db.insert(agents).values({ id: foreignAgentId,
companyId: f.run.companyId, name: "Other process owner", role: "engineer", adapterType: "process" });
await db.update(heartbeatRunEvents).set({
...(evidence === "wrong_agent" ? { agentId: foreignAgentId } : {}),
...(evidence === "native_source" ? { sourceEventId: "provider-event" } : {}),
payload: { ...event.payload, ...(evidence === "wrong_birth"
? { processStartedAt: new Date(0).toISOString() } : evidence === "foreign_host" ? { hostname: "foreign-host" }
: evidence === "wrong_pid" ? { processPid: child.pid + 1 }
: evidence === "wrong_group" ? { processGroupId: child.pid + 1 } : {}) },
}).where(eq(heartbeatRunEvents.id, event.id));
}
if (evidence === "remote_workspace") await db.update(heartbeatRuns).set({
runnerProfileJson: { nativeWorkspaceSync: { schema: "paperclip.native-workspace-sync/v3", state: "prepared",
workspaceId: randomUUID(), leaseId: randomUUID(), providerLeaseId: randomUUID(), remoteCwd: "/work",
descriptorSha256: "a".repeat(64), baselineSha256: "b".repeat(64), finalHostSha256: null, resourceDisposition: "destroy" } },
}).where(eq(heartbeatRuns.id, f.run.id));
if (evidence === "remote_lease") await db.insert(environmentLeases).values({ companyId: f.run.companyId,
heartbeatRunId: f.run.id, issueId: f.run.issueId, status: "active", provider: "daytona", providerLeaseId: randomUUID() });
const before = await f.current();
await createHeartbeatRecovery(db, f.deps).reapOrphanedRuns();
expect(await f.current()).toMatchObject({ processPid: child.pid, processGroupId: child.pid,
processStartedAt: before.processStartedAt, errorCode: "native_session_interrupted" });
expect(f.deps.executeRun).not.toHaveBeenCalled();
expect(f.deps.terminateHeartbeatRunProcess).not.toHaveBeenCalled();
expect(await hasNativeLocalProcessStop(db, f.run.companyId, f.run.id)).toBe(false);
} finally { await f.cleanup(); }
},
);
it("clears a verified stopped local retry identity and records its stop", async () => {
const child = await ownedProcess();
await child.stop();
const f = await nativeProcessFixture(child.pid, "local", "failed");
try {
await createHeartbeatRecovery(db, f.deps).reapOrphanedRuns();
expect(await f.current()).toMatchObject({ processPid: null, processGroupId: null, processStartedAt: null });
expect(await hasNativeLocalProcessStop(db, f.run.companyId, f.run.id)).toBe(true);
expect(f.deps.terminateHeartbeatRunProcess).not.toHaveBeenCalled();
} finally { await f.cleanup(); }
});
it("retires a remote identity only with an exact authenticated termination receipt", async () => {
const child = await ownedProcess();
const f = await nativeProcessFixture(child.pid, "remote", "failed");
try {
await db.update(heartbeatRuns).set({ runnerProfileJson: { sessionCheckpoint: { fixture: true } } })
.where(eq(heartbeatRuns.id, f.run.id));
await db.update(nativeRunFinalizations).set({ phase: "terminal_failure" })
.where(eq(nativeRunFinalizations.runId, f.run.id));
const [lease] = await db.insert(environmentLeases).values({ companyId: f.run.companyId,
heartbeatRunId: f.run.id, issueId: f.run.issueId, status: "active", provider: "daytona", providerLeaseId: randomUUID() }).returning();
expect(await releaseSettledNativeRunProcess(db, f.run.id)).toBe("pending");
const receipt = remoteTerminationReceipt(lease, { state: "destroyed", providerLeaseId: lease.providerLeaseId })!;
await db.update(environmentLeases).set({ status: "released", cleanupStatus: "success", releasedAt: new Date(),
metadata: { remoteExecutionTermination: { ...receipt, runId: randomUUID() } } }).where(eq(environmentLeases.id, lease.id));
expect(await releaseSettledNativeRunProcess(db, f.run.id)).toBe("pending");
expect(await f.current()).toMatchObject({ processPid: child.pid, processGroupId: child.pid });
await db.update(environmentLeases).set({ metadata: { remoteExecutionTermination: receipt } })
.where(eq(environmentLeases.id, lease.id));
expect(await releaseSettledNativeRunProcess(db, f.run.id)).toBe("released");
expect(await f.current()).toMatchObject({ processPid: null, processGroupId: null, processStartedAt: null });
expect(await hasNativeLocalProcessStop(db, f.run.companyId, f.run.id)).toBe(false);
expect(child.alive()).toBe(true);
} finally { try { await f.cleanup(); } finally { await child.stop(); } }
});
it.each(["ephemeral", "reuse_by_environment", "live_owner", "live_executor", "mismatched_source", "finalizing"] as const)(
"reconstructs missing native preservation during orphan cleanup without replay: %s", async state => {
const run = await seedRun();
@@ -857,11 +1158,12 @@ describePostgres("heartbeat recovery module ownership", () => {
await Promise.all([...deps.activeRunExecutionPromises]);
});
it("rearms the persisted native retry deadline through the service callback", async () => {
it.each(["future", "overdue-owner"])("rearms the persisted native retry deadline through the service callback: %s", async scenario => {
const run = await seedRun();
const nextAttemptAt = new Date(Date.now() + 60_000);
const nextAttemptAt = new Date(Date.now() + (scenario === "future" ? 60_000 : -1_000));
await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: run.issueId, status: "failed" }).where(eq(heartbeatRuns.id, run.id));
await db.insert(nativeRunFinalizations).values({ runId: run.id, companyId: run.companyId, issueId: run.issueId!, phase: "retryable_failure", nextAttemptAt });
await db.insert(nativeRunFinalizations).values({ runId: run.id, companyId: run.companyId, issueId: run.issueId!, phase: "retryable_failure", nextAttemptAt,
failureCode: scenario === "overdue-owner" ? "native_controller_recovery_waiting_for_owner" : null });
const deps = callbacks(db);
const result = await createHeartbeatRecovery(db, deps).recoverNativeRunsAfterRestart();
expect(result.scheduledRetryRunIds).toEqual([run.id]);
@@ -872,6 +1174,16 @@ describePostgres("heartbeat recovery module ownership", () => {
await db.delete(heartbeatRuns).where(eq(heartbeatRuns.id, run.id));
});
it("retains a live remote owner's actual lease deadline without dispatching competing work", async () => {
const deps = callbacks(db);
const runId = randomUUID(), retryAt = new Date(Date.now() + 60_000);
native.claims.mockResolvedValue([{ kind: "awaiting_evidence", runId, reason: "remote_controller_lease_live", retryAt }]);
const result = await createHeartbeatRecovery(db, deps).recoverNativeRunsAfterRestart();
expect(result.awaitingEvidenceRunIds).toEqual([runId]);
expect(deps.scheduleNativeSessionResumeDispatch).toHaveBeenCalledWith(runId, retryAt, "restart");
expect(deps.executeRun).not.toHaveBeenCalled();
});
it("retains shutdown ownership until background native cleanup physically settles", async () => {
const deps = callbacks(db);
let settle!: () => void;
+66 -14
View File
@@ -1,3 +1,4 @@
import { NATIVE_RETAINED_OWNER_WAIT } from "../native-runtime/native-controller-recovery.js";
import { nativeEnvironmentSettlementDisposition, nativeEnvironmentSettlementWork } from "../native-runtime/native-environment-settlement-work.js";
import { nativeWorkspaceFinalizationWork } from "../native-runtime/native-workspace-finalization-work.js";
import { attemptNativeWorkspaceFinalization } from "../native-runtime/native-workspace-finalizer.js";
@@ -18,7 +19,7 @@ import {
hasStopOnlyCleanup,
settleStopOnlyCleanup,
} from "../sandbox-stop-and-retain.js";
import { discoverSettledNativeRunProcesses, recordNativeLocalProcessStop, retryPendingSettledNativeRunProcesses } from "../native-local-process-stop.js";
import { discoverSettledNativeRunProcesses, PROCESS_IDENTITY_RECORDED, PROCESS_START_REQUESTED, recordNativeLocalProcessStop, retryPendingSettledNativeRunProcesses } from "../native-local-process-stop.js";
import { pendingCaseHandoffForRun } from "./case-handoff-lifecycle.js";
import {
legacyControllerBootId,
@@ -30,9 +31,11 @@ import { runUsedConversationAdapter } from "../conversation-continuation.js";
import { legacyExecutionNeedsReconciliationWithEvidence, terminalizeLegacyExecution } from "../legacy-execution-recovery.js";
import { reconcileRetainedWorkspaces } from "../workspace-recovery.js";
import { randomUUID } from "node:crypto";
import os from "node:os";
import {
and,
asc,
desc,
eq,
gt,
inArray,
@@ -46,6 +49,7 @@ import type { Db } from "@paperclipai/db";
import {
agents,
environmentLeases,
heartbeatRunEvents,
heartbeatRuns,
issues,
nativeRunFinalizations,
@@ -123,7 +127,7 @@ export interface HeartbeatRecoveryDependencies extends Pick<HeartbeatRetryDepend
nativeRestartRecovery?: NativeRestartRecoveryClaim;
onNativeExecutionLeaseAcquired?: () => Promise<void>;
}) => Promise<void>;
scheduleNativeSessionResumeDispatch: (runId: string, nextAttemptAt: Date) => void;
scheduleNativeSessionResumeDispatch: (runId: string, nextAttemptAt: Date, mode?: "restart") => void;
cancelHeartbeatNativeRun: (input: {
db: Db; runId: string; reason: string; runtimeMode: string | null;
}) => Promise<unknown>;
@@ -953,7 +957,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
inArray(heartbeatRuns.status, ["running", "failed"]),
isNull(nativeRunFinalizations.resultId),
eq(nativeRunFinalizations.phase, "retryable_failure"),
gt(nativeRunFinalizations.nextAttemptAt, now),
or(gt(nativeRunFinalizations.nextAttemptAt, now), eq(nativeRunFinalizations.failureCode, NATIVE_RETAINED_OWNER_WAIT)),
),
);
for (const scheduled of scheduledNativeRetries) {
@@ -1001,6 +1005,9 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
disposition.kind === "bootstrap_incomplete",
);
for (const disposition of dispositions) {
if (disposition.kind === "awaiting_evidence" && disposition.retryAt) {
scheduleNativeSessionResumeDispatch(disposition.runId, disposition.retryAt, "restart");
}
const run = await getRun(disposition.runId);
if (run) {
const isClaim =
@@ -1836,6 +1843,29 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
return { swept: rows.length, destroyed, capped };
}
async function nativeProcessIsLocal(run: HeartbeatRun) {
if (parseObject(run.runnerProfileJson).nativeWorkspaceSync != null) return false;
const [identity] = await db.select({
eventType: heartbeatRunEvents.eventType,
agentId: heartbeatRunEvents.agentId,
payload: heartbeatRunEvents.payload,
}).from(heartbeatRunEvents).where(and(
eq(heartbeatRunEvents.companyId, run.companyId),
eq(heartbeatRunEvents.runId, run.id),
isNull(heartbeatRunEvents.sourceEventId),
inArray(heartbeatRunEvents.eventType, [PROCESS_START_REQUESTED, PROCESS_IDENTITY_RECORDED]),
)).orderBy(desc(heartbeatRunEvents.seq)).limit(1);
const recorded = identity?.payload;
if (identity?.eventType !== PROCESS_IDENTITY_RECORDED || identity.agentId !== run.agentId ||
recorded?.targetKind !== "local" || recorded.hostname !== os.hostname() ||
recorded.processPid !== run.processPid || recorded.processGroupId !== run.processGroupId ||
!run.processStartedAt || recorded.processStartedAt !== run.processStartedAt.toISOString()) return false;
const leases = await db.select({ provider: environmentLeases.provider }).from(environmentLeases).where(and(
eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id),
));
return leases.every(lease => lease.provider === "local");
}
async function markNativeOwnershipUnverified(
run: typeof heartbeatRuns.$inferSelect,
evidence: {
@@ -2030,13 +2060,12 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
);
});
// A retryable native run can retain process identifiers from the failed
// attempt. Inspect them before the recovery claim: a live identifier is
// unowned and blocks recovery, while identifiers that are all dead can be
// cleared with a compare-and-set so the explicit retryable failure becomes
// claimable in this same sweep.
// Retryable runs can retain process identifiers from the failed attempt.
// Only verified local identities may be checked and cleared on this host.
// Due remote retries instead re-enter the existing restart-recovery owner,
// which verifies the saved checkpoint, controller lease, and remote target.
const retryableNativeProcesses = await db
.select({ run: heartbeatRuns })
.select({ run: heartbeatRuns, nextAttemptAt: nativeRunFinalizations.nextAttemptAt })
.from(heartbeatRuns)
.innerJoin(
nativeRunFinalizations,
@@ -2051,12 +2080,27 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
),
);
const claimableNativeRunIds = new Set<string>();
for (const { run } of retryableNativeProcesses) {
const remoteRetryRunIds: string[] = [];
for (const { run, nextAttemptAt } of retryableNativeProcesses) {
if (isNativeRunnerOwnershipHeld(run)) continue;
if (!run.processPid && !run.processGroupId) {
claimableNativeRunIds.add(run.id);
continue;
}
if (!await nativeProcessIsLocal(run)) {
if (nextAttemptAt && nextAttemptAt > now) continue;
if (activeRunExecutions.has(run.id) || runningProcesses.has(run.id)) continue;
if (parseObject(run.runnerProfileJson).nativeWorkspaceSync != null) {
remoteRetryRunIds.push(run.id);
} else {
const [taskLease] = await db.select({ id: environmentLeases.id }).from(environmentLeases).where(and(
eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id),
sql`${environmentLeases.metadata}->>'runnerTaskLease' = 'true'`,
)).limit(1);
if (taskLease) remoteRetryRunIds.push(run.id);
}
continue;
}
const processPidAlive =
!!run.processPid && isProcessAlive(run.processPid);
const processGroupAlive =
@@ -2082,6 +2126,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
and(
eq(heartbeatRuns.id, run.id),
eq(heartbeatRuns.runtimeMode, "native"),
eq(heartbeatRuns.nextEventSeq, run.nextEventSeq),
run.processPid === null
? isNull(heartbeatRuns.processPid)
: eq(heartbeatRuns.processPid, run.processPid),
@@ -2101,6 +2146,10 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
if (cleared) claimableNativeRunIds.add(cleared.id);
}
const remoteRetryClaims = remoteRetryRunIds.length
? (await recoverNativeRunsAfterRestart(now, remoteRetryRunIds)).claims
: [];
const executingRun = (runId: string) => activeRunExecutions.has(runId) || runningProcesses.has(runId);
if (!settledProcessesDiscovered) {
settledProcessesDiscovered = await discoverSettledNativeRunProcesses(db, { skip: executingRun }).then(() => true, (error) => {
@@ -2148,7 +2197,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
return [];
});
const resumedRunIds = new Set(
nativeResumeClaims.map((claim) => claim.runId),
[...nativeResumeClaims, ...remoteRetryClaims].map((claim) => claim.runId),
);
// A terminal issue transition writes this intent in the same transaction
@@ -2300,10 +2349,12 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
// preserves its explicit physical-copyback hold. Keep its source lease.
if (nativeWorkspaceFinalizationPending) continue;
const nativeRun = run.runtimeMode === "native";
const nativeLocalProcess = nativeRun && !!(run.processPid || run.processGroupId) &&
await nativeProcessIsLocal(run);
const nativeProcessPidAlive =
nativeRun && !!run.processPid && isProcessAlive(run.processPid);
nativeLocalProcess && !!run.processPid && isProcessAlive(run.processPid);
const nativeProcessGroupAlive =
nativeRun &&
nativeLocalProcess &&
!!run.processGroupId &&
isProcessGroupAlive(run.processGroupId);
const coordinatorOwnedByCurrentController =
@@ -2360,6 +2411,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
});
continue;
}
if (nativeRun && !locallyTracked && (run.processPid || run.processGroupId) && !nativeLocalProcess) continue;
if (resumedRunIds.has(run.id)) continue;
if (locallyTracked) continue;
if (await hasLiveLegacyController(db, run)) continue;
@@ -2379,7 +2431,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
// for a read-only liveness check so a lost in-memory handle cannot cause
// overlapping provider/tool execution while that child is still alive.
const checksPersistedChildLiveness =
currentAdapterTracksLocalChild || run.runtimeMode === "native";
nativeRun ? nativeLocalProcess : currentAdapterTracksLocalChild;
const processPidAlive =
checksPersistedChildLiveness &&
run.processPid &&
@@ -309,9 +309,16 @@ describe.skipIf(!support.supported && !process.env.PAPERCLIP_TEST_DATABASE_URL)(
it("retains native completion metadata when reconciliation already committed the same terminal status", async () => {
const finishedAt = new Date("2026-10-09T12:00:00Z");
const f = await fixture({ runtimeMode: "native", status: "succeeded", finishedAt, resultJson: { durable: "accepted" }, usageJson: { durableReceipt: "native" } });
f.input.adapterResult.nativeFinalization = { schema: "paperclip.native-finalization.v1", runtimeMode: "native", runId: f.run.id, companyId: f.run.companyId, issueId: "unused", result: {}, terminal: { schema: "paperclip.prp.terminal.v1", turnTerminalState: "completed", runTerminalState: "succeeded", reportedWorkDisposition: "done" }, turnId: null, sourceInstanceId: "test", normalizedSessionId: "native-session", providerSessionId: null, driverKind: "test", driverVersion: "1", nativeEventCount: 1, highestContiguousSourceSeq: 1, workspaceFinalizeStatus: "succeeded" };
await db.update(agents).set({ adapterType: "paperclip_runner" }).where(eq(agents.id, f.agent.id));
f.agent.adapterType = "paperclip_runner";
f.input.sessionCodec = getAdapterSessionCodec(f.agent.adapterType);
const nativeSessionId = randomUUID(), providerSessionId = randomUUID();
f.input.adapterResult.sessionId = nativeSessionId;
delete f.input.adapterResult.sessionParams;
f.input.adapterResult.sessionDisplayId = providerSessionId;
f.input.adapterResult.nativeFinalization = { schema: "paperclip.native-finalization.v1", runtimeMode: "native", runId: f.run.id, companyId: f.run.companyId, issueId: "unused", result: {}, terminal: { schema: "paperclip.prp.terminal.v1", turnTerminalState: "completed", runTerminalState: "succeeded", reportedWorkDisposition: "done" }, turnId: null, sourceInstanceId: "test", normalizedSessionId: nativeSessionId, providerSessionId, driverKind: "test", driverVersion: "1", nativeEventCount: 1, highestContiguousSourceSeq: 1, workspaceFinalizeStatus: "succeeded" };
await f.completion.completeRun(f.input);
expect(await read(f.run.id)).toMatchObject({ status: "succeeded", finishedAt, sessionIdAfter: "provider-session", resultJson: { durable: "accepted" }, usageJson: { durableReceipt: "native", inputTokens: 20 } });
expect(await read(f.run.id)).toMatchObject({ status: "succeeded", finishedAt, sessionIdAfter: providerSessionId, resultJson: { durable: "accepted" }, usageJson: { durableReceipt: "native", inputTokens: 20 } });
expect(f.deps.updateRuntimeState).toHaveBeenCalledOnce();
});
@@ -1,4 +1,5 @@
import { preserveAcceptedRunUsageProjectionSql } from "../usage-receipts.js";
import { nativeTaskSessionPublicationHandled } from "../native-runtime/native-task-session-publication.js";
import { notifyChatRunMilestones } from "../chat-run-milestone-notifications.js";
import { measureRunPhase } from "../execution-preparation-timing.js";
import { PROJECT_ACCESS_DENIED, ProjectAccessDeniedError } from "../project-access-denial.js";
@@ -489,6 +490,13 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC
normalizedUsage,
);
if (!taskKey) return;
if (nativeTaskSessionPublicationHandled(finalizedRun.runnerProfileJson, {
runId: finalizedRun.id,
companyId: finalizedRun.companyId,
agentId: agent.id,
taskKey,
adapterType: agent.adapterType,
})) return;
if (
adapterResult.clearSession ||
(!nextSessionState.params && !nextSessionState.displayId)
@@ -1,3 +1,5 @@
import { NativeControllerRecoveryWaitingError } from "../native-runtime/native-controller-recovery.js";
import { createNativeUsageDelivery } from "../native-runtime/native-usage-delivery.js";
import type { PrpEvent } from "../../vendor/paperclip-runner/index.js";
import { randomUUID } from "node:crypto";
@@ -10,7 +12,7 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vites
import { trace } from "@opentelemetry/api";
import { InMemorySpanExporter, SimpleSpanProcessor, TracerProvider } from "@opentelemetry/sdk-trace";
import { createRunPreparationTiming, measureRunPhase, measureWorkspaceSyncPhase, withRunExecutionTiming } from "../execution-preparation-timing.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
import { recoveryPostgresSupport as support, startRecoveryPostgres } from "../../__tests__/helpers/recovery-postgres.js";
import * as pluginWorkspaces from "../plugin-workspaces.js";
import * as legacyRecovery from "../legacy-execution-recovery.js";
import { createAgentIdentityRedactor } from "../agent-identity-redaction.js";
@@ -43,15 +45,14 @@ vi.mock("./run-preparation.js", async original => ({
revokeHeartbeatRunGatewayTokens: vi.fn(async () => {}),
}));
const support = await getEmbeddedPostgresTestSupport();
describe.skipIf(!support.supported)("heartbeat runtime execution boundary", () => {
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let database: Awaited<ReturnType<typeof startRecoveryPostgres>>;
let db: Db;
let home: string;
beforeAll(async () => {
home = await mkdtemp(join(tmpdir(), "paperclip-runtime-execution-"));
vi.stubEnv("PAPERCLIP_HOME", home);
database = await startEmbeddedPostgresTestDatabase("paperclip-runtime-execution");
database = await startRecoveryPostgres("paperclip-runtime-execution");
db = createDb(database.connectionString);
await db.execute(sql`set client_min_messages = warning`);
}, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
@@ -283,6 +284,18 @@ describe.skipIf(!support.supported)("heartbeat runtime execution boundary", () =
expectCleanup(input);
});
it("preserves the retained sandbox while native admission waits for its current owner", async () => {
const { input } = await fixture(true);
const error = new NativeControllerRecoveryWaitingError(new Date(Date.now() + 60_000));
vi.mocked(executePaperclipNativeSession).mockRejectedValueOnce(error);
await expect(executeHeartbeatRuntime(db, input)).rejects.toMatchObject({
name: "NativeSessionResumeScheduledError", original: error,
});
expect(input.effects.onNativeSessionResumeScheduled).toHaveBeenCalledOnce();
expect(input.effects.onProviderResourceDisposition).not.toHaveBeenCalledWith("stop_and_retain");
expect(await finalizations()).toHaveLength(0);
});
it("records native restoration inside the workspace finalization operation", async () => {
const { input, run } = await fixture(true);
const endpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
@@ -1,3 +1,5 @@
import { NativeControllerRecoveryWaitingError } from "../native-runtime/native-controller-recovery.js";
import { usesNativeUsageDelivery } from "../native-runtime/native-usage-accounting.js";
import { DELIVERY_QUEUES } from "../delivery-work-notifications.js";
import { writeExecutionControl } from "../execution-control-notifications.js";
@@ -559,6 +561,20 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
sessionGoalControl.requestId,
));
if (completedGoalControl) sessionGoalControl = null;
// Goal checkpoints already publish their task session. Goal-less turns
// need the same durable pointer before cleanup can delay host completion.
const taskSessionPublication = taskKey && !sessionGoalControl &&
context.resumeSessionGoalHeartbeat !== true && !completedGoalControl
? {
taskKey,
adapterType: agent.adapterType,
sessionParamsJson: attachPaperclipSessionMetadataToSessionParams(
{ ...runtimeSessionParamsForAdapter, cwd: executionWorkspace.cwd },
configuredModel,
sessionConfigMetadata,
),
}
: undefined;
const nativeDispatchAtMs = Date.now();
const runCreatedAtMs = run.createdAt.getTime();
const runStartedAtMs = (run.startedAt ?? run.createdAt).getTime();
@@ -608,6 +624,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
resumeSessionGoalHeartbeat:
context.resumeSessionGoalHeartbeat === true ||
completedGoalControl,
taskSessionPublication,
onGoalCheckpoint: async (snapshot) => {
if (!taskKey) return;
const params =
@@ -1026,6 +1043,10 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
preserveRunGatewayTokens = true;
throw adapterErr;
}
if (adapterErr instanceof NativeControllerRecoveryWaitingError) {
input.effects.onNativeSessionResumeScheduled();
throw new NativeSessionResumeScheduledError(adapterErr);
}
if (adapterErr instanceof NativeRunnerOwnershipUnverifiedError) {
input.effects.onNativeOwnershipHeld();
throw adapterErr;
@@ -4,6 +4,7 @@ import { inspectNativeRunnerUpgradeHeader } from "../native-runtime/native-runne
import { readNativeWorkspaceSyncReference } from "../native-runtime/native-workspace-sync.js";
import { REMOTE_RUNNER_RECONNECT_GRACE_MS } from "../native-runtime/native-runner-recovery-contract.js";
import { currentNativeControllerIdentity } from "../native-runtime/native-restart-recovery.js";
import { readNativeControllerRecoveryInventory } from "../native-runtime/native-controller-recovery-inventory.js";
const MAX_UPGRADE_RUNS = 1_000;
type Counters = { activeRuns: number; pendingWakes: number; draining: boolean };
@@ -60,8 +61,16 @@ export async function readUpgradeReadiness(db: Db, input: {
&& identity.runnerInstanceId === row.runnerId && identity.normalizedSessionId === row.sessionId
&& identity.environmentLeaseId === (row.pluginLeaseId ?? workspace?.workspaceId)) };
});
return summarizeUpgradeReadiness({ rows: candidates, before, after: input.counters(),
// Additive evidence does not strengthen or redefine the existing ready flag.
const retained = await readNativeControllerRecoveryInventory(db).catch(() => ({
version: 1 as const, observedAt: new Date().toISOString(), coverage: "unknown" as const,
reasons: { inventory_unavailable: 1 }, fullPayloadsValidated: false as const,
}));
const summary = summarizeUpgradeReadiness({ rows: candidates, before, after: input.counters(),
activeRunIds: [...new Set([...beforeIds, ...input.activeRunIds])], controllerBootId: owner.bootId });
return { ...summary, nativeControllerRecovery: { activeExecutionAdoption: {
recoverableRuns: summary.recoverableRuns, blockingRuns: summary.blockingRuns, truncated: summary.truncated,
}, retained } };
}
export function summarizeUpgradeReadiness(input: {
+26 -7
View File
@@ -39,7 +39,7 @@ import { requestRestorePointDeletion } from "./git-restore-point-ledger.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
import { recordChatHandoff, recordAcceptedChatHandoff, lockAcceptedChatHandoffSource, recordChatCompletion, existingChatCompletionReply, acknowledgeChatCompletionReply } from "./chat-completion-delivery.js";
import { mirrorSlackBoardComment, slackBoardReplyBindings } from "./slack-board-messages.js";
import { assertAgentRunWriteAllowed } from "../agent-run-cancellation.js";
import { agentRunWritesRevoked, assertAgentRunWriteAllowed } from "../agent-run-cancellation.js";
import { assertLegacyRunTaskOwnership } from "../legacy-run-task-ownership.js";
import { retryIdempotentDatabaseOperation } from "../database-retry.js";
import { externalConversationStateSql, nonIdleSlackIssueCondition, resumeSlackConversation } from "./slack-conversation-state.js";
@@ -153,7 +153,7 @@ import {
isUuidLike,
normalizeIssueIdentifier as normalizeIssueReferenceIdentifier,
} from "@paperclipai/shared";
import { conflict, HttpError, notFound, unprocessable } from "../errors.js";
import { conflict, forbidden, HttpError, notFound, unprocessable } from "../errors.js";
import { isForeignKeyViolation } from "../db-errors.js";
import { logger } from "../middleware/logger.js";
import { parseObject } from "../adapters/utils.js";
@@ -1695,6 +1695,7 @@ function withAgentCommentAuthorizationMetadata(
const reason = authorizationReason?.trim() || "internal_agent_write";
return {
version: 1,
...(metadata?.chatMessage ? { chatMessage: metadata.chatMessage } : {}),
...(metadata?.sourceRunId !== undefined
? { sourceRunId: metadata.sourceRunId }
: {}),
@@ -8653,7 +8654,7 @@ export function issueService(db: Db) {
companyId: string,
issueId: string,
userId: string,
readAt: Date = new Date(),
readAt: Date | SQL = new Date(),
) => {
const now = new Date();
const [row] = await db
@@ -8672,7 +8673,7 @@ export function issueService(db: Db) {
issueReadStates.userId,
],
set: {
lastReadAt: readAt,
lastReadAt: sql`greatest(${issueReadStates.lastReadAt}, excluded.last_read_at)`,
updatedAt: now,
},
})
@@ -12937,6 +12938,7 @@ export function issueService(db: Db) {
sourceTrust?: typeof issueComments.$inferInsert.sourceTrust;
createdAt?: Date | string | null;
clientRequestId?: string;
proactiveChatMessage?: { sourceTaskId: string; userId: string };
/** Server-only: authenticated Paperclip messages also belong in the Slack thread. */
mirrorToSlack?: boolean;
},
@@ -12993,11 +12995,28 @@ export function issueService(db: Db) {
return existing;
}
}
if (options?.metadata?.chatMessage && !options.proactiveChatMessage) {
throw forbidden("Chat message provenance is server-owned");
}
if (issue.conversationAgentId && actor.agentId && issue.conversationAgentId !== actor.agentId) {
throw forbidden("Only this conversation's agent can post agent messages");
}
if (issue.conversationAgentId && actor.agentId && !actor.runId) {
throw forbidden("Agent chat messages require an authenticated run");
}
if (issue.conversationAgentId && actor.runId) {
const [run] = await dbOrTx.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, actor.runId));
const [current] = await dbOrTx.select().from(issues).where(eq(issues.id, issueId));
if (run?.status === "cancelled") throw conflict("This conversation turn was cancelled; it cannot post a reply");
if (run?.contextSnapshot?.conversationSessionGeneration !== current.conversationSessionGeneration) {
if (agentRunWritesRevoked(run)) throw conflict("This conversation turn was cancelled; it cannot post a reply");
const runTaskId = run?.nativeIssueId ?? run?.issueId ?? run?.contextSnapshot?.issueId ?? run?.contextSnapshot?.taskId;
if (options?.proactiveChatMessage) {
if (run?.status !== "running" || run.agentId !== actor.agentId || run.companyId !== issue.companyId
|| runTaskId !== options.proactiveChatMessage.sourceTaskId || runTaskId === issueId
|| current.conversationUserId !== options.proactiveChatMessage.userId
|| run.responsibleUserId !== current.conversationUserId) {
throw forbidden("Invalid background chat message authority");
}
} else if (runTaskId !== issueId || run?.contextSnapshot?.conversationSessionGeneration !== current.conversationSessionGeneration) {
throw conflict("Conversation session changed; this reply belongs to an earlier session");
}
}
@@ -13132,7 +13151,7 @@ export function issueService(db: Db) {
: (options?.metadata ?? null),
);
let comment: typeof issueComments.$inferSelect | null = null;
if (createdByRunId) {
if (createdByRunId && !options?.proactiveChatMessage) {
const existing = await dbOrTx
.select()
.from(issueComments)
@@ -0,0 +1,120 @@
import { randomUUID } from "node:crypto";
import { runAdapterExecutionTargetShellCommand, type AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target";
import { kimiMetadataProbeDigest, kimiMetadataProbeNodeDigest, probeKimiMetadata, validateKimiMetadata } from "../vendor/paperclip-runner/live/index.js";
import { unprocessable } from "../errors.js";
import { createSshCommandManagedRuntimeRunner, runSshCommand } from "@paperclipai/adapter-utils/ssh";
import { QUALIFIED_ACPX_PROFILES } from "../vendor/paperclip-runner/index.js";
import { resolveKimiCandidateAdmission } from "./native-runtime/acpx-qualification.js";
const REQUEST_FAILED = () => unprocessable("Kimi Code metadata verification failed.", { code: "KIMI_REQUEST_FAILED" });
const INSTALLATION_INVALID = () => unprocessable("Install the current verified Kimi Code runtime in this execution environment.", { code: "KIMI_INSTALLATION_INVALID" });
const SSH_INPUT_PROBE = 'try { const input = JSON.parse(require("node:fs").readFileSync(0, "utf8")); if (typeof input.apiKey !== "string" || typeof input.model !== "string") process.exit(1); process.env.KIMI_API_KEY = input.apiKey; process.env.PAPERCLIP_KIMI_PROBE_MODEL = input.model; if (input.candidateAdmission !== undefined) process.env.PAPERCLIP_ACPX_CANDIDATE_ADMISSION = JSON.stringify(input.candidateAdmission); require(process.argv[1]); } catch { process.exit(1); }';
export async function probeKimiExecutionTarget(apiKey: string, target: AdapterExecutionTarget | null | undefined, model: string, context: { companyId?: string } = {}) {
if (typeof apiKey !== "string" || !apiKey.trim() || /[\0\r\n]/.test(apiKey) || Buffer.byteLength(apiKey) > 16 * 1024) {
throw unprocessable("Bind a valid company secret to KIMI_API_KEY.", { code: "KIMI_AUTH_REQUIRED" });
}
if (typeof model !== "string" || !model.trim() || model.includes("\0") || Buffer.byteLength(model) > 512 || Buffer.from(model).toString("utf8") !== model) {
throw unprocessable("Select the exact Kimi Code model.", { code: "KIMI_MODEL_UNAVAILABLE" });
}
let candidateAdmission: Parameters<typeof probeKimiMetadata>[2];
if (QUALIFIED_ACPX_PROFILES.kimi.qualificationStatus === "pending") {
try {
const admission = resolveKimiCandidateAdmission({ companyId: context.companyId, model,
target: target?.kind === "remote" ? { platform: "linux", architecture: "x64" } : { platform: process.platform, architecture: process.arch } });
if (!admission) throw new Error("Missing candidate admission");
const probeId = `kimi-metadata-${randomUUID()}`;
candidateAdmission = { ...admission, runId: probeId, normalizedSessionId: `metadata-${randomUUID()}` };
} catch {
throw unprocessable("Kimi Code qualification requires current trusted operator admission for this company and execution target.", { code: "KIMI_QUALIFICATION_REQUIRED" });
}
}
if (!target || target.kind === "local") {
const result = validateKimiMetadata(await probeKimiMetadata(model, apiKey, candidateAdmission), model);
if (result.status !== "verified") throw unprocessable(result.message, { code: result.code });
return result;
}
if (target.transport === "sandbox" && target.runnerTask) {
throw unprocessable("Kimi Code metadata verification is unavailable in this hosting environment.", { code: "KIMI_INSTALLATION_INVALID" });
}
let locations = ["/opt/paperclip-runner/provider-pack", '"$HOME/.local/share/paperclip-runner/provider-pack"'];
const selectedPackRoot = process.env.PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH?.trim();
if (selectedPackRoot) {
try {
// Test uses the same complete-pack installer as native execution. It
// needs no native run/session or DB context and opens no model session.
const { prepareRemoteProviderPackOnTarget, resolveRemoteProviderPackSource } = await import("./native-runtime/native-session-executor.js");
const selected = resolveRemoteProviderPackSource(selectedPackRoot);
if (selected.manifest.payload.target.platform !== "linux" || selected.manifest.payload.target.architecture !== "x64") throw INSTALLATION_INVALID();
const runner = target.transport === "ssh" ? createSshCommandManagedRuntimeRunner({ spec: target.spec, defaultCwd: target.remoteCwd }) : target.runner;
if (!runner) throw INSTALLATION_INVALID();
const prepared = await prepareRemoteProviderPackOnTarget({ target, runner,
configuredProviderPackRoot: selected.configuredRoot, manifest: selected.manifest });
locations = [`'${prepared.root.replaceAll("'", "'\\''")}'`];
} catch { throw INSTALLATION_INVALID(); }
}
// Preparation retains the normal installer's bounded stages. The existing
// metadata/HTTP budget begins only after that preparation has completed.
const deadlineAt = Date.now() + 35_000;
let digest: string;
try { digest = await kimiMetadataProbeDigest(); }
catch { throw INSTALLATION_INVALID(); }
const nodeDigest = kimiMetadataProbeNodeDigest();
if (!/^sha256:[a-f0-9]{64}$/.test(digest) || !/^sha256:[a-f0-9]{64}$/.test(nodeDigest)) throw INSTALLATION_INVALID();
for (const location of locations) {
const remainingMs = deadlineAt - Date.now();
if (remainingMs <= 3_000) throw REQUEST_FAILED();
const command = [
`pack=${location}`,
'entry="$pack/dist/cli/kimi-metadata-probe.cjs"; node="$pack/provider-assets/kimi/linux-x64/runtime/node/bin/node"',
'[ -f "$pack/provider-pack.json" ] && [ -f "$entry" ] && [ ! -L "$entry" ] && [ -x "$node" ] && [ ! -L "$node" ] || exit 127',
'unset NODE_OPTIONS NODE_PATH NODE_EXTRA_CA_CERTS NODE_USE_ENV_PROXY NODE_TLS_REJECT_UNAUTHORIZED OPENSSL_CONF SSL_CERT_FILE SSL_CERT_DIR HTTP_PROXY HTTPS_PROXY ALL_PROXY NO_PROXY http_proxy https_proxy all_proxy no_proxy LD_PRELOAD LD_LIBRARY_PATH LD_AUDIT GLIBC_TUNABLES DYLD_INSERT_LIBRARIES DYLD_LIBRARY_PATH DYLD_FRAMEWORK_PATH',
'PATH=/usr/bin:/bin; export PATH',
'actual=$(/usr/bin/env -i /usr/bin/sha256sum "$entry" 2>/dev/null) || exit 126',
`actual=\${actual%% *}; [ "$actual" = "${digest.slice(7)}" ] || exit 126`,
'actual=$(/usr/bin/env -i /usr/bin/sha256sum "$node" 2>/dev/null) || exit 126',
`actual=\${actual%% *}; [ "$actual" = "${nodeDigest.slice(7)}" ] || exit 126`,
target.transport === "ssh"
? `exec /usr/bin/env -i PAPERCLIP_ACPX_PROVIDER_PACKAGE_ROOT="$pack" PAPERCLIP_ACPX_PROVIDER_PACKAGE_MANIFEST="$pack/package.json" "$node" -e '${SSH_INPUT_PROBE}' "$entry"`
: ['exec /usr/bin/env -i KIMI_API_KEY="$KIMI_API_KEY" PAPERCLIP_KIMI_PROBE_MODEL="$PAPERCLIP_KIMI_PROBE_MODEL"',
...(candidateAdmission ? ['PAPERCLIP_ACPX_CANDIDATE_ADMISSION="$PAPERCLIP_ACPX_CANDIDATE_ADMISSION"'] : []),
'PAPERCLIP_ACPX_PROVIDER_PACKAGE_ROOT="$pack" PAPERCLIP_ACPX_PROVIDER_PACKAGE_MANIFEST="$pack/package.json" "$node" "$entry"'].join(" "),
].join("; ");
let response: Pick<Awaited<ReturnType<typeof runAdapterExecutionTargetShellCommand>>, "stdout" | "exitCode" | "signal" | "timedOut">;
try {
if (target.transport === "ssh") {
try {
const result = await runSshCommand(target.spec, command, {
stdin: JSON.stringify({ apiKey, model, ...(candidateAdmission ? { candidateAdmission } : {}) }), timeoutMs: Math.max(1, Math.floor((remainingMs - 3_000) / 1000)) * 1000,
});
response = { stdout: result.stdout, exitCode: 0, signal: null, timedOut: false };
} catch (error) {
const failure = error as { code?: unknown; stdout?: unknown; signal?: string | null };
if (typeof failure.code !== "number") throw error;
response = { stdout: typeof failure.stdout === "string" ? failure.stdout : "", exitCode: failure.code,
signal: failure.signal ?? null, timedOut: false };
}
} else {
response = await runAdapterExecutionTargetShellCommand(`kimi-metadata-${randomUUID()}`, target, command, {
cwd: target.remoteCwd, env: { KIMI_API_KEY: apiKey, PAPERCLIP_KIMI_PROBE_MODEL: model,
...(candidateAdmission ? { PAPERCLIP_ACPX_CANDIDATE_ADMISSION: JSON.stringify(candidateAdmission) } : {}) },
timeoutSec: Math.max(1, Math.floor((remainingMs - 3_000) / 1000)), graceSec: 3,
});
}
} catch { throw REQUEST_FAILED(); }
if (Date.now() > deadlineAt || response.timedOut || response.signal || Buffer.byteLength(response.stdout) > 64 * 1024) throw REQUEST_FAILED();
if ([126, 127].includes(response.exitCode ?? -1)) continue;
let payload: unknown;
try { payload = JSON.parse(response.stdout.trim()); } catch { throw REQUEST_FAILED(); }
const result = validateKimiMetadata(payload, model);
if (result.status !== "verified") {
if (result.code === "KIMI_INSTALLATION_INVALID") continue;
throw unprocessable(result.message, { code: result.code });
}
if (response.exitCode !== 0) throw REQUEST_FAILED();
if (result.target !== "linux-x64") throw INSTALLATION_INVALID();
return result;
}
throw INSTALLATION_INVALID();
}
@@ -1,6 +1,7 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { readFileSync } from "node:fs";
import { ACPX_QUALIFICATION_ENV, resolveAcpxQualification } from "./acpx-qualification.js";
import { ACPX_QUALIFICATION_ENV, ANTIGRAVITY_CANDIDATE_ADMISSION_ENV, resolveAcpxQualification, resolveKimiCandidateAdmission } from "./acpx-qualification.js";
import { QUALIFIED_ACPX_PROFILES, KIMI_DISTRIBUTION_CLOSURE_SHA256 } from "../../vendor/paperclip-runner/index.js";
import type { NativeExecutionInput } from "../../vendor/paperclip-runner/index.js";
import { resolvePaperclipRunnerProviderProfile, resolvePaperclipRunnerNativeProviderInput } from "./provider-profile.js";
@@ -9,6 +10,20 @@ const authorize = (value: unknown) => ({ [ACPX_QUALIFICATION_ENV]: JSON.stringif
describe("host ACPX qualification admission", () => {
afterEach(() => vi.unstubAllEnvs());
it.each([undefined, "malformed", JSON.stringify(kimiAdmission("linux", "x64"))])("admits released Kimi without candidate authority (%s)", authority => {
vi.stubEnv(ANTIGRAVITY_CANDIDATE_ADMISSION_ENV, authority);
vi.stubEnv(ACPX_QUALIFICATION_ENV, "malformed-unrelated-authority");
const config = { provider: "acpx", acpxAgent: "kimi", model: "kimi-for-coding" };
expect(QUALIFIED_ACPX_PROFILES.kimi.agentProfileVersion).toBe(17);
expect(QUALIFIED_ACPX_PROFILES.kimi.qualificationStatus).not.toBe("pending");
expect(resolveKimiCandidateAdmission({ companyId: "candidate-company", model: config.model })).toBeUndefined();
expect(resolvePaperclipRunnerProviderProfile(config, { companyId: "candidate-company" })).toMatchObject(config);
expect(resolvePaperclipRunnerNativeProviderInput({ companyId: "candidate-company", backend: "acpx_runtime", adapterConfig: config }))
.toMatchObject({ ...config, acpxPermissionMode: "approve-all" });
expect(() => resolvePaperclipRunnerProviderProfile({ ...config, model: "" }))
.toThrow(expect.objectContaining({ code: "paperclip_runner_acpx_model_required" }));
});
it.each([undefined, "malformed", JSON.stringify([{ agent: "antigravity", model: "exact-model" }])])(
"keeps pending Antigravity closed despite host authorization (%s)", host => {
vi.stubEnv(ACPX_QUALIFICATION_ENV, host);
@@ -97,3 +112,9 @@ it.each([undefined, "off", "low", "high", "max"] as const)("validates saved Pi t
expect(resolvePaperclipRunnerNativeProviderInput({ backend: "acpx_runtime", adapterConfig })).toMatchObject({ piThinkingLevel: piThinkingLevel ?? "low" });
for (const alias of ["medium", "minimal", "xhigh"]) expect(() => resolvePaperclipRunnerProviderProfile({ ...adapterConfig, piThinkingLevel: alias })).toThrow(expect.objectContaining({ code: "paperclip_runner_pi_thinking_invalid" }));
});
function kimiAdmission(platform: "darwin" | "linux", architecture: "arm64" | "x64") {
return { schema: "paperclip.acpx-candidate-admission.v1", companyId: "candidate-company", agent: "kimi", model: "kimi-for-coding",
platform, architecture, expiresAt: Date.now() + 60_000, profileDigest: QUALIFIED_ACPX_PROFILES.kimi.commandDigest,
nativeClosureDigest: `sha256:${KIMI_DISTRIBUTION_CLOSURE_SHA256[`${platform}-${architecture}` as "darwin-arm64" | "linux-x64"]}` };
}
@@ -1,5 +1,5 @@
import { PAPERCLIP_RUNNER_ACPX_PROFILES } from "@paperclipai/adapter-utils";
import { parseAcpxCandidateAdmission, type AcpxCandidateAdmission, type NativeExecutionInput } from "../../vendor/paperclip-runner/index.js";
import { parseAcpxCandidateAdmission, QUALIFIED_ACPX_PROFILES, type AcpxCandidateAdmission, type NativeExecutionInput } from "../../vendor/paperclip-runner/index.js";
/** Trusted operator startup authority, separate from saved agent configuration. */
export const ANTIGRAVITY_CANDIDATE_ADMISSION_ENV = "PAPERCLIP_ACPX_CANDIDATE_ADMISSION";
@@ -23,9 +23,31 @@ export function resolveAntigravityCandidateAdmission(
});
}
export function resolveKimiCandidateAdmission(
input: { companyId?: string; model?: string; target?: { platform: string; architecture: string } },
hostEnvironment: NodeJS.ProcessEnv = process.env,
): AcpxCandidateAdmission | undefined {
if (QUALIFIED_ACPX_PROFILES.kimi.qualificationStatus !== "pending") return undefined;
const encoded = hostEnvironment[ANTIGRAVITY_CANDIDATE_ADMISSION_ENV];
if (!encoded || !input.companyId) return undefined;
let value: unknown;
try { value = JSON.parse(encoded); } catch { throw new Error("Invalid Kimi candidate admission"); }
if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("Invalid Kimi candidate admission");
const record = value as Record<string, unknown>;
if (record.companyId !== input.companyId || record.agent !== "kimi") return undefined;
const admission = parseAcpxCandidateAdmission(value, {
companyId: input.companyId, agent: "kimi",
model: input.model ?? (typeof record.model === "string" ? record.model : ""),
});
if (input.target && (admission.platform !== input.target.platform || admission.architecture !== input.target.architecture)) {
throw new Error("Kimi candidate admission does not match the execution target");
}
return admission;
}
/** Operator-only admission for an exact candidate/model during qualification. */
export const ACPX_QUALIFICATION_ENV = "PAPERCLIP_RUNNER_ACPX_QUALIFICATION";
export type AcpxQualificationCandidate = "cursor" | "copilot" | "pi";
export type AcpxQualificationCandidate = "cursor" | "copilot" | "pi" | "kimi";
export function resolveAcpxQualification(
provider: NativeExecutionInput["provider"] | { kind: "acpx"; agent: string; model: string },
@@ -45,14 +67,14 @@ export function resolveAcpxQualification(
if (!encoded) return undefined;
let entries: unknown;
try { entries = JSON.parse(encoded); } catch { throw new Error("Invalid ACPX qualification authorization"); }
if (!Array.isArray(entries) || entries.length < 1 || entries.length > 3) {
if (!Array.isArray(entries) || entries.length < 1 || entries.length > 4) {
throw new Error("Invalid ACPX qualification authorization");
}
const agents = new Set<string>();
for (const entry of entries) {
if (!entry || typeof entry !== "object" || Array.isArray(entry)
|| Object.keys(entry).sort().join(",") !== "agent,model"
|| !["cursor", "copilot", "pi"].includes(entry.agent)
|| !["cursor", "copilot", "pi", "kimi"].includes(entry.agent)
|| typeof entry.model !== "string" || !entry.model.trim() || entry.model !== entry.model.trim()
|| agents.has(entry.agent)) throw new Error("Invalid ACPX qualification authorization");
agents.add(entry.agent);
@@ -0,0 +1,149 @@
import { randomUUID } from "node:crypto";
import { and, eq } from "drizzle-orm";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { agents, authUsers, companies, completionContracts, createDb, heartbeatRunEvents, heartbeatRuns, issueComments, issues,
nativeRunFinalizations, nativeRunResults, type Db } from "@paperclipai/db";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js";
import { NativeRunCoordinatorStore } from "./native-run-coordinator-store.js";
import { finalizeNativeRun, recoverCommittedNativeChatResponse } from "./native-run-finalizer.js";
import { nativeResponsePresentationWork } from "./native-response-presentation-work.js";
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_AGENT_CHAT_RESPONSE_TEST_DATABASE_URL");
const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport();
describe.skipIf(!support.supported)("committed Agent Chat reply recovery", () => {
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
let db: Db;
beforeAll(async () => {
if (!external) database = await startEmbeddedPostgresTestDatabase("agent-chat-response-");
db = createDb(external ?? database!.connectionString);
}, 60_000);
afterAll(async () => { await db?.$client.end(); await database?.cleanup(); });
async function fixture(conversation = true) {
const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID();
const sessionId = randomUUID(), runnerId = randomUUID(), userId = `fixture-${randomUUID()}`, wakeId = randomUUID();
await db.insert(authUsers).values({ id: userId, name: "Conversation owner", email: `${userId}@example.test`,
createdAt: new Date("2020-01-01T00:00:00Z"), updatedAt: new Date("2020-01-01T00:00:00Z") });
await db.insert(companies).values({ id: companyId, name: "Reply recovery fixture", issuePrefix: companyId.slice(0, 8) });
await db.insert(agents).values({ id: agentId, companyId, name: "Conversation agent", adapterType: "paperclip_runner" });
await db.insert(issues).values({ id: issueId, companyId, title: "Conversation reply", status: "in_progress", assigneeAgentId: agentId,
...(conversation ? { conversationAgentId: agentId, conversationUserId: userId, conversationState: "active", conversationSessionGeneration: 0 } : {}) });
await db.insert(issueComments).values({ id: wakeId, companyId, issueId, authorUserId: userId, body: "Hire one teammate and tell me when ready.",
createdAt: new Date("2020-01-01T00:00:00Z") });
const [contract] = await db.insert(completionContracts).values({ companyId, issueId, revision: 1,
schemaVersion: "paperclip.completion-contract.v1", policyVersion: "test", risk: "standard", completionAuthority: "server_arbiter",
incompleteCriteriaPolicy: "preserve_non_terminal", contractJson: { revision: "test", objective: "Answer the user", criteria: [] },
canonicalSha256: "fixture-conversation-contract", createdByActorType: "system", createdByActorId: "test" }).returning();
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: issueId, nativeSessionId: sessionId,
runnerInstanceId: runnerId, completionContractId: contract.id, completionContractSha256: contract.canonicalSha256,
status: "running", runtimeMode: "native", responsibleUserId: userId,
contextSnapshot: { issueId, source: "issue.comment", wakeCommentId: wakeId, ...(conversation ? { conversationSessionGeneration: 0 } : {}) } });
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId));
const port = new NativeRunCoordinatorStore(db, { companyId, issueId, runId, agentId, normalizedSessionId: sessionId,
runnerSourceInstanceId: runnerId, completionContractId: contract.id, completionContractSha256: contract.canonicalSha256,
completionContractRevision: "test", completionContractCriterionIds: [] });
const reply = "The teammate is hired and ready for your next instruction.";
const prpEvent = { runId, normalizedSessionId: sessionId, turnId: "hire-turn", sourceEventId: "fixture-final-answer",
payload: { kind: "agentMessage", channel: "final", providerPhase: "final", text: reply } };
await db.insert(heartbeatRunEvents).values({ companyId, runId, agentId, seq: 170, eventType: "item.completed", payload: { prpEvent } });
await port.completeRun({ result: { schema: "paperclip.run_result.v1", reportedWorkDisposition: "done",
summary: "A completed hire, with its final message preserved separately.", completionClaim: { contractRevision: "test",
objectiveSatisfied: true, criteria: [], remainingWork: [] }, evidence: [], verification: [], attentionRequests: [], artifacts: [] },
terminal: { schema: "paperclip.prp.terminal.v1", runTerminalState: "succeeded", turnTerminalState: "completed", reportedWorkDisposition: "done" }, turnId: "hire-turn" });
await finalizeNativeRun({ db, runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: false });
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date("2020-01-01T00:02:00Z") }).where(eq(heartbeatRuns.id, runId));
expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)))[0].phase).toBe("committed");
const input = { companyId, issueId, runId };
return { ...input, input, agentId, userId, wakeId, prpEvent, reply };
}
const comments = (runId: string) => db.select().from(issueComments).where(eq(issueComments.createdByRunId, runId));
const issue = async (issueId: string) => (await db.select().from(issues).where(eq(issues.id, issueId)))[0];
it("finds a committed final Agent Chat turn, persists the exact reply once and only then settles waiting", async () => {
const f = await fixture();
expect(await comments(f.runId)).toEqual([]);
expect(await issue(f.issueId)).toMatchObject({ status: "in_progress", conversationState: "active" });
const work = nativeResponsePresentationWork(db), entries = await work.load(f.input);
expect(entries).toEqual([expect.objectContaining({ id: f.runId })]);
await expect(work.check(entries[0])).resolves.toBeNull();
expect(await comments(f.runId)).toEqual([expect.objectContaining({ body: f.reply, authorAgentId: f.agentId, createdByRunId: f.runId })]);
expect(await issue(f.issueId)).toMatchObject({ status: "in_review", conversationState: "waiting" });
expect(await work.load(f.input)).toEqual([]);
await expect(recoverCommittedNativeChatResponse(db, f.input)).resolves.toBe(false);
expect(await comments(f.runId)).toHaveLength(1);
});
it("keeps a later user message actionable after recovering the earlier exact reply", async () => {
const f = await fixture();
await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId, authorUserId: f.userId, body: "Another instruction",
createdAt: new Date("2020-01-01T00:01:00Z") });
await expect(recoverCommittedNativeChatResponse(db, f.input)).resolves.toBe(true);
expect(await comments(f.runId)).toHaveLength(1);
expect(await issue(f.issueId)).toMatchObject({ status: "in_progress", conversationState: "active" });
});
it("does not duplicate a reply under concurrent recovery", async () => {
const f = await fixture();
expect((await Promise.all([recoverCommittedNativeChatResponse(db, f.input), recoverCommittedNativeChatResponse(db, f.input)])).sort())
.toEqual([false, true]);
expect(await comments(f.runId)).toHaveLength(1);
});
it.each(["skip", "generation", "agent", "user", "execution", "waiting", "turn", "event-run", "session", "channel", "kind", "digest", "partial"])(
"keeps %s mismatches unchanged instead of publishing an unauthorized or incomplete reply", async fault => {
const f = await fixture();
if (["skip", "generation"].includes(fault)) await db.update(heartbeatRuns).set({ contextSnapshot: { issueId: f.issueId,
source: "issue.comment", wakeCommentId: f.wakeId, conversationSessionGeneration: fault === "generation" ? 1 : 0,
...(fault === "skip" ? { skipIssueComment: true } : {}) } }).where(eq(heartbeatRuns.id, f.runId));
if (fault === "user") await db.update(heartbeatRuns).set({ responsibleUserId: "another-user" }).where(eq(heartbeatRuns.id, f.runId));
if (fault === "agent") {
const other = randomUUID(); await db.insert(agents).values({ id: other, companyId: f.companyId, name: "Other agent" });
await db.update(issues).set({ conversationAgentId: other, assigneeAgentId: other }).where(eq(issues.id, f.issueId));
}
if (fault === "execution") {
const other = randomUUID(); await db.insert(heartbeatRuns).values({ id: other, companyId: f.companyId, agentId: f.agentId });
await db.update(issues).set({ executionRunId: other }).where(eq(issues.id, f.issueId));
}
if (fault === "waiting") await db.update(issues).set({ executionRunId: null, status: "in_review", conversationState: "waiting" }).where(eq(issues.id, f.issueId));
if (["turn", "event-run", "session", "channel", "kind"].includes(fault)) {
const event = structuredClone(f.prpEvent);
if (fault === "turn") event.turnId = "foreign-turn";
if (fault === "event-run") event.runId = randomUUID();
if (fault === "session") event.normalizedSessionId = randomUUID();
if (fault === "channel") event.payload.channel = "analysis";
if (fault === "kind") event.payload.kind = "reasoning";
await db.update(heartbeatRunEvents).set({ payload: { prpEvent: event } }).where(and(eq(heartbeatRunEvents.runId, f.runId), eq(heartbeatRunEvents.seq, 170)));
}
if (fault === "digest") await db.update(nativeRunResults).set({ canonicalSha256: "invalid-digest" }).where(eq(nativeRunResults.runId, f.runId));
if (fault === "partial") await db.update(nativeRunFinalizations).set({ phase: "copying_back" }).where(eq(nativeRunFinalizations.runId, f.runId));
const before = await issue(f.issueId);
await expect(recoverCommittedNativeChatResponse(db, f.input)).resolves.toBe(false);
expect(await comments(f.runId)).toEqual([]); expect(await issue(f.issueId)).toEqual(before);
},
);
it.each([false, true])("preserves a selected reply without replacing or resurrecting it (deleted=%s)", async deleted => {
const f = await fixture();
const [saved] = await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId, authorAgentId: f.agentId,
createdByRunId: f.runId, body: "Already selected", ...(deleted ? { deletedAt: new Date("2020-01-01T00:03:00Z") } : {}) }).returning();
await db.update(heartbeatRuns).set({ resultJson: { presentationDecision: { commentId: saved.id, commentAction: "create" } } }).where(eq(heartbeatRuns.id, f.runId));
await expect(recoverCommittedNativeChatResponse(db, f.input)).resolves.toBe(false);
expect(await comments(f.runId)).toEqual([saved]);
});
it("never crosses company scope and keeps ordinary internal task response recovery intact", async () => {
const f = await fixture(false);
await expect(recoverCommittedNativeChatResponse(db, { ...f.input, companyId: randomUUID() })).resolves.toBe(false);
const work = nativeResponsePresentationWork(db), entries = await work.load(f.input);
expect(entries).toHaveLength(1); await expect(work.check(entries[0])).resolves.toBeNull();
expect(await comments(f.runId)).toEqual([expect.objectContaining({ body: f.reply })]);
expect((await issue(f.issueId)).conversationAgentId).toBeNull();
});
it("rolls back a partial reply write without changing the waiting state or consuming recovery", async () => {
const f = await fixture(), abort = new Error("fixture-commit-aborted");
const failing = new Proxy(db, { get(target, key) { return key === "transaction"
? (callback: Parameters<Db["transaction"]>[0]) => db.transaction(async tx => { await callback(tx); throw abort; })
: Reflect.get(target, key); } });
await expect(recoverCommittedNativeChatResponse(failing, f.input)).rejects.toBe(abort);
expect(await comments(f.runId)).toEqual([]);
expect(await issue(f.issueId)).toMatchObject({ status: "in_progress", conversationState: "active" });
await expect(recoverCommittedNativeChatResponse(db, f.input)).resolves.toBe(true);
expect(await comments(f.runId)).toHaveLength(1);
});
});
@@ -0,0 +1,61 @@
import type { Db } from "@paperclipai/db";
import { sql } from "drizzle-orm";
import { inspectNativeRunnerUpgradeHeader } from "./native-runner-checkpoint.js";
/** An explicit, paginated diagnostic. Never called by an idle/background scan.
* This inventories selected retained task-session pointers, not all history.
* Header evidence is preliminary: full snapshot/journal validation still occurs
* only when ordinary admission selects its exact continuation source. */
export async function readNativeControllerRecoveryInventory(db: Db, input: { afterId?: string; limit?: number } = {}) {
if (input.limit !== undefined && !Number.isFinite(input.limit)) throw new Error("native_recovery_inventory_limit_invalid");
const limit = Math.min(128, Math.max(1, Math.floor(input.limit ?? 128)));
const rows = await db.transaction(async tx => {
await tx.execute(sql`set local statement_timeout = '2s'`);
return tx.execute<{
id: string; runId: string | null; companyId: string; issueId: string | null; sessionId: string | null;
runnerId: string | null; metadata: Record<string, unknown> | null; material: Record<string, unknown> | null;
pending: boolean;
}>(sql`with selected as (
select id, company_id, agent_id, last_run_id from agent_task_sessions
where adapter_type = 'paperclip_runner' ${input.afterId ? sql`and id > ${input.afterId}::uuid` : sql``}
order by id limit ${limit + 1}
) select s.id, s.company_id as "companyId", r.id as "runId", r.native_issue_id as "issueId",
r.native_session_id as "sessionId", r.runner_instance_id as "runnerId",
case when pg_column_size(r.runner_profile_json->'nativeControllerRecovery') <= 4096
then r.runner_profile_json->'nativeControllerRecovery' else null end as metadata,
case when pg_column_size(c.material) <= 262144 and sum(pg_column_size(c.material)) over () <= 8388608
then c.material else null end as material,
(r.finished_at is null or r.status not in ('succeeded', 'failed', 'cancelled', 'interrupted', 'timed_out')
or f.run_id is null or f.lease_owner is not null or f.recovery_state is not null
or f.phase not in ('committed', 'terminal_failure')) is true as pending
from selected s left join heartbeat_runs r on r.id = s.last_run_id and r.company_id = s.company_id and r.agent_id = s.agent_id
left join native_run_finalizations f on f.run_id = r.id and f.company_id = s.company_id
left join native_runner_checkpoints c on c.run_id = r.id and c.company_id = s.company_id
and c.issue_id = r.native_issue_id and c.part = 'header'
order by s.id`);
}, { accessMode: "read only" });
const truncated = rows.length > limit;
const page = rows.slice(0, limit);
const reasons: Record<string, number> = {};
let selectedRetainedEvidence = 0, outstandingObligations = 0;
for (const row of page) {
let reason: string | null = null;
if (row.pending) { outstandingObligations++; reason = "persistence_or_handoff_pending"; }
else if (!row.runId || !row.issueId || !row.sessionId) reason = "selected_binding_missing";
else if (row.metadata?.schema !== "paperclip.native-controller-recovery.v1") reason = "metadata_unsupported_or_missing";
else {
const checkpoint = row.metadata.checkpoint as Record<string, unknown> | null;
const header = row.material ? inspectNativeRunnerUpgradeHeader({ companyId: row.companyId, issueId: row.issueId, runId: row.runId }, row.material) : null;
if (!checkpoint || checkpoint.runId !== row.runId || checkpoint.runnerInstanceId !== row.runnerId
|| checkpoint.normalizedSessionId !== row.sessionId || !header
|| header.runnerInstanceId !== row.runnerId || header.normalizedSessionId !== row.sessionId) reason = "checkpoint_binding_missing";
else selectedRetainedEvidence++;
}
if (reason) reasons[reason] = (reasons[reason] ?? 0) + 1;
}
return { version: 1 as const, observedAt: new Date().toISOString(), inspected: page.length,
coverage: truncated || input.afterId ? "partial" as const : "selected_task_session_pointers" as const,
truncated, nextAfterId: truncated ? page.at(-1)!.id : null,
selectedRetainedEvidence, outstandingObligations, reasons,
fullPayloadsValidated: false as const };
}
@@ -0,0 +1,165 @@
import { randomUUID } from "node:crypto";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { eq } from "drizzle-orm";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { agentTaskSessions, agents, companies, createDb, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults } from "@paperclipai/db";
import { DurablePrpControlPlane, type PersistedNativeSession } from "../../vendor/paperclip-runner/index.js";
import { startRecoveryPostgres, recoveryPostgresSupport } from "../../__tests__/helpers/recovery-postgres.js";
import { createIsolatedTestHome } from "../../__tests__/helpers/isolated-test-home.js";
import { prepareNativeHeartbeatRun } from "./prepare-native-run.js";
import { nativeRunnerCheckpointWriter } from "./native-runner-checkpoint.js";
import { createNativeControllerRecovery, deferNativeControllerRecovery, importNativeControllerRecovery, NATIVE_RETAINED_OWNER_WAIT, resolveNativeControllerRecovery } from "./native-controller-recovery.js";
import { claimNativeSessionResumptions } from "./native-finalization-reconciler.js";
import { claimNativeRestartRecoveries } from "./native-restart-recovery.js";
import { readNativeControllerRecoveryInventory } from "./native-controller-recovery-inventory.js";
describe.skipIf(!recoveryPostgresSupport.supported)("native controller durable discovery and import", () => {
let database: Awaited<ReturnType<typeof startRecoveryPostgres>>;
let db: ReturnType<typeof createDb>;
let home: ReturnType<typeof createIsolatedTestHome>;
let root: string;
const previousHome = process.env.PAPERCLIP_HOME;
const companyId = randomUUID(), agentId = randomUUID();
const compatibility = { configDigest: `sha256:${"a".repeat(64)}`, processConfigDigest: null };
beforeAll(async () => {
home = createIsolatedTestHome();
process.env.PAPERCLIP_HOME = home.home;
root = await mkdtemp(join(tmpdir(), "native-controller-discovery-"));
database = await startRecoveryPostgres("native-controller-discovery-");
db = createDb(database.connectionString);
await db.insert(companies).values({ id: companyId, name: "Controller discovery", issuePrefix: "NCD" });
await db.insert(agents).values({ id: agentId, companyId, name: "Runner", adapterType: "paperclip_runner", adapterConfig: { provider: "codex" } });
}, 60_000);
afterAll(async () => {
await database?.cleanup();
if (root) await rm(root, { recursive: true, force: true });
home?.cleanup();
if (previousHome === undefined) delete process.env.PAPERCLIP_HOME;
else process.env.PAPERCLIP_HOME = previousHome;
});
async function seed() {
const issueId = randomUUID(), runId = randomUUID();
await db.insert(issues).values({ id: issueId, companyId, title: "Continue retained execution", assigneeAgentId: agentId, status: "in_progress" });
const [run] = await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running", runtimeMode: "native", nativeIssueId: issueId }).returning();
const native = await prepareNativeHeartbeatRun({ db, run, issue: { id: issueId, title: "Continue retained execution", description: null, reviewPolicy: null }, environmentLeaseId: randomUUID() });
const owner = { companyId, issueId, runId, authorityRunId: runId, bootId: randomUUID(), generation: 1, leaseOwner: randomUUID() };
await db.insert(nativeRunFinalizations).values({ companyId, issueId, runId, phase: "observed", controllerBootId: owner.bootId,
controllerGeneration: 1, leaseOwner: owner.leaseOwner, leaseExpiresAt: new Date(Date.now() + 60_000) });
const core = new DurablePrpControlPlane({ stateDirectory: join(root, runId),
identity: { runId, normalizedSessionId: native.normalizedSessionId, runnerInstanceId: native.runnerInstanceId,
environmentLeaseId: native.environmentLeaseId, turnId: native.turnId, itemId: native.itemId },
expectedRunnerVersion: "discovery-fixture", expectedRunnerDigest: `sha256:${"a".repeat(64)}` });
core.setCheckpointPersistence(nativeRunnerCheckpointWriter(db, owner));
await core.flushCheckpoint();
await core.stop();
const snapshot: PersistedNativeSession = { backendKind: "mock", sessionId: native.normalizedSessionId,
identity: { companyId, agentId, issueId, runId, sessionId: native.normalizedSessionId } };
const [prepared] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
const [result] = await db.insert(nativeRunResults).values({ companyId, issueId, runId,
completionContractId: prepared.completionContractId!, serverFingerprint: "fixture", schemaStatus: "accepted",
resultJson: {}, canonicalSha256: "fixture" }).returning();
await db.update(nativeRunFinalizations).set({ phase: "committed", resultId: result.id, leaseOwner: null, leaseExpiresAt: null }).where(eq(nativeRunFinalizations.runId, runId));
const profile = { ...prepared.runnerProfileJson, sessionCheckpoint: snapshot };
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date(), runnerProfileJson: profile }).where(eq(heartbeatRuns.id, runId));
const scope = { companyId, agentId, issueId, runId, normalizedSessionId: native.normalizedSessionId };
return { scope, native, profile, snapshot, owner,
supervisor: { schema: "paperclip.native-session-supervisor.v1", ...compatibility, snapshot } };
}
it("imports a retained source once, rejects stale input, and resolves it without any local projection", async () => {
const fixture = await seed();
const input = { ...fixture.scope, expectedConfigDigest: compatibility.configDigest, supervisor: fixture.supervisor };
expect(await importNativeControllerRecovery(db, { ...input, supervisor: { ...input.supervisor, snapshot: { ...fixture.snapshot, providerSessionId: "stale" } } }))
.toEqual({ kind: "missing_evidence", reason: "stale_supervisor_snapshot" });
expect(await importNativeControllerRecovery(db, input)).toEqual({ kind: "imported" });
expect(await importNativeControllerRecovery(db, input)).toEqual({ kind: "unchanged" });
await rm(join(root, fixture.scope.runId), { recursive: true });
const currentRunId = randomUUID();
await db.insert(heartbeatRuns).values({ id: currentRunId, companyId, agentId, nativeIssueId: fixture.scope.issueId,
nativeSessionId: fixture.scope.normalizedSessionId, status: "running", runtimeMode: "native" });
const recovered = await resolveNativeControllerRecovery(db, { ...fixture.scope, currentRunId });
expect(recovered.kind).toBe("recoverable");
if (recovered.kind !== "recoverable") throw new Error(JSON.stringify(recovered));
expect(recovered.snapshot).toEqual(fixture.snapshot);
expect(recovered.checkpoint.recoveryEvidence.runnerInstanceId).toBe(fixture.native.runnerInstanceId);
const deadline = new Date(Date.now() + 60_000);
await db.update(nativeRunFinalizations).set({ leaseOwner: "other-controller", leaseExpiresAt: deadline }).where(eq(nativeRunFinalizations.runId, fixture.scope.runId));
expect(await importNativeControllerRecovery(db, input)).toEqual({ kind: "waiting_for_owner", deadline });
expect(await resolveNativeControllerRecovery(db, { ...fixture.scope, currentRunId })).toEqual({ kind: "waiting_for_owner", deadline });
});
it("holds mismatched snapshots and runner bindings instead of falling back to fresh execution", async () => {
const fixture = await seed(), currentRunId = randomUUID();
const metadata = createNativeControllerRecovery(fixture.snapshot, compatibility, { runId: fixture.scope.runId,
runnerInstanceId: fixture.native.runnerInstanceId, normalizedSessionId: fixture.scope.normalizedSessionId });
await db.insert(heartbeatRuns).values({ id: currentRunId, companyId, agentId, nativeIssueId: fixture.scope.issueId,
nativeSessionId: fixture.scope.normalizedSessionId, status: "running", runtimeMode: "native" });
for (const envelope of [{ ...metadata, snapshotSha256: "wrong" }, { ...metadata, checkpoint: { ...metadata.checkpoint!, runnerInstanceId: randomUUID() } }]) {
await db.update(heartbeatRuns).set({ runnerProfileJson: { ...fixture.profile, nativeControllerRecovery: envelope } }).where(eq(heartbeatRuns.id, fixture.scope.runId));
expect(await resolveNativeControllerRecovery(db, { ...fixture.scope, currentRunId }))
.toMatchObject({ kind: "missing_evidence", reason: "snapshot_digest_or_checkpoint_binding" });
}
});
it("keeps an unstarted follow-up on the same run until the retained owner's deadline", async () => {
const fixture = await seed(), runId = randomUUID();
const scope = { ...fixture.scope, runId };
const deadline = new Date(Date.now() + 60_000);
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: scope.issueId,
nativeSessionId: scope.normalizedSessionId, status: "running", runtimeMode: "native",
// The admission boundary has already stored its input; the deferral
// and resumption owners never execute this transport-independent fixture.
runnerProfileJson: { nativeExecutionInput: { binding: scope } } });
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, scope.issueId));
await db.insert(nativeRunFinalizations).values({ companyId, issueId: scope.issueId, runId, phase: "observed" });
await deferNativeControllerRecovery(db, { ...scope, retryAt: deadline });
const read = () => db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId));
expect((await read())[0]).toMatchObject({ phase: "retryable_failure", attempt: 0, leaseOwner: null,
nextAttemptAt: deadline, failureCode: NATIVE_RETAINED_OWNER_WAIT });
expect(await claimNativeSessionResumptions({ db, runnerInstanceId: "successor", runIds: [runId] })).toEqual([]);
// Even after expiry, restart adoption cannot reinterpret this pre-provider
// wait as a crashed provider. The ordinary resumption owner consumes it.
expect(await claimNativeRestartRecoveries({ db, runIds: [runId], restartKind: "hard", now: deadline })).toEqual([]);
const claims = await claimNativeSessionResumptions({ db, runnerInstanceId: "successor", runIds: [runId], now: deadline });
expect(claims).toHaveLength(1);
const nextDeadline = new Date(deadline.getTime() + 60_000);
await deferNativeControllerRecovery(db, { ...scope, leaseOwner: claims[0].leaseOwner, retryAt: nextDeadline });
expect((await read())[0]).toMatchObject({ attempt: 0, leaseOwner: null, nextAttemptAt: nextDeadline });
await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, runId));
expect(await claimNativeSessionResumptions({ db, runnerInstanceId: "successor", runIds: [runId], now: nextDeadline })).toEqual([]);
});
it.each(["live-owner", "expired-owner", "provider-attempt", "stopped", "operator-stop"])("does not publish an admission wait after authority changes: %s", async scenario => {
const fixture = await seed(), runId = randomUUID();
const scope = { ...fixture.scope, runId };
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: scope.issueId,
status: scenario === "stopped" ? "cancelled" : "running", runtimeMode: "native",
resultJson: scenario === "operator-stop" ? { cancellation: { source: "operator", expected: true,
initiator: { type: "user" }, reason: "Stop", recordedAt: new Date().toISOString() } } : null,
runnerProfileJson: { nativeExecutionInput: { binding: scope } } });
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, scope.issueId));
await db.insert(nativeRunFinalizations).values({ companyId, issueId: scope.issueId, runId, phase: "observed",
attempt: scenario === "provider-attempt" ? 1 : 0,
leaseOwner: scenario.endsWith("-owner") ? "other-controller" : null,
leaseExpiresAt: scenario === "expired-owner" ? new Date(Date.now() - 1_000) : null });
await expect(deferNativeControllerRecovery(db, { ...scope,
leaseOwner: scenario === "expired-owner" ? "other-controller" : undefined,
retryAt: new Date(Date.now() + 60_000) }))
.rejects.toThrow("wait_authority_changed");
expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)))[0].phase).toBe("observed");
});
it("reports bounded retained-pointer coverage without loading historical payloads", async () => {
const fixture = await seed();
await importNativeControllerRecovery(db, { ...fixture.scope, expectedConfigDigest: compatibility.configDigest, supervisor: fixture.supervisor });
await db.insert(agentTaskSessions).values([0, 1].map(index => ({ companyId, agentId, taskKey: `${fixture.scope.issueId}-${index}`,
adapterType: "paperclip_runner", lastRunId: fixture.scope.runId })));
const first = await readNativeControllerRecoveryInventory(db, { limit: 1 });
expect(first).toMatchObject({ inspected: 1, coverage: "partial", truncated: true, selectedRetainedEvidence: 1, fullPayloadsValidated: false });
const next = await readNativeControllerRecoveryInventory(db, { limit: 1, afterId: first.nextAfterId! });
expect(next).toMatchObject({ inspected: 1, coverage: "partial", truncated: false, selectedRetainedEvidence: 1 });
});
});
@@ -0,0 +1,43 @@
import { describe, expect, it } from "vitest";
import { createNativeControllerRecovery, readNativeControllerRecovery } from "./native-controller-recovery.js";
import { nativeSha256 } from "./canonical.js";
import { redactRegisteredSecretValues } from "../run-secret-redaction.js";
import type { PersistedNativeSession } from "../../vendor/paperclip-runner/index.js";
const snapshot: PersistedNativeSession = { backendKind: "mock", sessionId: "provider",
identity: { companyId: "company", agentId: "agent", issueId: "issue", runId: "run", sessionId: "session" } };
const compatibility = { configDigest: `sha256:${"a".repeat(64)}`, processConfigDigest: null };
const binding = { runId: "run", runnerInstanceId: "runner", normalizedSessionId: "session" };
describe("native controller recovery envelope", () => {
it("binds the stored snapshot, including redaction, without serializing its contents twice", () => {
const stored = { ...snapshot, providerSessionId: "[REDACTED]" };
const envelope = createNativeControllerRecovery(stored, compatibility, binding);
expect(envelope.snapshotSha256).toBe(nativeSha256(stored));
expect(readNativeControllerRecovery(envelope, JSON.parse(JSON.stringify(stored)))).toEqual(envelope);
expect(readNativeControllerRecovery(envelope, { ...stored, providerSessionId: "secret-before-redaction" })).toBeNull();
expect(JSON.stringify(envelope)).not.toContain("[REDACTED]");
expect(Buffer.byteLength(JSON.stringify(envelope))).toBeLessThan(4096);
});
it("does not certify a missing PRP publication or unknown format", () => {
const pending = createNativeControllerRecovery(snapshot, compatibility, null);
expect(readNativeControllerRecovery(pending, snapshot)).toBeNull();
const published = createNativeControllerRecovery(snapshot, compatibility, binding);
expect(readNativeControllerRecovery({ ...published, schema: "future" }, snapshot)).toBeNull();
expect(readNativeControllerRecovery({ ...published, processConfigDigest: "unverified" }, snapshot)).toBeNull();
});
it("enforces the metadata byte bound on both write and read", () => {
expect(() => createNativeControllerRecovery(snapshot, { ...compatibility, configDigest: "x".repeat(4096) }, binding))
.toThrow("metadata_too_large");
expect(readNativeControllerRecovery({ ...createNativeControllerRecovery(snapshot, compatibility, binding), extra: "x".repeat(4096) }, snapshot))
.toBeNull();
});
it("keeps compatibility metadata out of public run projections", () => {
expect(redactRegisteredSecretValues({ runnerProfileJson: { preserved: true,
nativeControllerRecovery: createNativeControllerRecovery(snapshot, compatibility, binding) } }, []))
.toEqual({ runnerProfileJson: { preserved: true } });
});
});
@@ -0,0 +1,270 @@
import { and, eq, ne, sql } from "drizzle-orm";
import { environmentLeases, heartbeatRuns, issues, nativeRunFinalizations, nativeRunnerCheckpoints, type Db } from "@paperclipai/db";
import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target";
import type { DurablePrpCheckpoint, PersistedNativeSession } from "../../vendor/paperclip-runner/index.js";
import { notifyNativeFinalizationWork } from "../native-finalization-notifications.js";
import { readRunCancellation } from "../run-cancellation.js";
import { nativeSha256 } from "./canonical.js";
import { findNativeSessionResumeRun, isNativeSessionId, isUnusedNativeSessionBootstrap, nativeSessionProviderEvidence } from "./native-session-resume.js";
import { loadNativeRunnerCheckpoint } from "./native-runner-checkpoint.js";
import { redactRunnerApiPayloads } from "./runner-api-log-redaction.js";
export const NATIVE_CONTROLLER_RECOVERY_MAX_BYTES = 4 * 1024;
/** Private run-profile metadata. No credentials or controller-local paths. */
export interface NativeControllerRecovery {
schema: "paperclip.native-controller-recovery.v1";
snapshotSha256: string;
configDigest: string;
processConfigDigest: string | null;
checkpoint: {
runId: string;
runnerInstanceId: string;
normalizedSessionId: string;
} | null;
}
export type NativeControllerCompatibility = Pick<NativeControllerRecovery, "configDigest" | "processConfigDigest">;
export function createNativeControllerRecovery(
snapshot: PersistedNativeSession,
compatibility: NativeControllerCompatibility,
checkpoint: NativeControllerRecovery["checkpoint"],
): NativeControllerRecovery {
const value: NativeControllerRecovery = {
schema: "paperclip.native-controller-recovery.v1",
snapshotSha256: nativeSha256(snapshot),
configDigest: compatibility.configDigest,
processConfigDigest: compatibility.processConfigDigest,
checkpoint,
};
if (Buffer.byteLength(JSON.stringify(value)) > NATIVE_CONTROLLER_RECOVERY_MAX_BYTES) {
throw new Error("native_controller_recovery_metadata_too_large");
}
return value;
}
function record(value: unknown): Record<string, unknown> {
return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
}
export function readNativeControllerRecovery(value: unknown, snapshot: unknown): NativeControllerRecovery | null {
const envelope = record(value), checkpoint = record(envelope.checkpoint);
if (Buffer.byteLength(JSON.stringify(value) ?? "null") > NATIVE_CONTROLLER_RECOVERY_MAX_BYTES
|| envelope.schema !== "paperclip.native-controller-recovery.v1"
|| typeof envelope.snapshotSha256 !== "string" || envelope.snapshotSha256 !== nativeSha256(snapshot)
|| typeof envelope.configDigest !== "string" || !/^sha256:[a-f0-9]{64}$/.test(envelope.configDigest)
|| !(envelope.processConfigDigest === null || (typeof envelope.processConfigDigest === "string" && /^[a-f0-9]{64}$/.test(envelope.processConfigDigest)))
|| ![checkpoint.runId, checkpoint.runnerInstanceId, checkpoint.normalizedSessionId].every(v => typeof v === "string" && v.length > 0)) return null;
return envelope as unknown as NativeControllerRecovery;
}
export type NativeControllerRecoveryOutcome =
| { kind: "recoverable"; snapshot: PersistedNativeSession; metadata: NativeControllerRecovery; checkpoint: DurablePrpCheckpoint }
| { kind: "waiting_for_owner"; deadline: Date }
| { kind: "missing_evidence" | "unsupported" | "dependency_unavailable"; reason: string; cause?: unknown }
| { kind: "no_prior_execution" };
export const NATIVE_RETAINED_OWNER_WAIT = "native_controller_recovery_waiting_for_owner";
export class NativeControllerRecoveryWaitingError extends Error {
constructor(readonly retryAt: Date) {
super(NATIVE_RETAINED_OWNER_WAIT);
this.name = "NativeControllerRecoveryWaitingError";
}
}
/** Admission has not crossed the provider boundary. Retain this same business
* run on the existing retry owner; never turn an ownership wait into a failure
* or invent a new provider attempt. The usual resumption gates recheck Stop,
* permissions and task ownership when this deadline arrives. */
export async function deferNativeControllerRecovery(db: Db, input: {
companyId: string; agentId: string; issueId: string; runId: string;
leaseOwner?: string; retryAt: Date;
}): Promise<void> {
await db.transaction(async tx => {
const [task] = await tx.select({ executionRunId: issues.executionRunId }).from(issues)
.where(and(eq(issues.companyId, input.companyId), eq(issues.id, input.issueId))).for("update");
const [owner] = await tx.select().from(nativeRunFinalizations).where(and(
eq(nativeRunFinalizations.companyId, input.companyId), eq(nativeRunFinalizations.issueId, input.issueId),
eq(nativeRunFinalizations.runId, input.runId))).for("update");
const [run] = await tx.select().from(heartbeatRuns).where(and(
eq(heartbeatRuns.id, input.runId), eq(heartbeatRuns.companyId, input.companyId),
eq(heartbeatRuns.agentId, input.agentId), eq(heartbeatRuns.nativeIssueId, input.issueId))).for("update");
const [authority] = await tx.select({ runId: nativeRunnerCheckpoints.runId }).from(nativeRunnerCheckpoints)
.where(and(eq(nativeRunnerCheckpoints.companyId, input.companyId), eq(nativeRunnerCheckpoints.runId, input.runId))).limit(1);
const progress = await nativeSessionProviderEvidence(tx as unknown as Db, [input.runId]);
const cancellation = record(run?.resultJson?.nativeCancellation);
const stop = readRunCancellation(run?.resultJson);
const [clock] = await tx.execute<{ now: Date }>(sql`select clock_timestamp() as now`);
if (!run || run.runtimeMode !== "native" || run.status !== "running" || task?.executionRunId !== input.runId
|| !owner || owner.phase !== "observed" || owner.resultId || owner.attempt !== 0
|| owner.controllerGeneration !== 0 || owner.leaseOwner !== (input.leaseOwner ?? null)
|| (input.leaseOwner && (!owner.leaseExpiresAt || owner.leaseExpiresAt <= new Date(clock.now)))
|| run.processPid || run.processGroupId || run.processStartedAt || authority || progress.size
|| !record(run.runnerProfileJson).nativeExecutionInput || run.resultJson?.startupCancellation
|| (stop?.source === "operator" && stop.expected)
|| (["run", "issue"].includes(String(cancellation.scope)) && ["pending", "acknowledged"].includes(String(cancellation.dispatchState)))) {
throw new Error("native_controller_recovery_wait_authority_changed");
}
await notifyNativeFinalizationWork(tx, input);
await tx.update(nativeRunFinalizations).set({
phase: "retryable_failure", leaseOwner: null, leaseExpiresAt: null, controlDeadlineAt: null,
nextAttemptAt: input.retryAt, recoveryState: "resuming_session", failureCode: NATIVE_RETAINED_OWNER_WAIT,
failureDetail: { originalFailureCode: NATIVE_RETAINED_OWNER_WAIT,
message: "Waiting for the retained native controller owner to release its lease.",
recoveryMode: record(run.runnerProfileJson).sessionCheckpoint ? "same_session_resume" : "bootstrap_retry",
providerSessionEstablished: false, providerEventsExist: false },
updatedAt: new Date(),
}).where(eq(nativeRunFinalizations.runId, input.runId));
});
}
/** Bind a read of retained Runner lifecycle to the surviving sandbox. The
* ordinary admission owner still checks current permissions before execution. */
export async function assertNativeRetainedSandboxBinding(db: Db, input: {
companyId: string; issueId: string; runId: string;
target: Extract<AdapterExecutionTarget, { kind: "remote" }>;
}): Promise<void> {
const { target } = input;
if (target.transport !== "sandbox" || target.sandboxLeaseAcquisition?.outcome !== "resumed"
|| !target.providerKey || !target.environmentId || !target.leaseId) {
throw new Error("native_controller_recovery_dependency_unavailable:retained_sandbox");
}
const [owner] = await db.select().from(nativeRunFinalizations).where(and(
eq(nativeRunFinalizations.companyId, input.companyId), eq(nativeRunFinalizations.issueId, input.issueId),
eq(nativeRunFinalizations.runId, input.runId)));
const leases = await db.select({ metadata: environmentLeases.metadata, status: environmentLeases.status }).from(environmentLeases).where(and(
eq(environmentLeases.companyId, input.companyId), eq(environmentLeases.issueId, input.issueId),
eq(environmentLeases.heartbeatRunId, input.runId), eq(environmentLeases.environmentId, target.environmentId),
eq(environmentLeases.provider, target.providerKey), eq(environmentLeases.providerLeaseId, target.sandboxLeaseAcquisition.providerLeaseId)));
if (!owner || owner.leaseOwner || owner.phase !== "committed" || !owner.resultId || owner.recoveryState
|| leases.length !== 1 || leases[0].metadata?.remoteCwd !== target.remoteCwd
|| !["active", "released", "expired"].includes(leases[0].status)) {
throw new Error("native_controller_recovery_missing_evidence:retained_sandbox_binding");
}
}
/** Only the existing bounded selector chooses history. Never search backwards
* again when its selected record is incomplete or incompatible. This operation
* reads evidence; it grants neither command nor acknowledgement authority. */
export async function resolveNativeControllerRecovery(db: Db, input: {
companyId: string; agentId: string; issueId: string; normalizedSessionId: string; currentRunId: string;
}): Promise<NativeControllerRecoveryOutcome> {
if (!isNativeSessionId(input.normalizedSessionId)) return { kind: "missing_evidence", reason: "session_binding" };
const [current] = await db.select({ createdAt: heartbeatRuns.createdAt, processPid: heartbeatRuns.processPid,
processGroupId: heartbeatRuns.processGroupId, processStartedAt: heartbeatRuns.processStartedAt,
runnerProfileJson: heartbeatRuns.runnerProfileJson }).from(heartbeatRuns).where(and(
eq(heartbeatRuns.id, input.currentRunId), eq(heartbeatRuns.companyId, input.companyId),
eq(heartbeatRuns.agentId, input.agentId), eq(heartbeatRuns.nativeIssueId, input.issueId),
eq(heartbeatRuns.nativeSessionId, input.normalizedSessionId)));
if (!current) return { kind: "missing_evidence", reason: "current_binding" };
const source = await findNativeSessionResumeRun(db, { ...input, beforeCreatedAt: current.createdAt, includeProgressBarrier: true });
if (!source) {
// An exhausted candidate window is unknown history, not fresh-bootstrap
// proof. Read only IDs, and retain the selector's existing 32-row boundary.
const prior = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(
ne(heartbeatRuns.id, input.currentRunId), eq(heartbeatRuns.companyId, input.companyId),
eq(heartbeatRuns.agentId, input.agentId), eq(heartbeatRuns.nativeIssueId, input.issueId),
eq(heartbeatRuns.nativeSessionId, input.normalizedSessionId))).limit(33);
if (prior.length > 32) return { kind: "missing_evidence", reason: "candidate_window_exhausted" };
const progress = await nativeSessionProviderEvidence(db, [input.currentRunId]);
if (!isUnusedNativeSessionBootstrap(current, progress.has(input.currentRunId))) {
return { kind: "missing_evidence", reason: "current_execution_progress" };
}
// The normal admission and remote untouched-session claim still execute.
return { kind: "no_prior_execution" };
}
const [owner] = await db.select({ leaseOwner: nativeRunFinalizations.leaseOwner, leaseExpiresAt: nativeRunFinalizations.leaseExpiresAt }).from(nativeRunFinalizations)
.where(and(eq(nativeRunFinalizations.runId, source.id), eq(nativeRunFinalizations.companyId, input.companyId),
eq(nativeRunFinalizations.issueId, input.issueId)));
if (owner?.leaseOwner && owner.leaseExpiresAt && owner.leaseExpiresAt.getTime() > Date.now()) {
return { kind: "waiting_for_owner", deadline: owner.leaseExpiresAt };
}
if (source.runtimeMode !== "native" || !["succeeded", "failed", "cancelled", "interrupted", "timed_out"].includes(source.status)
|| source.createdAt > current.createdAt) return { kind: "missing_evidence", reason: "newer_progress_barrier" };
const profile = record(source.runnerProfileJson), snapshot = record(profile.sessionCheckpoint);
const identity = record(snapshot.identity);
if (identity.companyId !== input.companyId || identity.agentId !== input.agentId
|| identity.issueId !== input.issueId || identity.runId !== source.id || identity.sessionId !== input.normalizedSessionId) {
return { kind: "missing_evidence", reason: "snapshot_binding" };
}
if (record(profile.nativeControllerRecovery).schema !== "paperclip.native-controller-recovery.v1") {
return { kind: "unsupported", reason: "metadata_format" };
}
const metadata = readNativeControllerRecovery(profile.nativeControllerRecovery, snapshot);
if (!metadata?.checkpoint || metadata.checkpoint.runId !== source.id
|| metadata.checkpoint.normalizedSessionId !== input.normalizedSessionId
|| metadata.checkpoint.runnerInstanceId !== source.runnerInstanceId) {
return { kind: "missing_evidence", reason: "snapshot_digest_or_checkpoint_binding" };
}
let checkpoint: DurablePrpCheckpoint | null;
try {
checkpoint = await loadNativeRunnerCheckpoint(db, { companyId: input.companyId, issueId: input.issueId, runId: source.id });
} catch (cause) {
return { kind: "dependency_unavailable", reason: "checkpoint_authority_unreadable", cause };
}
if (!checkpoint || checkpoint.recoveryEvidence.runnerInstanceId !== metadata.checkpoint.runnerInstanceId
|| checkpoint.recoveryEvidence.normalizedSessionId !== input.normalizedSessionId) {
return { kind: "missing_evidence", reason: "checkpoint_binding" };
}
return { kind: "recoverable", snapshot: snapshot as unknown as PersistedNativeSession, metadata, checkpoint };
}
/** Bounded internal migration of ONE retained source while its disk is available.
* The migration owner supplies compatibility computed from the admitted input.
* This never repairs/replaces the saved snapshot or claims a live execution. */
export async function importNativeControllerRecovery(db: Db, input: {
companyId: string; agentId: string; issueId: string; runId: string; normalizedSessionId: string;
expectedConfigDigest: string;
supervisor: unknown;
}): Promise<{ kind: "imported" | "unchanged" } | Exclude<NativeControllerRecoveryOutcome, { kind: "recoverable" } | { kind: "no_prior_execution" }>> {
const supervisor = record(input.supervisor);
if (supervisor.schema !== "paperclip.native-session-supervisor.v1" || supervisor.configDigest !== input.expectedConfigDigest) {
return { kind: "unsupported", reason: "supervisor_compatibility" };
}
return db.transaction(async tx => {
const [task] = await tx.select({ executionRunId: issues.executionRunId }).from(issues)
.where(and(eq(issues.companyId, input.companyId), eq(issues.id, input.issueId))).for("share");
const [owner] = await tx.select().from(nativeRunFinalizations).where(and(
eq(nativeRunFinalizations.companyId, input.companyId), eq(nativeRunFinalizations.issueId, input.issueId),
eq(nativeRunFinalizations.runId, input.runId))).for("update");
if (!task || !owner) return { kind: "missing_evidence" as const, reason: "owner_binding" };
if (owner.leaseOwner || task.executionRunId) {
return owner.leaseExpiresAt && owner.leaseExpiresAt > new Date()
? { kind: "waiting_for_owner" as const, deadline: owner.leaseExpiresAt }
: { kind: "missing_evidence" as const, reason: "normal_ownership_recovery_required" };
}
if (owner.phase !== "committed" || !owner.resultId || owner.recoveryState) {
return { kind: "missing_evidence" as const, reason: "pending_finalization" };
}
const [run] = await tx.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, input.runId),
eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.agentId, input.agentId),
eq(heartbeatRuns.nativeIssueId, input.issueId), eq(heartbeatRuns.nativeSessionId, input.normalizedSessionId))).for("update");
const profile = record(run?.runnerProfileJson), snapshot = record(profile.sessionCheckpoint), identity = record(snapshot.identity);
if (run?.runtimeMode !== "native" || run.status !== "succeeded" || !run.finishedAt
|| identity.runId !== input.runId || identity.companyId !== input.companyId || identity.issueId !== input.issueId
|| identity.agentId !== input.agentId || identity.sessionId !== input.normalizedSessionId
|| nativeSha256(snapshot) !== nativeSha256(redactRunnerApiPayloads(supervisor.snapshot))) {
return { kind: "missing_evidence" as const, reason: "stale_supervisor_snapshot" };
}
const checkpoint = await loadNativeRunnerCheckpoint(tx, input);
if (!checkpoint || checkpoint.recoveryEvidence.normalizedSessionId !== input.normalizedSessionId
|| checkpoint.recoveryEvidence.runnerInstanceId !== run.runnerInstanceId) {
return { kind: "missing_evidence" as const, reason: "checkpoint_binding" };
}
const metadata = createNativeControllerRecovery(snapshot as unknown as PersistedNativeSession, {
configDigest: input.expectedConfigDigest,
processConfigDigest: typeof supervisor.processConfigDigest === "string" ? supervisor.processConfigDigest : null,
}, { runId: input.runId, runnerInstanceId: checkpoint.recoveryEvidence.runnerInstanceId, normalizedSessionId: input.normalizedSessionId });
if (!readNativeControllerRecovery(metadata, snapshot)) return { kind: "unsupported" as const, reason: "metadata_format" };
if (profile.nativeControllerRecovery) {
return nativeSha256(profile.nativeControllerRecovery) === nativeSha256(metadata)
? { kind: "unchanged" as const }
: { kind: "missing_evidence" as const, reason: "existing_metadata_conflict" };
}
await tx.update(heartbeatRuns).set({ runnerProfileJson: { ...profile, nativeControllerRecovery: metadata }, updatedAt: new Date() })
.where(eq(heartbeatRuns.id, input.runId));
return { kind: "imported" as const };
});
}
@@ -189,6 +189,8 @@ async function sessionRetentionBlocker(
id: heartbeatRuns.id,
status: heartbeatRuns.status,
finishedAt: heartbeatRuns.finishedAt,
incompleteRecovery: sql<boolean>`(${heartbeatRuns.runnerProfileJson}->'nativeControllerRecovery'->>'schema' = 'paperclip.native-controller-recovery.v1'
and ${heartbeatRuns.runnerProfileJson}->'nativeControllerRecovery'->'checkpoint' = 'null'::jsonb) is true`,
issueId: sql<string | null>`coalesce(${heartbeatRuns.nativeIssueId}, ${heartbeatRuns.issueId})`,
})
.from(heartbeatRuns)
@@ -202,6 +204,7 @@ async function sessionRetentionBlocker(
if (runs.some((run) => LIVE_RUN_STATUSES.includes(run.status) || !run.finishedAt)) {
return { reason: "live_run" };
}
if (runs.some(run => run.incompleteRecovery)) return { reason: "native_recovery_incomplete" };
if (input.checkpoint && runs.some(run => run.finishedAt! >= input.closedBefore)) return { reason: "recent_run" };
const issueIds = [...new Set(runs.map((run) => run.issueId).filter((id): id is string => Boolean(id)))];
if (issueIds.length === 0) return { reason: "issue_unknown" };
@@ -0,0 +1,92 @@
import { randomUUID } from "node:crypto";
import { PgDialect } from "drizzle-orm/pg-core";
import type { SQL } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { DurableRecoveryIdentity, PrpEvent } from "../../vendor/paperclip-runner/index.js";
import { buildNativeExecutionInput } from "./native-execution-input.js";
import { nativeRuntimeContextFixture } from "./runtime-context.test-fixture.js";
import { createNativeRecoveryUsageDelivery } from "./native-recovery-usage-delivery.js";
const accounting = vi.hoisted(() => ({ open: vi.fn(), capture: vi.fn() }));
vi.mock("./native-usage-delivery.js", () => ({ createNativeUsageContinuationDelivery: accounting.open }));
beforeEach(() => { vi.clearAllMocks(); accounting.capture.mockResolvedValue(undefined); accounting.open.mockResolvedValue({ capture: accounting.capture }); });
function fixture() {
const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), sessionId = randomUUID();
const source: DurableRecoveryIdentity = { runId: randomUUID(), runnerInstanceId: randomUUID(), environmentLeaseId: randomUUID(),
normalizedSessionId: sessionId, turnId: randomUUID(), itemId: randomUUID() };
const build = (runId: string) => buildNativeExecutionInput({ companyId, agentId, runId, normalizedSessionId: sessionId,
provider: "codex", issue: { id: issueId, identifier: "REC-1", title: "Continue", description: null, workMode: "standard" },
taskPrompt: "Continue once", workspace: { id: randomUUID(), cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null },
completionContract: { id: randomUUID(), sha256: `sha256:${"a".repeat(64)}`, schemaVersion: "paperclip.run-result.v1",
contract: { revision: "1", objective: "Continue", criteria: [{ id: "objective", requirement: "Preserve the result" }] } },
runtimeContext: nativeRuntimeContextFixture() });
const execution = build(randomUUID()), previous = build(source.runId);
const row = { run: { status: "succeeded", finishedAt: new Date() as Date | null, runtimeMode: "native", runnerInstanceId: source.runnerInstanceId,
nativeSessionId: sessionId, runnerProfileJson: { nativeExecutionInput: previous } },
owner: { phase: "committed", resultId: randomUUID() as string | null, leaseOwner: null as string | null, companyId, issueId } };
const where = vi.fn(async (_predicate: SQL) => [row]);
const select = vi.fn(() => ({ from: () => ({ innerJoin: () => ({ where }) }) }));
const db = { select } as unknown as Db;
const matchesExecution = vi.fn(() => true);
const deliver = () => createNativeRecoveryUsageDelivery({ db, execution, sourceIdentity: { ...source }, matchesExecution });
const event: PrpEvent = { schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", sourceInstanceId: source.runnerInstanceId,
sourceEventId: "source-usage", sourceSeq: 1, runId: source.runId, normalizedSessionId: sessionId, turnId: source.turnId,
eventType: "usage.reported", priority: 0, emittedAt: new Date().toISOString(), payload: {} };
return { source, execution, previous, row, select, where, matchesExecution, deliver, event };
}
describe("source accounting recovery authority", () => {
it("pins the relational source tuple and hydrates once without binding target accounting", async () => {
const f = fixture(), deliver = f.deliver();
await deliver(f.event, f.source);
await deliver({ ...f.event, sourceEventId: "source-terminal", sourceSeq: 2, eventType: "turn.completed" }, f.source);
expect(f.select).toHaveBeenCalledOnce();
expect(new PgDialect().sqlToQuery(f.where.mock.calls[0][0]).params).toEqual([
f.source.runId, f.execution.binding.companyId, f.execution.binding.agentId, f.execution.binding.issueId,
]);
expect(f.matchesExecution).toHaveBeenCalledWith(f.previous);
expect(accounting.open).toHaveBeenCalledExactlyOnceWith(expect.anything(), {
companyId: f.execution.binding.companyId, runId: f.source.runId, runnerInstanceId: f.source.runnerInstanceId, provider: f.previous.provider,
});
expect(accounting.capture).toHaveBeenCalledTimes(2);
expect(accounting.capture).toHaveBeenCalledWith(f.event);
});
it.each(["runId", "runnerInstanceId", "environmentLeaseId", "normalizedSessionId", "turnId", "itemId"] as const)(
"rejects a changed pinned %s before source hydration", async key => {
const f = fixture();
await expect(f.deliver()(f.event, { ...f.source, [key]: randomUUID() })).rejects.toThrow("native_recovery_usage_source_unproven");
expect(f.select).not.toHaveBeenCalled();
expect(accounting.open).not.toHaveBeenCalled();
});
it.each(["runId", "sourceInstanceId", "normalizedSessionId"] as const)("rejects mismatched event %s before source hydration", async key => {
const f = fixture();
await expect(f.deliver()({ ...f.event, [key]: randomUUID() }, f.source)).rejects.toThrow("native_recovery_usage_source_unproven");
expect(f.select).not.toHaveBeenCalled();
});
it.each(["nonterminal", "unfinished", "uncommitted", "no-result", "held", "company", "issue", "scope"])(
"keeps an unproven source held: %s", async change => {
const f = fixture();
if (change === "nonterminal") f.row.run.status = "running";
if (change === "unfinished") f.row.run.finishedAt = null;
if (change === "uncommitted") f.row.owner.phase = "observed";
if (change === "no-result") f.row.owner.resultId = null;
if (change === "held") f.row.owner.leaseOwner = "live-owner";
if (change === "company") f.row.owner.companyId = randomUUID();
if (change === "issue") f.row.owner.issueId = randomUUID();
if (change === "scope") f.matchesExecution.mockReturnValue(false);
await expect(f.deliver()(f.event, f.source)).rejects.toThrow("native_recovery_usage_source_unproven");
expect(accounting.open).not.toHaveBeenCalled();
expect(accounting.capture).not.toHaveBeenCalled();
});
it("keeps a durable source receipt failure unacknowledged", async () => {
const f = fixture();
accounting.capture.mockRejectedValue(new Error("native_usage_after_finalization_requires_reconciliation"));
await expect(f.deliver()(f.event, f.source)).rejects.toThrow("native_usage_after_finalization_requires_reconciliation");
});
});
@@ -0,0 +1,47 @@
import { and, eq } from "drizzle-orm";
import { heartbeatRuns, nativeRunFinalizations, type Db } from "@paperclipai/db";
import { parseNativeExecutionInput, type DurableRecoveryIdentity, type NativeExecutionInput, type PrpEvent } from "../../vendor/paperclip-runner/index.js";
import { createNativeUsageContinuationDelivery } from "./native-usage-delivery.js";
const identityKeys = ["runId", "runnerInstanceId", "environmentLeaseId", "normalizedSessionId", "turnId", "itemId"] as const;
/** A cold handoff may discover accounting that belongs to its completed source.
* Hydrate that producer once; never route its receipts through the new run. */
export function createNativeRecoveryUsageDelivery(input: {
db: Db;
execution: NativeExecutionInput;
sourceIdentity: Record<string, unknown>;
matchesExecution(previous: NativeExecutionInput): boolean;
}) {
let delivery: ReturnType<typeof createNativeUsageContinuationDelivery> | undefined;
const deny = (): never => { throw new Error("native_recovery_usage_source_unproven"); };
const open = async (source: DurableRecoveryIdentity) => {
const { binding } = input.execution;
const [prior] = await input.db.select({ run: heartbeatRuns, owner: nativeRunFinalizations })
.from(heartbeatRuns).innerJoin(nativeRunFinalizations, eq(nativeRunFinalizations.runId, heartbeatRuns.id))
.where(and(eq(heartbeatRuns.id, source.runId), eq(heartbeatRuns.companyId, binding.companyId),
eq(heartbeatRuns.agentId, binding.agentId), eq(heartbeatRuns.nativeIssueId, binding.issueId)));
if (!prior || prior.run.status !== "succeeded" || prior.run.finishedAt === null ||
prior.run.runtimeMode !== "native" || prior.run.runnerInstanceId !== source.runnerInstanceId ||
prior.run.nativeSessionId !== source.normalizedSessionId || prior.owner.phase !== "committed" ||
!prior.owner.resultId || prior.owner.leaseOwner || prior.owner.companyId !== binding.companyId ||
prior.owner.issueId !== binding.issueId) return deny();
const previous = parseNativeExecutionInput(prior.run.runnerProfileJson?.nativeExecutionInput);
if (previous.binding.companyId !== binding.companyId || previous.binding.agentId !== binding.agentId ||
previous.binding.issueId !== binding.issueId || previous.binding.runId !== source.runId ||
!input.matchesExecution(previous)) return deny();
return createNativeUsageContinuationDelivery(input.db, {
companyId: binding.companyId, runId: source.runId,
runnerInstanceId: source.runnerInstanceId, provider: previous.provider,
});
};
return async (event: PrpEvent, source: DurableRecoveryIdentity) => {
if (source.runId === input.execution.binding.runId ||
identityKeys.some(key => typeof input.sourceIdentity[key] !== "string" ||
!input.sourceIdentity[key] || source[key] !== input.sourceIdentity[key]) ||
event.runId !== source.runId || event.sourceInstanceId !== source.runnerInstanceId ||
event.normalizedSessionId !== source.normalizedSessionId) return deny();
delivery ??= open(source);
await (await delivery).capture(event);
};
}
@@ -1,6 +1,6 @@
import { QueryBuilder } from "drizzle-orm/pg-core";
import { and, asc, eq, gt, inArray, isNotNull, isNull, notExists, or, sql } from "drizzle-orm";
import { heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults, statusDecisions, statusDecisionEffects } from "@paperclipai/db";
import { and, asc, eq, exists, gt, inArray, isNotNull, isNull, notExists, or, sql } from "drizzle-orm";
import { heartbeatRunEvents, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults, statusDecisions, statusDecisionEffects } from "@paperclipai/db";
import type { NativeResponsePresentationScope } from "../native-response-presentation-notifications.js";
export function nativeResponsePresentationCandidates(scope?: NativeResponsePresentationScope, after?: string, runId?: string) {
@@ -30,7 +30,24 @@ export function nativeResponsePresentationCandidates(scope?: NativeResponsePrese
sql`${nativeRunResults.resultJson} #>> '{terminal,turnTerminalState}' = 'completed'`,
or(
and(eq(heartbeatRuns.status, "succeeded"),
sql`(coalesce(${issues.conversationAgentId}::text, '') = '' or coalesce(${issues.conversationUserId}, '') = '')`,
or(sql`(coalesce(${issues.conversationAgentId}::text, '') = '' or coalesce(${issues.conversationUserId}, '') = '')`,
and(eq(statusDecisions.reasonCode, "conversation_turn_finished"),
eq(issues.conversationAgentId, heartbeatRuns.agentId), eq(issues.conversationUserId, heartbeatRuns.responsibleUserId),
eq(issues.status, "in_progress"), eq(issues.conversationState, "active"),
or(isNull(issues.executionRunId), eq(issues.executionRunId, heartbeatRuns.id)),
sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issues.id}::text`,
sql`jsonb_typeof(${heartbeatRuns.contextSnapshot}->'conversationSessionGeneration') = 'number'`,
sql`${heartbeatRuns.contextSnapshot}->>'conversationSessionGeneration' = ${issues.conversationSessionGeneration}::text`,
exists(db.select({ id: heartbeatRunEvents.id }).from(heartbeatRunEvents).where(and(
eq(heartbeatRunEvents.companyId, heartbeatRuns.companyId), eq(heartbeatRunEvents.runId, heartbeatRuns.id),
eq(heartbeatRunEvents.agentId, heartbeatRuns.agentId), eq(heartbeatRunEvents.eventType, "item.completed"),
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,runId}' = ${heartbeatRuns.id}::text`,
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,normalizedSessionId}' = ${heartbeatRuns.nativeSessionId}::text`,
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,turnId}' = ${nativeRunResults.turnId}`,
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,payload,kind}' = 'agentMessage'`,
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,payload,channel}' = 'final'`,
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,payload,text}' ~ '[^[:space:]]'`,
))))),
sql`${nativeRunResults.resultJson} #>> '{result,reportedWorkDisposition}' = 'done'`,
sql`${nativeRunResults.resultJson} #>> '{terminal,reportedWorkDisposition}' = 'done'`),
and(sql`${nativeRunResults.resultJson} #>> '{result,reportedWorkDisposition}' = 'yielded'`,
@@ -56,6 +56,7 @@ export function nativeResponsePresentationWork(db: Db) {
function register(coordinator: ReturnType<typeof createDeliveryWorkCoordinator>, canRun: () => boolean) {
const worker = registerScopedDeadlineWork(coordinator, {
queue: DELIVERY_QUEUES.nativeResponsePresentation,
concurrentChecks: { limit: 4, key: (entry: Entry) => `${entry.companyId}:${entry.issueId}` },
decodeScope: (hint: string): NativeResponsePresentationScope | undefined => hint === "*" ? undefined : JSON.parse(hint),
matches: (entry: Entry, scope?: NativeResponsePresentationScope) => !scope || entry.companyId === scope.companyId && (!scope.issueId || entry.issueId === scope.issueId) && (!scope.runId || entry.id === scope.runId) && (!scope.agentId || entry.agentId === scope.agentId),
load: (scope, signal) => load(scope, signal, canRun),
@@ -8,6 +8,18 @@ import {
} from "./native-restart-recovery.js";
describe("native restart recovery classification", () => {
it.each([false, true])("recognizes verified staged remote evidence with target snapshot=%s", hasCheckpoint => {
const evidence = { remoteSandbox: true, runnerPidAlive: false, runnerGroupAlive: false,
processStartMatches: false, hasCheckpoint, hasProviderEvidence: false };
expect(classifyNativeRunnerRecoveryEvidence(evidence).claimKind).toBeNull();
expect(classifyNativeRunnerRecoveryEvidence({ ...evidence, stagedAttachmentVerified: true })).toEqual({
claimKind: "reattach_remote_runner", reason: "staged_remote_attachment_requires_sandbox_verification",
});
expect(classifyNativeRunnerRecoveryEvidence({ ...evidence, stagedAttachmentVerified: true,
checkpointFailed: true }).claimKind).toBeNull();
expect(classifyNativeRunnerRecoveryEvidence({ ...evidence, remoteSandbox: false,
stagedAttachmentVerified: true }).claimKind).toBe(hasCheckpoint ? null : "bootstrap_incomplete");
});
it.each([true, false])("uses the remote fenced lease instead of the old container PID (live=%s)", async live => {
const now = new Date();
const probe = vi.fn(() => { throw new Error("must not probe another container's PID"); });
@@ -1,3 +1,4 @@
import { NATIVE_RETAINED_OWNER_WAIT } from "./native-controller-recovery.js";
import { notifyNativeEnvironmentSettlement } from "../native-environment-settlement-notifications.js";
import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
import { notifyNativeStatusReassessment } from "../native-status-reassessment-notifications.js";
@@ -6,7 +7,8 @@ import { notifyCancelledWorkspacePreservation } from "../cancelled-workspace-pre
import { notifyInstructionCollection } from "../instruction-collection-notifications.js";
import { notifyExecutionRunWork } from "../execution-control-notifications.js";
import { preserveNativeFailedWorkspace } from "./native-failed-workspace-recovery.js";
import { hasNativeRunnerCheckpoint } from "./native-runner-checkpoint.js";
import { hasNativeRunnerCheckpoint, loadNativeRunnerCheckpointHeader } from "./native-runner-checkpoint.js";
import { resolveNativeStagedAttachmentRecovery, type NativeStagedAttachmentRecovery } from "./native-staged-attachment-recovery.js";
import { readNativeWorkspaceSyncReference } from "./native-workspace-sync.js";
import { activeNativeExecutionRunIds, NATIVE_PREPARATION_FENCING_VERSION } from "./native-active-runs.js";
import {
@@ -15,7 +17,7 @@ import {
recordNativeLocalProcessStop,
} from "../native-local-process-stop.js";
import { randomUUID } from "node:crypto";
import { and, desc, eq, inArray, isNotNull, isNull, lte, or, sql } from "drizzle-orm";
import { and, desc, eq, inArray, isNotNull, isNull, lte, ne, or, sql } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import {
agents,
@@ -68,6 +70,8 @@ export type NativeRestartRecoveryClaim =
recoveryRequestId: string | null;
remote: { kind?: "sandbox"; providerLeaseId: string; remoteCwd: string }
| { kind: "plugin_task"; leaseId: string };
/** Discovery evidence only; executor revalidates the encrypted header. */
stagedAttachment?: Pick<NativeStagedAttachmentRecovery, "sourceIdentity" | "targetIdentity" | "templateDigest">;
}
| {
kind: "resume_dead_runner";
@@ -94,6 +98,7 @@ export type NativeRestartRecoveryDisposition =
kind: "awaiting_evidence" | "blocked" | "already_finalized";
runId: string;
reason: string;
retryAt?: Date;
};
export function nextNativeProviderAttempt(
@@ -320,6 +325,7 @@ export function classifyNativeRunnerRecoveryEvidence(input: {
checkpointIdentityMatches?: boolean;
hasProviderEvidence: boolean;
checkpointFailed?: boolean;
stagedAttachmentVerified?: boolean;
providerAttempt?: number;
}): {
claimKind: NativeRestartRecoveryClaim["kind"] | null;
@@ -334,6 +340,9 @@ export function classifyNativeRunnerRecoveryEvidence(input: {
// inspection; the executor must prove the original lease and authenticate
// the surviving runner before it can resume delivery.
if (input.remoteSandbox) {
if (input.stagedAttachmentVerified === true) {
return { claimKind: "reattach_remote_runner", reason: "staged_remote_attachment_requires_sandbox_verification" };
}
if (input.unclaimedBootstrap === true && !input.hasCheckpoint && !input.hasProviderEvidence) {
return { claimKind: "bootstrap_incomplete", reason: "remote_execution_never_claimed" };
}
@@ -514,6 +523,9 @@ export async function claimNativeRestartRecoveries(input: {
eq(agents.adapterType, "paperclip_runner"),
inArray(heartbeatRuns.status, ["running", "failed"]),
isNull(nativeRunFinalizations.resultId),
// A pre-provider retained-owner wait belongs to normal same-run
// resumption, including when its deadline elapsed while we were down.
or(isNull(nativeRunFinalizations.failureCode), ne(nativeRunFinalizations.failureCode, NATIVE_RETAINED_OWNER_WAIT)),
or(
eq(nativeRunFinalizations.phase, "observed"),
and(
@@ -668,9 +680,21 @@ export async function claimNativeRestartRecoveries(input: {
kind: "awaiting_evidence",
runId: row.run.id,
reason: takeover.reason,
...(durableAuthority && row.coordinator.leaseExpiresAt ? { retryAt: row.coordinator.leaseExpiresAt } : {}),
} as const;
}
// Only the classic remote path stages attachments. After the owner is
// eligible for takeover, inspect its bounded header even if activation
// already published a runtime snapshot. Never scan command/event payloads.
const stagedHeader = remoteWorkspace && !pluginTask && durableAuthority
? await loadNativeRunnerCheckpointHeader(tx, {
companyId: row.run.companyId, issueId: row.coordinator.issueId, runId: row.run.id,
}) : null;
const stagedAttachment = stagedHeader ? await resolveNativeStagedAttachmentRecovery({
db: tx, currentRun: row.run, header: stagedHeader,
}) : null;
const runnerPidAlive =
!remoteProcess && processIsAlive(row.run.processPid);
const runnerGroupAlive =
@@ -859,6 +883,7 @@ export async function claimNativeRestartRecoveries(input: {
knownProviderProcessIdentityAmbiguous:
providerProcesses.ambiguousLivePids.length > 0,
hasCheckpoint,
stagedAttachmentVerified: stagedAttachment !== null,
checkpointIdentityMatches:
checkpointIdentityMatches &&
(!remoteSandbox || remoteWorkspace !== null),
@@ -1109,6 +1134,10 @@ export async function claimNativeRestartRecoveries(input: {
return {
kind: claimKind,
...common,
...(stagedAttachment ? { stagedAttachment: {
sourceIdentity: stagedAttachment.sourceIdentity, targetIdentity: stagedAttachment.targetIdentity,
templateDigest: stagedAttachment.templateDigest,
} } : {}),
remote: pluginTask ? { kind: "plugin_task", leaseId: pluginTask.id } : {
providerLeaseId: remoteWorkspace!.providerLeaseId,
remoteCwd: remoteWorkspace!.remoteCwd,
@@ -29,7 +29,7 @@ import { readPersistedNativeFailureEvidence } from "./native-provider-failure-ev
import { dismissAutomaticCompletionReviews } from "./automatic-completion-reviews.js";
import { findSupersededExternalReviewRequests } from "./superseded-external-review-requests.js";
import { getNativeReviewAssignment, readNativeReviewAssignmentContext } from "./native-review-participant.js";
import { conversationNativeDecision, isConversation } from "../agent-conversations.js";
import { conversationNativeDecision, isConversation, settleConversationTurnForRun } from "../agent-conversations.js";
import { issueTreeControlService } from "../issue-tree-control.js";
import { randomUUID } from "node:crypto";
import { preserveNativeWorkspaceExportLease } from "./native-workspace-export-resume.js";
@@ -747,6 +747,7 @@ async function recoverCommittedNativeChatResponseInScope(
): Promise<boolean> {
let agentId: string | null = null;
let materialized = false;
let conversationReply = false;
const entered = beginRunPhase("response.transaction.entry");
try {
materialized = await measureRunPhase("response.transaction", () => db.transaction(async (transaction) => {
@@ -789,6 +790,7 @@ async function recoverCommittedNativeChatResponseInScope(
id: heartbeatRuns.id,
companyId: heartbeatRuns.companyId,
agentId: heartbeatRuns.agentId,
responsibleUserId: heartbeatRuns.responsibleUserId,
nativeIssueId: heartbeatRuns.nativeIssueId,
nativeSessionId: heartbeatRuns.nativeSessionId,
completionContractId: heartbeatRuns.completionContractId,
@@ -803,6 +805,8 @@ async function recoverCommittedNativeChatResponseInScope(
contextSnapshot: sql<Record<string, unknown> | null>`case when ${heartbeatRuns.contextSnapshot} is null then null
else jsonb_build_object(
'skipIssueComment', ${heartbeatRuns.contextSnapshot}->'skipIssueComment',
'issueId', ${heartbeatRuns.contextSnapshot}->'issueId',
'conversationSessionGeneration', ${heartbeatRuns.contextSnapshot}->'conversationSessionGeneration',
'source', ${heartbeatRuns.contextSnapshot}->'source',
'externalChatContinuation', ${heartbeatRuns.contextSnapshot}->'externalChatContinuation',
'paperclipWake', jsonb_build_object('externalInteractionContinuation',
@@ -909,12 +913,16 @@ async function recoverCommittedNativeChatResponseInScope(
agentId = run.agentId;
return true;
}
// Copyback can be owned by reconciliation before the live heartbeat
// reaches its presentation step. Recover a completed internal task's
// exact final reply from the same accepted turn, without rerunning work
// or granting external-chat publication authority.
const conversation = isConversation(issue);
const currentConversation = conversation && decision.reasonCode === "conversation_turn_finished"
&& issue.conversationAgentId === run.agentId && issue.conversationUserId === run.responsibleUserId
&& issue.status === "in_progress" && issue.conversationState === "active"
&& (!issue.executionRunId || issue.executionRunId === run.id)
&& run.contextSnapshot?.issueId === issue.id
&& typeof run.contextSnapshot.conversationSessionGeneration === "number"
&& run.contextSnapshot.conversationSessionGeneration === issue.conversationSessionGeneration;
if (
!isConversation(issue) &&
(!conversation || currentConversation) &&
run.status === "succeeded" &&
record(run.contextSnapshot).skipIssueComment !== true &&
!isExternalChatPresentationContext(run.contextSnapshot) &&
@@ -929,6 +937,9 @@ async function recoverCommittedNativeChatResponseInScope(
eq(heartbeatRunEvents.companyId, input.companyId),
eq(heartbeatRunEvents.runId, run.id),
eq(heartbeatRunEvents.eventType, "item.completed"),
conversation ? and(eq(heartbeatRunEvents.agentId, run.agentId),
sql`${heartbeatRunEvents.payload}->'prpEvent'->>'runId' = ${run.id}`,
sql`${heartbeatRunEvents.payload}->'prpEvent'->>'normalizedSessionId' = ${run.nativeSessionId}`) : undefined,
sql`${heartbeatRunEvents.payload}->'prpEvent'->>'turnId' = ${accepted.turnId}`,
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,payload,kind}' = 'agentMessage'`,
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,payload,channel}' = 'final'`,
@@ -963,6 +974,7 @@ async function recoverCommittedNativeChatResponseInScope(
})}::jsonb`, updatedAt: new Date(),
}).where(eq(heartbeatRuns.id, run.id));
agentId = run.agentId;
conversationReply = conversation;
return true;
}
if (
@@ -1039,12 +1051,14 @@ async function recoverCommittedNativeChatResponseInScope(
} finally {
entered();
}
if (materialized && agentId)
if (materialized && agentId) {
if (conversationReply) await settleConversationTurnForRun(db, input.companyId, input.runId);
publishChatPublicationCommitSignal({
...input,
agentId,
eventType: "run.presentation.resolved",
});
}
return materialized;
}
@@ -5,7 +5,7 @@ import { createLocalEncryptedCipher } from "../../secrets/local-encrypted-provid
import { legacyControllerBootId } from "../legacy-controller-lease.js";
type Binding = { companyId: string; issueId: string; runId: string };
type Owner = Binding & { bootId: string; leaseOwner: string; generation: number; authorityRunId: string };
export type NativeRunnerCheckpointOwner = Binding & { bootId: string; leaseOwner: string; generation: number; authorityRunId: string };
function encryptParts(binding: Binding, records: ReadonlyMap<string, string>) {
if (!records.size) return [];
const cipher = createLocalEncryptedCipher("write");
@@ -25,49 +25,79 @@ function decryptPart(cipher: ReturnType<typeof createLocalEncryptedCipher>, bind
return envelope.contents;
}
export function nativeRunnerCheckpointWriter(db: Db, owner: Owner) {
type CheckpointTransaction = Parameters<Parameters<Db["transaction"]>[0]>[0];
/** A dependent run-lock query can consume this CTE to preserve coordinator-first
* lock order without adding a second database round trip. */
export function nativeRunnerCheckpointOwnerLock(tx: CheckpointTransaction, owner: NativeRunnerCheckpointOwner) {
return tx.$with("checkpoint_owner").as(tx.select({
leaseOwner: nativeRunFinalizations.leaseOwner,
leaseExpiresAt: nativeRunFinalizations.leaseExpiresAt,
resultId: nativeRunFinalizations.resultId,
}).from(nativeRunFinalizations).where(and(
eq(nativeRunFinalizations.companyId, owner.companyId), eq(nativeRunFinalizations.issueId, owner.issueId),
eq(nativeRunFinalizations.runId, owner.authorityRunId), eq(nativeRunFinalizations.controllerBootId, owner.bootId),
eq(nativeRunFinalizations.controllerGeneration, owner.generation))).for("update"));
}
/** Locks the coordinator before callers lock a run. A caller that subsequently
* waits for another lock must recheck the fence before writing. */
export async function lockNativeRunnerCheckpointOwner(
tx: CheckpointTransaction, owner: NativeRunnerCheckpointOwner,
) {
// A locking CTE acquires ownership before the outer projection samples
// wall time. Sampling in the locking SELECT can precede its lock wait.
// Keep both steps in one statement without another database round trip.
const locked = nativeRunnerCheckpointOwnerLock(tx, owner);
const [owned] = await tx.with(locked).select({ leaseOwner: locked.leaseOwner,
leaseExpiresAt: locked.leaseExpiresAt, resultId: locked.resultId, now: sql<Date>`clock_timestamp()` }).from(locked);
if (!owned) throw new Error("native_runner_checkpoint_owner_lost");
return owned;
}
export async function validateNativeRunnerCheckpointOwner(
tx: CheckpointTransaction, owner: NativeRunnerCheckpointOwner,
owned: Awaited<ReturnType<typeof lockNativeRunnerCheckpointOwner>>, newWork: boolean,
): Promise<void> {
const current = owned;
const activeOwner = current.leaseOwner === owner.leaseOwner
&& current.leaseExpiresAt !== null && current.leaseExpiresAt > new Date(owned.now);
if (!activeOwner) {
const [task] = await tx.select({ executionRunId: issues.executionRunId }).from(issues)
.where(and(eq(issues.companyId, owner.companyId), eq(issues.id, owner.issueId)));
const [successor] = task?.executionRunId && task.executionRunId !== owner.runId
? await tx.select({ runId: nativeRunFinalizations.runId }).from(nativeRunFinalizations).where(and(
eq(nativeRunFinalizations.runId, task.executionRunId), eq(nativeRunFinalizations.companyId, owner.companyId),
eq(nativeRunFinalizations.issueId, owner.issueId), eq(nativeRunFinalizations.controllerBootId, owner.bootId),
gt(nativeRunFinalizations.leaseExpiresAt, sql`clock_timestamp()`))) : [];
const [preparingSuccessor] = !successor && task?.executionRunId && task.executionRunId !== owner.runId
? await tx.select({ runId: heartbeatRuns.id }).from(heartbeatRuns).where(and(
eq(heartbeatRuns.id, task.executionRunId), eq(heartbeatRuns.companyId, owner.companyId),
eq(heartbeatRuns.issueId, owner.issueId), eq(heartbeatRuns.runtimeMode, "legacy"),
eq(heartbeatRuns.status, "running"), eq(heartbeatRuns.executionStage, "preparing"),
eq(heartbeatRuns.controllerBootId, legacyControllerBootId),
gt(heartbeatRuns.controllerLeaseExpiresAt, sql`clock_timestamp()`),
notExists(tx.select({ runId: nativeRunFinalizations.runId }).from(nativeRunFinalizations)
.where(eq(nativeRunFinalizations.runId, heartbeatRuns.id))))) : [];
if (!current.resultId || newWork || !task || (task.executionRunId && task.executionRunId !== owner.runId && !successor && !preparingSuccessor)) {
throw new Error("native_runner_checkpoint_owner_lost");
}
}
}
export async function assertNativeRunnerCheckpointOwner(tx: CheckpointTransaction, owner: NativeRunnerCheckpointOwner, newWork: boolean) {
const locked = await lockNativeRunnerCheckpointOwner(tx, owner);
await validateNativeRunnerCheckpointOwner(tx, owner, locked, newWork);
}
export function nativeRunnerCheckpointWriter(db: Db, owner: NativeRunnerCheckpointOwner) {
const persistenceDb = checkpointDb(db);
let previousWork = new Set<string>();
return async (checkpoint: DurablePrpCheckpoint): Promise<void> => {
if (checkpoint.runId !== owner.runId) throw new Error("native_runner_checkpoint_binding_mismatch");
const changed = encryptParts(owner, checkpoint.changedRecords);
await persistenceDb.transaction(async tx => {
// A locking CTE acquires ownership before the outer projection samples
// wall time. Sampling in the locking SELECT can precede its lock wait.
// Keep both steps in one statement without another database round trip.
const locked = tx.$with("checkpoint_owner").as(tx.select().from(nativeRunFinalizations)
.where(and(eq(nativeRunFinalizations.companyId, owner.companyId),
eq(nativeRunFinalizations.issueId, owner.issueId), eq(nativeRunFinalizations.runId, owner.authorityRunId),
eq(nativeRunFinalizations.controllerBootId, owner.bootId),
eq(nativeRunFinalizations.controllerGeneration, owner.generation))).for("update"));
const [owned] = await tx.with(locked).select({ leaseOwner: locked.leaseOwner,
leaseExpiresAt: locked.leaseExpiresAt, resultId: locked.resultId, now: sql<Date>`clock_timestamp()` }).from(locked);
if (!owned) throw new Error("native_runner_checkpoint_owner_lost");
const current = owned;
const activeOwner = current.leaseOwner === owner.leaseOwner
&& current.leaseExpiresAt !== null && current.leaseExpiresAt > new Date(owned.now);
if (!activeOwner) {
const newWork = checkpoint.workCommandIds.some(id => !previousWork.has(id));
const [task] = await tx.select({ executionRunId: issues.executionRunId }).from(issues)
.where(and(eq(issues.companyId, owner.companyId), eq(issues.id, owner.issueId)));
const [successor] = task?.executionRunId && task.executionRunId !== owner.runId
? await tx.select({ runId: nativeRunFinalizations.runId }).from(nativeRunFinalizations).where(and(
eq(nativeRunFinalizations.runId, task.executionRunId), eq(nativeRunFinalizations.companyId, owner.companyId),
eq(nativeRunFinalizations.issueId, owner.issueId), eq(nativeRunFinalizations.controllerBootId, owner.bootId),
gt(nativeRunFinalizations.leaseExpiresAt, sql`clock_timestamp()`))) : [];
const [preparingSuccessor] = !successor && task?.executionRunId && task.executionRunId !== owner.runId
? await tx.select({ runId: heartbeatRuns.id }).from(heartbeatRuns).where(and(
eq(heartbeatRuns.id, task.executionRunId), eq(heartbeatRuns.companyId, owner.companyId),
eq(heartbeatRuns.issueId, owner.issueId), eq(heartbeatRuns.runtimeMode, "legacy"),
eq(heartbeatRuns.status, "running"), eq(heartbeatRuns.executionStage, "preparing"),
eq(heartbeatRuns.controllerBootId, legacyControllerBootId),
gt(heartbeatRuns.controllerLeaseExpiresAt, sql`clock_timestamp()`),
notExists(tx.select({ runId: nativeRunFinalizations.runId }).from(nativeRunFinalizations)
.where(eq(nativeRunFinalizations.runId, heartbeatRuns.id))))) : [];
if (!current.resultId || newWork || !task || (task.executionRunId && task.executionRunId !== owner.runId && !successor && !preparingSuccessor)) {
throw new Error("native_runner_checkpoint_owner_lost");
}
}
await assertNativeRunnerCheckpointOwner(tx, owner, checkpoint.workCommandIds.some(id => !previousWork.has(id)));
if (checkpoint.predecessor) {
const [prior] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(
eq(heartbeatRuns.id, checkpoint.predecessor.runId), eq(heartbeatRuns.companyId, owner.companyId),
@@ -150,7 +180,17 @@ export async function registerNativeRunnerCheckpoint(input: {
await input.authority.flushCheckpoint();
}
export async function loadNativeRunnerCheckpoint(db: Db, binding: Binding): Promise<DurablePrpCheckpoint | null> {
/** Exact-run bounded metadata read; never loads command/event payload records. */
export async function loadNativeRunnerCheckpointHeader(db: Pick<Db, "select">, binding: Binding): Promise<string | null> {
const [row] = await db.select({ material: nativeRunnerCheckpoints.material }).from(nativeRunnerCheckpoints).where(and(
eq(nativeRunnerCheckpoints.companyId, binding.companyId), eq(nativeRunnerCheckpoints.issueId, binding.issueId),
eq(nativeRunnerCheckpoints.runId, binding.runId), eq(nativeRunnerCheckpoints.part, "header"))).limit(1);
if (!row) return null;
try { return decryptPart(createLocalEncryptedCipher("read"), binding, "header", row.material); }
catch { return null; }
}
export async function loadNativeRunnerCheckpoint(db: Pick<Db, "select">, binding: Binding): Promise<DurablePrpCheckpoint | null> {
const rows = await db.select().from(nativeRunnerCheckpoints).where(and(
eq(nativeRunnerCheckpoints.companyId, binding.companyId), eq(nativeRunnerCheckpoints.issueId, binding.issueId),
eq(nativeRunnerCheckpoints.runId, binding.runId)));
@@ -6,9 +6,12 @@ import { prepareNativeRemoteRunner, createNativeRunnerExecutionDriver } from "./
const transport = vi.hoisted(() => ({
register: vi.fn(),
connect: vi.fn(),
checkpoint: vi.fn(),
checkpointRegistration: vi.fn(),
}));
vi.mock("../../realtime/runner-prp-ws.js", () => ({ registerRunnerPrpAuthority: transport.register }));
vi.mock("../../realtime/runner-prp-outbound.js", () => ({ connectRunnerPrpIngress: transport.connect }));
vi.mock("./native-runner-checkpoint.js", () => ({ nativeRunnerCheckpointRegistration: transport.checkpointRegistration }));
const binding = { companyId: "company", issueId: "issue", agentId: "agent", runId: "run-a" };
const authority = {} as Parameters<NonNullable<ReturnType<typeof createNativeRunnerExecutionDriver>["controlPlaneRegistration"]>>[0];
@@ -30,9 +33,41 @@ beforeEach(() => {
vi.clearAllMocks();
transport.register.mockResolvedValue({ connectUrl: "ws://127.0.0.1/core", release: vi.fn() });
transport.connect.mockReturnValue({ ready: Promise.resolve(), close: vi.fn(async () => {}) });
transport.checkpointRegistration.mockReturnValue(transport.checkpoint);
transport.checkpoint.mockResolvedValue(undefined);
});
describe("shared Runner execution driver", () => {
it("installs both staged checkpoint writers under the business lease without acquiring endpoints", async () => {
const prepare = vi.fn();
const persistence = { db: {} as never, bootId: "boot-c", authorityRunId: "run-c" };
const driver = createNativeRunnerExecutionDriver({ target: sandbox(), binding: { ...binding, runId: "run-c" },
controllerPersistence: persistence, prepareRemote: prepare });
const sourceIdentity = { runnerInstanceId: "runner", environmentLeaseId: "lease", normalizedSessionId: "session",
runId: "run-a", turnId: "turn-a", itemId: "item-a" };
const targetIdentity = { ...sourceIdentity, runId: "run-c", turnId: "turn-c", itemId: "item-c" };
await driver.onStagedRunAttachment!(authority, { sourceIdentity, targetIdentity, templateDigest: `sha256:${"a".repeat(64)}` });
expect(transport.checkpointRegistration).toHaveBeenCalledExactlyOnceWith({ ...persistence,
binding: { companyId: binding.companyId, issueId: binding.issueId, runId: "run-c" } });
expect(transport.checkpoint.mock.calls).toEqual([[authority, sourceIdentity], [authority, targetIdentity]]);
expect(prepare).not.toHaveBeenCalled();
expect(transport.register).not.toHaveBeenCalled();
expect(transport.connect).not.toHaveBeenCalled();
});
it("installs the target writer first when restoring an already persisted stage", async () => {
const sourceIdentity = { runnerInstanceId: "runner", environmentLeaseId: "lease", normalizedSessionId: "session",
runId: "run-a", turnId: "turn-a", itemId: "item-a" };
const targetIdentity = { ...sourceIdentity, runId: "run-c", turnId: "turn-c", itemId: "item-c" };
const restored = { store: { state: { stagedRunAttachment: { sourceIdentity, targetIdentity } } } } as unknown as typeof authority;
const driver = createNativeRunnerExecutionDriver({ target: sandbox(), binding: { ...binding, runId: "run-c" },
controllerPersistence: { db: {} as never, bootId: "boot-c", authorityRunId: "run-c" } });
await driver.onStagedRunAttachment!(restored, { sourceIdentity, targetIdentity, templateDigest: `sha256:${"a".repeat(64)}` });
expect(transport.checkpoint.mock.calls).toEqual([[restored, targetIdentity], [restored, sourceIdentity]]);
expect(transport.register).not.toHaveBeenCalled();
expect(transport.connect).not.toHaveBeenCalled();
});
it("keeps local execution on the registered loopback route", async () => {
const driver = createNativeRunnerExecutionDriver({ target: { kind: "local" }, binding });
const connection = await driver.controlPlaneRegistration!(authority);
@@ -92,6 +92,20 @@ export async function prepareNativeRemoteRunner(input: RemotePreparation) {
export function createNativeRunnerExecutionDriver(input: NativeRunnerExecutionDriverInput): RunnerExecutionDriver {
return {
...input.process,
onStagedRunAttachment: input.controllerPersistence && input.binding.issueId
? async (authority, proof) => {
// Both journals use the current business lease. Install persistence
// before staging; endpoint acquisition still belongs to attachment.
const register = nativeRunnerCheckpointRegistration({ ...input.controllerPersistence!,
binding: { companyId: input.binding.companyId, issueId: input.binding.issueId!, runId: input.binding.runId },
});
// A restored stage already publishes under the target. Its writer
// must exist before registration flushes that restored revision.
const identities = authority.store?.state.stagedRunAttachment
? [proof.targetIdentity, proof.sourceIdentity] : [proof.sourceIdentity, proof.targetIdentity];
for (const identity of identities) await register(authority, identity);
}
: undefined,
controlPlaneRegistration: async (authority, attachmentIdentity) => {
await input.beforeConnect?.(attachmentIdentity, authority);
if (input.controllerPersistence && input.binding.issueId) {
@@ -60,6 +60,7 @@ import { readProcessStartedAt } from "../hot-restart.js";
import { resolvePaperclipInstanceRoot } from "../../home-paths.js";
import { prepareNativeHeartbeatRun } from "./prepare-native-run.js";
import { buildNativeExecutionInput } from "./native-execution-input.js";
import { buildNativeExecutionWithCheckpoint, findNativeSessionResumeRun, nativeToolContractFingerprintForTarget } from "./native-session-resume.js";
import { nativeRuntimeContextFixture } from "./runtime-context.test-fixture.js";
import { materializeAsset } from "./runtime-context.js";
import { createRunnerTestTlsTunnel, dockerCommandRunner } from "../../__tests__/helpers/docker-command-runner.js";
@@ -321,8 +322,10 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
};
}
it.skipIf(!process.env.PAPERCLIP_TEST_RUNNER_CONTAINER).each(["warm", "hard", "legacy", "changed", "same-server"])("preserves remote warm continuity across completed turns: %s", async (scenario) => {
it.skipIf(!process.env.PAPERCLIP_TEST_RUNNER_CONTAINER).each(["warm", "hard", "legacy", "changed", "same-server", "diskless", "diskless-per-turn", "diskless-per-turn-read-retry"])("preserves remote warm continuity across completed turns: %s", async (scenario) => {
const hardRestart = !["warm", "same-server"].includes(scenario);
const diskless = scenario.startsWith("diskless");
const perTurn = scenario.startsWith("diskless-per-turn");
const container = process.env.PAPERCLIP_TEST_RUNNER_CONTAINER!;
const tunnel = await createRunnerTestTlsTunnel({ port: Number(process.env.PAPERCLIP_TEST_RUNNER_PORT),
key: process.env.PAPERCLIP_TEST_RUNNER_TLS_KEY!, cert: process.env.NODE_EXTRA_CA_CERTS! });
@@ -346,12 +349,13 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
taskPrompt: "Complete the first turn.",
workspace: { id: randomUUID(), cwd: workspace, repoUrl: null, repoRef: null, branchName: null },
normalizedSessionId: fixture.native.normalizedSessionId,
lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 },
lifecyclePolicy: perTurn ? { mode: "per_turn", idleTimeoutMs: null } : { mode: "warm", idleTimeoutMs: 60_000 },
runtimeContext: context,
completionContract: await boundContract(fixture.runId),
});
await fixture.db.update(heartbeatRuns).set({ runnerProfileJson: {
...fixture.preparedProfile, nativeExecutionInput: execution,
nativeToolContractFingerprint: nativeToolContractFingerprintForTarget("remote", execution.provider.kind),
} }).where(eq(heartbeatRuns.id, fixture.runId));
const remoteCwd = `/tmp/paperclip-continuity-${fixture.issueId}`;
expect((await runner.execute({ command: "mkdir", args: ["-p", remoteCwd] })).exitCode).toBe(0);
@@ -385,6 +389,26 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
onLog: async (_stream: "stdout" | "stderr", text: string) => { logs.push(text); },
};
let controller: ReturnType<typeof spawn> | undefined;
const executeInController = (executionOptions: Parameters<typeof executePaperclipNativeSession>[0]) => new Promise<{ exitCode: number | null }>((resolveResult, reject) => {
controller = spawn(process.execPath, ["--import", createRequire(import.meta.url).resolve("tsx"), resolve("server/src/__tests__/helpers/remote-warm-controller.test-worker.ts")],
{ stdio: ["ignore", "pipe", "pipe", "ipc"], env: process.env });
let timer = setTimeout(() => { controller?.kill("SIGKILL"); reject(new Error("Controller fixture imports exceeded 120 seconds")); }, 120_000);
controller.stdout?.on("data", data => logs.push(String(data)));
controller.stderr?.on("data", data => logs.push(String(data)));
controller.on("error", error => { clearTimeout(timer); reject(error); });
controller.on("exit", (code, signal) => { clearTimeout(timer); reject(new Error(`Warm controller exited: ${code}/${signal}: ${logs.join("").slice(-3_000)}`)); });
controller.on("message", (message: { ready?: boolean; log?: { text: string }; result?: { exitCode: number | null }; error?: string }) => {
if (message.ready) {
clearTimeout(timer);
timer = setTimeout(() => { controller?.kill("SIGKILL"); reject(new Error(`Warm controller admission exceeded 60 seconds: ${logs.join("").slice(-3_000)}`)); }, 60_000);
controller!.send?.({ connectionString: temporary.connectionString, container, ingressUrl: `wss://127.0.0.1:${tunnel.port}`,
options: { ...executionOptions, db: undefined, onLog: undefined, runnerExecutionTarget: { ...executionOptions.runnerExecutionTarget, runner: undefined } } });
}
if (message.log) logs.push(message.log.text);
if (message.error) { clearTimeout(timer); reject(new Error(message.error)); }
if (message.result) { clearTimeout(timer); resolveResult(message.result); }
});
});
try {
const firstOptions = { ...options, execution, runnerExecutionTarget: target };
const executeFirst = async () => {
@@ -454,28 +478,13 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
expect(nextProcess.stdout).toBe(priorProcess.stdout);
return result;
};
const first = hardRestart ? await new Promise<{ exitCode: number | null }>((resolveResult, reject) => {
controller = spawn(process.execPath, ["--import", createRequire(import.meta.url).resolve("tsx"), resolve("server/src/__tests__/helpers/remote-warm-controller.test-worker.ts")],
{ stdio: ["ignore", "pipe", "pipe", "ipc"], env: process.env });
const timer = setTimeout(() => { controller?.kill("SIGKILL"); reject(new Error("Warm controller startup exceeded 60 seconds")); }, 60_000);
controller.stdout?.on("data", data => logs.push(String(data)));
controller.stderr?.on("data", data => logs.push(String(data)));
controller.on("error", error => { clearTimeout(timer); reject(error); });
controller.on("exit", (code, signal) => { clearTimeout(timer); reject(new Error(`Warm controller exited: ${code}/${signal}: ${logs.join("").slice(-3_000)}`)); });
controller.on("message", (message: { log?: { text: string }; result?: { exitCode: number | null }; error?: string }) => {
if (message.log) logs.push(message.log.text);
if (message.error) { clearTimeout(timer); reject(new Error(message.error)); }
if (message.result) { clearTimeout(timer); resolveResult(message.result); }
});
controller.send?.({ connectionString: temporary.connectionString, container, ingressUrl: `wss://127.0.0.1:${tunnel.port}`,
options: { ...firstOptions, db: undefined, onLog: undefined, runnerExecutionTarget: { ...target, runner: undefined } } });
}) : await executeFirst();
const first = hardRestart ? await executeInController(firstOptions) : await executeFirst();
expect(first.exitCode).toBe(0);
await finalizeNativeRun({ db: fixture.db, runId: fixture.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
const snapshot = await runner.execute({ command: "sh", args: ["-c", `cat '${remoteCwd}'/.paperclip-runtime/paperclip-runner/sessions/*/runner/runner-state.json`] });
expect(snapshot.exitCode).toBe(0);
const saved = JSON.parse(snapshot.stdout);
expect(saved.lifecycle).toBe("ready");
expect(saved.lifecycle).toBe(perTurn ? "suspended" : "ready");
const processBefore = await runner.execute({ command: "sh", args: ["-c", `cat '${remoteCwd}'/.paperclip-runtime/paperclip-runner/sessions/*/runner/runner-process.identity`] });
expect(processBefore.exitCode).toBe(0);
const historyBefore = await runner.execute({ command: "sh", args: ["-c", `cat '${remoteCwd}'/.paperclip-runtime/paperclip-runner/sessions/*/filesystem/codex-home/fake-codex-state.json`] });
@@ -502,12 +511,11 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
runnerExecutionTarget: { ...target, leaseId: nextLeaseId, sandboxLeaseAcquisition: { outcome: "resumed" as const, providerLeaseId: container } },
};
if (controller) {
await expect(executePaperclipNativeSession(nextOptions)).rejects.toThrow("prior_owner_terminal_state_indeterminate");
const exited = new Promise<void>(resolveExit => controller!.once("exit", () => resolveExit()));
controller.kill("SIGKILL");
(controller as ReturnType<typeof spawn>).kill("SIGKILL");
await exited;
controller = undefined;
await expect(executePaperclipNativeSession({ ...nextOptions,
if (!perTurn) await expect(executePaperclipNativeSession({ ...nextOptions,
runnerExecutionTarget: { ...nextOptions.runnerExecutionTarget,
sandboxLeaseAcquisition: { outcome: "replacement", providerLeaseId: "unrelated-sandbox" } },
})).rejects.toThrow("prior_owner_terminal_state_indeterminate");
@@ -526,10 +534,43 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
}
expect(changed).toBe(1);
}
const [nextPrepared] = await fixture.db.select({ profile: heartbeatRuns.runnerProfileJson }).from(heartbeatRuns).where(eq(heartbeatRuns.id, nextRunId));
await fixture.db.update(heartbeatRuns).set({ runnerProfileJson: { ...nextPrepared.profile,
nativeExecutionInput: nextOptions.execution } }).where(eq(heartbeatRuns.id, nextRunId));
const second = await executePaperclipNativeSession(nextOptions);
if (diskless) {
// Preserve sandbox/provider storage and workspace dependencies. Delete
// both native controller projections; admission must discover the source.
await rm(resolve(resolvePaperclipInstanceRoot(), "runtime", "paperclip-runner"), { recursive: true, force: true });
}
// Exercise the existing admission selector/rebinder, which supplies per-turn
// runtime snapshots. It does not restore PRP or supervisor files.
const admitContinuation = async (current: typeof execution) => {
const [row] = await fixture.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, current.binding.runId));
const selected = await findNativeSessionResumeRun(fixture.db, { ...current.binding, currentRunId: row.id,
normalizedSessionId: fixture.native.normalizedSessionId, beforeCreatedAt: row.createdAt });
const admission = buildNativeExecutionWithCheckpoint({ previousRun: selected,
normalizedSessionId: fixture.native.normalizedSessionId, executionTargetKind: "remote", buildExecution: () => current });
if (perTurn) expect(admission.checkpoint).not.toBeNull();
await fixture.db.update(heartbeatRuns).set({ runnerProfileJson: { ...row.runnerProfileJson,
nativeExecutionInput: current, nativeToolContractFingerprint: nativeToolContractFingerprintForTarget("remote", current.provider.kind),
...(perTurn ? { sessionCheckpoint: admission.checkpoint } : {}) } }).where(eq(heartbeatRuns.id, row.id));
};
await admitContinuation(nextOptions.execution);
if (scenario === "diskless-per-turn-read-retry") {
let rejectedReads = 0;
await expect(executePaperclipNativeSession({ ...nextOptions,
runnerExecutionTarget: { ...nextOptions.runnerExecutionTarget, runner: { ...runner,
execute: async request => {
if (request.args?.some(arg => arg.includes("base64 <") && arg.includes("runner-state.json"))) {
rejectedReads++;
throw new Error("fixture_remote_state_timeout");
}
return runner.execute(request);
},
} },
})).rejects.toThrow("fixture_remote_state_timeout");
expect(rejectedReads).toBe(1);
// Retry with the partial PRP projection left by the failed read. No
// fixture writes a suspended receipt or changes the surviving Runner.
}
const second = diskless ? await executeInController(nextOptions) : await executePaperclipNativeSession(nextOptions);
expect(second.exitCode).toBe(0);
const historyAfter = await runner.execute({ command: "sh", args: ["-c", `cat '${remoteCwd}'/.paperclip-runtime/paperclip-runner/sessions/*/filesystem/codex-home/fake-codex-state.json`] });
const providerAfter = JSON.parse(historyAfter.stdout);
@@ -540,12 +581,19 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
expect(calls.stdout.split("\n").filter(line => line === "thread/start")).toHaveLength(1);
const providerStarts = calls.stdout.split("\n").filter(line => line === "process-start").length;
expect(providerStarts).toBeGreaterThanOrEqual(1);
expect(providerStarts).toBeLessThanOrEqual(["legacy", "changed"].includes(scenario) ? 3 : hardRestart ? 2 : 1);
expect(providerStarts).toBeLessThanOrEqual(perTurn || ["legacy", "changed"].includes(scenario) ? 3 : hardRestart ? 2 : 1);
const processAfter = await runner.execute({ command: "sh", args: ["-c", `cat '${remoteCwd}'/.paperclip-runtime/paperclip-runner/sessions/*/runner/runner-process.identity`] });
expect(processAfter.exitCode).toBe(0);
if (["legacy", "changed"].includes(scenario)) expect(processAfter.stdout).not.toBe(processBefore.stdout);
if (perTurn || ["legacy", "changed"].includes(scenario)) expect(processAfter.stdout).not.toBe(processBefore.stdout);
else expect(processAfter.stdout).toBe(processBefore.stdout);
await finalizeNativeRun({ db: fixture.db, runId: nextRunId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
if (diskless && controller) {
const exited = new Promise<void>(resolveExit => controller!.once("exit", () => resolveExit()));
(controller as ReturnType<typeof spawn>).kill("SIGKILL");
await exited;
controller = undefined;
await rm(resolve(resolvePaperclipInstanceRoot(), "runtime", "paperclip-runner"), { recursive: true, force: true });
}
const thirdRunId = randomUUID();
const thirdLeaseId = randomUUID();
const [thirdRun] = await fixture.db.insert(heartbeatRuns).values({
@@ -562,9 +610,11 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
issue: { id: fixture.issueId, title: "Continue the same conversation", description: null, reviewPolicy: null },
environmentLeaseId: thirdLeaseId });
await fixture.db.insert(nativeRunFinalizations).values({ runId: thirdRunId, companyId, issueId: fixture.issueId, phase: "observed" });
const thirdExecution = { ...nextOptions.execution, completionContract: await boundContract(thirdRunId),
binding: { ...execution.binding, runId: thirdRunId }, task: { ...execution.task, prompt: "Complete the third conversation turn." } };
await admitContinuation(thirdExecution);
const third = await executePaperclipNativeSession({ ...nextOptions, runnerInstanceId: thirdNative.runnerInstanceId,
execution: { ...nextOptions.execution, completionContract: await boundContract(thirdRunId),
binding: { ...execution.binding, runId: thirdRunId }, task: { ...execution.task, prompt: "Complete the third conversation turn." } },
execution: thirdExecution,
runnerExecutionTarget: { ...nextOptions.runnerExecutionTarget, leaseId: thirdLeaseId },
});
expect(third.exitCode).toBe(0);
@@ -573,10 +623,11 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
const finalCalls = await runner.execute({ command: "cat", args: [`${remoteCwd}/provider-calls.log`] });
expect(finalCalls.stdout.split("\n").filter(line => line === "turn/start")).toHaveLength(3);
expect(finalCalls.stdout.split("\n").filter(line => line === "thread/start")).toHaveLength(1);
expect(finalCalls.stdout.split("\n").filter(line => line === "process-start")).toHaveLength(providerStarts);
expect(finalCalls.stdout.split("\n").filter(line => line === "process-start")).toHaveLength(providerStarts + (perTurn ? 1 : 0));
const finalProcess = await runner.execute({ command: "sh", args: ["-c", `cat '${remoteCwd}'/.paperclip-runtime/paperclip-runner/sessions/*/runner/runner-process.identity`] });
expect(finalProcess.stdout).toBe(processAfter.stdout);
expect(await closeIdleWarmNativeSessionsForRestart()).toEqual({ closed: 1, busy: 0, failed: 0 });
if (perTurn) expect(finalProcess.stdout).not.toBe(processAfter.stdout);
else expect(finalProcess.stdout).toBe(processAfter.stdout);
expect(await closeIdleWarmNativeSessionsForRestart()).toEqual({ closed: perTurn ? 0 : 1, busy: 0, failed: 0 });
const suspended = await runner.execute({ command: "sh", args: ["-c", `cat '${remoteCwd}'/.paperclip-runtime/paperclip-runner/sessions/*/runner/runner-state.json`] });
expect(suspended.exitCode).toBe(0);
expect(JSON.parse(suspended.stdout).lifecycle).toBe("suspended");
@@ -588,7 +639,7 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
await closeIdleWarmNativeSessionsForRestart();
await tunnel.close();
}
}, 180_000);
}, 360_000);
async function persistRestartEvidence(input: {
fixture: Awaited<ReturnType<typeof seedRun>>;
@@ -1277,10 +1328,11 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
} finally { await authority.stop(); }
});
it.each(["idle", "preparing_successor", "unclaimed_successor", "claimed_successor", "expired_preparation", "dispatching_successor", "foreign_preparation", "foreign_native_preparation", "wrong_issue", "new_work", "replaced_controller"] as const)(
"settles a completed warm checkpoint during early retirement without granting work: %s",
async scenario => {
const fixture = await seedRun(`WARM-EARLY-RETIREMENT-${scenario}`);
it.each((["idle", "preparing_successor", "unclaimed_successor", "claimed_successor", "expired_preparation", "dispatching_successor", "foreign_preparation", "foreign_native_preparation", "wrong_issue", "new_work", "replaced_controller", "unaccepted"] as const).flatMap(scenario =>
(["runner.suspend", "turn.stop"] as const).map(command => ({ scenario, command }))))(
"settles a completed warm checkpoint during early retirement without granting work: $scenario/$command",
async ({ scenario, command }) => {
const fixture = await seedRun(`WARM-EARLY-RETIREMENT-${scenario}-${command}`);
const binding = { companyId, issueId: fixture.issueId, runId: fixture.runId };
const identity = transportOptions(fixture, runtimeRoot).prpIdentity;
const authority = new DurablePrpControlPlane({
@@ -1305,7 +1357,7 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
});
await fixture.db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() })
.where(eq(heartbeatRuns.id, fixture.runId));
await fixture.db.update(nativeRunFinalizations).set({ phase: "committed", resultId,
await fixture.db.update(nativeRunFinalizations).set({ phase: "committed", resultId: scenario === "unaccepted" ? null : resultId,
leaseOwner: null, leaseExpiresAt: null,
...(scenario === "replaced_controller" ? { controllerBootId: randomUUID(), controllerGeneration: 2 } : {}),
}).where(eq(nativeRunFinalizations.runId, fixture.runId));
@@ -1314,7 +1366,7 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
if (scenario !== "idle") {
nextRunId = randomUUID();
const preparing = scenario !== "claimed_successor" && scenario !== "unclaimed_successor";
const otherIssue = scenario === "wrong_issue" ? (await seedRun("WARM-OTHER-ISSUE")).issueId : fixture.issueId;
const otherIssue = scenario === "wrong_issue" ? (await seedRun(`WARM-OTHER-ISSUE-${command}`)).issueId : fixture.issueId;
await fixture.db.insert(heartbeatRuns).values({ id: nextRunId, companyId, agentId,
issueId: otherIssue, scopeKind: "issue", contextSnapshot: { issueId: otherIssue },
...(preparing ? { runtimeMode: "legacy", executionStage: scenario === "dispatching_successor" ? "dispatching" : "preparing",
@@ -1331,7 +1383,7 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
}
await fixture.db.update(issues).set({ executionRunId: nextRunId }).where(eq(issues.id, fixture.issueId));
const before = (await loadNativeRunnerCheckpoint(fixture.db, binding))!;
authority.queueCommand(scenario === "new_work" ? "turn.start" : "runner.suspend", {}, `early-retirement-${scenario}`);
authority.queueCommand(scenario === "new_work" ? "turn.start" : command, {}, `early-retirement-${scenario}`);
if (!["idle", "preparing_successor", "claimed_successor"].includes(scenario)) {
await expect(authority.flushCheckpoint()).rejects.toThrow("native_runner_checkpoint_owner_lost");
expect((await loadNativeRunnerCheckpoint(fixture.db, binding))?.records).toEqual(before.records);
@@ -1342,7 +1394,7 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
saved.restoreSessionDirectory(restoredRoot);
const restored = JSON.parse(await readFile(resolve(restoredRoot, "control-plane", "control-plane-state.json"), "utf8"));
expect(restored.identity).toEqual(identity);
expect(restored.commands).toContainEqual(expect.objectContaining({ type: "runner.suspend", status: "pending" }));
expect(restored.commands).toContainEqual(expect.objectContaining({ type: command, status: "pending" }));
}
const [task] = await fixture.db.select({ executionRunId: issues.executionRunId }).from(issues)
.where(eq(issues.id, fixture.issueId));
@@ -1351,6 +1403,104 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
},
);
realProcessIt("durably retires an accepted warm Runner while its same-controller successor prepares", async () => {
const fixture = await seedRun("REAL-WARM-RETIREMENT");
const binding = { companyId, issueId: fixture.issueId, runId: fixture.runId };
const root = resolve(runtimeRoot, fixture.runId, "real-retirement");
const options = transportOptions(fixture, root);
const retirementController = await currentNativeControllerIdentity();
const diagnostics: string[] = [], checkpointFailures: unknown[] = [];
let authority!: DurablePrpControlPlane;
await fixture.db.update(nativeRunFinalizations).set({ controllerBootId: retirementController.bootId,
controllerGeneration: 1, leaseOwner: "real-retirement-owner", leaseExpiresAt: new Date(Date.now() + 60_000),
}).where(eq(nativeRunFinalizations.runId, fixture.runId));
const transport = createRunnerdCodexTransport({ ...options,
codexArgs: [...options.codexArgs, "--linger-after-turn-start"],
onDiagnostic: message => diagnostics.push(message),
controlPlaneRegistration: async core => {
authority = core;
const write = nativeRunnerCheckpointWriter(fixture.db, { ...binding, authorityRunId: fixture.runId,
bootId: retirementController.bootId, generation: 1, leaseOwner: "real-retirement-owner" });
core.setCheckpointPersistence(async checkpoint => {
try { await write(checkpoint); }
catch (error) { checkpointFailures.push(error); throw error; }
});
return registerRunnerPrpAuthority({ ...binding, authority: core });
},
});
let runnerPid: number | null = null, providerPid: number | null = null;
try {
await transport.transport.request("thread/start", { cwd: root, dynamicTools: [] });
await transport.transport.request("turn/start", { input: [{ type: "text", text: "Keep the provider alive after accepting its result." }] });
runnerPid = transport.evidence().runnerPid;
providerPid = transport.evidence().providerPid;
expect(runnerPid).toEqual(expect.any(Number));
expect(providerPid).toEqual(expect.any(Number));
await waitForCondition("active real provider before accepted-result retirement", () =>
authority.store.state.committedEvents.some(event => event.eventType === "turn.started"));
await authority.flushCheckpoint();
const [run] = await fixture.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId));
const resultId = randomUUID();
await fixture.db.insert(nativeRunResults).values({ id: resultId, ...binding, turnId: options.prpIdentity.turnId,
completionContractId: run.completionContractId!, serverFingerprint: "real-retirement-accepted",
schemaStatus: "accepted", canonicalSha256: "a".repeat(64),
resultJson: { schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", summary: "Accepted before provider cleanup." },
});
await fixture.db.update(nativeRunFinalizations).set({ phase: "committed", resultId, leaseOwner: null, leaseExpiresAt: null })
.where(eq(nativeRunFinalizations.runId, fixture.runId));
await fixture.db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() })
.where(eq(heartbeatRuns.id, fixture.runId));
const nextRunId = randomUUID();
const [nextRun] = await fixture.db.insert(heartbeatRuns).values({ id: nextRunId, companyId, agentId,
issueId: fixture.issueId, nativeIssueId: fixture.issueId, scopeKind: "issue", status: "running", runtimeMode: "native",
invocationSource: "assignment", contextSnapshot: { issueId: fixture.issueId } }).returning();
const next = await prepareNativeHeartbeatRun({ db: fixture.db, run: nextRun,
issue: { id: fixture.issueId, title: "Next conversation", description: null }, environmentLeaseId: randomUUID() });
await fixture.db.insert(nativeRunFinalizations).values({ runId: nextRunId, companyId, issueId: fixture.issueId, phase: "observed" });
await fixture.db.update(issues).set({ executionRunId: nextRunId }).where(eq(issues.id, fixture.issueId));
const [contract] = await fixture.db.select().from(completionContracts).where(eq(completionContracts.id, run.completionContractId!));
const nextExecution = buildNativeExecutionInput({ companyId, agentId, runId: nextRunId,
issue: { id: fixture.issueId, identifier: "NRR-NATIVE-RETIREMENT", title: "Next conversation", description: null, workMode: "standard" },
taskPrompt: "Start only after the previous provider durably suspends.",
workspace: { id: randomUUID(), cwd: root, repoUrl: null, repoRef: null, branchName: null },
normalizedSessionId: next.normalizedSessionId, runtimeContext: nativeRuntimeContextFixture(),
completionContract: { id: contract.id, sha256: contract.canonicalSha256, schemaVersion: contract.schemaVersion, contract: contract.contractJson as never },
});
const fixtureStoppedAfterRealClaim = new Error("fixture stopped before next provider dispatch");
await expect(executePaperclipNativeSession({ db: fixture.db, execution: nextExecution, runnerInstanceId: next.runnerInstanceId,
onExecutionLeaseAcquired: async () => {
const [claimed] = await fixture.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, nextRunId));
expect(claimed.controllerBootId).toBe(retirementController.bootId);
expect(claimed.leaseOwner).toEqual(expect.any(String));
expect(claimed.leaseExpiresAt!.getTime()).toBeGreaterThan(Date.now());
await expect(transport.transport.close(), diagnostics.join("\n")).resolves.toBeUndefined();
throw fixtureStoppedAfterRealClaim;
},
})).rejects.toBe(fixtureStoppedAfterRealClaim);
expect(checkpointFailures).toEqual([]);
const state = JSON.parse(await readFile(resolve(root, "runner", "runner-state.json"), "utf8"));
expect(state).toMatchObject({ lifecycle: "suspended", runId: fixture.runId,
normalizedSessionId: options.prpIdentity.normalizedSessionId, runnerInstanceId: options.prpIdentity.runnerInstanceId });
const saved = (await loadNativeRunnerCheckpoint(fixture.db, binding))!;
const commands = [...saved.records].filter(([key]) => key.startsWith("command:")).map(([, value]) => JSON.parse(value));
for (const type of ["turn.stop", "runner.drain", "runner.suspend"]) {
expect(commands, diagnostics.join("\n")).toContainEqual(expect.objectContaining({ type, status: "completed" }));
}
expect(commands.filter(command => command.type === "turn.start")).toHaveLength(1);
const [task] = await fixture.db.select().from(issues).where(eq(issues.id, fixture.issueId));
expect(task.executionRunId).toBe(nextRunId);
} catch (error) {
throw new Error(`${String(error)}; owned retirement diagnostics ${JSON.stringify({
checkpointFailures: checkpointFailures.map(String), diagnostics,
commands: authority?.store.state.commands.map(({ type, status }) => ({ type, status })),
})}`);
} finally {
await transport.transport.close().catch(() => undefined);
if (providerPid && processAlive(providerPid)) process.kill(providerPid, "SIGKILL");
await stopOwnedProcessGroup(runnerPid, root).catch(() => undefined);
}
}, 45_000);
it("applies checkpoint deltas in Postgres without rewriting authority and removes only evicted events", async () => {
const fixture = await seedRun("CHECKPOINT-DELTA");
const root = resolve(runtimeRoot, fixture.runId);
@@ -1463,8 +1613,8 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
realProcessIt("keeps checkpoint SQL independent of streamed output volume with a real Postgres writer", async () => {
const samples: Array<{ chunks: number; checkpointWrites: number; transactions: number;
changedRows: number; checkpointBytes: number; walBytes: number; elapsedMs: number; idleSql: number;
streamSql: number; streamTransactions: number; streamChangedRows: number }> = [];
for (const chunks of [8, 512]) {
streamSql: number; streamTransactions: number; streamChangedRows: number; renewalSql: number }> = [];
for (const chunks of [8, 512, 8192]) {
const fixture = await seedRun(`CHECKPOINT-COST-${chunks}`);
const root = resolve(runtimeRoot, fixture.runId);
const releaseOutput = resolve(root, "release-output");
@@ -1472,19 +1622,31 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
const owner = { ...binding, authorityRunId: fixture.runId, bootId: successor.bootId,
generation: 1, leaseOwner: "cost-controller" };
await fixture.db.update(nativeRunFinalizations).set({ controllerBootId: owner.bootId,
controllerGeneration: 1, leaseOwner: owner.leaseOwner, leaseExpiresAt: new Date(Date.now() + 120_000),
controllerPid: successor.pid, controllerProcessStartedAt: successor.processStartedAt,
controllerGeneration: 1, leaseOwner: owner.leaseOwner, leaseExpiresAt: new Date(Date.now() + 60_000),
}).where(eq(nativeRunFinalizations.runId, fixture.runId));
let authority!: DurablePrpControlPlane;
let measuring = false;
let transactions = 0, changedRows = 0, checkpointBytes = 0;
const statements: string[] = [];
const capturedClients = [fixture.db.$client, checkpointDb(fixture.db).$client];
const renewalStatements: string[] = [];
let leaseLost = false;
const renewal = startNativeSessionExecutionLeaseRenewal({ db: fixture.db, ...binding, leaseOwner: owner.leaseOwner,
attempt: 0, controller: successor, remote: true, onLost: async () => { leaseLost = true; } });
const capturedClients = [fixture.db.$client, checkpointDb(fixture.db).$client, livenessDb(fixture.db).$client];
const priorDebug = capturedClients.map(client => client.options.debug);
// Capture statements only, never SQL parameters (encrypted state or secrets).
for (const client of capturedClients) {
client.options.debug = (_connection, query) => { if (measuring) statements.push(query); };
}
capturedClients.forEach((client, index) => {
client.options.debug = (_connection, query) => {
if (!measuring) return;
// Renewals are existing liveness traffic, reported separately. A
// checkpoint query on any pool still counts against the zero oracle.
if (index === 2 && !query.includes("native_runner_checkpoints")) renewalStatements.push(query);
else statements.push(query);
};
});
const write = nativeRunnerCheckpointWriter(fixture.db, owner);
let costFailure: unknown;
const transport = createRunnerdCodexTransport({ ...transportOptions(fixture, root),
codexArgs: ["--state-file", resolve(root, "provider.json"), "--linger-after-turn-start",
"--held-output", "--held-output-count", String(chunks), "--held-output-release-file", releaseOutput],
@@ -1516,7 +1678,7 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
const beforeStream = { sql: statements.length, transactions, changedRows };
await writeFile(releaseOutput, "release\n", { flag: "wx" });
await waitForCondition(`the final scripted text chunk of ${chunks}`, () => authority.store.state.committedEvents
.some(event => event.eventType === "item.delta" && JSON.stringify(event.envelope).includes(`still computing ${chunks - 1} `)), 60_000)
.some(event => event.eventType === "item.delta" && JSON.stringify(event.envelope).includes(`still computing ${chunks - 1} `)), chunks > 4096 ? 480_000 : 60_000)
.catch(error => { console.info("checkpoint-cost-stream-diagnostic", JSON.stringify({ chunks,
cursor: authority.store.state.ackedSourceSeq,
receivedTextEvents: authority.store.state.committedEvents.filter(event => event.eventType === "item.delta").length })); throw error; });
@@ -1528,13 +1690,14 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
&& query.includes("native_runner_checkpoints")).length;
expect(checkpointWrites).toBeGreaterThan(0); // The collector must see real control writes.
const snapshot = { chunks, checkpointWrites, transactions, changedRows, checkpointBytes,
walBytes: Number(walEnd.bytes), elapsedMs, idleSql: 0,
walBytes: Number(walEnd.bytes), elapsedMs, idleSql: 0, renewalSql: renewalStatements.length,
streamSql: statements.length - beforeStream.sql,
streamTransactions: transactions - beforeStream.transactions,
streamChangedRows: changedRows - beforeStream.changedRows };
// Setup control messages may coalesce differently. The released text-only
// interval must add exactly zero work, independent of that scheduling.
expect(snapshot).toMatchObject({ streamSql: 0, streamTransactions: 0, streamChangedRows: 0 });
expect(leaseLost).toBe(false);
statements.length = 0;
measuring = true;
await new Promise(done => setTimeout(done, 100));
@@ -1547,20 +1710,34 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
expect(statements.some(query => /insert into "native_runner_checkpoints"/i.test(query))).toBe(true);
measuring = false;
samples.push(snapshot);
console.info("native-checkpoint-cost-sample", JSON.stringify(snapshot));
} catch (error) {
costFailure = error;
throw error;
} finally {
measuring = false;
capturedClients.forEach((client, index) => { client.options.debug = priorDebug[index]; });
await transport.transport.close();
try { await transport.transport.close(); }
catch (error) {
if (costFailure) throw new AggregateError([costFailure, error], "Checkpoint cost measurement and cleanup failed");
throw error;
}
finally {
await renewal.stop().catch(error => {
if (costFailure) throw new AggregateError([costFailure, error], "Checkpoint cost measurement and lease renewal failed");
throw error;
});
}
}
}
const report = { schema: "paperclip.checkpoint-cost/v1", samples,
scope: "isolated Postgres; control operations included; WAL is diagnostic; no production capacity claim" };
if (process.env.PAPERCLIP_TEST_CHECKPOINT_COST_REPORT) await writeFile(process.env.PAPERCLIP_TEST_CHECKPOINT_COST_REPORT,
`${JSON.stringify(report, null, 2)}\n`, { flag: "wx", mode: 0o600 });
// Both stream sizes were checked independently; total setup counts remain
// Each stream size was checked independently; total setup counts remain
// diagnostic because control-message batching is scheduling-dependent.
console.info("native-checkpoint-cost", JSON.stringify(report));
}, 120_000);
}, 540_000);
realProcessIt.each([false, true])("adopts the same runner from Postgres after losing the controller filesystem (prepared task: %s)", async preparedTask => {
const fixture = await seedRun(`EMPTY-CONTROLLER-DISK-${preparedTask}`);
@@ -1579,9 +1756,10 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
heartbeatRunId: fixture.runId, issueId: fixture.issueId, status: "active", leasePolicy: "ephemeral",
provider: "plugin", providerLeaseId: "saved-task", metadata: { agentId, runnerTaskLease: true, runnerTaskDispatched: true } });
}
const originalLeaseExpiresAt = new Date(Date.now() + 60_000);
await fixture.db.update(nativeRunFinalizations).set({ controllerBootId: bootId,
controllerGeneration: 1, controllerPid: process.pid, controllerProcessStartedAt: new Date(),
leaseOwner: "original-controller", leaseExpiresAt: new Date(Date.now() + 60_000),
leaseOwner: "original-controller", leaseExpiresAt: originalLeaseExpiresAt,
}).where(eq(nativeRunFinalizations.runId, fixture.runId));
const options = {
...transportOptions(fixture, controllerRoot),
@@ -1647,9 +1825,13 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
await new Promise(resolveOutage => setTimeout(resolveOutage, outageMs));
expect(processAlive(runnerPid)).toBe(true);
expect(processAlive(providerPid)).toBe(true);
await fixture.db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 1) })
.where(eq(nativeRunFinalizations.runId, fixture.runId));
bootId = randomUUID();
if (originalLeaseExpiresAt.getTime() > Date.now()) {
const [waiting] = await claimNativeRestartRecoveries({ db: fixture.db, controller: { ...successor, bootId },
restartKind: "hard", runIds: [fixture.runId] });
expect(waiting).toMatchObject({ kind: "awaiting_evidence", reason: "remote_controller_lease_live", retryAt: originalLeaseExpiresAt });
await new Promise(resolveExpiry => setTimeout(resolveExpiry, Math.max(0, originalLeaseExpiresAt.getTime() - Date.now()) + 20));
}
const [claim] = await claimNativeRestartRecoveries({ db: fixture.db, controller: { ...successor, bootId },
restartKind: "hard", runIds: [fixture.runId] });
expect(claim).toMatchObject({ kind: "reattach_remote_runner", providerAttempt: 0, controllerGeneration: 2 });
@@ -1,6 +1,8 @@
import { onRetainedNativeSessionCleanupRelease } from "../retained-native-session-cleanup-notifications.js";
import { readIdleLocalWork, markIdleStartupComplete, markIdleIngressTracked } from "../idle-local-work.js";
import * as runnerCheckpoint from "./native-runner-checkpoint.js";
import * as stagedAttachmentRecovery from "./native-staged-attachment-recovery.js";
import * as controllerRecovery from "./native-controller-recovery.js";
import { DurablePrpControlPlane } from "../../vendor/paperclip-runner/index.js";
import * as environmentTaskRuntime from "../environment-task-runtime.js";
import * as pluginTaskRuntime from "./plugin-runner-task.js";
@@ -47,6 +49,7 @@ import {
import {
canonicalNativeRuntimeContextDigest,
CURSOR_DISTRIBUTION_PINS,
KIMI_DISTRIBUTION_CLOSURE_SHA256,
QUALIFIED_ACPX_PROFILES,
acpxRuntimeSessionDirectoryName,
resolveAcpxRuntimeRoot,
@@ -96,6 +99,7 @@ import { buildNativeHeartbeatPreparationSpans, createNativeRunTrace } from "./na
import { NativeRunnerOwnershipUnverifiedError } from "./native-runner-ownership.js";
import type { AdapterRuntimeEvent } from "../../adapters/index.js";
import type { AdapterUsageCheckpoint } from "@paperclipai/adapter-utils";
import { acknowledgedNativeStopExecutionHasStopped, isAcknowledgedNativeStop } from "../acknowledged-native-stop.js";
vi.mock("node:fs", async (importOriginal) => {
const actual = await importOriginal<typeof import("node:fs")>();
@@ -391,6 +395,8 @@ import {
REMOTE_RUNNER_CHILD_LAUNCH_SCRIPT,
verifyRemoteRunnerReattachment,
readRemoteProviderPackManifest,
prepareRemoteProviderPackOnTarget,
resolveRemoteProviderPackSource,
readBundledRemoteProviderPackManifest,
providerSessionIdentityFromDurableProviderState,
durableProviderCheckpointFailureReason,
@@ -416,6 +422,10 @@ import {
import { onNativeRunnerEvidence } from "./native-runner-evidence-signal.js";
beforeEach(() => {
// These transport-wiring fixtures use a partial Db double and synthetic IDs.
// The real selector/bootstrap proof is exercised with Postgres by the durable
// discovery and diskless admission suites, not by this backend unit boundary.
vi.spyOn(controllerRecovery, "resolveNativeControllerRecovery").mockResolvedValue({ kind: "no_prior_execution" });
state.drainMaintenance.mockReset().mockResolvedValue(undefined);
usageRecorder.capture.mockReset().mockImplementation(async receipt => receipt);
usageRecorder.create.mockReset().mockResolvedValue({ capture: usageRecorder.capture,
@@ -1946,6 +1956,40 @@ describe("remote provider pack manifest", () => {
expect(readRemoteProviderPackManifest(root).payload.pins.opencode).toBe(
"1.18.34",
);
// Both execution and setup Test consume this same normal installer. Reuse
// the complete controller fixture here instead of inventing a thin pack.
const selectedPack = resolveRemoteProviderPackSource(root);
for (const mode of ["staged", "preinstalled", "uploaded", "failed-final", "missing-transport"] as const) {
let transferred = false;
const syncIn = vi.fn(async (operations: { files: { sourcePath?: string; targetPath: string; kind: string }[] }[]) => {
expect(operations[0]?.files[0]).toMatchObject({ sourcePath: root,
targetPath: "/workspace/.paperclip-runtime/paperclip-runner/provider-pack", kind: "directory" });
transferred = true;
return { operations: [] };
});
const execute = vi.fn(async (command: { command: string; args?: string[]; timeoutMs?: number }) => {
expect(command.timeoutMs).toBeGreaterThan(0);
expect(command.timeoutMs).toBeLessThanOrEqual(30_000);
const verification = command.args?.[0] === "-e";
const packRoot = command.args?.[2];
const valid = mode === "staged" || mode === "preinstalled" && packRoot === "/opt/paperclip-runner/provider-pack"
|| transferred && mode !== "failed-final";
return { pid: null, startedAt: null, exitCode: verification && !valid ? 1 : 0, timedOut: false, signal: null,
stdout: command.args?.[0] === "--version" ? "1.18.34\n"
: command.args?.some(value => value.includes("for candidate in")) ? "/opt/paperclip-runner/provider-pack\n" : "",
stderr: verification && !valid ? "manifest mismatch" : "" };
});
const runner = { execute, ...(mode === "missing-transport" ? {} : { syncIn }) };
const preparation = prepareRemoteProviderPackOnTarget({
target: { kind: "remote", transport: "sandbox", remoteCwd: "/workspace", runner } as never,
runner, configuredProviderPackRoot: selectedPack.configuredRoot, manifest: selectedPack.manifest,
});
if (mode === "failed-final") await expect(preparation).rejects.toThrow("provider pack verification");
else if (mode === "missing-transport") await expect(preparation).rejects.toThrow("transport cannot stage a provider pack");
else await expect(preparation).resolves.toMatchObject({ source: mode, root: "/workspace/.paperclip-runtime/paperclip-runner/provider-pack",
manifest: selectedPack.manifest });
expect(syncIn).toHaveBeenCalledTimes(mode === "uploaded" || mode === "failed-final" ? 1 : 0);
}
const cursorPath = "provider-assets/cursor/linux-x64";
await mkdir(join(root, cursorPath), { recursive: true });
await writeFile(join(root, cursorPath, "runtime"), "pinned Cursor runtime");
@@ -1980,6 +2024,78 @@ describe("remote provider pack manifest", () => {
await writeFile(join(root, cursorPath, "runtime"), "substitute Cursor runtime");
expect(() => readRemoteProviderPackManifest(root)).toThrow("asset tree digest mismatch");
await writeFile(join(root, cursorPath, "runtime"), "pinned Cursor runtime");
const kimiPath = "provider-assets/kimi/linux-x64";
await mkdir(join(root, kimiPath), { recursive: true });
await writeFile(join(root, kimiPath, "runtime"), "pinned Kimi runtime");
const kimi = { version: QUALIFIED_ACPX_PROFILES.kimi.agentServerVersion,
profileDigest: QUALIFIED_ACPX_PROFILES.kimi.commandDigest,
closureDigest: `sha256:${KIMI_DISTRIBUTION_CLOSURE_SHA256["linux-x64"]}`,
qualification: "qualified", path: kimiPath, sha256: sha256DirectoryTree(join(root, kimiPath)) };
Object.assign(payload, { providers: { cursor, kimi } });
await mkdir(releaseMetadata, { recursive: true });
const writeKimiManifest = async () => {
await writeManifest();
await cp(join(root, "provider-pack.json"), manifestPath);
};
await writeKimiManifest();
expect(readRemoteProviderPackManifest(root).payload.providers?.kimi).toEqual(kimi);
expect(readBundledRemoteProviderPackManifest(manifestPath).payload.providers?.kimi?.qualification).toBe("qualified");
for (const field of ["version", "profileDigest", "closureDigest"] as const) {
const previous = kimi[field];
kimi[field] = field === "version" ? "older-kimi" : digest(`old-kimi-${field}`);
await writeKimiManifest();
expect(() => readRemoteProviderPackManifest(root)).toThrow("Kimi profile or closure");
expect(() => readBundledRemoteProviderPackManifest(manifestPath)).toThrow("Kimi profile or closure");
kimi[field] = previous;
}
kimi.closureDigest = `sha256:${KIMI_DISTRIBUTION_CLOSURE_SHA256["darwin-arm64"]}`;
await writeKimiManifest();
expect(() => readBundledRemoteProviderPackManifest(manifestPath)).toThrow("Kimi profile or closure");
kimi.closureDigest = `sha256:${KIMI_DISTRIBUTION_CLOSURE_SHA256["linux-x64"]}`;
for (const target of [{ platform: "linux", architecture: "arm64" }, { platform: "darwin", architecture: "arm64" }]) {
payload.target = target;
kimi.path = `provider-assets/kimi/${target.platform}-${target.architecture}`;
Object.assign(payload, { providers: {} });
await writeKimiManifest();
expect(() => readBundledRemoteProviderPackManifest(manifestPath)).toThrow("bundled image pack must target linux-x64");
}
payload.target = originalTarget;
kimi.path = kimiPath;
Object.assign(payload, { providers: { cursor, kimi } });
kimi.qualification = "pending";
await writeKimiManifest();
expect(() => readBundledRemoteProviderPackManifest(manifestPath)).toThrow("invalid candidate identity");
kimi.qualification = "qualified";
Object.assign(payload, { providers: { cursor, kimi }, candidateProviders: { kimi } });
await writeKimiManifest();
expect(() => readBundledRemoteProviderPackManifest(manifestPath)).toThrow("Kimi profile or closure");
Object.assign(payload, { providers: { cursor, kimi } });
Reflect.deleteProperty(payload, "candidateProviders");
await writeKimiManifest();
await writeFile(join(root, kimiPath, "runtime"), "substitute Kimi runtime");
expect(() => readRemoteProviderPackManifest(root)).toThrow("candidate asset tree digest mismatch");
await writeFile(join(root, kimiPath, "runtime"), "pinned Kimi runtime");
const threeCandidates = { cursor,
pi: { ...kimi, version: "1.0.0", qualification: "qualified", path: "provider-assets/pi/linux-x64" },
copilot: { ...kimi, qualification: "pending", version: QUALIFIED_ACPX_PROFILES.copilot.agentServerVersion,
profileDigest: QUALIFIED_ACPX_PROFILES.copilot.commandDigest,
closureDigest: "sha256:a3d8f4367cfa1694d79e3f8b7930b6fbfe42a229c272b375b11951d631db8d07",
path: "provider-assets/copilot/linux-x64" } };
for (const candidate of [threeCandidates.pi, threeCandidates.copilot]) {
await mkdir(join(root, candidate.path), { recursive: true });
await writeFile(join(root, candidate.path, "runtime"), "candidate runtime");
candidate.sha256 = sha256DirectoryTree(join(root, candidate.path));
}
Object.assign(payload, { candidateProviders: threeCandidates });
await writeKimiManifest();
expect(Object.keys(readRemoteProviderPackManifest(root).payload.candidateProviders!)).toHaveLength(3);
Object.assign(payload, { candidateProviders: { ...threeCandidates, unknown: kimi } });
await writeKimiManifest();
expect(() => readBundledRemoteProviderPackManifest(manifestPath)).toThrow("invalid candidate inventory");
Object.assign(payload, { candidateProviders: { unknown: kimi } });
await writeKimiManifest();
expect(() => readBundledRemoteProviderPackManifest(manifestPath)).toThrow("invalid candidate identity");
await rm(releaseMetadata, { recursive: true, force: true });
const candidatePath = "provider-assets/pi/linux-x64";
await mkdir(join(root, candidatePath), { recursive: true });
await writeFile(join(root, candidatePath, "runtime"), "pinned runtime");
@@ -3791,10 +3907,10 @@ describe("runtime question fallback", () => {
});
describe("native provider bootstrap environment", () => {
it.each(["pi", "cursor", "copilot", "antigravity"] as const)("does not promote ambient %s credentials when bindings are omitted", agent => {
it.each(["pi", "cursor", "copilot", "antigravity", "kimi"] as const)("does not promote ambient %s credentials when bindings are omitted", agent => {
const provider = { kind: "acpx", agent } as NativeExecutionInput["provider"];
const host = { PATH: "/host/bin", HOME: "/host/home", OPENROUTER_API_KEY: "ambient-pi",
CURSOR_API_KEY: "ambient-cursor", CURSOR_AUTH_TOKEN: "ambient-cursor-login", COPILOT_GITHUB_TOKEN: "ambient-copilot",
CURSOR_API_KEY: "ambient-cursor", CURSOR_AUTH_TOKEN: "ambient-cursor-login", COPILOT_GITHUB_TOKEN: "ambient-copilot", KIMI_API_KEY: "ambient-kimi",
GEMINI_API_KEY: "ambient-google", GOOGLE_API_KEY: "ambient-other-google", GEMINI_HOME: "/ambient-google-home",
GOOGLE_APPLICATION_CREDENTIALS: "/ambient-google-credential.json", PAPERCLIP_ACPX_ANTIGRAVITY_AUTH_JSON_SECRET: "ambient-google-subscription",
PAPERCLIP_ACPX_CREDENTIAL_BINDING: "ambient-forged-binding" };
@@ -7754,6 +7870,27 @@ describe("native session cancellation", () => {
expect(state.persistActivity).not.toHaveBeenCalled();
});
it.each([
{ companyId: "another-company" }, { runId: "another-run" }, { issueId: "another-issue" },
{ scope: "issue" }, { effects: ["cancel_issue"] }, { intentAuditId: null },
])("does not fence a failed retry for a foreign or stale Stop receipt %j", async changed => {
const intent = { schema: "paperclip.native-cancellation.v1", intentId: "native-cancellation:earlier-stop",
...execution.binding, scope: "run", reasonCode: "cancellation_run_only", effects: ["release_run_resources"],
dispatchState: "acknowledged", dispatched: true, intentAuditId: "earlier-intent-audit",
acknowledgementAuditId: "earlier-ack-audit", ...changed };
const coordinator = { runId: execution.binding.runId, assessmentId: null, phase: "retryable_failure" };
const persistence = cancellationDb({ status: "failed", coordinator, resultJson: { nativeCancellation: intent } });
const before = structuredClone(persistence.getResultJson());
state.cancel.mockClear();
state.persistActivity.mockClear();
await expect(cancelNativeSession(execution.binding.runId, "retry Stop", { db: persistence.db, scope: "run" })).rejects.toThrow();
expect(persistence.updates).toEqual([]);
expect(coordinator.phase).toBe("retryable_failure");
expect(persistence.getResultJson()).toEqual(before);
expect(state.cancel).not.toHaveBeenCalled();
expect(state.persistActivity).not.toHaveBeenCalled();
});
it("does not mistake a recovered running attempt for the failed retry that reserved Stop", async () => {
const id = "11111111-1111-4111-8111-111111111111";
const persistence = cancellationDb({ status: "running",
@@ -7779,6 +7916,39 @@ describe("native session cancellation", () => {
expect(persistence.getResultJsonUpdateCount()).toBe(writes);
});
it("fences a failed retry whose earlier Stop is already acknowledged without changing its receipts", async () => {
const intent = { schema: "paperclip.native-cancellation.v1", intentId: "native-cancellation:earlier-stop",
...execution.binding, scope: "run", reasonCode: "cancellation_run_only",
effects: ["release_run_resources"], dispatchState: "acknowledged", dispatched: true,
intentAuditId: "earlier-intent-audit", acknowledgementAuditId: "earlier-ack-audit",
acknowledgedAt: "2026-01-01T00:00:00.000Z", decisionId: null, priorCoordinatorDecisionId: null };
const coordinator = { runId: execution.binding.runId, assessmentId: null, phase: "retryable_failure",
nextAttemptAt: new Date(Date.now() + 30_000) };
const persistence = cancellationDb({ status: "failed", coordinator,
resultJson: { nativeCancellation: intent, executionCancellation: { state: "requested", requestedAt: "later-stop" } } });
const before = structuredClone(persistence.getResultJson());
state.cancel.mockClear();
state.persistActivity.mockClear();
for (let attempt = 0; attempt < 2; attempt++) {
await expect(cancelNativeSession(execution.binding.runId, "retry Stop", { db: persistence.db, scope: "run" }))
.resolves.toMatchObject({ dispatched: true, auditId: intent.intentAuditId });
expect(coordinator).toMatchObject({ phase: "terminal_failure", failureCode: "native_retry_cancelled", nextAttemptAt: null });
expect(persistence.getResultJson()).toEqual(before);
}
expect(persistence.updates.filter(update => update.table === nativeRunFinalizations)).toHaveLength(1);
expect(persistence.getResultJsonUpdateCount()).toBe(0);
expect(state.cancel).not.toHaveBeenCalled();
expect(state.persistActivity).not.toHaveBeenCalled();
const stopped = { id: execution.binding.runId, companyId: execution.binding.companyId,
nativeIssueId: execution.binding.issueId, status: "cancelled", processPid: null, processGroupId: null,
resultJson: { ...persistence.getResultJson(), cancelledByActorType: "user", cancelledByUserId: "operator" } };
const lease = { provider: "daytona", providerLeaseId: "unconfirmed-provider-resource", status: "active",
releasedAt: null, cleanupStatus: null, metadata: {} };
const leaseDb = { select: () => ({ from: () => ({ where: async () => [lease] }) }) } as unknown as Db;
expect(isAcknowledgedNativeStop(stopped)).toBe(true);
await expect(acknowledgedNativeStopExecutionHasStopped(leaseDb, stopped)).resolves.toBe(false);
});
it("uses the reserved caller UUID and permits same-intent and default retries", async () => {
const id = "11111111-1111-4111-8111-111111111111";
const persistence = cancellationDb({ resultJson: { startupCancellation: { cancellationRequestId: id } } });
@@ -9202,6 +9372,44 @@ describe("native warm session supervision", () => {
expect(state.execute).toHaveBeenCalledTimes(2);
});
it("claims native execution authority before retiring a superseded warm identity", async () => {
const { base, next, run } = fixture("identity-native-claim-order");
const updates: Array<{ table: unknown; values: Record<string, unknown> }> = [];
let inspected = false;
const close = vi.fn(async () => {
if (inspected) return;
inspected = true;
expect(updates.some(update => update.table === nativeRunFinalizations
&& update.values.leaseOwner && update.values.controllerBootId)).toBe(true);
});
state.execute.mockReset().mockImplementationOnce(async options => {
options.onSession?.({ close }); return result;
}).mockImplementationOnce(async () => result);
await run(base);
// Normal runtime selection already converted preparation to native and
// inserted an unclaimed observed coordinator before calling this facade.
await executePaperclipNativeSession({ db: leaseDb(next, {}, {}, updates),
execution: next, runnerInstanceId: "runner", useRunnerd: true });
expect(close).toHaveBeenCalledOnce();
expect(state.execute).toHaveBeenCalledTimes(2);
});
it.each(["cancelled", "foreign-owner"] as const)("does not retire or stage a warm successor after claim denial: %s", async denial => {
const { base, next, run } = fixture(`identity-native-claim-denied-${denial}`);
const close = vi.fn(async () => undefined);
state.execute.mockReset().mockImplementationOnce(async options => {
options.onSession?.({ close }); return result;
});
await run(base);
state.stageNativeRunnerWakeAttachments.mockClear();
const db = leaseDb(next, denial === "foreign-owner" ? { leaseOwner: "different-controller" } : {},
{}, [], {}, denial === "cancelled" ? "cancelled" : "running");
await expect(executePaperclipNativeSession({ db, execution: next, runnerInstanceId: "runner", useRunnerd: true })).rejects.toThrow();
expect(close).not.toHaveBeenCalled();
expect(state.stageNativeRunnerWakeAttachments).not.toHaveBeenCalled();
expect(state.execute).toHaveBeenCalledTimes(1);
});
it("does not retire an active turn when a new session identity arrives", async () => {
const { base, next, run } = fixture("identity-active");
let finish!: () => void;
@@ -12183,6 +12391,55 @@ describe("runnerd provider runtime wiring", () => {
).toThrow("scope_closed");
});
it.each(["attached", "changed-admission", "changed-template", "changed-template-digest", "changed-preparation"] as const)(
"verifies frozen staged admission after normal wake attachment preparation: %s", async mode => {
const current = scopedNativeExecution(`staged-attachment-${mode}`);
current.task.prompt = "Inspect attachment ref:original";
const root = await initializeScopedRunnerRoot(current, "attachment-runner");
const sourceIdentity = { runnerInstanceId: "attachment-runner", environmentLeaseId: "attachment-lease",
runId: "attachment-source", normalizedSessionId: current.session.normalizedSessionId!, turnId: "source-turn", itemId: "source-item" };
const targetIdentity = { ...sourceIdentity, runId: current.binding.runId, turnId: "target-turn", itemId: "target-item" };
const controller = new DurablePrpControlPlane({ stateDirectory: join(root, "control-plane"), identity: sourceIdentity,
expectedRunnerVersion: "fixture", expectedRunnerDigest: `sha256:${"a".repeat(64)}` });
controller.stageRunAttachment(targetIdentity, { provider: { kind: "codex", instructions: "prepared attachment grant" } });
const path = join(root, "control-plane/control-plane-state.json");
const saved = JSON.parse(await readFile(path, "utf8"));
const canonicalTemplateDigest = saved.stagedRunAttachment.templateDigest;
if (mode === "changed-template" || mode === "changed-template-digest") {
saved.stagedRunAttachment.runAttachTemplate.provider.instructions = "substituted grant";
if (mode === "changed-template-digest") saved.stagedRunAttachment.templateDigest = `sha256:${nativeSha256(saved.stagedRunAttachment.runAttachTemplate)}`;
await writeFile(path, JSON.stringify(saved));
}
state.stageNativeRunnerWakeAttachments.mockImplementationOnce(async () => {
if (mode === "changed-preparation") current.task.prompt = "Inspect attachment ref:mutated-during-preparation";
return { attachments: [{ id: "attachment-original" }], cleanup: async () => undefined } as never;
});
state.renderNativeRunnerStagedAttachmentPrompt.mockReturnValueOnce("Materialized attachment: .paperclip-inbound/current/original");
const admission = structuredClone(current);
if (mode === "changed-admission") admission.task.prompt = "Inspect attachment ref:changed";
const reached = new Error("verified staged admission reached recheck");
const restore = vi.spyOn(runnerCheckpoint, "restoreNativeRunnerCheckpoint").mockResolvedValue(null);
const header = vi.spyOn(runnerCheckpoint, "loadNativeRunnerCheckpointHeader").mockResolvedValue("staged header");
const verify = vi.spyOn(stagedAttachmentRecovery, "resolveNativeStagedAttachmentRecovery")
.mockResolvedValueOnce({ currentExecution: admission, sourceIdentity, targetIdentity,
templateDigest: canonicalTemplateDigest } as never)
.mockRejectedValueOnce(reached);
try {
const resume = executePaperclipNativeSession({ db: leaseDb(current), execution: current, useRunnerd: true,
runnerInstanceId: "attachment-runner", restartRecovery: { kind: "reattach_remote_runner", runId: current.binding.runId,
leaseOwner: "controller", controllerGeneration: 2, providerAttempt: 1, restartKind: "hard", recoveryRequestId: null,
remote: { kind: "sandbox", leaseId: "attachment-lease", providerLeaseId: "sandbox", remoteCwd: "/workspace" } } as never,
runnerExecutionTarget: { kind: "remote", transport: "sandbox", leaseId: "attachment-lease", remoteCwd: "/workspace",
runner: { execute: vi.fn() }, sandboxLeaseAcquisition: { outcome: "resumed", providerLeaseId: "sandbox" } } as never });
if (mode === "attached") { await expect(resume).rejects.toBe(reached); expect(verify).toHaveBeenCalledTimes(2); }
else await expect(resume).rejects.toThrow(mode === "changed-template" ? "staged_attachment_binding_mismatch"
: mode === "changed-preparation" ? "native_attachment_admission_changed" : "staged_attachment_recovery_unproven");
expect(state.stageNativeRunnerWakeAttachments).toHaveBeenCalledWith(expect.objectContaining({ binding: expect.objectContaining({
companyId: current.binding.companyId, issueId: current.binding.issueId, agentId: current.binding.agentId, runId: current.binding.runId,
}) }));
} finally { restore.mockRestore(); header.mockRestore(); verify.mockRestore(); await controller.stop(); }
});
it("passes the run checkpoint active turn into restart recovery", async () => {
state.createBackend.mockClear();
state.createTransport.mockClear();
@@ -12213,6 +12470,27 @@ describe("runnerd provider runtime wiring", () => {
);
});
it.each(["warm", "per_turn"] as const)("defers a %s retained-owner wait before claiming a provider attempt", async mode => {
const fresh = scopedNativeExecution(`retained-owner-${mode}`);
fresh.session.lifecyclePolicy = mode === "warm"
? { mode: "warm", idleTimeoutMs: 300_000 } : { mode: "per_turn", idleTimeoutMs: null };
const deadline = new Date(Date.now() + 60_000);
vi.mocked(controllerRecovery.resolveNativeControllerRecovery).mockResolvedValueOnce({ kind: "waiting_for_owner", deadline });
const defer = vi.spyOn(controllerRecovery, "deferNativeControllerRecovery").mockResolvedValueOnce();
const onExecutionLeaseAcquired = vi.fn();
const updates: Array<{ table: unknown; values: Record<string, unknown> }> = [];
const db = leaseDb(fresh, {}, {}, updates);
await expect(executePaperclipNativeSession({ db, execution: fresh, runnerInstanceId: "waiting-controller",
useRunnerd: true, onExecutionLeaseAcquired,
runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: "/workspace",
environmentId: "retained-environment", leaseId: "retained-lease" },
})).rejects.toMatchObject({ name: "NativeControllerRecoveryWaitingError", retryAt: deadline });
expect(defer).toHaveBeenCalledWith(db, expect.objectContaining({ ...fresh.binding, retryAt: deadline }));
expect(onExecutionLeaseAcquired).not.toHaveBeenCalled();
expect(updates.some(update => update.table === nativeRunFinalizations && update.values.attempt !== undefined)).toBe(false);
expect(state.createBackend).not.toHaveBeenCalled();
});
it("rejects overlapping runs for the same runnerd provider session scope", async () => {
const first = {
...execution,
@@ -12576,7 +12854,7 @@ describe("runnerd provider runtime wiring", () => {
} as NativeExecutionInputV1;
const identity = { runId: current.binding.runId, normalizedSessionId: current.session.normalizedSessionId!,
runnerInstanceId: `admitted-${provider}`, environmentLeaseId: "admitted-lease",
turnId: `turn-${current.binding.runId}`, itemId: `item-${current.binding.runId}` };
turnId: `turn_attachment_${randomUUID()}`, itemId: `item_attachment_${randomUUID()}` };
const root = await initializeScopedRunnerRoot(current, identity.runnerInstanceId);
await mkdir(join(root, "control-plane"), { recursive: true });
await writeFile(join(root, "control-plane", "control-plane-state.json"), JSON.stringify(durableControlPlaneState(identity)));
@@ -12606,9 +12884,56 @@ describe("runnerd provider runtime wiring", () => {
runnerArtifact: artifact, adoptExistingRunner: { pid: 123, startedAt: "2026-09-19T10:00:00.000Z" } });
expect(state.resolveRemoteRunnerBinary).not.toHaveBeenCalled();
expect(execute.mock.calls.some(([request]) => request.args.includes("paperclip-runner-launch") || request.args.includes("--build-metadata"))).toBe(false);
const options = state.createTransport.mock.calls.at(-1)![0] as RunnerTransportOptions & { prepareExternalRunnerState: () => Promise<void> };
await expect(options.prepareExternalRunnerState()).resolves.toBeUndefined();
expect(execute.mock.calls.some(([request]) => request.args.includes("paperclip-runner-launch")
|| request.args.includes("paperclip-runner-claim-unstarted-session"))).toBe(false);
});
it.each(["claude", "codex"].flatMap(provider => (["per_turn", "warm"] as const).flatMap(policy => ["canonical", "local-receipt", "local-receipt-missing-terminal", "local-receipt-pending-command", "quarantined", "empty-runner", "empty-runner-quarantined", "nonempty-runner", "wrong-identity", "active", "foreign-scope", "no-stop-proof", "not-suspended", "pending-outbox", "pending-command", "missing-terminal", "ambiguous", "contradictory"].map(mode => [provider, policy, mode] as const))))(
it.each(["verified", "no-proof", "dead", "stale"] as const)("reuses only verified idle warm process during actual remote preparation: %s", async mode => {
const previous = scopedNativeExecution(`warm-live-preparation-${mode}`);
previous.session.lifecyclePolicy = { mode: "warm", idleTimeoutMs: 300_000 };
const current = { ...previous, binding: { ...previous.binding, runId: `${previous.binding.runId}-next` } };
const identity = { runId: previous.binding.runId, normalizedSessionId: previous.session.normalizedSessionId!,
runnerInstanceId: "retained-warm-runner", environmentLeaseId: "retained-lease", turnId: "prior-turn", itemId: "prior-item" };
const root = await initializeScopedRunnerRoot(previous, identity.runnerInstanceId);
await writeRunnerAuthority(root, identity, "suspended");
const projections = vi.spyOn(controllerRecovery, "resolveNativeControllerRecovery").mockResolvedValue({ kind: "no_prior_execution" });
onTestFinished(() => projections.mockRestore());
const db = leaseDb(current), ordinarySelect = db.select.bind(db);
db.select = ((fields?: Record<string, unknown>) => fields?.status === heartbeatRuns.status && fields?.runnerProfileJson === heartbeatRuns.runnerProfileJson
? priorNativeRunDb(previous).select(fields as never) : ordinarySelect(fields as never)) as Db["select"];
let alive = true;
const execute = vi.fn(async (request: { args: string[] }) => ({ exitCode: request.args[2] === "paperclip-runner-recovery-check" && !alive ? 45 : 0,
timedOut: false, stderr: "", stdout: request.args[0] === "--build-metadata" ? JSON.stringify({
schema: "paperclip-runner/runnerd-build-metadata/v1", binaryName: "paperclip-runnerd",
packageName: "@paperclipai/paperclip-runner", binaryContractVersion: 2,
durableSessionCapabilities: ["unlimited_runtime", "connection_lease_renewal"], prpTransportModes: ["listen_ws"],
}) : request.args[0] === "--version" ? "codex-cli 0.160.0" : request.args[1]?.includes("base64")
? Buffer.from(JSON.stringify({ ...identity, lifecycle: "ready", outbox: [], pendingTerminalDelivery: null })).toString("base64")
: request.args[1]?.includes("cat --") ? `nonce\n123\n2026-09-19T10:00:00.000Z\n${identity.runnerInstanceId}\nlinux:abcd-1234:100\n` : "" }));
const revalidate = vi.fn(async () => undefined);
const syncIn = vi.fn(async () => undefined);
await createRunnerdBackend({ db, execution: current, runnerInstanceId: identity.runnerInstanceId,
runnerIngressAuthorized: true,
...(mode === "no-proof" ? {} : { remoteWarmRecovery: { identity, reuseProcess: true, previousExecution: previous,
checkpoint: {} as never, remote: { providerLeaseId: "retained-sandbox", remoteCwd: "/workspace" }, revalidate } }),
runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: "/workspace", environmentId: "environment",
leaseId: identity.environmentLeaseId, providerKey: "daytona", effectiveCapabilities: { runnerWebSocketIngress: true },
sandboxLeaseAcquisition: { outcome: "resumed", providerLeaseId: "retained-sandbox" }, runner: { execute, syncIn } } as never });
state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!();
const options = state.createTransport.mock.calls.at(-1)![0] as RunnerTransportOptions & { prepareExternalRunnerState: () => Promise<void> };
if (mode === "dead") alive = false;
if (mode === "stale") revalidate.mockRejectedValueOnce(new Error("fixture_warm_owner_changed"));
if (mode === "verified") await expect(options.prepareExternalRunnerState()).resolves.toBeUndefined();
else await expect(options.prepareExternalRunnerState()).rejects.toThrow(mode === "no-proof" ? "resumed_not_suspended"
: mode === "dead" ? "runner_remote_process_identity_unavailable" : "fixture_warm_owner_changed");
expect(syncIn).not.toHaveBeenCalled();
expect(execute.mock.calls.some(([request]) => request.args.includes("paperclip-runner-launch")
|| request.args.includes("paperclip-runner-claim-unstarted-session"))).toBe(false);
});
it.each(["claude", "codex"].flatMap(provider => (["per_turn", "warm"] as const).flatMap(policy => ["canonical", "resumed", "resumed-timeout", "local-receipt", "local-receipt-missing-terminal", "local-receipt-pending-command", "quarantined", "empty-runner", "empty-runner-quarantined", "nonempty-runner", "wrong-identity", "active", "foreign-scope", "no-stop-proof", "not-suspended", "pending-outbox", "pending-command", "missing-terminal", "ambiguous", "contradictory"].filter(mode => policy === "per_turn" || !mode.startsWith("resumed")).map(mode => [provider, policy, mode] as const))))(
"recovers a blocked %s %s plugin checkpoint on a replacement lease: %s", async (provider, policy, mode) => {
const { permissionPolicy: _permissionPolicy, ...claudeProfile } = QUALIFIED_ACPX_PROFILES.claude;
const first = { ...scopedNativeExecution(`plugin-${provider}-${policy}-${mode}`),
@@ -12656,8 +12981,13 @@ describe("runnerd provider runtime wiring", () => {
if (mode === "no-stop-proof") throw new Error("Plugin Runner retained session is not authorized");
return "old-task";
});
let timedOut = false;
const task = vi.fn(async (operation: any) => {
environmentTaskOperationSchema.parse(operation);
if (mode === "resumed-timeout" && operation.kind === "read_state" && !timedOut) {
timedOut = true;
throw new Error("fixture_remote_state_timeout");
}
if (operation.kind === "read_state") return { kind: "state", taskId: "replacement-task", state: {
...durableRunnerState(identity, mode === "not-suspended" ? "ready" : "suspended"),
outbox: mode === "pending-outbox" ? [{}] : [], pendingTerminalDelivery: null,
@@ -12676,7 +13006,7 @@ describe("runnerd provider runtime wiring", () => {
managedAiCredentialHome: home, managedAiCredentialIdentity: "revision", managedAiAccountIdentity: "account",
runnerEnvironment: { ANTHROPIC_API_KEY: "synthetic-fixture" },
runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: ".", leaseId: "replacement-lease", runnerTask: task,
sandboxLeaseAcquisition: { outcome: "replacement", providerLeaseId: "replacement-task" } } as never };
sandboxLeaseAcquisition: { outcome: mode.startsWith("resumed") ? "resumed" : "replacement", providerLeaseId: "replacement-task" } } as never };
if (throughExecutor) {
const validated = validatePrpStructuredRunResult({ schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", summary: "done",
completionClaim: { contractRevision: next.completionContract.contract.revision, objectiveSatisfied: true, criteria: [], remainingWork: [] },
@@ -12686,12 +13016,19 @@ describe("runnerd provider runtime wiring", () => {
normalizedSessionId: next.session.normalizedSessionId, providerSessionId: "original-provider-history", driverKind: next.session.driverKind,
driverVersion: "1", nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null });
}
if (mode === "resumed-timeout") {
await expect(createRunnerdBackend(launch)).rejects.toThrow("fixture_remote_state_timeout");
expect(existsSync(join(scoped, "plugin-runner-state.json"))).toBe(false);
expect(await readFile(join(scoped, "control-plane/control-plane-state.json"), "utf8")).toBe(journal);
}
const backend = throughExecutor ? executePaperclipNativeSession({ ...launch, useRunnerd: true }) : createRunnerdBackend(launch);
if (["canonical", "local-receipt", "quarantined", "empty-runner", "empty-runner-quarantined"].includes(mode)) {
if (["canonical", "resumed", "resumed-timeout", "local-receipt", "quarantined", "empty-runner", "empty-runner-quarantined"].includes(mode)) {
await expect(backend).resolves.toBeDefined();
expect(state.createBackend).toHaveBeenCalledOnce();
expect(owner).toHaveBeenCalledWith(expect.objectContaining({ resumeFrom: "old-task", priorIdentity: identity }));
expect(resume).toHaveBeenCalledWith(db, expect.objectContaining({ leaseId: "replacement-lease", priorIdentity: identity, inspectCheckpoint: true }));
expect(owner).toHaveBeenCalledWith(expect.objectContaining({ priorIdentity: identity,
...(!mode.startsWith("resumed") ? { resumeFrom: "old-task" } : {}) }));
if (!mode.startsWith("resumed")) expect(resume).toHaveBeenCalledWith(db, expect.objectContaining({ leaseId: "replacement-lease", priorIdentity: identity, inspectCheckpoint: true }));
else expect(task).toHaveBeenCalledWith({ kind: "read_state", identity });
expect(JSON.parse(await readFile(join(scoped, "plugin-runner-state.json"), "utf8"))).toMatchObject({ ...identity, lifecycle: "suspended" });
expect(await readFile(join(scoped, "control-plane/control-plane-state.json"), "utf8")).toBe(journal);
} else {
File diff suppressed because it is too large. Load diff
@@ -5,6 +5,7 @@ import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js";
import { findNativeSessionResumeRun } from "./native-session-resume.js";
import { resolveNativeControllerRecovery } from "./native-controller-recovery.js";
const externalUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_TEST_DATABASE_URL");
const supported = Boolean(externalUrl) || (await getEmbeddedPostgresTestSupport()).supported;
@@ -58,7 +59,7 @@ const supported = Boolean(externalUrl) || (await getEmbeddedPostgresTestSupport(
const saved = source();
await db.insert(heartbeatRuns).values([saved, run(1), run(2)]);
const result = await select();
expect(result).toEqual({ ...saved, runnerProfileJson: { nativeExecutionInput, sessionCheckpoint: checkpoint, nativeToolContractFingerprint: "fingerprint" },
expect(result).toEqual({ ...saved, runnerProfileJson: { nativeExecutionInput, sessionCheckpoint: checkpoint, nativeToolContractFingerprint: "fingerprint", nativeControllerRecovery: null },
contextSnapshot: { paperclipWake, paperclipIssue } });
const stored = await db.select({ profile: heartbeatRuns.runnerProfileJson, context: heartbeatRuns.contextSnapshot }).from(heartbeatRuns).where(eq(heartbeatRuns.id, saved.id));
expect(stored[0].profile).toEqual(profile);
@@ -80,6 +81,16 @@ const supported = Boolean(externalUrl) || (await getEmbeddedPostgresTestSupport(
}
});
it("does not call missing or newer incompatible recovery material a fresh session", async () => {
const saved = source();
await db.insert(heartbeatRuns).values([saved, run(30, { id: currentRunId, status: "running" })]);
const { beforeCreatedAt: _before, ...binding } = input;
expect(await resolveNativeControllerRecovery(db, binding)).toEqual({ kind: "missing_evidence", reason: "snapshot_binding" });
// The newest progress barrier remains authoritative even without a snapshot.
await db.insert(heartbeatRuns).values(run(10, { runtimeMode: "legacy" }));
expect(await resolveNativeControllerRecovery(db, binding)).toEqual({ kind: "missing_evidence", reason: "newer_progress_barrier" });
});
it("skips only unused SQL-null, JSON-null, scalar, array and checkpoint-null profiles", async () => {
const saved = source(), newer = run(1, { runnerProfileJson: null });
await db.insert(heartbeatRuns).values([saved, newer]);
@@ -223,6 +223,8 @@ export async function findNativeSessionResumeRun(
normalizedSessionId: string;
currentRunId: string;
beforeCreatedAt: Date;
/** Recovery discovery must retain an ineligible newest progress barrier. */
includeProgressBarrier?: boolean;
},
) {
if (!isNativeSessionId(input.normalizedSessionId)) return null;
@@ -287,10 +289,12 @@ export async function findNativeSessionResumeRun(
nativeToolContractFingerprint: unknown;
nativeExecutionInput: unknown;
sessionCheckpoint: unknown;
nativeControllerRecovery: unknown;
}>`jsonb_build_object(
'nativeToolContractFingerprint', ${heartbeatRuns.runnerProfileJson}->'nativeToolContractFingerprint',
'nativeExecutionInput', ${heartbeatRuns.runnerProfileJson}->'nativeExecutionInput',
'sessionCheckpoint', ${heartbeatRuns.runnerProfileJson}->'sessionCheckpoint'
'sessionCheckpoint', ${heartbeatRuns.runnerProfileJson}->'sessionCheckpoint',
'nativeControllerRecovery', ${heartbeatRuns.runnerProfileJson}->'nativeControllerRecovery'
)`,
contextSnapshot: sql<{
paperclipWake: unknown;
@@ -306,14 +310,16 @@ export async function findNativeSessionResumeRun(
.from(heartbeatRuns)
.innerJoin(barrier, eq(heartbeatRuns.id, barrier.id))
.where(and(
eq(barrier.runtimeMode, "native"),
inArray(barrier.status, terminalStatuses),
eq(barrier.hasCheckpoint, true),
...(input.includeProgressBarrier ? [] : [
eq(barrier.runtimeMode, "native"),
inArray(barrier.status, terminalStatuses),
eq(barrier.hasCheckpoint, true),
sql`date_trunc('milliseconds', ${barrier.createdAt}) <= ${input.beforeCreatedAt.toISOString()}::timestamptz`,
]),
sql`${heartbeatRuns.companyId}::text = ${input.companyId}`,
sql`${heartbeatRuns.agentId}::text = ${input.agentId}`,
sql`${heartbeatRuns.nativeIssueId}::text = ${input.issueId}`,
sql`${heartbeatRuns.nativeSessionId}::text = ${input.normalizedSessionId}`,
sql`date_trunc('milliseconds', ${barrier.createdAt}) <= ${input.beforeCreatedAt.toISOString()}::timestamptz`,
));
return run ?? null;
}
@@ -0,0 +1,187 @@
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { Db } from "@paperclipai/db";
import { afterEach, describe, expect, it, vi } from "vitest";
import { DurablePrpControlPlane, type DurablePrpCheckpoint } from "../../vendor/paperclip-runner/index.js";
import { createLocalEncryptedCipher } from "../../secrets/local-encrypted-provider.js";
import { buildNativeExecutionInput } from "./native-execution-input.js";
import { nativeToolContractFingerprintForTarget } from "./native-session-resume.js";
import { nativeRuntimeContextFixture } from "./runtime-context.test-fixture.js";
import { resolveNativeStagedAttachmentRecovery } from "./native-staged-attachment-recovery.js";
const companyId = "10000000-0000-4000-8000-000000000001";
const issueId = "20000000-0000-4000-8000-000000000002";
const agentId = "30000000-0000-4000-8000-000000000003";
const sourceRunId = "40000000-0000-4000-8000-000000000004";
const currentRunId = "50000000-0000-4000-8000-000000000005";
const normalizedSessionId = "60000000-0000-4000-8000-000000000006";
const runnerInstanceId = "70000000-0000-4000-8000-000000000007";
const sourceIdentity = { runId: sourceRunId, normalizedSessionId, runnerInstanceId,
environmentLeaseId: "environment-1", turnId: "opaque-source-turn", itemId: "opaque-source-item" };
const targetIdentity = { ...sourceIdentity, runId: currentRunId, turnId: "opaque-target-turn", itemId: "opaque-target-item" };
const roots: string[] = [];
afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); });
function execution(runId: string) {
return buildNativeExecutionInput({ companyId, runId, agentId, normalizedSessionId,
issue: { id: issueId, identifier: "TEST-1", title: "Continue", description: null, workMode: "standard" },
taskPrompt: "Continue normally", provider: "codex", workspace: { id: "workspace-1", cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null },
completionContract: { id: "80000000-0000-4000-8000-000000000008", sha256: `sha256:${"a".repeat(64)}`,
schemaVersion: "paperclip.run-result.v1", contract: { revision: "1", objective: "Continue", criteria: [{ id: "done", requirement: "Answer" }] } },
runtimeContext: nativeRuntimeContextFixture(),
});
}
async function fixture() {
const root = await mkdtemp(join(tmpdir(), "native-staged-proof-")); roots.push(root);
const core = new DurablePrpControlPlane({ stateDirectory: root, identity: sourceIdentity,
expectedRunnerVersion: "0.0.0", expectedRunnerDigest: `sha256:${"f".repeat(64)}` });
let saved!: DurablePrpCheckpoint;
core.setCheckpointPersistence(async checkpoint => { saved = checkpoint; });
core.persistRunAttachTemplate({ provider: { provider: "codex", command: "codex" }, authorizedTools: {} });
await core.flushCheckpoint();
const sourceHeader = saved.records.get("header")!;
core.stageRunAttachment(targetIdentity, { provider: { provider: "codex", command: "codex" }, authorizedTools: {} });
await core.flushCheckpoint();
const header = saved.records.get("header")!;
const base = { companyId, agentId, issueId, nativeIssueId: issueId, runnerInstanceId,
nativeSessionId: normalizedSessionId, runtimeMode: "native", status: "running", finishedAt: null as Date | null };
const currentRun = { ...base, id: currentRunId, runnerProfileJson: { nativeExecutionInput: execution(currentRunId) } };
const sourceRun = { ...base, id: sourceRunId, status: "succeeded", finishedAt: new Date(), runnerProfileJson: {
nativeExecutionInput: execution(sourceRunId), nativeToolContractFingerprint: nativeToolContractFingerprintForTarget("remote", "codex"),
sessionCheckpoint: { sessionId: "provider-thread-1", providerSessionId: "provider-thread-1", identity: {
companyId, issueId, agentId, runId: sourceRunId, sessionId: normalizedSessionId },
},
} };
const source = { run: sourceRun, coordinator: { runId: sourceRunId, companyId, issueId,
phase: "committed", resultId: "result-1", leaseOwner: null, leaseExpiresAt: null }, resultId: "result-1", resultStatus: "accepted" };
function database(sourceRows = [source], contents = sourceHeader, envelope = {}) {
const material = createLocalEncryptedCipher("write").encrypt(JSON.stringify({ companyId, issueId, runId: sourceRunId, part: "header", contents, ...envelope }));
const batches: unknown[][] = [sourceRows, [{ material }]];
const select = vi.fn(() => {
const chain = { from: () => chain, innerJoin: () => chain, where: () => chain,
limit: vi.fn(async () => batches.shift() ?? []) };
return chain;
});
return { db: { select } as unknown as Pick<Db, "select">, select };
}
return { header, sourceHeader, source, sourceRun, currentRun, database };
}
describe("private staged attachment restart evidence", () => {
it("proves both frozen run bindings before the target has a session snapshot", async () => {
const f = await fixture(); const { db, select } = f.database();
const matchesExecution = vi.fn(() => true);
const result = await resolveNativeStagedAttachmentRecovery({ db, header: f.header, currentRun: f.currentRun,
expectedTarget: targetIdentity, matchesExecution });
expect(result).toMatchObject({ sourceIdentity, targetIdentity, sourceRun: { id: sourceRunId }, hasTargetCheckpoint: false, targetCheckpoint: null,
protocolIdentity: sourceIdentity, phase: "source_bootstrap", receipt: null });
expect(matchesExecution).toHaveBeenCalledOnce();
expect(select).toHaveBeenCalledTimes(2); // one source point lookup and one encrypted header point lookup
expect(f.currentRun.runnerProfileJson).not.toHaveProperty("sessionCheckpoint");
});
it("retains staged evidence when the target runtime has already published its snapshot", async () => {
const f = await fixture();
const snapshot = structuredClone(f.sourceRun.runnerProfileJson.sessionCheckpoint);
snapshot.identity.runId = currentRunId;
Object.assign(snapshot, { cursor: "target-cursor", activeTurnId: "target-current-turn", pendingRuntimeRequests: [{ requestId: "target-request" }] });
Object.assign(f.currentRun.runnerProfileJson, { sessionCheckpoint: snapshot });
const result = await resolveNativeStagedAttachmentRecovery({ ...f.database(), header: f.header, currentRun: f.currentRun });
expect(result).toMatchObject({ sourceIdentity, targetIdentity, checkpoint: { identity: { runId: currentRunId } },
hasTargetCheckpoint: true, targetCheckpoint: snapshot });
expect(f.currentRun.runnerProfileJson).toHaveProperty("sessionCheckpoint", snapshot);
});
it("returns the codec's queued phase without assuming the Runner has activated the target", async () => {
const f = await fixture(); const header = JSON.parse(f.header);
// Header inspection deliberately does not certify separate command records.
// Full restoration/authentication owns that later gate; no RPC runs here.
header.state.stagedRunAttachment.commandId = "staged-attach-command";
const result = await resolveNativeStagedAttachmentRecovery({ ...f.database(), header: JSON.stringify(header), currentRun: f.currentRun });
expect(result).toMatchObject({ commandId: "staged-attach-command", protocolIdentity: sourceIdentity,
phase: "attachment_queued", receipt: null, targetIdentity });
});
it.each(["runId", "companyId", "issueId", "agentId", "sessionId", "providerSessionId"])(
"rejects a present target checkpoint with changed %s instead of replaying the source", async key => {
const f = await fixture(); const snapshot = structuredClone(f.sourceRun.runnerProfileJson.sessionCheckpoint);
snapshot.identity.runId = currentRunId;
if (key === "providerSessionId") snapshot.providerSessionId = "foreign";
else Object.assign(snapshot.identity, { [key]: "foreign" });
Object.assign(f.currentRun.runnerProfileJson, { sessionCheckpoint: snapshot });
expect(await resolveNativeStagedAttachmentRecovery({ ...f.database(), header: f.header, currentRun: f.currentRun })).toBeNull();
});
it.each(["runId", "runnerInstanceId", "environmentLeaseId", "normalizedSessionId", "turnId", "itemId"] as const)(
"rejects a different expected target %s before querying the source", async key => {
const f = await fixture(); const { db, select } = f.database();
expect(await resolveNativeStagedAttachmentRecovery({ db, header: f.header, currentRun: f.currentRun,
expectedTarget: { ...targetIdentity, [key]: "foreign" } })).toBeNull();
expect(select).not.toHaveBeenCalled();
});
it.each(["agentId", "companyId", "nativeIssueId", "nativeSessionId", "runnerInstanceId"] as const)(
"rejects mixed source DB %s", async key => {
const f = await fixture(); Object.assign(f.source.run, { [key]: "foreign" });
expect(await resolveNativeStagedAttachmentRecovery({ ...f.database(), header: f.header, currentRun: f.currentRun })).toBeNull();
});
it.each(["binding", "schema", "runner", "session"])("rejects current frozen evidence mismatch: %s", async kind => {
const f = await fixture(); const { db, select } = f.database();
if (kind === "binding") f.currentRun.runnerProfileJson.nativeExecutionInput.binding.agentId = sourceRunId;
if (kind === "schema") Object.assign(f.currentRun.runnerProfileJson.nativeExecutionInput, { schema: "unsupported" });
if (kind === "runner") f.currentRun.runnerInstanceId = sourceRunId;
if (kind === "session") f.currentRun.nativeSessionId = sourceRunId;
expect(await resolveNativeStagedAttachmentRecovery({ db, header: f.header, currentRun: f.currentRun })).toBeNull();
expect(select).not.toHaveBeenCalled();
});
it.each(["phase", "leaseOwner", "resultStatus", "status", "finishedAt", "frozenRun", "workspace", "provider", "sourceTuple", "artifactVersion", "artifactDigest"])(
"rejects unsupported or stale source evidence: %s", async kind => {
const f = await fixture(); let sourceHeader = f.sourceHeader;
if (kind === "phase") f.source.coordinator.phase = "observed";
if (kind === "leaseOwner") Object.assign(f.source.coordinator, { leaseOwner: "another-owner", leaseExpiresAt: new Date(0) });
if (kind === "resultStatus") f.source.resultStatus = "rejected";
if (kind === "status") f.source.run.status = "running";
if (kind === "finishedAt") Object.assign(f.source.run, { finishedAt: null });
if (kind === "frozenRun") f.source.run.runnerProfileJson.nativeExecutionInput.binding.runId = currentRunId;
if (kind === "workspace") f.source.run.runnerProfileJson.nativeExecutionInput.binding.executionWorkspaceId = "foreign";
if (kind === "provider") Object.assign(f.source.run.runnerProfileJson.nativeExecutionInput.provider, { model: "another-model" });
if (kind === "sourceTuple") { const h = JSON.parse(sourceHeader); h.state.identity.turnId = "foreign"; sourceHeader = JSON.stringify(h); }
if (kind === "artifactVersion") { const h = JSON.parse(sourceHeader); h.runnerVersion = "other-version"; sourceHeader = JSON.stringify(h); }
if (kind === "artifactDigest") { const h = JSON.parse(sourceHeader); h.runnerDigest = `sha256:${"d".repeat(64)}`; sourceHeader = JSON.stringify(h); }
expect(await resolveNativeStagedAttachmentRecovery({ ...f.database([f.source], sourceHeader), header: f.header, currentRun: f.currentRun })).toBeNull();
});
it("keeps private executor scope/config verification mandatory when supplied", async () => {
const f = await fixture();
expect(await resolveNativeStagedAttachmentRecovery({ ...f.database(), header: f.header, currentRun: f.currentRun,
matchesExecution: () => false })).toBeNull();
});
it.each(["companyId", "issueId", "runId", "part"])("rejects encrypted source header with another %s", async key => {
const f = await fixture();
expect(await resolveNativeStagedAttachmentRecovery({ ...f.database([f.source], f.sourceHeader, { [key]: "foreign" }),
header: f.header, currentRun: f.currentRun })).toBeNull();
});
it("does not accept a source lacking the current tool contract or provider identity", async () => {
const f = await fixture();
f.source.run.runnerProfileJson.nativeToolContractFingerprint = "old-contract";
expect(await resolveNativeStagedAttachmentRecovery({ ...f.database(), header: f.header, currentRun: f.currentRun })).toBeNull();
f.source.run.runnerProfileJson.nativeToolContractFingerprint = nativeToolContractFingerprintForTarget("remote", "codex");
f.source.run.runnerProfileJson.sessionCheckpoint.providerSessionId = "";
expect(await resolveNativeStagedAttachmentRecovery({ ...f.database(), header: f.header, currentRun: f.currentRun })).toBeNull();
});
it("rejects corrupted or ordinary headers instead of promoting them to staged evidence", async () => {
const f = await fixture();
for (const header of [f.sourceHeader, "{}", "not-json"]) {
const { db, select } = f.database();
expect(await resolveNativeStagedAttachmentRecovery({ db, header, currentRun: f.currentRun })).toBeNull();
expect(select).not.toHaveBeenCalled();
}
});
});
@@ -0,0 +1,124 @@
import { isDeepStrictEqual } from "node:util";
import { and, eq } from "drizzle-orm";
import { heartbeatRuns, nativeRunFinalizations, nativeRunResults, type Db } from "@paperclipai/db";
import {
inspectDurablePrpCheckpointHeader,
inspectStagedRunAttachmentHeader,
parseNativeExecutionInput,
type DurableRecoveryIdentity,
type NativeExecutionInput,
type PersistedNativeSession,
type StagedRunAttachment,
} from "../../vendor/paperclip-runner/index.js";
import { loadNativeRunnerCheckpointHeader } from "./native-runner-checkpoint.js";
import { rebindNativeSessionCheckpoint } from "./native-session-resume.js";
type FrozenRun = Pick<typeof heartbeatRuns.$inferSelect,
"id" | "companyId" | "agentId" | "nativeIssueId" | "nativeSessionId" | "runnerInstanceId" |
"runtimeMode" | "status" | "runnerProfileJson" | "finishedAt">;
type StagedHeaderProof = NonNullable<ReturnType<typeof inspectStagedRunAttachmentHeader>>;
export type NativeStagedAttachmentRecovery = Omit<StagedRunAttachment, "runAttachTemplate"> &
Pick<StagedHeaderProof, "protocolIdentity" | "phase" | "receipt"> & {
sourceRun: FrozenRun;
sourceExecution: NativeExecutionInput;
currentExecution: NativeExecutionInput;
/** In-memory input only. It is not a published/resumable target snapshot. */
checkpoint: PersistedNativeSession;
/** A published target snapshot takes precedence over the source rebound. */
hasTargetCheckpoint: boolean;
targetCheckpoint: PersistedNativeSession | null;
runnerVersion: string;
runnerDigest: string;
};
function record(value: unknown): Record<string, unknown> {
return value !== null && typeof value === "object" && !Array.isArray(value)
? value as Record<string, unknown> : {};
}
function boundTargetCheckpoint(run: FrozenRun, source: PersistedNativeSession): PersistedNativeSession | null {
const snapshot = record(run.runnerProfileJson?.sessionCheckpoint), identity = record(snapshot.identity);
if (typeof snapshot.sessionId !== "string" || !snapshot.sessionId || identity.runId !== run.id ||
identity.companyId !== run.companyId || identity.issueId !== run.nativeIssueId || identity.agentId !== run.agentId ||
identity.sessionId !== run.nativeSessionId) return null;
// A target checkpoint may advance cursor/goal/tool progress, but cannot change
// the provider session retained by this attachment. Do not rebound it again.
if ((source.providerSessionId != null && source.providerSessionId !== snapshot.providerSessionId) ||
(source.providerIdentity != null && !isDeepStrictEqual(source.providerIdentity, snapshot.providerIdentity))) return null;
return structuredClone(snapshot) as unknown as PersistedNativeSession;
}
function frozenExecution(run: FrozenRun): NativeExecutionInput | null {
try {
const execution = parseNativeExecutionInput(run.runnerProfileJson?.nativeExecutionInput);
return run.runtimeMode === "native" && execution.binding.runId === run.id &&
execution.binding.companyId === run.companyId && execution.binding.agentId === run.agentId &&
execution.binding.issueId === run.nativeIssueId && execution.session.normalizedSessionId === run.nativeSessionId
? execution : null;
} catch { return null; }
}
function matchesRun(identity: DurableRecoveryIdentity, run: FrozenRun): boolean {
return identity.runId === run.id && identity.runnerInstanceId === run.runnerInstanceId &&
identity.normalizedSessionId === run.nativeSessionId;
}
/** Staging proves which prior attachment may be inspected, not that it is alive
* or that work may be dispatched. Classification reads only two indexed point
* lookups: the named source/result and its encrypted header. The executor must
* re-read this proof with its private scope/config matcher before any RPC, then
* independently prove the physical Runner and authenticate the target tuple. */
export async function resolveNativeStagedAttachmentRecovery(input: {
db: Pick<Db, "select">;
currentRun: FrozenRun;
header: string;
expectedTarget?: DurableRecoveryIdentity;
matchesExecution?: (previous: NativeExecutionInput, current: NativeExecutionInput) => boolean;
}): Promise<NativeStagedAttachmentRecovery | null> {
const proof = inspectStagedRunAttachmentHeader(input.header);
const current = frozenExecution(input.currentRun);
if (!proof || !current || current.provider.kind !== "codex" || !["running", "failed"].includes(input.currentRun.status) ||
!matchesRun(proof.attachment.targetIdentity, input.currentRun) ||
(input.expectedTarget && !isDeepStrictEqual(proof.attachment.targetIdentity, input.expectedTarget))) return null;
const { attachment } = proof;
const [source] = await input.db.select({ run: heartbeatRuns, coordinator: nativeRunFinalizations,
resultId: nativeRunResults.id, resultStatus: nativeRunResults.schemaStatus }).from(heartbeatRuns)
.innerJoin(nativeRunFinalizations, and(eq(nativeRunFinalizations.runId, heartbeatRuns.id),
eq(nativeRunFinalizations.companyId, heartbeatRuns.companyId), eq(nativeRunFinalizations.issueId, heartbeatRuns.nativeIssueId)))
.innerJoin(nativeRunResults, and(eq(nativeRunResults.id, nativeRunFinalizations.resultId),
eq(nativeRunResults.runId, heartbeatRuns.id), eq(nativeRunResults.companyId, heartbeatRuns.companyId),
eq(nativeRunResults.issueId, heartbeatRuns.nativeIssueId)))
.where(and(eq(heartbeatRuns.id, attachment.sourceIdentity.runId), eq(heartbeatRuns.companyId, input.currentRun.companyId),
eq(heartbeatRuns.agentId, input.currentRun.agentId), eq(heartbeatRuns.nativeIssueId, current.binding.issueId)))
.limit(1);
if (!source || source.run.companyId !== current.binding.companyId || source.run.agentId !== current.binding.agentId ||
source.run.nativeIssueId !== current.binding.issueId || !matchesRun(attachment.sourceIdentity, source.run) ||
source.run.status !== "succeeded" || source.run.finishedAt === null || source.coordinator.phase !== "committed" ||
source.coordinator.companyId !== current.binding.companyId || source.coordinator.issueId !== current.binding.issueId ||
source.coordinator.runId !== source.run.id || !source.coordinator.resultId ||
source.coordinator.resultId !== source.resultId || source.resultStatus !== "accepted" ||
source.coordinator.leaseOwner !== null || source.coordinator.leaseExpiresAt !== null) return null;
const previous = frozenExecution(source.run);
if (!previous || (input.matchesExecution && !input.matchesExecution(previous, current))) return null;
// Reuse the existing provider, tool-contract, workspace, runtime-context and
// normalized-session compatibility owner; no second set of digest formulas.
const checkpoint = rebindNativeSessionCheckpoint({ previousRun: source.run, currentExecution: current, executionTargetKind: "remote" });
if (!checkpoint || !(typeof checkpoint.providerSessionId === "string" && checkpoint.providerSessionId.length > 0 ||
checkpoint.providerIdentity && Object.keys(checkpoint.providerIdentity).length > 0)) return null;
const hasTargetCheckpoint = input.currentRun.runnerProfileJson?.sessionCheckpoint != null;
const targetCheckpoint = hasTargetCheckpoint ? boundTargetCheckpoint(input.currentRun, checkpoint) : null;
if (hasTargetCheckpoint && !targetCheckpoint) return null;
const sourceHeader = await loadNativeRunnerCheckpointHeader(input.db, {
companyId: source.run.companyId, issueId: current.binding.issueId, runId: source.run.id,
});
if (!sourceHeader || !isDeepStrictEqual(inspectDurablePrpCheckpointHeader(sourceHeader), attachment.sourceIdentity)) return null;
// The codec above has validated this bounded header's artifact fields. A
// staged target cannot silently select different bytes from its source.
const sourceArtifact = JSON.parse(sourceHeader) as { runnerVersion: string; runnerDigest: string };
if (sourceArtifact.runnerVersion !== proof.runnerVersion || sourceArtifact.runnerDigest !== proof.runnerDigest) return null;
return { ...attachment, sourceRun: source.run, sourceExecution: previous, currentExecution: current,
checkpoint, hasTargetCheckpoint, targetCheckpoint,
protocolIdentity: proof.protocolIdentity, phase: proof.phase, receipt: proof.receipt,
runnerVersion: proof.runnerVersion, runnerDigest: proof.runnerDigest };
}
@@ -0,0 +1,319 @@
import { createRequire } from "node:module";
import { randomUUID } from "node:crypto";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { eq, sql } from "drizzle-orm";
import { drizzle } from "drizzle-orm/postgres-js";
import { agents, agentTaskSessions, companies, completionContracts, createDb, heartbeatRuns, issues,
nativeRunFinalizations, nativeRunResults, type Db } from "@paperclipai/db";
import { CONTROL_PLANE_CONFORMANCE_OPEN, CONTROL_PLANE_CONFORMANCE_RESULT,
CONTROL_PLANE_CONFORMANCE_TERMINAL } from "../../vendor/paperclip-runner/testing.js";
import type { PersistedNativeSession } from "../../vendor/paperclip-runner/index.js";
import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js";
import { startRecoveryPostgres } from "../../__tests__/helpers/recovery-postgres.js";
import { redactRegisteredSecretValues } from "../run-secret-redaction.js";
import { nativeSha256 } from "./canonical.js";
import { PaperclipControlPlanePort, type PaperclipControlPlaneBinding } from "./paperclip-control-plane-port.js";
import { nativeTaskSessionPublicationHandled, publishNativeTaskSessionInTransaction, NATIVE_TASK_SESSION_PUBLICATION_KEY,
type NativeTaskSessionPublicationContext } from "./native-task-session-publication.js";
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_NATIVE_PUBLICATION_TEST_DATABASE_URL");
describe("accepted native task session publication", () => {
let temporary: Awaited<ReturnType<typeof startRecoveryPostgres>> | null = null;
let db: Db;
let measured: Db;
let client: Db["$client"];
const statements: string[] = [];
beforeAll(async () => {
if (!external) temporary = await startRecoveryPostgres("native-publication-");
const url = external ?? temporary!.connectionString;
db = createDb(url, { maxConnections: 2 });
// Test-only wire observer. Production still uses the existing Db/pools.
const postgres = createRequire(import.meta.resolve("@paperclipai/db"))("postgres") as
(url: string, options: object) => Db["$client"];
client = postgres(url, { max: 1, prepare: false, connection: { application_name: "native-publication-observer" },
debug: (_id: number, query: string) => { statements.push(query); } });
measured = drizzle(client) as unknown as Db;
});
afterAll(async () => {
await client?.end({ timeout: 1 });
await db?.$client.end({ timeout: 1 });
await temporary?.cleanup();
});
async function fixture(source: "runner" | "control" = "runner", accepted = true) {
const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID();
const sessionId = randomUUID(), contractId = randomUUID(), runner = randomUUID(), control = randomUUID();
await db.insert(companies).values({ id: companyId, name: "Native pointer", issuePrefix: `P${companyId.slice(0, 7)}` });
await db.insert(agents).values({ id: agentId, companyId, name: "Native pointer", adapterType: "codex_local", status: "running" });
await db.insert(issues).values({ id: issueId, companyId, title: "Native pointer", status: "in_progress", assigneeAgentId: agentId });
await db.insert(completionContracts).values({ id: contractId, companyId, issueId, revision: 1,
schemaVersion: "paperclip.completion-contract.v1", policyVersion: "standalone-v1", risk: "standard",
completionAuthority: "server_arbiter", incompleteCriteriaPolicy: "preserve_non_terminal",
contractJson: { revision: "standalone-v1", objective: "Conformance", criteria: [{ id: "objective", requirement: "Complete" }] },
canonicalSha256: "publication-contract", createdByActorType: "system", createdByActorId: "test" });
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running", runtimeMode: "native",
nativeIssueId: issueId, nativeSessionId: sessionId, runnerInstanceId: runner, completionContractId: contractId,
completionContractSha256: "publication-contract", startedAt: new Date(Date.now() - 2_000), runnerProfileJson: {} });
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId));
const owner = { companyId, issueId, runId, authorityRunId: runId, bootId: "publication-boot", generation: 1, leaseOwner: "publication-owner" };
await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId, phase: "observed",
controllerBootId: owner.bootId, controllerGeneration: owner.generation, leaseOwner: owner.leaseOwner,
leaseExpiresAt: new Date(Date.now() + 60_000) });
const binding: PaperclipControlPlaneBinding = { companyId, agentId, issueId, runId, sessionId,
completionContractId: contractId, completionContractSha256: "publication-contract", sourceInstanceId: runner,
controlPlaneSourceInstanceId: control };
const context: NativeTaskSessionPublicationContext = { taskKey: issueId, adapterType: "codex_local",
sessionParamsJson: { cwd: "/working", paperclipSessionConfigFingerprint: "prepared-once", opaque: "private-prepared-param" } };
const snapshot: PersistedNativeSession = { backendKind: "mock", sessionId, identity: { companyId, agentId, issueId, runId, sessionId },
providerSessionId: "provider-thread", activeTurnId: null, pendingRuntimeRequests: [], goal: null,
semanticResult: CONTROL_PLANE_CONFORMANCE_RESULT, terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL };
const port = new PaperclipControlPlanePort(measured, binding, { taskSessionPublication: context, taskSessionPublicationOwner: owner });
await port.openRun({ ...CONTROL_PLANE_CONFORMANCE_OPEN, identity: snapshot.identity, sourceInstanceId: runner });
if (accepted) await new PaperclipControlPlanePort(db, binding).completeRun({ result: CONTROL_PLANE_CONFORMANCE_RESULT,
terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, turnId: "publication-turn" });
await db.insert(agentTaskSessions).values({ companyId, agentId, adapterType: context.adapterType, taskKey: issueId,
goalSourceId: `${source === "runner" ? runner : control}:${runId}`, goalSourceCursor: 4 });
const read = async () => {
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
const [session] = await db.select().from(agentTaskSessions).where(eq(agentTaskSessions.taskKey, issueId));
return { run, session };
};
return { binding, owner, context, snapshot, port, read };
}
it.each(["runner", "control"] as const)("publishes from the exact accepted %s epoch in one checkpoint transaction", async source => {
const f = await fixture(source); statements.length = 0;
await f.port.checkpointSession(f.snapshot);
const queries = [...statements], { run, session } = await f.read();
expect(session).toMatchObject({ sessionParamsJson: { ...f.context.sessionParamsJson, sessionId: f.binding.sessionId },
sessionDisplayId: "provider-thread", lastRunId: f.binding.runId });
expect(nativeTaskSessionPublicationHandled(run.runnerProfileJson, { ...f.binding, ...f.context })).toBe(true);
expect(queries.filter(query => /^begin\b/i.test(query.trim()))).toHaveLength(1);
expect(queries.filter(query => /^commit\b/i.test(query.trim()))).toHaveLength(1);
expect(queries.filter(query => /native_run_results/i.test(query))).toHaveLength(1);
expect(queries.filter(query => /^select .*from "agent_task_sessions"/i.test(query.trim()))).toHaveLength(1);
expect(queries.filter(query => /^update "agent_task_sessions"/i.test(query.trim()))).toHaveLength(1);
expect(queries).toHaveLength(7);
expect(JSON.stringify(run.runnerProfileJson)).not.toContain("private-prepared-param");
expect(JSON.stringify({ [NATIVE_TASK_SESSION_PUBLICATION_KEY]: run.runnerProfileJson![NATIVE_TASK_SESSION_PUBLICATION_KEY] }).length).toBeLessThan(4_096);
expect(redactRegisteredSecretValues(run.runnerProfileJson, [])).not.toHaveProperty(NATIVE_TASK_SESSION_PUBLICATION_KEY);
expect(nativeTaskSessionPublicationHandled(run.runnerProfileJson, { ...f.binding, ...f.context, companyId: randomUUID() })).toBe(false);
expect(nativeTaskSessionPublicationHandled(run.runnerProfileJson, { ...f.binding, ...f.context, agentId: randomUUID() })).toBe(false);
const malformed = structuredClone(run.runnerProfileJson!);
(malformed[NATIVE_TASK_SESSION_PUBLICATION_KEY] as Record<string, unknown>).resultId = "not-an-id";
expect(nativeTaskSessionPublicationHandled(malformed, { ...f.binding, ...f.context })).toBe(false);
});
it("does not recreate a reset pointer or query task state on repeated terminal maintenance", async () => {
const f = await fixture(); await f.port.checkpointSession(f.snapshot);
await db.delete(agentTaskSessions).where(eq(agentTaskSessions.taskKey, f.binding.issueId)); statements.length = 0;
await f.port.checkpointSession(f.snapshot);
const repeated = [...statements];
expect((await f.read()).session).toBeUndefined();
expect(repeated.filter(query => /agent_task_sessions|native_run_results/i.test(query))).toHaveLength(0);
expect(repeated.filter(query => /^begin\b/i.test(query.trim()))).toHaveLength(1);
expect(repeated.filter(query => /^update /i.test(query.trim()))).toHaveLength(1);
});
it("persists a pre-acceptance terminal observation and publishes only at the existing accepted checkpoint", async () => {
const f = await fixture("runner", false);
// Driver event/startup checkpoints can expose a semantic terminal observation
// before the runtime calls completeRun. That observation is not acceptance.
await f.port.checkpointSession(f.snapshot);
const observed = await f.read();
expect(observed.run.runnerProfileJson?.sessionCheckpoint).toMatchObject({ semanticResult: CONTROL_PLANE_CONFORMANCE_RESULT });
expect(observed.run.runnerProfileJson).not.toHaveProperty(NATIVE_TASK_SESSION_PUBLICATION_KEY);
expect(observed.session?.sessionParamsJson).toBeNull();
expect(nativeTaskSessionPublicationHandled(observed.run.runnerProfileJson, { ...f.binding, ...f.context })).toBe(false);
await f.port.completeRun({ result: CONTROL_PLANE_CONFORMANCE_RESULT,
terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, turnId: "publication-turn" });
expect((await f.read()).session?.sessionParamsJson).toBeNull();
await f.port.checkpointSession(f.snapshot);
const completed = await f.read();
expect(completed.session?.lastRunId).toBe(f.binding.runId);
expect(nativeTaskSessionPublicationHandled(completed.run.runnerProfileJson, { ...f.binding, ...f.context })).toBe(true);
});
it("does not hold the issue parent lock while a result replay waits for the status coordinator", async () => {
const f = await fixture();
let committed = 0;
const port = new PaperclipControlPlanePort(db, f.binding, { nativeRecovery: { owner: f.owner,
compatibility: { configDigest: nativeSha256("config"), processConfigDigest: null } },
onResultCommitted: async () => { committed++; } });
await port.openRun({ ...CONTROL_PLANE_CONFORMANCE_OPEN, identity: f.snapshot.identity, sourceInstanceId: f.binding.sourceInstanceId });
let replay: Promise<unknown> | undefined;
let statusLockError: unknown = null;
try {
await db.transaction(async tx => {
// This is commitNativeStatusDecision's actual lock prefix. Hold its
// coordinator first, then overlap an identical remote result replay.
await tx.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.binding.runId)).for("update");
const [backend] = await tx.execute(sql`select pg_backend_pid() as pid`);
replay = port.completeRun({ result: CONTROL_PLANE_CONFORMANCE_RESULT,
terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, turnId: "publication-turn" }).then(() => null, error => error);
let waiting = false;
for (let attempt = 0; attempt < 100; attempt++) {
const [state] = await measured.execute(sql`select exists (
select 1 from pg_stat_activity where ${backend.pid} = any(pg_blocking_pids(pid))) as waiting`);
if (state.waiting) { waiting = true; break; }
await new Promise(resolve => setTimeout(resolve, 10));
}
expect(waiting).toBe(true);
// A bounded test-local timeout reports the inverse-order cycle without
// relying on which transaction PostgreSQL chooses as deadlock victim.
await tx.execute(sql`set local lock_timeout = '500ms'`);
await tx.select().from(issues).where(eq(issues.id, f.binding.issueId)).for("update");
});
} catch (error) { statusLockError = error; }
const replayError = await replay;
expect(statusLockError).toBeNull();
expect(replayError).toBeNull();
expect(committed).toBe(0);
expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, f.binding.runId))).toHaveLength(1);
await db.update(nativeRunFinalizations).set({ controllerGeneration: 2 }).where(eq(nativeRunFinalizations.runId, f.binding.runId));
await expect(port.completeRun({ result: CONTROL_PLANE_CONFORMANCE_RESULT,
terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, turnId: "publication-turn" })).rejects.toThrow("checkpoint_owner_lost");
});
it.each(["missing", "foreign_epoch", "newer_pointer", "newer_execution", "generation_reset", "stop"])("fences %s and records a handled suppression", async kind => {
const f = await fixture();
if (kind === "missing") await db.delete(agentTaskSessions).where(eq(agentTaskSessions.taskKey, f.binding.issueId));
if (kind === "foreign_epoch") await db.update(agentTaskSessions).set({ goalSourceId: `${randomUUID()}:${f.binding.runId}` }).where(eq(agentTaskSessions.taskKey, f.binding.issueId));
if (kind === "newer_pointer" || kind === "newer_execution") {
const newer = randomUUID();
await db.insert(heartbeatRuns).values({ id: newer, companyId: f.binding.companyId, agentId: f.binding.agentId,
status: "running", issueId: f.binding.issueId, startedAt: new Date() });
if (kind === "newer_pointer") await db.update(agentTaskSessions).set({ lastRunId: newer, sessionParamsJson: { sessionId: "newer" } }).where(eq(agentTaskSessions.taskKey, f.binding.issueId));
else await db.update(issues).set({ executionRunId: newer }).where(eq(issues.id, f.binding.issueId));
}
if (kind === "generation_reset") await db.update(issues).set({ conversationAgentId: f.binding.agentId,
conversationUserId: "publication-test-user", conversationState: "active", conversationSessionGeneration: 2 }).where(eq(issues.id, f.binding.issueId));
if (kind === "stop") await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, f.binding.runId));
await f.port.checkpointSession(f.snapshot);
const { run, session } = await f.read();
expect(nativeTaskSessionPublicationHandled(run.runnerProfileJson, { ...f.binding, ...f.context })).toBe(true);
expect(session?.sessionParamsJson?.sessionId).not.toBe(f.binding.sessionId);
});
it("keeps unsupported task keys and missing remote recovery headers outside handled publication", async () => {
const f = await fixture();
const context = { ...f.context, taskKey: "custom-task" };
const port = new PaperclipControlPlanePort(measured, f.binding, { taskSessionPublication: context, taskSessionPublicationOwner: f.owner });
await port.openRun({ ...CONTROL_PLANE_CONFORMANCE_OPEN, identity: f.snapshot.identity, sourceInstanceId: f.binding.sourceInstanceId });
await port.checkpointSession(f.snapshot);
const { run, session } = await f.read();
expect(session?.sessionParamsJson).toBeNull();
expect(nativeTaskSessionPublicationHandled(run.runnerProfileJson, { ...f.binding, ...context })).toBe(false);
const remote = await fixture();
const remotePort = new PaperclipControlPlanePort(measured, remote.binding, { taskSessionPublication: remote.context,
taskSessionPublicationOwner: remote.owner, nativeRecovery: { owner: remote.owner,
compatibility: { configDigest: nativeSha256("config"), processConfigDigest: null } } });
await remotePort.openRun({ ...CONTROL_PLANE_CONFORMANCE_OPEN, identity: remote.snapshot.identity, sourceInstanceId: remote.binding.sourceInstanceId });
await remotePort.checkpointSession(remote.snapshot);
expect((await remote.read()).session?.sessionParamsJson).toBeNull();
expect(nativeTaskSessionPublicationHandled((await remote.read()).run.runnerProfileJson, { ...remote.binding, ...remote.context })).toBe(false);
});
it("rejects stale owners and a different accepted result without partially publishing", async () => {
const stale = await fixture();
await db.update(nativeRunFinalizations).set({ controllerGeneration: 2 }).where(eq(nativeRunFinalizations.runId, stale.binding.runId));
await expect(stale.port.checkpointSession(stale.snapshot)).rejects.toThrow("checkpoint_owner_lost");
expect((await stale.read()).session?.sessionParamsJson).toBeNull();
const wrong = await fixture();
await expect(wrong.port.checkpointSession({ ...wrong.snapshot, semanticResult: { ...CONTROL_PLANE_CONFORMANCE_RESULT, summary: "different" } })).rejects.toThrow("publication_result_mismatch");
expect((await wrong.read()).session?.sessionParamsJson).toBeNull();
const foreign = await fixture();
const foreignResult = await fixture();
const [otherCoordinator] = await db.select({ resultId: nativeRunFinalizations.resultId }).from(nativeRunFinalizations)
.where(eq(nativeRunFinalizations.runId, foreignResult.binding.runId));
await expect(db.update(nativeRunFinalizations).set({ resultId: otherCoordinator.resultId })
.where(eq(nativeRunFinalizations.runId, foreign.binding.runId)))
.rejects.toMatchObject({ cause: { constraint_name: "native_run_finalizations_result_owner_fk" } });
expect((await foreign.read()).session?.sessionParamsJson).toBeNull();
await db.update(nativeRunFinalizations).set({ resultId: null }).where(eq(nativeRunFinalizations.runId, foreign.binding.runId));
// The CP defers preacceptance snapshots, while the publication helper itself
// still requires accepted-result authority if called directly.
await expect(publishNativeTaskSessionInTransaction({} as never, { binding: foreign.binding,
context: foreign.context, snapshot: foreign.snapshot, run: (await foreign.read()).run,
issue: null, resultId: null })).rejects.toThrow("publication_result_missing");
});
it("adds no publication SQL for nonterminal or goal snapshots", async () => {
const f = await fixture(); statements.length = 0;
await f.port.checkpointSession({ ...f.snapshot, semanticResult: null, terminal: null });
expect(statements.some(query => /agent_task_sessions|native_run_results|publication_issue/i.test(query))).toBe(false);
statements.length = 0;
await f.port.checkpointSession({ ...f.snapshot, goal: {} as NonNullable<PersistedNativeSession["goal"]> });
expect(statements.some(query => /agent_task_sessions|native_run_results|publication_issue/i.test(query))).toBe(false);
expect((await f.read()).session?.sessionParamsJson).toBeNull();
});
it.each(["completed_predecessor", "expired_accepted_lease"])("preserves legitimate %s maintenance", async kind => {
const f = await fixture();
if (kind === "completed_predecessor") {
const prior = randomUUID();
await db.insert(heartbeatRuns).values({ id: prior, companyId: f.binding.companyId, agentId: f.binding.agentId,
issueId: f.binding.issueId, status: "succeeded", finishedAt: new Date(Date.now() - 5_000) });
await db.update(agentTaskSessions).set({ lastRunId: prior, sessionParamsJson: { sessionId: "prior" } })
.where(eq(agentTaskSessions.taskKey, f.binding.issueId));
} else {
await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 1_000) })
.where(eq(nativeRunFinalizations.runId, f.binding.runId));
}
await f.port.checkpointSession(f.snapshot);
expect((await f.read()).session?.lastRunId).toBe(f.binding.runId);
});
it("does not block completeRun's issue KEY SHARE while publishing", async () => {
const f = await fixture(); let release!: () => void; let locked!: () => void;
const ready = new Promise<void>(resolve => { locked = resolve; });
const hold = new Promise<void>(resolve => { release = resolve; });
const blocker = db.transaction(async tx => {
await tx.select().from(issues).where(eq(issues.id, f.binding.issueId)).for("key share"); locked(); await hold;
});
await ready;
try { await f.port.checkpointSession(f.snapshot); }
finally { release(); await blocker; }
expect((await f.read()).session?.lastRunId).toBe(f.binding.runId);
});
it("rechecks the owner clock and issue authority after an issue lock wait", async () => {
const f = await fixture(); let release!: () => void; let locked!: () => void;
await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() + 500) }).where(eq(nativeRunFinalizations.runId, f.binding.runId));
const ready = new Promise<void>(resolve => { locked = resolve; });
const hold = new Promise<void>(resolve => { release = resolve; });
const newer = randomUUID();
await db.insert(heartbeatRuns).values({ id: newer, companyId: f.binding.companyId, agentId: f.binding.agentId,
issueId: f.binding.issueId, status: "running", startedAt: new Date() });
const blocker = db.transaction(async tx => {
await tx.select().from(issues).where(eq(issues.id, f.binding.issueId)).for("no key update");
await tx.update(issues).set({ executionRunId: newer }).where(eq(issues.id, f.binding.issueId));
locked(); await hold;
});
await ready;
const publication = f.port.checkpointSession(f.snapshot);
const rejected = expect(publication).rejects.toThrow("checkpoint_owner_lost");
try {
let waiting = false;
for (let attempt = 0; attempt < 100; attempt++) {
const rows = await db.execute(sql`select 1 from pg_stat_activity where application_name = 'native-publication-observer'
and wait_event_type = 'Lock' and query like '%publication_issue%'`);
if (rows.length) { waiting = true; break; }
await new Promise(resolve => setTimeout(resolve, 10));
}
expect(waiting).toBe(true);
await new Promise(resolve => setTimeout(resolve, 600));
} finally { release(); await blocker; }
await rejected;
expect((await f.read()).session?.sessionParamsJson).toBeNull();
});
it("rolls back the pointer if combined private metadata exceeds its existing bound", async () => {
const f = await fixture();
await db.update(heartbeatRuns).set({ runnerProfileJson: { nativeControllerRecovery: { padding: "x".repeat(4_096) } } }).where(eq(heartbeatRuns.id, f.binding.runId));
await expect(f.port.checkpointSession(f.snapshot)).rejects.toThrow("publication_metadata_too_large");
const { run, session } = await f.read();
expect(session?.sessionParamsJson).toBeNull();
expect(run.runnerProfileJson).not.toHaveProperty(NATIVE_TASK_SESSION_PUBLICATION_KEY);
});
});
@@ -0,0 +1,145 @@
import { and, eq } from "drizzle-orm";
import { agentTaskSessions, heartbeatRuns, nativeRunResults, type Db, type issues } from "@paperclipai/db";
import { validatePrpStructuredRunResult, type PersistedNativeSession } from "../../vendor/paperclip-runner/index.js";
import { nativeSha256 } from "./canonical.js";
import type { PaperclipControlPlaneBinding } from "./paperclip-control-plane-port.js";
import { readRunCancellation } from "../run-cancellation.js";
/** Prepared once by heartbeat; never copied into recovery metadata or receipts. */
export interface NativeTaskSessionPublicationContext {
taskKey: string;
adapterType: string;
sessionParamsJson: Record<string, unknown> | null;
}
export const NATIVE_TASK_SESSION_PUBLICATION_KEY = "nativeTaskSessionPublication";
type Transaction = Parameters<Parameters<Db["transaction"]>[0]>[0];
type Outcome = "published" | "suppressed_reset" | "suppressed_newer" | "suppressed_authority" | "missing_stub" | "unsupported";
interface Receipt {
schema: "paperclip.native-task-session-publication.v1";
companyId: string;
issueId: string;
agentId: string;
runId: string;
taskKey: string;
adapterType: string;
sessionId: string;
resultId: string;
outcome: Outcome;
}
function record(value: unknown): Record<string, unknown> {
return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
}
const uuid = (value: unknown): value is string => typeof value === "string" && /^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/i.test(value);
const outcomes: readonly string[] = ["published", "suppressed_reset", "suppressed_newer", "suppressed_authority", "missing_stub", "unsupported"];
function receiptFromProfile(profile: unknown): Receipt | null {
const root = record(profile), value = record(root[NATIVE_TASK_SESSION_PUBLICATION_KEY]);
if (value.schema !== "paperclip.native-task-session-publication.v1"
|| ![value.companyId, value.issueId, value.agentId, value.runId, value.sessionId, value.resultId].every(uuid)
|| typeof value.taskKey !== "string" || value.taskKey.length > 512
|| typeof value.adapterType !== "string" || !value.adapterType.length || value.adapterType.length > 128
|| !outcomes.includes(String(value.outcome)) || Object.keys(value).length !== 10) return null;
const snapshot = record(root.sessionCheckpoint), identity = record(snapshot.identity);
if (identity.runId !== value.runId || identity.companyId !== value.companyId || identity.issueId !== value.issueId
|| identity.agentId !== value.agentId || identity.sessionId !== value.sessionId) return null;
return value as unknown as Receipt;
}
/** Only the exact server-written outcome may suppress heartbeat's late fallback. */
export function nativeTaskSessionPublicationHandled(profile: unknown, binding: {
companyId: string; agentId: string; runId: string; taskKey: string; adapterType: string;
}): boolean {
const receipt = receiptFromProfile(profile);
return !!receipt && receipt.companyId === binding.companyId && receipt.agentId === binding.agentId
&& receipt.runId === binding.runId && receipt.taskKey === binding.taskKey
&& receipt.adapterType === binding.adapterType && ["published", "suppressed_reset", "suppressed_newer", "suppressed_authority", "missing_stub"].includes(receipt.outcome);
}
export function eligibleNativeTaskSessionPublication(snapshot: PersistedNativeSession): boolean {
return snapshot.goal == null && snapshot.semanticResult != null && snapshot.terminal?.runTerminalState === "succeeded"
&& snapshot.activeTurnId == null && !(snapshot.pendingRuntimeRequests?.length);
}
export type NativeTaskSessionPublicationRun = Pick<typeof heartbeatRuns.$inferSelect,
"id" | "companyId" | "agentId" | "nativeIssueId" | "nativeSessionId" | "status" | "resultJson" | "contextSnapshot" | "startedAt" | "createdAt" | "runnerProfileJson">;
export type NativeTaskSessionPublicationIssue = Pick<typeof issues.$inferSelect,
"id" | "companyId" | "executionRunId" | "conversationAgentId" | "conversationSessionGeneration">;
/** Caller holds coordinator → issue NO KEY UPDATE → run. This helper owns only
* the task-row lock and UPDATE; never starts a transaction or creates a stub. */
export async function publishNativeTaskSessionInTransaction(tx: Transaction, input: {
binding: PaperclipControlPlaneBinding;
context: NativeTaskSessionPublicationContext;
snapshot: PersistedNativeSession;
run: NativeTaskSessionPublicationRun;
issue: NativeTaskSessionPublicationIssue | null;
resultId: string | null;
recoveryReady?: boolean;
}): Promise<Receipt> {
const { binding, context, snapshot, run, issue } = input;
const existingReceipt = receiptFromProfile(run.runnerProfileJson);
if (existingReceipt) {
if (existingReceipt.companyId !== binding.companyId || existingReceipt.issueId !== binding.issueId
|| existingReceipt.agentId !== binding.agentId || existingReceipt.runId !== binding.runId
|| existingReceipt.sessionId !== snapshot.identity.sessionId || existingReceipt.taskKey !== context.taskKey
|| existingReceipt.adapterType !== context.adapterType || existingReceipt.resultId !== input.resultId) {
throw new Error("native_task_session_publication_receipt_mismatch");
}
return existingReceipt;
}
if (record(run.runnerProfileJson)[NATIVE_TASK_SESSION_PUBLICATION_KEY] !== undefined) {
throw new Error("native_task_session_publication_receipt_invalid");
}
if (!input.resultId) throw new Error("native_task_session_publication_result_missing");
const [accepted] = await tx.select({ id: nativeRunResults.id, canonicalSha256: nativeRunResults.canonicalSha256,
turnId: nativeRunResults.turnId }).from(nativeRunResults).where(and(
eq(nativeRunResults.id, input.resultId), eq(nativeRunResults.runId, binding.runId),
eq(nativeRunResults.companyId, binding.companyId), eq(nativeRunResults.issueId, binding.issueId),
eq(nativeRunResults.completionContractId, binding.completionContractId), eq(nativeRunResults.schemaStatus, "accepted"),
)).limit(1);
const validated = validatePrpStructuredRunResult(snapshot.semanticResult);
if (!accepted || !validated.ok || accepted.canonicalSha256 !== nativeSha256({
binding, result: validated.result, terminal: snapshot.terminal, turnId: accepted.turnId ?? null,
})) throw new Error("native_task_session_publication_result_mismatch");
const receipt = (outcome: Outcome): Receipt => ({ schema: "paperclip.native-task-session-publication.v1",
companyId: binding.companyId, issueId: binding.issueId, agentId: binding.agentId, runId: binding.runId,
taskKey: context.taskKey, adapterType: context.adapterType, sessionId: snapshot.identity.sessionId,
resultId: accepted.id, outcome });
if (context.taskKey !== binding.issueId || input.recoveryReady === false) return receipt("unsupported");
const cancellation = record(run.resultJson?.nativeCancellation), stop = readRunCancellation(run.resultJson);
if (!issue || ["cancelled", "interrupted", "failed", "timed_out"].includes(run.status) || issue.companyId !== binding.companyId
|| run.resultJson?.startupCancellation || (stop?.source === "operator" && stop.expected)
|| (["run", "issue"].includes(String(cancellation.scope)) && ["pending", "acknowledged"].includes(String(cancellation.dispatchState)))
|| (issue.conversationAgentId && issue.conversationAgentId !== binding.agentId)) return receipt("suppressed_authority");
if (issue.conversationAgentId
&& run.contextSnapshot?.conversationSessionGeneration !== issue.conversationSessionGeneration) return receipt("suppressed_reset");
if (issue.executionRunId && issue.executionRunId !== binding.runId) return receipt("suppressed_newer");
const [session] = await tx.select({ id: agentTaskSessions.id, goalSourceId: agentTaskSessions.goalSourceId,
lastRunId: agentTaskSessions.lastRunId }).from(agentTaskSessions).where(and(
eq(agentTaskSessions.companyId, binding.companyId), eq(agentTaskSessions.agentId, binding.agentId),
eq(agentTaskSessions.adapterType, context.adapterType), eq(agentTaskSessions.taskKey, context.taskKey),
)).for("update").limit(1);
// Supported native turns synchronously project a stub before this terminal
// snapshot. Absence can be an operator deletion; never recreate it later.
if (!session) return receipt("missing_stub");
const boundSources = [binding.sourceInstanceId, binding.controlPlaneSourceInstanceId]
.filter((value): value is string => typeof value === "string" && value.length > 0)
.map(source => `${source}:${binding.runId}`);
if (!session.goalSourceId || !boundSources.includes(session.goalSourceId)) return receipt("suppressed_newer");
if (session.lastRunId && session.lastRunId !== binding.runId) {
const [prior] = await tx.select({ companyId: heartbeatRuns.companyId, agentId: heartbeatRuns.agentId,
issueId: heartbeatRuns.issueId, nativeIssueId: heartbeatRuns.nativeIssueId, finishedAt: heartbeatRuns.finishedAt,
}).from(heartbeatRuns).where(eq(heartbeatRuns.id, session.lastRunId)).limit(1);
if (!prior || prior.companyId !== binding.companyId || prior.agentId !== binding.agentId
|| (prior.nativeIssueId ?? prior.issueId) !== binding.issueId || !prior.finishedAt || !run.startedAt
|| prior.finishedAt > run.startedAt) return receipt("suppressed_newer");
}
await tx.update(agentTaskSessions).set({
sessionParamsJson: { ...context.sessionParamsJson, sessionId: snapshot.identity.sessionId },
sessionDisplayId: snapshot.providerSessionId ?? snapshot.sessionId, lastRunId: binding.runId,
lastError: null, updatedAt: new Date(),
}).where(eq(agentTaskSessions.id, session.id));
return receipt("published");
}
@@ -12,8 +12,8 @@ import { agents, budgetPolicies, companies, costEvents, createDb, heartbeatRuns,
import { type PrpEvent } from "../../vendor/paperclip-runner/index.js";
import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
import { accountRunCost } from "../run-cost-accounting.js";
import { usageReceiptSpoolPath } from "../usage-receipts.js";
import { createNativeUsageDelivery } from "./native-usage-delivery.js";
import { usageReceiptSpoolPath, type UsageReceiptEnvelope } from "../usage-receipts.js";
import { createNativeUsageContinuationDelivery, createNativeUsageDelivery } from "./native-usage-delivery.js";
const runnerRoot = fileURLToPath(new URL("../../../../packages/paperclip-runner/runner/", import.meta.url));
const fixtureProfile = process.env.CI ? "release" : "debug";
@@ -44,6 +44,89 @@ describe("producer-owned native usage", () => {
payload: type === "usage.reported" ? { runDeltaAvailable: true, runDeltaBasis: "run", runDelta: { inputTokens: 120, outputTokens: 10, providerCostUsd: amount } } : {},
} as PrpEvent);
const finalizedSource = async () => {
const input = await seed();
await db.update(heartbeatRuns).set({ runnerInstanceId: input.runnerInstanceId }).where(eq(heartbeatRuns.id, input.runId));
const delivery = await createNativeUsageDelivery(db, input);
const report = event(input, 1, "usage.reported"), done = event(input, 2, "turn.completed");
await delivery.capture(report);
await delivery.capture(done);
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, input.runId));
expect(await accountRunCost(db, input.runId)).toBe(true);
const { billing: _billing, ...source } = input;
return { input, source, report, done };
};
it("continues exact finalized source receipts without acquiring ownership or resetting accounting", async () => {
const { input, source, report, done } = await finalizedSource();
const before = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId));
await expect(createNativeUsageDelivery(db, input)).rejects.toThrow("native_usage_run_not_available");
const continuation = await createNativeUsageContinuationDelivery(db, source);
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId))).toEqual(before);
await continuation.capture(report);
await continuation.capture(done);
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId))).toEqual(before);
expect(await db.select().from(runUsageReceipts).where(eq(runUsageReceipts.runId, input.runId))).toHaveLength(2);
expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId))).toHaveLength(1);
await expect(continuation.capture({ ...report, sourceInstanceId: randomUUID() })).rejects.toThrow("native_usage_source_mismatch");
await expect(continuation.capture({ ...report, payload: { ...report.payload as object, runDeltaAvailable: false } }))
.rejects.toThrow("native_usage_source_conflict");
});
it("journals an equal finalized source tail using the source billing without repricing", async () => {
const { input, source } = await finalizedSource();
const before = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId));
// Even a structurally supplied extra current-run classification cannot own
// the old receipt: the continuation derives billing from saved source data.
const withCurrentBilling = { ...source, billing: { provider: "anthropic", biller: "current-grant", billingType: "subscription_included" } };
const continuation = await createNativeUsageContinuationDelivery(db, withCurrentBilling);
await continuation.capture(event(input, 3, "usage.reported"));
const receipts = await db.select().from(runUsageReceipts).where(eq(runUsageReceipts.runId, input.runId));
expect(receipts).toHaveLength(3);
expect(receipts.find(row => row.sequence === 3)?.receiptJson.receipt).toMatchObject(billing);
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId))).toEqual(before);
expect((await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId)))[0].costCents).toBe(25);
});
it.each(["tokens", "cost", "incomplete", "unpriceable", "basis"])(
"holds a differing finalized source tail before acknowledgement: %s", async change => {
const { input, source } = await finalizedSource();
const continuation = await createNativeUsageContinuationDelivery(db, source);
const late = event(input, 3, "usage.reported");
if (change === "incomplete") (late.payload as Record<string, unknown>).runDeltaAvailable = false;
if (change === "basis") delete (late.payload as Record<string, unknown>).runDeltaBasis;
if (["tokens", "cost", "unpriceable"].includes(change)) (late.payload as Record<string, unknown>).runDelta = {
inputTokens: change === "tokens" ? 150 : change === "unpriceable" ? "unknown" : 120,
outputTokens: 10, providerCostUsd: change === "cost" ? 0.5 : 0.25,
};
await expect(continuation.capture(late)).rejects.toThrow("native_usage_after_finalization_requires_reconciliation");
await expect(continuation.capture(late)).rejects.toThrow("native_usage_after_finalization_requires_reconciliation");
expect(await db.select().from(runUsageReceipts).where(eq(runUsageReceipts.runId, input.runId))).toHaveLength(2);
expect((await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId)))[0].costCents).toBe(25);
});
it.each(["company", "run", "runner", "source"])("rejects a changed continuation binding without mutation: %s", async change => {
const { input, source } = await finalizedSource();
if (change === "source") await db.update(heartbeatRuns).set({ usageJson: sql`${heartbeatRuns.usageJson} || ${JSON.stringify({ accountingReceiptSourceId: randomUUID() })}::jsonb` })
.where(eq(heartbeatRuns.id, input.runId));
const before = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId));
await expect(createNativeUsageContinuationDelivery(db, { ...source,
...(change === "company" ? { companyId: randomUUID() } : {}),
...(change === "run" ? { runId: randomUUID() } : {}),
...(change === "runner" ? { runnerInstanceId: randomUUID() } : {}),
})).rejects.toThrow("native_usage_continuation_source_mismatch");
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId))).toEqual(before);
});
it("rejects changed immutable source receipt facts during hydration", async () => {
const { input, source } = await finalizedSource();
const [receipt] = await db.select().from(runUsageReceipts).where(and(eq(runUsageReceipts.runId, input.runId), eq(runUsageReceipts.sequence, 1)));
const changed = structuredClone(receipt.receiptJson) as UsageReceiptEnvelope;
changed.producerEvent!.usage!.runDelta = { inputTokens: 999, outputTokens: 10 };
await db.update(runUsageReceipts).set({ receiptJson: changed }).where(eq(runUsageReceipts.id, receipt.id));
await expect(createNativeUsageContinuationDelivery(db, source)).rejects.toThrow("native_usage_source_conflict");
});
it("retains the same Runner report through a failed Postgres receipt and an empty replacement receiver", async () => {
const input = await seed();
expect(existsSync(usageReceiptSpoolPath())).toBe(false);
@@ -16,11 +16,13 @@ const uuid = (identity: string) => {
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-5${hex.slice(13, 16)}-a${hex.slice(17, 20)}-${hex.slice(20, 32)}`;
};
export async function createNativeUsageDelivery(db: Db, input: {
type NativeUsageDeliveryInput = {
companyId: string; runId: string; runnerInstanceId: string;
provider: NativeExecutionInput["provider"];
billing: Pick<AdapterUsageCheckpoint, "provider" | "biller" | "billingType">;
}) {
};
export async function createNativeUsageDelivery(db: Db, input: NativeUsageDeliveryInput) {
const sourceId = uuid(`${input.runId}:${input.runnerInstanceId}`);
const [bound] = await db.update(heartbeatRuns).set({ usageJson: sql`
coalesce(${heartbeatRuns.usageJson}, '{}'::jsonb) || '{"accountingReceiptOwner":"native_producer"}'::jsonb ||
@@ -29,9 +31,39 @@ export async function createNativeUsageDelivery(db: Db, input: {
}).where(and(eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.runId),
isNull(heartbeatRuns.costAccountedAt))).returning({ id: heartbeatRuns.id });
if (!bound) throw new Error("native_usage_run_not_available");
const saved = await db.select({ receipt: runUsageReceipts.receiptJson }).from(runUsageReceipts).where(and(
return createNativeUsageReceiver(db, input, sourceId, await loadNativeUsageReceipts(db, input, sourceId));
}
/** Drain the authenticated previous producer's accounting without acquiring or
* resetting its ownership. Finalized sources retain normal receipt equality
* and reconciliation rules; this grants no business/tool execution authority. */
export async function createNativeUsageContinuationDelivery(db: Db, input: Omit<NativeUsageDeliveryInput, "billing">) {
const sourceId = uuid(`${input.runId}:${input.runnerInstanceId}`);
const [source] = await db.select({ runnerInstanceId: heartbeatRuns.runnerInstanceId, usage: heartbeatRuns.usageJson })
.from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.runId)));
if (source?.runnerInstanceId !== input.runnerInstanceId || source.usage?.accountingReceiptOwner !== "native_producer"
|| source.usage.accountingReceiptSourceId !== sourceId) throw new Error("native_usage_continuation_source_mismatch");
const saved = await loadNativeUsageReceipts(db, input, sourceId);
if (!saved.length) throw new Error("native_usage_continuation_receipts_missing");
const classification = (receipt: AdapterUsageCheckpoint) => ({
provider: receipt.provider, biller: receipt.biller, billingType: receipt.billingType,
});
const receipts = saved.map(row => (row.receipt as UsageReceiptEnvelope).receipt as AdapterUsageCheckpoint);
const billing = classification(receipts[0]);
if (receipts.some(receipt => receiptFingerprint(classification(receipt)) !== receiptFingerprint(billing)
|| receipt.model !== (input.provider.model ?? "unknown"))) throw new Error("native_usage_source_conflict");
return createNativeUsageReceiver(db, { ...input, billing }, sourceId, saved, true);
}
function loadNativeUsageReceipts(db: Db, input: Pick<NativeUsageDeliveryInput, "companyId" | "runId">, sourceId: string) {
return db.select({ receipt: runUsageReceipts.receiptJson, hash: runUsageReceipts.receiptHash,
sequence: runUsageReceipts.sequence }).from(runUsageReceipts).where(and(
eq(runUsageReceipts.companyId, input.companyId), eq(runUsageReceipts.runId, input.runId),
eq(runUsageReceipts.sourceId, sourceId))).orderBy(asc(runUsageReceipts.sequence));
}
function createNativeUsageReceiver(db: Db, input: NativeUsageDeliveryInput, sourceId: string,
saved: Awaited<ReturnType<typeof loadNativeUsageReceipts>>, verifySaved = false) {
const accepted = new Map<string, UsageReceiptEnvelope>();
const turnAccounting = ["opencode", "acpx"].includes(input.provider.kind) ? createNativeTurnAccounting(input.provider) : null;
let observedUsage: Record<string, unknown> | null = null;
@@ -60,6 +92,12 @@ export async function createNativeUsageDelivery(db: Db, input: {
for (const row of saved) {
const envelope = row.receipt as UsageReceiptEnvelope;
if (!envelope.producerEvent) throw new Error("native_usage_receipt_missing_producer");
if (verifySaved && (envelope.companyId !== input.companyId || envelope.runId !== input.runId
|| envelope.sourceId !== sourceId || envelope.sequence !== row.sequence || envelope.adapterType !== "paperclip_runner"
|| envelope.id !== uuid(`${input.runId}:${input.runnerInstanceId}:${envelope.producerEvent.sourceEventId}`)
|| receiptFingerprint(envelope) !== row.hash || accepted.has(envelope.producerEvent.sourceEventId))) {
throw new Error("native_usage_source_conflict");
}
accepted.set(envelope.producerEvent.sourceEventId, envelope);
observe(envelope);
}
@@ -48,6 +48,7 @@ import {
import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
import { startRecoveryPostgres } from "../../__tests__/helpers/recovery-postgres.js";
import { PaperclipControlPlanePort } from "./paperclip-control-plane-port.js";
import { nativeSha256 } from "./canonical.js";
import { NativeRunCoordinatorStore } from "./native-run-coordinator-store.js";
import { finalizeNativeRun, repairCommittedNativeChatResponse } from "./native-run-finalizer.js";
import { nativeRuntimeContextFixture } from "./runtime-context.test-fixture.js";
@@ -313,6 +314,63 @@ describe("PaperclipControlPlanePort conformance", () => {
}
});
it("publishes native recovery metadata in the existing snapshot transaction and fences stale generations", async () => {
const identity = { ...CONTROL_PLANE_CONFORMANCE_OPEN.identity, runId: randomUUID(), sessionId: randomUUID() };
const runnerId = randomUUID();
const owner = { ...identity, authorityRunId: identity.runId, bootId: randomUUID(), generation: 1, leaseOwner: "snapshot-owner" };
await db.insert(heartbeatRuns).values({ id: identity.runId, companyId: identity.companyId, agentId: identity.agentId,
status: "running", runtimeMode: "native", nativeIssueId: identity.issueId, nativeSessionId: identity.sessionId,
runnerInstanceId: runnerId, completionContractId: contractId, completionContractSha256: contractSha,
runnerProfileJson: { unrelated: "preserve" } });
await db.insert(nativeRunFinalizations).values({ runId: identity.runId, companyId: identity.companyId, issueId: identity.issueId,
phase: "observed",
controllerBootId: owner.bootId, controllerGeneration: owner.generation, leaseOwner: owner.leaseOwner,
leaseExpiresAt: new Date(Date.now() + 60_000) });
const port = new PaperclipControlPlanePort(db, { ...identity, completionContractId: contractId,
completionContractSha256: contractSha, sourceInstanceId: runnerId, controlPlaneSourceInstanceId: "snapshot-control" }, {
nativeRecovery: { owner, compatibility: { configDigest: `sha256:${"a".repeat(64)}`, processConfigDigest: null } },
});
await port.openRun({ identity, backendKind: "mock", sourceInstanceId: runnerId });
const snapshot = { backendKind: "mock" as const, sessionId: identity.sessionId, identity };
const statements: string[] = [];
const previousDebug = db.$client.options.debug;
db.$client.options.debug = (_connection, statement) => { statements.push(statement); };
try { await port.checkpointSession(snapshot); }
finally { db.$client.options.debug = previousDebug; }
expect(statements.filter(statement => /^begin\b/i.test(statement.trim()))).toHaveLength(1);
expect(statements.filter(statement => /^update\s+"heartbeat_runs"/i.test(statement.trim()))).toHaveLength(1);
expect(statements.filter(statement => /^(?:with|select)\b/i.test(statement.trim()))).toHaveLength(1);
const [saved] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, identity.runId));
expect(saved.runnerProfileJson).toMatchObject({ unrelated: "preserve", sessionCheckpoint: snapshot,
nativeControllerRecovery: { schema: "paperclip.native-controller-recovery.v1", snapshotSha256: nativeSha256(saved.runnerProfileJson?.sessionCheckpoint),
checkpoint: null, processConfigDigest: null } });
// Missing PRP is an explicit incomplete publication, never fabricated authority.
await db.update(nativeRunFinalizations).set({ controllerGeneration: 2 }).where(eq(nativeRunFinalizations.runId, identity.runId));
await expect(port.checkpointSession({ ...snapshot, providerSessionId: "stale" })).rejects.toThrow("checkpoint_owner_lost");
await expect(port.completeRun({ result: CONTROL_PLANE_CONFORMANCE_RESULT, terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL }))
.rejects.toThrow("checkpoint_owner_lost");
const [unchanged] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, identity.runId));
expect(unchanged.runnerProfileJson).toEqual(saved.runnerProfileJson);
// A valid coordinator lock does not certify a lease that expires while the
// subsequent run-row lock waits. Recheck time at the actual write boundary.
await db.update(nativeRunFinalizations).set({ controllerGeneration: 1, leaseExpiresAt: new Date(Date.now() + 150) })
.where(eq(nativeRunFinalizations.runId, identity.runId));
let locked!: () => void, release!: () => void;
const acquired = new Promise<void>(resolve => { locked = resolve; });
const held = new Promise<void>(resolve => { release = resolve; });
const blocker = db.transaction(async tx => {
await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(eq(heartbeatRuns.id, identity.runId)).for("update");
locked();
await held;
});
await acquired;
const waiting = port.checkpointSession(snapshot);
const rejected = expect(waiting).rejects.toThrow("checkpoint_owner_lost");
try { await new Promise(resolve => setTimeout(resolve, 200)); }
finally { release(); await blocker; }
await rejected;
});
it("permits only read-only admission while an existing startup drains", async () => {
const identity = { ...CONTROL_PLANE_CONFORMANCE_OPEN.identity, runId: randomUUID(), sessionId: randomUUID() };
const runnerId = randomUUID();
@@ -1,7 +1,7 @@
import { notifyNativeResultCommit } from "../native-result-commit-notifications.js";
import { notifyNativeWorkspaceFinalization } from "../native-workspace-finalization-notifications.js";
import { createAgentIdentityRedactor } from "../agent-identity-redaction.js";
import { and, asc, eq, gt, or } from "drizzle-orm";
import { and, asc, eq, gt, or, sql } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import {
heartbeatRunEvents,
@@ -9,6 +9,7 @@ import {
issues,
nativeRunFinalizations,
nativeRunResults,
nativeRunnerCheckpoints,
} from "@paperclipai/db";
import type {
CompleteControlPlaneRunInput,
@@ -30,6 +31,10 @@ import { appendHeartbeatRunEvent } from "../heartbeat-run-events.js";
import { publishLiveEvent } from "../live-events.js";
import { nativeSha256 } from "./canonical.js";
import { redactRunnerApiEvent, redactRunnerApiPayloads } from "./runner-api-log-redaction.js";
import { createNativeControllerRecovery, NATIVE_CONTROLLER_RECOVERY_MAX_BYTES, type NativeControllerCompatibility } from "./native-controller-recovery.js";
import { eligibleNativeTaskSessionPublication, publishNativeTaskSessionInTransaction, NATIVE_TASK_SESSION_PUBLICATION_KEY,
type NativeTaskSessionPublicationContext, type NativeTaskSessionPublicationIssue } from "./native-task-session-publication.js";
import { lockNativeRunnerCheckpointOwner, nativeRunnerCheckpointOwnerLock, validateNativeRunnerCheckpointOwner, inspectNativeRunnerUpgradeHeader, type NativeRunnerCheckpointOwner } from "./native-runner-checkpoint.js";
export interface PaperclipControlPlaneBinding {
companyId: string;
@@ -90,6 +95,9 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
readonly #onEventPersistence?: (event: PrpEvent, startedAtMs: number, endedAtMs: number) => Promise<void>;
readonly #onCommittedEvent?: (event: PrpEvent, cursor: number) => Promise<void>;
readonly #onDuplicateEvent?: (event: PrpEvent, cursor: number) => Promise<void>;
readonly #nativeRecovery?: { compatibility: NativeControllerCompatibility; owner: NativeRunnerCheckpointOwner };
readonly #taskSessionPublication?: NativeTaskSessionPublicationContext;
readonly #taskSessionPublicationOwner?: NativeRunnerCheckpointOwner;
constructor(
db: Db,
@@ -105,6 +113,9 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
onEventPersistence?: (event: PrpEvent, startedAtMs: number, endedAtMs: number) => Promise<void>;
onCommittedEvent?: (event: PrpEvent, cursor: number) => Promise<void>;
onDuplicateEvent?: (event: PrpEvent, cursor: number) => Promise<void>;
nativeRecovery?: { compatibility: NativeControllerCompatibility; owner: NativeRunnerCheckpointOwner };
taskSessionPublication?: NativeTaskSessionPublicationContext;
taskSessionPublicationOwner?: NativeRunnerCheckpointOwner;
} = {},
) {
this.#db = db;
@@ -116,6 +127,9 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
this.#onEventPersistence = options.onEventPersistence;
this.#onResultCommitted = options.onResultCommitted;
this.#onDuplicateEvent = options.onDuplicateEvent;
this.#nativeRecovery = options.nativeRecovery;
this.#taskSessionPublication = options.taskSessionPublication ? structuredClone(options.taskSessionPublication) : undefined;
this.#taskSessionPublicationOwner = options.taskSessionPublicationOwner;
}
#assertActive(options?: { signal: AbortSignal }): void {
@@ -123,7 +137,9 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
options?.signal.throwIfAborted();
}
#matchesPersistedBinding(run: typeof heartbeatRuns.$inferSelect): boolean {
#matchesPersistedBinding(run: Pick<typeof heartbeatRuns.$inferSelect,
"companyId" | "agentId" | "runtimeMode" | "nativeIssueId" | "nativeSessionId" |
"runnerInstanceId" | "completionContractId" | "completionContractSha256">): boolean {
return run.companyId === this.#binding.companyId
&& run.agentId === this.#binding.agentId
&& run.runtimeMode === "native"
@@ -200,19 +216,105 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
|| identity.agentId !== this.#binding.agentId
|| this.#sessionId !== identity.sessionId
) throw new Error("native_session_checkpoint_binding_mismatch");
const publication = this.#taskSessionPublication && eligibleNativeTaskSessionPublication(snapshot)
? this.#taskSessionPublication : null;
if (publication && !this.#taskSessionPublicationOwner) throw new Error("native_task_session_publication_owner_required");
await this.#db.transaction(async (tx) => {
const run = await tx.select().from(heartbeatRuns)
.where(eq(heartbeatRuns.id, this.#binding.runId)).for("update").limit(1)
const recovery = this.#nativeRecovery;
const owner = recovery?.owner ?? (publication ? this.#taskSessionPublicationOwner : undefined);
if (publication && recovery && nativeSha256(recovery.owner) !== nativeSha256(this.#taskSessionPublicationOwner)) {
throw new Error("native_task_session_publication_owner_mismatch");
}
const ownerLock = owner ? nativeRunnerCheckpointOwnerLock(tx, owner) : null;
// NO KEY UPDATE serializes reset/admission fields while remaining compatible
// with completeRun's parent KEY SHARE. Keep coordinator → issue → run order.
const publicationIssue = publication ? tx.$with("publication_issue").as(tx.select({
id: issues.id, companyId: issues.companyId, executionRunId: issues.executionRunId,
conversationAgentId: issues.conversationAgentId, conversationSessionGeneration: issues.conversationSessionGeneration,
}).from(issues).where(and(eq(issues.id, this.#binding.issueId), eq(issues.companyId, this.#binding.companyId),
ownerLock ? sql`exists (select 1 from ${ownerLock})` : undefined)).for("no key update")) : null;
const lockedRun = tx.$with("snapshot_run").as(tx.select({
id: heartbeatRuns.id, status: heartbeatRuns.status,
contextSnapshot: (publication ? sql<Record<string, unknown>>`jsonb_build_object('conversationSessionGeneration', ${heartbeatRuns.contextSnapshot}->'conversationSessionGeneration')` : sql<null>`null`).as("publication_context"),
resultJson: (publication ? sql<Record<string, unknown>>`jsonb_build_object('nativeCancellation', ${heartbeatRuns.resultJson}->'nativeCancellation',
'cancellation', ${heartbeatRuns.resultJson}->'cancellation', 'startupCancellation', ${heartbeatRuns.resultJson}->'startupCancellation')` : sql<null>`null`).as("publication_cancellation"),
startedAt: heartbeatRuns.startedAt, createdAt: heartbeatRuns.createdAt,
companyId: heartbeatRuns.companyId, agentId: heartbeatRuns.agentId,
runtimeMode: heartbeatRuns.runtimeMode, nativeIssueId: heartbeatRuns.nativeIssueId,
nativeSessionId: heartbeatRuns.nativeSessionId, runnerInstanceId: heartbeatRuns.runnerInstanceId,
completionContractId: heartbeatRuns.completionContractId,
completionContractSha256: heartbeatRuns.completionContractSha256,
runnerProfileJson: heartbeatRuns.runnerProfileJson,
}).from(heartbeatRuns)
.where(and(eq(heartbeatRuns.id, this.#binding.runId), ownerLock
? sql`exists (select 1 from ${ownerLock})` : undefined,
publicationIssue ? sql`exists (select 1 from ${publicationIssue})` : undefined)).for("update"));
// Project the locking CTE's tuples, including any update committed while
// the lock waited. A second scan of heartbeat_runs could see an old MVCC tuple.
// The run CTE depends on the coordinator CTE: both locks precede the
// outer clock sample, including when either lock waits.
const selected = await tx.with(...(ownerLock ? [ownerLock] : []), ...(publicationIssue ? [publicationIssue] : []), lockedRun).select({ run: {
id: lockedRun.id, status: lockedRun.status, contextSnapshot: lockedRun.contextSnapshot, resultJson: lockedRun.resultJson,
startedAt: lockedRun.startedAt, createdAt: lockedRun.createdAt,
companyId: lockedRun.companyId, agentId: lockedRun.agentId,
runtimeMode: lockedRun.runtimeMode, nativeIssueId: lockedRun.nativeIssueId,
nativeSessionId: lockedRun.nativeSessionId, runnerInstanceId: lockedRun.runnerInstanceId,
completionContractId: lockedRun.completionContractId,
completionContractSha256: lockedRun.completionContractSha256,
runnerProfileJson: lockedRun.runnerProfileJson,
}, now: sql<Date>`clock_timestamp()`, ...(ownerLock ? {
leaseOwner: sql<string | null>`(select ${ownerLock.leaseOwner} from ${ownerLock})`,
leaseExpiresAt: sql<Date | null>`(select ${ownerLock.leaseExpiresAt} from ${ownerLock})`.mapWith(nativeRunFinalizations.leaseExpiresAt),
resultId: sql<string | null>`(select ${ownerLock.resultId} from ${ownerLock})`,
} : {}), ...(recovery ? {
checkpointHeader: sql<Record<string, unknown> | null>`(select material from ${nativeRunnerCheckpoints}
where ${nativeRunnerCheckpoints.runId} = ${this.#binding.runId}
and ${nativeRunnerCheckpoints.companyId} = ${this.#binding.companyId}
and ${nativeRunnerCheckpoints.issueId} = ${this.#binding.issueId}
and ${nativeRunnerCheckpoints.part} = 'header')`,
} : {}), ...(publicationIssue ? { publicationIssue: sql<NativeTaskSessionPublicationIssue>`(select jsonb_build_object(
'id', ${publicationIssue.id}, 'companyId', ${publicationIssue.companyId}, 'executionRunId', ${publicationIssue.executionRunId},
'conversationAgentId', ${publicationIssue.conversationAgentId}, 'conversationSessionGeneration', ${publicationIssue.conversationSessionGeneration}) from ${publicationIssue})` } : {}),
}).from(lockedRun).limit(1)
.then((rows) => rows[0] ?? null);
const run = selected?.run;
this.#assertActive(options);
if (owner && !selected) throw new Error("native_runner_checkpoint_owner_lost");
if (!run || !this.#matchesPersistedBinding(run)) {
throw new Error("native_session_checkpoint_binding_mismatch");
}
await tx.update(heartbeatRuns).set({
runnerProfileJson: {
// The run row lock may have waited beyond our lease. Generation cannot
// change while we hold the coordinator, but wall-clock expiry still can.
if (owner && selected) await validateNativeRunnerCheckpointOwner(tx, owner,
{ leaseOwner: selected.leaseOwner ?? null, leaseExpiresAt: selected.leaseExpiresAt ?? null,
resultId: selected.resultId ?? null, now: selected.now }, !snapshot.semanticResult);
const header = recovery && selected?.checkpointHeader
? inspectNativeRunnerUpgradeHeader(this.#binding, selected.checkpointHeader) : null;
if (header && (header.normalizedSessionId !== identity.sessionId || header.runnerInstanceId !== this.#binding.sourceInstanceId)) {
throw new Error("native_session_checkpoint_prp_binding_mismatch");
}
const profile: Record<string, unknown> = {
...record(run.runnerProfileJson),
sessionCheckpoint: structuredClone(snapshot) as unknown as Record<string, unknown>,
},
...(recovery ? { nativeControllerRecovery: createNativeControllerRecovery(snapshot, recovery.compatibility,
header ? { runId: this.#binding.runId, runnerInstanceId: header.runnerInstanceId,
normalizedSessionId: header.normalizedSessionId } : null) } : {}),
};
// Driver/live snapshots may already describe a terminal result before
// completeRun accepts it. Persist that observation without a handled
// receipt; the existing post-acceptance checkpoint publishes the pointer.
if (publication && selected?.resultId) {
const receipt = await publishNativeTaskSessionInTransaction(tx, { binding: this.#binding, context: publication,
snapshot, run, issue: selected.publicationIssue ?? null, resultId: selected.resultId ?? null,
recoveryReady: !recovery || header !== null });
profile[NATIVE_TASK_SESSION_PUBLICATION_KEY] = receipt;
if (Buffer.byteLength(JSON.stringify({ nativeControllerRecovery: profile.nativeControllerRecovery,
[NATIVE_TASK_SESSION_PUBLICATION_KEY]: receipt })) > NATIVE_CONTROLLER_RECOVERY_MAX_BYTES) {
throw new Error("native_task_session_publication_metadata_too_large");
}
}
await tx.update(heartbeatRuns).set({
runnerProfileJson: profile,
sessionIdAfter: snapshot.providerSessionId ?? snapshot.sessionId,
nativePhase: snapshot.semanticResult ? "workspace_finalizing" : "observed",
nativePhaseUpdatedAt: new Date(),
@@ -386,9 +488,12 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
});
const insertedResult = await this.#db.transaction(async (tx) => {
// Result insertion checks the task foreign key. Acquire that parent lock
// before the run, matching task mutations that subsequently update a run.
// Otherwise concurrent chat/status writes can deadlock after generation.
// Status commit and snapshot publication lock coordinator → issue → run.
// A replay must follow that order too: holding the issue while waiting
// for the coordinator deadlocks a status commit's issue UPDATE lock.
const recovery = this.#nativeRecovery;
const owner = recovery ? await lockNativeRunnerCheckpointOwner(tx, recovery.owner) : null;
// Result insertion checks the task FK. Keep its parent lock before run.
await tx.select({ id: issues.id }).from(issues)
.where(and(eq(issues.id, this.#binding.issueId), eq(issues.companyId, this.#binding.companyId)))
.for("key share");
@@ -399,6 +504,10 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
if (!run || !this.#matchesPersistedBinding(run)) {
throw new Error("native_result_binding_mismatch");
}
if (recovery && owner) {
const [clock] = await tx.execute<{ now: Date }>(sql`select clock_timestamp() as now`);
await validateNativeRunnerCheckpointOwner(tx, recovery.owner, { ...owner, now: clock.now }, false);
}
const callerConditions = [eq(nativeRunResults.serverFingerprint, serverFingerprint)];
if (value.callerResultId) callerConditions.push(eq(nativeRunResults.callerResultId, value.callerResultId));
@@ -411,6 +520,7 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
if (existing.canonicalSha256 !== canonicalSha256) throw new Error("structured_result_replay_conflict");
return false;
}
if (owner?.resultId) throw new Error("structured_result_replay_conflict");
const resultJson = {
result: validated.result,
terminal: value.terminal,
@@ -120,7 +120,7 @@ describe("PaperclipRunnerToolAuthority", () => {
issueId,
runId,
});
expect(authority.definitions()).toHaveLength(50);
expect(authority.definitions()).toHaveLength(51);
expect(authority.definitions().map(tool => tool.name)).toEqual(expect.arrayContaining([
"get_workspace", "list_workspaces", "select_workspace", "prepare_repository",
]));
@@ -28,6 +28,7 @@ import { createReadStream } from "node:fs";
import { publicChatTaskUrl } from "../chat-task-url.js";
import { agentCommentaryToolInputSchema } from "@paperclipai/shared";
import { submitAgentCommentary } from "../agent-commentary.js";
import { sendAgentChatMessage } from "../agent-chat-messages.js";
import type { createAssignedMcpTools } from "./assigned-mcp-tools.js";
import { captureRunnerToolResult, fitRunnerToolResultPage, READ_TOOL_RESULT, READ_TOOL_RESULT_DEFINITION, readToolResultSchema, RUNNER_TOOL_RESULT_MAX_BYTES } from "./runner-tool-results.js";
import { assertAssignableAgent } from "../agent-assignability.js";
@@ -126,6 +127,7 @@ import {
const EXTERNAL_WORKSPACE_OPERATIONS = new Set(["get_workspace", "list_workspaces", "select_workspace", "prepare_repository", "register_deliverable"]);
const IMPLEMENTED_OPERATIONS = new Set([
"send_chat_message",
"submit_complaint", "submit_suggestion",
"read_agent_instructions", "update_agent_instructions", "get_agent_instruction_history", "restore_agent_instructions",
"search_api", "call_api", "hire_agent",
@@ -434,6 +436,7 @@ export class PaperclipRunnerToolAuthority {
// retry, without ever falling through to an uncertain mutation.
let recovered: unknown;
if (call.tool === "set_task_title") recovered = await this.#setTaskTitle(input, true);
else if (call.tool === "send_chat_message") recovered = await this.#sendChatMessage(input, this.#privacyActor(run), true);
else if (call.tool === "submit_complaint" || call.tool === "submit_suggestion") recovered = await this.#submitCommentary(call.tool, input, true);
else if (call.tool === "create_task") recovered = await this.#createTask(input, null, true);
else if (call.tool === "request_human_input") recovered = await this.#requestHumanInput(input, null, true);
@@ -959,6 +962,7 @@ export class PaperclipRunnerToolAuthority {
return { approval, tasks: tasks.filter(row => !readable || readable.has(row.issue.id)).map(row => row.issue) };
}
case "report_progress": return this.#reportProgress(input);
case "send_chat_message": return this.#sendChatMessage(input, this.#privacyActor(context.run));
case "request_human_input": return this.#requestHumanInput(input,
(await captureRunIdentity(this.db, this.binding, { readScope: "identity" })).context?.id ?? null);
case "create_task": return this.#createTask(input,
@@ -1486,6 +1490,14 @@ export class PaperclipRunnerToolAuthority {
return { interactionId: review.interaction.id, status: expectedStatus };
}
async #sendChatMessage(input: Record<string, unknown>, actor: AuthorizationActor, receiptOnly = false) {
const result = await sendAgentChatMessage(this.db, { ...this.binding, keyScope: actor.keyScope,
onBehalfOfUserId: actor.onBehalfOfUserId, authorizingUserId: actor.authorizingUserId, source: actor.source }, input, {
authorizeNative: tx => this.#lockAuthorizedMutationContext(tx), receiptOnly,
});
return result ?? undefined;
}
async #reportProgress(input: Record<string, unknown>): Promise<unknown> {
const body = typeof input.body === "string" ? input.body.trim() : "";
const idempotencyKey = typeof input.idempotencyKey === "string" ? input.idempotencyKey.trim() : "";
@@ -91,6 +91,23 @@ it.skipIf(!tar || !hasZstd)("reuses only byte-matched image files, keeps modes,
expect(await fs.readlink(path.join(targetPath, "tool"))).toBe("bin/tool");
});
it.skipIf(!tar || !hasZstd)("stages a complete layered pack into a fresh nested runtime directory", async () => {
const { root, pack, archive } = await layeredFixture();
const remote = await runner(root);
const workspace = path.join(root, "workspace");
await fs.mkdir(workspace);
const targetPath = path.join(workspace, ".paperclip-runtime", "paperclip-runner", "provider-pack");
await expect(fs.lstat(path.dirname(targetPath))).rejects.toMatchObject({ code: "ENOENT" });
expect(await stageProviderPackArchive({ archive, runner: remote, targetPath, reuseRoots: [pack] })).toBe(true);
expect(await fs.readFile(path.join(targetPath, "bin/large"))).toEqual(await fs.readFile(path.join(pack, "bin/large")));
expect(await fs.readFile(path.join(targetPath, "provider-pack.json"))).toEqual(await fs.readFile(path.join(pack, "provider-pack.json")));
expect(await fs.readlink(path.join(targetPath, "tool"))).toBe("bin/tool");
for (const directory of [targetPath, path.dirname(targetPath), path.dirname(path.dirname(targetPath))]) {
expect((await fs.stat(directory)).mode & 0o777).toBe(0o700);
}
expect((await fs.stat(path.join(targetPath, "bin/large"))).ino).not.toBe((await fs.stat(path.join(pack, "bin/large"))).ino);
});
it.skipIf(!tar || !hasZstd).each(["missing", "wrong-digest", "symlink", "ancestor-symlink"])("transfers a large layer when the image file is %s", async reason => {
const { root, pack, archive } = await layeredFixture();
const remote = await runner(root);
@@ -135,13 +152,18 @@ it.skipIf(!tar || !hasZstd)("rejects a corrupt missing layer before transfer and
expect(remote.syncIn).not.toHaveBeenCalled();
});
it.skipIf(!tar || !hasZstd)("does not follow a layered target symlink or accept invalid reuse receipts", async () => {
it.skipIf(!tar || !hasZstd).each(["directory", "file", "symlink"])("rejects an existing layered target %s or invalid reuse receipts", async kind => {
const { root, archive } = await layeredFixture();
const remote = await runner(root);
const targetPath = path.join(root, "target"), other = path.join(root, "other");
await fs.mkdir(other); await fs.symlink(other, targetPath);
await fs.mkdir(other);
if (kind === "directory") await fs.mkdir(targetPath);
else if (kind === "file") await fs.writeFile(targetPath, "untouched");
else await fs.symlink(other, targetPath);
await expect(stageProviderPackArchive({ archive, runner: remote, targetPath })).rejects.toThrow("runner_remote_provider_archive_reuse_failed");
expect(await fs.readdir(other)).toEqual([]);
if (kind === "directory") expect(await fs.readdir(targetPath)).toEqual([]);
if (kind === "file") expect(await fs.readFile(targetPath, "utf8")).toBe("untouched");
remote.execute.mockResolvedValue({ exitCode: 0, timedOut: false, stdout: "[999999]", stderr: "", signal: null, pid: null, startedAt: null });
await expect(stageProviderPackArchive({ archive, runner: remote, targetPath: `${targetPath}-2` })).rejects.toThrow("runner_remote_provider_archive_reuse_failed");
expect(remote.syncIn).not.toHaveBeenCalled();
@@ -70,7 +70,8 @@ const reuseScript = `
(async()=>{
const fs=require('node:fs'),p=require('node:path'),c=require('node:crypto');
const spec=JSON.parse(Buffer.from(process.argv[1],'base64').toString('utf8'));
const f=fs.promises;await f.mkdir(spec.target,{mode:448});
const f=fs.promises;await f.mkdir(p.dirname(spec.target),{recursive:true,mode:448});
await f.mkdir(spec.target,{mode:448});
const missing=[];
const regularPath=async(root,relative)=>{
let current=root;if(!(await f.lstat(current)).isDirectory())return false;
@@ -18,7 +18,7 @@ import {
AGENTCORE_QUALIFIED_MODEL,
CLAUDE_MANAGED_QUALIFIED_MODEL,
} from "../provider-profile-qualification.js";
import { resolveAcpxQualification, resolveAntigravityCandidateAdmission, type AcpxQualificationCandidate } from "./acpx-qualification.js";
import { resolveAcpxQualification, resolveAntigravityCandidateAdmission, resolveKimiCandidateAdmission, type AcpxQualificationCandidate } from "./acpx-qualification.js";
import { compatibleCodexModel } from "./codex-model-fallback.js";
export const QUALIFIED_OPENCODE_RUNNER_VERSION = "1.18.34" as const;
@@ -34,6 +34,7 @@ export const DEFAULT_ACPX_RUNNER_MODELS = {
cursor: null,
copilot: null,
pi: null,
kimi: null,
} as const;
export type QualifiedPaperclipRunnerAcpxAgent =
@@ -564,6 +565,15 @@ export function resolvePaperclipRunnerProviderProfile(
throw new PaperclipRunnerProviderProfileError("paperclip_runner_acpx_qualification_invalid", "Antigravity candidate admission requires the authorized company, exact model and runtime identity.");
}
}
if (pendingAcpxProfile.value === "kimi") {
try {
if (resolveKimiCandidateAdmission({ companyId: context.companyId, model }, process.env)) {
return { provider: "acpx", backend: "acpx_runtime", model, acpxAgent: "kimi" };
}
} catch {
throw new PaperclipRunnerProviderProfileError("paperclip_runner_acpx_qualification_invalid", "Kimi candidate admission requires the authorized company, exact model, target and runtime identity.");
}
}
// Qualification authority belongs to the isolated server operator. Never
// read this allowlist from adapter config, credential refs, or run env.
let qualification: AcpxQualificationCandidate | undefined;
@@ -577,10 +587,10 @@ export function resolvePaperclipRunnerProviderProfile(
}
throw new PaperclipRunnerProviderProfileError("paperclip_runner_acpx_agent_unavailable", `${pendingAcpxProfile.label} is awaiting native Runner acceptance. Its profile is not enabled for production runs.`);
}
if ((acpxAgent === "cursor" || acpxAgent === "pi") && !model) {
if ((acpxAgent === "cursor" || acpxAgent === "pi" || acpxAgent === "kimi") && !model) {
throw new PaperclipRunnerProviderProfileError("paperclip_runner_acpx_model_required", "This ACPX agent requires an explicit model ID; there is no default model.");
}
if (acpxAgent !== "claude" && acpxAgent !== "codex" && acpxAgent !== "grok" && acpxAgent !== "cursor" && acpxAgent !== "copilot" && acpxAgent !== "pi") {
if (acpxAgent !== "claude" && acpxAgent !== "codex" && acpxAgent !== "grok" && acpxAgent !== "cursor" && acpxAgent !== "copilot" && acpxAgent !== "pi" && acpxAgent !== "kimi") {
throw new PaperclipRunnerProviderProfileError(
"paperclip_runner_acpx_agent_unavailable",
"Paperclip Runner ACPX requires a qualified agent profile.",
@@ -10,7 +10,7 @@ export function shouldUseBundledAcpxImageAssets(input: {
}): boolean {
return input.requiresRemoteProviderPack && !input.configuredProviderPackRoot &&
input.provider.kind === "acpx" &&
["cursor", "copilot"].includes(input.provider.agent);
["cursor", "copilot", "kimi"].includes(input.provider.agent);
}
type VerificationCommandResult = {
@@ -0,0 +1,43 @@
import { describe, expect, it } from "vitest";
import { isSettledRemoteWarmControlState } from "./remote-warm-session-recovery.js";
const identity = { runId: "run", normalizedSessionId: "session", runnerInstanceId: "runner", environmentLeaseId: "lease", turnId: "turn", itemId: "item" };
const terminal = { eventType: "run.terminal", envelope: { ...identity } };
const goal = { schema: "paperclip.session_goal.snapshot.v1", goal: null, workingNow: false };
const notification = { schema: "paperclip.prp.event.v1", eventType: "session.goal.snapshot", runId: identity.runId,
normalizedSessionId: identity.normalizedSessionId, sourceInstanceId: "runner", sourceKind: "runner", payload: goal };
const snapshot = { eventType: "session.goal.snapshot", envelope: { ...identity, payload: notification } };
const state = (events: unknown[]) => ({ identity, commands: [{ status: "completed" }], committedEvents: events });
describe("settled remote warm control evidence", () => {
it.each(["paperclip.prp.event.v1", "paperclip.prp.event.v2"])("accepts the terminal and an authenticated %s idle goal-read suffix without changing its order", schema => {
const suffix = { ...snapshot, envelope: { ...snapshot.envelope, payload: { ...notification, schema } } };
const control = state([terminal, suffix]);
expect(isSettledRemoteWarmControlState(control, identity)).toBe(true);
expect(control.committedEvents).toEqual([terminal, suffix]);
expect(isSettledRemoteWarmControlState(state([terminal]), identity)).toBe(true);
});
it.each([
["active goal", { ...snapshot, envelope: { ...snapshot.envelope, payload: { ...notification, payload: { ...goal, goal: { status: "active" } } } } }],
["provider work", { ...snapshot, envelope: { ...snapshot.envelope, payload: { ...notification, payload: { ...goal, workingNow: true } } } }],
["unknown event", { ...snapshot, eventType: "unknown" }],
["unknown protocol version", { ...snapshot, envelope: { ...snapshot.envelope, payload: { ...notification, schema: "paperclip.prp.event.v3" } } }],
["new turn", { ...snapshot, eventType: "turn.started" }],
["foreign run", { ...snapshot, envelope: { ...snapshot.envelope, runId: "other" } }],
["foreign session", { ...snapshot, envelope: { ...snapshot.envelope, normalizedSessionId: "other" } }],
["foreign source", { ...snapshot, envelope: { ...snapshot.envelope, payload: { ...notification, sourceInstanceId: "other" } } }],
["foreign lease", { ...snapshot, envelope: { ...snapshot.envelope, environmentLeaseId: "other" } }],
["foreign notification", { ...snapshot, envelope: { ...snapshot.envelope, payload: { ...notification, runId: "other" } } }],
["missing payload", { ...snapshot, envelope: { ...identity } }],
])("holds a %s suffix", (_name, suffix) => {
expect(isSettledRemoteWarmControlState(state([terminal, suffix]), identity)).toBe(false);
});
it("holds missing terminals, pending commands and mixed core identities", () => {
expect(isSettledRemoteWarmControlState(state([snapshot]), identity)).toBe(false);
expect(isSettledRemoteWarmControlState({ ...state([terminal, snapshot]), commands: [{ status: "dispatched" }] }, identity)).toBe(false);
expect(isSettledRemoteWarmControlState({ ...state([terminal, snapshot]), identity: { ...identity, runnerInstanceId: "other" } }, identity)).toBe(false);
expect(isSettledRemoteWarmControlState(state([{ ...terminal, envelope: { ...identity, runId: "other" } }, snapshot]), identity)).toBe(false);
});
});
@@ -3,6 +3,7 @@ import { agents, companies, environmentLeases, heartbeatRuns, issues, nativeRunF
import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target";
import { parseNativeExecutionInput, type NativeExecutionInput, type PersistedNativeSession } from "../../vendor/paperclip-runner/index.js";
import { currentNativeControllerIdentity, evaluateNativeControllerTakeover } from "./native-restart-recovery.js";
import { readNativeControllerRecovery } from "./native-controller-recovery.js";
export type RemoteWarmSessionRecovery = {
identity: Record<string, string>;
@@ -13,6 +14,36 @@ export type RemoteWarmSessionRecovery = {
revalidate: (leaseOwner: string) => Promise<void>;
};
/** A goal read can publish an idle snapshot after the terminal. It carries no
* new work; every other suffix still requires reconciliation before handoff. */
export function isSettledRemoteWarmControlState(control: Record<string, unknown>, identity: Record<string, string>): boolean {
const record = (value: unknown): Record<string, unknown> =>
value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
const commands = control.commands;
const events = control.committedEvents;
if (!Array.isArray(commands) || commands.some(command => record(command).status !== "completed") ||
!Array.isArray(events) || Object.entries(identity).some(([key, value]) => record(control.identity)[key] !== value)) return false;
const terminalIndex = events.findLastIndex(event => record(event).eventType === "run.terminal");
if (terminalIndex < 0) return false;
const matchesSession = (event: unknown) => {
const envelope = record(record(event).envelope);
return envelope.runId === identity.runId && envelope.normalizedSessionId === identity.normalizedSessionId;
};
if (!matchesSession(events[terminalIndex])) return false;
return events.slice(terminalIndex + 1).every(event => {
const envelope = record(record(event).envelope);
const notification = record(envelope.payload);
const payload = record(notification.payload);
return record(event).eventType === "session.goal.snapshot" && matchesSession(event) &&
Object.entries(identity).every(([key, value]) => envelope[key] === value) &&
(notification.schema === "paperclip.prp.event.v1" || notification.schema === "paperclip.prp.event.v2") &&
notification.eventType === "session.goal.snapshot" &&
notification.sourceInstanceId === identity.runnerInstanceId && notification.sourceKind === "runner" &&
notification.runId === identity.runId && notification.normalizedSessionId === identity.normalizedSessionId &&
payload.schema === "paperclip.session_goal.snapshot.v1" && payload.goal === null && payload.workingNow === false;
});
}
export async function inspectRemoteWarmSessionRecovery(input: {
db: Db;
execution: NativeExecutionInput;
@@ -52,15 +83,14 @@ export async function inspectRemoteWarmSessionRecovery(input: {
checkpoint.driverKind !== previous.session.driverKind || !checkpoint.providerSessionId ||
checkpoint.activeTurnId !== null || checkpoint.pendingRuntimeRequests?.length !== 0) return deny();
const control = input.readControlState();
const commands = control.commands;
const events = control.committedEvents;
const terminal = Array.isArray(events) ? events.at(-1) : undefined;
if (!Array.isArray(commands) || commands.some(command => command.status !== "completed") ||
terminal?.eventType !== "run.terminal" || terminal?.envelope?.runId !== identity.runId ||
terminal?.envelope?.normalizedSessionId !== identity.normalizedSessionId ||
Object.entries(identity).some(([key, value]) => (control.identity as Record<string, unknown>)?.[key] !== value)) return deny();
if (!isSettledRemoteWarmControlState(control, identity)) return deny();
const assertPriorStopped = async () => {
const stopped = await evaluateNativeControllerTakeover({ owner: prior.owner, now: new Date(), requireStoppedController: true });
const persistedRecovery = readNativeControllerRecovery(prior.run.runnerProfileJson?.nativeControllerRecovery,
prior.run.runnerProfileJson?.sessionCheckpoint);
// A PID in another webserver container cannot prove life or death. New
// durable controllers are fenced by the database lease and current turn.
const stopped = await evaluateNativeControllerTakeover({ owner: prior.owner, now: new Date(), requireStoppedController: true,
remoteLeaseFenced: persistedRecovery !== null });
if (!stopped.allowed) return deny();
};
await assertPriorStopped();
@@ -41,6 +41,7 @@ function words(text: string): string[] {
}
function dedicatedTools(method: string, path: string): string[] {
if (method === "POST" && path === "/api/agents/me/chat/messages") return ["send_chat_message"];
if (/\/companies\/\{[^}]+\}\/workflows$/.test(path)) return method === "GET" ? ["list_workflows"] : method === "POST" ? ["create_workflow"] : [];
if (/\/workflows\/\{[^}]+\}$/.test(path)) return method === "GET" ? ["read_workflow"] : method === "PATCH" ? ["update_workflow"] : [];
if (/\/issues\/\{[^}]+\}\/title$/.test(path) && method === "PUT") return ["set_task_title"];
@@ -10,6 +10,7 @@ import { getConversationOwnershipBlocker } from "./conversation-continuation.js"
import { isExternalChatPresentationContext } from "./heartbeat-run-summary.js";
import { finalizeNativeRun, recoverCommittedNativeChatResponse, repairCommittedNativeChatResponse } from "./native-runtime/native-run-finalizer.js";
import { nativeResponsePresentationWork } from "./native-runtime/native-response-presentation-work.js";
import { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js";
import { resolveExternalTestDatabaseUrl } from "../__tests__/helpers/external-test-database.js";
@@ -24,10 +25,12 @@ describe.skipIf(!support.supported)("presentation and ownership PostgreSQL read
}, 60_000);
afterAll(async () => { await db?.$client.end(); await database?.cleanup(); });
async function fixture(patch: Partial<typeof heartbeatRuns.$inferInsert> = {}) {
const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID();
await db.insert(companies).values({ id: companyId, name: "Read fixture", issuePrefix: companyId.slice(0, 8) });
await db.insert(agents).values({ id: agentId, companyId, name: "Read agent", adapterType: "codex_local" });
async function fixture(patch: Partial<typeof heartbeatRuns.$inferInsert> = {}, identity?: { companyId: string; agentId: string }) {
const companyId = identity?.companyId ?? randomUUID(), agentId = identity?.agentId ?? randomUUID(), issueId = randomUUID();
if (!identity) {
await db.insert(companies).values({ id: companyId, name: "Read fixture", issuePrefix: companyId.slice(0, 8) });
await db.insert(agents).values({ id: agentId, companyId, name: "Read agent", adapterType: "codex_local" });
}
await db.insert(issues).values({ id: issueId, companyId, title: "Read task", status: "blocked", assigneeAgentId: agentId });
const payload = randomBytes(128 * 1024).toString("base64");
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, nativeIssueId: issueId,
@@ -138,10 +141,10 @@ describe.skipIf(!support.supported)("presentation and ownership PostgreSQL read
expect(await repairCommittedNativeChatResponse(db, { companyId: f.companyId, issueId: other.id, runId: f.run.id })).toBe(false);
});
async function unpublishedAcceptedReply() {
async function unpublishedAcceptedReply(id?: string, identity?: { companyId: string; agentId: string }) {
const payload = randomBytes(128 * 1024).toString("base64");
const f = await fixture({ runtimeMode: "native", status: "running", finishedAt: null,
contextSnapshot: { source: "internal", prompt: payload, paperclipWake: { prompt: payload } } });
const f = await fixture({ id, runtimeMode: "native", status: "running", finishedAt: null,
contextSnapshot: { source: "internal", prompt: payload, paperclipWake: { prompt: payload } } }, identity);
const [contract] = await db.insert(completionContracts).values({ companyId: f.companyId, issueId: f.issueId,
revision: 1, schemaVersion: "paperclip.completion-contract.v1", policyVersion: "test", risk: "standard",
completionAuthority: "server_arbiter", incompleteCriteriaPolicy: "preserve_non_terminal",
@@ -173,6 +176,40 @@ describe.skipIf(!support.supported)("presentation and ownership PostgreSQL read
return { ...f, before, reply };
}
it("saves an independent task reply while an earlier task waits for its database lock", async () => {
const suffix = randomUUID().replaceAll("-", "").slice(0, 12);
const first = await unpublishedAcceptedReply(`00000000-0000-4000-8000-${suffix}`);
const second = await unpublishedAcceptedReply(`00000000-0000-4001-8000-${suffix}`, first);
let release!: () => void;
let locked!: () => void;
const hold = new Promise<void>(resolve => { release = resolve; });
const acquired = new Promise<void>(resolve => { locked = resolve; });
const blocker = db.transaction(async tx => {
await tx.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, first.run.id)).for("update");
locked(); await hold;
});
await Promise.race([acquired, blocker]);
const onError = vi.fn();
const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError });
const worker = nativeResponsePresentationWork(db).register(coordinator, () => true);
try {
await vi.waitFor(async () => {
const rows = await db.select().from(issueComments).where(eq(issueComments.createdByRunId, second.run.id));
expect(rows).toHaveLength(1);
expect(rows[0].body).toBe(second.reply);
}, { timeout: 5000 });
expect(await db.select().from(issueComments).where(eq(issueComments.createdByRunId, first.run.id))).toHaveLength(0);
} finally {
release(); await blocker; await worker.ready; worker.stop(); await coordinator.stop();
}
for (const f of [first, second]) {
const rows = await db.select().from(issueComments).where(eq(issueComments.createdByRunId, f.run.id));
expect(rows).toHaveLength(1);
expect(rows[0].body).toBe(f.reply);
}
expect(onError).not.toHaveBeenCalled();
});
async function repairFixture(patch: Partial<typeof heartbeatRuns.$inferInsert>) {
+1 -1
View File
@@ -52,7 +52,7 @@ export function redactRegisteredSecretValues<T>(input: T, values: string[], cred
if (!record) return input;
return Object.fromEntries(
Object.entries(record)
.filter(([key]) => key !== REGISTRY_KEY)
.filter(([key]) => key !== REGISTRY_KEY && key !== "nativeControllerRecovery" && key !== "nativeTaskSessionPublication")
.map(([key, value]) => [key, key === "credentialAccess"
? (credentialConnectionName ? { connectionName: redactText(credentialConnectionName, values) } : {})
: redactRegisteredSecretValues(value, values, credentialConnectionName)]),
+26 -6
View File
@@ -12,6 +12,7 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
queue: DeliveryQueue;
batchSize?: number;
minimumPassIntervalMs?: number;
concurrentChecks?: { limit: number; key(entry: Entry): string };
reconstructionIntervalMs?: number | ((scope: Scope | undefined) => number);
beforePass?(): Promise<void>;
decodeScope(hint: string): Scope | undefined;
@@ -21,6 +22,9 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
},
canRun: () => boolean,
) {
if (options.concurrentChecks && (!Number.isInteger(options.concurrentChecks.limit) || options.concurrentChecks.limit < 1)) {
throw new RangeError("Concurrent check limit must be a positive integer");
}
const dirty = new Set<string>(["*"]);
const versions = new Map<string, number>();
const scans = new Map<string, { version: number; seen: Set<string> }>();
@@ -112,14 +116,30 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
&& Math.max(entry.dueAt, failures.get(entry.id)?.retryAt ?? 0) <= Date.now())
.sort((a, b) => a.dueAt - b.dueAt || a.id.localeCompare(b.id));
const batch = takeCompanyWorkBatch(due, options.batchSize ?? 25, lastCompany); lastCompany = batch.lastCompany;
const groups = new Map<string, Entry[]>();
for (const entry of batch.entries) {
if (signal.aborted || !canRun()) break;
try {
const current = await options.check(entry, signal);
if (current) entries.set(entry.id, current); else entries.delete(entry.id);
failures.delete(entry.id);
} catch (error) { failed(failures, entry.id); errors.push(error); }
const key = options.concurrentChecks?.key(entry) ?? "";
const group = groups.get(key) ?? [];
group.push(entry); groups.set(key, group);
}
const pending = [...groups.values()];
let nextGroup = 0;
const checkGroups = async () => {
while (!signal.aborted && canRun()) {
const group = pending[nextGroup++];
if (!group) return;
for (const entry of group) {
if (signal.aborted || !canRun()) return;
try {
const current = await options.check(entry, signal);
if (current) entries.set(entry.id, current); else entries.delete(entry.id);
failures.delete(entry.id);
} catch (error) { failed(failures, entry.id); errors.push(error); }
}
}
};
const checks = await Promise.allSettled(Array.from({ length: Math.min(options.concurrentChecks?.limit ?? 1, pending.length) }, checkGroups));
for (const result of checks) if (result.status === "rejected") errors.push(result.reason);
if (errors.length) throw new AggregateError(errors, `${options.queue} could not settle`);
},
// Retain scopes/deadlines for resume without retaining a settled write's
+5 -1
View File
@@ -753,7 +753,11 @@ export function statusCardService(
createdByAgentId: input.actor?.agentId ?? null,
createdByUserId: input.actor?.userId ?? null,
hiddenAt: now,
idempotencyKey: `status-card-update:${card.id}:${fingerprintHash}`,
// Claim ownership is revision-scoped too. A superseded refresh may cancel
// its unclaimed task; it must not share that task with a newer revision
// which has the same watched fingerprint but has not claimed the card yet.
// Concurrent refreshes of this revision still deduplicate normally.
idempotencyKey: `status-card-update:${card.id}:${revision}:${fingerprintHash}`,
onDeduplicated: (reason) => { deduplicated = reason === "idempotency_key"; },
});
const reopened = deduplicated && TERMINAL_ISSUE_STATUSES.has(created.status)
+3
View File
@@ -2113,6 +2113,9 @@ export function toolAccessService(
sourceTemplateKey: galleryEntry.slug,
connectionMethodKey: method?.key,
methodConfig: normalizedMethodConfig?.values ?? {},
...(galleryEntry.slug === "google-workspace" && retainedConnection?.status === "draft"
? { workspaceSetup: retainedConnection.config.workspaceSetup }
: {}),
...(method?.defaults?.mcpSessionRequired === true
? { mcpSessionRequired: true }
: {}),
@@ -1,13 +1,13 @@
import { notifyIssueReviewState } from "../../issue-review-state-notifications.js";
import { vercelConnectHttpError, vercelCredentialFor, vercelConnectResourcesFor } from "../vercel-credentials.js";
import { credentialScope, managedConnectorProfile } from "./client-settings.js";
import { createOAuthClientRegistration } from "./client-registration.js";
import { assertOAuthClientBindingUnchanged, createOAuthClientRegistration } from "./client-registration.js";
import type { ActorInfo } from "../types.js";
import { createOAuthEndpointPolicy } from "./endpoint-policy.js";
import { createOAuthDiscovery } from "./discovery.js";
import { oauthConfig, normalizeOauthScopes, originOf } from "./metadata.js";
import { asRecord, oauthConfig, normalizeOauthScopes, originOf } from "./metadata.js";
import { connectionMethodForConnection } from "../connection-methods.js";
import { GOOGLE_WORKSPACE_DEFAULT_SELECTION, googleWorkspaceSelectedScopes, type GoogleWorkspaceServiceSelection, isGoogleWorkspaceScopeGrant } from "@paperclipai/shared";
import { googleWorkspaceDraftRequestedScopes, googleWorkspaceSelectedScopes, type GoogleWorkspaceServiceSelection, isGoogleWorkspaceScopeGrant } from "@paperclipai/shared";
import { and, eq, lt } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import { companies, agents, connectionGrants, issueThreadInteractions, toolConnections, toolOauthStates } from "@paperclipai/db";
@@ -72,17 +72,37 @@ export function createOAuthAuthorizationStart(db: Db, dependencies: Dependencies
const galleryMethod = galleryEntry
? connectionMethodForConnection(galleryEntry, connection)
: null;
const starterBinding = actorBinding(input.actor);
const fixedPersonalIdentity = await fixedPersonalIdentityForReconnect(
connection,
input.subjectUserId,
input.actor,
);
const authorizationSubjectUserId =
fixedPersonalIdentity?.subjectUserId ?? input.subjectUserId;
const authorizationSubjectAgentId = input.subjectAgentId;
const workspaceDraftBinding = {
companyId, connectionId: connection.id,
actorType: starterBinding.actorType, actorId: starterBinding.actorId,
grantKind: connection.credentialPolicy === "per_user" ? "user" as const
: connection.credentialPolicy === "per_agent" ? "agent" as const : "organization" as const,
subjectId: connection.credentialPolicy === "per_user" ? authorizationSubjectUserId ?? null
: connection.credentialPolicy === "per_agent" ? authorizationSubjectAgentId ?? null : null,
};
const requestedScopes = await (async () => {
if (input.workspaceServices && sourceTemplateKey !== "google-workspace") throw badRequest("Service selection requires Google Workspace");
if (sourceTemplateKey === "google-workspace") {
if (input.workspaceServices && input.scopes) throw badRequest("Choose services or scopes, not both");
const allowed = normalizeOauthScopes(galleryMethod?.defaults?.scopesHint);
let scopes = input.scopes ?? (input.workspaceServices ? googleWorkspaceSelectedScopes(input.workspaceServices) : undefined);
if (!scopes && connection.status !== "draft") {
if (!scopes && connection.status === "draft") {
scopes = googleWorkspaceDraftRequestedScopes(connection.config.workspaceSetup, workspaceDraftBinding) ?? undefined;
}
if (!scopes && input.actor.managementContext?.actor.type === "agent") return null;
if (!scopes) {
// Account-page reconnect has no new service choice. Never replace its
// subject's saved request with the new-account write defaults.
const binding = actorBinding(input.actor);
const subjectUserId = input.subjectUserId ?? (binding.actorType === "user" ? binding.actorId : null);
const subjectUserId = authorizationSubjectUserId ?? (starterBinding.actorType === "user" ? starterBinding.actorId : null);
const kind = connection.credentialPolicy === "per_user" ? "user"
: connection.credentialPolicy === "per_agent" ? "agent" : "organization";
const [grant] = await db.select().from(connectionGrants).where(and(
@@ -94,11 +114,12 @@ export function createOAuthAuthorizationStart(db: Db, dependencies: Dependencies
: eq(connectionGrants.isDefault, true),
)).limit(1);
const saved = grant?.providerTenant?.oauth;
scopes = normalizeOauthScopes(saved?.requestedScopes ?? saved?.scopes)
scopes = normalizeOauthScopes(saved?.requestedScopes ?? (connection.status !== "draft" ? saved?.scopes : undefined))
.filter(scope => allowed.includes(scope));
if (scopes.length === 0) throw badRequest("Choose your Google Workspace services before reconnecting");
if (scopes.length === 0) throw badRequest(connection.status === "draft"
? "Choose your Google Workspace services before continuing setup"
: "Choose your Google Workspace services before reconnecting");
}
scopes ??= googleWorkspaceSelectedScopes(GOOGLE_WORKSPACE_DEFAULT_SELECTION);
if (!isGoogleWorkspaceScopeGrant(scopes) || scopes.some(scope => !allowed.includes(scope))) throw badRequest("Select at least one supported Google service");
return scopes;
}
@@ -118,15 +139,6 @@ export function createOAuthAuthorizationStart(db: Db, dependencies: Dependencies
}
return requested;
})();
const starterBinding = actorBinding(input.actor);
const fixedPersonalIdentity = await fixedPersonalIdentityForReconnect(
connection,
input.subjectUserId,
input.actor,
);
const authorizationSubjectUserId =
fixedPersonalIdentity?.subjectUserId ?? input.subjectUserId;
const authorizationSubjectAgentId = input.subjectAgentId;
if (authorizationSubjectAgentId) {
const [subjectAgent] = await db
.select({ id: agents.id })
@@ -209,6 +221,37 @@ export function createOAuthAuthorizationStart(db: Db, dependencies: Dependencies
return { connectionId: connection.id, provider: sourceTemplateKey ?? "oauth",
authorizationUrl: destination.toString(), expiresAt: null };
}
if (sourceTemplateKey === "google-workspace" && connection.status === "draft") {
const workspaceSetup = { version: 1, ...workspaceDraftBinding, requestedScopes };
const updated = await db.transaction(async tx => {
const [saved] = await tx.select().from(toolConnections).where(and(
eq(toolConnections.id, connection.id), eq(toolConnections.companyId, companyId), eq(toolConnections.status, "draft"),
)).for("no key update");
if (!saved || (["credentialPolicy", "credentialSource", "authKind", "transport", "createdByUserId"] as const)
.some(key => saved[key] !== connection[key])
|| ["sourceTemplateKey", "connectionMethodKey"].some(key => saved.config[key] !== connection.config[key]
|| saved.transportConfig[key] !== connection.transportConfig[key])) {
throw conflict("Connection setup changed. Reload before continuing");
}
const config: Record<string, unknown> = { ...saved.config, workspaceSetup };
const transportConfig: Record<string, unknown> = { ...saved.transportConfig, workspaceSetup };
if (starterBinding.actorType === "user" && starterBinding.actorId) {
for (const target of [config, transportConfig]) {
const oauth = asRecord(target.oauth);
const pending = { ...asRecord(oauth.pendingWorkspaceServiceRequests) };
if (Object.hasOwn(pending, starterBinding.actorId)) {
delete pending[starterBinding.actorId];
target.oauth = { ...oauth, pendingWorkspaceServiceRequests: pending };
}
}
}
const [row] = await tx.update(toolConnections).set({ config, transportConfig, updatedAt: now() })
.where(and(eq(toolConnections.id, saved.id), eq(toolConnections.companyId, companyId), eq(toolConnections.status, "draft"))).returning();
return row;
});
if (!updated) throw conflict("Connection setup changed. Reload before continuing");
connection = updated;
}
if (connection.credentialSource === "vercel_connect") {
if (!vercelConnect)
throw vercelConnectHttpError(
@@ -629,10 +672,13 @@ export function createOAuthAuthorizationStart(db: Db, dependencies: Dependencies
});
}
const nextConfig = {
...connection.config,
oauth: {
...oauthConfig(connection),
await db.transaction(async tx => {
const [saved] = await tx.select().from(toolConnections).where(and(
eq(toolConnections.id, connection.id), eq(toolConnections.companyId, companyId),
)).for("no key update");
if (!saved) throw notFound("Tool connection not found");
assertOAuthClientBindingUnchanged(connection, saved, endpoints, input.redirectUri);
const oauth = {
provider: endpoints.provider,
authorizationUrl: endpoints.authorizationUrl,
tokenUrl: endpoints.tokenUrl,
@@ -655,27 +701,29 @@ export function createOAuthAuthorizationStart(db: Db, dependencies: Dependencies
// Persist the issuer and resource this authorization run is bound to.
// `iss` on the callback is validated against `expectedIssuer`, and
// refresh/reconnect/revoke reuse the same pair rather than re-deriving it.
issuer: endpoints.issuer ?? oauthConfig(connection).issuer ?? null,
issuer: endpoints.issuer ?? oauthConfig(saved).issuer ?? null,
expectedIssuer: endpoints.issuer ?? null,
resource:
endpoints.resource ?? oauthConfig(connection).resource ?? null,
endpoints.resource ?? oauthConfig(saved).resource ?? null,
clientIdMetadataDocumentSupported:
endpoints.clientIdMetadataDocumentSupported === true,
},
};
await db
.update(toolConnections)
.set({
};
const nextConfig = { ...saved.config, oauth: { ...oauthConfig(saved), ...oauth } };
const nextTransportConfig = { ...saved.transportConfig, oauth: { ...asRecord(saved.transportConfig.oauth), ...oauth } };
await tx
.update(toolConnections)
.set({
// A generic URL connection starts life as `authKind: none`. Once it has
// completed OAuth discovery and client resolution it is an OAuth
// connection, and refresh, reconnect, revoke and diagnostics must all
// treat it as one.
authKind: "oauth",
config: nextConfig,
transportConfig: nextConfig,
updatedAt: new Date(),
})
.where(eq(toolConnections.id, connection.id));
authKind: "oauth",
config: nextConfig,
transportConfig: nextTransportConfig,
updatedAt: new Date(),
})
.where(and(eq(toolConnections.id, connection.id), eq(toolConnections.companyId, companyId)));
});
return {
connectionId: connection.id,
@@ -1,6 +1,6 @@
import type { ActorInfo } from "../types.js";
import { createOAuthClientProtocol } from "./client.js";
import { oauthConfig, sameOAuthIssuer } from "./metadata.js";
import { asRecord, oauthConfig, sameOAuthIssuer } from "./metadata.js";
import { resolveOAuthClientIdMetadataDocumentUrl } from "./client-metadata.js";
import type { OAuthProviderEndpoints, OAuthClientRegistrationSource } from "./types.js";
import { connectionMethodForConnection } from "../connection-methods.js";
@@ -8,7 +8,7 @@ import { and, eq } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import { toolConnections } from "@paperclipai/db";
import type { AppDefinition } from "@paperclipai/shared";
import { notFound, unprocessable } from "../../../errors.js";
import { conflict, notFound, unprocessable } from "../../../errors.js";
import { type RemoteHttpEndpointLookup } from "../../remote-http-endpoint-guard.js";
import type { OAuthCredentialStore } from "./credential-store.js";
@@ -24,6 +24,34 @@ type RegistrationDependencies = {
requestOAuthClientRegistration: ReturnType<typeof createOAuthClientProtocol>["requestOAuthClientRegistration"];
};
export function assertOAuthClientBindingUnchanged(
expected: typeof toolConnections.$inferSelect,
current: typeof toolConnections.$inferSelect,
endpoints: OAuthProviderEndpoints,
redirectUri: string,
) {
const changedSetup = expected.config.sourceTemplateKey === "google-workspace" && expected.status === "draft"
&& (current.status !== "draft" || (["credentialPolicy", "credentialSource", "authKind", "transport", "createdByUserId"] as const)
.some(key => current[key] !== expected[key])
|| ["sourceTemplateKey", "connectionMethodKey"].some(key => current.config[key] !== expected.config[key]
|| current.transportConfig[key] !== expected.transportConfig[key]));
const bindings: Record<string, unknown> = {
clientRedirectUri: redirectUri, clientIssuer: endpoints.issuer ?? null,
clientResource: endpoints.resource ?? null, clientCompanyId: expected.companyId,
issuer: endpoints.issuer ?? null, resource: endpoints.resource ?? null,
};
const changedClient = expected.ownership !== current.ownership
|| JSON.stringify(oauthSecretRef(expected, "oauth.client_secret")) !== JSON.stringify(oauthSecretRef(current, "oauth.client_secret"))
|| (["config", "transportConfig"] as const).some(target => {
const before = asRecord(expected[target].oauth);
const after = asRecord(current[target].oauth);
return ["clientId", "clientRegistrationSource", "clientTokenEndpointAuthMethod"].some(key => before[key] !== after[key])
|| Object.entries(bindings).some(([key, value]) => before[key] !== after[key]
&& !(before[key] == null && (after[key] == null || after[key] === value)));
});
if (changedSetup || changedClient) throw conflict("Connection sign-in settings changed. Reload before continuing");
}
export function createOAuthClientRegistration(db: Db, { now, options, store, getConnectionRow, requestOAuthClientRegistration }: RegistrationDependencies) {
const { createOrRotateOAuthSecret, oauthClientForConnection, syncCredentialBindings } = store;
async function registerOAuthClient(input: {
@@ -300,46 +328,55 @@ async function stampOAuthClientBinding(
endpoints: OAuthProviderEndpoints,
redirectUri: string,
): Promise<typeof toolConnections.$inferSelect> {
const originalOAuth = oauthConfig(connection);
const oauth = reviewedOAuthClientBinding(connection, endpoints);
const nextBinding = {
clientRedirectUri: redirectUri,
clientIssuer:
typeof oauth.clientIssuer === "string" && oauth.clientIssuer
? oauth.clientIssuer
: (endpoints.issuer ?? null),
clientResource:
typeof oauth.clientResource === "string" && oauth.clientResource
? oauth.clientResource
: (endpoints.resource ?? null),
clientCompanyId:
typeof oauth.clientCompanyId === "string" && oauth.clientCompanyId
? oauth.clientCompanyId
: connection.companyId,
};
const unchanged = Object.entries(nextBinding).every(
([key, value]) => originalOAuth[key] === value,
);
if (unchanged) return connection;
const nextConfig = {
...connection.config,
oauth: { ...oauth, ...nextBinding },
};
const [updated] = await db
.update(toolConnections)
.set({
config: nextConfig,
transportConfig: nextConfig,
updatedAt: now(),
})
.where(
and(
eq(toolConnections.id, connection.id),
eq(toolConnections.companyId, connection.companyId),
),
)
.returning();
return updated ?? connection;
return db.transaction(async tx => {
const [saved] = await tx.select().from(toolConnections).where(and(
eq(toolConnections.id, connection.id), eq(toolConnections.companyId, connection.companyId),
)).for("no key update");
if (!saved) throw notFound("Tool connection not found");
assertOAuthClientBindingUnchanged(connection, saved, endpoints, redirectUri);
const originalOAuth = oauthConfig(saved);
const oauth = reviewedOAuthClientBinding(saved, endpoints);
const nextBinding = {
clientRedirectUri: redirectUri,
clientIssuer:
typeof oauth.clientIssuer === "string" && oauth.clientIssuer
? oauth.clientIssuer
: (endpoints.issuer ?? null),
clientResource:
typeof oauth.clientResource === "string" && oauth.clientResource
? oauth.clientResource
: (endpoints.resource ?? null),
clientCompanyId:
typeof oauth.clientCompanyId === "string" && oauth.clientCompanyId
? oauth.clientCompanyId
: saved.companyId,
};
const unchanged = Object.entries(nextBinding).every(
([key, value]) => originalOAuth[key] === value,
);
if (unchanged) return saved;
const nextConfig = {
...saved.config,
oauth: { ...oauth, ...nextBinding },
};
const nextTransportConfig = { ...saved.transportConfig, oauth: { ...asRecord(saved.transportConfig.oauth), ...nextBinding } };
const [updated] = await tx
.update(toolConnections)
.set({
config: nextConfig,
transportConfig: nextTransportConfig,
updatedAt: now(),
})
.where(
and(
eq(toolConnections.id, connection.id),
eq(toolConnections.companyId, connection.companyId),
),
)
.returning();
if (!updated) throw notFound("Tool connection not found");
return updated;
});
}
/**
@@ -48,7 +48,7 @@ export function createToolProfileService(db: Db, now: () => Date) {
eq(toolProfileEntries.profileId, profile.id),
),
)
.orderBy(asc(toolProfileEntries.createdAt)),
.orderBy(asc(toolProfileEntries.createdAt), asc(toolProfileEntries.id)),
dbClient
.select()
.from(toolProfileBindings)
@@ -387,7 +387,7 @@ export function createToolProfileService(db: Db, now: () => Date) {
inArray(toolProfileEntries.profileId, profileIds),
),
)
.orderBy(asc(toolProfileEntries.createdAt)),
.orderBy(asc(toolProfileEntries.createdAt), asc(toolProfileEntries.id)),
db
.select()
.from(toolProfileBindings)
+48 -3
View File
@@ -1572,7 +1572,7 @@ describe("automatic retained workspace recovery", () => {
return { ...t, queued: current, wake: claimedWake, followup, expectedIds: ids, latestAuthor: kind === "deferred-different" ? "third-user" : kind === "active" ? "second-user" : "operator" };
}
async function exerciseQueuedReset(nativeDescriptor: boolean, coalesced = false, ordinary?: "active" | "deferred" | "deferred-different" | "system-deferred") {
async function exerciseQueuedReset(nativeDescriptor: boolean, coalesced = false, ordinary?: "active" | "deferred" | "deferred-different" | "system-deferred", earlierCandidates = false) {
const mixed = ordinary ? await ordinaryMixedQueuedFixture(ordinary) : null;
const t = mixed ?? await queuedResetFixture(nativeDescriptor);
const { f, original, reset, wake, queued, deps, control, execution, queueDeps, queue } = t;
@@ -1595,12 +1595,43 @@ describe("automatic retained workspace recovery", () => {
deps.activeRunExecutions.delete(f.runId); execution.finish(); adapterExecutionControls.delete(f.runId);
await control.resumeRemoteStopComments(await run(f.runId));
expect(readSystemRecovery((await action(f.actionId)).evidence)!.nextAttemptAt!).toBeLessThanOrEqual(Date.now());
const earlier: Awaited<ReturnType<typeof seed>>[] = [];
if (earlierCandidates) for (let index = 1; index <= 2; index++) {
const candidate = await seed("native", false, "keep_running");
const id = `00000000-0000-4000-8000-${String(index).padStart(12, "0")}`;
await db.update(issueRecoveryActions).set({ id }).where(and(eq(issueRecoveryActions.id, candidate.actionId), eq(issueRecoveryActions.companyId, candidate.companyId)));
candidate.actionId = id; candidate.input.actionId = id;
await db.update(environmentLeases).set({ status: "active", releasedAt: null, cleanupStatus: null, metadata: candidate.lease.metadata }).where(eq(environmentLeases.id, candidate.lease.id));
await requestWorkspaceRecovery({ ...candidate.input, idempotencyKey: `earlier-${index}`, actorId: "board" });
earlier.push(candidate);
}
const ownedCompanies = [f.companyId, ...earlier.map(candidate => candidate.companyId)];
const seededRuns = earlierCandidates ? await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(inArray(heartbeatRuns.companyId, ownedCompanies)) : [];
if (earlierCandidates) {
await adoptHistoricalWorkspaceRecoveries(db);
expect((await retainedWorkspaceCandidates(db, undefined, true)).slice(0, 2).map(row => row.action.id)).toEqual(earlier.map(candidate => candidate.actionId));
}
let ownedRecoveryObserved = false;
const onRecovered = vi.fn(async (current: Awaited<ReturnType<typeof run>>) => {
if (current.id !== f.runId) return;
if (nativeDescriptor) expect(await inspectWorkspaceSource(db, { companyId: f.companyId, issueId: f.issueId,
runId: f.runId, actionId: f.actionId })).toMatchObject({ status: "verified", scope: "recorded_source_identity" });
await Promise.all([control.resumeRemoteStopComments(current), control.resumeRemoteStopComments(current)]);
ownedRecoveryObserved = true;
});
const repair = async () => { await Promise.all(await reconcileRetainedWorkspaces(db, f.environmentRuntime, onRecovered) ?? []); };
const sweepSizes: number[] = [];
let firstSweepOwned: boolean | undefined;
const repair = async () => {
await vi.waitFor(async () => {
const work = await reconcileRetainedWorkspaces(db, f.environmentRuntime, onRecovered) ?? [];
sweepSizes.push(work.length);
await Promise.all(work);
firstSweepOwned ??= ownedRecoveryObserved;
expect(ownedRecoveryObserved).toBe(true);
expect(readSystemRecovery((await action(f.actionId)).evidence)?.state).toBe("complete");
}, { timeout: 10_000, interval: 10 });
expect(sweepSizes.every(size => size <= 2)).toBe(true);
};
if (!nativeDescriptor) await repair();
expect(await queue.startNextQueuedRunForAgent(original.agentId)).toEqual([]);
expect(await queue.claimQueuedRun(await run(queued.id))).toBeNull();
@@ -1629,9 +1660,16 @@ describe("automatic retained workspace recovery", () => {
}
expect(queueDeps.executeRun).not.toHaveBeenCalled();
if (nativeDescriptor) await repair();
expect(onRecovered).toHaveBeenCalled();
expect(onRecovered).toHaveBeenCalledWith(expect.objectContaining({ id: f.runId, companyId: f.companyId }));
expect(readSystemRecovery((await action(f.actionId)).evidence)?.state).toBe("complete");
if (nativeDescriptor) expect(await readFile(path.join(f.host, "saved.txt"), "utf8")).toBe("finished work\n");
if (earlierCandidates) {
expect(sweepSizes[0]).toBe(2);
expect(firstSweepOwned).toBe(false);
expect(sweepSizes.length).toBeGreaterThan(1);
expect((await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(inArray(heartbeatRuns.companyId, ownedCompanies))).map(row => row.id).sort()).toEqual(seededRuns.map(row => row.id).sort());
for (const candidate of [f, ...earlier]) expect(await readFile(path.join(candidate.remote, "saved.txt"), "utf8")).toBe("finished work\n");
}
await Promise.all([control.resumeRemoteStopComments(await run(f.runId)),
control.resumeRemoteStopComments(await run(f.runId))]);
const [afterWake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id));
@@ -1664,6 +1702,13 @@ describe("automatic retained workspace recovery", () => {
{ nativeDescriptor: false, title: "does not replay a snapshot-only source without a verified native baseline" },
])("$title", async ({ nativeDescriptor }) => exerciseQueuedReset(nativeDescriptor), 30_000);
it("reaches its queued snapshot-only reset after earlier recovery candidates without starting agent work", async () => {
const previous = db;
db = createDb(connectionString);
try { await exerciseQueuedReset(false, false, undefined, true); }
finally { await db.$client.end(); db = previous; }
}, 30_000);
it("preserves coalesced queued user comments until real native copyback replays the original wake", async () => {
await exerciseQueuedReset(true, true);
}, 30_000);