mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-02 08:28:48 +01:00
Merge qualified backup expiration parent into GitHub connection work
Co-Authored-By: Paperclip <noreply@paperclip.ing> * commit 'aae32f2f493848199f6cb8c0a9996af3ee0061b1': (30 commits) Scope status-card refresh tasks to their claim revision test(chat): retain completion identity before link metadata loads test(chat): locate completion through its rendered task link fix(chat): recover committed native Agent Chat replies (#18465) perf(heartbeat): drive retry promotion from durable deadlines (#18387) fix(ui): make the Coordinator composer footer opaque (#18455) fix(testops): verify unexpanded result job runs (#18462) test(server): observe restored Retry before draining unrelated work test(runner): own listener cleanup across setup failures test(runner): buffer upgrade frames until the wire subscribes fix(runner): preserve OpenCode written-answer defaults (#18456) fix(native): fence retries when replaying an audited Stop (#18457) test: await async outcomes and drive saved deadlines deterministically (#18416) fix(ui): retain startup autosave through session recovery (#18412) perf(server): recover independent native replies concurrently (#18421) perf(chat): scope task questions and batch inbox queue checks (#18422) fix(native): recover controllers across webserver replacement (#18343) fix(ui): restore task document authoring and add persistence journey (#18356) fix(tools): stabilize tied profile entry order (#18427) fix(adapter-utils): tolerate entries removed during workspace snapshots (#18426) ...
This commit is contained in:
commit
f522ab07e2
484 files changed
+33834
-1620
No files matched your search
@@ -229,7 +229,9 @@ describe("run failure diagnostics", () => {
|
||||
const result = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics(run({ errorCode: "adapter_failed" }), { error: thrown }));
|
||||
expect(result.execution).toEqual({ remoteRunnerRecoveryStage: "process_verification", remoteRunnerRecoveryReason: "process_not_running" });
|
||||
expect(result.exceptions).toMatchObject([{ code: "runner_remote_process_identity_unavailable" }]);
|
||||
expect(JSON.stringify(result)).not.toContain("private");
|
||||
expect(JSON.stringify(result)).not.toContain("private-path");
|
||||
expect(result.exceptions[0]).not.toHaveProperty("path");
|
||||
expect(result.exceptions[0]).not.toHaveProperty("pid");
|
||||
|
||||
const transport = attachRemoteRunnerRecoveryDiagnostic(new Error("socket closed"),
|
||||
{ stage: "identity_marker", reason: "probe_transport_failed" });
|
||||
|
||||
@@ -480,6 +480,10 @@ export function activityService(db: Db) {
|
||||
.where(
|
||||
and(
|
||||
eq(heartbeatRuns.companyId, companyId),
|
||||
sql`(not exists (select 1 from ${issues} where ${issues.id} = ${issueId}
|
||||
and ${issues.companyId} = ${companyId} and ${issues.conversationAgentId} is not null)
|
||||
or coalesce(${heartbeatRuns.nativeIssueId}::text, ${heartbeatRuns.issueId}::text,
|
||||
${heartbeatRuns.contextSnapshot} ->> 'issueId', ${heartbeatRuns.contextSnapshot} ->> 'taskId') = ${issueId})`,
|
||||
...(selectedRunIds ? [inArray(heartbeatRuns.id, selectedRunIds)] : []),
|
||||
or(
|
||||
eq(heartbeatRuns.issueId, issueId),
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { and, eq, isNull, sql } from "drizzle-orm";
|
||||
import { agentApiKeys, agents, companies, companyMemberships, heartbeatRuns, issueComments, issues, nativeRunResults, type Db } from "@paperclipai/db";
|
||||
import { agentApiKeyScopeSchema, isUuidLike, sendChatMessageSchema, type AgentApiKeyScope } from "@paperclipai/shared";
|
||||
import { badRequest, conflict, forbidden, notFound, tooManyRequests } from "../errors.js";
|
||||
import { agentRunWritesRevoked, assertAgentRunWriteAllowed } from "../agent-run-cancellation.js";
|
||||
import { authorizationService, canActorReadIssuePrivacy, type AuthorizationActor } from "./authorization.js";
|
||||
import { persistActivity, publishActivity, type ActivityPublication } from "./activity-log.js";
|
||||
import { instanceSettingsService } from "./instance-settings.js";
|
||||
import { issueService } from "./issues.js";
|
||||
import { issueTreeControlService } from "./issue-tree-control.js";
|
||||
import { resolveActorSourceTrustForIssue } from "./source-trust.js";
|
||||
import { resolveTaskWatchdogMutationScope } from "./task-watchdog-scope.js";
|
||||
import { readNativeReviewAssignmentContext } from "./native-runtime/native-review-participant.js";
|
||||
import { nativeRunKeyScope } from "./native-runtime/native-skill-test-scope.js";
|
||||
import { crossIssueInfluenceLimitError, observeCrossIssueInfluence } from "./cross-issue-influence-limit.js";
|
||||
|
||||
type Actor = { companyId: string; agentId: string; runId: string; agentApiKeyId?: string | null;
|
||||
keyScope?: AgentApiKeyScope | null; onBehalfOfUserId?: string | null;
|
||||
authorizingUserId?: string | null; source?: AuthorizationActor["source"] };
|
||||
export interface ChatMessageReceipt {
|
||||
commentId: string;
|
||||
conversationId: string;
|
||||
chatUrl: string;
|
||||
sourceTaskId: string;
|
||||
replayed: boolean;
|
||||
}
|
||||
|
||||
function runTask(run: typeof heartbeatRuns.$inferSelect) {
|
||||
return run.nativeIssueId ?? run.issueId ?? run.contextSnapshot?.issueId ?? run.contextSnapshot?.taskId;
|
||||
}
|
||||
|
||||
export async function sendAgentChatMessage(
|
||||
db: Db,
|
||||
actor: Actor,
|
||||
value: unknown,
|
||||
options: { authorizeNative?: (tx: Db) => Promise<unknown>; receiptOnly?: boolean } = {},
|
||||
): Promise<ChatMessageReceipt | null> {
|
||||
const parsed = sendChatMessageSchema.safeParse(value);
|
||||
if (!parsed.success) throw badRequest("Chat messages require body and idempotencyKey only", parsed.error.issues);
|
||||
if (![actor.companyId, actor.agentId, actor.runId].every(isUuidLike)) throw forbidden("An authenticated task run is required");
|
||||
const credentialScope = agentApiKeyScopeSchema.safeParse(actor.keyScope ?? { kind: "standard" });
|
||||
if (!credentialScope.success) throw forbidden("The agent credential scope is invalid");
|
||||
if (credentialScope.data.kind !== "standard") throw forbidden("Scoped test and task bridge credentials cannot publish agent chat messages");
|
||||
const input = parsed.data;
|
||||
const contentHash = createHash("sha256").update(input.body).digest("hex");
|
||||
const publications: ActivityPublication[] = [];
|
||||
const result = await db.transaction(async transaction => {
|
||||
const tx = transaction as unknown as Db;
|
||||
await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${`issue-privacy-tree:${actor.companyId}`}, 0))`);
|
||||
const runWhere = and(eq(heartbeatRuns.id, actor.runId), eq(heartbeatRuns.companyId, actor.companyId), eq(heartbeatRuns.agentId, actor.agentId));
|
||||
const [initial] = await tx.select().from(heartbeatRuns).where(runWhere);
|
||||
const sourceId = initial && runTask(initial);
|
||||
if (typeof sourceId !== "string" || !isUuidLike(sourceId)) throw forbidden("Chat delivery requires a task-bound run");
|
||||
const sourceQuery = () => tx.select().from(issues).where(and(eq(issues.id, sourceId), eq(issues.companyId, actor.companyId)));
|
||||
const [initialSource] = await sourceQuery();
|
||||
if (!initialSource) throw forbidden("This run no longer owns the source task");
|
||||
let source = initialSource;
|
||||
const contextQuery = () => tx.select({ run: heartbeatRuns, agent: agents }).from(heartbeatRuns)
|
||||
.innerJoin(agents, and(eq(agents.id, heartbeatRuns.agentId), eq(agents.companyId, heartbeatRuns.companyId)))
|
||||
.where(runWhere);
|
||||
const validateContext = (context: { run: typeof heartbeatRuns.$inferSelect; agent: typeof agents.$inferSelect } | undefined) => {
|
||||
if (!context || context.run.status !== "running" || agentRunWritesRevoked(context.run)
|
||||
|| runTask(context.run) !== sourceId || source.assigneeAgentId !== actor.agentId
|
||||
|| source.executionRunId !== actor.runId || (source.checkoutRunId && source.checkoutRunId !== actor.runId)
|
||||
|| ["paused", "terminated", "pending_approval", "error"].includes(context.agent.status)) {
|
||||
throw forbidden("This run no longer owns the source task");
|
||||
}
|
||||
if (source.conversationAgentId) throw forbidden("Reply normally in this conversation; send_chat_message is for execution tasks");
|
||||
if (nativeRunKeyScope({ contextSnapshot: context.run.contextSnapshot, issueId: source.id, workMode: source.workMode }).kind === "skill_test") {
|
||||
throw forbidden("Skill-test runs cannot publish agent chat messages");
|
||||
}
|
||||
if (source.workMode !== "standard" || ["done", "cancelled"].includes(source.status)
|
||||
|| readNativeReviewAssignmentContext(context.run.contextSnapshot)) {
|
||||
throw forbidden("Chat delivery requires an ordinary task in standard execution mode");
|
||||
}
|
||||
return context;
|
||||
};
|
||||
const [initialContext] = await contextQuery();
|
||||
const context = validateContext(initialContext);
|
||||
if (!(await instanceSettingsService(tx).getExperimental()).enableAgentChat) throw notFound("Agent Chat is disabled");
|
||||
const userId = context.run.responsibleUserId;
|
||||
if (!userId) throw forbidden("This task has no responsible human to receive the message");
|
||||
if (actor.onBehalfOfUserId !== undefined && actor.onBehalfOfUserId !== userId) throw forbidden("The authenticated responsible human no longer matches this run");
|
||||
const assertCurrentKey = async (lock: boolean) => {
|
||||
if (!actor.agentApiKeyId) return;
|
||||
const query = tx.select().from(agentApiKeys).where(and(eq(agentApiKeys.id, actor.agentApiKeyId),
|
||||
eq(agentApiKeys.companyId, actor.companyId), eq(agentApiKeys.agentId, actor.agentId), isNull(agentApiKeys.revokedAt)));
|
||||
const [key] = lock ? await query.for("share") : await query;
|
||||
const currentScope = key && agentApiKeyScopeSchema.safeParse(key.scopeConfig ?? { kind: "standard" });
|
||||
if (!key || !currentScope?.success || JSON.stringify(currentScope.data) !== JSON.stringify(credentialScope.data)
|
||||
|| (key.responsibleUserId?.trim() || null) !== (actor.onBehalfOfUserId ?? userId)) {
|
||||
throw forbidden("The agent key was revoked or its authority changed");
|
||||
}
|
||||
};
|
||||
await assertCurrentKey(false);
|
||||
const [membership] = await tx.select({ id: companyMemberships.id }).from(companyMemberships).where(and(
|
||||
eq(companyMemberships.companyId, actor.companyId), eq(companyMemberships.principalType, "user"),
|
||||
eq(companyMemberships.principalId, userId), eq(companyMemberships.status, "active"),
|
||||
)).for("share");
|
||||
if (!membership) throw forbidden("The responsible human is no longer an active company member");
|
||||
const authActor: AuthorizationActor = { type: "agent", ...actor, keyId: actor.agentApiKeyId, keyScope: credentialScope.data,
|
||||
source: actor.source ?? "agent_jwt", onBehalfOfUserId: actor.onBehalfOfUserId ?? userId };
|
||||
const authorize = async (issueId: string, action: "issue:read" | "issue:comment") => {
|
||||
const decision = await authorizationService(tx).decide({ actor: authActor, action,
|
||||
resource: { type: "issue", companyId: actor.companyId, issueId } });
|
||||
if (!decision.allowed) throw forbidden(decision.explanation);
|
||||
};
|
||||
await authorize(source.id, "issue:read");
|
||||
if (!(await canActorReadIssuePrivacy(tx, { type: "none" }, source))) {
|
||||
throw forbidden("Restricted task content cannot be sent to a company-visible chat");
|
||||
}
|
||||
const svc = issueService(tx);
|
||||
let conversation = await svc.getConversation(actor.companyId, actor.agentId, userId);
|
||||
if (!conversation && !options.receiptOnly) {
|
||||
conversation = await svc.create(actor.companyId, {
|
||||
title: `Chat with ${context.agent.name}`, assigneeAgentId: actor.agentId,
|
||||
conversationAgentId: actor.agentId, conversationUserId: userId,
|
||||
conversationState: "waiting", status: "in_review", createdByAgentId: actor.agentId,
|
||||
responsibleUserId: userId, trustExplicitResponsibleUserId: true,
|
||||
});
|
||||
const opened = await persistActivity(tx, { companyId: actor.companyId, actorType: "agent", actorId: actor.agentId,
|
||||
agentId: actor.agentId, runId: actor.runId, action: "issue.conversation_opened", entityType: "issue", entityId: conversation.id,
|
||||
details: { agentId: actor.agentId, conversationUserId: userId } });
|
||||
publications.push(opened.publication);
|
||||
}
|
||||
if (conversation) await tx.select({ id: issues.id }).from(issues).where(and(
|
||||
eq(issues.id, conversation.id), eq(issues.companyId, actor.companyId),
|
||||
)).for("update");
|
||||
const [lockedSource] = await sourceQuery().for("update");
|
||||
if (!lockedSource) throw forbidden("This run no longer owns the source task");
|
||||
source = lockedSource;
|
||||
if (options.authorizeNative) await options.authorizeNative(tx);
|
||||
const [lockedContext] = await contextQuery().for("update");
|
||||
const current = validateContext(lockedContext);
|
||||
if (current.run.responsibleUserId !== userId) throw forbidden("The responsible human changed before this message was saved");
|
||||
await assertCurrentKey(true);
|
||||
if ((await resolveTaskWatchdogMutationScope(tx, { ...actor, type: "agent" })).kind !== "none") {
|
||||
throw forbidden("Chat delivery requires an ordinary task in standard execution mode");
|
||||
}
|
||||
await assertAgentRunWriteAllowed(tx, actor.companyId, actor);
|
||||
if (await issueTreeControlService(tx).getActivePauseHoldGate(actor.companyId, source.id)) throw forbidden("This task is paused");
|
||||
const [finished] = await tx.select({ id: nativeRunResults.id }).from(nativeRunResults).where(and(
|
||||
eq(nativeRunResults.companyId, actor.companyId), eq(nativeRunResults.runId, actor.runId), eq(nativeRunResults.schemaStatus, "accepted"),
|
||||
)).limit(1);
|
||||
if (finished) throw forbidden("Send the message before finishing the run");
|
||||
if (!(await instanceSettingsService(tx).getExperimental()).enableAgentChat) throw notFound("Agent Chat is disabled");
|
||||
await authorize(source.id, "issue:read");
|
||||
if (!conversation) return null;
|
||||
await authorize(conversation.id, "issue:comment");
|
||||
const recipientDecision = await authorizationService(tx).decide({ actor: { type: "board", userId, source: "session", ignoreInstanceAdmin: true },
|
||||
action: "issue:read", resource: { type: "issue", companyId: actor.companyId, issueId: conversation.id } });
|
||||
if (!recipientDecision.allowed) throw forbidden("The responsible human cannot access this chat");
|
||||
const requestId = `chat:${source.id}:${createHash("sha256").update(input.idempotencyKey).digest("hex")}`;
|
||||
const [prior] = await tx.select().from(issueComments).where(and(eq(issueComments.companyId, actor.companyId),
|
||||
eq(issueComments.issueId, conversation.id), eq(issueComments.authorAgentId, actor.agentId), eq(issueComments.clientRequestId, requestId)));
|
||||
const [company] = await tx.select({ prefix: companies.issuePrefix }).from(companies).where(eq(companies.id, actor.companyId));
|
||||
const receipt = (commentId: string, replayed: boolean): ChatMessageReceipt => ({ commentId, conversationId: conversation!.id,
|
||||
chatUrl: `/${company.prefix}/chats/${actor.agentId}`, sourceTaskId: source.id, replayed });
|
||||
if (prior) {
|
||||
if (prior.metadata?.chatMessage?.contentHash !== contentHash) {
|
||||
if (options.receiptOnly) return null;
|
||||
throw conflict("Chat message retry key was already used with different content");
|
||||
}
|
||||
return receipt(prior.id, true);
|
||||
}
|
||||
if (options.receiptOnly) return null;
|
||||
const influence = await observeCrossIssueInfluence(tx, { companyId: actor.companyId, agentId: actor.agentId,
|
||||
runId: actor.runId, responsibleUserId: userId, targetIssueId: conversation.id,
|
||||
targetIssueIdentifier: conversation.identifier, kind: "comment" });
|
||||
if (influence && !influence.allowed) {
|
||||
const denial = crossIssueInfluenceLimitError(influence, { actorLabel: current.agent.name,
|
||||
assigneeLabel: current.agent.name, issueIdentifier: conversation.identifier });
|
||||
throw tooManyRequests(denial.error, denial.details);
|
||||
}
|
||||
const sourceTrust = await resolveActorSourceTrustForIssue({ db: tx, issue: source,
|
||||
actor: { actorType: "agent", actorId: actor.agentId, agentId: actor.agentId, runId: actor.runId } });
|
||||
const comment = await svc.addComment(conversation.id, input.body, { agentId: actor.agentId, runId: actor.runId }, {
|
||||
clientRequestId: requestId, sourceTrust, proactiveChatMessage: { sourceTaskId: source.id, userId },
|
||||
metadata: { version: 1, chatMessage: { sourceTaskId: source.id, sourceTaskIdentifier: source.identifier!, sourceRunId: actor.runId, contentHash },
|
||||
sections: [{ rows: [{ type: "issue_link", issueId: source.id, identifier: source.identifier, label: "Source task" }] }] },
|
||||
}, tx);
|
||||
const activity = await persistActivity(tx, { companyId: actor.companyId, actorType: "agent", actorId: actor.agentId,
|
||||
agentId: actor.agentId, runId: actor.runId, agentApiKeyId: actor.agentApiKeyId,
|
||||
action: "issue.comment_added", entityType: "issue", entityId: conversation.id,
|
||||
details: { commentId: comment.id, identifier: conversation.identifier, conversationUserId: userId,
|
||||
conversationAgentId: actor.agentId, sourceTaskId: source.id } });
|
||||
publications.push(activity.publication);
|
||||
return receipt(comment.id, false);
|
||||
});
|
||||
for (const publication of publications) publishActivity(publication);
|
||||
return result;
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
import { and, eq, inArray, sql } from "drizzle-orm";
|
||||
import { issueComments, issueReadStates, type Db } from "@paperclipai/db";
|
||||
|
||||
export async function withConversationReadState<T extends { id: string }>(db: Db, companyId: string, userId: string, conversations: T[]) {
|
||||
if (!conversations.length) return [] as Array<T & { isUnreadForMe: boolean; lastExternalCommentAt: Date | null }>;
|
||||
const rows = await db.select({
|
||||
issueId: issueComments.issueId,
|
||||
latest: sql<Date>`max(${issueComments.createdAt})`,
|
||||
unread: sql<boolean>`bool_or(${issueComments.createdAt} > coalesce(${issueReadStates.lastReadAt}, to_timestamp(0)))`,
|
||||
}).from(issueComments).leftJoin(issueReadStates, and(
|
||||
eq(issueReadStates.companyId, companyId), eq(issueReadStates.issueId, issueComments.issueId), eq(issueReadStates.userId, userId),
|
||||
)).where(and(eq(issueComments.companyId, companyId), inArray(issueComments.issueId, conversations.map(chat => chat.id)),
|
||||
sql`${issueComments.deletedAt} is null`, sql`${issueComments.authorAgentId} is not null`,
|
||||
sql`${issueComments.authorType} is distinct from 'system'`,
|
||||
sql`${issueComments.presentation} ->> 'kind' is distinct from 'system_notice'`,
|
||||
eq(issueComments.origin, "comment"),
|
||||
)).groupBy(issueComments.issueId);
|
||||
const byId = new Map(rows.map(row => [row.issueId, row]));
|
||||
return conversations.map(chat => ({ ...chat, isUnreadForMe: byId.get(chat.id)?.unread ?? false,
|
||||
lastExternalCommentAt: byId.get(chat.id)?.latest ?? null }));
|
||||
}
|
||||
@@ -530,6 +530,9 @@ export async function undeliveredConversationComments(
|
||||
eq(issueComments.companyId, companyId),
|
||||
eq(issueComments.issueId, issueId),
|
||||
isNull(issueComments.deletedAt),
|
||||
sql`${issueComments.authorUserId} is not null`,
|
||||
isNull(issueComments.authorAgentId),
|
||||
eq(issueComments.origin, "comment"),
|
||||
sql`${issueComments.clientRequestId} is not null`,
|
||||
sql`not exists (select 1 from ${agentWakeupRequests} where ${agentWakeupRequests.companyId} = ${companyId}
|
||||
and ${agentWakeupRequests.idempotencyKey} = 'conversation-comment:' || ${issueComments.id}::text)`,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { notifyInstructionCollection } from "./instruction-collection-notifications.js";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { isDeepStrictEqual } from "node:util";
|
||||
import { and, eq, inArray, isNull, sql } from "drizzle-orm";
|
||||
import { agents, environmentLeases, environments, agentInstructionWorkingCopies as copies, type Db } from "@paperclipai/db";
|
||||
import { syncDirectoryToSsh, restoreWorkspaceFromSshExecution } from "@paperclipai/adapter-utils/ssh";
|
||||
@@ -20,6 +21,11 @@ import { cachedAgentFileManifest, captureAgentFileCheckpoint, checkpointBaseline
|
||||
import { logger } from "../middleware/logger.js";
|
||||
|
||||
type Copy = typeof copies.$inferSelect;
|
||||
function adoptionAuthority(row: Copy) {
|
||||
const { updatedAt: _updatedAt, nextAttemptAt: _nextAttemptAt, ...authority } = row;
|
||||
return authority;
|
||||
}
|
||||
|
||||
export class AgentDirectoryReuseInvalidatedError extends Error {}
|
||||
const completed = new Set(["saved", "unchanged", "resolved", "unavailable"]);
|
||||
const transports = new Map<string, PreparedAdapterExecutionTargetRuntime>();
|
||||
@@ -291,10 +297,22 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
|
||||
// adopt_remote constructs a collector on the successor target without
|
||||
// uploading the stale host mirror over the retained remote directory.
|
||||
const replacement = remote ? await transport({ ...prior, runId: input.runId }, remote, true) : undefined;
|
||||
const expectedPrior = prior;
|
||||
let adopted: Copy | null;
|
||||
try { adopted = await store.locked(input.companyId, input.agentId, bound, false, async (tx, _agent, canonical) => {
|
||||
if (remote && !validLeases(await tx.select().from(environmentLeases).where(and(
|
||||
eq(environmentLeases.companyId, input.companyId), inArray(environmentLeases.id, leaseIds))).for("update"))) return null;
|
||||
const [lockedPrior] = await tx.select().from(copies).where(and(eq(copies.companyId, input.companyId),
|
||||
eq(copies.runId, expectedPrior.runId), eq(copies.agentId, input.agentId))).for("update");
|
||||
if (!lockedPrior || !isDeepStrictEqual(adoptionAuthority(lockedPrior), adoptionAuthority(expectedPrior))) {
|
||||
const fields = ["state", "receipt", "baseHash", "localRoot", "executionRoot", "location", "processStoppedAt", "responsibleUserId", "entryFile", "attempts", "candidateHash", "errorCode", "errorMessage", "baseRevisionId", "candidateBase64", "createdAt"] as const;
|
||||
const diagnostic = { stage: "locked_adoption_authority", predecessorPresent: Boolean(lockedPrior), observedAfterInitialRead: true,
|
||||
changedFields: lockedPrior ? fields.filter(field => !isDeepStrictEqual(lockedPrior[field], expectedPrior[field])) : [] };
|
||||
logger.warn({ companyId: input.companyId, agentId: input.agentId, runId: input.runId, previousRunId: input.previousRunId,
|
||||
adoptionConflict: diagnostic }, "Agent directory adoption conflict");
|
||||
throw conflict("Agent directory ownership changed", diagnostic);
|
||||
}
|
||||
prior = lockedPrior;
|
||||
const current = await inspectAgentDirectory(canonical);
|
||||
const canonicalChanged = agentDirectoryBaselineDigest(current.snapshot) !== agentDirectoryBaselineDigest(baseline(prior));
|
||||
if (canonicalChanged && !input.allowRetirementHandoff) return null;
|
||||
@@ -304,7 +322,18 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
|
||||
.where(and(eq(copies.companyId, input.companyId), eq(copies.runId, prior.runId), eq(copies.agentId, input.agentId), eq(copies.state, "unchanged_turn"),
|
||||
eq(copies.updatedAt, prior.updatedAt), eq(copies.baseHash, prior.baseHash), eq(copies.localRoot, prior.localRoot),
|
||||
isNull(copies.processStoppedAt), sql`${copies.receipt}->>'retainedByRunId' IS NULL`)).returning();
|
||||
if (!claimed) throw conflict("Agent directory ownership changed");
|
||||
if (!claimed) {
|
||||
const [conditions] = await tx.select({ unchangedTurn: eq(copies.state, "unchanged_turn"),
|
||||
timestampMatches: eq(copies.updatedAt, prior.updatedAt), baselineMatches: eq(copies.baseHash, prior.baseHash),
|
||||
localRootMatches: eq(copies.localRoot, prior.localRoot), unstopped: isNull(copies.processStoppedAt),
|
||||
successorAbsent: sql<boolean>`${copies.receipt}->>'retainedByRunId' IS NULL`,
|
||||
receiptMatches: sql<boolean>`${copies.receipt} IS NOT DISTINCT FROM ${JSON.stringify(prior.receipt)}::jsonb`,
|
||||
}).from(copies).where(and(eq(copies.companyId, input.companyId), eq(copies.runId, prior.runId), eq(copies.agentId, input.agentId)));
|
||||
const diagnostic = { stage: "adoption_claim", observedAfterClaim: true, predecessorPresent: Boolean(conditions), conditions: conditions ?? null };
|
||||
logger.warn({ companyId: input.companyId, agentId: input.agentId, runId: input.runId, previousRunId: input.previousRunId,
|
||||
adoptionConflict: diagnostic }, "Agent directory adoption conflict");
|
||||
throw conflict("Agent directory ownership changed", diagnostic);
|
||||
}
|
||||
const { createdAt: _createdAt, updatedAt: _updatedAt, ...values } = prior;
|
||||
const [next] = await tx.insert(copies).values({ ...values, runId: input.runId, responsibleUserId: bound.onBehalfOfUserId!, state: "prepared",
|
||||
receipt: { ...prior.receipt, materializationRunId: prior.receipt?.materializationRunId ?? prior.runId,
|
||||
|
||||
@@ -1080,8 +1080,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
|
||||
Math.max(Math.trunc(serviceOptions.openDecisionLimit ?? OPEN_DECISION_DEFAULT_LIMIT), 1),
|
||||
OPEN_DECISION_MAX_LIMIT,
|
||||
);
|
||||
const service = {
|
||||
list: async (companyId: string, options: AttentionListOptions = {}): Promise<AttentionFeed> => {
|
||||
async function list(companyId: string, options: AttentionListOptions = {}, questionIssueIds?: readonly string[]): Promise<AttentionFeed> {
|
||||
if (options.all && !options.queue && !options.allowUnscopedAll) {
|
||||
throw badRequest("all requires a queue filter");
|
||||
}
|
||||
@@ -1105,7 +1104,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
|
||||
collected.push({ ...item, dismissal });
|
||||
};
|
||||
|
||||
if (!options.excludeSources?.includes("approval")) {
|
||||
if (!questionIssueIds && !options.excludeSources?.includes("approval")) {
|
||||
const pendingApprovals = await db
|
||||
.select({
|
||||
id: approvals.id,
|
||||
@@ -1196,6 +1195,8 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
|
||||
.where(and(
|
||||
eq(issueThreadInteractions.companyId, companyId),
|
||||
inArray(issueThreadInteractions.status, [...PENDING_INTERACTION_STATUSES]),
|
||||
questionIssueIds ? inArray(issueThreadInteractions.issueId, [...questionIssueIds]) : undefined,
|
||||
questionIssueIds ? eq(issueThreadInteractions.kind, "ask_user_questions") : undefined,
|
||||
))
|
||||
.orderBy(desc(issueThreadInteractions.updatedAt), desc(issueThreadInteractions.id));
|
||||
// Addressee invokability needs the org graph; the audience line also needs
|
||||
@@ -1285,6 +1286,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
|
||||
}));
|
||||
}
|
||||
|
||||
if (!questionIssueIds) {
|
||||
const openDecisionQuery = db.select({
|
||||
id: decisions.id,
|
||||
bundleId: decisions.bundleId,
|
||||
@@ -1865,6 +1867,8 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
|
||||
}));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
const deduped = new Map<string, AttentionItem>();
|
||||
for (const item of collected) {
|
||||
const current = deduped.get(item.dedupKey);
|
||||
@@ -1972,18 +1976,22 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
|
||||
}
|
||||
}
|
||||
return { ...summary, nextCursor, items };
|
||||
},
|
||||
};
|
||||
}
|
||||
return {
|
||||
...service,
|
||||
list: (companyId: string, options: AttentionListOptions = {}) => list(companyId, options),
|
||||
taskQuestions: async (companyId: string, issueIds: readonly string[], options: { actor: AuthorizationActor; userId: string }): Promise<AttentionItem[]> => {
|
||||
if (issueIds.length === 0) return [];
|
||||
const feed = await list(companyId, { ...options, all: true, allowUnscopedAll: true }, issueIds);
|
||||
return feed.items;
|
||||
},
|
||||
summary: async (companyId: string, options: Pick<AttentionListOptions, "userId" | "actor"> = {}): Promise<AttentionSummary> => {
|
||||
const { items: _items, nextCursor: _cursor, ...summary } = await service.list(companyId, {
|
||||
const { items: _items, nextCursor: _cursor, ...summary } = await list(companyId, {
|
||||
...options, all: true, allowUnscopedAll: true, summaryOnly: true,
|
||||
});
|
||||
return summary;
|
||||
},
|
||||
taskIndex: async (companyId: string, options: Pick<AttentionListOptions, "userId" | "actor"> = {}): Promise<AttentionTaskIndex> => {
|
||||
const { items, nextCursor: _cursor, ...summary } = await service.list(companyId, {
|
||||
const { items, nextCursor: _cursor, ...summary } = await list(companyId, {
|
||||
...options, all: true, allowUnscopedAll: true, summaryOnly: true,
|
||||
});
|
||||
return { ...summary, taskIds: [...new Set(items.flatMap(item => item.relatedIssue
|
||||
|
||||
@@ -6150,9 +6150,14 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo
|
||||
if (!versionId || (selectedKeys && !selectedKeys.has(key))) continue;
|
||||
const [version] = await db.select({ skillId: companySkillVersions.companySkillId }).from(companySkillVersions)
|
||||
.where(and(eq(companySkillVersions.companyId, companyId), eq(companySkillVersions.id, versionId)));
|
||||
skills = skills.filter(skill => skill.key !== key);
|
||||
// A saved JSONB selection map has a different key order from the fresh
|
||||
// name/key query. Replace existing rows in place so pinning versions does
|
||||
// not reorder the server-produced preference (and its session fingerprint).
|
||||
const index = skills.findIndex(skill => skill.key === key);
|
||||
if (index !== -1) skills.splice(index, 1);
|
||||
const owner = version ? await getById(companyId, version.skillId) : null;
|
||||
if (owner) skills.push({ ...owner, key, slug: owner.key === key ? owner.slug : key.split("/").at(-1)! });
|
||||
if (owner) skills.splice(index === -1 ? skills.length : index, 0,
|
||||
{ ...owner, key, slug: owner.key === key ? owner.slug : key.split("/").at(-1)! });
|
||||
}
|
||||
await packages.prefetch(companyId, skills.flatMap(skill => {
|
||||
const id = options.versionSelections?.get(skill.key) ?? skill.currentVersionId;
|
||||
|
||||
@@ -56,17 +56,7 @@ export function conversationDecisionReceiptsService(db: Db) {
|
||||
const taskIds = new Set([...handoffIds, ...directIds].filter(id => id !== conversationId));
|
||||
if (taskIds.size === 0) return [];
|
||||
|
||||
// One on-demand canonical company snapshot, not one query per task. The
|
||||
// attention owner applies current audience, privacy, dismissal, snooze and
|
||||
// archive policy, including its existing queue/retention bookkeeping.
|
||||
// Cost grows with the company feed, not only the returned task questions;
|
||||
// there is currently no task-scoped attention API. Do not poll this read.
|
||||
const feed = await attentionService(db).list(companyId, {
|
||||
all: true, allowUnscopedAll: true, actor: options.actor, userId: options.userId,
|
||||
});
|
||||
return feed.items.filter(item => item.sourceKind === "issue_thread_interaction"
|
||||
&& item.detail?.kind === "questions"
|
||||
&& item.relatedIssue !== null && taskIds.has(item.relatedIssue.id));
|
||||
return attentionService(db).taskQuestions(companyId, [...taskIds], options);
|
||||
},
|
||||
async listCreatedTasks(companyId: string, conversationId: string, options: {
|
||||
actor: AuthorizationActor; userId: string; cursor?: string; limit?: number;
|
||||
|
||||
@@ -456,34 +456,36 @@ export function decisionQueueService(db: Db) {
|
||||
});
|
||||
}
|
||||
let insertedAnyItem = false;
|
||||
for (const item of matchingItems) {
|
||||
const inserted = await txDb.insert(decisionQueueItems).values({
|
||||
for (let offset = 0; offset < matchingItems.length; offset += 500) {
|
||||
const inserted = await txDb.insert(decisionQueueItems).values(matchingItems.slice(offset, offset + 500).map(item => ({
|
||||
companyId,
|
||||
queueId: queue.id,
|
||||
sourceKind: item.sourceKind,
|
||||
sourceId: item.sourceId,
|
||||
addedByType: "system",
|
||||
}).onConflictDoNothing({
|
||||
}))).onConflictDoNothing({
|
||||
target: [decisionQueueItems.queueId, decisionQueueItems.sourceKind, decisionQueueItems.sourceId],
|
||||
}).returning();
|
||||
if (!inserted[0]) continue;
|
||||
if (inserted.length === 0) continue;
|
||||
insertedAnyItem = true;
|
||||
await txDb.insert(decisionTriageEvents).values({
|
||||
companyId,
|
||||
queueId: queue.id,
|
||||
sourceKind: item.sourceKind,
|
||||
sourceId: item.sourceId,
|
||||
action: "queue_item.seeded",
|
||||
...eventActorColumns(SYSTEM_ACTOR),
|
||||
details: { seedKey: seed.key },
|
||||
});
|
||||
await recordActivity(txDb, SYSTEM_ACTOR, {
|
||||
companyId,
|
||||
action: "decision_queue_item.seeded",
|
||||
entityType: "decision_queue",
|
||||
entityId: queue.id,
|
||||
details: { sourceKind: item.sourceKind, sourceId: item.sourceId, seedKey: seed.key },
|
||||
});
|
||||
for (const item of inserted) {
|
||||
await txDb.insert(decisionTriageEvents).values({
|
||||
companyId,
|
||||
queueId: queue.id,
|
||||
sourceKind: item.sourceKind,
|
||||
sourceId: item.sourceId,
|
||||
action: "queue_item.seeded",
|
||||
...eventActorColumns(SYSTEM_ACTOR),
|
||||
details: { seedKey: seed.key },
|
||||
});
|
||||
await recordActivity(txDb, SYSTEM_ACTOR, {
|
||||
companyId,
|
||||
action: "decision_queue_item.seeded",
|
||||
entityType: "decision_queue",
|
||||
entityId: queue.id,
|
||||
details: { sourceKind: item.sourceKind, sourceId: item.sourceId, seedKey: seed.key },
|
||||
});
|
||||
}
|
||||
}
|
||||
if (insertedAnyItem) {
|
||||
await txDb.update(decisionQueues).set({ updatedAt: new Date() })
|
||||
|
||||
@@ -2,6 +2,7 @@ import { registerDeferredReceiptWork } from "./heartbeat/deferred-receipt-work.j
|
||||
import { reconcileTaskChatRetryReceipts } from "./task-retry.js";
|
||||
import { registerIssueRecoveryWork } from "./recovery/work.js";
|
||||
import { registerRunDispatchWork } from "./heartbeat/dispatch-work.js";
|
||||
import { NATIVE_RETAINED_OWNER_WAIT } from "./native-runtime/native-controller-recovery.js";
|
||||
import { externalConversationSettlementWork } from "./external-conversation-settlement-work.js";
|
||||
import { nativeRecoveryNoticeWork } from "./native-runtime/native-recovery-notice-work.js";
|
||||
import { nativeRunProjectionWork } from "./native-runtime/native-run-projection-work.js";
|
||||
@@ -610,6 +611,9 @@ const nativeSessionResumeDispatchTimers = new Map<
|
||||
string,
|
||||
ReturnType<typeof setTimeout>
|
||||
>();
|
||||
// Routes and the scheduler can drain/resume through different service facades.
|
||||
// Rearm through the original owner so its database and dispatch mode survive.
|
||||
const nativeRecoveryDeadlines = new Map<string, { rearm: () => void }>();
|
||||
// Shared with HTTP admission so an idle hold fences work before inspection.
|
||||
export { computeTaskDrain, applyTaskDrain, startTaskDrain, stopTaskDrain } from "./task-admission.js";
|
||||
|
||||
@@ -1339,6 +1343,9 @@ export function heartbeatService(
|
||||
}
|
||||
|
||||
function resumeQueuedRunsAfterDrain(...args: Parameters<typeof heartbeatQueue.resumeQueuedRunsAfterDrain>) {
|
||||
for (const pending of nativeRecoveryDeadlines.values()) {
|
||||
pending.rearm();
|
||||
}
|
||||
const promise = heartbeatQueue.resumeQueuedRunsAfterDrain(...args);
|
||||
activeWakeupPromises.add(promise);
|
||||
void promise.then(
|
||||
@@ -1708,10 +1715,15 @@ export function heartbeatService(
|
||||
}
|
||||
}
|
||||
|
||||
// Native ownership waits survive a temporary drain without adding a polling
|
||||
// loop. A process restart rediscovers the same deadlines from coordinator rows.
|
||||
function scheduleNativeSessionResumeDispatch(
|
||||
runId: string,
|
||||
nextAttemptAt: Date,
|
||||
mode?: "restart",
|
||||
) {
|
||||
const pending = { rearm: () => scheduleNativeSessionResumeDispatch(runId, nextAttemptAt, mode) };
|
||||
nativeRecoveryDeadlines.set(runId, pending);
|
||||
const prior = nativeSessionResumeDispatchTimers.get(runId);
|
||||
if (prior) clearTimeout(prior);
|
||||
const delayMs = Math.max(0, nextAttemptAt.getTime() - Date.now());
|
||||
@@ -1720,6 +1732,12 @@ export function heartbeatService(
|
||||
nativeSessionResumeDispatchTimers.delete(runId);
|
||||
void (async () => {
|
||||
if ((await getSchedulingSuppression()).suppressed) return;
|
||||
if (nativeRecoveryDeadlines.get(runId) !== pending) return;
|
||||
nativeRecoveryDeadlines.delete(runId);
|
||||
if (mode === "restart") {
|
||||
await recoverNativeRunsAfterRestart(new Date(), [runId]);
|
||||
return;
|
||||
}
|
||||
await dispatchNativeSessionResumptions({
|
||||
db,
|
||||
runnerInstanceId:
|
||||
@@ -1741,6 +1759,9 @@ export function heartbeatService(
|
||||
},
|
||||
});
|
||||
})().catch((error) => {
|
||||
if (!nativeRecoveryDeadlines.has(runId)) {
|
||||
scheduleNativeSessionResumeDispatch(runId, new Date(Date.now() + 30_000), mode);
|
||||
}
|
||||
logger.error(
|
||||
{ err: error, runId },
|
||||
"failed to dispatch scheduled native session resume",
|
||||
@@ -3531,7 +3552,8 @@ export function heartbeatService(
|
||||
err.original instanceof Error
|
||||
? err.original.message
|
||||
: String(err.original ?? "");
|
||||
const retryReasonCode = /native_finalization_missing/i.test(
|
||||
const waitingForRetainedOwner = retryMessage === NATIVE_RETAINED_OWNER_WAIT;
|
||||
const retryReasonCode = waitingForRetainedOwner ? NATIVE_RETAINED_OWNER_WAIT : /native_finalization_missing/i.test(
|
||||
retryMessage,
|
||||
)
|
||||
? "semantic_result_missing"
|
||||
@@ -3551,7 +3573,8 @@ export function heartbeatService(
|
||||
stream: "system",
|
||||
level: "warn",
|
||||
message:
|
||||
retryReasonCode === "semantic_result_missing"
|
||||
waitingForRetainedOwner ? "native continuation waiting for the retained controller lease"
|
||||
: retryReasonCode === "semantic_result_missing"
|
||||
? "provider turn completed without a semantic result; same-run disposition recovery persisted"
|
||||
: "native session transport interrupted; same-run resume persisted",
|
||||
payload: {
|
||||
|
||||
@@ -49,7 +49,11 @@ export function bindHeartbeatRunnerHomeMode(config: Record<string, unknown>, per
|
||||
const saved = parseObject(persistedExecution);
|
||||
if (Object.keys(saved).length === 0) return config;
|
||||
// Execution schemas before v7 always used isolated homes.
|
||||
return { ...config, homeMode: saved.homeMode ?? "isolated" };
|
||||
const homeMode = saved.homeMode ?? "isolated";
|
||||
// Omission already means isolated. Recovery must not add a fingerprinted
|
||||
// configuration property merely to restate that unchanged default.
|
||||
if (homeMode === "isolated" && config.homeMode === undefined) return config;
|
||||
return { ...config, homeMode };
|
||||
}
|
||||
|
||||
export interface HeartbeatExecutionConfigurationInput {
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { heartbeatService, startTaskDrain, stopTaskDrain } from "../heartbeat.js";
|
||||
import type { HeartbeatRecoveryDependencies } from "./recovery.js";
|
||||
|
||||
const hooks = vi.hoisted(() => ({
|
||||
schedule: undefined as HeartbeatRecoveryDependencies["scheduleNativeSessionResumeDispatch"] | undefined,
|
||||
restart: vi.fn(async () => []),
|
||||
resume: vi.fn(async () => []),
|
||||
queue: vi.fn(async () => undefined),
|
||||
}));
|
||||
|
||||
vi.mock("./recovery.js", async original => {
|
||||
const module = await original<typeof import("./recovery.js")>();
|
||||
return { ...module, createHeartbeatRecovery: (db: Db, dependencies: HeartbeatRecoveryDependencies) => {
|
||||
hooks.schedule = dependencies.scheduleNativeSessionResumeDispatch;
|
||||
return { ...module.createHeartbeatRecovery(db, dependencies), recoverNativeRunsAfterRestart: hooks.restart };
|
||||
} };
|
||||
});
|
||||
vi.mock("./queue.js", async original => {
|
||||
const module = await original<typeof import("./queue.js")>();
|
||||
return { ...module, createHeartbeatQueue: (...args: Parameters<typeof module.createHeartbeatQueue>) => ({
|
||||
...module.createHeartbeatQueue(...args), resumeQueuedRunsAfterDrain: hooks.queue,
|
||||
}) };
|
||||
});
|
||||
vi.mock("../native-runtime/index.js", async original => ({
|
||||
...await original<typeof import("../native-runtime/index.js")>(),
|
||||
dispatchNativeSessionResumptions: hooks.resume,
|
||||
}));
|
||||
|
||||
describe("native deadline drain and resume", () => {
|
||||
let heartbeat: ReturnType<typeof heartbeatService>;
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
vi.useFakeTimers();
|
||||
stopTaskDrain();
|
||||
const db = new Proxy({}, { get: () => { throw new Error("Unexpected deadline database access"); } }) as Db;
|
||||
heartbeat = heartbeatService(db, { runtimeEnv: {} });
|
||||
});
|
||||
afterEach(async () => {
|
||||
stopTaskDrain();
|
||||
await heartbeat?.resumeQueuedRunsAfterDrain();
|
||||
await vi.runOnlyPendingTimersAsync();
|
||||
await heartbeat?.drainActiveRunExecutions();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it.each([undefined, "restart"] as const)("retains an overdue %s wait through drain with its original dispatch mode", async mode => {
|
||||
const deadline = new Date(Date.now() + 100);
|
||||
startTaskDrain({});
|
||||
hooks.schedule!("waiting-run", deadline, mode);
|
||||
await vi.advanceTimersByTimeAsync(100);
|
||||
expect(hooks.restart).not.toHaveBeenCalled();
|
||||
expect(hooks.resume).not.toHaveBeenCalled();
|
||||
stopTaskDrain();
|
||||
await heartbeat.resumeQueuedRunsAfterDrain();
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
const called = mode === "restart" ? hooks.restart : hooks.resume;
|
||||
const other = mode === "restart" ? hooks.resume : hooks.restart;
|
||||
expect(called).toHaveBeenCalledOnce();
|
||||
expect(other).not.toHaveBeenCalled();
|
||||
await heartbeat.resumeQueuedRunsAfterDrain();
|
||||
await vi.advanceTimersByTimeAsync(60_000);
|
||||
expect(called).toHaveBeenCalledOnce();
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
});
|
||||
|
||||
it("preserves a future follow-up deadline when draining cancels its timer", async () => {
|
||||
const deadline = new Date(Date.now() + 100);
|
||||
hooks.schedule!("future-run", deadline);
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
await heartbeat.resumeQueuedRunsAfterDrain();
|
||||
await vi.advanceTimersByTimeAsync(99);
|
||||
expect(hooks.resume).not.toHaveBeenCalled();
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
expect(hooks.resume).toHaveBeenCalledOnce();
|
||||
expect(hooks.restart).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rearms the original owner when another service facade ends the drain", async () => {
|
||||
hooks.schedule!("other-facade-run", new Date(Date.now() + 100));
|
||||
const db = new Proxy({}, { get: () => { throw new Error("Unexpected replacement facade database access"); } }) as Db;
|
||||
const other = heartbeatService(db, { runtimeEnv: {} });
|
||||
await other.drainActiveRunExecutions();
|
||||
await vi.advanceTimersByTimeAsync(100);
|
||||
await other.resumeQueuedRunsAfterDrain();
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(hooks.resume).toHaveBeenCalledOnce();
|
||||
expect(hooks.restart).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("retries a failed follow-up obligation without changing its dispatch mode", async () => {
|
||||
hooks.resume.mockRejectedValueOnce(new Error("temporary database failure"));
|
||||
hooks.schedule!("failed-follow-up", new Date());
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(hooks.resume).toHaveBeenCalledOnce();
|
||||
await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(hooks.resume).toHaveBeenCalledTimes(2);
|
||||
expect(hooks.restart).not.toHaveBeenCalled();
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,8 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { spawn } from "node:child_process";
|
||||
import { once } from "node:events";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { activityLog, agents, applyPendingMigrations, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, environments, environmentLeases, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, nativeRunnerCheckpoints } from "@paperclipai/db";
|
||||
import { activityLog, agents, applyPendingMigrations, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, environments, environmentLeases, heartbeatRunEvents, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, nativeRunnerCheckpoints } from "@paperclipai/db";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
|
||||
@@ -19,7 +21,10 @@ import { logger } from "../../middleware/logger.js";
|
||||
import { listExpiredNativeRunnerAuthorities } from "../native-runtime/native-runner-checkpoint.js";
|
||||
import { DELIVERY_QUEUES, subscribeDeliveryWork } from "../delivery-work-notifications.js";
|
||||
import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js";
|
||||
import { hasRemoteTerminationReceipt } from "../remote-execution-termination.js";
|
||||
import { hasRemoteTerminationReceipt, remoteTerminationReceipt } from "../remote-execution-termination.js";
|
||||
import { persistHeartbeatRunProcessMetadata } from "./run-lifecycle.js";
|
||||
import { appendHeartbeatRunEvent } from "../heartbeat-run-events.js";
|
||||
import { hasNativeLocalProcessStop, PROCESS_START_REQUESTED, releaseSettledNativeRunProcess } from "../native-local-process-stop.js";
|
||||
|
||||
type Run = typeof heartbeatRuns.$inferSelect;
|
||||
|
||||
@@ -202,6 +207,302 @@ describePostgres("heartbeat recovery module ownership", () => {
|
||||
return (await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, issueId, invocationSource: "automation", status: "running", runtimeMode: "legacy" }).returning())[0];
|
||||
}
|
||||
|
||||
async function ownedProcess() {
|
||||
const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)", `paperclip-recovery-${randomUUID()}`], {
|
||||
stdio: "ignore", detached: true, env: { PATH: process.env.PATH },
|
||||
});
|
||||
await once(child, "spawn");
|
||||
return {
|
||||
pid: child.pid!,
|
||||
async stop() {
|
||||
if (child.exitCode !== null || child.signalCode !== null) return;
|
||||
const exited = once(child, "exit");
|
||||
child.kill("SIGTERM");
|
||||
await exited;
|
||||
},
|
||||
alive() { return process.kill(child.pid!, 0); },
|
||||
};
|
||||
}
|
||||
|
||||
async function nativeProcessFixture(pid: number, targetKind: "local" | "remote", status: "running" | "failed") {
|
||||
const run = await seedRun();
|
||||
await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: run.issueId, status,
|
||||
nativePhase: status === "failed" ? "retryable_failure" : "running",
|
||||
errorCode: status === "failed" ? "native_session_interrupted" : null,
|
||||
error: status === "failed" ? "native session timed out after 120000ms" : null,
|
||||
finishedAt: status === "failed" ? new Date() : null,
|
||||
}).where(eq(heartbeatRuns.id, run.id));
|
||||
await db.insert(nativeRunFinalizations).values({ runId: run.id, companyId: run.companyId, issueId: run.issueId!,
|
||||
phase: status === "failed" ? "retryable_failure" : "running", nextAttemptAt: new Date(Date.now() - 1) });
|
||||
await persistHeartbeatRunProcessMetadata(db, run.id, { pid, processGroupId: pid,
|
||||
startedAt: new Date().toISOString(), targetKind });
|
||||
const deps = callbacks(db);
|
||||
deps.setRunStatusFromLive.mockImplementation(async (id, nextStatus, fromStatuses, patch) => {
|
||||
const [current] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id));
|
||||
if (!current || !fromStatuses.includes(current.status)) return { updated: false, run: current ?? null };
|
||||
const [updated] = await db.update(heartbeatRuns).set({ ...patch, status: nextStatus })
|
||||
.where(eq(heartbeatRuns.id, id)).returning();
|
||||
return { updated: true, run: updated };
|
||||
});
|
||||
return {
|
||||
run, deps,
|
||||
current: async () => (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0],
|
||||
async cleanup() {
|
||||
await Promise.all([...deps.activeRunExecutionPromises]);
|
||||
await db.delete(environmentLeases).where(eq(environmentLeases.heartbeatRunId, run.id));
|
||||
await db.delete(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id));
|
||||
await db.delete(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, run.id));
|
||||
await db.delete(heartbeatRuns).where(eq(heartbeatRuns.id, run.id));
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
it.each(["failed", "running"] as const)("does not attribute a remote %s run's PID to a live host process", async status => {
|
||||
const child = await ownedProcess();
|
||||
const f = await nativeProcessFixture(child.pid, "remote", status);
|
||||
try {
|
||||
await db.update(agents).set({ adapterType: "process" }).where(eq(agents.id, f.run.agentId));
|
||||
const before = await f.current();
|
||||
expect(await createHeartbeatRecovery(db, f.deps).reapOrphanedRuns()).toEqual({ reaped: 0, runIds: [] });
|
||||
expect(await f.current()).toMatchObject({ status, errorCode: before.errorCode, error: before.error,
|
||||
processPid: child.pid, processGroupId: child.pid, processStartedAt: before.processStartedAt });
|
||||
expect(f.deps.setRunStatusFromLive).not.toHaveBeenCalled();
|
||||
expect(f.deps.executeRun).not.toHaveBeenCalled();
|
||||
expect(f.deps.terminateHeartbeatRunProcess).not.toHaveBeenCalled();
|
||||
expect(await hasNativeLocalProcessStop(db, f.run.companyId, f.run.id)).toBe(false);
|
||||
expect(child.alive()).toBe(true);
|
||||
} finally { try { await f.cleanup(); } finally { await child.stop(); } }
|
||||
});
|
||||
|
||||
async function remoteRetryFixture(pid: number, target: "workspace" | "plugin_task" = "workspace") {
|
||||
const fixture = await nativeProcessFixture(pid, "remote", "failed");
|
||||
const sessionId = randomUUID();
|
||||
const [lease] = await db.insert(environmentLeases).values({ companyId: fixture.run.companyId,
|
||||
heartbeatRunId: fixture.run.id, issueId: fixture.run.issueId, status: "active", provider: "daytona",
|
||||
providerLeaseId: randomUUID(), metadata: target === "plugin_task" ? { runnerTaskLease: true } : {},
|
||||
}).returning();
|
||||
await db.update(agents).set({ adapterType: "paperclip_runner" }).where(eq(agents.id, fixture.run.agentId));
|
||||
await db.update(issues).set({ executionRunId: fixture.run.id }).where(eq(issues.id, fixture.run.issueId!));
|
||||
await db.update(heartbeatRuns).set({ nativeSessionId: sessionId, runnerProfileJson: {
|
||||
sessionCheckpoint: { identity: { runId: fixture.run.id, companyId: fixture.run.companyId,
|
||||
issueId: fixture.run.issueId, agentId: fixture.run.agentId, sessionId }, providerSessionId: "existing-provider" },
|
||||
...(target === "workspace" ? { nativeWorkspaceSync: { schema: "paperclip.native-workspace-sync/v3", state: "prepared",
|
||||
workspaceId: randomUUID(), leaseId: lease.id, providerLeaseId: lease.providerLeaseId, remoteCwd: "/work",
|
||||
descriptorSha256: "a".repeat(64), baselineSha256: "b".repeat(64), finalHostSha256: null,
|
||||
resourceDisposition: "keep_running" } } : {}),
|
||||
} }).where(eq(heartbeatRuns.id, fixture.run.id));
|
||||
await db.insert(nativeRunnerCheckpoints).values({ runId: fixture.run.id, companyId: fixture.run.companyId,
|
||||
issueId: fixture.run.issueId!, part: "opaque-checkpoint-record", material: { fixture: true } });
|
||||
fixture.deps.getRun.mockImplementation(async id => (await db.select().from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, id)))[0] ?? null);
|
||||
native.claims.mockImplementation(claimNativeRestartRecoveries);
|
||||
return { ...fixture, lease };
|
||||
}
|
||||
|
||||
it.each(["failed", "running"] as const)("routes a due remote %s retry through verified recovery without local PID inspection", async status => {
|
||||
const child = await ownedProcess();
|
||||
const f = await remoteRetryFixture(child.pid);
|
||||
const originalKill = process.kill.bind(process);
|
||||
const kill = vi.spyOn(process, "kill").mockImplementation(originalKill);
|
||||
try {
|
||||
await db.update(heartbeatRuns).set({ status }).where(eq(heartbeatRuns.id, f.run.id));
|
||||
await db.update(nativeRunFinalizations).set({ nextAttemptAt: null }).where(eq(nativeRunFinalizations.runId, f.run.id));
|
||||
const recovery = createHeartbeatRecovery(db, f.deps);
|
||||
await recovery.reapOrphanedRuns();
|
||||
await Promise.all([...f.deps.activeRunExecutionPromises]);
|
||||
expect(f.deps.executeRun).toHaveBeenCalledExactlyOnceWith(f.run.id, expect.objectContaining({
|
||||
nativeRestartRecovery: expect.objectContaining({ kind: "reattach_remote_runner", runId: f.run.id,
|
||||
remote: { providerLeaseId: f.lease.providerLeaseId, remoteCwd: "/work" } }),
|
||||
}));
|
||||
expect(native.claims).toHaveBeenCalledWith(expect.objectContaining({ runIds: [f.run.id] }));
|
||||
expect(await f.current()).toMatchObject({ status: "running", processPid: child.pid, processGroupId: child.pid });
|
||||
expect(kill.mock.calls.filter(([pid]) => Math.abs(pid as number) === child.pid)).toEqual([]);
|
||||
expect(await hasNativeLocalProcessStop(db, f.run.companyId, f.run.id)).toBe(false);
|
||||
await recovery.reapOrphanedRuns();
|
||||
expect(f.deps.executeRun).toHaveBeenCalledOnce();
|
||||
expect(f.deps.terminateHeartbeatRunProcess).not.toHaveBeenCalled();
|
||||
kill.mockRestore();
|
||||
expect(child.alive()).toBe(true);
|
||||
} finally { kill.mockRestore(); try { await f.cleanup(); } finally { await child.stop(); } }
|
||||
});
|
||||
|
||||
it("routes a due plugin-task retry through its existing remote lease", async () => {
|
||||
const child = await ownedProcess();
|
||||
const f = await remoteRetryFixture(child.pid, "plugin_task");
|
||||
try {
|
||||
await createHeartbeatRecovery(db, f.deps).reapOrphanedRuns();
|
||||
await Promise.all([...f.deps.activeRunExecutionPromises]);
|
||||
expect(f.deps.executeRun).toHaveBeenCalledExactlyOnceWith(f.run.id, expect.objectContaining({
|
||||
nativeRestartRecovery: expect.objectContaining({ kind: "reattach_remote_runner", runId: f.run.id,
|
||||
remote: { kind: "plugin_task", leaseId: f.lease.id } }),
|
||||
}));
|
||||
expect(await f.current()).toMatchObject({ processPid: child.pid, processGroupId: child.pid });
|
||||
expect(await hasNativeLocalProcessStop(db, f.run.companyId, f.run.id)).toBe(false);
|
||||
expect(child.alive()).toBe(true);
|
||||
} finally { try { await f.cleanup(); } finally { await child.stop(); } }
|
||||
});
|
||||
|
||||
it.each(["future_retry", "live_owner", "missing_checkpoint", "active_execution"] as const)(
|
||||
"does not execute a remote retry without current authority: %s", async condition => {
|
||||
const child = await ownedProcess();
|
||||
const f = await remoteRetryFixture(child.pid);
|
||||
try {
|
||||
if (condition === "future_retry") await db.update(nativeRunFinalizations)
|
||||
.set({ nextAttemptAt: new Date(Date.now() + 60_000) }).where(eq(nativeRunFinalizations.runId, f.run.id));
|
||||
if (condition === "live_owner") await db.update(nativeRunFinalizations)
|
||||
.set({ leaseOwner: "other-controller", leaseExpiresAt: new Date(Date.now() + 60_000) })
|
||||
.where(eq(nativeRunFinalizations.runId, f.run.id));
|
||||
if (condition === "missing_checkpoint") {
|
||||
const run = await f.current();
|
||||
await db.update(heartbeatRuns).set({ runnerProfileJson: { ...run.runnerProfileJson, sessionCheckpoint: null } })
|
||||
.where(eq(heartbeatRuns.id, f.run.id));
|
||||
}
|
||||
if (condition === "active_execution") f.deps.activeRunExecutions.add(f.run.id);
|
||||
await createHeartbeatRecovery(db, f.deps).reapOrphanedRuns();
|
||||
expect(f.deps.executeRun).not.toHaveBeenCalled();
|
||||
expect(f.deps.terminateHeartbeatRunProcess).not.toHaveBeenCalled();
|
||||
expect(await f.current()).toMatchObject({ processPid: child.pid, processGroupId: child.pid });
|
||||
if (condition === "future_retry") expect(native.claims).not.toHaveBeenCalled();
|
||||
if (condition === "missing_checkpoint") {
|
||||
const [coordinator] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.run.id));
|
||||
expect(coordinator).toMatchObject({ phase: "terminal_failure", recoveryState: "blocked",
|
||||
failureCode: "native_restart_recovery_blocked" });
|
||||
}
|
||||
expect(child.alive()).toBe(true);
|
||||
} finally { f.deps.activeRunExecutions.delete(f.run.id); try { await f.cleanup(); } finally { await child.stop(); } }
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["unknown", "remote"] as const)("exposes expired observed %s ownership without attributing host liveness", async target => {
|
||||
const child = await ownedProcess();
|
||||
const f = await nativeProcessFixture(child.pid, "remote", "running");
|
||||
const originalKill = process.kill.bind(process);
|
||||
let restoreKill: (() => void) | undefined;
|
||||
try {
|
||||
if (target === "unknown") await db.delete(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, f.run.id));
|
||||
await db.update(heartbeatRuns).set({ nativePhase: null })
|
||||
.where(eq(heartbeatRuns.id, f.run.id));
|
||||
await db.update(nativeRunFinalizations).set({ phase: "observed", attempt: 2,
|
||||
leaseOwner: "prior-native-owner", leaseExpiresAt: new Date(0), nextAttemptAt: null })
|
||||
.where(eq(nativeRunFinalizations.runId, f.run.id));
|
||||
const before = await f.current();
|
||||
const coordinatorBefore = await db.select().from(nativeRunFinalizations)
|
||||
.where(eq(nativeRunFinalizations.runId, f.run.id));
|
||||
const kill = vi.spyOn(process, "kill").mockImplementation(originalKill);
|
||||
restoreKill = () => kill.mockRestore();
|
||||
expect(await createHeartbeatRecovery(db, f.deps).reapOrphanedRuns()).toEqual({ reaped: 0, runIds: [] });
|
||||
expect(kill.mock.calls.filter(([pid]) => Math.abs(pid as number) === child.pid)).toEqual([]);
|
||||
expect(await f.current()).toMatchObject({ status: "running", nativePhase: null,
|
||||
processPid: child.pid, processGroupId: child.pid, processStartedAt: before.processStartedAt,
|
||||
errorCode: NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE });
|
||||
expect(f.deps.appendRunEvent).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({
|
||||
payload: { reason: "observed_owner_unverified" },
|
||||
}));
|
||||
expect(await db.select().from(nativeRunFinalizations)
|
||||
.where(eq(nativeRunFinalizations.runId, f.run.id))).toEqual(coordinatorBefore);
|
||||
expect(f.deps.executeRun).not.toHaveBeenCalled();
|
||||
expect(f.deps.terminateHeartbeatRunProcess).not.toHaveBeenCalled();
|
||||
kill.mockRestore(); restoreKill = undefined;
|
||||
expect(child.alive()).toBe(true);
|
||||
} finally { restoreKill?.(); try { await f.cleanup(); } finally { await child.stop(); } }
|
||||
});
|
||||
|
||||
it.each(["failed", "running"] as const)("protects a verified local %s run's live process", async status => {
|
||||
const child = await ownedProcess();
|
||||
const f = await nativeProcessFixture(child.pid, "local", status);
|
||||
try {
|
||||
await db.update(agents).set({ adapterType: "paperclip_runner", adapterConfig: { driver: "daytona" } })
|
||||
.where(eq(agents.id, f.run.agentId));
|
||||
await createHeartbeatRecovery(db, f.deps).reapOrphanedRuns();
|
||||
expect(await f.current()).toMatchObject({ status, processPid: child.pid, processGroupId: child.pid,
|
||||
errorCode: NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE });
|
||||
expect(f.deps.appendRunEvent).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({
|
||||
payload: { reason: "live_process_identifier", processPidAlive: true, processGroupAlive: true },
|
||||
}));
|
||||
expect(f.deps.executeRun).not.toHaveBeenCalled();
|
||||
expect(f.deps.terminateHeartbeatRunProcess).not.toHaveBeenCalled();
|
||||
expect(child.alive()).toBe(true);
|
||||
} finally { try { await f.cleanup(); } finally { await child.stop(); } }
|
||||
});
|
||||
|
||||
it.each(["missing", "new_launch", "wrong_birth", "foreign_host", "wrong_agent", "wrong_pid", "wrong_group", "native_source", "remote_workspace", "remote_lease"] as const)(
|
||||
"holds a native PID without local authority: %s", async evidence => {
|
||||
const child = await ownedProcess();
|
||||
await child.stop();
|
||||
const f = await nativeProcessFixture(child.pid, "local", "failed");
|
||||
try {
|
||||
if (evidence === "missing") await db.delete(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, f.run.id));
|
||||
if (evidence === "new_launch") await appendHeartbeatRunEvent(db, { companyId: f.run.companyId,
|
||||
runId: f.run.id, agentId: f.run.agentId, eventType: PROCESS_START_REQUESTED });
|
||||
if (["wrong_birth", "foreign_host", "wrong_agent", "wrong_pid", "wrong_group", "native_source"].includes(evidence)) {
|
||||
const [event] = await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, f.run.id));
|
||||
const foreignAgentId = randomUUID();
|
||||
if (evidence === "wrong_agent") await db.insert(agents).values({ id: foreignAgentId,
|
||||
companyId: f.run.companyId, name: "Other process owner", role: "engineer", adapterType: "process" });
|
||||
await db.update(heartbeatRunEvents).set({
|
||||
...(evidence === "wrong_agent" ? { agentId: foreignAgentId } : {}),
|
||||
...(evidence === "native_source" ? { sourceEventId: "provider-event" } : {}),
|
||||
payload: { ...event.payload, ...(evidence === "wrong_birth"
|
||||
? { processStartedAt: new Date(0).toISOString() } : evidence === "foreign_host" ? { hostname: "foreign-host" }
|
||||
: evidence === "wrong_pid" ? { processPid: child.pid + 1 }
|
||||
: evidence === "wrong_group" ? { processGroupId: child.pid + 1 } : {}) },
|
||||
}).where(eq(heartbeatRunEvents.id, event.id));
|
||||
}
|
||||
if (evidence === "remote_workspace") await db.update(heartbeatRuns).set({
|
||||
runnerProfileJson: { nativeWorkspaceSync: { schema: "paperclip.native-workspace-sync/v3", state: "prepared",
|
||||
workspaceId: randomUUID(), leaseId: randomUUID(), providerLeaseId: randomUUID(), remoteCwd: "/work",
|
||||
descriptorSha256: "a".repeat(64), baselineSha256: "b".repeat(64), finalHostSha256: null, resourceDisposition: "destroy" } },
|
||||
}).where(eq(heartbeatRuns.id, f.run.id));
|
||||
if (evidence === "remote_lease") await db.insert(environmentLeases).values({ companyId: f.run.companyId,
|
||||
heartbeatRunId: f.run.id, issueId: f.run.issueId, status: "active", provider: "daytona", providerLeaseId: randomUUID() });
|
||||
const before = await f.current();
|
||||
await createHeartbeatRecovery(db, f.deps).reapOrphanedRuns();
|
||||
expect(await f.current()).toMatchObject({ processPid: child.pid, processGroupId: child.pid,
|
||||
processStartedAt: before.processStartedAt, errorCode: "native_session_interrupted" });
|
||||
expect(f.deps.executeRun).not.toHaveBeenCalled();
|
||||
expect(f.deps.terminateHeartbeatRunProcess).not.toHaveBeenCalled();
|
||||
expect(await hasNativeLocalProcessStop(db, f.run.companyId, f.run.id)).toBe(false);
|
||||
} finally { await f.cleanup(); }
|
||||
},
|
||||
);
|
||||
|
||||
it("clears a verified stopped local retry identity and records its stop", async () => {
|
||||
const child = await ownedProcess();
|
||||
await child.stop();
|
||||
const f = await nativeProcessFixture(child.pid, "local", "failed");
|
||||
try {
|
||||
await createHeartbeatRecovery(db, f.deps).reapOrphanedRuns();
|
||||
expect(await f.current()).toMatchObject({ processPid: null, processGroupId: null, processStartedAt: null });
|
||||
expect(await hasNativeLocalProcessStop(db, f.run.companyId, f.run.id)).toBe(true);
|
||||
expect(f.deps.terminateHeartbeatRunProcess).not.toHaveBeenCalled();
|
||||
} finally { await f.cleanup(); }
|
||||
});
|
||||
|
||||
it("retires a remote identity only with an exact authenticated termination receipt", async () => {
|
||||
const child = await ownedProcess();
|
||||
const f = await nativeProcessFixture(child.pid, "remote", "failed");
|
||||
try {
|
||||
await db.update(heartbeatRuns).set({ runnerProfileJson: { sessionCheckpoint: { fixture: true } } })
|
||||
.where(eq(heartbeatRuns.id, f.run.id));
|
||||
await db.update(nativeRunFinalizations).set({ phase: "terminal_failure" })
|
||||
.where(eq(nativeRunFinalizations.runId, f.run.id));
|
||||
const [lease] = await db.insert(environmentLeases).values({ companyId: f.run.companyId,
|
||||
heartbeatRunId: f.run.id, issueId: f.run.issueId, status: "active", provider: "daytona", providerLeaseId: randomUUID() }).returning();
|
||||
expect(await releaseSettledNativeRunProcess(db, f.run.id)).toBe("pending");
|
||||
const receipt = remoteTerminationReceipt(lease, { state: "destroyed", providerLeaseId: lease.providerLeaseId })!;
|
||||
await db.update(environmentLeases).set({ status: "released", cleanupStatus: "success", releasedAt: new Date(),
|
||||
metadata: { remoteExecutionTermination: { ...receipt, runId: randomUUID() } } }).where(eq(environmentLeases.id, lease.id));
|
||||
expect(await releaseSettledNativeRunProcess(db, f.run.id)).toBe("pending");
|
||||
expect(await f.current()).toMatchObject({ processPid: child.pid, processGroupId: child.pid });
|
||||
await db.update(environmentLeases).set({ metadata: { remoteExecutionTermination: receipt } })
|
||||
.where(eq(environmentLeases.id, lease.id));
|
||||
expect(await releaseSettledNativeRunProcess(db, f.run.id)).toBe("released");
|
||||
expect(await f.current()).toMatchObject({ processPid: null, processGroupId: null, processStartedAt: null });
|
||||
expect(await hasNativeLocalProcessStop(db, f.run.companyId, f.run.id)).toBe(false);
|
||||
expect(child.alive()).toBe(true);
|
||||
} finally { try { await f.cleanup(); } finally { await child.stop(); } }
|
||||
});
|
||||
|
||||
it.each(["ephemeral", "reuse_by_environment", "live_owner", "live_executor", "mismatched_source", "finalizing"] as const)(
|
||||
"reconstructs missing native preservation during orphan cleanup without replay: %s", async state => {
|
||||
const run = await seedRun();
|
||||
@@ -857,11 +1158,12 @@ describePostgres("heartbeat recovery module ownership", () => {
|
||||
await Promise.all([...deps.activeRunExecutionPromises]);
|
||||
});
|
||||
|
||||
it("rearms the persisted native retry deadline through the service callback", async () => {
|
||||
it.each(["future", "overdue-owner"])("rearms the persisted native retry deadline through the service callback: %s", async scenario => {
|
||||
const run = await seedRun();
|
||||
const nextAttemptAt = new Date(Date.now() + 60_000);
|
||||
const nextAttemptAt = new Date(Date.now() + (scenario === "future" ? 60_000 : -1_000));
|
||||
await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: run.issueId, status: "failed" }).where(eq(heartbeatRuns.id, run.id));
|
||||
await db.insert(nativeRunFinalizations).values({ runId: run.id, companyId: run.companyId, issueId: run.issueId!, phase: "retryable_failure", nextAttemptAt });
|
||||
await db.insert(nativeRunFinalizations).values({ runId: run.id, companyId: run.companyId, issueId: run.issueId!, phase: "retryable_failure", nextAttemptAt,
|
||||
failureCode: scenario === "overdue-owner" ? "native_controller_recovery_waiting_for_owner" : null });
|
||||
const deps = callbacks(db);
|
||||
const result = await createHeartbeatRecovery(db, deps).recoverNativeRunsAfterRestart();
|
||||
expect(result.scheduledRetryRunIds).toEqual([run.id]);
|
||||
@@ -872,6 +1174,16 @@ describePostgres("heartbeat recovery module ownership", () => {
|
||||
await db.delete(heartbeatRuns).where(eq(heartbeatRuns.id, run.id));
|
||||
});
|
||||
|
||||
it("retains a live remote owner's actual lease deadline without dispatching competing work", async () => {
|
||||
const deps = callbacks(db);
|
||||
const runId = randomUUID(), retryAt = new Date(Date.now() + 60_000);
|
||||
native.claims.mockResolvedValue([{ kind: "awaiting_evidence", runId, reason: "remote_controller_lease_live", retryAt }]);
|
||||
const result = await createHeartbeatRecovery(db, deps).recoverNativeRunsAfterRestart();
|
||||
expect(result.awaitingEvidenceRunIds).toEqual([runId]);
|
||||
expect(deps.scheduleNativeSessionResumeDispatch).toHaveBeenCalledWith(runId, retryAt, "restart");
|
||||
expect(deps.executeRun).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("retains shutdown ownership until background native cleanup physically settles", async () => {
|
||||
const deps = callbacks(db);
|
||||
let settle!: () => void;
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { NATIVE_RETAINED_OWNER_WAIT } from "../native-runtime/native-controller-recovery.js";
|
||||
import { nativeEnvironmentSettlementDisposition, nativeEnvironmentSettlementWork } from "../native-runtime/native-environment-settlement-work.js";
|
||||
import { nativeWorkspaceFinalizationWork } from "../native-runtime/native-workspace-finalization-work.js";
|
||||
import { attemptNativeWorkspaceFinalization } from "../native-runtime/native-workspace-finalizer.js";
|
||||
@@ -18,7 +19,7 @@ import {
|
||||
hasStopOnlyCleanup,
|
||||
settleStopOnlyCleanup,
|
||||
} from "../sandbox-stop-and-retain.js";
|
||||
import { discoverSettledNativeRunProcesses, recordNativeLocalProcessStop, retryPendingSettledNativeRunProcesses } from "../native-local-process-stop.js";
|
||||
import { discoverSettledNativeRunProcesses, PROCESS_IDENTITY_RECORDED, PROCESS_START_REQUESTED, recordNativeLocalProcessStop, retryPendingSettledNativeRunProcesses } from "../native-local-process-stop.js";
|
||||
import { pendingCaseHandoffForRun } from "./case-handoff-lifecycle.js";
|
||||
import {
|
||||
legacyControllerBootId,
|
||||
@@ -30,9 +31,11 @@ import { runUsedConversationAdapter } from "../conversation-continuation.js";
|
||||
import { legacyExecutionNeedsReconciliationWithEvidence, terminalizeLegacyExecution } from "../legacy-execution-recovery.js";
|
||||
import { reconcileRetainedWorkspaces } from "../workspace-recovery.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import os from "node:os";
|
||||
import {
|
||||
and,
|
||||
asc,
|
||||
desc,
|
||||
eq,
|
||||
gt,
|
||||
inArray,
|
||||
@@ -46,6 +49,7 @@ import type { Db } from "@paperclipai/db";
|
||||
import {
|
||||
agents,
|
||||
environmentLeases,
|
||||
heartbeatRunEvents,
|
||||
heartbeatRuns,
|
||||
issues,
|
||||
nativeRunFinalizations,
|
||||
@@ -123,7 +127,7 @@ export interface HeartbeatRecoveryDependencies extends Pick<HeartbeatRetryDepend
|
||||
nativeRestartRecovery?: NativeRestartRecoveryClaim;
|
||||
onNativeExecutionLeaseAcquired?: () => Promise<void>;
|
||||
}) => Promise<void>;
|
||||
scheduleNativeSessionResumeDispatch: (runId: string, nextAttemptAt: Date) => void;
|
||||
scheduleNativeSessionResumeDispatch: (runId: string, nextAttemptAt: Date, mode?: "restart") => void;
|
||||
cancelHeartbeatNativeRun: (input: {
|
||||
db: Db; runId: string; reason: string; runtimeMode: string | null;
|
||||
}) => Promise<unknown>;
|
||||
@@ -953,7 +957,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
|
||||
inArray(heartbeatRuns.status, ["running", "failed"]),
|
||||
isNull(nativeRunFinalizations.resultId),
|
||||
eq(nativeRunFinalizations.phase, "retryable_failure"),
|
||||
gt(nativeRunFinalizations.nextAttemptAt, now),
|
||||
or(gt(nativeRunFinalizations.nextAttemptAt, now), eq(nativeRunFinalizations.failureCode, NATIVE_RETAINED_OWNER_WAIT)),
|
||||
),
|
||||
);
|
||||
for (const scheduled of scheduledNativeRetries) {
|
||||
@@ -1001,6 +1005,9 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
|
||||
disposition.kind === "bootstrap_incomplete",
|
||||
);
|
||||
for (const disposition of dispositions) {
|
||||
if (disposition.kind === "awaiting_evidence" && disposition.retryAt) {
|
||||
scheduleNativeSessionResumeDispatch(disposition.runId, disposition.retryAt, "restart");
|
||||
}
|
||||
const run = await getRun(disposition.runId);
|
||||
if (run) {
|
||||
const isClaim =
|
||||
@@ -1836,6 +1843,29 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
|
||||
return { swept: rows.length, destroyed, capped };
|
||||
}
|
||||
|
||||
async function nativeProcessIsLocal(run: HeartbeatRun) {
|
||||
if (parseObject(run.runnerProfileJson).nativeWorkspaceSync != null) return false;
|
||||
const [identity] = await db.select({
|
||||
eventType: heartbeatRunEvents.eventType,
|
||||
agentId: heartbeatRunEvents.agentId,
|
||||
payload: heartbeatRunEvents.payload,
|
||||
}).from(heartbeatRunEvents).where(and(
|
||||
eq(heartbeatRunEvents.companyId, run.companyId),
|
||||
eq(heartbeatRunEvents.runId, run.id),
|
||||
isNull(heartbeatRunEvents.sourceEventId),
|
||||
inArray(heartbeatRunEvents.eventType, [PROCESS_START_REQUESTED, PROCESS_IDENTITY_RECORDED]),
|
||||
)).orderBy(desc(heartbeatRunEvents.seq)).limit(1);
|
||||
const recorded = identity?.payload;
|
||||
if (identity?.eventType !== PROCESS_IDENTITY_RECORDED || identity.agentId !== run.agentId ||
|
||||
recorded?.targetKind !== "local" || recorded.hostname !== os.hostname() ||
|
||||
recorded.processPid !== run.processPid || recorded.processGroupId !== run.processGroupId ||
|
||||
!run.processStartedAt || recorded.processStartedAt !== run.processStartedAt.toISOString()) return false;
|
||||
const leases = await db.select({ provider: environmentLeases.provider }).from(environmentLeases).where(and(
|
||||
eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id),
|
||||
));
|
||||
return leases.every(lease => lease.provider === "local");
|
||||
}
|
||||
|
||||
async function markNativeOwnershipUnverified(
|
||||
run: typeof heartbeatRuns.$inferSelect,
|
||||
evidence: {
|
||||
@@ -2030,13 +2060,12 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
|
||||
);
|
||||
});
|
||||
|
||||
// A retryable native run can retain process identifiers from the failed
|
||||
// attempt. Inspect them before the recovery claim: a live identifier is
|
||||
// unowned and blocks recovery, while identifiers that are all dead can be
|
||||
// cleared with a compare-and-set so the explicit retryable failure becomes
|
||||
// claimable in this same sweep.
|
||||
// Retryable runs can retain process identifiers from the failed attempt.
|
||||
// Only verified local identities may be checked and cleared on this host.
|
||||
// Due remote retries instead re-enter the existing restart-recovery owner,
|
||||
// which verifies the saved checkpoint, controller lease, and remote target.
|
||||
const retryableNativeProcesses = await db
|
||||
.select({ run: heartbeatRuns })
|
||||
.select({ run: heartbeatRuns, nextAttemptAt: nativeRunFinalizations.nextAttemptAt })
|
||||
.from(heartbeatRuns)
|
||||
.innerJoin(
|
||||
nativeRunFinalizations,
|
||||
@@ -2051,12 +2080,27 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
|
||||
),
|
||||
);
|
||||
const claimableNativeRunIds = new Set<string>();
|
||||
for (const { run } of retryableNativeProcesses) {
|
||||
const remoteRetryRunIds: string[] = [];
|
||||
for (const { run, nextAttemptAt } of retryableNativeProcesses) {
|
||||
if (isNativeRunnerOwnershipHeld(run)) continue;
|
||||
if (!run.processPid && !run.processGroupId) {
|
||||
claimableNativeRunIds.add(run.id);
|
||||
continue;
|
||||
}
|
||||
if (!await nativeProcessIsLocal(run)) {
|
||||
if (nextAttemptAt && nextAttemptAt > now) continue;
|
||||
if (activeRunExecutions.has(run.id) || runningProcesses.has(run.id)) continue;
|
||||
if (parseObject(run.runnerProfileJson).nativeWorkspaceSync != null) {
|
||||
remoteRetryRunIds.push(run.id);
|
||||
} else {
|
||||
const [taskLease] = await db.select({ id: environmentLeases.id }).from(environmentLeases).where(and(
|
||||
eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id),
|
||||
sql`${environmentLeases.metadata}->>'runnerTaskLease' = 'true'`,
|
||||
)).limit(1);
|
||||
if (taskLease) remoteRetryRunIds.push(run.id);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const processPidAlive =
|
||||
!!run.processPid && isProcessAlive(run.processPid);
|
||||
const processGroupAlive =
|
||||
@@ -2082,6 +2126,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
|
||||
and(
|
||||
eq(heartbeatRuns.id, run.id),
|
||||
eq(heartbeatRuns.runtimeMode, "native"),
|
||||
eq(heartbeatRuns.nextEventSeq, run.nextEventSeq),
|
||||
run.processPid === null
|
||||
? isNull(heartbeatRuns.processPid)
|
||||
: eq(heartbeatRuns.processPid, run.processPid),
|
||||
@@ -2101,6 +2146,10 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
|
||||
if (cleared) claimableNativeRunIds.add(cleared.id);
|
||||
}
|
||||
|
||||
const remoteRetryClaims = remoteRetryRunIds.length
|
||||
? (await recoverNativeRunsAfterRestart(now, remoteRetryRunIds)).claims
|
||||
: [];
|
||||
|
||||
const executingRun = (runId: string) => activeRunExecutions.has(runId) || runningProcesses.has(runId);
|
||||
if (!settledProcessesDiscovered) {
|
||||
settledProcessesDiscovered = await discoverSettledNativeRunProcesses(db, { skip: executingRun }).then(() => true, (error) => {
|
||||
@@ -2148,7 +2197,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
|
||||
return [];
|
||||
});
|
||||
const resumedRunIds = new Set(
|
||||
nativeResumeClaims.map((claim) => claim.runId),
|
||||
[...nativeResumeClaims, ...remoteRetryClaims].map((claim) => claim.runId),
|
||||
);
|
||||
|
||||
// A terminal issue transition writes this intent in the same transaction
|
||||
@@ -2300,10 +2349,12 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
|
||||
// preserves its explicit physical-copyback hold. Keep its source lease.
|
||||
if (nativeWorkspaceFinalizationPending) continue;
|
||||
const nativeRun = run.runtimeMode === "native";
|
||||
const nativeLocalProcess = nativeRun && !!(run.processPid || run.processGroupId) &&
|
||||
await nativeProcessIsLocal(run);
|
||||
const nativeProcessPidAlive =
|
||||
nativeRun && !!run.processPid && isProcessAlive(run.processPid);
|
||||
nativeLocalProcess && !!run.processPid && isProcessAlive(run.processPid);
|
||||
const nativeProcessGroupAlive =
|
||||
nativeRun &&
|
||||
nativeLocalProcess &&
|
||||
!!run.processGroupId &&
|
||||
isProcessGroupAlive(run.processGroupId);
|
||||
const coordinatorOwnedByCurrentController =
|
||||
@@ -2360,6 +2411,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if (nativeRun && !locallyTracked && (run.processPid || run.processGroupId) && !nativeLocalProcess) continue;
|
||||
if (resumedRunIds.has(run.id)) continue;
|
||||
if (locallyTracked) continue;
|
||||
if (await hasLiveLegacyController(db, run)) continue;
|
||||
@@ -2379,7 +2431,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
|
||||
// for a read-only liveness check so a lost in-memory handle cannot cause
|
||||
// overlapping provider/tool execution while that child is still alive.
|
||||
const checksPersistedChildLiveness =
|
||||
currentAdapterTracksLocalChild || run.runtimeMode === "native";
|
||||
nativeRun ? nativeLocalProcess : currentAdapterTracksLocalChild;
|
||||
const processPidAlive =
|
||||
checksPersistedChildLiveness &&
|
||||
run.processPid &&
|
||||
|
||||
@@ -309,9 +309,16 @@ describe.skipIf(!support.supported && !process.env.PAPERCLIP_TEST_DATABASE_URL)(
|
||||
it("retains native completion metadata when reconciliation already committed the same terminal status", async () => {
|
||||
const finishedAt = new Date("2026-10-09T12:00:00Z");
|
||||
const f = await fixture({ runtimeMode: "native", status: "succeeded", finishedAt, resultJson: { durable: "accepted" }, usageJson: { durableReceipt: "native" } });
|
||||
f.input.adapterResult.nativeFinalization = { schema: "paperclip.native-finalization.v1", runtimeMode: "native", runId: f.run.id, companyId: f.run.companyId, issueId: "unused", result: {}, terminal: { schema: "paperclip.prp.terminal.v1", turnTerminalState: "completed", runTerminalState: "succeeded", reportedWorkDisposition: "done" }, turnId: null, sourceInstanceId: "test", normalizedSessionId: "native-session", providerSessionId: null, driverKind: "test", driverVersion: "1", nativeEventCount: 1, highestContiguousSourceSeq: 1, workspaceFinalizeStatus: "succeeded" };
|
||||
await db.update(agents).set({ adapterType: "paperclip_runner" }).where(eq(agents.id, f.agent.id));
|
||||
f.agent.adapterType = "paperclip_runner";
|
||||
f.input.sessionCodec = getAdapterSessionCodec(f.agent.adapterType);
|
||||
const nativeSessionId = randomUUID(), providerSessionId = randomUUID();
|
||||
f.input.adapterResult.sessionId = nativeSessionId;
|
||||
delete f.input.adapterResult.sessionParams;
|
||||
f.input.adapterResult.sessionDisplayId = providerSessionId;
|
||||
f.input.adapterResult.nativeFinalization = { schema: "paperclip.native-finalization.v1", runtimeMode: "native", runId: f.run.id, companyId: f.run.companyId, issueId: "unused", result: {}, terminal: { schema: "paperclip.prp.terminal.v1", turnTerminalState: "completed", runTerminalState: "succeeded", reportedWorkDisposition: "done" }, turnId: null, sourceInstanceId: "test", normalizedSessionId: nativeSessionId, providerSessionId, driverKind: "test", driverVersion: "1", nativeEventCount: 1, highestContiguousSourceSeq: 1, workspaceFinalizeStatus: "succeeded" };
|
||||
await f.completion.completeRun(f.input);
|
||||
expect(await read(f.run.id)).toMatchObject({ status: "succeeded", finishedAt, sessionIdAfter: "provider-session", resultJson: { durable: "accepted" }, usageJson: { durableReceipt: "native", inputTokens: 20 } });
|
||||
expect(await read(f.run.id)).toMatchObject({ status: "succeeded", finishedAt, sessionIdAfter: providerSessionId, resultJson: { durable: "accepted" }, usageJson: { durableReceipt: "native", inputTokens: 20 } });
|
||||
expect(f.deps.updateRuntimeState).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { preserveAcceptedRunUsageProjectionSql } from "../usage-receipts.js";
|
||||
import { nativeTaskSessionPublicationHandled } from "../native-runtime/native-task-session-publication.js";
|
||||
import { notifyChatRunMilestones } from "../chat-run-milestone-notifications.js";
|
||||
import { measureRunPhase } from "../execution-preparation-timing.js";
|
||||
import { PROJECT_ACCESS_DENIED, ProjectAccessDeniedError } from "../project-access-denial.js";
|
||||
@@ -489,6 +490,13 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC
|
||||
normalizedUsage,
|
||||
);
|
||||
if (!taskKey) return;
|
||||
if (nativeTaskSessionPublicationHandled(finalizedRun.runnerProfileJson, {
|
||||
runId: finalizedRun.id,
|
||||
companyId: finalizedRun.companyId,
|
||||
agentId: agent.id,
|
||||
taskKey,
|
||||
adapterType: agent.adapterType,
|
||||
})) return;
|
||||
if (
|
||||
adapterResult.clearSession ||
|
||||
(!nextSessionState.params && !nextSessionState.displayId)
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { NativeControllerRecoveryWaitingError } from "../native-runtime/native-controller-recovery.js";
|
||||
|
||||
import { createNativeUsageDelivery } from "../native-runtime/native-usage-delivery.js";
|
||||
import type { PrpEvent } from "../../vendor/paperclip-runner/index.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
@@ -10,7 +12,7 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vites
|
||||
import { trace } from "@opentelemetry/api";
|
||||
import { InMemorySpanExporter, SimpleSpanProcessor, TracerProvider } from "@opentelemetry/sdk-trace";
|
||||
import { createRunPreparationTiming, measureRunPhase, measureWorkspaceSyncPhase, withRunExecutionTiming } from "../execution-preparation-timing.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
|
||||
import { recoveryPostgresSupport as support, startRecoveryPostgres } from "../../__tests__/helpers/recovery-postgres.js";
|
||||
import * as pluginWorkspaces from "../plugin-workspaces.js";
|
||||
import * as legacyRecovery from "../legacy-execution-recovery.js";
|
||||
import { createAgentIdentityRedactor } from "../agent-identity-redaction.js";
|
||||
@@ -43,15 +45,14 @@ vi.mock("./run-preparation.js", async original => ({
|
||||
revokeHeartbeatRunGatewayTokens: vi.fn(async () => {}),
|
||||
}));
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
describe.skipIf(!support.supported)("heartbeat runtime execution boundary", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let database: Awaited<ReturnType<typeof startRecoveryPostgres>>;
|
||||
let db: Db;
|
||||
let home: string;
|
||||
beforeAll(async () => {
|
||||
home = await mkdtemp(join(tmpdir(), "paperclip-runtime-execution-"));
|
||||
vi.stubEnv("PAPERCLIP_HOME", home);
|
||||
database = await startEmbeddedPostgresTestDatabase("paperclip-runtime-execution");
|
||||
database = await startRecoveryPostgres("paperclip-runtime-execution");
|
||||
db = createDb(database.connectionString);
|
||||
await db.execute(sql`set client_min_messages = warning`);
|
||||
}, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
|
||||
@@ -283,6 +284,18 @@ describe.skipIf(!support.supported)("heartbeat runtime execution boundary", () =
|
||||
expectCleanup(input);
|
||||
});
|
||||
|
||||
it("preserves the retained sandbox while native admission waits for its current owner", async () => {
|
||||
const { input } = await fixture(true);
|
||||
const error = new NativeControllerRecoveryWaitingError(new Date(Date.now() + 60_000));
|
||||
vi.mocked(executePaperclipNativeSession).mockRejectedValueOnce(error);
|
||||
await expect(executeHeartbeatRuntime(db, input)).rejects.toMatchObject({
|
||||
name: "NativeSessionResumeScheduledError", original: error,
|
||||
});
|
||||
expect(input.effects.onNativeSessionResumeScheduled).toHaveBeenCalledOnce();
|
||||
expect(input.effects.onProviderResourceDisposition).not.toHaveBeenCalledWith("stop_and_retain");
|
||||
expect(await finalizations()).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("records native restoration inside the workspace finalization operation", async () => {
|
||||
const { input, run } = await fixture(true);
|
||||
const endpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { NativeControllerRecoveryWaitingError } from "../native-runtime/native-controller-recovery.js";
|
||||
|
||||
import { usesNativeUsageDelivery } from "../native-runtime/native-usage-accounting.js";
|
||||
import { DELIVERY_QUEUES } from "../delivery-work-notifications.js";
|
||||
import { writeExecutionControl } from "../execution-control-notifications.js";
|
||||
@@ -559,6 +561,20 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
|
||||
sessionGoalControl.requestId,
|
||||
));
|
||||
if (completedGoalControl) sessionGoalControl = null;
|
||||
// Goal checkpoints already publish their task session. Goal-less turns
|
||||
// need the same durable pointer before cleanup can delay host completion.
|
||||
const taskSessionPublication = taskKey && !sessionGoalControl &&
|
||||
context.resumeSessionGoalHeartbeat !== true && !completedGoalControl
|
||||
? {
|
||||
taskKey,
|
||||
adapterType: agent.adapterType,
|
||||
sessionParamsJson: attachPaperclipSessionMetadataToSessionParams(
|
||||
{ ...runtimeSessionParamsForAdapter, cwd: executionWorkspace.cwd },
|
||||
configuredModel,
|
||||
sessionConfigMetadata,
|
||||
),
|
||||
}
|
||||
: undefined;
|
||||
const nativeDispatchAtMs = Date.now();
|
||||
const runCreatedAtMs = run.createdAt.getTime();
|
||||
const runStartedAtMs = (run.startedAt ?? run.createdAt).getTime();
|
||||
@@ -608,6 +624,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
|
||||
resumeSessionGoalHeartbeat:
|
||||
context.resumeSessionGoalHeartbeat === true ||
|
||||
completedGoalControl,
|
||||
taskSessionPublication,
|
||||
onGoalCheckpoint: async (snapshot) => {
|
||||
if (!taskKey) return;
|
||||
const params =
|
||||
@@ -1026,6 +1043,10 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
|
||||
preserveRunGatewayTokens = true;
|
||||
throw adapterErr;
|
||||
}
|
||||
if (adapterErr instanceof NativeControllerRecoveryWaitingError) {
|
||||
input.effects.onNativeSessionResumeScheduled();
|
||||
throw new NativeSessionResumeScheduledError(adapterErr);
|
||||
}
|
||||
if (adapterErr instanceof NativeRunnerOwnershipUnverifiedError) {
|
||||
input.effects.onNativeOwnershipHeld();
|
||||
throw adapterErr;
|
||||
|
||||
@@ -4,6 +4,7 @@ import { inspectNativeRunnerUpgradeHeader } from "../native-runtime/native-runne
|
||||
import { readNativeWorkspaceSyncReference } from "../native-runtime/native-workspace-sync.js";
|
||||
import { REMOTE_RUNNER_RECONNECT_GRACE_MS } from "../native-runtime/native-runner-recovery-contract.js";
|
||||
import { currentNativeControllerIdentity } from "../native-runtime/native-restart-recovery.js";
|
||||
import { readNativeControllerRecoveryInventory } from "../native-runtime/native-controller-recovery-inventory.js";
|
||||
|
||||
const MAX_UPGRADE_RUNS = 1_000;
|
||||
type Counters = { activeRuns: number; pendingWakes: number; draining: boolean };
|
||||
@@ -60,8 +61,16 @@ export async function readUpgradeReadiness(db: Db, input: {
|
||||
&& identity.runnerInstanceId === row.runnerId && identity.normalizedSessionId === row.sessionId
|
||||
&& identity.environmentLeaseId === (row.pluginLeaseId ?? workspace?.workspaceId)) };
|
||||
});
|
||||
return summarizeUpgradeReadiness({ rows: candidates, before, after: input.counters(),
|
||||
// Additive evidence does not strengthen or redefine the existing ready flag.
|
||||
const retained = await readNativeControllerRecoveryInventory(db).catch(() => ({
|
||||
version: 1 as const, observedAt: new Date().toISOString(), coverage: "unknown" as const,
|
||||
reasons: { inventory_unavailable: 1 }, fullPayloadsValidated: false as const,
|
||||
}));
|
||||
const summary = summarizeUpgradeReadiness({ rows: candidates, before, after: input.counters(),
|
||||
activeRunIds: [...new Set([...beforeIds, ...input.activeRunIds])], controllerBootId: owner.bootId });
|
||||
return { ...summary, nativeControllerRecovery: { activeExecutionAdoption: {
|
||||
recoverableRuns: summary.recoverableRuns, blockingRuns: summary.blockingRuns, truncated: summary.truncated,
|
||||
}, retained } };
|
||||
}
|
||||
|
||||
export function summarizeUpgradeReadiness(input: {
|
||||
|
||||
@@ -39,7 +39,7 @@ import { requestRestorePointDeletion } from "./git-restore-point-ledger.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
import { recordChatHandoff, recordAcceptedChatHandoff, lockAcceptedChatHandoffSource, recordChatCompletion, existingChatCompletionReply, acknowledgeChatCompletionReply } from "./chat-completion-delivery.js";
|
||||
import { mirrorSlackBoardComment, slackBoardReplyBindings } from "./slack-board-messages.js";
|
||||
import { assertAgentRunWriteAllowed } from "../agent-run-cancellation.js";
|
||||
import { agentRunWritesRevoked, assertAgentRunWriteAllowed } from "../agent-run-cancellation.js";
|
||||
import { assertLegacyRunTaskOwnership } from "../legacy-run-task-ownership.js";
|
||||
import { retryIdempotentDatabaseOperation } from "../database-retry.js";
|
||||
import { externalConversationStateSql, nonIdleSlackIssueCondition, resumeSlackConversation } from "./slack-conversation-state.js";
|
||||
@@ -153,7 +153,7 @@ import {
|
||||
isUuidLike,
|
||||
normalizeIssueIdentifier as normalizeIssueReferenceIdentifier,
|
||||
} from "@paperclipai/shared";
|
||||
import { conflict, HttpError, notFound, unprocessable } from "../errors.js";
|
||||
import { conflict, forbidden, HttpError, notFound, unprocessable } from "../errors.js";
|
||||
import { isForeignKeyViolation } from "../db-errors.js";
|
||||
import { logger } from "../middleware/logger.js";
|
||||
import { parseObject } from "../adapters/utils.js";
|
||||
@@ -1695,6 +1695,7 @@ function withAgentCommentAuthorizationMetadata(
|
||||
const reason = authorizationReason?.trim() || "internal_agent_write";
|
||||
return {
|
||||
version: 1,
|
||||
...(metadata?.chatMessage ? { chatMessage: metadata.chatMessage } : {}),
|
||||
...(metadata?.sourceRunId !== undefined
|
||||
? { sourceRunId: metadata.sourceRunId }
|
||||
: {}),
|
||||
@@ -8653,7 +8654,7 @@ export function issueService(db: Db) {
|
||||
companyId: string,
|
||||
issueId: string,
|
||||
userId: string,
|
||||
readAt: Date = new Date(),
|
||||
readAt: Date | SQL = new Date(),
|
||||
) => {
|
||||
const now = new Date();
|
||||
const [row] = await db
|
||||
@@ -8672,7 +8673,7 @@ export function issueService(db: Db) {
|
||||
issueReadStates.userId,
|
||||
],
|
||||
set: {
|
||||
lastReadAt: readAt,
|
||||
lastReadAt: sql`greatest(${issueReadStates.lastReadAt}, excluded.last_read_at)`,
|
||||
updatedAt: now,
|
||||
},
|
||||
})
|
||||
@@ -12937,6 +12938,7 @@ export function issueService(db: Db) {
|
||||
sourceTrust?: typeof issueComments.$inferInsert.sourceTrust;
|
||||
createdAt?: Date | string | null;
|
||||
clientRequestId?: string;
|
||||
proactiveChatMessage?: { sourceTaskId: string; userId: string };
|
||||
/** Server-only: authenticated Paperclip messages also belong in the Slack thread. */
|
||||
mirrorToSlack?: boolean;
|
||||
},
|
||||
@@ -12993,11 +12995,28 @@ export function issueService(db: Db) {
|
||||
return existing;
|
||||
}
|
||||
}
|
||||
if (options?.metadata?.chatMessage && !options.proactiveChatMessage) {
|
||||
throw forbidden("Chat message provenance is server-owned");
|
||||
}
|
||||
if (issue.conversationAgentId && actor.agentId && issue.conversationAgentId !== actor.agentId) {
|
||||
throw forbidden("Only this conversation's agent can post agent messages");
|
||||
}
|
||||
if (issue.conversationAgentId && actor.agentId && !actor.runId) {
|
||||
throw forbidden("Agent chat messages require an authenticated run");
|
||||
}
|
||||
if (issue.conversationAgentId && actor.runId) {
|
||||
const [run] = await dbOrTx.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, actor.runId));
|
||||
const [current] = await dbOrTx.select().from(issues).where(eq(issues.id, issueId));
|
||||
if (run?.status === "cancelled") throw conflict("This conversation turn was cancelled; it cannot post a reply");
|
||||
if (run?.contextSnapshot?.conversationSessionGeneration !== current.conversationSessionGeneration) {
|
||||
if (agentRunWritesRevoked(run)) throw conflict("This conversation turn was cancelled; it cannot post a reply");
|
||||
const runTaskId = run?.nativeIssueId ?? run?.issueId ?? run?.contextSnapshot?.issueId ?? run?.contextSnapshot?.taskId;
|
||||
if (options?.proactiveChatMessage) {
|
||||
if (run?.status !== "running" || run.agentId !== actor.agentId || run.companyId !== issue.companyId
|
||||
|| runTaskId !== options.proactiveChatMessage.sourceTaskId || runTaskId === issueId
|
||||
|| current.conversationUserId !== options.proactiveChatMessage.userId
|
||||
|| run.responsibleUserId !== current.conversationUserId) {
|
||||
throw forbidden("Invalid background chat message authority");
|
||||
}
|
||||
} else if (runTaskId !== issueId || run?.contextSnapshot?.conversationSessionGeneration !== current.conversationSessionGeneration) {
|
||||
throw conflict("Conversation session changed; this reply belongs to an earlier session");
|
||||
}
|
||||
}
|
||||
@@ -13132,7 +13151,7 @@ export function issueService(db: Db) {
|
||||
: (options?.metadata ?? null),
|
||||
);
|
||||
let comment: typeof issueComments.$inferSelect | null = null;
|
||||
if (createdByRunId) {
|
||||
if (createdByRunId && !options?.proactiveChatMessage) {
|
||||
const existing = await dbOrTx
|
||||
.select()
|
||||
.from(issueComments)
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { runAdapterExecutionTargetShellCommand, type AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target";
|
||||
import { kimiMetadataProbeDigest, kimiMetadataProbeNodeDigest, probeKimiMetadata, validateKimiMetadata } from "../vendor/paperclip-runner/live/index.js";
|
||||
import { unprocessable } from "../errors.js";
|
||||
import { createSshCommandManagedRuntimeRunner, runSshCommand } from "@paperclipai/adapter-utils/ssh";
|
||||
import { QUALIFIED_ACPX_PROFILES } from "../vendor/paperclip-runner/index.js";
|
||||
import { resolveKimiCandidateAdmission } from "./native-runtime/acpx-qualification.js";
|
||||
|
||||
const REQUEST_FAILED = () => unprocessable("Kimi Code metadata verification failed.", { code: "KIMI_REQUEST_FAILED" });
|
||||
const INSTALLATION_INVALID = () => unprocessable("Install the current verified Kimi Code runtime in this execution environment.", { code: "KIMI_INSTALLATION_INVALID" });
|
||||
|
||||
const SSH_INPUT_PROBE = 'try { const input = JSON.parse(require("node:fs").readFileSync(0, "utf8")); if (typeof input.apiKey !== "string" || typeof input.model !== "string") process.exit(1); process.env.KIMI_API_KEY = input.apiKey; process.env.PAPERCLIP_KIMI_PROBE_MODEL = input.model; if (input.candidateAdmission !== undefined) process.env.PAPERCLIP_ACPX_CANDIDATE_ADMISSION = JSON.stringify(input.candidateAdmission); require(process.argv[1]); } catch { process.exit(1); }';
|
||||
|
||||
export async function probeKimiExecutionTarget(apiKey: string, target: AdapterExecutionTarget | null | undefined, model: string, context: { companyId?: string } = {}) {
|
||||
if (typeof apiKey !== "string" || !apiKey.trim() || /[\0\r\n]/.test(apiKey) || Buffer.byteLength(apiKey) > 16 * 1024) {
|
||||
throw unprocessable("Bind a valid company secret to KIMI_API_KEY.", { code: "KIMI_AUTH_REQUIRED" });
|
||||
}
|
||||
if (typeof model !== "string" || !model.trim() || model.includes("\0") || Buffer.byteLength(model) > 512 || Buffer.from(model).toString("utf8") !== model) {
|
||||
throw unprocessable("Select the exact Kimi Code model.", { code: "KIMI_MODEL_UNAVAILABLE" });
|
||||
}
|
||||
let candidateAdmission: Parameters<typeof probeKimiMetadata>[2];
|
||||
if (QUALIFIED_ACPX_PROFILES.kimi.qualificationStatus === "pending") {
|
||||
try {
|
||||
const admission = resolveKimiCandidateAdmission({ companyId: context.companyId, model,
|
||||
target: target?.kind === "remote" ? { platform: "linux", architecture: "x64" } : { platform: process.platform, architecture: process.arch } });
|
||||
if (!admission) throw new Error("Missing candidate admission");
|
||||
const probeId = `kimi-metadata-${randomUUID()}`;
|
||||
candidateAdmission = { ...admission, runId: probeId, normalizedSessionId: `metadata-${randomUUID()}` };
|
||||
} catch {
|
||||
throw unprocessable("Kimi Code qualification requires current trusted operator admission for this company and execution target.", { code: "KIMI_QUALIFICATION_REQUIRED" });
|
||||
}
|
||||
}
|
||||
if (!target || target.kind === "local") {
|
||||
const result = validateKimiMetadata(await probeKimiMetadata(model, apiKey, candidateAdmission), model);
|
||||
if (result.status !== "verified") throw unprocessable(result.message, { code: result.code });
|
||||
return result;
|
||||
}
|
||||
if (target.transport === "sandbox" && target.runnerTask) {
|
||||
throw unprocessable("Kimi Code metadata verification is unavailable in this hosting environment.", { code: "KIMI_INSTALLATION_INVALID" });
|
||||
}
|
||||
let locations = ["/opt/paperclip-runner/provider-pack", '"$HOME/.local/share/paperclip-runner/provider-pack"'];
|
||||
const selectedPackRoot = process.env.PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH?.trim();
|
||||
if (selectedPackRoot) {
|
||||
try {
|
||||
// Test uses the same complete-pack installer as native execution. It
|
||||
// needs no native run/session or DB context and opens no model session.
|
||||
const { prepareRemoteProviderPackOnTarget, resolveRemoteProviderPackSource } = await import("./native-runtime/native-session-executor.js");
|
||||
const selected = resolveRemoteProviderPackSource(selectedPackRoot);
|
||||
if (selected.manifest.payload.target.platform !== "linux" || selected.manifest.payload.target.architecture !== "x64") throw INSTALLATION_INVALID();
|
||||
const runner = target.transport === "ssh" ? createSshCommandManagedRuntimeRunner({ spec: target.spec, defaultCwd: target.remoteCwd }) : target.runner;
|
||||
if (!runner) throw INSTALLATION_INVALID();
|
||||
const prepared = await prepareRemoteProviderPackOnTarget({ target, runner,
|
||||
configuredProviderPackRoot: selected.configuredRoot, manifest: selected.manifest });
|
||||
locations = [`'${prepared.root.replaceAll("'", "'\\''")}'`];
|
||||
} catch { throw INSTALLATION_INVALID(); }
|
||||
}
|
||||
// Preparation retains the normal installer's bounded stages. The existing
|
||||
// metadata/HTTP budget begins only after that preparation has completed.
|
||||
const deadlineAt = Date.now() + 35_000;
|
||||
let digest: string;
|
||||
try { digest = await kimiMetadataProbeDigest(); }
|
||||
catch { throw INSTALLATION_INVALID(); }
|
||||
const nodeDigest = kimiMetadataProbeNodeDigest();
|
||||
if (!/^sha256:[a-f0-9]{64}$/.test(digest) || !/^sha256:[a-f0-9]{64}$/.test(nodeDigest)) throw INSTALLATION_INVALID();
|
||||
for (const location of locations) {
|
||||
const remainingMs = deadlineAt - Date.now();
|
||||
if (remainingMs <= 3_000) throw REQUEST_FAILED();
|
||||
const command = [
|
||||
`pack=${location}`,
|
||||
'entry="$pack/dist/cli/kimi-metadata-probe.cjs"; node="$pack/provider-assets/kimi/linux-x64/runtime/node/bin/node"',
|
||||
'[ -f "$pack/provider-pack.json" ] && [ -f "$entry" ] && [ ! -L "$entry" ] && [ -x "$node" ] && [ ! -L "$node" ] || exit 127',
|
||||
'unset NODE_OPTIONS NODE_PATH NODE_EXTRA_CA_CERTS NODE_USE_ENV_PROXY NODE_TLS_REJECT_UNAUTHORIZED OPENSSL_CONF SSL_CERT_FILE SSL_CERT_DIR HTTP_PROXY HTTPS_PROXY ALL_PROXY NO_PROXY http_proxy https_proxy all_proxy no_proxy LD_PRELOAD LD_LIBRARY_PATH LD_AUDIT GLIBC_TUNABLES DYLD_INSERT_LIBRARIES DYLD_LIBRARY_PATH DYLD_FRAMEWORK_PATH',
|
||||
'PATH=/usr/bin:/bin; export PATH',
|
||||
'actual=$(/usr/bin/env -i /usr/bin/sha256sum "$entry" 2>/dev/null) || exit 126',
|
||||
`actual=\${actual%% *}; [ "$actual" = "${digest.slice(7)}" ] || exit 126`,
|
||||
'actual=$(/usr/bin/env -i /usr/bin/sha256sum "$node" 2>/dev/null) || exit 126',
|
||||
`actual=\${actual%% *}; [ "$actual" = "${nodeDigest.slice(7)}" ] || exit 126`,
|
||||
target.transport === "ssh"
|
||||
? `exec /usr/bin/env -i PAPERCLIP_ACPX_PROVIDER_PACKAGE_ROOT="$pack" PAPERCLIP_ACPX_PROVIDER_PACKAGE_MANIFEST="$pack/package.json" "$node" -e '${SSH_INPUT_PROBE}' "$entry"`
|
||||
: ['exec /usr/bin/env -i KIMI_API_KEY="$KIMI_API_KEY" PAPERCLIP_KIMI_PROBE_MODEL="$PAPERCLIP_KIMI_PROBE_MODEL"',
|
||||
...(candidateAdmission ? ['PAPERCLIP_ACPX_CANDIDATE_ADMISSION="$PAPERCLIP_ACPX_CANDIDATE_ADMISSION"'] : []),
|
||||
'PAPERCLIP_ACPX_PROVIDER_PACKAGE_ROOT="$pack" PAPERCLIP_ACPX_PROVIDER_PACKAGE_MANIFEST="$pack/package.json" "$node" "$entry"'].join(" "),
|
||||
].join("; ");
|
||||
let response: Pick<Awaited<ReturnType<typeof runAdapterExecutionTargetShellCommand>>, "stdout" | "exitCode" | "signal" | "timedOut">;
|
||||
try {
|
||||
if (target.transport === "ssh") {
|
||||
try {
|
||||
const result = await runSshCommand(target.spec, command, {
|
||||
stdin: JSON.stringify({ apiKey, model, ...(candidateAdmission ? { candidateAdmission } : {}) }), timeoutMs: Math.max(1, Math.floor((remainingMs - 3_000) / 1000)) * 1000,
|
||||
});
|
||||
response = { stdout: result.stdout, exitCode: 0, signal: null, timedOut: false };
|
||||
} catch (error) {
|
||||
const failure = error as { code?: unknown; stdout?: unknown; signal?: string | null };
|
||||
if (typeof failure.code !== "number") throw error;
|
||||
response = { stdout: typeof failure.stdout === "string" ? failure.stdout : "", exitCode: failure.code,
|
||||
signal: failure.signal ?? null, timedOut: false };
|
||||
}
|
||||
} else {
|
||||
response = await runAdapterExecutionTargetShellCommand(`kimi-metadata-${randomUUID()}`, target, command, {
|
||||
cwd: target.remoteCwd, env: { KIMI_API_KEY: apiKey, PAPERCLIP_KIMI_PROBE_MODEL: model,
|
||||
...(candidateAdmission ? { PAPERCLIP_ACPX_CANDIDATE_ADMISSION: JSON.stringify(candidateAdmission) } : {}) },
|
||||
timeoutSec: Math.max(1, Math.floor((remainingMs - 3_000) / 1000)), graceSec: 3,
|
||||
});
|
||||
}
|
||||
} catch { throw REQUEST_FAILED(); }
|
||||
if (Date.now() > deadlineAt || response.timedOut || response.signal || Buffer.byteLength(response.stdout) > 64 * 1024) throw REQUEST_FAILED();
|
||||
if ([126, 127].includes(response.exitCode ?? -1)) continue;
|
||||
let payload: unknown;
|
||||
try { payload = JSON.parse(response.stdout.trim()); } catch { throw REQUEST_FAILED(); }
|
||||
const result = validateKimiMetadata(payload, model);
|
||||
if (result.status !== "verified") {
|
||||
if (result.code === "KIMI_INSTALLATION_INVALID") continue;
|
||||
throw unprocessable(result.message, { code: result.code });
|
||||
}
|
||||
if (response.exitCode !== 0) throw REQUEST_FAILED();
|
||||
if (result.target !== "linux-x64") throw INSTALLATION_INVALID();
|
||||
return result;
|
||||
}
|
||||
throw INSTALLATION_INVALID();
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { ACPX_QUALIFICATION_ENV, resolveAcpxQualification } from "./acpx-qualification.js";
|
||||
import { ACPX_QUALIFICATION_ENV, ANTIGRAVITY_CANDIDATE_ADMISSION_ENV, resolveAcpxQualification, resolveKimiCandidateAdmission } from "./acpx-qualification.js";
|
||||
import { QUALIFIED_ACPX_PROFILES, KIMI_DISTRIBUTION_CLOSURE_SHA256 } from "../../vendor/paperclip-runner/index.js";
|
||||
import type { NativeExecutionInput } from "../../vendor/paperclip-runner/index.js";
|
||||
import { resolvePaperclipRunnerProviderProfile, resolvePaperclipRunnerNativeProviderInput } from "./provider-profile.js";
|
||||
|
||||
@@ -9,6 +10,20 @@ const authorize = (value: unknown) => ({ [ACPX_QUALIFICATION_ENV]: JSON.stringif
|
||||
describe("host ACPX qualification admission", () => {
|
||||
afterEach(() => vi.unstubAllEnvs());
|
||||
|
||||
it.each([undefined, "malformed", JSON.stringify(kimiAdmission("linux", "x64"))])("admits released Kimi without candidate authority (%s)", authority => {
|
||||
vi.stubEnv(ANTIGRAVITY_CANDIDATE_ADMISSION_ENV, authority);
|
||||
vi.stubEnv(ACPX_QUALIFICATION_ENV, "malformed-unrelated-authority");
|
||||
const config = { provider: "acpx", acpxAgent: "kimi", model: "kimi-for-coding" };
|
||||
expect(QUALIFIED_ACPX_PROFILES.kimi.agentProfileVersion).toBe(17);
|
||||
expect(QUALIFIED_ACPX_PROFILES.kimi.qualificationStatus).not.toBe("pending");
|
||||
expect(resolveKimiCandidateAdmission({ companyId: "candidate-company", model: config.model })).toBeUndefined();
|
||||
expect(resolvePaperclipRunnerProviderProfile(config, { companyId: "candidate-company" })).toMatchObject(config);
|
||||
expect(resolvePaperclipRunnerNativeProviderInput({ companyId: "candidate-company", backend: "acpx_runtime", adapterConfig: config }))
|
||||
.toMatchObject({ ...config, acpxPermissionMode: "approve-all" });
|
||||
expect(() => resolvePaperclipRunnerProviderProfile({ ...config, model: "" }))
|
||||
.toThrow(expect.objectContaining({ code: "paperclip_runner_acpx_model_required" }));
|
||||
});
|
||||
|
||||
it.each([undefined, "malformed", JSON.stringify([{ agent: "antigravity", model: "exact-model" }])])(
|
||||
"keeps pending Antigravity closed despite host authorization (%s)", host => {
|
||||
vi.stubEnv(ACPX_QUALIFICATION_ENV, host);
|
||||
@@ -97,3 +112,9 @@ it.each([undefined, "off", "low", "high", "max"] as const)("validates saved Pi t
|
||||
expect(resolvePaperclipRunnerNativeProviderInput({ backend: "acpx_runtime", adapterConfig })).toMatchObject({ piThinkingLevel: piThinkingLevel ?? "low" });
|
||||
for (const alias of ["medium", "minimal", "xhigh"]) expect(() => resolvePaperclipRunnerProviderProfile({ ...adapterConfig, piThinkingLevel: alias })).toThrow(expect.objectContaining({ code: "paperclip_runner_pi_thinking_invalid" }));
|
||||
});
|
||||
|
||||
function kimiAdmission(platform: "darwin" | "linux", architecture: "arm64" | "x64") {
|
||||
return { schema: "paperclip.acpx-candidate-admission.v1", companyId: "candidate-company", agent: "kimi", model: "kimi-for-coding",
|
||||
platform, architecture, expiresAt: Date.now() + 60_000, profileDigest: QUALIFIED_ACPX_PROFILES.kimi.commandDigest,
|
||||
nativeClosureDigest: `sha256:${KIMI_DISTRIBUTION_CLOSURE_SHA256[`${platform}-${architecture}` as "darwin-arm64" | "linux-x64"]}` };
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import { PAPERCLIP_RUNNER_ACPX_PROFILES } from "@paperclipai/adapter-utils";
|
||||
import { parseAcpxCandidateAdmission, type AcpxCandidateAdmission, type NativeExecutionInput } from "../../vendor/paperclip-runner/index.js";
|
||||
import { parseAcpxCandidateAdmission, QUALIFIED_ACPX_PROFILES, type AcpxCandidateAdmission, type NativeExecutionInput } from "../../vendor/paperclip-runner/index.js";
|
||||
|
||||
/** Trusted operator startup authority, separate from saved agent configuration. */
|
||||
export const ANTIGRAVITY_CANDIDATE_ADMISSION_ENV = "PAPERCLIP_ACPX_CANDIDATE_ADMISSION";
|
||||
@@ -23,9 +23,31 @@ export function resolveAntigravityCandidateAdmission(
|
||||
});
|
||||
}
|
||||
|
||||
export function resolveKimiCandidateAdmission(
|
||||
input: { companyId?: string; model?: string; target?: { platform: string; architecture: string } },
|
||||
hostEnvironment: NodeJS.ProcessEnv = process.env,
|
||||
): AcpxCandidateAdmission | undefined {
|
||||
if (QUALIFIED_ACPX_PROFILES.kimi.qualificationStatus !== "pending") return undefined;
|
||||
const encoded = hostEnvironment[ANTIGRAVITY_CANDIDATE_ADMISSION_ENV];
|
||||
if (!encoded || !input.companyId) return undefined;
|
||||
let value: unknown;
|
||||
try { value = JSON.parse(encoded); } catch { throw new Error("Invalid Kimi candidate admission"); }
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("Invalid Kimi candidate admission");
|
||||
const record = value as Record<string, unknown>;
|
||||
if (record.companyId !== input.companyId || record.agent !== "kimi") return undefined;
|
||||
const admission = parseAcpxCandidateAdmission(value, {
|
||||
companyId: input.companyId, agent: "kimi",
|
||||
model: input.model ?? (typeof record.model === "string" ? record.model : ""),
|
||||
});
|
||||
if (input.target && (admission.platform !== input.target.platform || admission.architecture !== input.target.architecture)) {
|
||||
throw new Error("Kimi candidate admission does not match the execution target");
|
||||
}
|
||||
return admission;
|
||||
}
|
||||
|
||||
/** Operator-only admission for an exact candidate/model during qualification. */
|
||||
export const ACPX_QUALIFICATION_ENV = "PAPERCLIP_RUNNER_ACPX_QUALIFICATION";
|
||||
export type AcpxQualificationCandidate = "cursor" | "copilot" | "pi";
|
||||
export type AcpxQualificationCandidate = "cursor" | "copilot" | "pi" | "kimi";
|
||||
|
||||
export function resolveAcpxQualification(
|
||||
provider: NativeExecutionInput["provider"] | { kind: "acpx"; agent: string; model: string },
|
||||
@@ -45,14 +67,14 @@ export function resolveAcpxQualification(
|
||||
if (!encoded) return undefined;
|
||||
let entries: unknown;
|
||||
try { entries = JSON.parse(encoded); } catch { throw new Error("Invalid ACPX qualification authorization"); }
|
||||
if (!Array.isArray(entries) || entries.length < 1 || entries.length > 3) {
|
||||
if (!Array.isArray(entries) || entries.length < 1 || entries.length > 4) {
|
||||
throw new Error("Invalid ACPX qualification authorization");
|
||||
}
|
||||
const agents = new Set<string>();
|
||||
for (const entry of entries) {
|
||||
if (!entry || typeof entry !== "object" || Array.isArray(entry)
|
||||
|| Object.keys(entry).sort().join(",") !== "agent,model"
|
||||
|| !["cursor", "copilot", "pi"].includes(entry.agent)
|
||||
|| !["cursor", "copilot", "pi", "kimi"].includes(entry.agent)
|
||||
|| typeof entry.model !== "string" || !entry.model.trim() || entry.model !== entry.model.trim()
|
||||
|| agents.has(entry.agent)) throw new Error("Invalid ACPX qualification authorization");
|
||||
agents.add(entry.agent);
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { agents, authUsers, companies, completionContracts, createDb, heartbeatRunEvents, heartbeatRuns, issueComments, issues,
|
||||
nativeRunFinalizations, nativeRunResults, type Db } from "@paperclipai/db";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
|
||||
import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js";
|
||||
import { NativeRunCoordinatorStore } from "./native-run-coordinator-store.js";
|
||||
import { finalizeNativeRun, recoverCommittedNativeChatResponse } from "./native-run-finalizer.js";
|
||||
import { nativeResponsePresentationWork } from "./native-response-presentation-work.js";
|
||||
|
||||
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_AGENT_CHAT_RESPONSE_TEST_DATABASE_URL");
|
||||
const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
describe.skipIf(!support.supported)("committed Agent Chat reply recovery", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
|
||||
let db: Db;
|
||||
beforeAll(async () => {
|
||||
if (!external) database = await startEmbeddedPostgresTestDatabase("agent-chat-response-");
|
||||
db = createDb(external ?? database!.connectionString);
|
||||
}, 60_000);
|
||||
afterAll(async () => { await db?.$client.end(); await database?.cleanup(); });
|
||||
|
||||
async function fixture(conversation = true) {
|
||||
const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID();
|
||||
const sessionId = randomUUID(), runnerId = randomUUID(), userId = `fixture-${randomUUID()}`, wakeId = randomUUID();
|
||||
await db.insert(authUsers).values({ id: userId, name: "Conversation owner", email: `${userId}@example.test`,
|
||||
createdAt: new Date("2020-01-01T00:00:00Z"), updatedAt: new Date("2020-01-01T00:00:00Z") });
|
||||
await db.insert(companies).values({ id: companyId, name: "Reply recovery fixture", issuePrefix: companyId.slice(0, 8) });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Conversation agent", adapterType: "paperclip_runner" });
|
||||
await db.insert(issues).values({ id: issueId, companyId, title: "Conversation reply", status: "in_progress", assigneeAgentId: agentId,
|
||||
...(conversation ? { conversationAgentId: agentId, conversationUserId: userId, conversationState: "active", conversationSessionGeneration: 0 } : {}) });
|
||||
await db.insert(issueComments).values({ id: wakeId, companyId, issueId, authorUserId: userId, body: "Hire one teammate and tell me when ready.",
|
||||
createdAt: new Date("2020-01-01T00:00:00Z") });
|
||||
const [contract] = await db.insert(completionContracts).values({ companyId, issueId, revision: 1,
|
||||
schemaVersion: "paperclip.completion-contract.v1", policyVersion: "test", risk: "standard", completionAuthority: "server_arbiter",
|
||||
incompleteCriteriaPolicy: "preserve_non_terminal", contractJson: { revision: "test", objective: "Answer the user", criteria: [] },
|
||||
canonicalSha256: "fixture-conversation-contract", createdByActorType: "system", createdByActorId: "test" }).returning();
|
||||
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: issueId, nativeSessionId: sessionId,
|
||||
runnerInstanceId: runnerId, completionContractId: contract.id, completionContractSha256: contract.canonicalSha256,
|
||||
status: "running", runtimeMode: "native", responsibleUserId: userId,
|
||||
contextSnapshot: { issueId, source: "issue.comment", wakeCommentId: wakeId, ...(conversation ? { conversationSessionGeneration: 0 } : {}) } });
|
||||
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId));
|
||||
const port = new NativeRunCoordinatorStore(db, { companyId, issueId, runId, agentId, normalizedSessionId: sessionId,
|
||||
runnerSourceInstanceId: runnerId, completionContractId: contract.id, completionContractSha256: contract.canonicalSha256,
|
||||
completionContractRevision: "test", completionContractCriterionIds: [] });
|
||||
const reply = "The teammate is hired and ready for your next instruction.";
|
||||
const prpEvent = { runId, normalizedSessionId: sessionId, turnId: "hire-turn", sourceEventId: "fixture-final-answer",
|
||||
payload: { kind: "agentMessage", channel: "final", providerPhase: "final", text: reply } };
|
||||
await db.insert(heartbeatRunEvents).values({ companyId, runId, agentId, seq: 170, eventType: "item.completed", payload: { prpEvent } });
|
||||
await port.completeRun({ result: { schema: "paperclip.run_result.v1", reportedWorkDisposition: "done",
|
||||
summary: "A completed hire, with its final message preserved separately.", completionClaim: { contractRevision: "test",
|
||||
objectiveSatisfied: true, criteria: [], remainingWork: [] }, evidence: [], verification: [], attentionRequests: [], artifacts: [] },
|
||||
terminal: { schema: "paperclip.prp.terminal.v1", runTerminalState: "succeeded", turnTerminalState: "completed", reportedWorkDisposition: "done" }, turnId: "hire-turn" });
|
||||
await finalizeNativeRun({ db, runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: false });
|
||||
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date("2020-01-01T00:02:00Z") }).where(eq(heartbeatRuns.id, runId));
|
||||
expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)))[0].phase).toBe("committed");
|
||||
const input = { companyId, issueId, runId };
|
||||
return { ...input, input, agentId, userId, wakeId, prpEvent, reply };
|
||||
}
|
||||
const comments = (runId: string) => db.select().from(issueComments).where(eq(issueComments.createdByRunId, runId));
|
||||
const issue = async (issueId: string) => (await db.select().from(issues).where(eq(issues.id, issueId)))[0];
|
||||
|
||||
it("finds a committed final Agent Chat turn, persists the exact reply once and only then settles waiting", async () => {
|
||||
const f = await fixture();
|
||||
expect(await comments(f.runId)).toEqual([]);
|
||||
expect(await issue(f.issueId)).toMatchObject({ status: "in_progress", conversationState: "active" });
|
||||
const work = nativeResponsePresentationWork(db), entries = await work.load(f.input);
|
||||
expect(entries).toEqual([expect.objectContaining({ id: f.runId })]);
|
||||
await expect(work.check(entries[0])).resolves.toBeNull();
|
||||
expect(await comments(f.runId)).toEqual([expect.objectContaining({ body: f.reply, authorAgentId: f.agentId, createdByRunId: f.runId })]);
|
||||
expect(await issue(f.issueId)).toMatchObject({ status: "in_review", conversationState: "waiting" });
|
||||
expect(await work.load(f.input)).toEqual([]);
|
||||
await expect(recoverCommittedNativeChatResponse(db, f.input)).resolves.toBe(false);
|
||||
expect(await comments(f.runId)).toHaveLength(1);
|
||||
});
|
||||
it("keeps a later user message actionable after recovering the earlier exact reply", async () => {
|
||||
const f = await fixture();
|
||||
await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId, authorUserId: f.userId, body: "Another instruction",
|
||||
createdAt: new Date("2020-01-01T00:01:00Z") });
|
||||
await expect(recoverCommittedNativeChatResponse(db, f.input)).resolves.toBe(true);
|
||||
expect(await comments(f.runId)).toHaveLength(1);
|
||||
expect(await issue(f.issueId)).toMatchObject({ status: "in_progress", conversationState: "active" });
|
||||
});
|
||||
it("does not duplicate a reply under concurrent recovery", async () => {
|
||||
const f = await fixture();
|
||||
expect((await Promise.all([recoverCommittedNativeChatResponse(db, f.input), recoverCommittedNativeChatResponse(db, f.input)])).sort())
|
||||
.toEqual([false, true]);
|
||||
expect(await comments(f.runId)).toHaveLength(1);
|
||||
});
|
||||
it.each(["skip", "generation", "agent", "user", "execution", "waiting", "turn", "event-run", "session", "channel", "kind", "digest", "partial"])(
|
||||
"keeps %s mismatches unchanged instead of publishing an unauthorized or incomplete reply", async fault => {
|
||||
const f = await fixture();
|
||||
if (["skip", "generation"].includes(fault)) await db.update(heartbeatRuns).set({ contextSnapshot: { issueId: f.issueId,
|
||||
source: "issue.comment", wakeCommentId: f.wakeId, conversationSessionGeneration: fault === "generation" ? 1 : 0,
|
||||
...(fault === "skip" ? { skipIssueComment: true } : {}) } }).where(eq(heartbeatRuns.id, f.runId));
|
||||
if (fault === "user") await db.update(heartbeatRuns).set({ responsibleUserId: "another-user" }).where(eq(heartbeatRuns.id, f.runId));
|
||||
if (fault === "agent") {
|
||||
const other = randomUUID(); await db.insert(agents).values({ id: other, companyId: f.companyId, name: "Other agent" });
|
||||
await db.update(issues).set({ conversationAgentId: other, assigneeAgentId: other }).where(eq(issues.id, f.issueId));
|
||||
}
|
||||
if (fault === "execution") {
|
||||
const other = randomUUID(); await db.insert(heartbeatRuns).values({ id: other, companyId: f.companyId, agentId: f.agentId });
|
||||
await db.update(issues).set({ executionRunId: other }).where(eq(issues.id, f.issueId));
|
||||
}
|
||||
if (fault === "waiting") await db.update(issues).set({ executionRunId: null, status: "in_review", conversationState: "waiting" }).where(eq(issues.id, f.issueId));
|
||||
if (["turn", "event-run", "session", "channel", "kind"].includes(fault)) {
|
||||
const event = structuredClone(f.prpEvent);
|
||||
if (fault === "turn") event.turnId = "foreign-turn";
|
||||
if (fault === "event-run") event.runId = randomUUID();
|
||||
if (fault === "session") event.normalizedSessionId = randomUUID();
|
||||
if (fault === "channel") event.payload.channel = "analysis";
|
||||
if (fault === "kind") event.payload.kind = "reasoning";
|
||||
await db.update(heartbeatRunEvents).set({ payload: { prpEvent: event } }).where(and(eq(heartbeatRunEvents.runId, f.runId), eq(heartbeatRunEvents.seq, 170)));
|
||||
}
|
||||
if (fault === "digest") await db.update(nativeRunResults).set({ canonicalSha256: "invalid-digest" }).where(eq(nativeRunResults.runId, f.runId));
|
||||
if (fault === "partial") await db.update(nativeRunFinalizations).set({ phase: "copying_back" }).where(eq(nativeRunFinalizations.runId, f.runId));
|
||||
const before = await issue(f.issueId);
|
||||
await expect(recoverCommittedNativeChatResponse(db, f.input)).resolves.toBe(false);
|
||||
expect(await comments(f.runId)).toEqual([]); expect(await issue(f.issueId)).toEqual(before);
|
||||
},
|
||||
);
|
||||
it.each([false, true])("preserves a selected reply without replacing or resurrecting it (deleted=%s)", async deleted => {
|
||||
const f = await fixture();
|
||||
const [saved] = await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId, authorAgentId: f.agentId,
|
||||
createdByRunId: f.runId, body: "Already selected", ...(deleted ? { deletedAt: new Date("2020-01-01T00:03:00Z") } : {}) }).returning();
|
||||
await db.update(heartbeatRuns).set({ resultJson: { presentationDecision: { commentId: saved.id, commentAction: "create" } } }).where(eq(heartbeatRuns.id, f.runId));
|
||||
await expect(recoverCommittedNativeChatResponse(db, f.input)).resolves.toBe(false);
|
||||
expect(await comments(f.runId)).toEqual([saved]);
|
||||
});
|
||||
it("never crosses company scope and keeps ordinary internal task response recovery intact", async () => {
|
||||
const f = await fixture(false);
|
||||
await expect(recoverCommittedNativeChatResponse(db, { ...f.input, companyId: randomUUID() })).resolves.toBe(false);
|
||||
const work = nativeResponsePresentationWork(db), entries = await work.load(f.input);
|
||||
expect(entries).toHaveLength(1); await expect(work.check(entries[0])).resolves.toBeNull();
|
||||
expect(await comments(f.runId)).toEqual([expect.objectContaining({ body: f.reply })]);
|
||||
expect((await issue(f.issueId)).conversationAgentId).toBeNull();
|
||||
});
|
||||
it("rolls back a partial reply write without changing the waiting state or consuming recovery", async () => {
|
||||
const f = await fixture(), abort = new Error("fixture-commit-aborted");
|
||||
const failing = new Proxy(db, { get(target, key) { return key === "transaction"
|
||||
? (callback: Parameters<Db["transaction"]>[0]) => db.transaction(async tx => { await callback(tx); throw abort; })
|
||||
: Reflect.get(target, key); } });
|
||||
await expect(recoverCommittedNativeChatResponse(failing, f.input)).rejects.toBe(abort);
|
||||
expect(await comments(f.runId)).toEqual([]);
|
||||
expect(await issue(f.issueId)).toMatchObject({ status: "in_progress", conversationState: "active" });
|
||||
await expect(recoverCommittedNativeChatResponse(db, f.input)).resolves.toBe(true);
|
||||
expect(await comments(f.runId)).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,61 @@
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { sql } from "drizzle-orm";
|
||||
import { inspectNativeRunnerUpgradeHeader } from "./native-runner-checkpoint.js";
|
||||
|
||||
/** An explicit, paginated diagnostic. Never called by an idle/background scan.
|
||||
* This inventories selected retained task-session pointers, not all history.
|
||||
* Header evidence is preliminary: full snapshot/journal validation still occurs
|
||||
* only when ordinary admission selects its exact continuation source. */
|
||||
export async function readNativeControllerRecoveryInventory(db: Db, input: { afterId?: string; limit?: number } = {}) {
|
||||
if (input.limit !== undefined && !Number.isFinite(input.limit)) throw new Error("native_recovery_inventory_limit_invalid");
|
||||
const limit = Math.min(128, Math.max(1, Math.floor(input.limit ?? 128)));
|
||||
const rows = await db.transaction(async tx => {
|
||||
await tx.execute(sql`set local statement_timeout = '2s'`);
|
||||
return tx.execute<{
|
||||
id: string; runId: string | null; companyId: string; issueId: string | null; sessionId: string | null;
|
||||
runnerId: string | null; metadata: Record<string, unknown> | null; material: Record<string, unknown> | null;
|
||||
pending: boolean;
|
||||
}>(sql`with selected as (
|
||||
select id, company_id, agent_id, last_run_id from agent_task_sessions
|
||||
where adapter_type = 'paperclip_runner' ${input.afterId ? sql`and id > ${input.afterId}::uuid` : sql``}
|
||||
order by id limit ${limit + 1}
|
||||
) select s.id, s.company_id as "companyId", r.id as "runId", r.native_issue_id as "issueId",
|
||||
r.native_session_id as "sessionId", r.runner_instance_id as "runnerId",
|
||||
case when pg_column_size(r.runner_profile_json->'nativeControllerRecovery') <= 4096
|
||||
then r.runner_profile_json->'nativeControllerRecovery' else null end as metadata,
|
||||
case when pg_column_size(c.material) <= 262144 and sum(pg_column_size(c.material)) over () <= 8388608
|
||||
then c.material else null end as material,
|
||||
(r.finished_at is null or r.status not in ('succeeded', 'failed', 'cancelled', 'interrupted', 'timed_out')
|
||||
or f.run_id is null or f.lease_owner is not null or f.recovery_state is not null
|
||||
or f.phase not in ('committed', 'terminal_failure')) is true as pending
|
||||
from selected s left join heartbeat_runs r on r.id = s.last_run_id and r.company_id = s.company_id and r.agent_id = s.agent_id
|
||||
left join native_run_finalizations f on f.run_id = r.id and f.company_id = s.company_id
|
||||
left join native_runner_checkpoints c on c.run_id = r.id and c.company_id = s.company_id
|
||||
and c.issue_id = r.native_issue_id and c.part = 'header'
|
||||
order by s.id`);
|
||||
}, { accessMode: "read only" });
|
||||
const truncated = rows.length > limit;
|
||||
const page = rows.slice(0, limit);
|
||||
const reasons: Record<string, number> = {};
|
||||
let selectedRetainedEvidence = 0, outstandingObligations = 0;
|
||||
for (const row of page) {
|
||||
let reason: string | null = null;
|
||||
if (row.pending) { outstandingObligations++; reason = "persistence_or_handoff_pending"; }
|
||||
else if (!row.runId || !row.issueId || !row.sessionId) reason = "selected_binding_missing";
|
||||
else if (row.metadata?.schema !== "paperclip.native-controller-recovery.v1") reason = "metadata_unsupported_or_missing";
|
||||
else {
|
||||
const checkpoint = row.metadata.checkpoint as Record<string, unknown> | null;
|
||||
const header = row.material ? inspectNativeRunnerUpgradeHeader({ companyId: row.companyId, issueId: row.issueId, runId: row.runId }, row.material) : null;
|
||||
if (!checkpoint || checkpoint.runId !== row.runId || checkpoint.runnerInstanceId !== row.runnerId
|
||||
|| checkpoint.normalizedSessionId !== row.sessionId || !header
|
||||
|| header.runnerInstanceId !== row.runnerId || header.normalizedSessionId !== row.sessionId) reason = "checkpoint_binding_missing";
|
||||
else selectedRetainedEvidence++;
|
||||
}
|
||||
if (reason) reasons[reason] = (reasons[reason] ?? 0) + 1;
|
||||
}
|
||||
return { version: 1 as const, observedAt: new Date().toISOString(), inspected: page.length,
|
||||
coverage: truncated || input.afterId ? "partial" as const : "selected_task_session_pointers" as const,
|
||||
truncated, nextAfterId: truncated ? page.at(-1)!.id : null,
|
||||
selectedRetainedEvidence, outstandingObligations, reasons,
|
||||
fullPayloadsValidated: false as const };
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { agentTaskSessions, agents, companies, createDb, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults } from "@paperclipai/db";
|
||||
import { DurablePrpControlPlane, type PersistedNativeSession } from "../../vendor/paperclip-runner/index.js";
|
||||
import { startRecoveryPostgres, recoveryPostgresSupport } from "../../__tests__/helpers/recovery-postgres.js";
|
||||
import { createIsolatedTestHome } from "../../__tests__/helpers/isolated-test-home.js";
|
||||
import { prepareNativeHeartbeatRun } from "./prepare-native-run.js";
|
||||
import { nativeRunnerCheckpointWriter } from "./native-runner-checkpoint.js";
|
||||
import { createNativeControllerRecovery, deferNativeControllerRecovery, importNativeControllerRecovery, NATIVE_RETAINED_OWNER_WAIT, resolveNativeControllerRecovery } from "./native-controller-recovery.js";
|
||||
import { claimNativeSessionResumptions } from "./native-finalization-reconciler.js";
|
||||
import { claimNativeRestartRecoveries } from "./native-restart-recovery.js";
|
||||
import { readNativeControllerRecoveryInventory } from "./native-controller-recovery-inventory.js";
|
||||
|
||||
describe.skipIf(!recoveryPostgresSupport.supported)("native controller durable discovery and import", () => {
|
||||
let database: Awaited<ReturnType<typeof startRecoveryPostgres>>;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let home: ReturnType<typeof createIsolatedTestHome>;
|
||||
let root: string;
|
||||
const previousHome = process.env.PAPERCLIP_HOME;
|
||||
const companyId = randomUUID(), agentId = randomUUID();
|
||||
const compatibility = { configDigest: `sha256:${"a".repeat(64)}`, processConfigDigest: null };
|
||||
beforeAll(async () => {
|
||||
home = createIsolatedTestHome();
|
||||
process.env.PAPERCLIP_HOME = home.home;
|
||||
root = await mkdtemp(join(tmpdir(), "native-controller-discovery-"));
|
||||
database = await startRecoveryPostgres("native-controller-discovery-");
|
||||
db = createDb(database.connectionString);
|
||||
await db.insert(companies).values({ id: companyId, name: "Controller discovery", issuePrefix: "NCD" });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Runner", adapterType: "paperclip_runner", adapterConfig: { provider: "codex" } });
|
||||
}, 60_000);
|
||||
afterAll(async () => {
|
||||
await database?.cleanup();
|
||||
if (root) await rm(root, { recursive: true, force: true });
|
||||
home?.cleanup();
|
||||
if (previousHome === undefined) delete process.env.PAPERCLIP_HOME;
|
||||
else process.env.PAPERCLIP_HOME = previousHome;
|
||||
});
|
||||
|
||||
async function seed() {
|
||||
const issueId = randomUUID(), runId = randomUUID();
|
||||
await db.insert(issues).values({ id: issueId, companyId, title: "Continue retained execution", assigneeAgentId: agentId, status: "in_progress" });
|
||||
const [run] = await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running", runtimeMode: "native", nativeIssueId: issueId }).returning();
|
||||
const native = await prepareNativeHeartbeatRun({ db, run, issue: { id: issueId, title: "Continue retained execution", description: null, reviewPolicy: null }, environmentLeaseId: randomUUID() });
|
||||
const owner = { companyId, issueId, runId, authorityRunId: runId, bootId: randomUUID(), generation: 1, leaseOwner: randomUUID() };
|
||||
await db.insert(nativeRunFinalizations).values({ companyId, issueId, runId, phase: "observed", controllerBootId: owner.bootId,
|
||||
controllerGeneration: 1, leaseOwner: owner.leaseOwner, leaseExpiresAt: new Date(Date.now() + 60_000) });
|
||||
const core = new DurablePrpControlPlane({ stateDirectory: join(root, runId),
|
||||
identity: { runId, normalizedSessionId: native.normalizedSessionId, runnerInstanceId: native.runnerInstanceId,
|
||||
environmentLeaseId: native.environmentLeaseId, turnId: native.turnId, itemId: native.itemId },
|
||||
expectedRunnerVersion: "discovery-fixture", expectedRunnerDigest: `sha256:${"a".repeat(64)}` });
|
||||
core.setCheckpointPersistence(nativeRunnerCheckpointWriter(db, owner));
|
||||
await core.flushCheckpoint();
|
||||
await core.stop();
|
||||
const snapshot: PersistedNativeSession = { backendKind: "mock", sessionId: native.normalizedSessionId,
|
||||
identity: { companyId, agentId, issueId, runId, sessionId: native.normalizedSessionId } };
|
||||
const [prepared] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
|
||||
const [result] = await db.insert(nativeRunResults).values({ companyId, issueId, runId,
|
||||
completionContractId: prepared.completionContractId!, serverFingerprint: "fixture", schemaStatus: "accepted",
|
||||
resultJson: {}, canonicalSha256: "fixture" }).returning();
|
||||
await db.update(nativeRunFinalizations).set({ phase: "committed", resultId: result.id, leaseOwner: null, leaseExpiresAt: null }).where(eq(nativeRunFinalizations.runId, runId));
|
||||
const profile = { ...prepared.runnerProfileJson, sessionCheckpoint: snapshot };
|
||||
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date(), runnerProfileJson: profile }).where(eq(heartbeatRuns.id, runId));
|
||||
const scope = { companyId, agentId, issueId, runId, normalizedSessionId: native.normalizedSessionId };
|
||||
return { scope, native, profile, snapshot, owner,
|
||||
supervisor: { schema: "paperclip.native-session-supervisor.v1", ...compatibility, snapshot } };
|
||||
}
|
||||
|
||||
it("imports a retained source once, rejects stale input, and resolves it without any local projection", async () => {
|
||||
const fixture = await seed();
|
||||
const input = { ...fixture.scope, expectedConfigDigest: compatibility.configDigest, supervisor: fixture.supervisor };
|
||||
expect(await importNativeControllerRecovery(db, { ...input, supervisor: { ...input.supervisor, snapshot: { ...fixture.snapshot, providerSessionId: "stale" } } }))
|
||||
.toEqual({ kind: "missing_evidence", reason: "stale_supervisor_snapshot" });
|
||||
expect(await importNativeControllerRecovery(db, input)).toEqual({ kind: "imported" });
|
||||
expect(await importNativeControllerRecovery(db, input)).toEqual({ kind: "unchanged" });
|
||||
await rm(join(root, fixture.scope.runId), { recursive: true });
|
||||
const currentRunId = randomUUID();
|
||||
await db.insert(heartbeatRuns).values({ id: currentRunId, companyId, agentId, nativeIssueId: fixture.scope.issueId,
|
||||
nativeSessionId: fixture.scope.normalizedSessionId, status: "running", runtimeMode: "native" });
|
||||
const recovered = await resolveNativeControllerRecovery(db, { ...fixture.scope, currentRunId });
|
||||
expect(recovered.kind).toBe("recoverable");
|
||||
if (recovered.kind !== "recoverable") throw new Error(JSON.stringify(recovered));
|
||||
expect(recovered.snapshot).toEqual(fixture.snapshot);
|
||||
expect(recovered.checkpoint.recoveryEvidence.runnerInstanceId).toBe(fixture.native.runnerInstanceId);
|
||||
const deadline = new Date(Date.now() + 60_000);
|
||||
await db.update(nativeRunFinalizations).set({ leaseOwner: "other-controller", leaseExpiresAt: deadline }).where(eq(nativeRunFinalizations.runId, fixture.scope.runId));
|
||||
expect(await importNativeControllerRecovery(db, input)).toEqual({ kind: "waiting_for_owner", deadline });
|
||||
expect(await resolveNativeControllerRecovery(db, { ...fixture.scope, currentRunId })).toEqual({ kind: "waiting_for_owner", deadline });
|
||||
});
|
||||
|
||||
it("holds mismatched snapshots and runner bindings instead of falling back to fresh execution", async () => {
|
||||
const fixture = await seed(), currentRunId = randomUUID();
|
||||
const metadata = createNativeControllerRecovery(fixture.snapshot, compatibility, { runId: fixture.scope.runId,
|
||||
runnerInstanceId: fixture.native.runnerInstanceId, normalizedSessionId: fixture.scope.normalizedSessionId });
|
||||
await db.insert(heartbeatRuns).values({ id: currentRunId, companyId, agentId, nativeIssueId: fixture.scope.issueId,
|
||||
nativeSessionId: fixture.scope.normalizedSessionId, status: "running", runtimeMode: "native" });
|
||||
for (const envelope of [{ ...metadata, snapshotSha256: "wrong" }, { ...metadata, checkpoint: { ...metadata.checkpoint!, runnerInstanceId: randomUUID() } }]) {
|
||||
await db.update(heartbeatRuns).set({ runnerProfileJson: { ...fixture.profile, nativeControllerRecovery: envelope } }).where(eq(heartbeatRuns.id, fixture.scope.runId));
|
||||
expect(await resolveNativeControllerRecovery(db, { ...fixture.scope, currentRunId }))
|
||||
.toMatchObject({ kind: "missing_evidence", reason: "snapshot_digest_or_checkpoint_binding" });
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps an unstarted follow-up on the same run until the retained owner's deadline", async () => {
|
||||
const fixture = await seed(), runId = randomUUID();
|
||||
const scope = { ...fixture.scope, runId };
|
||||
const deadline = new Date(Date.now() + 60_000);
|
||||
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: scope.issueId,
|
||||
nativeSessionId: scope.normalizedSessionId, status: "running", runtimeMode: "native",
|
||||
// The admission boundary has already stored its input; the deferral
|
||||
// and resumption owners never execute this transport-independent fixture.
|
||||
runnerProfileJson: { nativeExecutionInput: { binding: scope } } });
|
||||
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, scope.issueId));
|
||||
await db.insert(nativeRunFinalizations).values({ companyId, issueId: scope.issueId, runId, phase: "observed" });
|
||||
await deferNativeControllerRecovery(db, { ...scope, retryAt: deadline });
|
||||
const read = () => db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId));
|
||||
expect((await read())[0]).toMatchObject({ phase: "retryable_failure", attempt: 0, leaseOwner: null,
|
||||
nextAttemptAt: deadline, failureCode: NATIVE_RETAINED_OWNER_WAIT });
|
||||
expect(await claimNativeSessionResumptions({ db, runnerInstanceId: "successor", runIds: [runId] })).toEqual([]);
|
||||
// Even after expiry, restart adoption cannot reinterpret this pre-provider
|
||||
// wait as a crashed provider. The ordinary resumption owner consumes it.
|
||||
expect(await claimNativeRestartRecoveries({ db, runIds: [runId], restartKind: "hard", now: deadline })).toEqual([]);
|
||||
const claims = await claimNativeSessionResumptions({ db, runnerInstanceId: "successor", runIds: [runId], now: deadline });
|
||||
expect(claims).toHaveLength(1);
|
||||
const nextDeadline = new Date(deadline.getTime() + 60_000);
|
||||
await deferNativeControllerRecovery(db, { ...scope, leaseOwner: claims[0].leaseOwner, retryAt: nextDeadline });
|
||||
expect((await read())[0]).toMatchObject({ attempt: 0, leaseOwner: null, nextAttemptAt: nextDeadline });
|
||||
await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, runId));
|
||||
expect(await claimNativeSessionResumptions({ db, runnerInstanceId: "successor", runIds: [runId], now: nextDeadline })).toEqual([]);
|
||||
});
|
||||
|
||||
it.each(["live-owner", "expired-owner", "provider-attempt", "stopped", "operator-stop"])("does not publish an admission wait after authority changes: %s", async scenario => {
|
||||
const fixture = await seed(), runId = randomUUID();
|
||||
const scope = { ...fixture.scope, runId };
|
||||
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: scope.issueId,
|
||||
status: scenario === "stopped" ? "cancelled" : "running", runtimeMode: "native",
|
||||
resultJson: scenario === "operator-stop" ? { cancellation: { source: "operator", expected: true,
|
||||
initiator: { type: "user" }, reason: "Stop", recordedAt: new Date().toISOString() } } : null,
|
||||
runnerProfileJson: { nativeExecutionInput: { binding: scope } } });
|
||||
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, scope.issueId));
|
||||
await db.insert(nativeRunFinalizations).values({ companyId, issueId: scope.issueId, runId, phase: "observed",
|
||||
attempt: scenario === "provider-attempt" ? 1 : 0,
|
||||
leaseOwner: scenario.endsWith("-owner") ? "other-controller" : null,
|
||||
leaseExpiresAt: scenario === "expired-owner" ? new Date(Date.now() - 1_000) : null });
|
||||
await expect(deferNativeControllerRecovery(db, { ...scope,
|
||||
leaseOwner: scenario === "expired-owner" ? "other-controller" : undefined,
|
||||
retryAt: new Date(Date.now() + 60_000) }))
|
||||
.rejects.toThrow("wait_authority_changed");
|
||||
expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)))[0].phase).toBe("observed");
|
||||
});
|
||||
|
||||
it("reports bounded retained-pointer coverage without loading historical payloads", async () => {
|
||||
const fixture = await seed();
|
||||
await importNativeControllerRecovery(db, { ...fixture.scope, expectedConfigDigest: compatibility.configDigest, supervisor: fixture.supervisor });
|
||||
await db.insert(agentTaskSessions).values([0, 1].map(index => ({ companyId, agentId, taskKey: `${fixture.scope.issueId}-${index}`,
|
||||
adapterType: "paperclip_runner", lastRunId: fixture.scope.runId })));
|
||||
const first = await readNativeControllerRecoveryInventory(db, { limit: 1 });
|
||||
expect(first).toMatchObject({ inspected: 1, coverage: "partial", truncated: true, selectedRetainedEvidence: 1, fullPayloadsValidated: false });
|
||||
const next = await readNativeControllerRecoveryInventory(db, { limit: 1, afterId: first.nextAfterId! });
|
||||
expect(next).toMatchObject({ inspected: 1, coverage: "partial", truncated: false, selectedRetainedEvidence: 1 });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,43 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { createNativeControllerRecovery, readNativeControllerRecovery } from "./native-controller-recovery.js";
|
||||
import { nativeSha256 } from "./canonical.js";
|
||||
import { redactRegisteredSecretValues } from "../run-secret-redaction.js";
|
||||
import type { PersistedNativeSession } from "../../vendor/paperclip-runner/index.js";
|
||||
|
||||
const snapshot: PersistedNativeSession = { backendKind: "mock", sessionId: "provider",
|
||||
identity: { companyId: "company", agentId: "agent", issueId: "issue", runId: "run", sessionId: "session" } };
|
||||
const compatibility = { configDigest: `sha256:${"a".repeat(64)}`, processConfigDigest: null };
|
||||
const binding = { runId: "run", runnerInstanceId: "runner", normalizedSessionId: "session" };
|
||||
|
||||
describe("native controller recovery envelope", () => {
|
||||
it("binds the stored snapshot, including redaction, without serializing its contents twice", () => {
|
||||
const stored = { ...snapshot, providerSessionId: "[REDACTED]" };
|
||||
const envelope = createNativeControllerRecovery(stored, compatibility, binding);
|
||||
expect(envelope.snapshotSha256).toBe(nativeSha256(stored));
|
||||
expect(readNativeControllerRecovery(envelope, JSON.parse(JSON.stringify(stored)))).toEqual(envelope);
|
||||
expect(readNativeControllerRecovery(envelope, { ...stored, providerSessionId: "secret-before-redaction" })).toBeNull();
|
||||
expect(JSON.stringify(envelope)).not.toContain("[REDACTED]");
|
||||
expect(Buffer.byteLength(JSON.stringify(envelope))).toBeLessThan(4096);
|
||||
});
|
||||
|
||||
it("does not certify a missing PRP publication or unknown format", () => {
|
||||
const pending = createNativeControllerRecovery(snapshot, compatibility, null);
|
||||
expect(readNativeControllerRecovery(pending, snapshot)).toBeNull();
|
||||
const published = createNativeControllerRecovery(snapshot, compatibility, binding);
|
||||
expect(readNativeControllerRecovery({ ...published, schema: "future" }, snapshot)).toBeNull();
|
||||
expect(readNativeControllerRecovery({ ...published, processConfigDigest: "unverified" }, snapshot)).toBeNull();
|
||||
});
|
||||
|
||||
it("enforces the metadata byte bound on both write and read", () => {
|
||||
expect(() => createNativeControllerRecovery(snapshot, { ...compatibility, configDigest: "x".repeat(4096) }, binding))
|
||||
.toThrow("metadata_too_large");
|
||||
expect(readNativeControllerRecovery({ ...createNativeControllerRecovery(snapshot, compatibility, binding), extra: "x".repeat(4096) }, snapshot))
|
||||
.toBeNull();
|
||||
});
|
||||
|
||||
it("keeps compatibility metadata out of public run projections", () => {
|
||||
expect(redactRegisteredSecretValues({ runnerProfileJson: { preserved: true,
|
||||
nativeControllerRecovery: createNativeControllerRecovery(snapshot, compatibility, binding) } }, []))
|
||||
.toEqual({ runnerProfileJson: { preserved: true } });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,270 @@
|
||||
import { and, eq, ne, sql } from "drizzle-orm";
|
||||
import { environmentLeases, heartbeatRuns, issues, nativeRunFinalizations, nativeRunnerCheckpoints, type Db } from "@paperclipai/db";
|
||||
import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target";
|
||||
import type { DurablePrpCheckpoint, PersistedNativeSession } from "../../vendor/paperclip-runner/index.js";
|
||||
import { notifyNativeFinalizationWork } from "../native-finalization-notifications.js";
|
||||
import { readRunCancellation } from "../run-cancellation.js";
|
||||
import { nativeSha256 } from "./canonical.js";
|
||||
import { findNativeSessionResumeRun, isNativeSessionId, isUnusedNativeSessionBootstrap, nativeSessionProviderEvidence } from "./native-session-resume.js";
|
||||
import { loadNativeRunnerCheckpoint } from "./native-runner-checkpoint.js";
|
||||
import { redactRunnerApiPayloads } from "./runner-api-log-redaction.js";
|
||||
|
||||
export const NATIVE_CONTROLLER_RECOVERY_MAX_BYTES = 4 * 1024;
|
||||
|
||||
/** Private run-profile metadata. No credentials or controller-local paths. */
|
||||
export interface NativeControllerRecovery {
|
||||
schema: "paperclip.native-controller-recovery.v1";
|
||||
snapshotSha256: string;
|
||||
configDigest: string;
|
||||
processConfigDigest: string | null;
|
||||
checkpoint: {
|
||||
runId: string;
|
||||
runnerInstanceId: string;
|
||||
normalizedSessionId: string;
|
||||
} | null;
|
||||
}
|
||||
|
||||
export type NativeControllerCompatibility = Pick<NativeControllerRecovery, "configDigest" | "processConfigDigest">;
|
||||
|
||||
export function createNativeControllerRecovery(
|
||||
snapshot: PersistedNativeSession,
|
||||
compatibility: NativeControllerCompatibility,
|
||||
checkpoint: NativeControllerRecovery["checkpoint"],
|
||||
): NativeControllerRecovery {
|
||||
const value: NativeControllerRecovery = {
|
||||
schema: "paperclip.native-controller-recovery.v1",
|
||||
snapshotSha256: nativeSha256(snapshot),
|
||||
configDigest: compatibility.configDigest,
|
||||
processConfigDigest: compatibility.processConfigDigest,
|
||||
checkpoint,
|
||||
};
|
||||
if (Buffer.byteLength(JSON.stringify(value)) > NATIVE_CONTROLLER_RECOVERY_MAX_BYTES) {
|
||||
throw new Error("native_controller_recovery_metadata_too_large");
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function record(value: unknown): Record<string, unknown> {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||||
}
|
||||
|
||||
export function readNativeControllerRecovery(value: unknown, snapshot: unknown): NativeControllerRecovery | null {
|
||||
const envelope = record(value), checkpoint = record(envelope.checkpoint);
|
||||
if (Buffer.byteLength(JSON.stringify(value) ?? "null") > NATIVE_CONTROLLER_RECOVERY_MAX_BYTES
|
||||
|| envelope.schema !== "paperclip.native-controller-recovery.v1"
|
||||
|| typeof envelope.snapshotSha256 !== "string" || envelope.snapshotSha256 !== nativeSha256(snapshot)
|
||||
|| typeof envelope.configDigest !== "string" || !/^sha256:[a-f0-9]{64}$/.test(envelope.configDigest)
|
||||
|| !(envelope.processConfigDigest === null || (typeof envelope.processConfigDigest === "string" && /^[a-f0-9]{64}$/.test(envelope.processConfigDigest)))
|
||||
|| ![checkpoint.runId, checkpoint.runnerInstanceId, checkpoint.normalizedSessionId].every(v => typeof v === "string" && v.length > 0)) return null;
|
||||
return envelope as unknown as NativeControllerRecovery;
|
||||
}
|
||||
|
||||
export type NativeControllerRecoveryOutcome =
|
||||
| { kind: "recoverable"; snapshot: PersistedNativeSession; metadata: NativeControllerRecovery; checkpoint: DurablePrpCheckpoint }
|
||||
| { kind: "waiting_for_owner"; deadline: Date }
|
||||
| { kind: "missing_evidence" | "unsupported" | "dependency_unavailable"; reason: string; cause?: unknown }
|
||||
| { kind: "no_prior_execution" };
|
||||
|
||||
export const NATIVE_RETAINED_OWNER_WAIT = "native_controller_recovery_waiting_for_owner";
|
||||
|
||||
export class NativeControllerRecoveryWaitingError extends Error {
|
||||
constructor(readonly retryAt: Date) {
|
||||
super(NATIVE_RETAINED_OWNER_WAIT);
|
||||
this.name = "NativeControllerRecoveryWaitingError";
|
||||
}
|
||||
}
|
||||
|
||||
/** Admission has not crossed the provider boundary. Retain this same business
|
||||
* run on the existing retry owner; never turn an ownership wait into a failure
|
||||
* or invent a new provider attempt. The usual resumption gates recheck Stop,
|
||||
* permissions and task ownership when this deadline arrives. */
|
||||
export async function deferNativeControllerRecovery(db: Db, input: {
|
||||
companyId: string; agentId: string; issueId: string; runId: string;
|
||||
leaseOwner?: string; retryAt: Date;
|
||||
}): Promise<void> {
|
||||
await db.transaction(async tx => {
|
||||
const [task] = await tx.select({ executionRunId: issues.executionRunId }).from(issues)
|
||||
.where(and(eq(issues.companyId, input.companyId), eq(issues.id, input.issueId))).for("update");
|
||||
const [owner] = await tx.select().from(nativeRunFinalizations).where(and(
|
||||
eq(nativeRunFinalizations.companyId, input.companyId), eq(nativeRunFinalizations.issueId, input.issueId),
|
||||
eq(nativeRunFinalizations.runId, input.runId))).for("update");
|
||||
const [run] = await tx.select().from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.id, input.runId), eq(heartbeatRuns.companyId, input.companyId),
|
||||
eq(heartbeatRuns.agentId, input.agentId), eq(heartbeatRuns.nativeIssueId, input.issueId))).for("update");
|
||||
const [authority] = await tx.select({ runId: nativeRunnerCheckpoints.runId }).from(nativeRunnerCheckpoints)
|
||||
.where(and(eq(nativeRunnerCheckpoints.companyId, input.companyId), eq(nativeRunnerCheckpoints.runId, input.runId))).limit(1);
|
||||
const progress = await nativeSessionProviderEvidence(tx as unknown as Db, [input.runId]);
|
||||
const cancellation = record(run?.resultJson?.nativeCancellation);
|
||||
const stop = readRunCancellation(run?.resultJson);
|
||||
const [clock] = await tx.execute<{ now: Date }>(sql`select clock_timestamp() as now`);
|
||||
if (!run || run.runtimeMode !== "native" || run.status !== "running" || task?.executionRunId !== input.runId
|
||||
|| !owner || owner.phase !== "observed" || owner.resultId || owner.attempt !== 0
|
||||
|| owner.controllerGeneration !== 0 || owner.leaseOwner !== (input.leaseOwner ?? null)
|
||||
|| (input.leaseOwner && (!owner.leaseExpiresAt || owner.leaseExpiresAt <= new Date(clock.now)))
|
||||
|| run.processPid || run.processGroupId || run.processStartedAt || authority || progress.size
|
||||
|| !record(run.runnerProfileJson).nativeExecutionInput || run.resultJson?.startupCancellation
|
||||
|| (stop?.source === "operator" && stop.expected)
|
||||
|| (["run", "issue"].includes(String(cancellation.scope)) && ["pending", "acknowledged"].includes(String(cancellation.dispatchState)))) {
|
||||
throw new Error("native_controller_recovery_wait_authority_changed");
|
||||
}
|
||||
await notifyNativeFinalizationWork(tx, input);
|
||||
await tx.update(nativeRunFinalizations).set({
|
||||
phase: "retryable_failure", leaseOwner: null, leaseExpiresAt: null, controlDeadlineAt: null,
|
||||
nextAttemptAt: input.retryAt, recoveryState: "resuming_session", failureCode: NATIVE_RETAINED_OWNER_WAIT,
|
||||
failureDetail: { originalFailureCode: NATIVE_RETAINED_OWNER_WAIT,
|
||||
message: "Waiting for the retained native controller owner to release its lease.",
|
||||
recoveryMode: record(run.runnerProfileJson).sessionCheckpoint ? "same_session_resume" : "bootstrap_retry",
|
||||
providerSessionEstablished: false, providerEventsExist: false },
|
||||
updatedAt: new Date(),
|
||||
}).where(eq(nativeRunFinalizations.runId, input.runId));
|
||||
});
|
||||
}
|
||||
|
||||
/** Bind a read of retained Runner lifecycle to the surviving sandbox. The
|
||||
* ordinary admission owner still checks current permissions before execution. */
|
||||
export async function assertNativeRetainedSandboxBinding(db: Db, input: {
|
||||
companyId: string; issueId: string; runId: string;
|
||||
target: Extract<AdapterExecutionTarget, { kind: "remote" }>;
|
||||
}): Promise<void> {
|
||||
const { target } = input;
|
||||
if (target.transport !== "sandbox" || target.sandboxLeaseAcquisition?.outcome !== "resumed"
|
||||
|| !target.providerKey || !target.environmentId || !target.leaseId) {
|
||||
throw new Error("native_controller_recovery_dependency_unavailable:retained_sandbox");
|
||||
}
|
||||
const [owner] = await db.select().from(nativeRunFinalizations).where(and(
|
||||
eq(nativeRunFinalizations.companyId, input.companyId), eq(nativeRunFinalizations.issueId, input.issueId),
|
||||
eq(nativeRunFinalizations.runId, input.runId)));
|
||||
const leases = await db.select({ metadata: environmentLeases.metadata, status: environmentLeases.status }).from(environmentLeases).where(and(
|
||||
eq(environmentLeases.companyId, input.companyId), eq(environmentLeases.issueId, input.issueId),
|
||||
eq(environmentLeases.heartbeatRunId, input.runId), eq(environmentLeases.environmentId, target.environmentId),
|
||||
eq(environmentLeases.provider, target.providerKey), eq(environmentLeases.providerLeaseId, target.sandboxLeaseAcquisition.providerLeaseId)));
|
||||
if (!owner || owner.leaseOwner || owner.phase !== "committed" || !owner.resultId || owner.recoveryState
|
||||
|| leases.length !== 1 || leases[0].metadata?.remoteCwd !== target.remoteCwd
|
||||
|| !["active", "released", "expired"].includes(leases[0].status)) {
|
||||
throw new Error("native_controller_recovery_missing_evidence:retained_sandbox_binding");
|
||||
}
|
||||
}
|
||||
|
||||
/** Only the existing bounded selector chooses history. Never search backwards
|
||||
* again when its selected record is incomplete or incompatible. This operation
|
||||
* reads evidence; it grants neither command nor acknowledgement authority. */
|
||||
export async function resolveNativeControllerRecovery(db: Db, input: {
|
||||
companyId: string; agentId: string; issueId: string; normalizedSessionId: string; currentRunId: string;
|
||||
}): Promise<NativeControllerRecoveryOutcome> {
|
||||
if (!isNativeSessionId(input.normalizedSessionId)) return { kind: "missing_evidence", reason: "session_binding" };
|
||||
const [current] = await db.select({ createdAt: heartbeatRuns.createdAt, processPid: heartbeatRuns.processPid,
|
||||
processGroupId: heartbeatRuns.processGroupId, processStartedAt: heartbeatRuns.processStartedAt,
|
||||
runnerProfileJson: heartbeatRuns.runnerProfileJson }).from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.id, input.currentRunId), eq(heartbeatRuns.companyId, input.companyId),
|
||||
eq(heartbeatRuns.agentId, input.agentId), eq(heartbeatRuns.nativeIssueId, input.issueId),
|
||||
eq(heartbeatRuns.nativeSessionId, input.normalizedSessionId)));
|
||||
if (!current) return { kind: "missing_evidence", reason: "current_binding" };
|
||||
const source = await findNativeSessionResumeRun(db, { ...input, beforeCreatedAt: current.createdAt, includeProgressBarrier: true });
|
||||
if (!source) {
|
||||
// An exhausted candidate window is unknown history, not fresh-bootstrap
|
||||
// proof. Read only IDs, and retain the selector's existing 32-row boundary.
|
||||
const prior = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(
|
||||
ne(heartbeatRuns.id, input.currentRunId), eq(heartbeatRuns.companyId, input.companyId),
|
||||
eq(heartbeatRuns.agentId, input.agentId), eq(heartbeatRuns.nativeIssueId, input.issueId),
|
||||
eq(heartbeatRuns.nativeSessionId, input.normalizedSessionId))).limit(33);
|
||||
if (prior.length > 32) return { kind: "missing_evidence", reason: "candidate_window_exhausted" };
|
||||
const progress = await nativeSessionProviderEvidence(db, [input.currentRunId]);
|
||||
if (!isUnusedNativeSessionBootstrap(current, progress.has(input.currentRunId))) {
|
||||
return { kind: "missing_evidence", reason: "current_execution_progress" };
|
||||
}
|
||||
// The normal admission and remote untouched-session claim still execute.
|
||||
return { kind: "no_prior_execution" };
|
||||
}
|
||||
const [owner] = await db.select({ leaseOwner: nativeRunFinalizations.leaseOwner, leaseExpiresAt: nativeRunFinalizations.leaseExpiresAt }).from(nativeRunFinalizations)
|
||||
.where(and(eq(nativeRunFinalizations.runId, source.id), eq(nativeRunFinalizations.companyId, input.companyId),
|
||||
eq(nativeRunFinalizations.issueId, input.issueId)));
|
||||
if (owner?.leaseOwner && owner.leaseExpiresAt && owner.leaseExpiresAt.getTime() > Date.now()) {
|
||||
return { kind: "waiting_for_owner", deadline: owner.leaseExpiresAt };
|
||||
}
|
||||
if (source.runtimeMode !== "native" || !["succeeded", "failed", "cancelled", "interrupted", "timed_out"].includes(source.status)
|
||||
|| source.createdAt > current.createdAt) return { kind: "missing_evidence", reason: "newer_progress_barrier" };
|
||||
const profile = record(source.runnerProfileJson), snapshot = record(profile.sessionCheckpoint);
|
||||
const identity = record(snapshot.identity);
|
||||
if (identity.companyId !== input.companyId || identity.agentId !== input.agentId
|
||||
|| identity.issueId !== input.issueId || identity.runId !== source.id || identity.sessionId !== input.normalizedSessionId) {
|
||||
return { kind: "missing_evidence", reason: "snapshot_binding" };
|
||||
}
|
||||
if (record(profile.nativeControllerRecovery).schema !== "paperclip.native-controller-recovery.v1") {
|
||||
return { kind: "unsupported", reason: "metadata_format" };
|
||||
}
|
||||
const metadata = readNativeControllerRecovery(profile.nativeControllerRecovery, snapshot);
|
||||
if (!metadata?.checkpoint || metadata.checkpoint.runId !== source.id
|
||||
|| metadata.checkpoint.normalizedSessionId !== input.normalizedSessionId
|
||||
|| metadata.checkpoint.runnerInstanceId !== source.runnerInstanceId) {
|
||||
return { kind: "missing_evidence", reason: "snapshot_digest_or_checkpoint_binding" };
|
||||
}
|
||||
let checkpoint: DurablePrpCheckpoint | null;
|
||||
try {
|
||||
checkpoint = await loadNativeRunnerCheckpoint(db, { companyId: input.companyId, issueId: input.issueId, runId: source.id });
|
||||
} catch (cause) {
|
||||
return { kind: "dependency_unavailable", reason: "checkpoint_authority_unreadable", cause };
|
||||
}
|
||||
if (!checkpoint || checkpoint.recoveryEvidence.runnerInstanceId !== metadata.checkpoint.runnerInstanceId
|
||||
|| checkpoint.recoveryEvidence.normalizedSessionId !== input.normalizedSessionId) {
|
||||
return { kind: "missing_evidence", reason: "checkpoint_binding" };
|
||||
}
|
||||
return { kind: "recoverable", snapshot: snapshot as unknown as PersistedNativeSession, metadata, checkpoint };
|
||||
}
|
||||
|
||||
/** Bounded internal migration of ONE retained source while its disk is available.
|
||||
* The migration owner supplies compatibility computed from the admitted input.
|
||||
* This never repairs/replaces the saved snapshot or claims a live execution. */
|
||||
export async function importNativeControllerRecovery(db: Db, input: {
|
||||
companyId: string; agentId: string; issueId: string; runId: string; normalizedSessionId: string;
|
||||
expectedConfigDigest: string;
|
||||
supervisor: unknown;
|
||||
}): Promise<{ kind: "imported" | "unchanged" } | Exclude<NativeControllerRecoveryOutcome, { kind: "recoverable" } | { kind: "no_prior_execution" }>> {
|
||||
const supervisor = record(input.supervisor);
|
||||
if (supervisor.schema !== "paperclip.native-session-supervisor.v1" || supervisor.configDigest !== input.expectedConfigDigest) {
|
||||
return { kind: "unsupported", reason: "supervisor_compatibility" };
|
||||
}
|
||||
return db.transaction(async tx => {
|
||||
const [task] = await tx.select({ executionRunId: issues.executionRunId }).from(issues)
|
||||
.where(and(eq(issues.companyId, input.companyId), eq(issues.id, input.issueId))).for("share");
|
||||
const [owner] = await tx.select().from(nativeRunFinalizations).where(and(
|
||||
eq(nativeRunFinalizations.companyId, input.companyId), eq(nativeRunFinalizations.issueId, input.issueId),
|
||||
eq(nativeRunFinalizations.runId, input.runId))).for("update");
|
||||
if (!task || !owner) return { kind: "missing_evidence" as const, reason: "owner_binding" };
|
||||
if (owner.leaseOwner || task.executionRunId) {
|
||||
return owner.leaseExpiresAt && owner.leaseExpiresAt > new Date()
|
||||
? { kind: "waiting_for_owner" as const, deadline: owner.leaseExpiresAt }
|
||||
: { kind: "missing_evidence" as const, reason: "normal_ownership_recovery_required" };
|
||||
}
|
||||
if (owner.phase !== "committed" || !owner.resultId || owner.recoveryState) {
|
||||
return { kind: "missing_evidence" as const, reason: "pending_finalization" };
|
||||
}
|
||||
const [run] = await tx.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, input.runId),
|
||||
eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.agentId, input.agentId),
|
||||
eq(heartbeatRuns.nativeIssueId, input.issueId), eq(heartbeatRuns.nativeSessionId, input.normalizedSessionId))).for("update");
|
||||
const profile = record(run?.runnerProfileJson), snapshot = record(profile.sessionCheckpoint), identity = record(snapshot.identity);
|
||||
if (run?.runtimeMode !== "native" || run.status !== "succeeded" || !run.finishedAt
|
||||
|| identity.runId !== input.runId || identity.companyId !== input.companyId || identity.issueId !== input.issueId
|
||||
|| identity.agentId !== input.agentId || identity.sessionId !== input.normalizedSessionId
|
||||
|| nativeSha256(snapshot) !== nativeSha256(redactRunnerApiPayloads(supervisor.snapshot))) {
|
||||
return { kind: "missing_evidence" as const, reason: "stale_supervisor_snapshot" };
|
||||
}
|
||||
const checkpoint = await loadNativeRunnerCheckpoint(tx, input);
|
||||
if (!checkpoint || checkpoint.recoveryEvidence.normalizedSessionId !== input.normalizedSessionId
|
||||
|| checkpoint.recoveryEvidence.runnerInstanceId !== run.runnerInstanceId) {
|
||||
return { kind: "missing_evidence" as const, reason: "checkpoint_binding" };
|
||||
}
|
||||
const metadata = createNativeControllerRecovery(snapshot as unknown as PersistedNativeSession, {
|
||||
configDigest: input.expectedConfigDigest,
|
||||
processConfigDigest: typeof supervisor.processConfigDigest === "string" ? supervisor.processConfigDigest : null,
|
||||
}, { runId: input.runId, runnerInstanceId: checkpoint.recoveryEvidence.runnerInstanceId, normalizedSessionId: input.normalizedSessionId });
|
||||
if (!readNativeControllerRecovery(metadata, snapshot)) return { kind: "unsupported" as const, reason: "metadata_format" };
|
||||
if (profile.nativeControllerRecovery) {
|
||||
return nativeSha256(profile.nativeControllerRecovery) === nativeSha256(metadata)
|
||||
? { kind: "unchanged" as const }
|
||||
: { kind: "missing_evidence" as const, reason: "existing_metadata_conflict" };
|
||||
}
|
||||
await tx.update(heartbeatRuns).set({ runnerProfileJson: { ...profile, nativeControllerRecovery: metadata }, updatedAt: new Date() })
|
||||
.where(eq(heartbeatRuns.id, input.runId));
|
||||
return { kind: "imported" as const };
|
||||
});
|
||||
}
|
||||
@@ -189,6 +189,8 @@ async function sessionRetentionBlocker(
|
||||
id: heartbeatRuns.id,
|
||||
status: heartbeatRuns.status,
|
||||
finishedAt: heartbeatRuns.finishedAt,
|
||||
incompleteRecovery: sql<boolean>`(${heartbeatRuns.runnerProfileJson}->'nativeControllerRecovery'->>'schema' = 'paperclip.native-controller-recovery.v1'
|
||||
and ${heartbeatRuns.runnerProfileJson}->'nativeControllerRecovery'->'checkpoint' = 'null'::jsonb) is true`,
|
||||
issueId: sql<string | null>`coalesce(${heartbeatRuns.nativeIssueId}, ${heartbeatRuns.issueId})`,
|
||||
})
|
||||
.from(heartbeatRuns)
|
||||
@@ -202,6 +204,7 @@ async function sessionRetentionBlocker(
|
||||
if (runs.some((run) => LIVE_RUN_STATUSES.includes(run.status) || !run.finishedAt)) {
|
||||
return { reason: "live_run" };
|
||||
}
|
||||
if (runs.some(run => run.incompleteRecovery)) return { reason: "native_recovery_incomplete" };
|
||||
if (input.checkpoint && runs.some(run => run.finishedAt! >= input.closedBefore)) return { reason: "recent_run" };
|
||||
const issueIds = [...new Set(runs.map((run) => run.issueId).filter((id): id is string => Boolean(id)))];
|
||||
if (issueIds.length === 0) return { reason: "issue_unknown" };
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { PgDialect } from "drizzle-orm/pg-core";
|
||||
import type { SQL } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { DurableRecoveryIdentity, PrpEvent } from "../../vendor/paperclip-runner/index.js";
|
||||
import { buildNativeExecutionInput } from "./native-execution-input.js";
|
||||
import { nativeRuntimeContextFixture } from "./runtime-context.test-fixture.js";
|
||||
import { createNativeRecoveryUsageDelivery } from "./native-recovery-usage-delivery.js";
|
||||
|
||||
const accounting = vi.hoisted(() => ({ open: vi.fn(), capture: vi.fn() }));
|
||||
vi.mock("./native-usage-delivery.js", () => ({ createNativeUsageContinuationDelivery: accounting.open }));
|
||||
beforeEach(() => { vi.clearAllMocks(); accounting.capture.mockResolvedValue(undefined); accounting.open.mockResolvedValue({ capture: accounting.capture }); });
|
||||
|
||||
function fixture() {
|
||||
const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), sessionId = randomUUID();
|
||||
const source: DurableRecoveryIdentity = { runId: randomUUID(), runnerInstanceId: randomUUID(), environmentLeaseId: randomUUID(),
|
||||
normalizedSessionId: sessionId, turnId: randomUUID(), itemId: randomUUID() };
|
||||
const build = (runId: string) => buildNativeExecutionInput({ companyId, agentId, runId, normalizedSessionId: sessionId,
|
||||
provider: "codex", issue: { id: issueId, identifier: "REC-1", title: "Continue", description: null, workMode: "standard" },
|
||||
taskPrompt: "Continue once", workspace: { id: randomUUID(), cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null },
|
||||
completionContract: { id: randomUUID(), sha256: `sha256:${"a".repeat(64)}`, schemaVersion: "paperclip.run-result.v1",
|
||||
contract: { revision: "1", objective: "Continue", criteria: [{ id: "objective", requirement: "Preserve the result" }] } },
|
||||
runtimeContext: nativeRuntimeContextFixture() });
|
||||
const execution = build(randomUUID()), previous = build(source.runId);
|
||||
const row = { run: { status: "succeeded", finishedAt: new Date() as Date | null, runtimeMode: "native", runnerInstanceId: source.runnerInstanceId,
|
||||
nativeSessionId: sessionId, runnerProfileJson: { nativeExecutionInput: previous } },
|
||||
owner: { phase: "committed", resultId: randomUUID() as string | null, leaseOwner: null as string | null, companyId, issueId } };
|
||||
const where = vi.fn(async (_predicate: SQL) => [row]);
|
||||
const select = vi.fn(() => ({ from: () => ({ innerJoin: () => ({ where }) }) }));
|
||||
const db = { select } as unknown as Db;
|
||||
const matchesExecution = vi.fn(() => true);
|
||||
const deliver = () => createNativeRecoveryUsageDelivery({ db, execution, sourceIdentity: { ...source }, matchesExecution });
|
||||
const event: PrpEvent = { schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", sourceInstanceId: source.runnerInstanceId,
|
||||
sourceEventId: "source-usage", sourceSeq: 1, runId: source.runId, normalizedSessionId: sessionId, turnId: source.turnId,
|
||||
eventType: "usage.reported", priority: 0, emittedAt: new Date().toISOString(), payload: {} };
|
||||
return { source, execution, previous, row, select, where, matchesExecution, deliver, event };
|
||||
}
|
||||
|
||||
describe("source accounting recovery authority", () => {
|
||||
it("pins the relational source tuple and hydrates once without binding target accounting", async () => {
|
||||
const f = fixture(), deliver = f.deliver();
|
||||
await deliver(f.event, f.source);
|
||||
await deliver({ ...f.event, sourceEventId: "source-terminal", sourceSeq: 2, eventType: "turn.completed" }, f.source);
|
||||
expect(f.select).toHaveBeenCalledOnce();
|
||||
expect(new PgDialect().sqlToQuery(f.where.mock.calls[0][0]).params).toEqual([
|
||||
f.source.runId, f.execution.binding.companyId, f.execution.binding.agentId, f.execution.binding.issueId,
|
||||
]);
|
||||
expect(f.matchesExecution).toHaveBeenCalledWith(f.previous);
|
||||
expect(accounting.open).toHaveBeenCalledExactlyOnceWith(expect.anything(), {
|
||||
companyId: f.execution.binding.companyId, runId: f.source.runId, runnerInstanceId: f.source.runnerInstanceId, provider: f.previous.provider,
|
||||
});
|
||||
expect(accounting.capture).toHaveBeenCalledTimes(2);
|
||||
expect(accounting.capture).toHaveBeenCalledWith(f.event);
|
||||
});
|
||||
|
||||
it.each(["runId", "runnerInstanceId", "environmentLeaseId", "normalizedSessionId", "turnId", "itemId"] as const)(
|
||||
"rejects a changed pinned %s before source hydration", async key => {
|
||||
const f = fixture();
|
||||
await expect(f.deliver()(f.event, { ...f.source, [key]: randomUUID() })).rejects.toThrow("native_recovery_usage_source_unproven");
|
||||
expect(f.select).not.toHaveBeenCalled();
|
||||
expect(accounting.open).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(["runId", "sourceInstanceId", "normalizedSessionId"] as const)("rejects mismatched event %s before source hydration", async key => {
|
||||
const f = fixture();
|
||||
await expect(f.deliver()({ ...f.event, [key]: randomUUID() }, f.source)).rejects.toThrow("native_recovery_usage_source_unproven");
|
||||
expect(f.select).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(["nonterminal", "unfinished", "uncommitted", "no-result", "held", "company", "issue", "scope"])(
|
||||
"keeps an unproven source held: %s", async change => {
|
||||
const f = fixture();
|
||||
if (change === "nonterminal") f.row.run.status = "running";
|
||||
if (change === "unfinished") f.row.run.finishedAt = null;
|
||||
if (change === "uncommitted") f.row.owner.phase = "observed";
|
||||
if (change === "no-result") f.row.owner.resultId = null;
|
||||
if (change === "held") f.row.owner.leaseOwner = "live-owner";
|
||||
if (change === "company") f.row.owner.companyId = randomUUID();
|
||||
if (change === "issue") f.row.owner.issueId = randomUUID();
|
||||
if (change === "scope") f.matchesExecution.mockReturnValue(false);
|
||||
await expect(f.deliver()(f.event, f.source)).rejects.toThrow("native_recovery_usage_source_unproven");
|
||||
expect(accounting.open).not.toHaveBeenCalled();
|
||||
expect(accounting.capture).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps a durable source receipt failure unacknowledged", async () => {
|
||||
const f = fixture();
|
||||
accounting.capture.mockRejectedValue(new Error("native_usage_after_finalization_requires_reconciliation"));
|
||||
await expect(f.deliver()(f.event, f.source)).rejects.toThrow("native_usage_after_finalization_requires_reconciliation");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,47 @@
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { heartbeatRuns, nativeRunFinalizations, type Db } from "@paperclipai/db";
|
||||
import { parseNativeExecutionInput, type DurableRecoveryIdentity, type NativeExecutionInput, type PrpEvent } from "../../vendor/paperclip-runner/index.js";
|
||||
import { createNativeUsageContinuationDelivery } from "./native-usage-delivery.js";
|
||||
|
||||
const identityKeys = ["runId", "runnerInstanceId", "environmentLeaseId", "normalizedSessionId", "turnId", "itemId"] as const;
|
||||
|
||||
/** A cold handoff may discover accounting that belongs to its completed source.
|
||||
* Hydrate that producer once; never route its receipts through the new run. */
|
||||
export function createNativeRecoveryUsageDelivery(input: {
|
||||
db: Db;
|
||||
execution: NativeExecutionInput;
|
||||
sourceIdentity: Record<string, unknown>;
|
||||
matchesExecution(previous: NativeExecutionInput): boolean;
|
||||
}) {
|
||||
let delivery: ReturnType<typeof createNativeUsageContinuationDelivery> | undefined;
|
||||
const deny = (): never => { throw new Error("native_recovery_usage_source_unproven"); };
|
||||
const open = async (source: DurableRecoveryIdentity) => {
|
||||
const { binding } = input.execution;
|
||||
const [prior] = await input.db.select({ run: heartbeatRuns, owner: nativeRunFinalizations })
|
||||
.from(heartbeatRuns).innerJoin(nativeRunFinalizations, eq(nativeRunFinalizations.runId, heartbeatRuns.id))
|
||||
.where(and(eq(heartbeatRuns.id, source.runId), eq(heartbeatRuns.companyId, binding.companyId),
|
||||
eq(heartbeatRuns.agentId, binding.agentId), eq(heartbeatRuns.nativeIssueId, binding.issueId)));
|
||||
if (!prior || prior.run.status !== "succeeded" || prior.run.finishedAt === null ||
|
||||
prior.run.runtimeMode !== "native" || prior.run.runnerInstanceId !== source.runnerInstanceId ||
|
||||
prior.run.nativeSessionId !== source.normalizedSessionId || prior.owner.phase !== "committed" ||
|
||||
!prior.owner.resultId || prior.owner.leaseOwner || prior.owner.companyId !== binding.companyId ||
|
||||
prior.owner.issueId !== binding.issueId) return deny();
|
||||
const previous = parseNativeExecutionInput(prior.run.runnerProfileJson?.nativeExecutionInput);
|
||||
if (previous.binding.companyId !== binding.companyId || previous.binding.agentId !== binding.agentId ||
|
||||
previous.binding.issueId !== binding.issueId || previous.binding.runId !== source.runId ||
|
||||
!input.matchesExecution(previous)) return deny();
|
||||
return createNativeUsageContinuationDelivery(input.db, {
|
||||
companyId: binding.companyId, runId: source.runId,
|
||||
runnerInstanceId: source.runnerInstanceId, provider: previous.provider,
|
||||
});
|
||||
};
|
||||
return async (event: PrpEvent, source: DurableRecoveryIdentity) => {
|
||||
if (source.runId === input.execution.binding.runId ||
|
||||
identityKeys.some(key => typeof input.sourceIdentity[key] !== "string" ||
|
||||
!input.sourceIdentity[key] || source[key] !== input.sourceIdentity[key]) ||
|
||||
event.runId !== source.runId || event.sourceInstanceId !== source.runnerInstanceId ||
|
||||
event.normalizedSessionId !== source.normalizedSessionId) return deny();
|
||||
delivery ??= open(source);
|
||||
await (await delivery).capture(event);
|
||||
};
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
import { QueryBuilder } from "drizzle-orm/pg-core";
|
||||
import { and, asc, eq, gt, inArray, isNotNull, isNull, notExists, or, sql } from "drizzle-orm";
|
||||
import { heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults, statusDecisions, statusDecisionEffects } from "@paperclipai/db";
|
||||
import { and, asc, eq, exists, gt, inArray, isNotNull, isNull, notExists, or, sql } from "drizzle-orm";
|
||||
import { heartbeatRunEvents, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults, statusDecisions, statusDecisionEffects } from "@paperclipai/db";
|
||||
import type { NativeResponsePresentationScope } from "../native-response-presentation-notifications.js";
|
||||
|
||||
export function nativeResponsePresentationCandidates(scope?: NativeResponsePresentationScope, after?: string, runId?: string) {
|
||||
@@ -30,7 +30,24 @@ export function nativeResponsePresentationCandidates(scope?: NativeResponsePrese
|
||||
sql`${nativeRunResults.resultJson} #>> '{terminal,turnTerminalState}' = 'completed'`,
|
||||
or(
|
||||
and(eq(heartbeatRuns.status, "succeeded"),
|
||||
sql`(coalesce(${issues.conversationAgentId}::text, '') = '' or coalesce(${issues.conversationUserId}, '') = '')`,
|
||||
or(sql`(coalesce(${issues.conversationAgentId}::text, '') = '' or coalesce(${issues.conversationUserId}, '') = '')`,
|
||||
and(eq(statusDecisions.reasonCode, "conversation_turn_finished"),
|
||||
eq(issues.conversationAgentId, heartbeatRuns.agentId), eq(issues.conversationUserId, heartbeatRuns.responsibleUserId),
|
||||
eq(issues.status, "in_progress"), eq(issues.conversationState, "active"),
|
||||
or(isNull(issues.executionRunId), eq(issues.executionRunId, heartbeatRuns.id)),
|
||||
sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issues.id}::text`,
|
||||
sql`jsonb_typeof(${heartbeatRuns.contextSnapshot}->'conversationSessionGeneration') = 'number'`,
|
||||
sql`${heartbeatRuns.contextSnapshot}->>'conversationSessionGeneration' = ${issues.conversationSessionGeneration}::text`,
|
||||
exists(db.select({ id: heartbeatRunEvents.id }).from(heartbeatRunEvents).where(and(
|
||||
eq(heartbeatRunEvents.companyId, heartbeatRuns.companyId), eq(heartbeatRunEvents.runId, heartbeatRuns.id),
|
||||
eq(heartbeatRunEvents.agentId, heartbeatRuns.agentId), eq(heartbeatRunEvents.eventType, "item.completed"),
|
||||
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,runId}' = ${heartbeatRuns.id}::text`,
|
||||
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,normalizedSessionId}' = ${heartbeatRuns.nativeSessionId}::text`,
|
||||
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,turnId}' = ${nativeRunResults.turnId}`,
|
||||
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,payload,kind}' = 'agentMessage'`,
|
||||
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,payload,channel}' = 'final'`,
|
||||
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,payload,text}' ~ '[^[:space:]]'`,
|
||||
))))),
|
||||
sql`${nativeRunResults.resultJson} #>> '{result,reportedWorkDisposition}' = 'done'`,
|
||||
sql`${nativeRunResults.resultJson} #>> '{terminal,reportedWorkDisposition}' = 'done'`),
|
||||
and(sql`${nativeRunResults.resultJson} #>> '{result,reportedWorkDisposition}' = 'yielded'`,
|
||||
|
||||
@@ -56,6 +56,7 @@ export function nativeResponsePresentationWork(db: Db) {
|
||||
function register(coordinator: ReturnType<typeof createDeliveryWorkCoordinator>, canRun: () => boolean) {
|
||||
const worker = registerScopedDeadlineWork(coordinator, {
|
||||
queue: DELIVERY_QUEUES.nativeResponsePresentation,
|
||||
concurrentChecks: { limit: 4, key: (entry: Entry) => `${entry.companyId}:${entry.issueId}` },
|
||||
decodeScope: (hint: string): NativeResponsePresentationScope | undefined => hint === "*" ? undefined : JSON.parse(hint),
|
||||
matches: (entry: Entry, scope?: NativeResponsePresentationScope) => !scope || entry.companyId === scope.companyId && (!scope.issueId || entry.issueId === scope.issueId) && (!scope.runId || entry.id === scope.runId) && (!scope.agentId || entry.agentId === scope.agentId),
|
||||
load: (scope, signal) => load(scope, signal, canRun),
|
||||
|
||||
@@ -8,6 +8,18 @@ import {
|
||||
} from "./native-restart-recovery.js";
|
||||
|
||||
describe("native restart recovery classification", () => {
|
||||
it.each([false, true])("recognizes verified staged remote evidence with target snapshot=%s", hasCheckpoint => {
|
||||
const evidence = { remoteSandbox: true, runnerPidAlive: false, runnerGroupAlive: false,
|
||||
processStartMatches: false, hasCheckpoint, hasProviderEvidence: false };
|
||||
expect(classifyNativeRunnerRecoveryEvidence(evidence).claimKind).toBeNull();
|
||||
expect(classifyNativeRunnerRecoveryEvidence({ ...evidence, stagedAttachmentVerified: true })).toEqual({
|
||||
claimKind: "reattach_remote_runner", reason: "staged_remote_attachment_requires_sandbox_verification",
|
||||
});
|
||||
expect(classifyNativeRunnerRecoveryEvidence({ ...evidence, stagedAttachmentVerified: true,
|
||||
checkpointFailed: true }).claimKind).toBeNull();
|
||||
expect(classifyNativeRunnerRecoveryEvidence({ ...evidence, remoteSandbox: false,
|
||||
stagedAttachmentVerified: true }).claimKind).toBe(hasCheckpoint ? null : "bootstrap_incomplete");
|
||||
});
|
||||
it.each([true, false])("uses the remote fenced lease instead of the old container PID (live=%s)", async live => {
|
||||
const now = new Date();
|
||||
const probe = vi.fn(() => { throw new Error("must not probe another container's PID"); });
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { NATIVE_RETAINED_OWNER_WAIT } from "./native-controller-recovery.js";
|
||||
import { notifyNativeEnvironmentSettlement } from "../native-environment-settlement-notifications.js";
|
||||
import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
|
||||
import { notifyNativeStatusReassessment } from "../native-status-reassessment-notifications.js";
|
||||
@@ -6,7 +7,8 @@ import { notifyCancelledWorkspacePreservation } from "../cancelled-workspace-pre
|
||||
import { notifyInstructionCollection } from "../instruction-collection-notifications.js";
|
||||
import { notifyExecutionRunWork } from "../execution-control-notifications.js";
|
||||
import { preserveNativeFailedWorkspace } from "./native-failed-workspace-recovery.js";
|
||||
import { hasNativeRunnerCheckpoint } from "./native-runner-checkpoint.js";
|
||||
import { hasNativeRunnerCheckpoint, loadNativeRunnerCheckpointHeader } from "./native-runner-checkpoint.js";
|
||||
import { resolveNativeStagedAttachmentRecovery, type NativeStagedAttachmentRecovery } from "./native-staged-attachment-recovery.js";
|
||||
import { readNativeWorkspaceSyncReference } from "./native-workspace-sync.js";
|
||||
import { activeNativeExecutionRunIds, NATIVE_PREPARATION_FENCING_VERSION } from "./native-active-runs.js";
|
||||
import {
|
||||
@@ -15,7 +17,7 @@ import {
|
||||
recordNativeLocalProcessStop,
|
||||
} from "../native-local-process-stop.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { and, desc, eq, inArray, isNotNull, isNull, lte, or, sql } from "drizzle-orm";
|
||||
import { and, desc, eq, inArray, isNotNull, isNull, lte, ne, or, sql } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import {
|
||||
agents,
|
||||
@@ -68,6 +70,8 @@ export type NativeRestartRecoveryClaim =
|
||||
recoveryRequestId: string | null;
|
||||
remote: { kind?: "sandbox"; providerLeaseId: string; remoteCwd: string }
|
||||
| { kind: "plugin_task"; leaseId: string };
|
||||
/** Discovery evidence only; executor revalidates the encrypted header. */
|
||||
stagedAttachment?: Pick<NativeStagedAttachmentRecovery, "sourceIdentity" | "targetIdentity" | "templateDigest">;
|
||||
}
|
||||
| {
|
||||
kind: "resume_dead_runner";
|
||||
@@ -94,6 +98,7 @@ export type NativeRestartRecoveryDisposition =
|
||||
kind: "awaiting_evidence" | "blocked" | "already_finalized";
|
||||
runId: string;
|
||||
reason: string;
|
||||
retryAt?: Date;
|
||||
};
|
||||
|
||||
export function nextNativeProviderAttempt(
|
||||
@@ -320,6 +325,7 @@ export function classifyNativeRunnerRecoveryEvidence(input: {
|
||||
checkpointIdentityMatches?: boolean;
|
||||
hasProviderEvidence: boolean;
|
||||
checkpointFailed?: boolean;
|
||||
stagedAttachmentVerified?: boolean;
|
||||
providerAttempt?: number;
|
||||
}): {
|
||||
claimKind: NativeRestartRecoveryClaim["kind"] | null;
|
||||
@@ -334,6 +340,9 @@ export function classifyNativeRunnerRecoveryEvidence(input: {
|
||||
// inspection; the executor must prove the original lease and authenticate
|
||||
// the surviving runner before it can resume delivery.
|
||||
if (input.remoteSandbox) {
|
||||
if (input.stagedAttachmentVerified === true) {
|
||||
return { claimKind: "reattach_remote_runner", reason: "staged_remote_attachment_requires_sandbox_verification" };
|
||||
}
|
||||
if (input.unclaimedBootstrap === true && !input.hasCheckpoint && !input.hasProviderEvidence) {
|
||||
return { claimKind: "bootstrap_incomplete", reason: "remote_execution_never_claimed" };
|
||||
}
|
||||
@@ -514,6 +523,9 @@ export async function claimNativeRestartRecoveries(input: {
|
||||
eq(agents.adapterType, "paperclip_runner"),
|
||||
inArray(heartbeatRuns.status, ["running", "failed"]),
|
||||
isNull(nativeRunFinalizations.resultId),
|
||||
// A pre-provider retained-owner wait belongs to normal same-run
|
||||
// resumption, including when its deadline elapsed while we were down.
|
||||
or(isNull(nativeRunFinalizations.failureCode), ne(nativeRunFinalizations.failureCode, NATIVE_RETAINED_OWNER_WAIT)),
|
||||
or(
|
||||
eq(nativeRunFinalizations.phase, "observed"),
|
||||
and(
|
||||
@@ -668,9 +680,21 @@ export async function claimNativeRestartRecoveries(input: {
|
||||
kind: "awaiting_evidence",
|
||||
runId: row.run.id,
|
||||
reason: takeover.reason,
|
||||
...(durableAuthority && row.coordinator.leaseExpiresAt ? { retryAt: row.coordinator.leaseExpiresAt } : {}),
|
||||
} as const;
|
||||
}
|
||||
|
||||
// Only the classic remote path stages attachments. After the owner is
|
||||
// eligible for takeover, inspect its bounded header even if activation
|
||||
// already published a runtime snapshot. Never scan command/event payloads.
|
||||
const stagedHeader = remoteWorkspace && !pluginTask && durableAuthority
|
||||
? await loadNativeRunnerCheckpointHeader(tx, {
|
||||
companyId: row.run.companyId, issueId: row.coordinator.issueId, runId: row.run.id,
|
||||
}) : null;
|
||||
const stagedAttachment = stagedHeader ? await resolveNativeStagedAttachmentRecovery({
|
||||
db: tx, currentRun: row.run, header: stagedHeader,
|
||||
}) : null;
|
||||
|
||||
const runnerPidAlive =
|
||||
!remoteProcess && processIsAlive(row.run.processPid);
|
||||
const runnerGroupAlive =
|
||||
@@ -859,6 +883,7 @@ export async function claimNativeRestartRecoveries(input: {
|
||||
knownProviderProcessIdentityAmbiguous:
|
||||
providerProcesses.ambiguousLivePids.length > 0,
|
||||
hasCheckpoint,
|
||||
stagedAttachmentVerified: stagedAttachment !== null,
|
||||
checkpointIdentityMatches:
|
||||
checkpointIdentityMatches &&
|
||||
(!remoteSandbox || remoteWorkspace !== null),
|
||||
@@ -1109,6 +1134,10 @@ export async function claimNativeRestartRecoveries(input: {
|
||||
return {
|
||||
kind: claimKind,
|
||||
...common,
|
||||
...(stagedAttachment ? { stagedAttachment: {
|
||||
sourceIdentity: stagedAttachment.sourceIdentity, targetIdentity: stagedAttachment.targetIdentity,
|
||||
templateDigest: stagedAttachment.templateDigest,
|
||||
} } : {}),
|
||||
remote: pluginTask ? { kind: "plugin_task", leaseId: pluginTask.id } : {
|
||||
providerLeaseId: remoteWorkspace!.providerLeaseId,
|
||||
remoteCwd: remoteWorkspace!.remoteCwd,
|
||||
|
||||
@@ -29,7 +29,7 @@ import { readPersistedNativeFailureEvidence } from "./native-provider-failure-ev
|
||||
import { dismissAutomaticCompletionReviews } from "./automatic-completion-reviews.js";
|
||||
import { findSupersededExternalReviewRequests } from "./superseded-external-review-requests.js";
|
||||
import { getNativeReviewAssignment, readNativeReviewAssignmentContext } from "./native-review-participant.js";
|
||||
import { conversationNativeDecision, isConversation } from "../agent-conversations.js";
|
||||
import { conversationNativeDecision, isConversation, settleConversationTurnForRun } from "../agent-conversations.js";
|
||||
import { issueTreeControlService } from "../issue-tree-control.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { preserveNativeWorkspaceExportLease } from "./native-workspace-export-resume.js";
|
||||
@@ -747,6 +747,7 @@ async function recoverCommittedNativeChatResponseInScope(
|
||||
): Promise<boolean> {
|
||||
let agentId: string | null = null;
|
||||
let materialized = false;
|
||||
let conversationReply = false;
|
||||
const entered = beginRunPhase("response.transaction.entry");
|
||||
try {
|
||||
materialized = await measureRunPhase("response.transaction", () => db.transaction(async (transaction) => {
|
||||
@@ -789,6 +790,7 @@ async function recoverCommittedNativeChatResponseInScope(
|
||||
id: heartbeatRuns.id,
|
||||
companyId: heartbeatRuns.companyId,
|
||||
agentId: heartbeatRuns.agentId,
|
||||
responsibleUserId: heartbeatRuns.responsibleUserId,
|
||||
nativeIssueId: heartbeatRuns.nativeIssueId,
|
||||
nativeSessionId: heartbeatRuns.nativeSessionId,
|
||||
completionContractId: heartbeatRuns.completionContractId,
|
||||
@@ -803,6 +805,8 @@ async function recoverCommittedNativeChatResponseInScope(
|
||||
contextSnapshot: sql<Record<string, unknown> | null>`case when ${heartbeatRuns.contextSnapshot} is null then null
|
||||
else jsonb_build_object(
|
||||
'skipIssueComment', ${heartbeatRuns.contextSnapshot}->'skipIssueComment',
|
||||
'issueId', ${heartbeatRuns.contextSnapshot}->'issueId',
|
||||
'conversationSessionGeneration', ${heartbeatRuns.contextSnapshot}->'conversationSessionGeneration',
|
||||
'source', ${heartbeatRuns.contextSnapshot}->'source',
|
||||
'externalChatContinuation', ${heartbeatRuns.contextSnapshot}->'externalChatContinuation',
|
||||
'paperclipWake', jsonb_build_object('externalInteractionContinuation',
|
||||
@@ -909,12 +913,16 @@ async function recoverCommittedNativeChatResponseInScope(
|
||||
agentId = run.agentId;
|
||||
return true;
|
||||
}
|
||||
// Copyback can be owned by reconciliation before the live heartbeat
|
||||
// reaches its presentation step. Recover a completed internal task's
|
||||
// exact final reply from the same accepted turn, without rerunning work
|
||||
// or granting external-chat publication authority.
|
||||
const conversation = isConversation(issue);
|
||||
const currentConversation = conversation && decision.reasonCode === "conversation_turn_finished"
|
||||
&& issue.conversationAgentId === run.agentId && issue.conversationUserId === run.responsibleUserId
|
||||
&& issue.status === "in_progress" && issue.conversationState === "active"
|
||||
&& (!issue.executionRunId || issue.executionRunId === run.id)
|
||||
&& run.contextSnapshot?.issueId === issue.id
|
||||
&& typeof run.contextSnapshot.conversationSessionGeneration === "number"
|
||||
&& run.contextSnapshot.conversationSessionGeneration === issue.conversationSessionGeneration;
|
||||
if (
|
||||
!isConversation(issue) &&
|
||||
(!conversation || currentConversation) &&
|
||||
run.status === "succeeded" &&
|
||||
record(run.contextSnapshot).skipIssueComment !== true &&
|
||||
!isExternalChatPresentationContext(run.contextSnapshot) &&
|
||||
@@ -929,6 +937,9 @@ async function recoverCommittedNativeChatResponseInScope(
|
||||
eq(heartbeatRunEvents.companyId, input.companyId),
|
||||
eq(heartbeatRunEvents.runId, run.id),
|
||||
eq(heartbeatRunEvents.eventType, "item.completed"),
|
||||
conversation ? and(eq(heartbeatRunEvents.agentId, run.agentId),
|
||||
sql`${heartbeatRunEvents.payload}->'prpEvent'->>'runId' = ${run.id}`,
|
||||
sql`${heartbeatRunEvents.payload}->'prpEvent'->>'normalizedSessionId' = ${run.nativeSessionId}`) : undefined,
|
||||
sql`${heartbeatRunEvents.payload}->'prpEvent'->>'turnId' = ${accepted.turnId}`,
|
||||
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,payload,kind}' = 'agentMessage'`,
|
||||
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,payload,channel}' = 'final'`,
|
||||
@@ -963,6 +974,7 @@ async function recoverCommittedNativeChatResponseInScope(
|
||||
})}::jsonb`, updatedAt: new Date(),
|
||||
}).where(eq(heartbeatRuns.id, run.id));
|
||||
agentId = run.agentId;
|
||||
conversationReply = conversation;
|
||||
return true;
|
||||
}
|
||||
if (
|
||||
@@ -1039,12 +1051,14 @@ async function recoverCommittedNativeChatResponseInScope(
|
||||
} finally {
|
||||
entered();
|
||||
}
|
||||
if (materialized && agentId)
|
||||
if (materialized && agentId) {
|
||||
if (conversationReply) await settleConversationTurnForRun(db, input.companyId, input.runId);
|
||||
publishChatPublicationCommitSignal({
|
||||
...input,
|
||||
agentId,
|
||||
eventType: "run.presentation.resolved",
|
||||
});
|
||||
}
|
||||
return materialized;
|
||||
}
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ import { createLocalEncryptedCipher } from "../../secrets/local-encrypted-provid
|
||||
import { legacyControllerBootId } from "../legacy-controller-lease.js";
|
||||
|
||||
type Binding = { companyId: string; issueId: string; runId: string };
|
||||
type Owner = Binding & { bootId: string; leaseOwner: string; generation: number; authorityRunId: string };
|
||||
export type NativeRunnerCheckpointOwner = Binding & { bootId: string; leaseOwner: string; generation: number; authorityRunId: string };
|
||||
function encryptParts(binding: Binding, records: ReadonlyMap<string, string>) {
|
||||
if (!records.size) return [];
|
||||
const cipher = createLocalEncryptedCipher("write");
|
||||
@@ -25,49 +25,79 @@ function decryptPart(cipher: ReturnType<typeof createLocalEncryptedCipher>, bind
|
||||
return envelope.contents;
|
||||
}
|
||||
|
||||
export function nativeRunnerCheckpointWriter(db: Db, owner: Owner) {
|
||||
type CheckpointTransaction = Parameters<Parameters<Db["transaction"]>[0]>[0];
|
||||
|
||||
/** A dependent run-lock query can consume this CTE to preserve coordinator-first
|
||||
* lock order without adding a second database round trip. */
|
||||
export function nativeRunnerCheckpointOwnerLock(tx: CheckpointTransaction, owner: NativeRunnerCheckpointOwner) {
|
||||
return tx.$with("checkpoint_owner").as(tx.select({
|
||||
leaseOwner: nativeRunFinalizations.leaseOwner,
|
||||
leaseExpiresAt: nativeRunFinalizations.leaseExpiresAt,
|
||||
resultId: nativeRunFinalizations.resultId,
|
||||
}).from(nativeRunFinalizations).where(and(
|
||||
eq(nativeRunFinalizations.companyId, owner.companyId), eq(nativeRunFinalizations.issueId, owner.issueId),
|
||||
eq(nativeRunFinalizations.runId, owner.authorityRunId), eq(nativeRunFinalizations.controllerBootId, owner.bootId),
|
||||
eq(nativeRunFinalizations.controllerGeneration, owner.generation))).for("update"));
|
||||
}
|
||||
|
||||
/** Locks the coordinator before callers lock a run. A caller that subsequently
|
||||
* waits for another lock must recheck the fence before writing. */
|
||||
export async function lockNativeRunnerCheckpointOwner(
|
||||
tx: CheckpointTransaction, owner: NativeRunnerCheckpointOwner,
|
||||
) {
|
||||
// A locking CTE acquires ownership before the outer projection samples
|
||||
// wall time. Sampling in the locking SELECT can precede its lock wait.
|
||||
// Keep both steps in one statement without another database round trip.
|
||||
const locked = nativeRunnerCheckpointOwnerLock(tx, owner);
|
||||
const [owned] = await tx.with(locked).select({ leaseOwner: locked.leaseOwner,
|
||||
leaseExpiresAt: locked.leaseExpiresAt, resultId: locked.resultId, now: sql<Date>`clock_timestamp()` }).from(locked);
|
||||
if (!owned) throw new Error("native_runner_checkpoint_owner_lost");
|
||||
return owned;
|
||||
}
|
||||
|
||||
export async function validateNativeRunnerCheckpointOwner(
|
||||
tx: CheckpointTransaction, owner: NativeRunnerCheckpointOwner,
|
||||
owned: Awaited<ReturnType<typeof lockNativeRunnerCheckpointOwner>>, newWork: boolean,
|
||||
): Promise<void> {
|
||||
const current = owned;
|
||||
const activeOwner = current.leaseOwner === owner.leaseOwner
|
||||
&& current.leaseExpiresAt !== null && current.leaseExpiresAt > new Date(owned.now);
|
||||
if (!activeOwner) {
|
||||
const [task] = await tx.select({ executionRunId: issues.executionRunId }).from(issues)
|
||||
.where(and(eq(issues.companyId, owner.companyId), eq(issues.id, owner.issueId)));
|
||||
const [successor] = task?.executionRunId && task.executionRunId !== owner.runId
|
||||
? await tx.select({ runId: nativeRunFinalizations.runId }).from(nativeRunFinalizations).where(and(
|
||||
eq(nativeRunFinalizations.runId, task.executionRunId), eq(nativeRunFinalizations.companyId, owner.companyId),
|
||||
eq(nativeRunFinalizations.issueId, owner.issueId), eq(nativeRunFinalizations.controllerBootId, owner.bootId),
|
||||
gt(nativeRunFinalizations.leaseExpiresAt, sql`clock_timestamp()`))) : [];
|
||||
const [preparingSuccessor] = !successor && task?.executionRunId && task.executionRunId !== owner.runId
|
||||
? await tx.select({ runId: heartbeatRuns.id }).from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.id, task.executionRunId), eq(heartbeatRuns.companyId, owner.companyId),
|
||||
eq(heartbeatRuns.issueId, owner.issueId), eq(heartbeatRuns.runtimeMode, "legacy"),
|
||||
eq(heartbeatRuns.status, "running"), eq(heartbeatRuns.executionStage, "preparing"),
|
||||
eq(heartbeatRuns.controllerBootId, legacyControllerBootId),
|
||||
gt(heartbeatRuns.controllerLeaseExpiresAt, sql`clock_timestamp()`),
|
||||
notExists(tx.select({ runId: nativeRunFinalizations.runId }).from(nativeRunFinalizations)
|
||||
.where(eq(nativeRunFinalizations.runId, heartbeatRuns.id))))) : [];
|
||||
if (!current.resultId || newWork || !task || (task.executionRunId && task.executionRunId !== owner.runId && !successor && !preparingSuccessor)) {
|
||||
throw new Error("native_runner_checkpoint_owner_lost");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function assertNativeRunnerCheckpointOwner(tx: CheckpointTransaction, owner: NativeRunnerCheckpointOwner, newWork: boolean) {
|
||||
const locked = await lockNativeRunnerCheckpointOwner(tx, owner);
|
||||
await validateNativeRunnerCheckpointOwner(tx, owner, locked, newWork);
|
||||
}
|
||||
|
||||
export function nativeRunnerCheckpointWriter(db: Db, owner: NativeRunnerCheckpointOwner) {
|
||||
const persistenceDb = checkpointDb(db);
|
||||
let previousWork = new Set<string>();
|
||||
return async (checkpoint: DurablePrpCheckpoint): Promise<void> => {
|
||||
if (checkpoint.runId !== owner.runId) throw new Error("native_runner_checkpoint_binding_mismatch");
|
||||
const changed = encryptParts(owner, checkpoint.changedRecords);
|
||||
await persistenceDb.transaction(async tx => {
|
||||
// A locking CTE acquires ownership before the outer projection samples
|
||||
// wall time. Sampling in the locking SELECT can precede its lock wait.
|
||||
// Keep both steps in one statement without another database round trip.
|
||||
const locked = tx.$with("checkpoint_owner").as(tx.select().from(nativeRunFinalizations)
|
||||
.where(and(eq(nativeRunFinalizations.companyId, owner.companyId),
|
||||
eq(nativeRunFinalizations.issueId, owner.issueId), eq(nativeRunFinalizations.runId, owner.authorityRunId),
|
||||
eq(nativeRunFinalizations.controllerBootId, owner.bootId),
|
||||
eq(nativeRunFinalizations.controllerGeneration, owner.generation))).for("update"));
|
||||
const [owned] = await tx.with(locked).select({ leaseOwner: locked.leaseOwner,
|
||||
leaseExpiresAt: locked.leaseExpiresAt, resultId: locked.resultId, now: sql<Date>`clock_timestamp()` }).from(locked);
|
||||
if (!owned) throw new Error("native_runner_checkpoint_owner_lost");
|
||||
const current = owned;
|
||||
const activeOwner = current.leaseOwner === owner.leaseOwner
|
||||
&& current.leaseExpiresAt !== null && current.leaseExpiresAt > new Date(owned.now);
|
||||
if (!activeOwner) {
|
||||
const newWork = checkpoint.workCommandIds.some(id => !previousWork.has(id));
|
||||
const [task] = await tx.select({ executionRunId: issues.executionRunId }).from(issues)
|
||||
.where(and(eq(issues.companyId, owner.companyId), eq(issues.id, owner.issueId)));
|
||||
const [successor] = task?.executionRunId && task.executionRunId !== owner.runId
|
||||
? await tx.select({ runId: nativeRunFinalizations.runId }).from(nativeRunFinalizations).where(and(
|
||||
eq(nativeRunFinalizations.runId, task.executionRunId), eq(nativeRunFinalizations.companyId, owner.companyId),
|
||||
eq(nativeRunFinalizations.issueId, owner.issueId), eq(nativeRunFinalizations.controllerBootId, owner.bootId),
|
||||
gt(nativeRunFinalizations.leaseExpiresAt, sql`clock_timestamp()`))) : [];
|
||||
const [preparingSuccessor] = !successor && task?.executionRunId && task.executionRunId !== owner.runId
|
||||
? await tx.select({ runId: heartbeatRuns.id }).from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.id, task.executionRunId), eq(heartbeatRuns.companyId, owner.companyId),
|
||||
eq(heartbeatRuns.issueId, owner.issueId), eq(heartbeatRuns.runtimeMode, "legacy"),
|
||||
eq(heartbeatRuns.status, "running"), eq(heartbeatRuns.executionStage, "preparing"),
|
||||
eq(heartbeatRuns.controllerBootId, legacyControllerBootId),
|
||||
gt(heartbeatRuns.controllerLeaseExpiresAt, sql`clock_timestamp()`),
|
||||
notExists(tx.select({ runId: nativeRunFinalizations.runId }).from(nativeRunFinalizations)
|
||||
.where(eq(nativeRunFinalizations.runId, heartbeatRuns.id))))) : [];
|
||||
if (!current.resultId || newWork || !task || (task.executionRunId && task.executionRunId !== owner.runId && !successor && !preparingSuccessor)) {
|
||||
throw new Error("native_runner_checkpoint_owner_lost");
|
||||
}
|
||||
}
|
||||
await assertNativeRunnerCheckpointOwner(tx, owner, checkpoint.workCommandIds.some(id => !previousWork.has(id)));
|
||||
if (checkpoint.predecessor) {
|
||||
const [prior] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.id, checkpoint.predecessor.runId), eq(heartbeatRuns.companyId, owner.companyId),
|
||||
@@ -150,7 +180,17 @@ export async function registerNativeRunnerCheckpoint(input: {
|
||||
await input.authority.flushCheckpoint();
|
||||
}
|
||||
|
||||
export async function loadNativeRunnerCheckpoint(db: Db, binding: Binding): Promise<DurablePrpCheckpoint | null> {
|
||||
/** Exact-run bounded metadata read; never loads command/event payload records. */
|
||||
export async function loadNativeRunnerCheckpointHeader(db: Pick<Db, "select">, binding: Binding): Promise<string | null> {
|
||||
const [row] = await db.select({ material: nativeRunnerCheckpoints.material }).from(nativeRunnerCheckpoints).where(and(
|
||||
eq(nativeRunnerCheckpoints.companyId, binding.companyId), eq(nativeRunnerCheckpoints.issueId, binding.issueId),
|
||||
eq(nativeRunnerCheckpoints.runId, binding.runId), eq(nativeRunnerCheckpoints.part, "header"))).limit(1);
|
||||
if (!row) return null;
|
||||
try { return decryptPart(createLocalEncryptedCipher("read"), binding, "header", row.material); }
|
||||
catch { return null; }
|
||||
}
|
||||
|
||||
export async function loadNativeRunnerCheckpoint(db: Pick<Db, "select">, binding: Binding): Promise<DurablePrpCheckpoint | null> {
|
||||
const rows = await db.select().from(nativeRunnerCheckpoints).where(and(
|
||||
eq(nativeRunnerCheckpoints.companyId, binding.companyId), eq(nativeRunnerCheckpoints.issueId, binding.issueId),
|
||||
eq(nativeRunnerCheckpoints.runId, binding.runId)));
|
||||
|
||||
@@ -6,9 +6,12 @@ import { prepareNativeRemoteRunner, createNativeRunnerExecutionDriver } from "./
|
||||
const transport = vi.hoisted(() => ({
|
||||
register: vi.fn(),
|
||||
connect: vi.fn(),
|
||||
checkpoint: vi.fn(),
|
||||
checkpointRegistration: vi.fn(),
|
||||
}));
|
||||
vi.mock("../../realtime/runner-prp-ws.js", () => ({ registerRunnerPrpAuthority: transport.register }));
|
||||
vi.mock("../../realtime/runner-prp-outbound.js", () => ({ connectRunnerPrpIngress: transport.connect }));
|
||||
vi.mock("./native-runner-checkpoint.js", () => ({ nativeRunnerCheckpointRegistration: transport.checkpointRegistration }));
|
||||
|
||||
const binding = { companyId: "company", issueId: "issue", agentId: "agent", runId: "run-a" };
|
||||
const authority = {} as Parameters<NonNullable<ReturnType<typeof createNativeRunnerExecutionDriver>["controlPlaneRegistration"]>>[0];
|
||||
@@ -30,9 +33,41 @@ beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
transport.register.mockResolvedValue({ connectUrl: "ws://127.0.0.1/core", release: vi.fn() });
|
||||
transport.connect.mockReturnValue({ ready: Promise.resolve(), close: vi.fn(async () => {}) });
|
||||
transport.checkpointRegistration.mockReturnValue(transport.checkpoint);
|
||||
transport.checkpoint.mockResolvedValue(undefined);
|
||||
});
|
||||
|
||||
describe("shared Runner execution driver", () => {
|
||||
it("installs both staged checkpoint writers under the business lease without acquiring endpoints", async () => {
|
||||
const prepare = vi.fn();
|
||||
const persistence = { db: {} as never, bootId: "boot-c", authorityRunId: "run-c" };
|
||||
const driver = createNativeRunnerExecutionDriver({ target: sandbox(), binding: { ...binding, runId: "run-c" },
|
||||
controllerPersistence: persistence, prepareRemote: prepare });
|
||||
const sourceIdentity = { runnerInstanceId: "runner", environmentLeaseId: "lease", normalizedSessionId: "session",
|
||||
runId: "run-a", turnId: "turn-a", itemId: "item-a" };
|
||||
const targetIdentity = { ...sourceIdentity, runId: "run-c", turnId: "turn-c", itemId: "item-c" };
|
||||
await driver.onStagedRunAttachment!(authority, { sourceIdentity, targetIdentity, templateDigest: `sha256:${"a".repeat(64)}` });
|
||||
expect(transport.checkpointRegistration).toHaveBeenCalledExactlyOnceWith({ ...persistence,
|
||||
binding: { companyId: binding.companyId, issueId: binding.issueId, runId: "run-c" } });
|
||||
expect(transport.checkpoint.mock.calls).toEqual([[authority, sourceIdentity], [authority, targetIdentity]]);
|
||||
expect(prepare).not.toHaveBeenCalled();
|
||||
expect(transport.register).not.toHaveBeenCalled();
|
||||
expect(transport.connect).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("installs the target writer first when restoring an already persisted stage", async () => {
|
||||
const sourceIdentity = { runnerInstanceId: "runner", environmentLeaseId: "lease", normalizedSessionId: "session",
|
||||
runId: "run-a", turnId: "turn-a", itemId: "item-a" };
|
||||
const targetIdentity = { ...sourceIdentity, runId: "run-c", turnId: "turn-c", itemId: "item-c" };
|
||||
const restored = { store: { state: { stagedRunAttachment: { sourceIdentity, targetIdentity } } } } as unknown as typeof authority;
|
||||
const driver = createNativeRunnerExecutionDriver({ target: sandbox(), binding: { ...binding, runId: "run-c" },
|
||||
controllerPersistence: { db: {} as never, bootId: "boot-c", authorityRunId: "run-c" } });
|
||||
await driver.onStagedRunAttachment!(restored, { sourceIdentity, targetIdentity, templateDigest: `sha256:${"a".repeat(64)}` });
|
||||
expect(transport.checkpoint.mock.calls).toEqual([[restored, targetIdentity], [restored, sourceIdentity]]);
|
||||
expect(transport.register).not.toHaveBeenCalled();
|
||||
expect(transport.connect).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps local execution on the registered loopback route", async () => {
|
||||
const driver = createNativeRunnerExecutionDriver({ target: { kind: "local" }, binding });
|
||||
const connection = await driver.controlPlaneRegistration!(authority);
|
||||
|
||||
@@ -92,6 +92,20 @@ export async function prepareNativeRemoteRunner(input: RemotePreparation) {
|
||||
export function createNativeRunnerExecutionDriver(input: NativeRunnerExecutionDriverInput): RunnerExecutionDriver {
|
||||
return {
|
||||
...input.process,
|
||||
onStagedRunAttachment: input.controllerPersistence && input.binding.issueId
|
||||
? async (authority, proof) => {
|
||||
// Both journals use the current business lease. Install persistence
|
||||
// before staging; endpoint acquisition still belongs to attachment.
|
||||
const register = nativeRunnerCheckpointRegistration({ ...input.controllerPersistence!,
|
||||
binding: { companyId: input.binding.companyId, issueId: input.binding.issueId!, runId: input.binding.runId },
|
||||
});
|
||||
// A restored stage already publishes under the target. Its writer
|
||||
// must exist before registration flushes that restored revision.
|
||||
const identities = authority.store?.state.stagedRunAttachment
|
||||
? [proof.targetIdentity, proof.sourceIdentity] : [proof.sourceIdentity, proof.targetIdentity];
|
||||
for (const identity of identities) await register(authority, identity);
|
||||
}
|
||||
: undefined,
|
||||
controlPlaneRegistration: async (authority, attachmentIdentity) => {
|
||||
await input.beforeConnect?.(attachmentIdentity, authority);
|
||||
if (input.controllerPersistence && input.binding.issueId) {
|
||||
|
||||
+239
-57
@@ -60,6 +60,7 @@ import { readProcessStartedAt } from "../hot-restart.js";
|
||||
import { resolvePaperclipInstanceRoot } from "../../home-paths.js";
|
||||
import { prepareNativeHeartbeatRun } from "./prepare-native-run.js";
|
||||
import { buildNativeExecutionInput } from "./native-execution-input.js";
|
||||
import { buildNativeExecutionWithCheckpoint, findNativeSessionResumeRun, nativeToolContractFingerprintForTarget } from "./native-session-resume.js";
|
||||
import { nativeRuntimeContextFixture } from "./runtime-context.test-fixture.js";
|
||||
import { materializeAsset } from "./runtime-context.js";
|
||||
import { createRunnerTestTlsTunnel, dockerCommandRunner } from "../../__tests__/helpers/docker-command-runner.js";
|
||||
@@ -321,8 +322,10 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
};
|
||||
}
|
||||
|
||||
it.skipIf(!process.env.PAPERCLIP_TEST_RUNNER_CONTAINER).each(["warm", "hard", "legacy", "changed", "same-server"])("preserves remote warm continuity across completed turns: %s", async (scenario) => {
|
||||
it.skipIf(!process.env.PAPERCLIP_TEST_RUNNER_CONTAINER).each(["warm", "hard", "legacy", "changed", "same-server", "diskless", "diskless-per-turn", "diskless-per-turn-read-retry"])("preserves remote warm continuity across completed turns: %s", async (scenario) => {
|
||||
const hardRestart = !["warm", "same-server"].includes(scenario);
|
||||
const diskless = scenario.startsWith("diskless");
|
||||
const perTurn = scenario.startsWith("diskless-per-turn");
|
||||
const container = process.env.PAPERCLIP_TEST_RUNNER_CONTAINER!;
|
||||
const tunnel = await createRunnerTestTlsTunnel({ port: Number(process.env.PAPERCLIP_TEST_RUNNER_PORT),
|
||||
key: process.env.PAPERCLIP_TEST_RUNNER_TLS_KEY!, cert: process.env.NODE_EXTRA_CA_CERTS! });
|
||||
@@ -346,12 +349,13 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
taskPrompt: "Complete the first turn.",
|
||||
workspace: { id: randomUUID(), cwd: workspace, repoUrl: null, repoRef: null, branchName: null },
|
||||
normalizedSessionId: fixture.native.normalizedSessionId,
|
||||
lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 },
|
||||
lifecyclePolicy: perTurn ? { mode: "per_turn", idleTimeoutMs: null } : { mode: "warm", idleTimeoutMs: 60_000 },
|
||||
runtimeContext: context,
|
||||
completionContract: await boundContract(fixture.runId),
|
||||
});
|
||||
await fixture.db.update(heartbeatRuns).set({ runnerProfileJson: {
|
||||
...fixture.preparedProfile, nativeExecutionInput: execution,
|
||||
nativeToolContractFingerprint: nativeToolContractFingerprintForTarget("remote", execution.provider.kind),
|
||||
} }).where(eq(heartbeatRuns.id, fixture.runId));
|
||||
const remoteCwd = `/tmp/paperclip-continuity-${fixture.issueId}`;
|
||||
expect((await runner.execute({ command: "mkdir", args: ["-p", remoteCwd] })).exitCode).toBe(0);
|
||||
@@ -385,6 +389,26 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
onLog: async (_stream: "stdout" | "stderr", text: string) => { logs.push(text); },
|
||||
};
|
||||
let controller: ReturnType<typeof spawn> | undefined;
|
||||
const executeInController = (executionOptions: Parameters<typeof executePaperclipNativeSession>[0]) => new Promise<{ exitCode: number | null }>((resolveResult, reject) => {
|
||||
controller = spawn(process.execPath, ["--import", createRequire(import.meta.url).resolve("tsx"), resolve("server/src/__tests__/helpers/remote-warm-controller.test-worker.ts")],
|
||||
{ stdio: ["ignore", "pipe", "pipe", "ipc"], env: process.env });
|
||||
let timer = setTimeout(() => { controller?.kill("SIGKILL"); reject(new Error("Controller fixture imports exceeded 120 seconds")); }, 120_000);
|
||||
controller.stdout?.on("data", data => logs.push(String(data)));
|
||||
controller.stderr?.on("data", data => logs.push(String(data)));
|
||||
controller.on("error", error => { clearTimeout(timer); reject(error); });
|
||||
controller.on("exit", (code, signal) => { clearTimeout(timer); reject(new Error(`Warm controller exited: ${code}/${signal}: ${logs.join("").slice(-3_000)}`)); });
|
||||
controller.on("message", (message: { ready?: boolean; log?: { text: string }; result?: { exitCode: number | null }; error?: string }) => {
|
||||
if (message.ready) {
|
||||
clearTimeout(timer);
|
||||
timer = setTimeout(() => { controller?.kill("SIGKILL"); reject(new Error(`Warm controller admission exceeded 60 seconds: ${logs.join("").slice(-3_000)}`)); }, 60_000);
|
||||
controller!.send?.({ connectionString: temporary.connectionString, container, ingressUrl: `wss://127.0.0.1:${tunnel.port}`,
|
||||
options: { ...executionOptions, db: undefined, onLog: undefined, runnerExecutionTarget: { ...executionOptions.runnerExecutionTarget, runner: undefined } } });
|
||||
}
|
||||
if (message.log) logs.push(message.log.text);
|
||||
if (message.error) { clearTimeout(timer); reject(new Error(message.error)); }
|
||||
if (message.result) { clearTimeout(timer); resolveResult(message.result); }
|
||||
});
|
||||
});
|
||||
try {
|
||||
const firstOptions = { ...options, execution, runnerExecutionTarget: target };
|
||||
const executeFirst = async () => {
|
||||
@@ -454,28 +478,13 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
expect(nextProcess.stdout).toBe(priorProcess.stdout);
|
||||
return result;
|
||||
};
|
||||
const first = hardRestart ? await new Promise<{ exitCode: number | null }>((resolveResult, reject) => {
|
||||
controller = spawn(process.execPath, ["--import", createRequire(import.meta.url).resolve("tsx"), resolve("server/src/__tests__/helpers/remote-warm-controller.test-worker.ts")],
|
||||
{ stdio: ["ignore", "pipe", "pipe", "ipc"], env: process.env });
|
||||
const timer = setTimeout(() => { controller?.kill("SIGKILL"); reject(new Error("Warm controller startup exceeded 60 seconds")); }, 60_000);
|
||||
controller.stdout?.on("data", data => logs.push(String(data)));
|
||||
controller.stderr?.on("data", data => logs.push(String(data)));
|
||||
controller.on("error", error => { clearTimeout(timer); reject(error); });
|
||||
controller.on("exit", (code, signal) => { clearTimeout(timer); reject(new Error(`Warm controller exited: ${code}/${signal}: ${logs.join("").slice(-3_000)}`)); });
|
||||
controller.on("message", (message: { log?: { text: string }; result?: { exitCode: number | null }; error?: string }) => {
|
||||
if (message.log) logs.push(message.log.text);
|
||||
if (message.error) { clearTimeout(timer); reject(new Error(message.error)); }
|
||||
if (message.result) { clearTimeout(timer); resolveResult(message.result); }
|
||||
});
|
||||
controller.send?.({ connectionString: temporary.connectionString, container, ingressUrl: `wss://127.0.0.1:${tunnel.port}`,
|
||||
options: { ...firstOptions, db: undefined, onLog: undefined, runnerExecutionTarget: { ...target, runner: undefined } } });
|
||||
}) : await executeFirst();
|
||||
const first = hardRestart ? await executeInController(firstOptions) : await executeFirst();
|
||||
expect(first.exitCode).toBe(0);
|
||||
await finalizeNativeRun({ db: fixture.db, runId: fixture.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
|
||||
const snapshot = await runner.execute({ command: "sh", args: ["-c", `cat '${remoteCwd}'/.paperclip-runtime/paperclip-runner/sessions/*/runner/runner-state.json`] });
|
||||
expect(snapshot.exitCode).toBe(0);
|
||||
const saved = JSON.parse(snapshot.stdout);
|
||||
expect(saved.lifecycle).toBe("ready");
|
||||
expect(saved.lifecycle).toBe(perTurn ? "suspended" : "ready");
|
||||
const processBefore = await runner.execute({ command: "sh", args: ["-c", `cat '${remoteCwd}'/.paperclip-runtime/paperclip-runner/sessions/*/runner/runner-process.identity`] });
|
||||
expect(processBefore.exitCode).toBe(0);
|
||||
const historyBefore = await runner.execute({ command: "sh", args: ["-c", `cat '${remoteCwd}'/.paperclip-runtime/paperclip-runner/sessions/*/filesystem/codex-home/fake-codex-state.json`] });
|
||||
@@ -502,12 +511,11 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
runnerExecutionTarget: { ...target, leaseId: nextLeaseId, sandboxLeaseAcquisition: { outcome: "resumed" as const, providerLeaseId: container } },
|
||||
};
|
||||
if (controller) {
|
||||
await expect(executePaperclipNativeSession(nextOptions)).rejects.toThrow("prior_owner_terminal_state_indeterminate");
|
||||
const exited = new Promise<void>(resolveExit => controller!.once("exit", () => resolveExit()));
|
||||
controller.kill("SIGKILL");
|
||||
(controller as ReturnType<typeof spawn>).kill("SIGKILL");
|
||||
await exited;
|
||||
controller = undefined;
|
||||
await expect(executePaperclipNativeSession({ ...nextOptions,
|
||||
if (!perTurn) await expect(executePaperclipNativeSession({ ...nextOptions,
|
||||
runnerExecutionTarget: { ...nextOptions.runnerExecutionTarget,
|
||||
sandboxLeaseAcquisition: { outcome: "replacement", providerLeaseId: "unrelated-sandbox" } },
|
||||
})).rejects.toThrow("prior_owner_terminal_state_indeterminate");
|
||||
@@ -526,10 +534,43 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
}
|
||||
expect(changed).toBe(1);
|
||||
}
|
||||
const [nextPrepared] = await fixture.db.select({ profile: heartbeatRuns.runnerProfileJson }).from(heartbeatRuns).where(eq(heartbeatRuns.id, nextRunId));
|
||||
await fixture.db.update(heartbeatRuns).set({ runnerProfileJson: { ...nextPrepared.profile,
|
||||
nativeExecutionInput: nextOptions.execution } }).where(eq(heartbeatRuns.id, nextRunId));
|
||||
const second = await executePaperclipNativeSession(nextOptions);
|
||||
if (diskless) {
|
||||
// Preserve sandbox/provider storage and workspace dependencies. Delete
|
||||
// both native controller projections; admission must discover the source.
|
||||
await rm(resolve(resolvePaperclipInstanceRoot(), "runtime", "paperclip-runner"), { recursive: true, force: true });
|
||||
}
|
||||
// Exercise the existing admission selector/rebinder, which supplies per-turn
|
||||
// runtime snapshots. It does not restore PRP or supervisor files.
|
||||
const admitContinuation = async (current: typeof execution) => {
|
||||
const [row] = await fixture.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, current.binding.runId));
|
||||
const selected = await findNativeSessionResumeRun(fixture.db, { ...current.binding, currentRunId: row.id,
|
||||
normalizedSessionId: fixture.native.normalizedSessionId, beforeCreatedAt: row.createdAt });
|
||||
const admission = buildNativeExecutionWithCheckpoint({ previousRun: selected,
|
||||
normalizedSessionId: fixture.native.normalizedSessionId, executionTargetKind: "remote", buildExecution: () => current });
|
||||
if (perTurn) expect(admission.checkpoint).not.toBeNull();
|
||||
await fixture.db.update(heartbeatRuns).set({ runnerProfileJson: { ...row.runnerProfileJson,
|
||||
nativeExecutionInput: current, nativeToolContractFingerprint: nativeToolContractFingerprintForTarget("remote", current.provider.kind),
|
||||
...(perTurn ? { sessionCheckpoint: admission.checkpoint } : {}) } }).where(eq(heartbeatRuns.id, row.id));
|
||||
};
|
||||
await admitContinuation(nextOptions.execution);
|
||||
if (scenario === "diskless-per-turn-read-retry") {
|
||||
let rejectedReads = 0;
|
||||
await expect(executePaperclipNativeSession({ ...nextOptions,
|
||||
runnerExecutionTarget: { ...nextOptions.runnerExecutionTarget, runner: { ...runner,
|
||||
execute: async request => {
|
||||
if (request.args?.some(arg => arg.includes("base64 <") && arg.includes("runner-state.json"))) {
|
||||
rejectedReads++;
|
||||
throw new Error("fixture_remote_state_timeout");
|
||||
}
|
||||
return runner.execute(request);
|
||||
},
|
||||
} },
|
||||
})).rejects.toThrow("fixture_remote_state_timeout");
|
||||
expect(rejectedReads).toBe(1);
|
||||
// Retry with the partial PRP projection left by the failed read. No
|
||||
// fixture writes a suspended receipt or changes the surviving Runner.
|
||||
}
|
||||
const second = diskless ? await executeInController(nextOptions) : await executePaperclipNativeSession(nextOptions);
|
||||
expect(second.exitCode).toBe(0);
|
||||
const historyAfter = await runner.execute({ command: "sh", args: ["-c", `cat '${remoteCwd}'/.paperclip-runtime/paperclip-runner/sessions/*/filesystem/codex-home/fake-codex-state.json`] });
|
||||
const providerAfter = JSON.parse(historyAfter.stdout);
|
||||
@@ -540,12 +581,19 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
expect(calls.stdout.split("\n").filter(line => line === "thread/start")).toHaveLength(1);
|
||||
const providerStarts = calls.stdout.split("\n").filter(line => line === "process-start").length;
|
||||
expect(providerStarts).toBeGreaterThanOrEqual(1);
|
||||
expect(providerStarts).toBeLessThanOrEqual(["legacy", "changed"].includes(scenario) ? 3 : hardRestart ? 2 : 1);
|
||||
expect(providerStarts).toBeLessThanOrEqual(perTurn || ["legacy", "changed"].includes(scenario) ? 3 : hardRestart ? 2 : 1);
|
||||
const processAfter = await runner.execute({ command: "sh", args: ["-c", `cat '${remoteCwd}'/.paperclip-runtime/paperclip-runner/sessions/*/runner/runner-process.identity`] });
|
||||
expect(processAfter.exitCode).toBe(0);
|
||||
if (["legacy", "changed"].includes(scenario)) expect(processAfter.stdout).not.toBe(processBefore.stdout);
|
||||
if (perTurn || ["legacy", "changed"].includes(scenario)) expect(processAfter.stdout).not.toBe(processBefore.stdout);
|
||||
else expect(processAfter.stdout).toBe(processBefore.stdout);
|
||||
await finalizeNativeRun({ db: fixture.db, runId: nextRunId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
|
||||
if (diskless && controller) {
|
||||
const exited = new Promise<void>(resolveExit => controller!.once("exit", () => resolveExit()));
|
||||
(controller as ReturnType<typeof spawn>).kill("SIGKILL");
|
||||
await exited;
|
||||
controller = undefined;
|
||||
await rm(resolve(resolvePaperclipInstanceRoot(), "runtime", "paperclip-runner"), { recursive: true, force: true });
|
||||
}
|
||||
const thirdRunId = randomUUID();
|
||||
const thirdLeaseId = randomUUID();
|
||||
const [thirdRun] = await fixture.db.insert(heartbeatRuns).values({
|
||||
@@ -562,9 +610,11 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
issue: { id: fixture.issueId, title: "Continue the same conversation", description: null, reviewPolicy: null },
|
||||
environmentLeaseId: thirdLeaseId });
|
||||
await fixture.db.insert(nativeRunFinalizations).values({ runId: thirdRunId, companyId, issueId: fixture.issueId, phase: "observed" });
|
||||
const thirdExecution = { ...nextOptions.execution, completionContract: await boundContract(thirdRunId),
|
||||
binding: { ...execution.binding, runId: thirdRunId }, task: { ...execution.task, prompt: "Complete the third conversation turn." } };
|
||||
await admitContinuation(thirdExecution);
|
||||
const third = await executePaperclipNativeSession({ ...nextOptions, runnerInstanceId: thirdNative.runnerInstanceId,
|
||||
execution: { ...nextOptions.execution, completionContract: await boundContract(thirdRunId),
|
||||
binding: { ...execution.binding, runId: thirdRunId }, task: { ...execution.task, prompt: "Complete the third conversation turn." } },
|
||||
execution: thirdExecution,
|
||||
runnerExecutionTarget: { ...nextOptions.runnerExecutionTarget, leaseId: thirdLeaseId },
|
||||
});
|
||||
expect(third.exitCode).toBe(0);
|
||||
@@ -573,10 +623,11 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
const finalCalls = await runner.execute({ command: "cat", args: [`${remoteCwd}/provider-calls.log`] });
|
||||
expect(finalCalls.stdout.split("\n").filter(line => line === "turn/start")).toHaveLength(3);
|
||||
expect(finalCalls.stdout.split("\n").filter(line => line === "thread/start")).toHaveLength(1);
|
||||
expect(finalCalls.stdout.split("\n").filter(line => line === "process-start")).toHaveLength(providerStarts);
|
||||
expect(finalCalls.stdout.split("\n").filter(line => line === "process-start")).toHaveLength(providerStarts + (perTurn ? 1 : 0));
|
||||
const finalProcess = await runner.execute({ command: "sh", args: ["-c", `cat '${remoteCwd}'/.paperclip-runtime/paperclip-runner/sessions/*/runner/runner-process.identity`] });
|
||||
expect(finalProcess.stdout).toBe(processAfter.stdout);
|
||||
expect(await closeIdleWarmNativeSessionsForRestart()).toEqual({ closed: 1, busy: 0, failed: 0 });
|
||||
if (perTurn) expect(finalProcess.stdout).not.toBe(processAfter.stdout);
|
||||
else expect(finalProcess.stdout).toBe(processAfter.stdout);
|
||||
expect(await closeIdleWarmNativeSessionsForRestart()).toEqual({ closed: perTurn ? 0 : 1, busy: 0, failed: 0 });
|
||||
const suspended = await runner.execute({ command: "sh", args: ["-c", `cat '${remoteCwd}'/.paperclip-runtime/paperclip-runner/sessions/*/runner/runner-state.json`] });
|
||||
expect(suspended.exitCode).toBe(0);
|
||||
expect(JSON.parse(suspended.stdout).lifecycle).toBe("suspended");
|
||||
@@ -588,7 +639,7 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
await closeIdleWarmNativeSessionsForRestart();
|
||||
await tunnel.close();
|
||||
}
|
||||
}, 180_000);
|
||||
}, 360_000);
|
||||
|
||||
async function persistRestartEvidence(input: {
|
||||
fixture: Awaited<ReturnType<typeof seedRun>>;
|
||||
@@ -1277,10 +1328,11 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
} finally { await authority.stop(); }
|
||||
});
|
||||
|
||||
it.each(["idle", "preparing_successor", "unclaimed_successor", "claimed_successor", "expired_preparation", "dispatching_successor", "foreign_preparation", "foreign_native_preparation", "wrong_issue", "new_work", "replaced_controller"] as const)(
|
||||
"settles a completed warm checkpoint during early retirement without granting work: %s",
|
||||
async scenario => {
|
||||
const fixture = await seedRun(`WARM-EARLY-RETIREMENT-${scenario}`);
|
||||
it.each((["idle", "preparing_successor", "unclaimed_successor", "claimed_successor", "expired_preparation", "dispatching_successor", "foreign_preparation", "foreign_native_preparation", "wrong_issue", "new_work", "replaced_controller", "unaccepted"] as const).flatMap(scenario =>
|
||||
(["runner.suspend", "turn.stop"] as const).map(command => ({ scenario, command }))))(
|
||||
"settles a completed warm checkpoint during early retirement without granting work: $scenario/$command",
|
||||
async ({ scenario, command }) => {
|
||||
const fixture = await seedRun(`WARM-EARLY-RETIREMENT-${scenario}-${command}`);
|
||||
const binding = { companyId, issueId: fixture.issueId, runId: fixture.runId };
|
||||
const identity = transportOptions(fixture, runtimeRoot).prpIdentity;
|
||||
const authority = new DurablePrpControlPlane({
|
||||
@@ -1305,7 +1357,7 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
});
|
||||
await fixture.db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() })
|
||||
.where(eq(heartbeatRuns.id, fixture.runId));
|
||||
await fixture.db.update(nativeRunFinalizations).set({ phase: "committed", resultId,
|
||||
await fixture.db.update(nativeRunFinalizations).set({ phase: "committed", resultId: scenario === "unaccepted" ? null : resultId,
|
||||
leaseOwner: null, leaseExpiresAt: null,
|
||||
...(scenario === "replaced_controller" ? { controllerBootId: randomUUID(), controllerGeneration: 2 } : {}),
|
||||
}).where(eq(nativeRunFinalizations.runId, fixture.runId));
|
||||
@@ -1314,7 +1366,7 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
if (scenario !== "idle") {
|
||||
nextRunId = randomUUID();
|
||||
const preparing = scenario !== "claimed_successor" && scenario !== "unclaimed_successor";
|
||||
const otherIssue = scenario === "wrong_issue" ? (await seedRun("WARM-OTHER-ISSUE")).issueId : fixture.issueId;
|
||||
const otherIssue = scenario === "wrong_issue" ? (await seedRun(`WARM-OTHER-ISSUE-${command}`)).issueId : fixture.issueId;
|
||||
await fixture.db.insert(heartbeatRuns).values({ id: nextRunId, companyId, agentId,
|
||||
issueId: otherIssue, scopeKind: "issue", contextSnapshot: { issueId: otherIssue },
|
||||
...(preparing ? { runtimeMode: "legacy", executionStage: scenario === "dispatching_successor" ? "dispatching" : "preparing",
|
||||
@@ -1331,7 +1383,7 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
}
|
||||
await fixture.db.update(issues).set({ executionRunId: nextRunId }).where(eq(issues.id, fixture.issueId));
|
||||
const before = (await loadNativeRunnerCheckpoint(fixture.db, binding))!;
|
||||
authority.queueCommand(scenario === "new_work" ? "turn.start" : "runner.suspend", {}, `early-retirement-${scenario}`);
|
||||
authority.queueCommand(scenario === "new_work" ? "turn.start" : command, {}, `early-retirement-${scenario}`);
|
||||
if (!["idle", "preparing_successor", "claimed_successor"].includes(scenario)) {
|
||||
await expect(authority.flushCheckpoint()).rejects.toThrow("native_runner_checkpoint_owner_lost");
|
||||
expect((await loadNativeRunnerCheckpoint(fixture.db, binding))?.records).toEqual(before.records);
|
||||
@@ -1342,7 +1394,7 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
saved.restoreSessionDirectory(restoredRoot);
|
||||
const restored = JSON.parse(await readFile(resolve(restoredRoot, "control-plane", "control-plane-state.json"), "utf8"));
|
||||
expect(restored.identity).toEqual(identity);
|
||||
expect(restored.commands).toContainEqual(expect.objectContaining({ type: "runner.suspend", status: "pending" }));
|
||||
expect(restored.commands).toContainEqual(expect.objectContaining({ type: command, status: "pending" }));
|
||||
}
|
||||
const [task] = await fixture.db.select({ executionRunId: issues.executionRunId }).from(issues)
|
||||
.where(eq(issues.id, fixture.issueId));
|
||||
@@ -1351,6 +1403,104 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
},
|
||||
);
|
||||
|
||||
realProcessIt("durably retires an accepted warm Runner while its same-controller successor prepares", async () => {
|
||||
const fixture = await seedRun("REAL-WARM-RETIREMENT");
|
||||
const binding = { companyId, issueId: fixture.issueId, runId: fixture.runId };
|
||||
const root = resolve(runtimeRoot, fixture.runId, "real-retirement");
|
||||
const options = transportOptions(fixture, root);
|
||||
const retirementController = await currentNativeControllerIdentity();
|
||||
const diagnostics: string[] = [], checkpointFailures: unknown[] = [];
|
||||
let authority!: DurablePrpControlPlane;
|
||||
await fixture.db.update(nativeRunFinalizations).set({ controllerBootId: retirementController.bootId,
|
||||
controllerGeneration: 1, leaseOwner: "real-retirement-owner", leaseExpiresAt: new Date(Date.now() + 60_000),
|
||||
}).where(eq(nativeRunFinalizations.runId, fixture.runId));
|
||||
const transport = createRunnerdCodexTransport({ ...options,
|
||||
codexArgs: [...options.codexArgs, "--linger-after-turn-start"],
|
||||
onDiagnostic: message => diagnostics.push(message),
|
||||
controlPlaneRegistration: async core => {
|
||||
authority = core;
|
||||
const write = nativeRunnerCheckpointWriter(fixture.db, { ...binding, authorityRunId: fixture.runId,
|
||||
bootId: retirementController.bootId, generation: 1, leaseOwner: "real-retirement-owner" });
|
||||
core.setCheckpointPersistence(async checkpoint => {
|
||||
try { await write(checkpoint); }
|
||||
catch (error) { checkpointFailures.push(error); throw error; }
|
||||
});
|
||||
return registerRunnerPrpAuthority({ ...binding, authority: core });
|
||||
},
|
||||
});
|
||||
let runnerPid: number | null = null, providerPid: number | null = null;
|
||||
try {
|
||||
await transport.transport.request("thread/start", { cwd: root, dynamicTools: [] });
|
||||
await transport.transport.request("turn/start", { input: [{ type: "text", text: "Keep the provider alive after accepting its result." }] });
|
||||
runnerPid = transport.evidence().runnerPid;
|
||||
providerPid = transport.evidence().providerPid;
|
||||
expect(runnerPid).toEqual(expect.any(Number));
|
||||
expect(providerPid).toEqual(expect.any(Number));
|
||||
await waitForCondition("active real provider before accepted-result retirement", () =>
|
||||
authority.store.state.committedEvents.some(event => event.eventType === "turn.started"));
|
||||
await authority.flushCheckpoint();
|
||||
const [run] = await fixture.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId));
|
||||
const resultId = randomUUID();
|
||||
await fixture.db.insert(nativeRunResults).values({ id: resultId, ...binding, turnId: options.prpIdentity.turnId,
|
||||
completionContractId: run.completionContractId!, serverFingerprint: "real-retirement-accepted",
|
||||
schemaStatus: "accepted", canonicalSha256: "a".repeat(64),
|
||||
resultJson: { schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", summary: "Accepted before provider cleanup." },
|
||||
});
|
||||
await fixture.db.update(nativeRunFinalizations).set({ phase: "committed", resultId, leaseOwner: null, leaseExpiresAt: null })
|
||||
.where(eq(nativeRunFinalizations.runId, fixture.runId));
|
||||
await fixture.db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() })
|
||||
.where(eq(heartbeatRuns.id, fixture.runId));
|
||||
const nextRunId = randomUUID();
|
||||
const [nextRun] = await fixture.db.insert(heartbeatRuns).values({ id: nextRunId, companyId, agentId,
|
||||
issueId: fixture.issueId, nativeIssueId: fixture.issueId, scopeKind: "issue", status: "running", runtimeMode: "native",
|
||||
invocationSource: "assignment", contextSnapshot: { issueId: fixture.issueId } }).returning();
|
||||
const next = await prepareNativeHeartbeatRun({ db: fixture.db, run: nextRun,
|
||||
issue: { id: fixture.issueId, title: "Next conversation", description: null }, environmentLeaseId: randomUUID() });
|
||||
await fixture.db.insert(nativeRunFinalizations).values({ runId: nextRunId, companyId, issueId: fixture.issueId, phase: "observed" });
|
||||
await fixture.db.update(issues).set({ executionRunId: nextRunId }).where(eq(issues.id, fixture.issueId));
|
||||
const [contract] = await fixture.db.select().from(completionContracts).where(eq(completionContracts.id, run.completionContractId!));
|
||||
const nextExecution = buildNativeExecutionInput({ companyId, agentId, runId: nextRunId,
|
||||
issue: { id: fixture.issueId, identifier: "NRR-NATIVE-RETIREMENT", title: "Next conversation", description: null, workMode: "standard" },
|
||||
taskPrompt: "Start only after the previous provider durably suspends.",
|
||||
workspace: { id: randomUUID(), cwd: root, repoUrl: null, repoRef: null, branchName: null },
|
||||
normalizedSessionId: next.normalizedSessionId, runtimeContext: nativeRuntimeContextFixture(),
|
||||
completionContract: { id: contract.id, sha256: contract.canonicalSha256, schemaVersion: contract.schemaVersion, contract: contract.contractJson as never },
|
||||
});
|
||||
const fixtureStoppedAfterRealClaim = new Error("fixture stopped before next provider dispatch");
|
||||
await expect(executePaperclipNativeSession({ db: fixture.db, execution: nextExecution, runnerInstanceId: next.runnerInstanceId,
|
||||
onExecutionLeaseAcquired: async () => {
|
||||
const [claimed] = await fixture.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, nextRunId));
|
||||
expect(claimed.controllerBootId).toBe(retirementController.bootId);
|
||||
expect(claimed.leaseOwner).toEqual(expect.any(String));
|
||||
expect(claimed.leaseExpiresAt!.getTime()).toBeGreaterThan(Date.now());
|
||||
await expect(transport.transport.close(), diagnostics.join("\n")).resolves.toBeUndefined();
|
||||
throw fixtureStoppedAfterRealClaim;
|
||||
},
|
||||
})).rejects.toBe(fixtureStoppedAfterRealClaim);
|
||||
expect(checkpointFailures).toEqual([]);
|
||||
const state = JSON.parse(await readFile(resolve(root, "runner", "runner-state.json"), "utf8"));
|
||||
expect(state).toMatchObject({ lifecycle: "suspended", runId: fixture.runId,
|
||||
normalizedSessionId: options.prpIdentity.normalizedSessionId, runnerInstanceId: options.prpIdentity.runnerInstanceId });
|
||||
const saved = (await loadNativeRunnerCheckpoint(fixture.db, binding))!;
|
||||
const commands = [...saved.records].filter(([key]) => key.startsWith("command:")).map(([, value]) => JSON.parse(value));
|
||||
for (const type of ["turn.stop", "runner.drain", "runner.suspend"]) {
|
||||
expect(commands, diagnostics.join("\n")).toContainEqual(expect.objectContaining({ type, status: "completed" }));
|
||||
}
|
||||
expect(commands.filter(command => command.type === "turn.start")).toHaveLength(1);
|
||||
const [task] = await fixture.db.select().from(issues).where(eq(issues.id, fixture.issueId));
|
||||
expect(task.executionRunId).toBe(nextRunId);
|
||||
} catch (error) {
|
||||
throw new Error(`${String(error)}; owned retirement diagnostics ${JSON.stringify({
|
||||
checkpointFailures: checkpointFailures.map(String), diagnostics,
|
||||
commands: authority?.store.state.commands.map(({ type, status }) => ({ type, status })),
|
||||
})}`);
|
||||
} finally {
|
||||
await transport.transport.close().catch(() => undefined);
|
||||
if (providerPid && processAlive(providerPid)) process.kill(providerPid, "SIGKILL");
|
||||
await stopOwnedProcessGroup(runnerPid, root).catch(() => undefined);
|
||||
}
|
||||
}, 45_000);
|
||||
|
||||
it("applies checkpoint deltas in Postgres without rewriting authority and removes only evicted events", async () => {
|
||||
const fixture = await seedRun("CHECKPOINT-DELTA");
|
||||
const root = resolve(runtimeRoot, fixture.runId);
|
||||
@@ -1463,8 +1613,8 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
realProcessIt("keeps checkpoint SQL independent of streamed output volume with a real Postgres writer", async () => {
|
||||
const samples: Array<{ chunks: number; checkpointWrites: number; transactions: number;
|
||||
changedRows: number; checkpointBytes: number; walBytes: number; elapsedMs: number; idleSql: number;
|
||||
streamSql: number; streamTransactions: number; streamChangedRows: number }> = [];
|
||||
for (const chunks of [8, 512]) {
|
||||
streamSql: number; streamTransactions: number; streamChangedRows: number; renewalSql: number }> = [];
|
||||
for (const chunks of [8, 512, 8192]) {
|
||||
const fixture = await seedRun(`CHECKPOINT-COST-${chunks}`);
|
||||
const root = resolve(runtimeRoot, fixture.runId);
|
||||
const releaseOutput = resolve(root, "release-output");
|
||||
@@ -1472,19 +1622,31 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
const owner = { ...binding, authorityRunId: fixture.runId, bootId: successor.bootId,
|
||||
generation: 1, leaseOwner: "cost-controller" };
|
||||
await fixture.db.update(nativeRunFinalizations).set({ controllerBootId: owner.bootId,
|
||||
controllerGeneration: 1, leaseOwner: owner.leaseOwner, leaseExpiresAt: new Date(Date.now() + 120_000),
|
||||
controllerPid: successor.pid, controllerProcessStartedAt: successor.processStartedAt,
|
||||
controllerGeneration: 1, leaseOwner: owner.leaseOwner, leaseExpiresAt: new Date(Date.now() + 60_000),
|
||||
}).where(eq(nativeRunFinalizations.runId, fixture.runId));
|
||||
let authority!: DurablePrpControlPlane;
|
||||
let measuring = false;
|
||||
let transactions = 0, changedRows = 0, checkpointBytes = 0;
|
||||
const statements: string[] = [];
|
||||
const capturedClients = [fixture.db.$client, checkpointDb(fixture.db).$client];
|
||||
const renewalStatements: string[] = [];
|
||||
let leaseLost = false;
|
||||
const renewal = startNativeSessionExecutionLeaseRenewal({ db: fixture.db, ...binding, leaseOwner: owner.leaseOwner,
|
||||
attempt: 0, controller: successor, remote: true, onLost: async () => { leaseLost = true; } });
|
||||
const capturedClients = [fixture.db.$client, checkpointDb(fixture.db).$client, livenessDb(fixture.db).$client];
|
||||
const priorDebug = capturedClients.map(client => client.options.debug);
|
||||
// Capture statements only, never SQL parameters (encrypted state or secrets).
|
||||
for (const client of capturedClients) {
|
||||
client.options.debug = (_connection, query) => { if (measuring) statements.push(query); };
|
||||
}
|
||||
capturedClients.forEach((client, index) => {
|
||||
client.options.debug = (_connection, query) => {
|
||||
if (!measuring) return;
|
||||
// Renewals are existing liveness traffic, reported separately. A
|
||||
// checkpoint query on any pool still counts against the zero oracle.
|
||||
if (index === 2 && !query.includes("native_runner_checkpoints")) renewalStatements.push(query);
|
||||
else statements.push(query);
|
||||
};
|
||||
});
|
||||
const write = nativeRunnerCheckpointWriter(fixture.db, owner);
|
||||
let costFailure: unknown;
|
||||
const transport = createRunnerdCodexTransport({ ...transportOptions(fixture, root),
|
||||
codexArgs: ["--state-file", resolve(root, "provider.json"), "--linger-after-turn-start",
|
||||
"--held-output", "--held-output-count", String(chunks), "--held-output-release-file", releaseOutput],
|
||||
@@ -1516,7 +1678,7 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
const beforeStream = { sql: statements.length, transactions, changedRows };
|
||||
await writeFile(releaseOutput, "release\n", { flag: "wx" });
|
||||
await waitForCondition(`the final scripted text chunk of ${chunks}`, () => authority.store.state.committedEvents
|
||||
.some(event => event.eventType === "item.delta" && JSON.stringify(event.envelope).includes(`still computing ${chunks - 1} `)), 60_000)
|
||||
.some(event => event.eventType === "item.delta" && JSON.stringify(event.envelope).includes(`still computing ${chunks - 1} `)), chunks > 4096 ? 480_000 : 60_000)
|
||||
.catch(error => { console.info("checkpoint-cost-stream-diagnostic", JSON.stringify({ chunks,
|
||||
cursor: authority.store.state.ackedSourceSeq,
|
||||
receivedTextEvents: authority.store.state.committedEvents.filter(event => event.eventType === "item.delta").length })); throw error; });
|
||||
@@ -1528,13 +1690,14 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
&& query.includes("native_runner_checkpoints")).length;
|
||||
expect(checkpointWrites).toBeGreaterThan(0); // The collector must see real control writes.
|
||||
const snapshot = { chunks, checkpointWrites, transactions, changedRows, checkpointBytes,
|
||||
walBytes: Number(walEnd.bytes), elapsedMs, idleSql: 0,
|
||||
walBytes: Number(walEnd.bytes), elapsedMs, idleSql: 0, renewalSql: renewalStatements.length,
|
||||
streamSql: statements.length - beforeStream.sql,
|
||||
streamTransactions: transactions - beforeStream.transactions,
|
||||
streamChangedRows: changedRows - beforeStream.changedRows };
|
||||
// Setup control messages may coalesce differently. The released text-only
|
||||
// interval must add exactly zero work, independent of that scheduling.
|
||||
expect(snapshot).toMatchObject({ streamSql: 0, streamTransactions: 0, streamChangedRows: 0 });
|
||||
expect(leaseLost).toBe(false);
|
||||
statements.length = 0;
|
||||
measuring = true;
|
||||
await new Promise(done => setTimeout(done, 100));
|
||||
@@ -1547,20 +1710,34 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
expect(statements.some(query => /insert into "native_runner_checkpoints"/i.test(query))).toBe(true);
|
||||
measuring = false;
|
||||
samples.push(snapshot);
|
||||
console.info("native-checkpoint-cost-sample", JSON.stringify(snapshot));
|
||||
} catch (error) {
|
||||
costFailure = error;
|
||||
throw error;
|
||||
} finally {
|
||||
measuring = false;
|
||||
capturedClients.forEach((client, index) => { client.options.debug = priorDebug[index]; });
|
||||
await transport.transport.close();
|
||||
try { await transport.transport.close(); }
|
||||
catch (error) {
|
||||
if (costFailure) throw new AggregateError([costFailure, error], "Checkpoint cost measurement and cleanup failed");
|
||||
throw error;
|
||||
}
|
||||
finally {
|
||||
await renewal.stop().catch(error => {
|
||||
if (costFailure) throw new AggregateError([costFailure, error], "Checkpoint cost measurement and lease renewal failed");
|
||||
throw error;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
const report = { schema: "paperclip.checkpoint-cost/v1", samples,
|
||||
scope: "isolated Postgres; control operations included; WAL is diagnostic; no production capacity claim" };
|
||||
if (process.env.PAPERCLIP_TEST_CHECKPOINT_COST_REPORT) await writeFile(process.env.PAPERCLIP_TEST_CHECKPOINT_COST_REPORT,
|
||||
`${JSON.stringify(report, null, 2)}\n`, { flag: "wx", mode: 0o600 });
|
||||
// Both stream sizes were checked independently; total setup counts remain
|
||||
// Each stream size was checked independently; total setup counts remain
|
||||
// diagnostic because control-message batching is scheduling-dependent.
|
||||
console.info("native-checkpoint-cost", JSON.stringify(report));
|
||||
}, 120_000);
|
||||
}, 540_000);
|
||||
|
||||
realProcessIt.each([false, true])("adopts the same runner from Postgres after losing the controller filesystem (prepared task: %s)", async preparedTask => {
|
||||
const fixture = await seedRun(`EMPTY-CONTROLLER-DISK-${preparedTask}`);
|
||||
@@ -1579,9 +1756,10 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
heartbeatRunId: fixture.runId, issueId: fixture.issueId, status: "active", leasePolicy: "ephemeral",
|
||||
provider: "plugin", providerLeaseId: "saved-task", metadata: { agentId, runnerTaskLease: true, runnerTaskDispatched: true } });
|
||||
}
|
||||
const originalLeaseExpiresAt = new Date(Date.now() + 60_000);
|
||||
await fixture.db.update(nativeRunFinalizations).set({ controllerBootId: bootId,
|
||||
controllerGeneration: 1, controllerPid: process.pid, controllerProcessStartedAt: new Date(),
|
||||
leaseOwner: "original-controller", leaseExpiresAt: new Date(Date.now() + 60_000),
|
||||
leaseOwner: "original-controller", leaseExpiresAt: originalLeaseExpiresAt,
|
||||
}).where(eq(nativeRunFinalizations.runId, fixture.runId));
|
||||
const options = {
|
||||
...transportOptions(fixture, controllerRoot),
|
||||
@@ -1647,9 +1825,13 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
await new Promise(resolveOutage => setTimeout(resolveOutage, outageMs));
|
||||
expect(processAlive(runnerPid)).toBe(true);
|
||||
expect(processAlive(providerPid)).toBe(true);
|
||||
await fixture.db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 1) })
|
||||
.where(eq(nativeRunFinalizations.runId, fixture.runId));
|
||||
bootId = randomUUID();
|
||||
if (originalLeaseExpiresAt.getTime() > Date.now()) {
|
||||
const [waiting] = await claimNativeRestartRecoveries({ db: fixture.db, controller: { ...successor, bootId },
|
||||
restartKind: "hard", runIds: [fixture.runId] });
|
||||
expect(waiting).toMatchObject({ kind: "awaiting_evidence", reason: "remote_controller_lease_live", retryAt: originalLeaseExpiresAt });
|
||||
await new Promise(resolveExpiry => setTimeout(resolveExpiry, Math.max(0, originalLeaseExpiresAt.getTime() - Date.now()) + 20));
|
||||
}
|
||||
const [claim] = await claimNativeRestartRecoveries({ db: fixture.db, controller: { ...successor, bootId },
|
||||
restartKind: "hard", runIds: [fixture.runId] });
|
||||
expect(claim).toMatchObject({ kind: "reattach_remote_runner", providerAttempt: 0, controllerGeneration: 2 });
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { onRetainedNativeSessionCleanupRelease } from "../retained-native-session-cleanup-notifications.js";
|
||||
import { readIdleLocalWork, markIdleStartupComplete, markIdleIngressTracked } from "../idle-local-work.js";
|
||||
import * as runnerCheckpoint from "./native-runner-checkpoint.js";
|
||||
import * as stagedAttachmentRecovery from "./native-staged-attachment-recovery.js";
|
||||
import * as controllerRecovery from "./native-controller-recovery.js";
|
||||
import { DurablePrpControlPlane } from "../../vendor/paperclip-runner/index.js";
|
||||
import * as environmentTaskRuntime from "../environment-task-runtime.js";
|
||||
import * as pluginTaskRuntime from "./plugin-runner-task.js";
|
||||
@@ -47,6 +49,7 @@ import {
|
||||
import {
|
||||
canonicalNativeRuntimeContextDigest,
|
||||
CURSOR_DISTRIBUTION_PINS,
|
||||
KIMI_DISTRIBUTION_CLOSURE_SHA256,
|
||||
QUALIFIED_ACPX_PROFILES,
|
||||
acpxRuntimeSessionDirectoryName,
|
||||
resolveAcpxRuntimeRoot,
|
||||
@@ -96,6 +99,7 @@ import { buildNativeHeartbeatPreparationSpans, createNativeRunTrace } from "./na
|
||||
import { NativeRunnerOwnershipUnverifiedError } from "./native-runner-ownership.js";
|
||||
import type { AdapterRuntimeEvent } from "../../adapters/index.js";
|
||||
import type { AdapterUsageCheckpoint } from "@paperclipai/adapter-utils";
|
||||
import { acknowledgedNativeStopExecutionHasStopped, isAcknowledgedNativeStop } from "../acknowledged-native-stop.js";
|
||||
|
||||
vi.mock("node:fs", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("node:fs")>();
|
||||
@@ -391,6 +395,8 @@ import {
|
||||
REMOTE_RUNNER_CHILD_LAUNCH_SCRIPT,
|
||||
verifyRemoteRunnerReattachment,
|
||||
readRemoteProviderPackManifest,
|
||||
prepareRemoteProviderPackOnTarget,
|
||||
resolveRemoteProviderPackSource,
|
||||
readBundledRemoteProviderPackManifest,
|
||||
providerSessionIdentityFromDurableProviderState,
|
||||
durableProviderCheckpointFailureReason,
|
||||
@@ -416,6 +422,10 @@ import {
|
||||
import { onNativeRunnerEvidence } from "./native-runner-evidence-signal.js";
|
||||
|
||||
beforeEach(() => {
|
||||
// These transport-wiring fixtures use a partial Db double and synthetic IDs.
|
||||
// The real selector/bootstrap proof is exercised with Postgres by the durable
|
||||
// discovery and diskless admission suites, not by this backend unit boundary.
|
||||
vi.spyOn(controllerRecovery, "resolveNativeControllerRecovery").mockResolvedValue({ kind: "no_prior_execution" });
|
||||
state.drainMaintenance.mockReset().mockResolvedValue(undefined);
|
||||
usageRecorder.capture.mockReset().mockImplementation(async receipt => receipt);
|
||||
usageRecorder.create.mockReset().mockResolvedValue({ capture: usageRecorder.capture,
|
||||
@@ -1946,6 +1956,40 @@ describe("remote provider pack manifest", () => {
|
||||
expect(readRemoteProviderPackManifest(root).payload.pins.opencode).toBe(
|
||||
"1.18.34",
|
||||
);
|
||||
// Both execution and setup Test consume this same normal installer. Reuse
|
||||
// the complete controller fixture here instead of inventing a thin pack.
|
||||
const selectedPack = resolveRemoteProviderPackSource(root);
|
||||
for (const mode of ["staged", "preinstalled", "uploaded", "failed-final", "missing-transport"] as const) {
|
||||
let transferred = false;
|
||||
const syncIn = vi.fn(async (operations: { files: { sourcePath?: string; targetPath: string; kind: string }[] }[]) => {
|
||||
expect(operations[0]?.files[0]).toMatchObject({ sourcePath: root,
|
||||
targetPath: "/workspace/.paperclip-runtime/paperclip-runner/provider-pack", kind: "directory" });
|
||||
transferred = true;
|
||||
return { operations: [] };
|
||||
});
|
||||
const execute = vi.fn(async (command: { command: string; args?: string[]; timeoutMs?: number }) => {
|
||||
expect(command.timeoutMs).toBeGreaterThan(0);
|
||||
expect(command.timeoutMs).toBeLessThanOrEqual(30_000);
|
||||
const verification = command.args?.[0] === "-e";
|
||||
const packRoot = command.args?.[2];
|
||||
const valid = mode === "staged" || mode === "preinstalled" && packRoot === "/opt/paperclip-runner/provider-pack"
|
||||
|| transferred && mode !== "failed-final";
|
||||
return { pid: null, startedAt: null, exitCode: verification && !valid ? 1 : 0, timedOut: false, signal: null,
|
||||
stdout: command.args?.[0] === "--version" ? "1.18.34\n"
|
||||
: command.args?.some(value => value.includes("for candidate in")) ? "/opt/paperclip-runner/provider-pack\n" : "",
|
||||
stderr: verification && !valid ? "manifest mismatch" : "" };
|
||||
});
|
||||
const runner = { execute, ...(mode === "missing-transport" ? {} : { syncIn }) };
|
||||
const preparation = prepareRemoteProviderPackOnTarget({
|
||||
target: { kind: "remote", transport: "sandbox", remoteCwd: "/workspace", runner } as never,
|
||||
runner, configuredProviderPackRoot: selectedPack.configuredRoot, manifest: selectedPack.manifest,
|
||||
});
|
||||
if (mode === "failed-final") await expect(preparation).rejects.toThrow("provider pack verification");
|
||||
else if (mode === "missing-transport") await expect(preparation).rejects.toThrow("transport cannot stage a provider pack");
|
||||
else await expect(preparation).resolves.toMatchObject({ source: mode, root: "/workspace/.paperclip-runtime/paperclip-runner/provider-pack",
|
||||
manifest: selectedPack.manifest });
|
||||
expect(syncIn).toHaveBeenCalledTimes(mode === "uploaded" || mode === "failed-final" ? 1 : 0);
|
||||
}
|
||||
const cursorPath = "provider-assets/cursor/linux-x64";
|
||||
await mkdir(join(root, cursorPath), { recursive: true });
|
||||
await writeFile(join(root, cursorPath, "runtime"), "pinned Cursor runtime");
|
||||
@@ -1980,6 +2024,78 @@ describe("remote provider pack manifest", () => {
|
||||
await writeFile(join(root, cursorPath, "runtime"), "substitute Cursor runtime");
|
||||
expect(() => readRemoteProviderPackManifest(root)).toThrow("asset tree digest mismatch");
|
||||
await writeFile(join(root, cursorPath, "runtime"), "pinned Cursor runtime");
|
||||
const kimiPath = "provider-assets/kimi/linux-x64";
|
||||
await mkdir(join(root, kimiPath), { recursive: true });
|
||||
await writeFile(join(root, kimiPath, "runtime"), "pinned Kimi runtime");
|
||||
const kimi = { version: QUALIFIED_ACPX_PROFILES.kimi.agentServerVersion,
|
||||
profileDigest: QUALIFIED_ACPX_PROFILES.kimi.commandDigest,
|
||||
closureDigest: `sha256:${KIMI_DISTRIBUTION_CLOSURE_SHA256["linux-x64"]}`,
|
||||
qualification: "qualified", path: kimiPath, sha256: sha256DirectoryTree(join(root, kimiPath)) };
|
||||
Object.assign(payload, { providers: { cursor, kimi } });
|
||||
await mkdir(releaseMetadata, { recursive: true });
|
||||
const writeKimiManifest = async () => {
|
||||
await writeManifest();
|
||||
await cp(join(root, "provider-pack.json"), manifestPath);
|
||||
};
|
||||
await writeKimiManifest();
|
||||
expect(readRemoteProviderPackManifest(root).payload.providers?.kimi).toEqual(kimi);
|
||||
expect(readBundledRemoteProviderPackManifest(manifestPath).payload.providers?.kimi?.qualification).toBe("qualified");
|
||||
for (const field of ["version", "profileDigest", "closureDigest"] as const) {
|
||||
const previous = kimi[field];
|
||||
kimi[field] = field === "version" ? "older-kimi" : digest(`old-kimi-${field}`);
|
||||
await writeKimiManifest();
|
||||
expect(() => readRemoteProviderPackManifest(root)).toThrow("Kimi profile or closure");
|
||||
expect(() => readBundledRemoteProviderPackManifest(manifestPath)).toThrow("Kimi profile or closure");
|
||||
kimi[field] = previous;
|
||||
}
|
||||
kimi.closureDigest = `sha256:${KIMI_DISTRIBUTION_CLOSURE_SHA256["darwin-arm64"]}`;
|
||||
await writeKimiManifest();
|
||||
expect(() => readBundledRemoteProviderPackManifest(manifestPath)).toThrow("Kimi profile or closure");
|
||||
kimi.closureDigest = `sha256:${KIMI_DISTRIBUTION_CLOSURE_SHA256["linux-x64"]}`;
|
||||
for (const target of [{ platform: "linux", architecture: "arm64" }, { platform: "darwin", architecture: "arm64" }]) {
|
||||
payload.target = target;
|
||||
kimi.path = `provider-assets/kimi/${target.platform}-${target.architecture}`;
|
||||
Object.assign(payload, { providers: {} });
|
||||
await writeKimiManifest();
|
||||
expect(() => readBundledRemoteProviderPackManifest(manifestPath)).toThrow("bundled image pack must target linux-x64");
|
||||
}
|
||||
payload.target = originalTarget;
|
||||
kimi.path = kimiPath;
|
||||
Object.assign(payload, { providers: { cursor, kimi } });
|
||||
kimi.qualification = "pending";
|
||||
await writeKimiManifest();
|
||||
expect(() => readBundledRemoteProviderPackManifest(manifestPath)).toThrow("invalid candidate identity");
|
||||
kimi.qualification = "qualified";
|
||||
Object.assign(payload, { providers: { cursor, kimi }, candidateProviders: { kimi } });
|
||||
await writeKimiManifest();
|
||||
expect(() => readBundledRemoteProviderPackManifest(manifestPath)).toThrow("Kimi profile or closure");
|
||||
Object.assign(payload, { providers: { cursor, kimi } });
|
||||
Reflect.deleteProperty(payload, "candidateProviders");
|
||||
await writeKimiManifest();
|
||||
await writeFile(join(root, kimiPath, "runtime"), "substitute Kimi runtime");
|
||||
expect(() => readRemoteProviderPackManifest(root)).toThrow("candidate asset tree digest mismatch");
|
||||
await writeFile(join(root, kimiPath, "runtime"), "pinned Kimi runtime");
|
||||
const threeCandidates = { cursor,
|
||||
pi: { ...kimi, version: "1.0.0", qualification: "qualified", path: "provider-assets/pi/linux-x64" },
|
||||
copilot: { ...kimi, qualification: "pending", version: QUALIFIED_ACPX_PROFILES.copilot.agentServerVersion,
|
||||
profileDigest: QUALIFIED_ACPX_PROFILES.copilot.commandDigest,
|
||||
closureDigest: "sha256:a3d8f4367cfa1694d79e3f8b7930b6fbfe42a229c272b375b11951d631db8d07",
|
||||
path: "provider-assets/copilot/linux-x64" } };
|
||||
for (const candidate of [threeCandidates.pi, threeCandidates.copilot]) {
|
||||
await mkdir(join(root, candidate.path), { recursive: true });
|
||||
await writeFile(join(root, candidate.path, "runtime"), "candidate runtime");
|
||||
candidate.sha256 = sha256DirectoryTree(join(root, candidate.path));
|
||||
}
|
||||
Object.assign(payload, { candidateProviders: threeCandidates });
|
||||
await writeKimiManifest();
|
||||
expect(Object.keys(readRemoteProviderPackManifest(root).payload.candidateProviders!)).toHaveLength(3);
|
||||
Object.assign(payload, { candidateProviders: { ...threeCandidates, unknown: kimi } });
|
||||
await writeKimiManifest();
|
||||
expect(() => readBundledRemoteProviderPackManifest(manifestPath)).toThrow("invalid candidate inventory");
|
||||
Object.assign(payload, { candidateProviders: { unknown: kimi } });
|
||||
await writeKimiManifest();
|
||||
expect(() => readBundledRemoteProviderPackManifest(manifestPath)).toThrow("invalid candidate identity");
|
||||
await rm(releaseMetadata, { recursive: true, force: true });
|
||||
const candidatePath = "provider-assets/pi/linux-x64";
|
||||
await mkdir(join(root, candidatePath), { recursive: true });
|
||||
await writeFile(join(root, candidatePath, "runtime"), "pinned runtime");
|
||||
@@ -3791,10 +3907,10 @@ describe("runtime question fallback", () => {
|
||||
});
|
||||
|
||||
describe("native provider bootstrap environment", () => {
|
||||
it.each(["pi", "cursor", "copilot", "antigravity"] as const)("does not promote ambient %s credentials when bindings are omitted", agent => {
|
||||
it.each(["pi", "cursor", "copilot", "antigravity", "kimi"] as const)("does not promote ambient %s credentials when bindings are omitted", agent => {
|
||||
const provider = { kind: "acpx", agent } as NativeExecutionInput["provider"];
|
||||
const host = { PATH: "/host/bin", HOME: "/host/home", OPENROUTER_API_KEY: "ambient-pi",
|
||||
CURSOR_API_KEY: "ambient-cursor", CURSOR_AUTH_TOKEN: "ambient-cursor-login", COPILOT_GITHUB_TOKEN: "ambient-copilot",
|
||||
CURSOR_API_KEY: "ambient-cursor", CURSOR_AUTH_TOKEN: "ambient-cursor-login", COPILOT_GITHUB_TOKEN: "ambient-copilot", KIMI_API_KEY: "ambient-kimi",
|
||||
GEMINI_API_KEY: "ambient-google", GOOGLE_API_KEY: "ambient-other-google", GEMINI_HOME: "/ambient-google-home",
|
||||
GOOGLE_APPLICATION_CREDENTIALS: "/ambient-google-credential.json", PAPERCLIP_ACPX_ANTIGRAVITY_AUTH_JSON_SECRET: "ambient-google-subscription",
|
||||
PAPERCLIP_ACPX_CREDENTIAL_BINDING: "ambient-forged-binding" };
|
||||
@@ -7754,6 +7870,27 @@ describe("native session cancellation", () => {
|
||||
expect(state.persistActivity).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ companyId: "another-company" }, { runId: "another-run" }, { issueId: "another-issue" },
|
||||
{ scope: "issue" }, { effects: ["cancel_issue"] }, { intentAuditId: null },
|
||||
])("does not fence a failed retry for a foreign or stale Stop receipt %j", async changed => {
|
||||
const intent = { schema: "paperclip.native-cancellation.v1", intentId: "native-cancellation:earlier-stop",
|
||||
...execution.binding, scope: "run", reasonCode: "cancellation_run_only", effects: ["release_run_resources"],
|
||||
dispatchState: "acknowledged", dispatched: true, intentAuditId: "earlier-intent-audit",
|
||||
acknowledgementAuditId: "earlier-ack-audit", ...changed };
|
||||
const coordinator = { runId: execution.binding.runId, assessmentId: null, phase: "retryable_failure" };
|
||||
const persistence = cancellationDb({ status: "failed", coordinator, resultJson: { nativeCancellation: intent } });
|
||||
const before = structuredClone(persistence.getResultJson());
|
||||
state.cancel.mockClear();
|
||||
state.persistActivity.mockClear();
|
||||
await expect(cancelNativeSession(execution.binding.runId, "retry Stop", { db: persistence.db, scope: "run" })).rejects.toThrow();
|
||||
expect(persistence.updates).toEqual([]);
|
||||
expect(coordinator.phase).toBe("retryable_failure");
|
||||
expect(persistence.getResultJson()).toEqual(before);
|
||||
expect(state.cancel).not.toHaveBeenCalled();
|
||||
expect(state.persistActivity).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not mistake a recovered running attempt for the failed retry that reserved Stop", async () => {
|
||||
const id = "11111111-1111-4111-8111-111111111111";
|
||||
const persistence = cancellationDb({ status: "running",
|
||||
@@ -7779,6 +7916,39 @@ describe("native session cancellation", () => {
|
||||
expect(persistence.getResultJsonUpdateCount()).toBe(writes);
|
||||
});
|
||||
|
||||
it("fences a failed retry whose earlier Stop is already acknowledged without changing its receipts", async () => {
|
||||
const intent = { schema: "paperclip.native-cancellation.v1", intentId: "native-cancellation:earlier-stop",
|
||||
...execution.binding, scope: "run", reasonCode: "cancellation_run_only",
|
||||
effects: ["release_run_resources"], dispatchState: "acknowledged", dispatched: true,
|
||||
intentAuditId: "earlier-intent-audit", acknowledgementAuditId: "earlier-ack-audit",
|
||||
acknowledgedAt: "2026-01-01T00:00:00.000Z", decisionId: null, priorCoordinatorDecisionId: null };
|
||||
const coordinator = { runId: execution.binding.runId, assessmentId: null, phase: "retryable_failure",
|
||||
nextAttemptAt: new Date(Date.now() + 30_000) };
|
||||
const persistence = cancellationDb({ status: "failed", coordinator,
|
||||
resultJson: { nativeCancellation: intent, executionCancellation: { state: "requested", requestedAt: "later-stop" } } });
|
||||
const before = structuredClone(persistence.getResultJson());
|
||||
state.cancel.mockClear();
|
||||
state.persistActivity.mockClear();
|
||||
for (let attempt = 0; attempt < 2; attempt++) {
|
||||
await expect(cancelNativeSession(execution.binding.runId, "retry Stop", { db: persistence.db, scope: "run" }))
|
||||
.resolves.toMatchObject({ dispatched: true, auditId: intent.intentAuditId });
|
||||
expect(coordinator).toMatchObject({ phase: "terminal_failure", failureCode: "native_retry_cancelled", nextAttemptAt: null });
|
||||
expect(persistence.getResultJson()).toEqual(before);
|
||||
}
|
||||
expect(persistence.updates.filter(update => update.table === nativeRunFinalizations)).toHaveLength(1);
|
||||
expect(persistence.getResultJsonUpdateCount()).toBe(0);
|
||||
expect(state.cancel).not.toHaveBeenCalled();
|
||||
expect(state.persistActivity).not.toHaveBeenCalled();
|
||||
const stopped = { id: execution.binding.runId, companyId: execution.binding.companyId,
|
||||
nativeIssueId: execution.binding.issueId, status: "cancelled", processPid: null, processGroupId: null,
|
||||
resultJson: { ...persistence.getResultJson(), cancelledByActorType: "user", cancelledByUserId: "operator" } };
|
||||
const lease = { provider: "daytona", providerLeaseId: "unconfirmed-provider-resource", status: "active",
|
||||
releasedAt: null, cleanupStatus: null, metadata: {} };
|
||||
const leaseDb = { select: () => ({ from: () => ({ where: async () => [lease] }) }) } as unknown as Db;
|
||||
expect(isAcknowledgedNativeStop(stopped)).toBe(true);
|
||||
await expect(acknowledgedNativeStopExecutionHasStopped(leaseDb, stopped)).resolves.toBe(false);
|
||||
});
|
||||
|
||||
it("uses the reserved caller UUID and permits same-intent and default retries", async () => {
|
||||
const id = "11111111-1111-4111-8111-111111111111";
|
||||
const persistence = cancellationDb({ resultJson: { startupCancellation: { cancellationRequestId: id } } });
|
||||
@@ -9202,6 +9372,44 @@ describe("native warm session supervision", () => {
|
||||
expect(state.execute).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("claims native execution authority before retiring a superseded warm identity", async () => {
|
||||
const { base, next, run } = fixture("identity-native-claim-order");
|
||||
const updates: Array<{ table: unknown; values: Record<string, unknown> }> = [];
|
||||
let inspected = false;
|
||||
const close = vi.fn(async () => {
|
||||
if (inspected) return;
|
||||
inspected = true;
|
||||
expect(updates.some(update => update.table === nativeRunFinalizations
|
||||
&& update.values.leaseOwner && update.values.controllerBootId)).toBe(true);
|
||||
});
|
||||
state.execute.mockReset().mockImplementationOnce(async options => {
|
||||
options.onSession?.({ close }); return result;
|
||||
}).mockImplementationOnce(async () => result);
|
||||
await run(base);
|
||||
// Normal runtime selection already converted preparation to native and
|
||||
// inserted an unclaimed observed coordinator before calling this facade.
|
||||
await executePaperclipNativeSession({ db: leaseDb(next, {}, {}, updates),
|
||||
execution: next, runnerInstanceId: "runner", useRunnerd: true });
|
||||
expect(close).toHaveBeenCalledOnce();
|
||||
expect(state.execute).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it.each(["cancelled", "foreign-owner"] as const)("does not retire or stage a warm successor after claim denial: %s", async denial => {
|
||||
const { base, next, run } = fixture(`identity-native-claim-denied-${denial}`);
|
||||
const close = vi.fn(async () => undefined);
|
||||
state.execute.mockReset().mockImplementationOnce(async options => {
|
||||
options.onSession?.({ close }); return result;
|
||||
});
|
||||
await run(base);
|
||||
state.stageNativeRunnerWakeAttachments.mockClear();
|
||||
const db = leaseDb(next, denial === "foreign-owner" ? { leaseOwner: "different-controller" } : {},
|
||||
{}, [], {}, denial === "cancelled" ? "cancelled" : "running");
|
||||
await expect(executePaperclipNativeSession({ db, execution: next, runnerInstanceId: "runner", useRunnerd: true })).rejects.toThrow();
|
||||
expect(close).not.toHaveBeenCalled();
|
||||
expect(state.stageNativeRunnerWakeAttachments).not.toHaveBeenCalled();
|
||||
expect(state.execute).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not retire an active turn when a new session identity arrives", async () => {
|
||||
const { base, next, run } = fixture("identity-active");
|
||||
let finish!: () => void;
|
||||
@@ -12183,6 +12391,55 @@ describe("runnerd provider runtime wiring", () => {
|
||||
).toThrow("scope_closed");
|
||||
});
|
||||
|
||||
it.each(["attached", "changed-admission", "changed-template", "changed-template-digest", "changed-preparation"] as const)(
|
||||
"verifies frozen staged admission after normal wake attachment preparation: %s", async mode => {
|
||||
const current = scopedNativeExecution(`staged-attachment-${mode}`);
|
||||
current.task.prompt = "Inspect attachment ref:original";
|
||||
const root = await initializeScopedRunnerRoot(current, "attachment-runner");
|
||||
const sourceIdentity = { runnerInstanceId: "attachment-runner", environmentLeaseId: "attachment-lease",
|
||||
runId: "attachment-source", normalizedSessionId: current.session.normalizedSessionId!, turnId: "source-turn", itemId: "source-item" };
|
||||
const targetIdentity = { ...sourceIdentity, runId: current.binding.runId, turnId: "target-turn", itemId: "target-item" };
|
||||
const controller = new DurablePrpControlPlane({ stateDirectory: join(root, "control-plane"), identity: sourceIdentity,
|
||||
expectedRunnerVersion: "fixture", expectedRunnerDigest: `sha256:${"a".repeat(64)}` });
|
||||
controller.stageRunAttachment(targetIdentity, { provider: { kind: "codex", instructions: "prepared attachment grant" } });
|
||||
const path = join(root, "control-plane/control-plane-state.json");
|
||||
const saved = JSON.parse(await readFile(path, "utf8"));
|
||||
const canonicalTemplateDigest = saved.stagedRunAttachment.templateDigest;
|
||||
if (mode === "changed-template" || mode === "changed-template-digest") {
|
||||
saved.stagedRunAttachment.runAttachTemplate.provider.instructions = "substituted grant";
|
||||
if (mode === "changed-template-digest") saved.stagedRunAttachment.templateDigest = `sha256:${nativeSha256(saved.stagedRunAttachment.runAttachTemplate)}`;
|
||||
await writeFile(path, JSON.stringify(saved));
|
||||
}
|
||||
state.stageNativeRunnerWakeAttachments.mockImplementationOnce(async () => {
|
||||
if (mode === "changed-preparation") current.task.prompt = "Inspect attachment ref:mutated-during-preparation";
|
||||
return { attachments: [{ id: "attachment-original" }], cleanup: async () => undefined } as never;
|
||||
});
|
||||
state.renderNativeRunnerStagedAttachmentPrompt.mockReturnValueOnce("Materialized attachment: .paperclip-inbound/current/original");
|
||||
const admission = structuredClone(current);
|
||||
if (mode === "changed-admission") admission.task.prompt = "Inspect attachment ref:changed";
|
||||
const reached = new Error("verified staged admission reached recheck");
|
||||
const restore = vi.spyOn(runnerCheckpoint, "restoreNativeRunnerCheckpoint").mockResolvedValue(null);
|
||||
const header = vi.spyOn(runnerCheckpoint, "loadNativeRunnerCheckpointHeader").mockResolvedValue("staged header");
|
||||
const verify = vi.spyOn(stagedAttachmentRecovery, "resolveNativeStagedAttachmentRecovery")
|
||||
.mockResolvedValueOnce({ currentExecution: admission, sourceIdentity, targetIdentity,
|
||||
templateDigest: canonicalTemplateDigest } as never)
|
||||
.mockRejectedValueOnce(reached);
|
||||
try {
|
||||
const resume = executePaperclipNativeSession({ db: leaseDb(current), execution: current, useRunnerd: true,
|
||||
runnerInstanceId: "attachment-runner", restartRecovery: { kind: "reattach_remote_runner", runId: current.binding.runId,
|
||||
leaseOwner: "controller", controllerGeneration: 2, providerAttempt: 1, restartKind: "hard", recoveryRequestId: null,
|
||||
remote: { kind: "sandbox", leaseId: "attachment-lease", providerLeaseId: "sandbox", remoteCwd: "/workspace" } } as never,
|
||||
runnerExecutionTarget: { kind: "remote", transport: "sandbox", leaseId: "attachment-lease", remoteCwd: "/workspace",
|
||||
runner: { execute: vi.fn() }, sandboxLeaseAcquisition: { outcome: "resumed", providerLeaseId: "sandbox" } } as never });
|
||||
if (mode === "attached") { await expect(resume).rejects.toBe(reached); expect(verify).toHaveBeenCalledTimes(2); }
|
||||
else await expect(resume).rejects.toThrow(mode === "changed-template" ? "staged_attachment_binding_mismatch"
|
||||
: mode === "changed-preparation" ? "native_attachment_admission_changed" : "staged_attachment_recovery_unproven");
|
||||
expect(state.stageNativeRunnerWakeAttachments).toHaveBeenCalledWith(expect.objectContaining({ binding: expect.objectContaining({
|
||||
companyId: current.binding.companyId, issueId: current.binding.issueId, agentId: current.binding.agentId, runId: current.binding.runId,
|
||||
}) }));
|
||||
} finally { restore.mockRestore(); header.mockRestore(); verify.mockRestore(); await controller.stop(); }
|
||||
});
|
||||
|
||||
it("passes the run checkpoint active turn into restart recovery", async () => {
|
||||
state.createBackend.mockClear();
|
||||
state.createTransport.mockClear();
|
||||
@@ -12213,6 +12470,27 @@ describe("runnerd provider runtime wiring", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it.each(["warm", "per_turn"] as const)("defers a %s retained-owner wait before claiming a provider attempt", async mode => {
|
||||
const fresh = scopedNativeExecution(`retained-owner-${mode}`);
|
||||
fresh.session.lifecyclePolicy = mode === "warm"
|
||||
? { mode: "warm", idleTimeoutMs: 300_000 } : { mode: "per_turn", idleTimeoutMs: null };
|
||||
const deadline = new Date(Date.now() + 60_000);
|
||||
vi.mocked(controllerRecovery.resolveNativeControllerRecovery).mockResolvedValueOnce({ kind: "waiting_for_owner", deadline });
|
||||
const defer = vi.spyOn(controllerRecovery, "deferNativeControllerRecovery").mockResolvedValueOnce();
|
||||
const onExecutionLeaseAcquired = vi.fn();
|
||||
const updates: Array<{ table: unknown; values: Record<string, unknown> }> = [];
|
||||
const db = leaseDb(fresh, {}, {}, updates);
|
||||
await expect(executePaperclipNativeSession({ db, execution: fresh, runnerInstanceId: "waiting-controller",
|
||||
useRunnerd: true, onExecutionLeaseAcquired,
|
||||
runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: "/workspace",
|
||||
environmentId: "retained-environment", leaseId: "retained-lease" },
|
||||
})).rejects.toMatchObject({ name: "NativeControllerRecoveryWaitingError", retryAt: deadline });
|
||||
expect(defer).toHaveBeenCalledWith(db, expect.objectContaining({ ...fresh.binding, retryAt: deadline }));
|
||||
expect(onExecutionLeaseAcquired).not.toHaveBeenCalled();
|
||||
expect(updates.some(update => update.table === nativeRunFinalizations && update.values.attempt !== undefined)).toBe(false);
|
||||
expect(state.createBackend).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects overlapping runs for the same runnerd provider session scope", async () => {
|
||||
const first = {
|
||||
...execution,
|
||||
@@ -12576,7 +12854,7 @@ describe("runnerd provider runtime wiring", () => {
|
||||
} as NativeExecutionInputV1;
|
||||
const identity = { runId: current.binding.runId, normalizedSessionId: current.session.normalizedSessionId!,
|
||||
runnerInstanceId: `admitted-${provider}`, environmentLeaseId: "admitted-lease",
|
||||
turnId: `turn-${current.binding.runId}`, itemId: `item-${current.binding.runId}` };
|
||||
turnId: `turn_attachment_${randomUUID()}`, itemId: `item_attachment_${randomUUID()}` };
|
||||
const root = await initializeScopedRunnerRoot(current, identity.runnerInstanceId);
|
||||
await mkdir(join(root, "control-plane"), { recursive: true });
|
||||
await writeFile(join(root, "control-plane", "control-plane-state.json"), JSON.stringify(durableControlPlaneState(identity)));
|
||||
@@ -12606,9 +12884,56 @@ describe("runnerd provider runtime wiring", () => {
|
||||
runnerArtifact: artifact, adoptExistingRunner: { pid: 123, startedAt: "2026-09-19T10:00:00.000Z" } });
|
||||
expect(state.resolveRemoteRunnerBinary).not.toHaveBeenCalled();
|
||||
expect(execute.mock.calls.some(([request]) => request.args.includes("paperclip-runner-launch") || request.args.includes("--build-metadata"))).toBe(false);
|
||||
const options = state.createTransport.mock.calls.at(-1)![0] as RunnerTransportOptions & { prepareExternalRunnerState: () => Promise<void> };
|
||||
await expect(options.prepareExternalRunnerState()).resolves.toBeUndefined();
|
||||
expect(execute.mock.calls.some(([request]) => request.args.includes("paperclip-runner-launch")
|
||||
|| request.args.includes("paperclip-runner-claim-unstarted-session"))).toBe(false);
|
||||
});
|
||||
|
||||
it.each(["claude", "codex"].flatMap(provider => (["per_turn", "warm"] as const).flatMap(policy => ["canonical", "local-receipt", "local-receipt-missing-terminal", "local-receipt-pending-command", "quarantined", "empty-runner", "empty-runner-quarantined", "nonempty-runner", "wrong-identity", "active", "foreign-scope", "no-stop-proof", "not-suspended", "pending-outbox", "pending-command", "missing-terminal", "ambiguous", "contradictory"].map(mode => [provider, policy, mode] as const))))(
|
||||
it.each(["verified", "no-proof", "dead", "stale"] as const)("reuses only verified idle warm process during actual remote preparation: %s", async mode => {
|
||||
const previous = scopedNativeExecution(`warm-live-preparation-${mode}`);
|
||||
previous.session.lifecyclePolicy = { mode: "warm", idleTimeoutMs: 300_000 };
|
||||
const current = { ...previous, binding: { ...previous.binding, runId: `${previous.binding.runId}-next` } };
|
||||
const identity = { runId: previous.binding.runId, normalizedSessionId: previous.session.normalizedSessionId!,
|
||||
runnerInstanceId: "retained-warm-runner", environmentLeaseId: "retained-lease", turnId: "prior-turn", itemId: "prior-item" };
|
||||
const root = await initializeScopedRunnerRoot(previous, identity.runnerInstanceId);
|
||||
await writeRunnerAuthority(root, identity, "suspended");
|
||||
const projections = vi.spyOn(controllerRecovery, "resolveNativeControllerRecovery").mockResolvedValue({ kind: "no_prior_execution" });
|
||||
onTestFinished(() => projections.mockRestore());
|
||||
const db = leaseDb(current), ordinarySelect = db.select.bind(db);
|
||||
db.select = ((fields?: Record<string, unknown>) => fields?.status === heartbeatRuns.status && fields?.runnerProfileJson === heartbeatRuns.runnerProfileJson
|
||||
? priorNativeRunDb(previous).select(fields as never) : ordinarySelect(fields as never)) as Db["select"];
|
||||
let alive = true;
|
||||
const execute = vi.fn(async (request: { args: string[] }) => ({ exitCode: request.args[2] === "paperclip-runner-recovery-check" && !alive ? 45 : 0,
|
||||
timedOut: false, stderr: "", stdout: request.args[0] === "--build-metadata" ? JSON.stringify({
|
||||
schema: "paperclip-runner/runnerd-build-metadata/v1", binaryName: "paperclip-runnerd",
|
||||
packageName: "@paperclipai/paperclip-runner", binaryContractVersion: 2,
|
||||
durableSessionCapabilities: ["unlimited_runtime", "connection_lease_renewal"], prpTransportModes: ["listen_ws"],
|
||||
}) : request.args[0] === "--version" ? "codex-cli 0.160.0" : request.args[1]?.includes("base64")
|
||||
? Buffer.from(JSON.stringify({ ...identity, lifecycle: "ready", outbox: [], pendingTerminalDelivery: null })).toString("base64")
|
||||
: request.args[1]?.includes("cat --") ? `nonce\n123\n2026-09-19T10:00:00.000Z\n${identity.runnerInstanceId}\nlinux:abcd-1234:100\n` : "" }));
|
||||
const revalidate = vi.fn(async () => undefined);
|
||||
const syncIn = vi.fn(async () => undefined);
|
||||
await createRunnerdBackend({ db, execution: current, runnerInstanceId: identity.runnerInstanceId,
|
||||
runnerIngressAuthorized: true,
|
||||
...(mode === "no-proof" ? {} : { remoteWarmRecovery: { identity, reuseProcess: true, previousExecution: previous,
|
||||
checkpoint: {} as never, remote: { providerLeaseId: "retained-sandbox", remoteCwd: "/workspace" }, revalidate } }),
|
||||
runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: "/workspace", environmentId: "environment",
|
||||
leaseId: identity.environmentLeaseId, providerKey: "daytona", effectiveCapabilities: { runnerWebSocketIngress: true },
|
||||
sandboxLeaseAcquisition: { outcome: "resumed", providerLeaseId: "retained-sandbox" }, runner: { execute, syncIn } } as never });
|
||||
state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!();
|
||||
const options = state.createTransport.mock.calls.at(-1)![0] as RunnerTransportOptions & { prepareExternalRunnerState: () => Promise<void> };
|
||||
if (mode === "dead") alive = false;
|
||||
if (mode === "stale") revalidate.mockRejectedValueOnce(new Error("fixture_warm_owner_changed"));
|
||||
if (mode === "verified") await expect(options.prepareExternalRunnerState()).resolves.toBeUndefined();
|
||||
else await expect(options.prepareExternalRunnerState()).rejects.toThrow(mode === "no-proof" ? "resumed_not_suspended"
|
||||
: mode === "dead" ? "runner_remote_process_identity_unavailable" : "fixture_warm_owner_changed");
|
||||
expect(syncIn).not.toHaveBeenCalled();
|
||||
expect(execute.mock.calls.some(([request]) => request.args.includes("paperclip-runner-launch")
|
||||
|| request.args.includes("paperclip-runner-claim-unstarted-session"))).toBe(false);
|
||||
});
|
||||
|
||||
it.each(["claude", "codex"].flatMap(provider => (["per_turn", "warm"] as const).flatMap(policy => ["canonical", "resumed", "resumed-timeout", "local-receipt", "local-receipt-missing-terminal", "local-receipt-pending-command", "quarantined", "empty-runner", "empty-runner-quarantined", "nonempty-runner", "wrong-identity", "active", "foreign-scope", "no-stop-proof", "not-suspended", "pending-outbox", "pending-command", "missing-terminal", "ambiguous", "contradictory"].filter(mode => policy === "per_turn" || !mode.startsWith("resumed")).map(mode => [provider, policy, mode] as const))))(
|
||||
"recovers a blocked %s %s plugin checkpoint on a replacement lease: %s", async (provider, policy, mode) => {
|
||||
const { permissionPolicy: _permissionPolicy, ...claudeProfile } = QUALIFIED_ACPX_PROFILES.claude;
|
||||
const first = { ...scopedNativeExecution(`plugin-${provider}-${policy}-${mode}`),
|
||||
@@ -12656,8 +12981,13 @@ describe("runnerd provider runtime wiring", () => {
|
||||
if (mode === "no-stop-proof") throw new Error("Plugin Runner retained session is not authorized");
|
||||
return "old-task";
|
||||
});
|
||||
let timedOut = false;
|
||||
const task = vi.fn(async (operation: any) => {
|
||||
environmentTaskOperationSchema.parse(operation);
|
||||
if (mode === "resumed-timeout" && operation.kind === "read_state" && !timedOut) {
|
||||
timedOut = true;
|
||||
throw new Error("fixture_remote_state_timeout");
|
||||
}
|
||||
if (operation.kind === "read_state") return { kind: "state", taskId: "replacement-task", state: {
|
||||
...durableRunnerState(identity, mode === "not-suspended" ? "ready" : "suspended"),
|
||||
outbox: mode === "pending-outbox" ? [{}] : [], pendingTerminalDelivery: null,
|
||||
@@ -12676,7 +13006,7 @@ describe("runnerd provider runtime wiring", () => {
|
||||
managedAiCredentialHome: home, managedAiCredentialIdentity: "revision", managedAiAccountIdentity: "account",
|
||||
runnerEnvironment: { ANTHROPIC_API_KEY: "synthetic-fixture" },
|
||||
runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: ".", leaseId: "replacement-lease", runnerTask: task,
|
||||
sandboxLeaseAcquisition: { outcome: "replacement", providerLeaseId: "replacement-task" } } as never };
|
||||
sandboxLeaseAcquisition: { outcome: mode.startsWith("resumed") ? "resumed" : "replacement", providerLeaseId: "replacement-task" } } as never };
|
||||
if (throughExecutor) {
|
||||
const validated = validatePrpStructuredRunResult({ schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", summary: "done",
|
||||
completionClaim: { contractRevision: next.completionContract.contract.revision, objectiveSatisfied: true, criteria: [], remainingWork: [] },
|
||||
@@ -12686,12 +13016,19 @@ describe("runnerd provider runtime wiring", () => {
|
||||
normalizedSessionId: next.session.normalizedSessionId, providerSessionId: "original-provider-history", driverKind: next.session.driverKind,
|
||||
driverVersion: "1", nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null });
|
||||
}
|
||||
if (mode === "resumed-timeout") {
|
||||
await expect(createRunnerdBackend(launch)).rejects.toThrow("fixture_remote_state_timeout");
|
||||
expect(existsSync(join(scoped, "plugin-runner-state.json"))).toBe(false);
|
||||
expect(await readFile(join(scoped, "control-plane/control-plane-state.json"), "utf8")).toBe(journal);
|
||||
}
|
||||
const backend = throughExecutor ? executePaperclipNativeSession({ ...launch, useRunnerd: true }) : createRunnerdBackend(launch);
|
||||
if (["canonical", "local-receipt", "quarantined", "empty-runner", "empty-runner-quarantined"].includes(mode)) {
|
||||
if (["canonical", "resumed", "resumed-timeout", "local-receipt", "quarantined", "empty-runner", "empty-runner-quarantined"].includes(mode)) {
|
||||
await expect(backend).resolves.toBeDefined();
|
||||
expect(state.createBackend).toHaveBeenCalledOnce();
|
||||
expect(owner).toHaveBeenCalledWith(expect.objectContaining({ resumeFrom: "old-task", priorIdentity: identity }));
|
||||
expect(resume).toHaveBeenCalledWith(db, expect.objectContaining({ leaseId: "replacement-lease", priorIdentity: identity, inspectCheckpoint: true }));
|
||||
expect(owner).toHaveBeenCalledWith(expect.objectContaining({ priorIdentity: identity,
|
||||
...(!mode.startsWith("resumed") ? { resumeFrom: "old-task" } : {}) }));
|
||||
if (!mode.startsWith("resumed")) expect(resume).toHaveBeenCalledWith(db, expect.objectContaining({ leaseId: "replacement-lease", priorIdentity: identity, inspectCheckpoint: true }));
|
||||
else expect(task).toHaveBeenCalledWith({ kind: "read_state", identity });
|
||||
expect(JSON.parse(await readFile(join(scoped, "plugin-runner-state.json"), "utf8"))).toMatchObject({ ...identity, lifecycle: "suspended" });
|
||||
expect(await readFile(join(scoped, "control-plane/control-plane-state.json"), "utf8")).toBe(journal);
|
||||
} else {
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -5,6 +5,7 @@ import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
|
||||
import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js";
|
||||
import { findNativeSessionResumeRun } from "./native-session-resume.js";
|
||||
import { resolveNativeControllerRecovery } from "./native-controller-recovery.js";
|
||||
|
||||
const externalUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_TEST_DATABASE_URL");
|
||||
const supported = Boolean(externalUrl) || (await getEmbeddedPostgresTestSupport()).supported;
|
||||
@@ -58,7 +59,7 @@ const supported = Boolean(externalUrl) || (await getEmbeddedPostgresTestSupport(
|
||||
const saved = source();
|
||||
await db.insert(heartbeatRuns).values([saved, run(1), run(2)]);
|
||||
const result = await select();
|
||||
expect(result).toEqual({ ...saved, runnerProfileJson: { nativeExecutionInput, sessionCheckpoint: checkpoint, nativeToolContractFingerprint: "fingerprint" },
|
||||
expect(result).toEqual({ ...saved, runnerProfileJson: { nativeExecutionInput, sessionCheckpoint: checkpoint, nativeToolContractFingerprint: "fingerprint", nativeControllerRecovery: null },
|
||||
contextSnapshot: { paperclipWake, paperclipIssue } });
|
||||
const stored = await db.select({ profile: heartbeatRuns.runnerProfileJson, context: heartbeatRuns.contextSnapshot }).from(heartbeatRuns).where(eq(heartbeatRuns.id, saved.id));
|
||||
expect(stored[0].profile).toEqual(profile);
|
||||
@@ -80,6 +81,16 @@ const supported = Boolean(externalUrl) || (await getEmbeddedPostgresTestSupport(
|
||||
}
|
||||
});
|
||||
|
||||
it("does not call missing or newer incompatible recovery material a fresh session", async () => {
|
||||
const saved = source();
|
||||
await db.insert(heartbeatRuns).values([saved, run(30, { id: currentRunId, status: "running" })]);
|
||||
const { beforeCreatedAt: _before, ...binding } = input;
|
||||
expect(await resolveNativeControllerRecovery(db, binding)).toEqual({ kind: "missing_evidence", reason: "snapshot_binding" });
|
||||
// The newest progress barrier remains authoritative even without a snapshot.
|
||||
await db.insert(heartbeatRuns).values(run(10, { runtimeMode: "legacy" }));
|
||||
expect(await resolveNativeControllerRecovery(db, binding)).toEqual({ kind: "missing_evidence", reason: "newer_progress_barrier" });
|
||||
});
|
||||
|
||||
it("skips only unused SQL-null, JSON-null, scalar, array and checkpoint-null profiles", async () => {
|
||||
const saved = source(), newer = run(1, { runnerProfileJson: null });
|
||||
await db.insert(heartbeatRuns).values([saved, newer]);
|
||||
|
||||
@@ -223,6 +223,8 @@ export async function findNativeSessionResumeRun(
|
||||
normalizedSessionId: string;
|
||||
currentRunId: string;
|
||||
beforeCreatedAt: Date;
|
||||
/** Recovery discovery must retain an ineligible newest progress barrier. */
|
||||
includeProgressBarrier?: boolean;
|
||||
},
|
||||
) {
|
||||
if (!isNativeSessionId(input.normalizedSessionId)) return null;
|
||||
@@ -287,10 +289,12 @@ export async function findNativeSessionResumeRun(
|
||||
nativeToolContractFingerprint: unknown;
|
||||
nativeExecutionInput: unknown;
|
||||
sessionCheckpoint: unknown;
|
||||
nativeControllerRecovery: unknown;
|
||||
}>`jsonb_build_object(
|
||||
'nativeToolContractFingerprint', ${heartbeatRuns.runnerProfileJson}->'nativeToolContractFingerprint',
|
||||
'nativeExecutionInput', ${heartbeatRuns.runnerProfileJson}->'nativeExecutionInput',
|
||||
'sessionCheckpoint', ${heartbeatRuns.runnerProfileJson}->'sessionCheckpoint'
|
||||
'sessionCheckpoint', ${heartbeatRuns.runnerProfileJson}->'sessionCheckpoint',
|
||||
'nativeControllerRecovery', ${heartbeatRuns.runnerProfileJson}->'nativeControllerRecovery'
|
||||
)`,
|
||||
contextSnapshot: sql<{
|
||||
paperclipWake: unknown;
|
||||
@@ -306,14 +310,16 @@ export async function findNativeSessionResumeRun(
|
||||
.from(heartbeatRuns)
|
||||
.innerJoin(barrier, eq(heartbeatRuns.id, barrier.id))
|
||||
.where(and(
|
||||
eq(barrier.runtimeMode, "native"),
|
||||
inArray(barrier.status, terminalStatuses),
|
||||
eq(barrier.hasCheckpoint, true),
|
||||
...(input.includeProgressBarrier ? [] : [
|
||||
eq(barrier.runtimeMode, "native"),
|
||||
inArray(barrier.status, terminalStatuses),
|
||||
eq(barrier.hasCheckpoint, true),
|
||||
sql`date_trunc('milliseconds', ${barrier.createdAt}) <= ${input.beforeCreatedAt.toISOString()}::timestamptz`,
|
||||
]),
|
||||
sql`${heartbeatRuns.companyId}::text = ${input.companyId}`,
|
||||
sql`${heartbeatRuns.agentId}::text = ${input.agentId}`,
|
||||
sql`${heartbeatRuns.nativeIssueId}::text = ${input.issueId}`,
|
||||
sql`${heartbeatRuns.nativeSessionId}::text = ${input.normalizedSessionId}`,
|
||||
sql`date_trunc('milliseconds', ${barrier.createdAt}) <= ${input.beforeCreatedAt.toISOString()}::timestamptz`,
|
||||
));
|
||||
return run ?? null;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,187 @@
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { DurablePrpControlPlane, type DurablePrpCheckpoint } from "../../vendor/paperclip-runner/index.js";
|
||||
import { createLocalEncryptedCipher } from "../../secrets/local-encrypted-provider.js";
|
||||
import { buildNativeExecutionInput } from "./native-execution-input.js";
|
||||
import { nativeToolContractFingerprintForTarget } from "./native-session-resume.js";
|
||||
import { nativeRuntimeContextFixture } from "./runtime-context.test-fixture.js";
|
||||
import { resolveNativeStagedAttachmentRecovery } from "./native-staged-attachment-recovery.js";
|
||||
|
||||
const companyId = "10000000-0000-4000-8000-000000000001";
|
||||
const issueId = "20000000-0000-4000-8000-000000000002";
|
||||
const agentId = "30000000-0000-4000-8000-000000000003";
|
||||
const sourceRunId = "40000000-0000-4000-8000-000000000004";
|
||||
const currentRunId = "50000000-0000-4000-8000-000000000005";
|
||||
const normalizedSessionId = "60000000-0000-4000-8000-000000000006";
|
||||
const runnerInstanceId = "70000000-0000-4000-8000-000000000007";
|
||||
const sourceIdentity = { runId: sourceRunId, normalizedSessionId, runnerInstanceId,
|
||||
environmentLeaseId: "environment-1", turnId: "opaque-source-turn", itemId: "opaque-source-item" };
|
||||
const targetIdentity = { ...sourceIdentity, runId: currentRunId, turnId: "opaque-target-turn", itemId: "opaque-target-item" };
|
||||
const roots: string[] = [];
|
||||
afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); });
|
||||
|
||||
function execution(runId: string) {
|
||||
return buildNativeExecutionInput({ companyId, runId, agentId, normalizedSessionId,
|
||||
issue: { id: issueId, identifier: "TEST-1", title: "Continue", description: null, workMode: "standard" },
|
||||
taskPrompt: "Continue normally", provider: "codex", workspace: { id: "workspace-1", cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null },
|
||||
completionContract: { id: "80000000-0000-4000-8000-000000000008", sha256: `sha256:${"a".repeat(64)}`,
|
||||
schemaVersion: "paperclip.run-result.v1", contract: { revision: "1", objective: "Continue", criteria: [{ id: "done", requirement: "Answer" }] } },
|
||||
runtimeContext: nativeRuntimeContextFixture(),
|
||||
});
|
||||
}
|
||||
|
||||
async function fixture() {
|
||||
const root = await mkdtemp(join(tmpdir(), "native-staged-proof-")); roots.push(root);
|
||||
const core = new DurablePrpControlPlane({ stateDirectory: root, identity: sourceIdentity,
|
||||
expectedRunnerVersion: "0.0.0", expectedRunnerDigest: `sha256:${"f".repeat(64)}` });
|
||||
let saved!: DurablePrpCheckpoint;
|
||||
core.setCheckpointPersistence(async checkpoint => { saved = checkpoint; });
|
||||
core.persistRunAttachTemplate({ provider: { provider: "codex", command: "codex" }, authorizedTools: {} });
|
||||
await core.flushCheckpoint();
|
||||
const sourceHeader = saved.records.get("header")!;
|
||||
core.stageRunAttachment(targetIdentity, { provider: { provider: "codex", command: "codex" }, authorizedTools: {} });
|
||||
await core.flushCheckpoint();
|
||||
const header = saved.records.get("header")!;
|
||||
const base = { companyId, agentId, issueId, nativeIssueId: issueId, runnerInstanceId,
|
||||
nativeSessionId: normalizedSessionId, runtimeMode: "native", status: "running", finishedAt: null as Date | null };
|
||||
const currentRun = { ...base, id: currentRunId, runnerProfileJson: { nativeExecutionInput: execution(currentRunId) } };
|
||||
const sourceRun = { ...base, id: sourceRunId, status: "succeeded", finishedAt: new Date(), runnerProfileJson: {
|
||||
nativeExecutionInput: execution(sourceRunId), nativeToolContractFingerprint: nativeToolContractFingerprintForTarget("remote", "codex"),
|
||||
sessionCheckpoint: { sessionId: "provider-thread-1", providerSessionId: "provider-thread-1", identity: {
|
||||
companyId, issueId, agentId, runId: sourceRunId, sessionId: normalizedSessionId },
|
||||
},
|
||||
} };
|
||||
const source = { run: sourceRun, coordinator: { runId: sourceRunId, companyId, issueId,
|
||||
phase: "committed", resultId: "result-1", leaseOwner: null, leaseExpiresAt: null }, resultId: "result-1", resultStatus: "accepted" };
|
||||
function database(sourceRows = [source], contents = sourceHeader, envelope = {}) {
|
||||
const material = createLocalEncryptedCipher("write").encrypt(JSON.stringify({ companyId, issueId, runId: sourceRunId, part: "header", contents, ...envelope }));
|
||||
const batches: unknown[][] = [sourceRows, [{ material }]];
|
||||
const select = vi.fn(() => {
|
||||
const chain = { from: () => chain, innerJoin: () => chain, where: () => chain,
|
||||
limit: vi.fn(async () => batches.shift() ?? []) };
|
||||
return chain;
|
||||
});
|
||||
return { db: { select } as unknown as Pick<Db, "select">, select };
|
||||
}
|
||||
return { header, sourceHeader, source, sourceRun, currentRun, database };
|
||||
}
|
||||
|
||||
describe("private staged attachment restart evidence", () => {
|
||||
it("proves both frozen run bindings before the target has a session snapshot", async () => {
|
||||
const f = await fixture(); const { db, select } = f.database();
|
||||
const matchesExecution = vi.fn(() => true);
|
||||
const result = await resolveNativeStagedAttachmentRecovery({ db, header: f.header, currentRun: f.currentRun,
|
||||
expectedTarget: targetIdentity, matchesExecution });
|
||||
expect(result).toMatchObject({ sourceIdentity, targetIdentity, sourceRun: { id: sourceRunId }, hasTargetCheckpoint: false, targetCheckpoint: null,
|
||||
protocolIdentity: sourceIdentity, phase: "source_bootstrap", receipt: null });
|
||||
expect(matchesExecution).toHaveBeenCalledOnce();
|
||||
expect(select).toHaveBeenCalledTimes(2); // one source point lookup and one encrypted header point lookup
|
||||
expect(f.currentRun.runnerProfileJson).not.toHaveProperty("sessionCheckpoint");
|
||||
});
|
||||
|
||||
it("retains staged evidence when the target runtime has already published its snapshot", async () => {
|
||||
const f = await fixture();
|
||||
const snapshot = structuredClone(f.sourceRun.runnerProfileJson.sessionCheckpoint);
|
||||
snapshot.identity.runId = currentRunId;
|
||||
Object.assign(snapshot, { cursor: "target-cursor", activeTurnId: "target-current-turn", pendingRuntimeRequests: [{ requestId: "target-request" }] });
|
||||
Object.assign(f.currentRun.runnerProfileJson, { sessionCheckpoint: snapshot });
|
||||
const result = await resolveNativeStagedAttachmentRecovery({ ...f.database(), header: f.header, currentRun: f.currentRun });
|
||||
expect(result).toMatchObject({ sourceIdentity, targetIdentity, checkpoint: { identity: { runId: currentRunId } },
|
||||
hasTargetCheckpoint: true, targetCheckpoint: snapshot });
|
||||
expect(f.currentRun.runnerProfileJson).toHaveProperty("sessionCheckpoint", snapshot);
|
||||
});
|
||||
|
||||
it("returns the codec's queued phase without assuming the Runner has activated the target", async () => {
|
||||
const f = await fixture(); const header = JSON.parse(f.header);
|
||||
// Header inspection deliberately does not certify separate command records.
|
||||
// Full restoration/authentication owns that later gate; no RPC runs here.
|
||||
header.state.stagedRunAttachment.commandId = "staged-attach-command";
|
||||
const result = await resolveNativeStagedAttachmentRecovery({ ...f.database(), header: JSON.stringify(header), currentRun: f.currentRun });
|
||||
expect(result).toMatchObject({ commandId: "staged-attach-command", protocolIdentity: sourceIdentity,
|
||||
phase: "attachment_queued", receipt: null, targetIdentity });
|
||||
});
|
||||
|
||||
it.each(["runId", "companyId", "issueId", "agentId", "sessionId", "providerSessionId"])(
|
||||
"rejects a present target checkpoint with changed %s instead of replaying the source", async key => {
|
||||
const f = await fixture(); const snapshot = structuredClone(f.sourceRun.runnerProfileJson.sessionCheckpoint);
|
||||
snapshot.identity.runId = currentRunId;
|
||||
if (key === "providerSessionId") snapshot.providerSessionId = "foreign";
|
||||
else Object.assign(snapshot.identity, { [key]: "foreign" });
|
||||
Object.assign(f.currentRun.runnerProfileJson, { sessionCheckpoint: snapshot });
|
||||
expect(await resolveNativeStagedAttachmentRecovery({ ...f.database(), header: f.header, currentRun: f.currentRun })).toBeNull();
|
||||
});
|
||||
|
||||
it.each(["runId", "runnerInstanceId", "environmentLeaseId", "normalizedSessionId", "turnId", "itemId"] as const)(
|
||||
"rejects a different expected target %s before querying the source", async key => {
|
||||
const f = await fixture(); const { db, select } = f.database();
|
||||
expect(await resolveNativeStagedAttachmentRecovery({ db, header: f.header, currentRun: f.currentRun,
|
||||
expectedTarget: { ...targetIdentity, [key]: "foreign" } })).toBeNull();
|
||||
expect(select).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(["agentId", "companyId", "nativeIssueId", "nativeSessionId", "runnerInstanceId"] as const)(
|
||||
"rejects mixed source DB %s", async key => {
|
||||
const f = await fixture(); Object.assign(f.source.run, { [key]: "foreign" });
|
||||
expect(await resolveNativeStagedAttachmentRecovery({ ...f.database(), header: f.header, currentRun: f.currentRun })).toBeNull();
|
||||
});
|
||||
|
||||
it.each(["binding", "schema", "runner", "session"])("rejects current frozen evidence mismatch: %s", async kind => {
|
||||
const f = await fixture(); const { db, select } = f.database();
|
||||
if (kind === "binding") f.currentRun.runnerProfileJson.nativeExecutionInput.binding.agentId = sourceRunId;
|
||||
if (kind === "schema") Object.assign(f.currentRun.runnerProfileJson.nativeExecutionInput, { schema: "unsupported" });
|
||||
if (kind === "runner") f.currentRun.runnerInstanceId = sourceRunId;
|
||||
if (kind === "session") f.currentRun.nativeSessionId = sourceRunId;
|
||||
expect(await resolveNativeStagedAttachmentRecovery({ db, header: f.header, currentRun: f.currentRun })).toBeNull();
|
||||
expect(select).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(["phase", "leaseOwner", "resultStatus", "status", "finishedAt", "frozenRun", "workspace", "provider", "sourceTuple", "artifactVersion", "artifactDigest"])(
|
||||
"rejects unsupported or stale source evidence: %s", async kind => {
|
||||
const f = await fixture(); let sourceHeader = f.sourceHeader;
|
||||
if (kind === "phase") f.source.coordinator.phase = "observed";
|
||||
if (kind === "leaseOwner") Object.assign(f.source.coordinator, { leaseOwner: "another-owner", leaseExpiresAt: new Date(0) });
|
||||
if (kind === "resultStatus") f.source.resultStatus = "rejected";
|
||||
if (kind === "status") f.source.run.status = "running";
|
||||
if (kind === "finishedAt") Object.assign(f.source.run, { finishedAt: null });
|
||||
if (kind === "frozenRun") f.source.run.runnerProfileJson.nativeExecutionInput.binding.runId = currentRunId;
|
||||
if (kind === "workspace") f.source.run.runnerProfileJson.nativeExecutionInput.binding.executionWorkspaceId = "foreign";
|
||||
if (kind === "provider") Object.assign(f.source.run.runnerProfileJson.nativeExecutionInput.provider, { model: "another-model" });
|
||||
if (kind === "sourceTuple") { const h = JSON.parse(sourceHeader); h.state.identity.turnId = "foreign"; sourceHeader = JSON.stringify(h); }
|
||||
if (kind === "artifactVersion") { const h = JSON.parse(sourceHeader); h.runnerVersion = "other-version"; sourceHeader = JSON.stringify(h); }
|
||||
if (kind === "artifactDigest") { const h = JSON.parse(sourceHeader); h.runnerDigest = `sha256:${"d".repeat(64)}`; sourceHeader = JSON.stringify(h); }
|
||||
expect(await resolveNativeStagedAttachmentRecovery({ ...f.database([f.source], sourceHeader), header: f.header, currentRun: f.currentRun })).toBeNull();
|
||||
});
|
||||
|
||||
it("keeps private executor scope/config verification mandatory when supplied", async () => {
|
||||
const f = await fixture();
|
||||
expect(await resolveNativeStagedAttachmentRecovery({ ...f.database(), header: f.header, currentRun: f.currentRun,
|
||||
matchesExecution: () => false })).toBeNull();
|
||||
});
|
||||
|
||||
it.each(["companyId", "issueId", "runId", "part"])("rejects encrypted source header with another %s", async key => {
|
||||
const f = await fixture();
|
||||
expect(await resolveNativeStagedAttachmentRecovery({ ...f.database([f.source], f.sourceHeader, { [key]: "foreign" }),
|
||||
header: f.header, currentRun: f.currentRun })).toBeNull();
|
||||
});
|
||||
|
||||
it("does not accept a source lacking the current tool contract or provider identity", async () => {
|
||||
const f = await fixture();
|
||||
f.source.run.runnerProfileJson.nativeToolContractFingerprint = "old-contract";
|
||||
expect(await resolveNativeStagedAttachmentRecovery({ ...f.database(), header: f.header, currentRun: f.currentRun })).toBeNull();
|
||||
f.source.run.runnerProfileJson.nativeToolContractFingerprint = nativeToolContractFingerprintForTarget("remote", "codex");
|
||||
f.source.run.runnerProfileJson.sessionCheckpoint.providerSessionId = "";
|
||||
expect(await resolveNativeStagedAttachmentRecovery({ ...f.database(), header: f.header, currentRun: f.currentRun })).toBeNull();
|
||||
});
|
||||
|
||||
it("rejects corrupted or ordinary headers instead of promoting them to staged evidence", async () => {
|
||||
const f = await fixture();
|
||||
for (const header of [f.sourceHeader, "{}", "not-json"]) {
|
||||
const { db, select } = f.database();
|
||||
expect(await resolveNativeStagedAttachmentRecovery({ db, header, currentRun: f.currentRun })).toBeNull();
|
||||
expect(select).not.toHaveBeenCalled();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,124 @@
|
||||
import { isDeepStrictEqual } from "node:util";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { heartbeatRuns, nativeRunFinalizations, nativeRunResults, type Db } from "@paperclipai/db";
|
||||
import {
|
||||
inspectDurablePrpCheckpointHeader,
|
||||
inspectStagedRunAttachmentHeader,
|
||||
parseNativeExecutionInput,
|
||||
type DurableRecoveryIdentity,
|
||||
type NativeExecutionInput,
|
||||
type PersistedNativeSession,
|
||||
type StagedRunAttachment,
|
||||
} from "../../vendor/paperclip-runner/index.js";
|
||||
import { loadNativeRunnerCheckpointHeader } from "./native-runner-checkpoint.js";
|
||||
import { rebindNativeSessionCheckpoint } from "./native-session-resume.js";
|
||||
|
||||
type FrozenRun = Pick<typeof heartbeatRuns.$inferSelect,
|
||||
"id" | "companyId" | "agentId" | "nativeIssueId" | "nativeSessionId" | "runnerInstanceId" |
|
||||
"runtimeMode" | "status" | "runnerProfileJson" | "finishedAt">;
|
||||
|
||||
type StagedHeaderProof = NonNullable<ReturnType<typeof inspectStagedRunAttachmentHeader>>;
|
||||
export type NativeStagedAttachmentRecovery = Omit<StagedRunAttachment, "runAttachTemplate"> &
|
||||
Pick<StagedHeaderProof, "protocolIdentity" | "phase" | "receipt"> & {
|
||||
sourceRun: FrozenRun;
|
||||
sourceExecution: NativeExecutionInput;
|
||||
currentExecution: NativeExecutionInput;
|
||||
/** In-memory input only. It is not a published/resumable target snapshot. */
|
||||
checkpoint: PersistedNativeSession;
|
||||
/** A published target snapshot takes precedence over the source rebound. */
|
||||
hasTargetCheckpoint: boolean;
|
||||
targetCheckpoint: PersistedNativeSession | null;
|
||||
runnerVersion: string;
|
||||
runnerDigest: string;
|
||||
};
|
||||
|
||||
function record(value: unknown): Record<string, unknown> {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value)
|
||||
? value as Record<string, unknown> : {};
|
||||
}
|
||||
|
||||
function boundTargetCheckpoint(run: FrozenRun, source: PersistedNativeSession): PersistedNativeSession | null {
|
||||
const snapshot = record(run.runnerProfileJson?.sessionCheckpoint), identity = record(snapshot.identity);
|
||||
if (typeof snapshot.sessionId !== "string" || !snapshot.sessionId || identity.runId !== run.id ||
|
||||
identity.companyId !== run.companyId || identity.issueId !== run.nativeIssueId || identity.agentId !== run.agentId ||
|
||||
identity.sessionId !== run.nativeSessionId) return null;
|
||||
// A target checkpoint may advance cursor/goal/tool progress, but cannot change
|
||||
// the provider session retained by this attachment. Do not rebound it again.
|
||||
if ((source.providerSessionId != null && source.providerSessionId !== snapshot.providerSessionId) ||
|
||||
(source.providerIdentity != null && !isDeepStrictEqual(source.providerIdentity, snapshot.providerIdentity))) return null;
|
||||
return structuredClone(snapshot) as unknown as PersistedNativeSession;
|
||||
}
|
||||
|
||||
function frozenExecution(run: FrozenRun): NativeExecutionInput | null {
|
||||
try {
|
||||
const execution = parseNativeExecutionInput(run.runnerProfileJson?.nativeExecutionInput);
|
||||
return run.runtimeMode === "native" && execution.binding.runId === run.id &&
|
||||
execution.binding.companyId === run.companyId && execution.binding.agentId === run.agentId &&
|
||||
execution.binding.issueId === run.nativeIssueId && execution.session.normalizedSessionId === run.nativeSessionId
|
||||
? execution : null;
|
||||
} catch { return null; }
|
||||
}
|
||||
|
||||
function matchesRun(identity: DurableRecoveryIdentity, run: FrozenRun): boolean {
|
||||
return identity.runId === run.id && identity.runnerInstanceId === run.runnerInstanceId &&
|
||||
identity.normalizedSessionId === run.nativeSessionId;
|
||||
}
|
||||
|
||||
/** Staging proves which prior attachment may be inspected, not that it is alive
|
||||
* or that work may be dispatched. Classification reads only two indexed point
|
||||
* lookups: the named source/result and its encrypted header. The executor must
|
||||
* re-read this proof with its private scope/config matcher before any RPC, then
|
||||
* independently prove the physical Runner and authenticate the target tuple. */
|
||||
export async function resolveNativeStagedAttachmentRecovery(input: {
|
||||
db: Pick<Db, "select">;
|
||||
currentRun: FrozenRun;
|
||||
header: string;
|
||||
expectedTarget?: DurableRecoveryIdentity;
|
||||
matchesExecution?: (previous: NativeExecutionInput, current: NativeExecutionInput) => boolean;
|
||||
}): Promise<NativeStagedAttachmentRecovery | null> {
|
||||
const proof = inspectStagedRunAttachmentHeader(input.header);
|
||||
const current = frozenExecution(input.currentRun);
|
||||
if (!proof || !current || current.provider.kind !== "codex" || !["running", "failed"].includes(input.currentRun.status) ||
|
||||
!matchesRun(proof.attachment.targetIdentity, input.currentRun) ||
|
||||
(input.expectedTarget && !isDeepStrictEqual(proof.attachment.targetIdentity, input.expectedTarget))) return null;
|
||||
const { attachment } = proof;
|
||||
const [source] = await input.db.select({ run: heartbeatRuns, coordinator: nativeRunFinalizations,
|
||||
resultId: nativeRunResults.id, resultStatus: nativeRunResults.schemaStatus }).from(heartbeatRuns)
|
||||
.innerJoin(nativeRunFinalizations, and(eq(nativeRunFinalizations.runId, heartbeatRuns.id),
|
||||
eq(nativeRunFinalizations.companyId, heartbeatRuns.companyId), eq(nativeRunFinalizations.issueId, heartbeatRuns.nativeIssueId)))
|
||||
.innerJoin(nativeRunResults, and(eq(nativeRunResults.id, nativeRunFinalizations.resultId),
|
||||
eq(nativeRunResults.runId, heartbeatRuns.id), eq(nativeRunResults.companyId, heartbeatRuns.companyId),
|
||||
eq(nativeRunResults.issueId, heartbeatRuns.nativeIssueId)))
|
||||
.where(and(eq(heartbeatRuns.id, attachment.sourceIdentity.runId), eq(heartbeatRuns.companyId, input.currentRun.companyId),
|
||||
eq(heartbeatRuns.agentId, input.currentRun.agentId), eq(heartbeatRuns.nativeIssueId, current.binding.issueId)))
|
||||
.limit(1);
|
||||
if (!source || source.run.companyId !== current.binding.companyId || source.run.agentId !== current.binding.agentId ||
|
||||
source.run.nativeIssueId !== current.binding.issueId || !matchesRun(attachment.sourceIdentity, source.run) ||
|
||||
source.run.status !== "succeeded" || source.run.finishedAt === null || source.coordinator.phase !== "committed" ||
|
||||
source.coordinator.companyId !== current.binding.companyId || source.coordinator.issueId !== current.binding.issueId ||
|
||||
source.coordinator.runId !== source.run.id || !source.coordinator.resultId ||
|
||||
source.coordinator.resultId !== source.resultId || source.resultStatus !== "accepted" ||
|
||||
source.coordinator.leaseOwner !== null || source.coordinator.leaseExpiresAt !== null) return null;
|
||||
const previous = frozenExecution(source.run);
|
||||
if (!previous || (input.matchesExecution && !input.matchesExecution(previous, current))) return null;
|
||||
// Reuse the existing provider, tool-contract, workspace, runtime-context and
|
||||
// normalized-session compatibility owner; no second set of digest formulas.
|
||||
const checkpoint = rebindNativeSessionCheckpoint({ previousRun: source.run, currentExecution: current, executionTargetKind: "remote" });
|
||||
if (!checkpoint || !(typeof checkpoint.providerSessionId === "string" && checkpoint.providerSessionId.length > 0 ||
|
||||
checkpoint.providerIdentity && Object.keys(checkpoint.providerIdentity).length > 0)) return null;
|
||||
const hasTargetCheckpoint = input.currentRun.runnerProfileJson?.sessionCheckpoint != null;
|
||||
const targetCheckpoint = hasTargetCheckpoint ? boundTargetCheckpoint(input.currentRun, checkpoint) : null;
|
||||
if (hasTargetCheckpoint && !targetCheckpoint) return null;
|
||||
const sourceHeader = await loadNativeRunnerCheckpointHeader(input.db, {
|
||||
companyId: source.run.companyId, issueId: current.binding.issueId, runId: source.run.id,
|
||||
});
|
||||
if (!sourceHeader || !isDeepStrictEqual(inspectDurablePrpCheckpointHeader(sourceHeader), attachment.sourceIdentity)) return null;
|
||||
// The codec above has validated this bounded header's artifact fields. A
|
||||
// staged target cannot silently select different bytes from its source.
|
||||
const sourceArtifact = JSON.parse(sourceHeader) as { runnerVersion: string; runnerDigest: string };
|
||||
if (sourceArtifact.runnerVersion !== proof.runnerVersion || sourceArtifact.runnerDigest !== proof.runnerDigest) return null;
|
||||
return { ...attachment, sourceRun: source.run, sourceExecution: previous, currentExecution: current,
|
||||
checkpoint, hasTargetCheckpoint, targetCheckpoint,
|
||||
protocolIdentity: proof.protocolIdentity, phase: proof.phase, receipt: proof.receipt,
|
||||
runnerVersion: proof.runnerVersion, runnerDigest: proof.runnerDigest };
|
||||
}
|
||||
@@ -0,0 +1,319 @@
|
||||
import { createRequire } from "node:module";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { drizzle } from "drizzle-orm/postgres-js";
|
||||
import { agents, agentTaskSessions, companies, completionContracts, createDb, heartbeatRuns, issues,
|
||||
nativeRunFinalizations, nativeRunResults, type Db } from "@paperclipai/db";
|
||||
import { CONTROL_PLANE_CONFORMANCE_OPEN, CONTROL_PLANE_CONFORMANCE_RESULT,
|
||||
CONTROL_PLANE_CONFORMANCE_TERMINAL } from "../../vendor/paperclip-runner/testing.js";
|
||||
import type { PersistedNativeSession } from "../../vendor/paperclip-runner/index.js";
|
||||
import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js";
|
||||
import { startRecoveryPostgres } from "../../__tests__/helpers/recovery-postgres.js";
|
||||
import { redactRegisteredSecretValues } from "../run-secret-redaction.js";
|
||||
import { nativeSha256 } from "./canonical.js";
|
||||
import { PaperclipControlPlanePort, type PaperclipControlPlaneBinding } from "./paperclip-control-plane-port.js";
|
||||
import { nativeTaskSessionPublicationHandled, publishNativeTaskSessionInTransaction, NATIVE_TASK_SESSION_PUBLICATION_KEY,
|
||||
type NativeTaskSessionPublicationContext } from "./native-task-session-publication.js";
|
||||
|
||||
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_NATIVE_PUBLICATION_TEST_DATABASE_URL");
|
||||
describe("accepted native task session publication", () => {
|
||||
let temporary: Awaited<ReturnType<typeof startRecoveryPostgres>> | null = null;
|
||||
let db: Db;
|
||||
let measured: Db;
|
||||
let client: Db["$client"];
|
||||
const statements: string[] = [];
|
||||
beforeAll(async () => {
|
||||
if (!external) temporary = await startRecoveryPostgres("native-publication-");
|
||||
const url = external ?? temporary!.connectionString;
|
||||
db = createDb(url, { maxConnections: 2 });
|
||||
// Test-only wire observer. Production still uses the existing Db/pools.
|
||||
const postgres = createRequire(import.meta.resolve("@paperclipai/db"))("postgres") as
|
||||
(url: string, options: object) => Db["$client"];
|
||||
client = postgres(url, { max: 1, prepare: false, connection: { application_name: "native-publication-observer" },
|
||||
debug: (_id: number, query: string) => { statements.push(query); } });
|
||||
measured = drizzle(client) as unknown as Db;
|
||||
});
|
||||
afterAll(async () => {
|
||||
await client?.end({ timeout: 1 });
|
||||
await db?.$client.end({ timeout: 1 });
|
||||
await temporary?.cleanup();
|
||||
});
|
||||
|
||||
async function fixture(source: "runner" | "control" = "runner", accepted = true) {
|
||||
const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID();
|
||||
const sessionId = randomUUID(), contractId = randomUUID(), runner = randomUUID(), control = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Native pointer", issuePrefix: `P${companyId.slice(0, 7)}` });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Native pointer", adapterType: "codex_local", status: "running" });
|
||||
await db.insert(issues).values({ id: issueId, companyId, title: "Native pointer", status: "in_progress", assigneeAgentId: agentId });
|
||||
await db.insert(completionContracts).values({ id: contractId, companyId, issueId, revision: 1,
|
||||
schemaVersion: "paperclip.completion-contract.v1", policyVersion: "standalone-v1", risk: "standard",
|
||||
completionAuthority: "server_arbiter", incompleteCriteriaPolicy: "preserve_non_terminal",
|
||||
contractJson: { revision: "standalone-v1", objective: "Conformance", criteria: [{ id: "objective", requirement: "Complete" }] },
|
||||
canonicalSha256: "publication-contract", createdByActorType: "system", createdByActorId: "test" });
|
||||
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running", runtimeMode: "native",
|
||||
nativeIssueId: issueId, nativeSessionId: sessionId, runnerInstanceId: runner, completionContractId: contractId,
|
||||
completionContractSha256: "publication-contract", startedAt: new Date(Date.now() - 2_000), runnerProfileJson: {} });
|
||||
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId));
|
||||
const owner = { companyId, issueId, runId, authorityRunId: runId, bootId: "publication-boot", generation: 1, leaseOwner: "publication-owner" };
|
||||
await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId, phase: "observed",
|
||||
controllerBootId: owner.bootId, controllerGeneration: owner.generation, leaseOwner: owner.leaseOwner,
|
||||
leaseExpiresAt: new Date(Date.now() + 60_000) });
|
||||
const binding: PaperclipControlPlaneBinding = { companyId, agentId, issueId, runId, sessionId,
|
||||
completionContractId: contractId, completionContractSha256: "publication-contract", sourceInstanceId: runner,
|
||||
controlPlaneSourceInstanceId: control };
|
||||
const context: NativeTaskSessionPublicationContext = { taskKey: issueId, adapterType: "codex_local",
|
||||
sessionParamsJson: { cwd: "/working", paperclipSessionConfigFingerprint: "prepared-once", opaque: "private-prepared-param" } };
|
||||
const snapshot: PersistedNativeSession = { backendKind: "mock", sessionId, identity: { companyId, agentId, issueId, runId, sessionId },
|
||||
providerSessionId: "provider-thread", activeTurnId: null, pendingRuntimeRequests: [], goal: null,
|
||||
semanticResult: CONTROL_PLANE_CONFORMANCE_RESULT, terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL };
|
||||
const port = new PaperclipControlPlanePort(measured, binding, { taskSessionPublication: context, taskSessionPublicationOwner: owner });
|
||||
await port.openRun({ ...CONTROL_PLANE_CONFORMANCE_OPEN, identity: snapshot.identity, sourceInstanceId: runner });
|
||||
if (accepted) await new PaperclipControlPlanePort(db, binding).completeRun({ result: CONTROL_PLANE_CONFORMANCE_RESULT,
|
||||
terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, turnId: "publication-turn" });
|
||||
await db.insert(agentTaskSessions).values({ companyId, agentId, adapterType: context.adapterType, taskKey: issueId,
|
||||
goalSourceId: `${source === "runner" ? runner : control}:${runId}`, goalSourceCursor: 4 });
|
||||
const read = async () => {
|
||||
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
|
||||
const [session] = await db.select().from(agentTaskSessions).where(eq(agentTaskSessions.taskKey, issueId));
|
||||
return { run, session };
|
||||
};
|
||||
return { binding, owner, context, snapshot, port, read };
|
||||
}
|
||||
|
||||
it.each(["runner", "control"] as const)("publishes from the exact accepted %s epoch in one checkpoint transaction", async source => {
|
||||
const f = await fixture(source); statements.length = 0;
|
||||
await f.port.checkpointSession(f.snapshot);
|
||||
const queries = [...statements], { run, session } = await f.read();
|
||||
expect(session).toMatchObject({ sessionParamsJson: { ...f.context.sessionParamsJson, sessionId: f.binding.sessionId },
|
||||
sessionDisplayId: "provider-thread", lastRunId: f.binding.runId });
|
||||
expect(nativeTaskSessionPublicationHandled(run.runnerProfileJson, { ...f.binding, ...f.context })).toBe(true);
|
||||
expect(queries.filter(query => /^begin\b/i.test(query.trim()))).toHaveLength(1);
|
||||
expect(queries.filter(query => /^commit\b/i.test(query.trim()))).toHaveLength(1);
|
||||
expect(queries.filter(query => /native_run_results/i.test(query))).toHaveLength(1);
|
||||
expect(queries.filter(query => /^select .*from "agent_task_sessions"/i.test(query.trim()))).toHaveLength(1);
|
||||
expect(queries.filter(query => /^update "agent_task_sessions"/i.test(query.trim()))).toHaveLength(1);
|
||||
expect(queries).toHaveLength(7);
|
||||
expect(JSON.stringify(run.runnerProfileJson)).not.toContain("private-prepared-param");
|
||||
expect(JSON.stringify({ [NATIVE_TASK_SESSION_PUBLICATION_KEY]: run.runnerProfileJson![NATIVE_TASK_SESSION_PUBLICATION_KEY] }).length).toBeLessThan(4_096);
|
||||
expect(redactRegisteredSecretValues(run.runnerProfileJson, [])).not.toHaveProperty(NATIVE_TASK_SESSION_PUBLICATION_KEY);
|
||||
expect(nativeTaskSessionPublicationHandled(run.runnerProfileJson, { ...f.binding, ...f.context, companyId: randomUUID() })).toBe(false);
|
||||
expect(nativeTaskSessionPublicationHandled(run.runnerProfileJson, { ...f.binding, ...f.context, agentId: randomUUID() })).toBe(false);
|
||||
const malformed = structuredClone(run.runnerProfileJson!);
|
||||
(malformed[NATIVE_TASK_SESSION_PUBLICATION_KEY] as Record<string, unknown>).resultId = "not-an-id";
|
||||
expect(nativeTaskSessionPublicationHandled(malformed, { ...f.binding, ...f.context })).toBe(false);
|
||||
});
|
||||
|
||||
it("does not recreate a reset pointer or query task state on repeated terminal maintenance", async () => {
|
||||
const f = await fixture(); await f.port.checkpointSession(f.snapshot);
|
||||
await db.delete(agentTaskSessions).where(eq(agentTaskSessions.taskKey, f.binding.issueId)); statements.length = 0;
|
||||
await f.port.checkpointSession(f.snapshot);
|
||||
const repeated = [...statements];
|
||||
expect((await f.read()).session).toBeUndefined();
|
||||
expect(repeated.filter(query => /agent_task_sessions|native_run_results/i.test(query))).toHaveLength(0);
|
||||
expect(repeated.filter(query => /^begin\b/i.test(query.trim()))).toHaveLength(1);
|
||||
expect(repeated.filter(query => /^update /i.test(query.trim()))).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("persists a pre-acceptance terminal observation and publishes only at the existing accepted checkpoint", async () => {
|
||||
const f = await fixture("runner", false);
|
||||
// Driver event/startup checkpoints can expose a semantic terminal observation
|
||||
// before the runtime calls completeRun. That observation is not acceptance.
|
||||
await f.port.checkpointSession(f.snapshot);
|
||||
const observed = await f.read();
|
||||
expect(observed.run.runnerProfileJson?.sessionCheckpoint).toMatchObject({ semanticResult: CONTROL_PLANE_CONFORMANCE_RESULT });
|
||||
expect(observed.run.runnerProfileJson).not.toHaveProperty(NATIVE_TASK_SESSION_PUBLICATION_KEY);
|
||||
expect(observed.session?.sessionParamsJson).toBeNull();
|
||||
expect(nativeTaskSessionPublicationHandled(observed.run.runnerProfileJson, { ...f.binding, ...f.context })).toBe(false);
|
||||
await f.port.completeRun({ result: CONTROL_PLANE_CONFORMANCE_RESULT,
|
||||
terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, turnId: "publication-turn" });
|
||||
expect((await f.read()).session?.sessionParamsJson).toBeNull();
|
||||
await f.port.checkpointSession(f.snapshot);
|
||||
const completed = await f.read();
|
||||
expect(completed.session?.lastRunId).toBe(f.binding.runId);
|
||||
expect(nativeTaskSessionPublicationHandled(completed.run.runnerProfileJson, { ...f.binding, ...f.context })).toBe(true);
|
||||
});
|
||||
|
||||
it("does not hold the issue parent lock while a result replay waits for the status coordinator", async () => {
|
||||
const f = await fixture();
|
||||
let committed = 0;
|
||||
const port = new PaperclipControlPlanePort(db, f.binding, { nativeRecovery: { owner: f.owner,
|
||||
compatibility: { configDigest: nativeSha256("config"), processConfigDigest: null } },
|
||||
onResultCommitted: async () => { committed++; } });
|
||||
await port.openRun({ ...CONTROL_PLANE_CONFORMANCE_OPEN, identity: f.snapshot.identity, sourceInstanceId: f.binding.sourceInstanceId });
|
||||
let replay: Promise<unknown> | undefined;
|
||||
let statusLockError: unknown = null;
|
||||
try {
|
||||
await db.transaction(async tx => {
|
||||
// This is commitNativeStatusDecision's actual lock prefix. Hold its
|
||||
// coordinator first, then overlap an identical remote result replay.
|
||||
await tx.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.binding.runId)).for("update");
|
||||
const [backend] = await tx.execute(sql`select pg_backend_pid() as pid`);
|
||||
replay = port.completeRun({ result: CONTROL_PLANE_CONFORMANCE_RESULT,
|
||||
terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, turnId: "publication-turn" }).then(() => null, error => error);
|
||||
let waiting = false;
|
||||
for (let attempt = 0; attempt < 100; attempt++) {
|
||||
const [state] = await measured.execute(sql`select exists (
|
||||
select 1 from pg_stat_activity where ${backend.pid} = any(pg_blocking_pids(pid))) as waiting`);
|
||||
if (state.waiting) { waiting = true; break; }
|
||||
await new Promise(resolve => setTimeout(resolve, 10));
|
||||
}
|
||||
expect(waiting).toBe(true);
|
||||
// A bounded test-local timeout reports the inverse-order cycle without
|
||||
// relying on which transaction PostgreSQL chooses as deadlock victim.
|
||||
await tx.execute(sql`set local lock_timeout = '500ms'`);
|
||||
await tx.select().from(issues).where(eq(issues.id, f.binding.issueId)).for("update");
|
||||
});
|
||||
} catch (error) { statusLockError = error; }
|
||||
const replayError = await replay;
|
||||
expect(statusLockError).toBeNull();
|
||||
expect(replayError).toBeNull();
|
||||
expect(committed).toBe(0);
|
||||
expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, f.binding.runId))).toHaveLength(1);
|
||||
await db.update(nativeRunFinalizations).set({ controllerGeneration: 2 }).where(eq(nativeRunFinalizations.runId, f.binding.runId));
|
||||
await expect(port.completeRun({ result: CONTROL_PLANE_CONFORMANCE_RESULT,
|
||||
terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, turnId: "publication-turn" })).rejects.toThrow("checkpoint_owner_lost");
|
||||
});
|
||||
|
||||
it.each(["missing", "foreign_epoch", "newer_pointer", "newer_execution", "generation_reset", "stop"])("fences %s and records a handled suppression", async kind => {
|
||||
const f = await fixture();
|
||||
if (kind === "missing") await db.delete(agentTaskSessions).where(eq(agentTaskSessions.taskKey, f.binding.issueId));
|
||||
if (kind === "foreign_epoch") await db.update(agentTaskSessions).set({ goalSourceId: `${randomUUID()}:${f.binding.runId}` }).where(eq(agentTaskSessions.taskKey, f.binding.issueId));
|
||||
if (kind === "newer_pointer" || kind === "newer_execution") {
|
||||
const newer = randomUUID();
|
||||
await db.insert(heartbeatRuns).values({ id: newer, companyId: f.binding.companyId, agentId: f.binding.agentId,
|
||||
status: "running", issueId: f.binding.issueId, startedAt: new Date() });
|
||||
if (kind === "newer_pointer") await db.update(agentTaskSessions).set({ lastRunId: newer, sessionParamsJson: { sessionId: "newer" } }).where(eq(agentTaskSessions.taskKey, f.binding.issueId));
|
||||
else await db.update(issues).set({ executionRunId: newer }).where(eq(issues.id, f.binding.issueId));
|
||||
}
|
||||
if (kind === "generation_reset") await db.update(issues).set({ conversationAgentId: f.binding.agentId,
|
||||
conversationUserId: "publication-test-user", conversationState: "active", conversationSessionGeneration: 2 }).where(eq(issues.id, f.binding.issueId));
|
||||
if (kind === "stop") await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, f.binding.runId));
|
||||
await f.port.checkpointSession(f.snapshot);
|
||||
const { run, session } = await f.read();
|
||||
expect(nativeTaskSessionPublicationHandled(run.runnerProfileJson, { ...f.binding, ...f.context })).toBe(true);
|
||||
expect(session?.sessionParamsJson?.sessionId).not.toBe(f.binding.sessionId);
|
||||
});
|
||||
|
||||
it("keeps unsupported task keys and missing remote recovery headers outside handled publication", async () => {
|
||||
const f = await fixture();
|
||||
const context = { ...f.context, taskKey: "custom-task" };
|
||||
const port = new PaperclipControlPlanePort(measured, f.binding, { taskSessionPublication: context, taskSessionPublicationOwner: f.owner });
|
||||
await port.openRun({ ...CONTROL_PLANE_CONFORMANCE_OPEN, identity: f.snapshot.identity, sourceInstanceId: f.binding.sourceInstanceId });
|
||||
await port.checkpointSession(f.snapshot);
|
||||
const { run, session } = await f.read();
|
||||
expect(session?.sessionParamsJson).toBeNull();
|
||||
expect(nativeTaskSessionPublicationHandled(run.runnerProfileJson, { ...f.binding, ...context })).toBe(false);
|
||||
const remote = await fixture();
|
||||
const remotePort = new PaperclipControlPlanePort(measured, remote.binding, { taskSessionPublication: remote.context,
|
||||
taskSessionPublicationOwner: remote.owner, nativeRecovery: { owner: remote.owner,
|
||||
compatibility: { configDigest: nativeSha256("config"), processConfigDigest: null } } });
|
||||
await remotePort.openRun({ ...CONTROL_PLANE_CONFORMANCE_OPEN, identity: remote.snapshot.identity, sourceInstanceId: remote.binding.sourceInstanceId });
|
||||
await remotePort.checkpointSession(remote.snapshot);
|
||||
expect((await remote.read()).session?.sessionParamsJson).toBeNull();
|
||||
expect(nativeTaskSessionPublicationHandled((await remote.read()).run.runnerProfileJson, { ...remote.binding, ...remote.context })).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects stale owners and a different accepted result without partially publishing", async () => {
|
||||
const stale = await fixture();
|
||||
await db.update(nativeRunFinalizations).set({ controllerGeneration: 2 }).where(eq(nativeRunFinalizations.runId, stale.binding.runId));
|
||||
await expect(stale.port.checkpointSession(stale.snapshot)).rejects.toThrow("checkpoint_owner_lost");
|
||||
expect((await stale.read()).session?.sessionParamsJson).toBeNull();
|
||||
const wrong = await fixture();
|
||||
await expect(wrong.port.checkpointSession({ ...wrong.snapshot, semanticResult: { ...CONTROL_PLANE_CONFORMANCE_RESULT, summary: "different" } })).rejects.toThrow("publication_result_mismatch");
|
||||
expect((await wrong.read()).session?.sessionParamsJson).toBeNull();
|
||||
const foreign = await fixture();
|
||||
const foreignResult = await fixture();
|
||||
const [otherCoordinator] = await db.select({ resultId: nativeRunFinalizations.resultId }).from(nativeRunFinalizations)
|
||||
.where(eq(nativeRunFinalizations.runId, foreignResult.binding.runId));
|
||||
await expect(db.update(nativeRunFinalizations).set({ resultId: otherCoordinator.resultId })
|
||||
.where(eq(nativeRunFinalizations.runId, foreign.binding.runId)))
|
||||
.rejects.toMatchObject({ cause: { constraint_name: "native_run_finalizations_result_owner_fk" } });
|
||||
expect((await foreign.read()).session?.sessionParamsJson).toBeNull();
|
||||
await db.update(nativeRunFinalizations).set({ resultId: null }).where(eq(nativeRunFinalizations.runId, foreign.binding.runId));
|
||||
// The CP defers preacceptance snapshots, while the publication helper itself
|
||||
// still requires accepted-result authority if called directly.
|
||||
await expect(publishNativeTaskSessionInTransaction({} as never, { binding: foreign.binding,
|
||||
context: foreign.context, snapshot: foreign.snapshot, run: (await foreign.read()).run,
|
||||
issue: null, resultId: null })).rejects.toThrow("publication_result_missing");
|
||||
});
|
||||
|
||||
it("adds no publication SQL for nonterminal or goal snapshots", async () => {
|
||||
const f = await fixture(); statements.length = 0;
|
||||
await f.port.checkpointSession({ ...f.snapshot, semanticResult: null, terminal: null });
|
||||
expect(statements.some(query => /agent_task_sessions|native_run_results|publication_issue/i.test(query))).toBe(false);
|
||||
statements.length = 0;
|
||||
await f.port.checkpointSession({ ...f.snapshot, goal: {} as NonNullable<PersistedNativeSession["goal"]> });
|
||||
expect(statements.some(query => /agent_task_sessions|native_run_results|publication_issue/i.test(query))).toBe(false);
|
||||
expect((await f.read()).session?.sessionParamsJson).toBeNull();
|
||||
});
|
||||
|
||||
it.each(["completed_predecessor", "expired_accepted_lease"])("preserves legitimate %s maintenance", async kind => {
|
||||
const f = await fixture();
|
||||
if (kind === "completed_predecessor") {
|
||||
const prior = randomUUID();
|
||||
await db.insert(heartbeatRuns).values({ id: prior, companyId: f.binding.companyId, agentId: f.binding.agentId,
|
||||
issueId: f.binding.issueId, status: "succeeded", finishedAt: new Date(Date.now() - 5_000) });
|
||||
await db.update(agentTaskSessions).set({ lastRunId: prior, sessionParamsJson: { sessionId: "prior" } })
|
||||
.where(eq(agentTaskSessions.taskKey, f.binding.issueId));
|
||||
} else {
|
||||
await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 1_000) })
|
||||
.where(eq(nativeRunFinalizations.runId, f.binding.runId));
|
||||
}
|
||||
await f.port.checkpointSession(f.snapshot);
|
||||
expect((await f.read()).session?.lastRunId).toBe(f.binding.runId);
|
||||
});
|
||||
|
||||
it("does not block completeRun's issue KEY SHARE while publishing", async () => {
|
||||
const f = await fixture(); let release!: () => void; let locked!: () => void;
|
||||
const ready = new Promise<void>(resolve => { locked = resolve; });
|
||||
const hold = new Promise<void>(resolve => { release = resolve; });
|
||||
const blocker = db.transaction(async tx => {
|
||||
await tx.select().from(issues).where(eq(issues.id, f.binding.issueId)).for("key share"); locked(); await hold;
|
||||
});
|
||||
await ready;
|
||||
try { await f.port.checkpointSession(f.snapshot); }
|
||||
finally { release(); await blocker; }
|
||||
expect((await f.read()).session?.lastRunId).toBe(f.binding.runId);
|
||||
});
|
||||
|
||||
it("rechecks the owner clock and issue authority after an issue lock wait", async () => {
|
||||
const f = await fixture(); let release!: () => void; let locked!: () => void;
|
||||
await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() + 500) }).where(eq(nativeRunFinalizations.runId, f.binding.runId));
|
||||
const ready = new Promise<void>(resolve => { locked = resolve; });
|
||||
const hold = new Promise<void>(resolve => { release = resolve; });
|
||||
const newer = randomUUID();
|
||||
await db.insert(heartbeatRuns).values({ id: newer, companyId: f.binding.companyId, agentId: f.binding.agentId,
|
||||
issueId: f.binding.issueId, status: "running", startedAt: new Date() });
|
||||
const blocker = db.transaction(async tx => {
|
||||
await tx.select().from(issues).where(eq(issues.id, f.binding.issueId)).for("no key update");
|
||||
await tx.update(issues).set({ executionRunId: newer }).where(eq(issues.id, f.binding.issueId));
|
||||
locked(); await hold;
|
||||
});
|
||||
await ready;
|
||||
const publication = f.port.checkpointSession(f.snapshot);
|
||||
const rejected = expect(publication).rejects.toThrow("checkpoint_owner_lost");
|
||||
try {
|
||||
let waiting = false;
|
||||
for (let attempt = 0; attempt < 100; attempt++) {
|
||||
const rows = await db.execute(sql`select 1 from pg_stat_activity where application_name = 'native-publication-observer'
|
||||
and wait_event_type = 'Lock' and query like '%publication_issue%'`);
|
||||
if (rows.length) { waiting = true; break; }
|
||||
await new Promise(resolve => setTimeout(resolve, 10));
|
||||
}
|
||||
expect(waiting).toBe(true);
|
||||
await new Promise(resolve => setTimeout(resolve, 600));
|
||||
} finally { release(); await blocker; }
|
||||
await rejected;
|
||||
expect((await f.read()).session?.sessionParamsJson).toBeNull();
|
||||
});
|
||||
|
||||
it("rolls back the pointer if combined private metadata exceeds its existing bound", async () => {
|
||||
const f = await fixture();
|
||||
await db.update(heartbeatRuns).set({ runnerProfileJson: { nativeControllerRecovery: { padding: "x".repeat(4_096) } } }).where(eq(heartbeatRuns.id, f.binding.runId));
|
||||
await expect(f.port.checkpointSession(f.snapshot)).rejects.toThrow("publication_metadata_too_large");
|
||||
const { run, session } = await f.read();
|
||||
expect(session?.sessionParamsJson).toBeNull();
|
||||
expect(run.runnerProfileJson).not.toHaveProperty(NATIVE_TASK_SESSION_PUBLICATION_KEY);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,145 @@
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { agentTaskSessions, heartbeatRuns, nativeRunResults, type Db, type issues } from "@paperclipai/db";
|
||||
import { validatePrpStructuredRunResult, type PersistedNativeSession } from "../../vendor/paperclip-runner/index.js";
|
||||
import { nativeSha256 } from "./canonical.js";
|
||||
import type { PaperclipControlPlaneBinding } from "./paperclip-control-plane-port.js";
|
||||
import { readRunCancellation } from "../run-cancellation.js";
|
||||
|
||||
/** Prepared once by heartbeat; never copied into recovery metadata or receipts. */
|
||||
export interface NativeTaskSessionPublicationContext {
|
||||
taskKey: string;
|
||||
adapterType: string;
|
||||
sessionParamsJson: Record<string, unknown> | null;
|
||||
}
|
||||
|
||||
export const NATIVE_TASK_SESSION_PUBLICATION_KEY = "nativeTaskSessionPublication";
|
||||
type Transaction = Parameters<Parameters<Db["transaction"]>[0]>[0];
|
||||
type Outcome = "published" | "suppressed_reset" | "suppressed_newer" | "suppressed_authority" | "missing_stub" | "unsupported";
|
||||
interface Receipt {
|
||||
schema: "paperclip.native-task-session-publication.v1";
|
||||
companyId: string;
|
||||
issueId: string;
|
||||
agentId: string;
|
||||
runId: string;
|
||||
taskKey: string;
|
||||
adapterType: string;
|
||||
sessionId: string;
|
||||
resultId: string;
|
||||
outcome: Outcome;
|
||||
}
|
||||
|
||||
function record(value: unknown): Record<string, unknown> {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||||
}
|
||||
const uuid = (value: unknown): value is string => typeof value === "string" && /^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/i.test(value);
|
||||
const outcomes: readonly string[] = ["published", "suppressed_reset", "suppressed_newer", "suppressed_authority", "missing_stub", "unsupported"];
|
||||
|
||||
function receiptFromProfile(profile: unknown): Receipt | null {
|
||||
const root = record(profile), value = record(root[NATIVE_TASK_SESSION_PUBLICATION_KEY]);
|
||||
if (value.schema !== "paperclip.native-task-session-publication.v1"
|
||||
|| ![value.companyId, value.issueId, value.agentId, value.runId, value.sessionId, value.resultId].every(uuid)
|
||||
|| typeof value.taskKey !== "string" || value.taskKey.length > 512
|
||||
|| typeof value.adapterType !== "string" || !value.adapterType.length || value.adapterType.length > 128
|
||||
|| !outcomes.includes(String(value.outcome)) || Object.keys(value).length !== 10) return null;
|
||||
const snapshot = record(root.sessionCheckpoint), identity = record(snapshot.identity);
|
||||
if (identity.runId !== value.runId || identity.companyId !== value.companyId || identity.issueId !== value.issueId
|
||||
|| identity.agentId !== value.agentId || identity.sessionId !== value.sessionId) return null;
|
||||
return value as unknown as Receipt;
|
||||
}
|
||||
|
||||
/** Only the exact server-written outcome may suppress heartbeat's late fallback. */
|
||||
export function nativeTaskSessionPublicationHandled(profile: unknown, binding: {
|
||||
companyId: string; agentId: string; runId: string; taskKey: string; adapterType: string;
|
||||
}): boolean {
|
||||
const receipt = receiptFromProfile(profile);
|
||||
return !!receipt && receipt.companyId === binding.companyId && receipt.agentId === binding.agentId
|
||||
&& receipt.runId === binding.runId && receipt.taskKey === binding.taskKey
|
||||
&& receipt.adapterType === binding.adapterType && ["published", "suppressed_reset", "suppressed_newer", "suppressed_authority", "missing_stub"].includes(receipt.outcome);
|
||||
}
|
||||
|
||||
export function eligibleNativeTaskSessionPublication(snapshot: PersistedNativeSession): boolean {
|
||||
return snapshot.goal == null && snapshot.semanticResult != null && snapshot.terminal?.runTerminalState === "succeeded"
|
||||
&& snapshot.activeTurnId == null && !(snapshot.pendingRuntimeRequests?.length);
|
||||
}
|
||||
|
||||
export type NativeTaskSessionPublicationRun = Pick<typeof heartbeatRuns.$inferSelect,
|
||||
"id" | "companyId" | "agentId" | "nativeIssueId" | "nativeSessionId" | "status" | "resultJson" | "contextSnapshot" | "startedAt" | "createdAt" | "runnerProfileJson">;
|
||||
export type NativeTaskSessionPublicationIssue = Pick<typeof issues.$inferSelect,
|
||||
"id" | "companyId" | "executionRunId" | "conversationAgentId" | "conversationSessionGeneration">;
|
||||
|
||||
/** Caller holds coordinator → issue NO KEY UPDATE → run. This helper owns only
|
||||
* the task-row lock and UPDATE; never starts a transaction or creates a stub. */
|
||||
export async function publishNativeTaskSessionInTransaction(tx: Transaction, input: {
|
||||
binding: PaperclipControlPlaneBinding;
|
||||
context: NativeTaskSessionPublicationContext;
|
||||
snapshot: PersistedNativeSession;
|
||||
run: NativeTaskSessionPublicationRun;
|
||||
issue: NativeTaskSessionPublicationIssue | null;
|
||||
resultId: string | null;
|
||||
recoveryReady?: boolean;
|
||||
}): Promise<Receipt> {
|
||||
const { binding, context, snapshot, run, issue } = input;
|
||||
const existingReceipt = receiptFromProfile(run.runnerProfileJson);
|
||||
if (existingReceipt) {
|
||||
if (existingReceipt.companyId !== binding.companyId || existingReceipt.issueId !== binding.issueId
|
||||
|| existingReceipt.agentId !== binding.agentId || existingReceipt.runId !== binding.runId
|
||||
|| existingReceipt.sessionId !== snapshot.identity.sessionId || existingReceipt.taskKey !== context.taskKey
|
||||
|| existingReceipt.adapterType !== context.adapterType || existingReceipt.resultId !== input.resultId) {
|
||||
throw new Error("native_task_session_publication_receipt_mismatch");
|
||||
}
|
||||
return existingReceipt;
|
||||
}
|
||||
if (record(run.runnerProfileJson)[NATIVE_TASK_SESSION_PUBLICATION_KEY] !== undefined) {
|
||||
throw new Error("native_task_session_publication_receipt_invalid");
|
||||
}
|
||||
if (!input.resultId) throw new Error("native_task_session_publication_result_missing");
|
||||
const [accepted] = await tx.select({ id: nativeRunResults.id, canonicalSha256: nativeRunResults.canonicalSha256,
|
||||
turnId: nativeRunResults.turnId }).from(nativeRunResults).where(and(
|
||||
eq(nativeRunResults.id, input.resultId), eq(nativeRunResults.runId, binding.runId),
|
||||
eq(nativeRunResults.companyId, binding.companyId), eq(nativeRunResults.issueId, binding.issueId),
|
||||
eq(nativeRunResults.completionContractId, binding.completionContractId), eq(nativeRunResults.schemaStatus, "accepted"),
|
||||
)).limit(1);
|
||||
const validated = validatePrpStructuredRunResult(snapshot.semanticResult);
|
||||
if (!accepted || !validated.ok || accepted.canonicalSha256 !== nativeSha256({
|
||||
binding, result: validated.result, terminal: snapshot.terminal, turnId: accepted.turnId ?? null,
|
||||
})) throw new Error("native_task_session_publication_result_mismatch");
|
||||
const receipt = (outcome: Outcome): Receipt => ({ schema: "paperclip.native-task-session-publication.v1",
|
||||
companyId: binding.companyId, issueId: binding.issueId, agentId: binding.agentId, runId: binding.runId,
|
||||
taskKey: context.taskKey, adapterType: context.adapterType, sessionId: snapshot.identity.sessionId,
|
||||
resultId: accepted.id, outcome });
|
||||
if (context.taskKey !== binding.issueId || input.recoveryReady === false) return receipt("unsupported");
|
||||
const cancellation = record(run.resultJson?.nativeCancellation), stop = readRunCancellation(run.resultJson);
|
||||
if (!issue || ["cancelled", "interrupted", "failed", "timed_out"].includes(run.status) || issue.companyId !== binding.companyId
|
||||
|| run.resultJson?.startupCancellation || (stop?.source === "operator" && stop.expected)
|
||||
|| (["run", "issue"].includes(String(cancellation.scope)) && ["pending", "acknowledged"].includes(String(cancellation.dispatchState)))
|
||||
|| (issue.conversationAgentId && issue.conversationAgentId !== binding.agentId)) return receipt("suppressed_authority");
|
||||
if (issue.conversationAgentId
|
||||
&& run.contextSnapshot?.conversationSessionGeneration !== issue.conversationSessionGeneration) return receipt("suppressed_reset");
|
||||
if (issue.executionRunId && issue.executionRunId !== binding.runId) return receipt("suppressed_newer");
|
||||
const [session] = await tx.select({ id: agentTaskSessions.id, goalSourceId: agentTaskSessions.goalSourceId,
|
||||
lastRunId: agentTaskSessions.lastRunId }).from(agentTaskSessions).where(and(
|
||||
eq(agentTaskSessions.companyId, binding.companyId), eq(agentTaskSessions.agentId, binding.agentId),
|
||||
eq(agentTaskSessions.adapterType, context.adapterType), eq(agentTaskSessions.taskKey, context.taskKey),
|
||||
)).for("update").limit(1);
|
||||
// Supported native turns synchronously project a stub before this terminal
|
||||
// snapshot. Absence can be an operator deletion; never recreate it later.
|
||||
if (!session) return receipt("missing_stub");
|
||||
const boundSources = [binding.sourceInstanceId, binding.controlPlaneSourceInstanceId]
|
||||
.filter((value): value is string => typeof value === "string" && value.length > 0)
|
||||
.map(source => `${source}:${binding.runId}`);
|
||||
if (!session.goalSourceId || !boundSources.includes(session.goalSourceId)) return receipt("suppressed_newer");
|
||||
if (session.lastRunId && session.lastRunId !== binding.runId) {
|
||||
const [prior] = await tx.select({ companyId: heartbeatRuns.companyId, agentId: heartbeatRuns.agentId,
|
||||
issueId: heartbeatRuns.issueId, nativeIssueId: heartbeatRuns.nativeIssueId, finishedAt: heartbeatRuns.finishedAt,
|
||||
}).from(heartbeatRuns).where(eq(heartbeatRuns.id, session.lastRunId)).limit(1);
|
||||
if (!prior || prior.companyId !== binding.companyId || prior.agentId !== binding.agentId
|
||||
|| (prior.nativeIssueId ?? prior.issueId) !== binding.issueId || !prior.finishedAt || !run.startedAt
|
||||
|| prior.finishedAt > run.startedAt) return receipt("suppressed_newer");
|
||||
}
|
||||
await tx.update(agentTaskSessions).set({
|
||||
sessionParamsJson: { ...context.sessionParamsJson, sessionId: snapshot.identity.sessionId },
|
||||
sessionDisplayId: snapshot.providerSessionId ?? snapshot.sessionId, lastRunId: binding.runId,
|
||||
lastError: null, updatedAt: new Date(),
|
||||
}).where(eq(agentTaskSessions.id, session.id));
|
||||
return receipt("published");
|
||||
}
|
||||
@@ -12,8 +12,8 @@ import { agents, budgetPolicies, companies, costEvents, createDb, heartbeatRuns,
|
||||
import { type PrpEvent } from "../../vendor/paperclip-runner/index.js";
|
||||
import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
|
||||
import { accountRunCost } from "../run-cost-accounting.js";
|
||||
import { usageReceiptSpoolPath } from "../usage-receipts.js";
|
||||
import { createNativeUsageDelivery } from "./native-usage-delivery.js";
|
||||
import { usageReceiptSpoolPath, type UsageReceiptEnvelope } from "../usage-receipts.js";
|
||||
import { createNativeUsageContinuationDelivery, createNativeUsageDelivery } from "./native-usage-delivery.js";
|
||||
|
||||
const runnerRoot = fileURLToPath(new URL("../../../../packages/paperclip-runner/runner/", import.meta.url));
|
||||
const fixtureProfile = process.env.CI ? "release" : "debug";
|
||||
@@ -44,6 +44,89 @@ describe("producer-owned native usage", () => {
|
||||
payload: type === "usage.reported" ? { runDeltaAvailable: true, runDeltaBasis: "run", runDelta: { inputTokens: 120, outputTokens: 10, providerCostUsd: amount } } : {},
|
||||
} as PrpEvent);
|
||||
|
||||
const finalizedSource = async () => {
|
||||
const input = await seed();
|
||||
await db.update(heartbeatRuns).set({ runnerInstanceId: input.runnerInstanceId }).where(eq(heartbeatRuns.id, input.runId));
|
||||
const delivery = await createNativeUsageDelivery(db, input);
|
||||
const report = event(input, 1, "usage.reported"), done = event(input, 2, "turn.completed");
|
||||
await delivery.capture(report);
|
||||
await delivery.capture(done);
|
||||
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, input.runId));
|
||||
expect(await accountRunCost(db, input.runId)).toBe(true);
|
||||
const { billing: _billing, ...source } = input;
|
||||
return { input, source, report, done };
|
||||
};
|
||||
|
||||
it("continues exact finalized source receipts without acquiring ownership or resetting accounting", async () => {
|
||||
const { input, source, report, done } = await finalizedSource();
|
||||
const before = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId));
|
||||
await expect(createNativeUsageDelivery(db, input)).rejects.toThrow("native_usage_run_not_available");
|
||||
const continuation = await createNativeUsageContinuationDelivery(db, source);
|
||||
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId))).toEqual(before);
|
||||
await continuation.capture(report);
|
||||
await continuation.capture(done);
|
||||
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId))).toEqual(before);
|
||||
expect(await db.select().from(runUsageReceipts).where(eq(runUsageReceipts.runId, input.runId))).toHaveLength(2);
|
||||
expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId))).toHaveLength(1);
|
||||
await expect(continuation.capture({ ...report, sourceInstanceId: randomUUID() })).rejects.toThrow("native_usage_source_mismatch");
|
||||
await expect(continuation.capture({ ...report, payload: { ...report.payload as object, runDeltaAvailable: false } }))
|
||||
.rejects.toThrow("native_usage_source_conflict");
|
||||
});
|
||||
|
||||
it("journals an equal finalized source tail using the source billing without repricing", async () => {
|
||||
const { input, source } = await finalizedSource();
|
||||
const before = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId));
|
||||
// Even a structurally supplied extra current-run classification cannot own
|
||||
// the old receipt: the continuation derives billing from saved source data.
|
||||
const withCurrentBilling = { ...source, billing: { provider: "anthropic", biller: "current-grant", billingType: "subscription_included" } };
|
||||
const continuation = await createNativeUsageContinuationDelivery(db, withCurrentBilling);
|
||||
await continuation.capture(event(input, 3, "usage.reported"));
|
||||
const receipts = await db.select().from(runUsageReceipts).where(eq(runUsageReceipts.runId, input.runId));
|
||||
expect(receipts).toHaveLength(3);
|
||||
expect(receipts.find(row => row.sequence === 3)?.receiptJson.receipt).toMatchObject(billing);
|
||||
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId))).toEqual(before);
|
||||
expect((await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId)))[0].costCents).toBe(25);
|
||||
});
|
||||
|
||||
it.each(["tokens", "cost", "incomplete", "unpriceable", "basis"])(
|
||||
"holds a differing finalized source tail before acknowledgement: %s", async change => {
|
||||
const { input, source } = await finalizedSource();
|
||||
const continuation = await createNativeUsageContinuationDelivery(db, source);
|
||||
const late = event(input, 3, "usage.reported");
|
||||
if (change === "incomplete") (late.payload as Record<string, unknown>).runDeltaAvailable = false;
|
||||
if (change === "basis") delete (late.payload as Record<string, unknown>).runDeltaBasis;
|
||||
if (["tokens", "cost", "unpriceable"].includes(change)) (late.payload as Record<string, unknown>).runDelta = {
|
||||
inputTokens: change === "tokens" ? 150 : change === "unpriceable" ? "unknown" : 120,
|
||||
outputTokens: 10, providerCostUsd: change === "cost" ? 0.5 : 0.25,
|
||||
};
|
||||
await expect(continuation.capture(late)).rejects.toThrow("native_usage_after_finalization_requires_reconciliation");
|
||||
await expect(continuation.capture(late)).rejects.toThrow("native_usage_after_finalization_requires_reconciliation");
|
||||
expect(await db.select().from(runUsageReceipts).where(eq(runUsageReceipts.runId, input.runId))).toHaveLength(2);
|
||||
expect((await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId)))[0].costCents).toBe(25);
|
||||
});
|
||||
|
||||
it.each(["company", "run", "runner", "source"])("rejects a changed continuation binding without mutation: %s", async change => {
|
||||
const { input, source } = await finalizedSource();
|
||||
if (change === "source") await db.update(heartbeatRuns).set({ usageJson: sql`${heartbeatRuns.usageJson} || ${JSON.stringify({ accountingReceiptSourceId: randomUUID() })}::jsonb` })
|
||||
.where(eq(heartbeatRuns.id, input.runId));
|
||||
const before = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId));
|
||||
await expect(createNativeUsageContinuationDelivery(db, { ...source,
|
||||
...(change === "company" ? { companyId: randomUUID() } : {}),
|
||||
...(change === "run" ? { runId: randomUUID() } : {}),
|
||||
...(change === "runner" ? { runnerInstanceId: randomUUID() } : {}),
|
||||
})).rejects.toThrow("native_usage_continuation_source_mismatch");
|
||||
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId))).toEqual(before);
|
||||
});
|
||||
|
||||
it("rejects changed immutable source receipt facts during hydration", async () => {
|
||||
const { input, source } = await finalizedSource();
|
||||
const [receipt] = await db.select().from(runUsageReceipts).where(and(eq(runUsageReceipts.runId, input.runId), eq(runUsageReceipts.sequence, 1)));
|
||||
const changed = structuredClone(receipt.receiptJson) as UsageReceiptEnvelope;
|
||||
changed.producerEvent!.usage!.runDelta = { inputTokens: 999, outputTokens: 10 };
|
||||
await db.update(runUsageReceipts).set({ receiptJson: changed }).where(eq(runUsageReceipts.id, receipt.id));
|
||||
await expect(createNativeUsageContinuationDelivery(db, source)).rejects.toThrow("native_usage_source_conflict");
|
||||
});
|
||||
|
||||
it("retains the same Runner report through a failed Postgres receipt and an empty replacement receiver", async () => {
|
||||
const input = await seed();
|
||||
expect(existsSync(usageReceiptSpoolPath())).toBe(false);
|
||||
|
||||
@@ -16,11 +16,13 @@ const uuid = (identity: string) => {
|
||||
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-5${hex.slice(13, 16)}-a${hex.slice(17, 20)}-${hex.slice(20, 32)}`;
|
||||
};
|
||||
|
||||
export async function createNativeUsageDelivery(db: Db, input: {
|
||||
type NativeUsageDeliveryInput = {
|
||||
companyId: string; runId: string; runnerInstanceId: string;
|
||||
provider: NativeExecutionInput["provider"];
|
||||
billing: Pick<AdapterUsageCheckpoint, "provider" | "biller" | "billingType">;
|
||||
}) {
|
||||
};
|
||||
|
||||
export async function createNativeUsageDelivery(db: Db, input: NativeUsageDeliveryInput) {
|
||||
const sourceId = uuid(`${input.runId}:${input.runnerInstanceId}`);
|
||||
const [bound] = await db.update(heartbeatRuns).set({ usageJson: sql`
|
||||
coalesce(${heartbeatRuns.usageJson}, '{}'::jsonb) || '{"accountingReceiptOwner":"native_producer"}'::jsonb ||
|
||||
@@ -29,9 +31,39 @@ export async function createNativeUsageDelivery(db: Db, input: {
|
||||
}).where(and(eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.runId),
|
||||
isNull(heartbeatRuns.costAccountedAt))).returning({ id: heartbeatRuns.id });
|
||||
if (!bound) throw new Error("native_usage_run_not_available");
|
||||
const saved = await db.select({ receipt: runUsageReceipts.receiptJson }).from(runUsageReceipts).where(and(
|
||||
return createNativeUsageReceiver(db, input, sourceId, await loadNativeUsageReceipts(db, input, sourceId));
|
||||
}
|
||||
|
||||
/** Drain the authenticated previous producer's accounting without acquiring or
|
||||
* resetting its ownership. Finalized sources retain normal receipt equality
|
||||
* and reconciliation rules; this grants no business/tool execution authority. */
|
||||
export async function createNativeUsageContinuationDelivery(db: Db, input: Omit<NativeUsageDeliveryInput, "billing">) {
|
||||
const sourceId = uuid(`${input.runId}:${input.runnerInstanceId}`);
|
||||
const [source] = await db.select({ runnerInstanceId: heartbeatRuns.runnerInstanceId, usage: heartbeatRuns.usageJson })
|
||||
.from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.runId)));
|
||||
if (source?.runnerInstanceId !== input.runnerInstanceId || source.usage?.accountingReceiptOwner !== "native_producer"
|
||||
|| source.usage.accountingReceiptSourceId !== sourceId) throw new Error("native_usage_continuation_source_mismatch");
|
||||
const saved = await loadNativeUsageReceipts(db, input, sourceId);
|
||||
if (!saved.length) throw new Error("native_usage_continuation_receipts_missing");
|
||||
const classification = (receipt: AdapterUsageCheckpoint) => ({
|
||||
provider: receipt.provider, biller: receipt.biller, billingType: receipt.billingType,
|
||||
});
|
||||
const receipts = saved.map(row => (row.receipt as UsageReceiptEnvelope).receipt as AdapterUsageCheckpoint);
|
||||
const billing = classification(receipts[0]);
|
||||
if (receipts.some(receipt => receiptFingerprint(classification(receipt)) !== receiptFingerprint(billing)
|
||||
|| receipt.model !== (input.provider.model ?? "unknown"))) throw new Error("native_usage_source_conflict");
|
||||
return createNativeUsageReceiver(db, { ...input, billing }, sourceId, saved, true);
|
||||
}
|
||||
|
||||
function loadNativeUsageReceipts(db: Db, input: Pick<NativeUsageDeliveryInput, "companyId" | "runId">, sourceId: string) {
|
||||
return db.select({ receipt: runUsageReceipts.receiptJson, hash: runUsageReceipts.receiptHash,
|
||||
sequence: runUsageReceipts.sequence }).from(runUsageReceipts).where(and(
|
||||
eq(runUsageReceipts.companyId, input.companyId), eq(runUsageReceipts.runId, input.runId),
|
||||
eq(runUsageReceipts.sourceId, sourceId))).orderBy(asc(runUsageReceipts.sequence));
|
||||
}
|
||||
|
||||
function createNativeUsageReceiver(db: Db, input: NativeUsageDeliveryInput, sourceId: string,
|
||||
saved: Awaited<ReturnType<typeof loadNativeUsageReceipts>>, verifySaved = false) {
|
||||
const accepted = new Map<string, UsageReceiptEnvelope>();
|
||||
const turnAccounting = ["opencode", "acpx"].includes(input.provider.kind) ? createNativeTurnAccounting(input.provider) : null;
|
||||
let observedUsage: Record<string, unknown> | null = null;
|
||||
@@ -60,6 +92,12 @@ export async function createNativeUsageDelivery(db: Db, input: {
|
||||
for (const row of saved) {
|
||||
const envelope = row.receipt as UsageReceiptEnvelope;
|
||||
if (!envelope.producerEvent) throw new Error("native_usage_receipt_missing_producer");
|
||||
if (verifySaved && (envelope.companyId !== input.companyId || envelope.runId !== input.runId
|
||||
|| envelope.sourceId !== sourceId || envelope.sequence !== row.sequence || envelope.adapterType !== "paperclip_runner"
|
||||
|| envelope.id !== uuid(`${input.runId}:${input.runnerInstanceId}:${envelope.producerEvent.sourceEventId}`)
|
||||
|| receiptFingerprint(envelope) !== row.hash || accepted.has(envelope.producerEvent.sourceEventId))) {
|
||||
throw new Error("native_usage_source_conflict");
|
||||
}
|
||||
accepted.set(envelope.producerEvent.sourceEventId, envelope);
|
||||
observe(envelope);
|
||||
}
|
||||
|
||||
@@ -48,6 +48,7 @@ import {
|
||||
import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
|
||||
import { startRecoveryPostgres } from "../../__tests__/helpers/recovery-postgres.js";
|
||||
import { PaperclipControlPlanePort } from "./paperclip-control-plane-port.js";
|
||||
import { nativeSha256 } from "./canonical.js";
|
||||
import { NativeRunCoordinatorStore } from "./native-run-coordinator-store.js";
|
||||
import { finalizeNativeRun, repairCommittedNativeChatResponse } from "./native-run-finalizer.js";
|
||||
import { nativeRuntimeContextFixture } from "./runtime-context.test-fixture.js";
|
||||
@@ -313,6 +314,63 @@ describe("PaperclipControlPlanePort conformance", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("publishes native recovery metadata in the existing snapshot transaction and fences stale generations", async () => {
|
||||
const identity = { ...CONTROL_PLANE_CONFORMANCE_OPEN.identity, runId: randomUUID(), sessionId: randomUUID() };
|
||||
const runnerId = randomUUID();
|
||||
const owner = { ...identity, authorityRunId: identity.runId, bootId: randomUUID(), generation: 1, leaseOwner: "snapshot-owner" };
|
||||
await db.insert(heartbeatRuns).values({ id: identity.runId, companyId: identity.companyId, agentId: identity.agentId,
|
||||
status: "running", runtimeMode: "native", nativeIssueId: identity.issueId, nativeSessionId: identity.sessionId,
|
||||
runnerInstanceId: runnerId, completionContractId: contractId, completionContractSha256: contractSha,
|
||||
runnerProfileJson: { unrelated: "preserve" } });
|
||||
await db.insert(nativeRunFinalizations).values({ runId: identity.runId, companyId: identity.companyId, issueId: identity.issueId,
|
||||
phase: "observed",
|
||||
controllerBootId: owner.bootId, controllerGeneration: owner.generation, leaseOwner: owner.leaseOwner,
|
||||
leaseExpiresAt: new Date(Date.now() + 60_000) });
|
||||
const port = new PaperclipControlPlanePort(db, { ...identity, completionContractId: contractId,
|
||||
completionContractSha256: contractSha, sourceInstanceId: runnerId, controlPlaneSourceInstanceId: "snapshot-control" }, {
|
||||
nativeRecovery: { owner, compatibility: { configDigest: `sha256:${"a".repeat(64)}`, processConfigDigest: null } },
|
||||
});
|
||||
await port.openRun({ identity, backendKind: "mock", sourceInstanceId: runnerId });
|
||||
const snapshot = { backendKind: "mock" as const, sessionId: identity.sessionId, identity };
|
||||
const statements: string[] = [];
|
||||
const previousDebug = db.$client.options.debug;
|
||||
db.$client.options.debug = (_connection, statement) => { statements.push(statement); };
|
||||
try { await port.checkpointSession(snapshot); }
|
||||
finally { db.$client.options.debug = previousDebug; }
|
||||
expect(statements.filter(statement => /^begin\b/i.test(statement.trim()))).toHaveLength(1);
|
||||
expect(statements.filter(statement => /^update\s+"heartbeat_runs"/i.test(statement.trim()))).toHaveLength(1);
|
||||
expect(statements.filter(statement => /^(?:with|select)\b/i.test(statement.trim()))).toHaveLength(1);
|
||||
const [saved] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, identity.runId));
|
||||
expect(saved.runnerProfileJson).toMatchObject({ unrelated: "preserve", sessionCheckpoint: snapshot,
|
||||
nativeControllerRecovery: { schema: "paperclip.native-controller-recovery.v1", snapshotSha256: nativeSha256(saved.runnerProfileJson?.sessionCheckpoint),
|
||||
checkpoint: null, processConfigDigest: null } });
|
||||
// Missing PRP is an explicit incomplete publication, never fabricated authority.
|
||||
await db.update(nativeRunFinalizations).set({ controllerGeneration: 2 }).where(eq(nativeRunFinalizations.runId, identity.runId));
|
||||
await expect(port.checkpointSession({ ...snapshot, providerSessionId: "stale" })).rejects.toThrow("checkpoint_owner_lost");
|
||||
await expect(port.completeRun({ result: CONTROL_PLANE_CONFORMANCE_RESULT, terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL }))
|
||||
.rejects.toThrow("checkpoint_owner_lost");
|
||||
const [unchanged] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, identity.runId));
|
||||
expect(unchanged.runnerProfileJson).toEqual(saved.runnerProfileJson);
|
||||
// A valid coordinator lock does not certify a lease that expires while the
|
||||
// subsequent run-row lock waits. Recheck time at the actual write boundary.
|
||||
await db.update(nativeRunFinalizations).set({ controllerGeneration: 1, leaseExpiresAt: new Date(Date.now() + 150) })
|
||||
.where(eq(nativeRunFinalizations.runId, identity.runId));
|
||||
let locked!: () => void, release!: () => void;
|
||||
const acquired = new Promise<void>(resolve => { locked = resolve; });
|
||||
const held = new Promise<void>(resolve => { release = resolve; });
|
||||
const blocker = db.transaction(async tx => {
|
||||
await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(eq(heartbeatRuns.id, identity.runId)).for("update");
|
||||
locked();
|
||||
await held;
|
||||
});
|
||||
await acquired;
|
||||
const waiting = port.checkpointSession(snapshot);
|
||||
const rejected = expect(waiting).rejects.toThrow("checkpoint_owner_lost");
|
||||
try { await new Promise(resolve => setTimeout(resolve, 200)); }
|
||||
finally { release(); await blocker; }
|
||||
await rejected;
|
||||
});
|
||||
|
||||
it("permits only read-only admission while an existing startup drains", async () => {
|
||||
const identity = { ...CONTROL_PLANE_CONFORMANCE_OPEN.identity, runId: randomUUID(), sessionId: randomUUID() };
|
||||
const runnerId = randomUUID();
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { notifyNativeResultCommit } from "../native-result-commit-notifications.js";
|
||||
import { notifyNativeWorkspaceFinalization } from "../native-workspace-finalization-notifications.js";
|
||||
import { createAgentIdentityRedactor } from "../agent-identity-redaction.js";
|
||||
import { and, asc, eq, gt, or } from "drizzle-orm";
|
||||
import { and, asc, eq, gt, or, sql } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import {
|
||||
heartbeatRunEvents,
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
issues,
|
||||
nativeRunFinalizations,
|
||||
nativeRunResults,
|
||||
nativeRunnerCheckpoints,
|
||||
} from "@paperclipai/db";
|
||||
import type {
|
||||
CompleteControlPlaneRunInput,
|
||||
@@ -30,6 +31,10 @@ import { appendHeartbeatRunEvent } from "../heartbeat-run-events.js";
|
||||
import { publishLiveEvent } from "../live-events.js";
|
||||
import { nativeSha256 } from "./canonical.js";
|
||||
import { redactRunnerApiEvent, redactRunnerApiPayloads } from "./runner-api-log-redaction.js";
|
||||
import { createNativeControllerRecovery, NATIVE_CONTROLLER_RECOVERY_MAX_BYTES, type NativeControllerCompatibility } from "./native-controller-recovery.js";
|
||||
import { eligibleNativeTaskSessionPublication, publishNativeTaskSessionInTransaction, NATIVE_TASK_SESSION_PUBLICATION_KEY,
|
||||
type NativeTaskSessionPublicationContext, type NativeTaskSessionPublicationIssue } from "./native-task-session-publication.js";
|
||||
import { lockNativeRunnerCheckpointOwner, nativeRunnerCheckpointOwnerLock, validateNativeRunnerCheckpointOwner, inspectNativeRunnerUpgradeHeader, type NativeRunnerCheckpointOwner } from "./native-runner-checkpoint.js";
|
||||
|
||||
export interface PaperclipControlPlaneBinding {
|
||||
companyId: string;
|
||||
@@ -90,6 +95,9 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
|
||||
readonly #onEventPersistence?: (event: PrpEvent, startedAtMs: number, endedAtMs: number) => Promise<void>;
|
||||
readonly #onCommittedEvent?: (event: PrpEvent, cursor: number) => Promise<void>;
|
||||
readonly #onDuplicateEvent?: (event: PrpEvent, cursor: number) => Promise<void>;
|
||||
readonly #nativeRecovery?: { compatibility: NativeControllerCompatibility; owner: NativeRunnerCheckpointOwner };
|
||||
readonly #taskSessionPublication?: NativeTaskSessionPublicationContext;
|
||||
readonly #taskSessionPublicationOwner?: NativeRunnerCheckpointOwner;
|
||||
|
||||
constructor(
|
||||
db: Db,
|
||||
@@ -105,6 +113,9 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
|
||||
onEventPersistence?: (event: PrpEvent, startedAtMs: number, endedAtMs: number) => Promise<void>;
|
||||
onCommittedEvent?: (event: PrpEvent, cursor: number) => Promise<void>;
|
||||
onDuplicateEvent?: (event: PrpEvent, cursor: number) => Promise<void>;
|
||||
nativeRecovery?: { compatibility: NativeControllerCompatibility; owner: NativeRunnerCheckpointOwner };
|
||||
taskSessionPublication?: NativeTaskSessionPublicationContext;
|
||||
taskSessionPublicationOwner?: NativeRunnerCheckpointOwner;
|
||||
} = {},
|
||||
) {
|
||||
this.#db = db;
|
||||
@@ -116,6 +127,9 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
|
||||
this.#onEventPersistence = options.onEventPersistence;
|
||||
this.#onResultCommitted = options.onResultCommitted;
|
||||
this.#onDuplicateEvent = options.onDuplicateEvent;
|
||||
this.#nativeRecovery = options.nativeRecovery;
|
||||
this.#taskSessionPublication = options.taskSessionPublication ? structuredClone(options.taskSessionPublication) : undefined;
|
||||
this.#taskSessionPublicationOwner = options.taskSessionPublicationOwner;
|
||||
}
|
||||
|
||||
#assertActive(options?: { signal: AbortSignal }): void {
|
||||
@@ -123,7 +137,9 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
|
||||
options?.signal.throwIfAborted();
|
||||
}
|
||||
|
||||
#matchesPersistedBinding(run: typeof heartbeatRuns.$inferSelect): boolean {
|
||||
#matchesPersistedBinding(run: Pick<typeof heartbeatRuns.$inferSelect,
|
||||
"companyId" | "agentId" | "runtimeMode" | "nativeIssueId" | "nativeSessionId" |
|
||||
"runnerInstanceId" | "completionContractId" | "completionContractSha256">): boolean {
|
||||
return run.companyId === this.#binding.companyId
|
||||
&& run.agentId === this.#binding.agentId
|
||||
&& run.runtimeMode === "native"
|
||||
@@ -200,19 +216,105 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
|
||||
|| identity.agentId !== this.#binding.agentId
|
||||
|| this.#sessionId !== identity.sessionId
|
||||
) throw new Error("native_session_checkpoint_binding_mismatch");
|
||||
const publication = this.#taskSessionPublication && eligibleNativeTaskSessionPublication(snapshot)
|
||||
? this.#taskSessionPublication : null;
|
||||
if (publication && !this.#taskSessionPublicationOwner) throw new Error("native_task_session_publication_owner_required");
|
||||
await this.#db.transaction(async (tx) => {
|
||||
const run = await tx.select().from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, this.#binding.runId)).for("update").limit(1)
|
||||
const recovery = this.#nativeRecovery;
|
||||
const owner = recovery?.owner ?? (publication ? this.#taskSessionPublicationOwner : undefined);
|
||||
if (publication && recovery && nativeSha256(recovery.owner) !== nativeSha256(this.#taskSessionPublicationOwner)) {
|
||||
throw new Error("native_task_session_publication_owner_mismatch");
|
||||
}
|
||||
const ownerLock = owner ? nativeRunnerCheckpointOwnerLock(tx, owner) : null;
|
||||
// NO KEY UPDATE serializes reset/admission fields while remaining compatible
|
||||
// with completeRun's parent KEY SHARE. Keep coordinator → issue → run order.
|
||||
const publicationIssue = publication ? tx.$with("publication_issue").as(tx.select({
|
||||
id: issues.id, companyId: issues.companyId, executionRunId: issues.executionRunId,
|
||||
conversationAgentId: issues.conversationAgentId, conversationSessionGeneration: issues.conversationSessionGeneration,
|
||||
}).from(issues).where(and(eq(issues.id, this.#binding.issueId), eq(issues.companyId, this.#binding.companyId),
|
||||
ownerLock ? sql`exists (select 1 from ${ownerLock})` : undefined)).for("no key update")) : null;
|
||||
const lockedRun = tx.$with("snapshot_run").as(tx.select({
|
||||
id: heartbeatRuns.id, status: heartbeatRuns.status,
|
||||
contextSnapshot: (publication ? sql<Record<string, unknown>>`jsonb_build_object('conversationSessionGeneration', ${heartbeatRuns.contextSnapshot}->'conversationSessionGeneration')` : sql<null>`null`).as("publication_context"),
|
||||
resultJson: (publication ? sql<Record<string, unknown>>`jsonb_build_object('nativeCancellation', ${heartbeatRuns.resultJson}->'nativeCancellation',
|
||||
'cancellation', ${heartbeatRuns.resultJson}->'cancellation', 'startupCancellation', ${heartbeatRuns.resultJson}->'startupCancellation')` : sql<null>`null`).as("publication_cancellation"),
|
||||
startedAt: heartbeatRuns.startedAt, createdAt: heartbeatRuns.createdAt,
|
||||
companyId: heartbeatRuns.companyId, agentId: heartbeatRuns.agentId,
|
||||
runtimeMode: heartbeatRuns.runtimeMode, nativeIssueId: heartbeatRuns.nativeIssueId,
|
||||
nativeSessionId: heartbeatRuns.nativeSessionId, runnerInstanceId: heartbeatRuns.runnerInstanceId,
|
||||
completionContractId: heartbeatRuns.completionContractId,
|
||||
completionContractSha256: heartbeatRuns.completionContractSha256,
|
||||
runnerProfileJson: heartbeatRuns.runnerProfileJson,
|
||||
}).from(heartbeatRuns)
|
||||
.where(and(eq(heartbeatRuns.id, this.#binding.runId), ownerLock
|
||||
? sql`exists (select 1 from ${ownerLock})` : undefined,
|
||||
publicationIssue ? sql`exists (select 1 from ${publicationIssue})` : undefined)).for("update"));
|
||||
// Project the locking CTE's tuples, including any update committed while
|
||||
// the lock waited. A second scan of heartbeat_runs could see an old MVCC tuple.
|
||||
// The run CTE depends on the coordinator CTE: both locks precede the
|
||||
// outer clock sample, including when either lock waits.
|
||||
const selected = await tx.with(...(ownerLock ? [ownerLock] : []), ...(publicationIssue ? [publicationIssue] : []), lockedRun).select({ run: {
|
||||
id: lockedRun.id, status: lockedRun.status, contextSnapshot: lockedRun.contextSnapshot, resultJson: lockedRun.resultJson,
|
||||
startedAt: lockedRun.startedAt, createdAt: lockedRun.createdAt,
|
||||
companyId: lockedRun.companyId, agentId: lockedRun.agentId,
|
||||
runtimeMode: lockedRun.runtimeMode, nativeIssueId: lockedRun.nativeIssueId,
|
||||
nativeSessionId: lockedRun.nativeSessionId, runnerInstanceId: lockedRun.runnerInstanceId,
|
||||
completionContractId: lockedRun.completionContractId,
|
||||
completionContractSha256: lockedRun.completionContractSha256,
|
||||
runnerProfileJson: lockedRun.runnerProfileJson,
|
||||
}, now: sql<Date>`clock_timestamp()`, ...(ownerLock ? {
|
||||
leaseOwner: sql<string | null>`(select ${ownerLock.leaseOwner} from ${ownerLock})`,
|
||||
leaseExpiresAt: sql<Date | null>`(select ${ownerLock.leaseExpiresAt} from ${ownerLock})`.mapWith(nativeRunFinalizations.leaseExpiresAt),
|
||||
resultId: sql<string | null>`(select ${ownerLock.resultId} from ${ownerLock})`,
|
||||
} : {}), ...(recovery ? {
|
||||
checkpointHeader: sql<Record<string, unknown> | null>`(select material from ${nativeRunnerCheckpoints}
|
||||
where ${nativeRunnerCheckpoints.runId} = ${this.#binding.runId}
|
||||
and ${nativeRunnerCheckpoints.companyId} = ${this.#binding.companyId}
|
||||
and ${nativeRunnerCheckpoints.issueId} = ${this.#binding.issueId}
|
||||
and ${nativeRunnerCheckpoints.part} = 'header')`,
|
||||
} : {}), ...(publicationIssue ? { publicationIssue: sql<NativeTaskSessionPublicationIssue>`(select jsonb_build_object(
|
||||
'id', ${publicationIssue.id}, 'companyId', ${publicationIssue.companyId}, 'executionRunId', ${publicationIssue.executionRunId},
|
||||
'conversationAgentId', ${publicationIssue.conversationAgentId}, 'conversationSessionGeneration', ${publicationIssue.conversationSessionGeneration}) from ${publicationIssue})` } : {}),
|
||||
}).from(lockedRun).limit(1)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
const run = selected?.run;
|
||||
this.#assertActive(options);
|
||||
if (owner && !selected) throw new Error("native_runner_checkpoint_owner_lost");
|
||||
if (!run || !this.#matchesPersistedBinding(run)) {
|
||||
throw new Error("native_session_checkpoint_binding_mismatch");
|
||||
}
|
||||
await tx.update(heartbeatRuns).set({
|
||||
runnerProfileJson: {
|
||||
// The run row lock may have waited beyond our lease. Generation cannot
|
||||
// change while we hold the coordinator, but wall-clock expiry still can.
|
||||
if (owner && selected) await validateNativeRunnerCheckpointOwner(tx, owner,
|
||||
{ leaseOwner: selected.leaseOwner ?? null, leaseExpiresAt: selected.leaseExpiresAt ?? null,
|
||||
resultId: selected.resultId ?? null, now: selected.now }, !snapshot.semanticResult);
|
||||
const header = recovery && selected?.checkpointHeader
|
||||
? inspectNativeRunnerUpgradeHeader(this.#binding, selected.checkpointHeader) : null;
|
||||
if (header && (header.normalizedSessionId !== identity.sessionId || header.runnerInstanceId !== this.#binding.sourceInstanceId)) {
|
||||
throw new Error("native_session_checkpoint_prp_binding_mismatch");
|
||||
}
|
||||
const profile: Record<string, unknown> = {
|
||||
...record(run.runnerProfileJson),
|
||||
sessionCheckpoint: structuredClone(snapshot) as unknown as Record<string, unknown>,
|
||||
},
|
||||
...(recovery ? { nativeControllerRecovery: createNativeControllerRecovery(snapshot, recovery.compatibility,
|
||||
header ? { runId: this.#binding.runId, runnerInstanceId: header.runnerInstanceId,
|
||||
normalizedSessionId: header.normalizedSessionId } : null) } : {}),
|
||||
};
|
||||
// Driver/live snapshots may already describe a terminal result before
|
||||
// completeRun accepts it. Persist that observation without a handled
|
||||
// receipt; the existing post-acceptance checkpoint publishes the pointer.
|
||||
if (publication && selected?.resultId) {
|
||||
const receipt = await publishNativeTaskSessionInTransaction(tx, { binding: this.#binding, context: publication,
|
||||
snapshot, run, issue: selected.publicationIssue ?? null, resultId: selected.resultId ?? null,
|
||||
recoveryReady: !recovery || header !== null });
|
||||
profile[NATIVE_TASK_SESSION_PUBLICATION_KEY] = receipt;
|
||||
if (Buffer.byteLength(JSON.stringify({ nativeControllerRecovery: profile.nativeControllerRecovery,
|
||||
[NATIVE_TASK_SESSION_PUBLICATION_KEY]: receipt })) > NATIVE_CONTROLLER_RECOVERY_MAX_BYTES) {
|
||||
throw new Error("native_task_session_publication_metadata_too_large");
|
||||
}
|
||||
}
|
||||
await tx.update(heartbeatRuns).set({
|
||||
runnerProfileJson: profile,
|
||||
sessionIdAfter: snapshot.providerSessionId ?? snapshot.sessionId,
|
||||
nativePhase: snapshot.semanticResult ? "workspace_finalizing" : "observed",
|
||||
nativePhaseUpdatedAt: new Date(),
|
||||
@@ -386,9 +488,12 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
|
||||
});
|
||||
|
||||
const insertedResult = await this.#db.transaction(async (tx) => {
|
||||
// Result insertion checks the task foreign key. Acquire that parent lock
|
||||
// before the run, matching task mutations that subsequently update a run.
|
||||
// Otherwise concurrent chat/status writes can deadlock after generation.
|
||||
// Status commit and snapshot publication lock coordinator → issue → run.
|
||||
// A replay must follow that order too: holding the issue while waiting
|
||||
// for the coordinator deadlocks a status commit's issue UPDATE lock.
|
||||
const recovery = this.#nativeRecovery;
|
||||
const owner = recovery ? await lockNativeRunnerCheckpointOwner(tx, recovery.owner) : null;
|
||||
// Result insertion checks the task FK. Keep its parent lock before run.
|
||||
await tx.select({ id: issues.id }).from(issues)
|
||||
.where(and(eq(issues.id, this.#binding.issueId), eq(issues.companyId, this.#binding.companyId)))
|
||||
.for("key share");
|
||||
@@ -399,6 +504,10 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
|
||||
if (!run || !this.#matchesPersistedBinding(run)) {
|
||||
throw new Error("native_result_binding_mismatch");
|
||||
}
|
||||
if (recovery && owner) {
|
||||
const [clock] = await tx.execute<{ now: Date }>(sql`select clock_timestamp() as now`);
|
||||
await validateNativeRunnerCheckpointOwner(tx, recovery.owner, { ...owner, now: clock.now }, false);
|
||||
}
|
||||
|
||||
const callerConditions = [eq(nativeRunResults.serverFingerprint, serverFingerprint)];
|
||||
if (value.callerResultId) callerConditions.push(eq(nativeRunResults.callerResultId, value.callerResultId));
|
||||
@@ -411,6 +520,7 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
|
||||
if (existing.canonicalSha256 !== canonicalSha256) throw new Error("structured_result_replay_conflict");
|
||||
return false;
|
||||
}
|
||||
if (owner?.resultId) throw new Error("structured_result_replay_conflict");
|
||||
const resultJson = {
|
||||
result: validated.result,
|
||||
terminal: value.terminal,
|
||||
|
||||
@@ -120,7 +120,7 @@ describe("PaperclipRunnerToolAuthority", () => {
|
||||
issueId,
|
||||
runId,
|
||||
});
|
||||
expect(authority.definitions()).toHaveLength(50);
|
||||
expect(authority.definitions()).toHaveLength(51);
|
||||
expect(authority.definitions().map(tool => tool.name)).toEqual(expect.arrayContaining([
|
||||
"get_workspace", "list_workspaces", "select_workspace", "prepare_repository",
|
||||
]));
|
||||
|
||||
@@ -28,6 +28,7 @@ import { createReadStream } from "node:fs";
|
||||
import { publicChatTaskUrl } from "../chat-task-url.js";
|
||||
import { agentCommentaryToolInputSchema } from "@paperclipai/shared";
|
||||
import { submitAgentCommentary } from "../agent-commentary.js";
|
||||
import { sendAgentChatMessage } from "../agent-chat-messages.js";
|
||||
import type { createAssignedMcpTools } from "./assigned-mcp-tools.js";
|
||||
import { captureRunnerToolResult, fitRunnerToolResultPage, READ_TOOL_RESULT, READ_TOOL_RESULT_DEFINITION, readToolResultSchema, RUNNER_TOOL_RESULT_MAX_BYTES } from "./runner-tool-results.js";
|
||||
import { assertAssignableAgent } from "../agent-assignability.js";
|
||||
@@ -126,6 +127,7 @@ import {
|
||||
|
||||
const EXTERNAL_WORKSPACE_OPERATIONS = new Set(["get_workspace", "list_workspaces", "select_workspace", "prepare_repository", "register_deliverable"]);
|
||||
const IMPLEMENTED_OPERATIONS = new Set([
|
||||
"send_chat_message",
|
||||
"submit_complaint", "submit_suggestion",
|
||||
"read_agent_instructions", "update_agent_instructions", "get_agent_instruction_history", "restore_agent_instructions",
|
||||
"search_api", "call_api", "hire_agent",
|
||||
@@ -434,6 +436,7 @@ export class PaperclipRunnerToolAuthority {
|
||||
// retry, without ever falling through to an uncertain mutation.
|
||||
let recovered: unknown;
|
||||
if (call.tool === "set_task_title") recovered = await this.#setTaskTitle(input, true);
|
||||
else if (call.tool === "send_chat_message") recovered = await this.#sendChatMessage(input, this.#privacyActor(run), true);
|
||||
else if (call.tool === "submit_complaint" || call.tool === "submit_suggestion") recovered = await this.#submitCommentary(call.tool, input, true);
|
||||
else if (call.tool === "create_task") recovered = await this.#createTask(input, null, true);
|
||||
else if (call.tool === "request_human_input") recovered = await this.#requestHumanInput(input, null, true);
|
||||
@@ -959,6 +962,7 @@ export class PaperclipRunnerToolAuthority {
|
||||
return { approval, tasks: tasks.filter(row => !readable || readable.has(row.issue.id)).map(row => row.issue) };
|
||||
}
|
||||
case "report_progress": return this.#reportProgress(input);
|
||||
case "send_chat_message": return this.#sendChatMessage(input, this.#privacyActor(context.run));
|
||||
case "request_human_input": return this.#requestHumanInput(input,
|
||||
(await captureRunIdentity(this.db, this.binding, { readScope: "identity" })).context?.id ?? null);
|
||||
case "create_task": return this.#createTask(input,
|
||||
@@ -1486,6 +1490,14 @@ export class PaperclipRunnerToolAuthority {
|
||||
return { interactionId: review.interaction.id, status: expectedStatus };
|
||||
}
|
||||
|
||||
async #sendChatMessage(input: Record<string, unknown>, actor: AuthorizationActor, receiptOnly = false) {
|
||||
const result = await sendAgentChatMessage(this.db, { ...this.binding, keyScope: actor.keyScope,
|
||||
onBehalfOfUserId: actor.onBehalfOfUserId, authorizingUserId: actor.authorizingUserId, source: actor.source }, input, {
|
||||
authorizeNative: tx => this.#lockAuthorizedMutationContext(tx), receiptOnly,
|
||||
});
|
||||
return result ?? undefined;
|
||||
}
|
||||
|
||||
async #reportProgress(input: Record<string, unknown>): Promise<unknown> {
|
||||
const body = typeof input.body === "string" ? input.body.trim() : "";
|
||||
const idempotencyKey = typeof input.idempotencyKey === "string" ? input.idempotencyKey.trim() : "";
|
||||
|
||||
@@ -91,6 +91,23 @@ it.skipIf(!tar || !hasZstd)("reuses only byte-matched image files, keeps modes,
|
||||
expect(await fs.readlink(path.join(targetPath, "tool"))).toBe("bin/tool");
|
||||
});
|
||||
|
||||
it.skipIf(!tar || !hasZstd)("stages a complete layered pack into a fresh nested runtime directory", async () => {
|
||||
const { root, pack, archive } = await layeredFixture();
|
||||
const remote = await runner(root);
|
||||
const workspace = path.join(root, "workspace");
|
||||
await fs.mkdir(workspace);
|
||||
const targetPath = path.join(workspace, ".paperclip-runtime", "paperclip-runner", "provider-pack");
|
||||
await expect(fs.lstat(path.dirname(targetPath))).rejects.toMatchObject({ code: "ENOENT" });
|
||||
expect(await stageProviderPackArchive({ archive, runner: remote, targetPath, reuseRoots: [pack] })).toBe(true);
|
||||
expect(await fs.readFile(path.join(targetPath, "bin/large"))).toEqual(await fs.readFile(path.join(pack, "bin/large")));
|
||||
expect(await fs.readFile(path.join(targetPath, "provider-pack.json"))).toEqual(await fs.readFile(path.join(pack, "provider-pack.json")));
|
||||
expect(await fs.readlink(path.join(targetPath, "tool"))).toBe("bin/tool");
|
||||
for (const directory of [targetPath, path.dirname(targetPath), path.dirname(path.dirname(targetPath))]) {
|
||||
expect((await fs.stat(directory)).mode & 0o777).toBe(0o700);
|
||||
}
|
||||
expect((await fs.stat(path.join(targetPath, "bin/large"))).ino).not.toBe((await fs.stat(path.join(pack, "bin/large"))).ino);
|
||||
});
|
||||
|
||||
it.skipIf(!tar || !hasZstd).each(["missing", "wrong-digest", "symlink", "ancestor-symlink"])("transfers a large layer when the image file is %s", async reason => {
|
||||
const { root, pack, archive } = await layeredFixture();
|
||||
const remote = await runner(root);
|
||||
@@ -135,13 +152,18 @@ it.skipIf(!tar || !hasZstd)("rejects a corrupt missing layer before transfer and
|
||||
expect(remote.syncIn).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.skipIf(!tar || !hasZstd)("does not follow a layered target symlink or accept invalid reuse receipts", async () => {
|
||||
it.skipIf(!tar || !hasZstd).each(["directory", "file", "symlink"])("rejects an existing layered target %s or invalid reuse receipts", async kind => {
|
||||
const { root, archive } = await layeredFixture();
|
||||
const remote = await runner(root);
|
||||
const targetPath = path.join(root, "target"), other = path.join(root, "other");
|
||||
await fs.mkdir(other); await fs.symlink(other, targetPath);
|
||||
await fs.mkdir(other);
|
||||
if (kind === "directory") await fs.mkdir(targetPath);
|
||||
else if (kind === "file") await fs.writeFile(targetPath, "untouched");
|
||||
else await fs.symlink(other, targetPath);
|
||||
await expect(stageProviderPackArchive({ archive, runner: remote, targetPath })).rejects.toThrow("runner_remote_provider_archive_reuse_failed");
|
||||
expect(await fs.readdir(other)).toEqual([]);
|
||||
if (kind === "directory") expect(await fs.readdir(targetPath)).toEqual([]);
|
||||
if (kind === "file") expect(await fs.readFile(targetPath, "utf8")).toBe("untouched");
|
||||
remote.execute.mockResolvedValue({ exitCode: 0, timedOut: false, stdout: "[999999]", stderr: "", signal: null, pid: null, startedAt: null });
|
||||
await expect(stageProviderPackArchive({ archive, runner: remote, targetPath: `${targetPath}-2` })).rejects.toThrow("runner_remote_provider_archive_reuse_failed");
|
||||
expect(remote.syncIn).not.toHaveBeenCalled();
|
||||
|
||||
@@ -70,7 +70,8 @@ const reuseScript = `
|
||||
(async()=>{
|
||||
const fs=require('node:fs'),p=require('node:path'),c=require('node:crypto');
|
||||
const spec=JSON.parse(Buffer.from(process.argv[1],'base64').toString('utf8'));
|
||||
const f=fs.promises;await f.mkdir(spec.target,{mode:448});
|
||||
const f=fs.promises;await f.mkdir(p.dirname(spec.target),{recursive:true,mode:448});
|
||||
await f.mkdir(spec.target,{mode:448});
|
||||
const missing=[];
|
||||
const regularPath=async(root,relative)=>{
|
||||
let current=root;if(!(await f.lstat(current)).isDirectory())return false;
|
||||
|
||||
@@ -18,7 +18,7 @@ import {
|
||||
AGENTCORE_QUALIFIED_MODEL,
|
||||
CLAUDE_MANAGED_QUALIFIED_MODEL,
|
||||
} from "../provider-profile-qualification.js";
|
||||
import { resolveAcpxQualification, resolveAntigravityCandidateAdmission, type AcpxQualificationCandidate } from "./acpx-qualification.js";
|
||||
import { resolveAcpxQualification, resolveAntigravityCandidateAdmission, resolveKimiCandidateAdmission, type AcpxQualificationCandidate } from "./acpx-qualification.js";
|
||||
import { compatibleCodexModel } from "./codex-model-fallback.js";
|
||||
|
||||
export const QUALIFIED_OPENCODE_RUNNER_VERSION = "1.18.34" as const;
|
||||
@@ -34,6 +34,7 @@ export const DEFAULT_ACPX_RUNNER_MODELS = {
|
||||
cursor: null,
|
||||
copilot: null,
|
||||
pi: null,
|
||||
kimi: null,
|
||||
} as const;
|
||||
|
||||
export type QualifiedPaperclipRunnerAcpxAgent =
|
||||
@@ -564,6 +565,15 @@ export function resolvePaperclipRunnerProviderProfile(
|
||||
throw new PaperclipRunnerProviderProfileError("paperclip_runner_acpx_qualification_invalid", "Antigravity candidate admission requires the authorized company, exact model and runtime identity.");
|
||||
}
|
||||
}
|
||||
if (pendingAcpxProfile.value === "kimi") {
|
||||
try {
|
||||
if (resolveKimiCandidateAdmission({ companyId: context.companyId, model }, process.env)) {
|
||||
return { provider: "acpx", backend: "acpx_runtime", model, acpxAgent: "kimi" };
|
||||
}
|
||||
} catch {
|
||||
throw new PaperclipRunnerProviderProfileError("paperclip_runner_acpx_qualification_invalid", "Kimi candidate admission requires the authorized company, exact model, target and runtime identity.");
|
||||
}
|
||||
}
|
||||
// Qualification authority belongs to the isolated server operator. Never
|
||||
// read this allowlist from adapter config, credential refs, or run env.
|
||||
let qualification: AcpxQualificationCandidate | undefined;
|
||||
@@ -577,10 +587,10 @@ export function resolvePaperclipRunnerProviderProfile(
|
||||
}
|
||||
throw new PaperclipRunnerProviderProfileError("paperclip_runner_acpx_agent_unavailable", `${pendingAcpxProfile.label} is awaiting native Runner acceptance. Its profile is not enabled for production runs.`);
|
||||
}
|
||||
if ((acpxAgent === "cursor" || acpxAgent === "pi") && !model) {
|
||||
if ((acpxAgent === "cursor" || acpxAgent === "pi" || acpxAgent === "kimi") && !model) {
|
||||
throw new PaperclipRunnerProviderProfileError("paperclip_runner_acpx_model_required", "This ACPX agent requires an explicit model ID; there is no default model.");
|
||||
}
|
||||
if (acpxAgent !== "claude" && acpxAgent !== "codex" && acpxAgent !== "grok" && acpxAgent !== "cursor" && acpxAgent !== "copilot" && acpxAgent !== "pi") {
|
||||
if (acpxAgent !== "claude" && acpxAgent !== "codex" && acpxAgent !== "grok" && acpxAgent !== "cursor" && acpxAgent !== "copilot" && acpxAgent !== "pi" && acpxAgent !== "kimi") {
|
||||
throw new PaperclipRunnerProviderProfileError(
|
||||
"paperclip_runner_acpx_agent_unavailable",
|
||||
"Paperclip Runner ACPX requires a qualified agent profile.",
|
||||
|
||||
@@ -10,7 +10,7 @@ export function shouldUseBundledAcpxImageAssets(input: {
|
||||
}): boolean {
|
||||
return input.requiresRemoteProviderPack && !input.configuredProviderPackRoot &&
|
||||
input.provider.kind === "acpx" &&
|
||||
["cursor", "copilot"].includes(input.provider.agent);
|
||||
["cursor", "copilot", "kimi"].includes(input.provider.agent);
|
||||
}
|
||||
|
||||
type VerificationCommandResult = {
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { isSettledRemoteWarmControlState } from "./remote-warm-session-recovery.js";
|
||||
|
||||
const identity = { runId: "run", normalizedSessionId: "session", runnerInstanceId: "runner", environmentLeaseId: "lease", turnId: "turn", itemId: "item" };
|
||||
const terminal = { eventType: "run.terminal", envelope: { ...identity } };
|
||||
const goal = { schema: "paperclip.session_goal.snapshot.v1", goal: null, workingNow: false };
|
||||
const notification = { schema: "paperclip.prp.event.v1", eventType: "session.goal.snapshot", runId: identity.runId,
|
||||
normalizedSessionId: identity.normalizedSessionId, sourceInstanceId: "runner", sourceKind: "runner", payload: goal };
|
||||
const snapshot = { eventType: "session.goal.snapshot", envelope: { ...identity, payload: notification } };
|
||||
const state = (events: unknown[]) => ({ identity, commands: [{ status: "completed" }], committedEvents: events });
|
||||
|
||||
describe("settled remote warm control evidence", () => {
|
||||
it.each(["paperclip.prp.event.v1", "paperclip.prp.event.v2"])("accepts the terminal and an authenticated %s idle goal-read suffix without changing its order", schema => {
|
||||
const suffix = { ...snapshot, envelope: { ...snapshot.envelope, payload: { ...notification, schema } } };
|
||||
const control = state([terminal, suffix]);
|
||||
expect(isSettledRemoteWarmControlState(control, identity)).toBe(true);
|
||||
expect(control.committedEvents).toEqual([terminal, suffix]);
|
||||
expect(isSettledRemoteWarmControlState(state([terminal]), identity)).toBe(true);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["active goal", { ...snapshot, envelope: { ...snapshot.envelope, payload: { ...notification, payload: { ...goal, goal: { status: "active" } } } } }],
|
||||
["provider work", { ...snapshot, envelope: { ...snapshot.envelope, payload: { ...notification, payload: { ...goal, workingNow: true } } } }],
|
||||
["unknown event", { ...snapshot, eventType: "unknown" }],
|
||||
["unknown protocol version", { ...snapshot, envelope: { ...snapshot.envelope, payload: { ...notification, schema: "paperclip.prp.event.v3" } } }],
|
||||
["new turn", { ...snapshot, eventType: "turn.started" }],
|
||||
["foreign run", { ...snapshot, envelope: { ...snapshot.envelope, runId: "other" } }],
|
||||
["foreign session", { ...snapshot, envelope: { ...snapshot.envelope, normalizedSessionId: "other" } }],
|
||||
["foreign source", { ...snapshot, envelope: { ...snapshot.envelope, payload: { ...notification, sourceInstanceId: "other" } } }],
|
||||
["foreign lease", { ...snapshot, envelope: { ...snapshot.envelope, environmentLeaseId: "other" } }],
|
||||
["foreign notification", { ...snapshot, envelope: { ...snapshot.envelope, payload: { ...notification, runId: "other" } } }],
|
||||
["missing payload", { ...snapshot, envelope: { ...identity } }],
|
||||
])("holds a %s suffix", (_name, suffix) => {
|
||||
expect(isSettledRemoteWarmControlState(state([terminal, suffix]), identity)).toBe(false);
|
||||
});
|
||||
|
||||
it("holds missing terminals, pending commands and mixed core identities", () => {
|
||||
expect(isSettledRemoteWarmControlState(state([snapshot]), identity)).toBe(false);
|
||||
expect(isSettledRemoteWarmControlState({ ...state([terminal, snapshot]), commands: [{ status: "dispatched" }] }, identity)).toBe(false);
|
||||
expect(isSettledRemoteWarmControlState({ ...state([terminal, snapshot]), identity: { ...identity, runnerInstanceId: "other" } }, identity)).toBe(false);
|
||||
expect(isSettledRemoteWarmControlState(state([{ ...terminal, envelope: { ...identity, runId: "other" } }, snapshot]), identity)).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -3,6 +3,7 @@ import { agents, companies, environmentLeases, heartbeatRuns, issues, nativeRunF
|
||||
import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target";
|
||||
import { parseNativeExecutionInput, type NativeExecutionInput, type PersistedNativeSession } from "../../vendor/paperclip-runner/index.js";
|
||||
import { currentNativeControllerIdentity, evaluateNativeControllerTakeover } from "./native-restart-recovery.js";
|
||||
import { readNativeControllerRecovery } from "./native-controller-recovery.js";
|
||||
|
||||
export type RemoteWarmSessionRecovery = {
|
||||
identity: Record<string, string>;
|
||||
@@ -13,6 +14,36 @@ export type RemoteWarmSessionRecovery = {
|
||||
revalidate: (leaseOwner: string) => Promise<void>;
|
||||
};
|
||||
|
||||
/** A goal read can publish an idle snapshot after the terminal. It carries no
|
||||
* new work; every other suffix still requires reconciliation before handoff. */
|
||||
export function isSettledRemoteWarmControlState(control: Record<string, unknown>, identity: Record<string, string>): boolean {
|
||||
const record = (value: unknown): Record<string, unknown> =>
|
||||
value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||||
const commands = control.commands;
|
||||
const events = control.committedEvents;
|
||||
if (!Array.isArray(commands) || commands.some(command => record(command).status !== "completed") ||
|
||||
!Array.isArray(events) || Object.entries(identity).some(([key, value]) => record(control.identity)[key] !== value)) return false;
|
||||
const terminalIndex = events.findLastIndex(event => record(event).eventType === "run.terminal");
|
||||
if (terminalIndex < 0) return false;
|
||||
const matchesSession = (event: unknown) => {
|
||||
const envelope = record(record(event).envelope);
|
||||
return envelope.runId === identity.runId && envelope.normalizedSessionId === identity.normalizedSessionId;
|
||||
};
|
||||
if (!matchesSession(events[terminalIndex])) return false;
|
||||
return events.slice(terminalIndex + 1).every(event => {
|
||||
const envelope = record(record(event).envelope);
|
||||
const notification = record(envelope.payload);
|
||||
const payload = record(notification.payload);
|
||||
return record(event).eventType === "session.goal.snapshot" && matchesSession(event) &&
|
||||
Object.entries(identity).every(([key, value]) => envelope[key] === value) &&
|
||||
(notification.schema === "paperclip.prp.event.v1" || notification.schema === "paperclip.prp.event.v2") &&
|
||||
notification.eventType === "session.goal.snapshot" &&
|
||||
notification.sourceInstanceId === identity.runnerInstanceId && notification.sourceKind === "runner" &&
|
||||
notification.runId === identity.runId && notification.normalizedSessionId === identity.normalizedSessionId &&
|
||||
payload.schema === "paperclip.session_goal.snapshot.v1" && payload.goal === null && payload.workingNow === false;
|
||||
});
|
||||
}
|
||||
|
||||
export async function inspectRemoteWarmSessionRecovery(input: {
|
||||
db: Db;
|
||||
execution: NativeExecutionInput;
|
||||
@@ -52,15 +83,14 @@ export async function inspectRemoteWarmSessionRecovery(input: {
|
||||
checkpoint.driverKind !== previous.session.driverKind || !checkpoint.providerSessionId ||
|
||||
checkpoint.activeTurnId !== null || checkpoint.pendingRuntimeRequests?.length !== 0) return deny();
|
||||
const control = input.readControlState();
|
||||
const commands = control.commands;
|
||||
const events = control.committedEvents;
|
||||
const terminal = Array.isArray(events) ? events.at(-1) : undefined;
|
||||
if (!Array.isArray(commands) || commands.some(command => command.status !== "completed") ||
|
||||
terminal?.eventType !== "run.terminal" || terminal?.envelope?.runId !== identity.runId ||
|
||||
terminal?.envelope?.normalizedSessionId !== identity.normalizedSessionId ||
|
||||
Object.entries(identity).some(([key, value]) => (control.identity as Record<string, unknown>)?.[key] !== value)) return deny();
|
||||
if (!isSettledRemoteWarmControlState(control, identity)) return deny();
|
||||
const assertPriorStopped = async () => {
|
||||
const stopped = await evaluateNativeControllerTakeover({ owner: prior.owner, now: new Date(), requireStoppedController: true });
|
||||
const persistedRecovery = readNativeControllerRecovery(prior.run.runnerProfileJson?.nativeControllerRecovery,
|
||||
prior.run.runnerProfileJson?.sessionCheckpoint);
|
||||
// A PID in another webserver container cannot prove life or death. New
|
||||
// durable controllers are fenced by the database lease and current turn.
|
||||
const stopped = await evaluateNativeControllerTakeover({ owner: prior.owner, now: new Date(), requireStoppedController: true,
|
||||
remoteLeaseFenced: persistedRecovery !== null });
|
||||
if (!stopped.allowed) return deny();
|
||||
};
|
||||
await assertPriorStopped();
|
||||
|
||||
@@ -41,6 +41,7 @@ function words(text: string): string[] {
|
||||
}
|
||||
|
||||
function dedicatedTools(method: string, path: string): string[] {
|
||||
if (method === "POST" && path === "/api/agents/me/chat/messages") return ["send_chat_message"];
|
||||
if (/\/companies\/\{[^}]+\}\/workflows$/.test(path)) return method === "GET" ? ["list_workflows"] : method === "POST" ? ["create_workflow"] : [];
|
||||
if (/\/workflows\/\{[^}]+\}$/.test(path)) return method === "GET" ? ["read_workflow"] : method === "PATCH" ? ["update_workflow"] : [];
|
||||
if (/\/issues\/\{[^}]+\}\/title$/.test(path) && method === "PUT") return ["set_task_title"];
|
||||
|
||||
@@ -10,6 +10,7 @@ import { getConversationOwnershipBlocker } from "./conversation-continuation.js"
|
||||
import { isExternalChatPresentationContext } from "./heartbeat-run-summary.js";
|
||||
import { finalizeNativeRun, recoverCommittedNativeChatResponse, repairCommittedNativeChatResponse } from "./native-runtime/native-run-finalizer.js";
|
||||
import { nativeResponsePresentationWork } from "./native-runtime/native-response-presentation-work.js";
|
||||
import { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js";
|
||||
|
||||
import { resolveExternalTestDatabaseUrl } from "../__tests__/helpers/external-test-database.js";
|
||||
|
||||
@@ -24,10 +25,12 @@ describe.skipIf(!support.supported)("presentation and ownership PostgreSQL read
|
||||
}, 60_000);
|
||||
afterAll(async () => { await db?.$client.end(); await database?.cleanup(); });
|
||||
|
||||
async function fixture(patch: Partial<typeof heartbeatRuns.$inferInsert> = {}) {
|
||||
const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Read fixture", issuePrefix: companyId.slice(0, 8) });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Read agent", adapterType: "codex_local" });
|
||||
async function fixture(patch: Partial<typeof heartbeatRuns.$inferInsert> = {}, identity?: { companyId: string; agentId: string }) {
|
||||
const companyId = identity?.companyId ?? randomUUID(), agentId = identity?.agentId ?? randomUUID(), issueId = randomUUID();
|
||||
if (!identity) {
|
||||
await db.insert(companies).values({ id: companyId, name: "Read fixture", issuePrefix: companyId.slice(0, 8) });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Read agent", adapterType: "codex_local" });
|
||||
}
|
||||
await db.insert(issues).values({ id: issueId, companyId, title: "Read task", status: "blocked", assigneeAgentId: agentId });
|
||||
const payload = randomBytes(128 * 1024).toString("base64");
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, nativeIssueId: issueId,
|
||||
@@ -138,10 +141,10 @@ describe.skipIf(!support.supported)("presentation and ownership PostgreSQL read
|
||||
expect(await repairCommittedNativeChatResponse(db, { companyId: f.companyId, issueId: other.id, runId: f.run.id })).toBe(false);
|
||||
});
|
||||
|
||||
async function unpublishedAcceptedReply() {
|
||||
async function unpublishedAcceptedReply(id?: string, identity?: { companyId: string; agentId: string }) {
|
||||
const payload = randomBytes(128 * 1024).toString("base64");
|
||||
const f = await fixture({ runtimeMode: "native", status: "running", finishedAt: null,
|
||||
contextSnapshot: { source: "internal", prompt: payload, paperclipWake: { prompt: payload } } });
|
||||
const f = await fixture({ id, runtimeMode: "native", status: "running", finishedAt: null,
|
||||
contextSnapshot: { source: "internal", prompt: payload, paperclipWake: { prompt: payload } } }, identity);
|
||||
const [contract] = await db.insert(completionContracts).values({ companyId: f.companyId, issueId: f.issueId,
|
||||
revision: 1, schemaVersion: "paperclip.completion-contract.v1", policyVersion: "test", risk: "standard",
|
||||
completionAuthority: "server_arbiter", incompleteCriteriaPolicy: "preserve_non_terminal",
|
||||
@@ -173,6 +176,40 @@ describe.skipIf(!support.supported)("presentation and ownership PostgreSQL read
|
||||
return { ...f, before, reply };
|
||||
}
|
||||
|
||||
it("saves an independent task reply while an earlier task waits for its database lock", async () => {
|
||||
const suffix = randomUUID().replaceAll("-", "").slice(0, 12);
|
||||
const first = await unpublishedAcceptedReply(`00000000-0000-4000-8000-${suffix}`);
|
||||
const second = await unpublishedAcceptedReply(`00000000-0000-4001-8000-${suffix}`, first);
|
||||
let release!: () => void;
|
||||
let locked!: () => void;
|
||||
const hold = new Promise<void>(resolve => { release = resolve; });
|
||||
const acquired = new Promise<void>(resolve => { locked = resolve; });
|
||||
const blocker = db.transaction(async tx => {
|
||||
await tx.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, first.run.id)).for("update");
|
||||
locked(); await hold;
|
||||
});
|
||||
await Promise.race([acquired, blocker]);
|
||||
const onError = vi.fn();
|
||||
const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError });
|
||||
const worker = nativeResponsePresentationWork(db).register(coordinator, () => true);
|
||||
try {
|
||||
await vi.waitFor(async () => {
|
||||
const rows = await db.select().from(issueComments).where(eq(issueComments.createdByRunId, second.run.id));
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0].body).toBe(second.reply);
|
||||
}, { timeout: 5000 });
|
||||
expect(await db.select().from(issueComments).where(eq(issueComments.createdByRunId, first.run.id))).toHaveLength(0);
|
||||
} finally {
|
||||
release(); await blocker; await worker.ready; worker.stop(); await coordinator.stop();
|
||||
}
|
||||
for (const f of [first, second]) {
|
||||
const rows = await db.select().from(issueComments).where(eq(issueComments.createdByRunId, f.run.id));
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0].body).toBe(f.reply);
|
||||
}
|
||||
expect(onError).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
async function repairFixture(patch: Partial<typeof heartbeatRuns.$inferInsert>) {
|
||||
|
||||
|
||||
|
||||
@@ -52,7 +52,7 @@ export function redactRegisteredSecretValues<T>(input: T, values: string[], cred
|
||||
if (!record) return input;
|
||||
return Object.fromEntries(
|
||||
Object.entries(record)
|
||||
.filter(([key]) => key !== REGISTRY_KEY)
|
||||
.filter(([key]) => key !== REGISTRY_KEY && key !== "nativeControllerRecovery" && key !== "nativeTaskSessionPublication")
|
||||
.map(([key, value]) => [key, key === "credentialAccess"
|
||||
? (credentialConnectionName ? { connectionName: redactText(credentialConnectionName, values) } : {})
|
||||
: redactRegisteredSecretValues(value, values, credentialConnectionName)]),
|
||||
|
||||
@@ -12,6 +12,7 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
|
||||
queue: DeliveryQueue;
|
||||
batchSize?: number;
|
||||
minimumPassIntervalMs?: number;
|
||||
concurrentChecks?: { limit: number; key(entry: Entry): string };
|
||||
reconstructionIntervalMs?: number | ((scope: Scope | undefined) => number);
|
||||
beforePass?(): Promise<void>;
|
||||
decodeScope(hint: string): Scope | undefined;
|
||||
@@ -21,6 +22,9 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
|
||||
},
|
||||
canRun: () => boolean,
|
||||
) {
|
||||
if (options.concurrentChecks && (!Number.isInteger(options.concurrentChecks.limit) || options.concurrentChecks.limit < 1)) {
|
||||
throw new RangeError("Concurrent check limit must be a positive integer");
|
||||
}
|
||||
const dirty = new Set<string>(["*"]);
|
||||
const versions = new Map<string, number>();
|
||||
const scans = new Map<string, { version: number; seen: Set<string> }>();
|
||||
@@ -112,14 +116,30 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
|
||||
&& Math.max(entry.dueAt, failures.get(entry.id)?.retryAt ?? 0) <= Date.now())
|
||||
.sort((a, b) => a.dueAt - b.dueAt || a.id.localeCompare(b.id));
|
||||
const batch = takeCompanyWorkBatch(due, options.batchSize ?? 25, lastCompany); lastCompany = batch.lastCompany;
|
||||
const groups = new Map<string, Entry[]>();
|
||||
for (const entry of batch.entries) {
|
||||
if (signal.aborted || !canRun()) break;
|
||||
try {
|
||||
const current = await options.check(entry, signal);
|
||||
if (current) entries.set(entry.id, current); else entries.delete(entry.id);
|
||||
failures.delete(entry.id);
|
||||
} catch (error) { failed(failures, entry.id); errors.push(error); }
|
||||
const key = options.concurrentChecks?.key(entry) ?? "";
|
||||
const group = groups.get(key) ?? [];
|
||||
group.push(entry); groups.set(key, group);
|
||||
}
|
||||
const pending = [...groups.values()];
|
||||
let nextGroup = 0;
|
||||
const checkGroups = async () => {
|
||||
while (!signal.aborted && canRun()) {
|
||||
const group = pending[nextGroup++];
|
||||
if (!group) return;
|
||||
for (const entry of group) {
|
||||
if (signal.aborted || !canRun()) return;
|
||||
try {
|
||||
const current = await options.check(entry, signal);
|
||||
if (current) entries.set(entry.id, current); else entries.delete(entry.id);
|
||||
failures.delete(entry.id);
|
||||
} catch (error) { failed(failures, entry.id); errors.push(error); }
|
||||
}
|
||||
}
|
||||
};
|
||||
const checks = await Promise.allSettled(Array.from({ length: Math.min(options.concurrentChecks?.limit ?? 1, pending.length) }, checkGroups));
|
||||
for (const result of checks) if (result.status === "rejected") errors.push(result.reason);
|
||||
if (errors.length) throw new AggregateError(errors, `${options.queue} could not settle`);
|
||||
},
|
||||
// Retain scopes/deadlines for resume without retaining a settled write's
|
||||
|
||||
@@ -753,7 +753,11 @@ export function statusCardService(
|
||||
createdByAgentId: input.actor?.agentId ?? null,
|
||||
createdByUserId: input.actor?.userId ?? null,
|
||||
hiddenAt: now,
|
||||
idempotencyKey: `status-card-update:${card.id}:${fingerprintHash}`,
|
||||
// Claim ownership is revision-scoped too. A superseded refresh may cancel
|
||||
// its unclaimed task; it must not share that task with a newer revision
|
||||
// which has the same watched fingerprint but has not claimed the card yet.
|
||||
// Concurrent refreshes of this revision still deduplicate normally.
|
||||
idempotencyKey: `status-card-update:${card.id}:${revision}:${fingerprintHash}`,
|
||||
onDeduplicated: (reason) => { deduplicated = reason === "idempotency_key"; },
|
||||
});
|
||||
const reopened = deduplicated && TERMINAL_ISSUE_STATUSES.has(created.status)
|
||||
|
||||
@@ -2113,6 +2113,9 @@ export function toolAccessService(
|
||||
sourceTemplateKey: galleryEntry.slug,
|
||||
connectionMethodKey: method?.key,
|
||||
methodConfig: normalizedMethodConfig?.values ?? {},
|
||||
...(galleryEntry.slug === "google-workspace" && retainedConnection?.status === "draft"
|
||||
? { workspaceSetup: retainedConnection.config.workspaceSetup }
|
||||
: {}),
|
||||
...(method?.defaults?.mcpSessionRequired === true
|
||||
? { mcpSessionRequired: true }
|
||||
: {}),
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
import { notifyIssueReviewState } from "../../issue-review-state-notifications.js";
|
||||
import { vercelConnectHttpError, vercelCredentialFor, vercelConnectResourcesFor } from "../vercel-credentials.js";
|
||||
import { credentialScope, managedConnectorProfile } from "./client-settings.js";
|
||||
import { createOAuthClientRegistration } from "./client-registration.js";
|
||||
import { assertOAuthClientBindingUnchanged, createOAuthClientRegistration } from "./client-registration.js";
|
||||
import type { ActorInfo } from "../types.js";
|
||||
import { createOAuthEndpointPolicy } from "./endpoint-policy.js";
|
||||
import { createOAuthDiscovery } from "./discovery.js";
|
||||
import { oauthConfig, normalizeOauthScopes, originOf } from "./metadata.js";
|
||||
import { asRecord, oauthConfig, normalizeOauthScopes, originOf } from "./metadata.js";
|
||||
import { connectionMethodForConnection } from "../connection-methods.js";
|
||||
import { GOOGLE_WORKSPACE_DEFAULT_SELECTION, googleWorkspaceSelectedScopes, type GoogleWorkspaceServiceSelection, isGoogleWorkspaceScopeGrant } from "@paperclipai/shared";
|
||||
import { googleWorkspaceDraftRequestedScopes, googleWorkspaceSelectedScopes, type GoogleWorkspaceServiceSelection, isGoogleWorkspaceScopeGrant } from "@paperclipai/shared";
|
||||
import { and, eq, lt } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { companies, agents, connectionGrants, issueThreadInteractions, toolConnections, toolOauthStates } from "@paperclipai/db";
|
||||
@@ -72,17 +72,37 @@ export function createOAuthAuthorizationStart(db: Db, dependencies: Dependencies
|
||||
const galleryMethod = galleryEntry
|
||||
? connectionMethodForConnection(galleryEntry, connection)
|
||||
: null;
|
||||
const starterBinding = actorBinding(input.actor);
|
||||
const fixedPersonalIdentity = await fixedPersonalIdentityForReconnect(
|
||||
connection,
|
||||
input.subjectUserId,
|
||||
input.actor,
|
||||
);
|
||||
const authorizationSubjectUserId =
|
||||
fixedPersonalIdentity?.subjectUserId ?? input.subjectUserId;
|
||||
const authorizationSubjectAgentId = input.subjectAgentId;
|
||||
const workspaceDraftBinding = {
|
||||
companyId, connectionId: connection.id,
|
||||
actorType: starterBinding.actorType, actorId: starterBinding.actorId,
|
||||
grantKind: connection.credentialPolicy === "per_user" ? "user" as const
|
||||
: connection.credentialPolicy === "per_agent" ? "agent" as const : "organization" as const,
|
||||
subjectId: connection.credentialPolicy === "per_user" ? authorizationSubjectUserId ?? null
|
||||
: connection.credentialPolicy === "per_agent" ? authorizationSubjectAgentId ?? null : null,
|
||||
};
|
||||
const requestedScopes = await (async () => {
|
||||
if (input.workspaceServices && sourceTemplateKey !== "google-workspace") throw badRequest("Service selection requires Google Workspace");
|
||||
if (sourceTemplateKey === "google-workspace") {
|
||||
if (input.workspaceServices && input.scopes) throw badRequest("Choose services or scopes, not both");
|
||||
const allowed = normalizeOauthScopes(galleryMethod?.defaults?.scopesHint);
|
||||
let scopes = input.scopes ?? (input.workspaceServices ? googleWorkspaceSelectedScopes(input.workspaceServices) : undefined);
|
||||
if (!scopes && connection.status !== "draft") {
|
||||
if (!scopes && connection.status === "draft") {
|
||||
scopes = googleWorkspaceDraftRequestedScopes(connection.config.workspaceSetup, workspaceDraftBinding) ?? undefined;
|
||||
}
|
||||
if (!scopes && input.actor.managementContext?.actor.type === "agent") return null;
|
||||
if (!scopes) {
|
||||
// Account-page reconnect has no new service choice. Never replace its
|
||||
// subject's saved request with the new-account write defaults.
|
||||
const binding = actorBinding(input.actor);
|
||||
const subjectUserId = input.subjectUserId ?? (binding.actorType === "user" ? binding.actorId : null);
|
||||
const subjectUserId = authorizationSubjectUserId ?? (starterBinding.actorType === "user" ? starterBinding.actorId : null);
|
||||
const kind = connection.credentialPolicy === "per_user" ? "user"
|
||||
: connection.credentialPolicy === "per_agent" ? "agent" : "organization";
|
||||
const [grant] = await db.select().from(connectionGrants).where(and(
|
||||
@@ -94,11 +114,12 @@ export function createOAuthAuthorizationStart(db: Db, dependencies: Dependencies
|
||||
: eq(connectionGrants.isDefault, true),
|
||||
)).limit(1);
|
||||
const saved = grant?.providerTenant?.oauth;
|
||||
scopes = normalizeOauthScopes(saved?.requestedScopes ?? saved?.scopes)
|
||||
scopes = normalizeOauthScopes(saved?.requestedScopes ?? (connection.status !== "draft" ? saved?.scopes : undefined))
|
||||
.filter(scope => allowed.includes(scope));
|
||||
if (scopes.length === 0) throw badRequest("Choose your Google Workspace services before reconnecting");
|
||||
if (scopes.length === 0) throw badRequest(connection.status === "draft"
|
||||
? "Choose your Google Workspace services before continuing setup"
|
||||
: "Choose your Google Workspace services before reconnecting");
|
||||
}
|
||||
scopes ??= googleWorkspaceSelectedScopes(GOOGLE_WORKSPACE_DEFAULT_SELECTION);
|
||||
if (!isGoogleWorkspaceScopeGrant(scopes) || scopes.some(scope => !allowed.includes(scope))) throw badRequest("Select at least one supported Google service");
|
||||
return scopes;
|
||||
}
|
||||
@@ -118,15 +139,6 @@ export function createOAuthAuthorizationStart(db: Db, dependencies: Dependencies
|
||||
}
|
||||
return requested;
|
||||
})();
|
||||
const starterBinding = actorBinding(input.actor);
|
||||
const fixedPersonalIdentity = await fixedPersonalIdentityForReconnect(
|
||||
connection,
|
||||
input.subjectUserId,
|
||||
input.actor,
|
||||
);
|
||||
const authorizationSubjectUserId =
|
||||
fixedPersonalIdentity?.subjectUserId ?? input.subjectUserId;
|
||||
const authorizationSubjectAgentId = input.subjectAgentId;
|
||||
if (authorizationSubjectAgentId) {
|
||||
const [subjectAgent] = await db
|
||||
.select({ id: agents.id })
|
||||
@@ -209,6 +221,37 @@ export function createOAuthAuthorizationStart(db: Db, dependencies: Dependencies
|
||||
return { connectionId: connection.id, provider: sourceTemplateKey ?? "oauth",
|
||||
authorizationUrl: destination.toString(), expiresAt: null };
|
||||
}
|
||||
if (sourceTemplateKey === "google-workspace" && connection.status === "draft") {
|
||||
const workspaceSetup = { version: 1, ...workspaceDraftBinding, requestedScopes };
|
||||
const updated = await db.transaction(async tx => {
|
||||
const [saved] = await tx.select().from(toolConnections).where(and(
|
||||
eq(toolConnections.id, connection.id), eq(toolConnections.companyId, companyId), eq(toolConnections.status, "draft"),
|
||||
)).for("no key update");
|
||||
if (!saved || (["credentialPolicy", "credentialSource", "authKind", "transport", "createdByUserId"] as const)
|
||||
.some(key => saved[key] !== connection[key])
|
||||
|| ["sourceTemplateKey", "connectionMethodKey"].some(key => saved.config[key] !== connection.config[key]
|
||||
|| saved.transportConfig[key] !== connection.transportConfig[key])) {
|
||||
throw conflict("Connection setup changed. Reload before continuing");
|
||||
}
|
||||
const config: Record<string, unknown> = { ...saved.config, workspaceSetup };
|
||||
const transportConfig: Record<string, unknown> = { ...saved.transportConfig, workspaceSetup };
|
||||
if (starterBinding.actorType === "user" && starterBinding.actorId) {
|
||||
for (const target of [config, transportConfig]) {
|
||||
const oauth = asRecord(target.oauth);
|
||||
const pending = { ...asRecord(oauth.pendingWorkspaceServiceRequests) };
|
||||
if (Object.hasOwn(pending, starterBinding.actorId)) {
|
||||
delete pending[starterBinding.actorId];
|
||||
target.oauth = { ...oauth, pendingWorkspaceServiceRequests: pending };
|
||||
}
|
||||
}
|
||||
}
|
||||
const [row] = await tx.update(toolConnections).set({ config, transportConfig, updatedAt: now() })
|
||||
.where(and(eq(toolConnections.id, saved.id), eq(toolConnections.companyId, companyId), eq(toolConnections.status, "draft"))).returning();
|
||||
return row;
|
||||
});
|
||||
if (!updated) throw conflict("Connection setup changed. Reload before continuing");
|
||||
connection = updated;
|
||||
}
|
||||
if (connection.credentialSource === "vercel_connect") {
|
||||
if (!vercelConnect)
|
||||
throw vercelConnectHttpError(
|
||||
@@ -629,10 +672,13 @@ export function createOAuthAuthorizationStart(db: Db, dependencies: Dependencies
|
||||
});
|
||||
}
|
||||
|
||||
const nextConfig = {
|
||||
...connection.config,
|
||||
oauth: {
|
||||
...oauthConfig(connection),
|
||||
await db.transaction(async tx => {
|
||||
const [saved] = await tx.select().from(toolConnections).where(and(
|
||||
eq(toolConnections.id, connection.id), eq(toolConnections.companyId, companyId),
|
||||
)).for("no key update");
|
||||
if (!saved) throw notFound("Tool connection not found");
|
||||
assertOAuthClientBindingUnchanged(connection, saved, endpoints, input.redirectUri);
|
||||
const oauth = {
|
||||
provider: endpoints.provider,
|
||||
authorizationUrl: endpoints.authorizationUrl,
|
||||
tokenUrl: endpoints.tokenUrl,
|
||||
@@ -655,27 +701,29 @@ export function createOAuthAuthorizationStart(db: Db, dependencies: Dependencies
|
||||
// Persist the issuer and resource this authorization run is bound to.
|
||||
// `iss` on the callback is validated against `expectedIssuer`, and
|
||||
// refresh/reconnect/revoke reuse the same pair rather than re-deriving it.
|
||||
issuer: endpoints.issuer ?? oauthConfig(connection).issuer ?? null,
|
||||
issuer: endpoints.issuer ?? oauthConfig(saved).issuer ?? null,
|
||||
expectedIssuer: endpoints.issuer ?? null,
|
||||
resource:
|
||||
endpoints.resource ?? oauthConfig(connection).resource ?? null,
|
||||
endpoints.resource ?? oauthConfig(saved).resource ?? null,
|
||||
clientIdMetadataDocumentSupported:
|
||||
endpoints.clientIdMetadataDocumentSupported === true,
|
||||
},
|
||||
};
|
||||
await db
|
||||
.update(toolConnections)
|
||||
.set({
|
||||
};
|
||||
const nextConfig = { ...saved.config, oauth: { ...oauthConfig(saved), ...oauth } };
|
||||
const nextTransportConfig = { ...saved.transportConfig, oauth: { ...asRecord(saved.transportConfig.oauth), ...oauth } };
|
||||
await tx
|
||||
.update(toolConnections)
|
||||
.set({
|
||||
// A generic URL connection starts life as `authKind: none`. Once it has
|
||||
// completed OAuth discovery and client resolution it is an OAuth
|
||||
// connection, and refresh, reconnect, revoke and diagnostics must all
|
||||
// treat it as one.
|
||||
authKind: "oauth",
|
||||
config: nextConfig,
|
||||
transportConfig: nextConfig,
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(eq(toolConnections.id, connection.id));
|
||||
authKind: "oauth",
|
||||
config: nextConfig,
|
||||
transportConfig: nextTransportConfig,
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(and(eq(toolConnections.id, connection.id), eq(toolConnections.companyId, companyId)));
|
||||
});
|
||||
|
||||
return {
|
||||
connectionId: connection.id,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { ActorInfo } from "../types.js";
|
||||
import { createOAuthClientProtocol } from "./client.js";
|
||||
import { oauthConfig, sameOAuthIssuer } from "./metadata.js";
|
||||
import { asRecord, oauthConfig, sameOAuthIssuer } from "./metadata.js";
|
||||
import { resolveOAuthClientIdMetadataDocumentUrl } from "./client-metadata.js";
|
||||
import type { OAuthProviderEndpoints, OAuthClientRegistrationSource } from "./types.js";
|
||||
import { connectionMethodForConnection } from "../connection-methods.js";
|
||||
@@ -8,7 +8,7 @@ import { and, eq } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { toolConnections } from "@paperclipai/db";
|
||||
import type { AppDefinition } from "@paperclipai/shared";
|
||||
import { notFound, unprocessable } from "../../../errors.js";
|
||||
import { conflict, notFound, unprocessable } from "../../../errors.js";
|
||||
import { type RemoteHttpEndpointLookup } from "../../remote-http-endpoint-guard.js";
|
||||
|
||||
import type { OAuthCredentialStore } from "./credential-store.js";
|
||||
@@ -24,6 +24,34 @@ type RegistrationDependencies = {
|
||||
requestOAuthClientRegistration: ReturnType<typeof createOAuthClientProtocol>["requestOAuthClientRegistration"];
|
||||
};
|
||||
|
||||
export function assertOAuthClientBindingUnchanged(
|
||||
expected: typeof toolConnections.$inferSelect,
|
||||
current: typeof toolConnections.$inferSelect,
|
||||
endpoints: OAuthProviderEndpoints,
|
||||
redirectUri: string,
|
||||
) {
|
||||
const changedSetup = expected.config.sourceTemplateKey === "google-workspace" && expected.status === "draft"
|
||||
&& (current.status !== "draft" || (["credentialPolicy", "credentialSource", "authKind", "transport", "createdByUserId"] as const)
|
||||
.some(key => current[key] !== expected[key])
|
||||
|| ["sourceTemplateKey", "connectionMethodKey"].some(key => current.config[key] !== expected.config[key]
|
||||
|| current.transportConfig[key] !== expected.transportConfig[key]));
|
||||
const bindings: Record<string, unknown> = {
|
||||
clientRedirectUri: redirectUri, clientIssuer: endpoints.issuer ?? null,
|
||||
clientResource: endpoints.resource ?? null, clientCompanyId: expected.companyId,
|
||||
issuer: endpoints.issuer ?? null, resource: endpoints.resource ?? null,
|
||||
};
|
||||
const changedClient = expected.ownership !== current.ownership
|
||||
|| JSON.stringify(oauthSecretRef(expected, "oauth.client_secret")) !== JSON.stringify(oauthSecretRef(current, "oauth.client_secret"))
|
||||
|| (["config", "transportConfig"] as const).some(target => {
|
||||
const before = asRecord(expected[target].oauth);
|
||||
const after = asRecord(current[target].oauth);
|
||||
return ["clientId", "clientRegistrationSource", "clientTokenEndpointAuthMethod"].some(key => before[key] !== after[key])
|
||||
|| Object.entries(bindings).some(([key, value]) => before[key] !== after[key]
|
||||
&& !(before[key] == null && (after[key] == null || after[key] === value)));
|
||||
});
|
||||
if (changedSetup || changedClient) throw conflict("Connection sign-in settings changed. Reload before continuing");
|
||||
}
|
||||
|
||||
export function createOAuthClientRegistration(db: Db, { now, options, store, getConnectionRow, requestOAuthClientRegistration }: RegistrationDependencies) {
|
||||
const { createOrRotateOAuthSecret, oauthClientForConnection, syncCredentialBindings } = store;
|
||||
async function registerOAuthClient(input: {
|
||||
@@ -300,46 +328,55 @@ async function stampOAuthClientBinding(
|
||||
endpoints: OAuthProviderEndpoints,
|
||||
redirectUri: string,
|
||||
): Promise<typeof toolConnections.$inferSelect> {
|
||||
const originalOAuth = oauthConfig(connection);
|
||||
const oauth = reviewedOAuthClientBinding(connection, endpoints);
|
||||
const nextBinding = {
|
||||
clientRedirectUri: redirectUri,
|
||||
clientIssuer:
|
||||
typeof oauth.clientIssuer === "string" && oauth.clientIssuer
|
||||
? oauth.clientIssuer
|
||||
: (endpoints.issuer ?? null),
|
||||
clientResource:
|
||||
typeof oauth.clientResource === "string" && oauth.clientResource
|
||||
? oauth.clientResource
|
||||
: (endpoints.resource ?? null),
|
||||
clientCompanyId:
|
||||
typeof oauth.clientCompanyId === "string" && oauth.clientCompanyId
|
||||
? oauth.clientCompanyId
|
||||
: connection.companyId,
|
||||
};
|
||||
const unchanged = Object.entries(nextBinding).every(
|
||||
([key, value]) => originalOAuth[key] === value,
|
||||
);
|
||||
if (unchanged) return connection;
|
||||
const nextConfig = {
|
||||
...connection.config,
|
||||
oauth: { ...oauth, ...nextBinding },
|
||||
};
|
||||
const [updated] = await db
|
||||
.update(toolConnections)
|
||||
.set({
|
||||
config: nextConfig,
|
||||
transportConfig: nextConfig,
|
||||
updatedAt: now(),
|
||||
})
|
||||
.where(
|
||||
and(
|
||||
eq(toolConnections.id, connection.id),
|
||||
eq(toolConnections.companyId, connection.companyId),
|
||||
),
|
||||
)
|
||||
.returning();
|
||||
return updated ?? connection;
|
||||
return db.transaction(async tx => {
|
||||
const [saved] = await tx.select().from(toolConnections).where(and(
|
||||
eq(toolConnections.id, connection.id), eq(toolConnections.companyId, connection.companyId),
|
||||
)).for("no key update");
|
||||
if (!saved) throw notFound("Tool connection not found");
|
||||
assertOAuthClientBindingUnchanged(connection, saved, endpoints, redirectUri);
|
||||
const originalOAuth = oauthConfig(saved);
|
||||
const oauth = reviewedOAuthClientBinding(saved, endpoints);
|
||||
const nextBinding = {
|
||||
clientRedirectUri: redirectUri,
|
||||
clientIssuer:
|
||||
typeof oauth.clientIssuer === "string" && oauth.clientIssuer
|
||||
? oauth.clientIssuer
|
||||
: (endpoints.issuer ?? null),
|
||||
clientResource:
|
||||
typeof oauth.clientResource === "string" && oauth.clientResource
|
||||
? oauth.clientResource
|
||||
: (endpoints.resource ?? null),
|
||||
clientCompanyId:
|
||||
typeof oauth.clientCompanyId === "string" && oauth.clientCompanyId
|
||||
? oauth.clientCompanyId
|
||||
: saved.companyId,
|
||||
};
|
||||
const unchanged = Object.entries(nextBinding).every(
|
||||
([key, value]) => originalOAuth[key] === value,
|
||||
);
|
||||
if (unchanged) return saved;
|
||||
const nextConfig = {
|
||||
...saved.config,
|
||||
oauth: { ...oauth, ...nextBinding },
|
||||
};
|
||||
const nextTransportConfig = { ...saved.transportConfig, oauth: { ...asRecord(saved.transportConfig.oauth), ...nextBinding } };
|
||||
const [updated] = await tx
|
||||
.update(toolConnections)
|
||||
.set({
|
||||
config: nextConfig,
|
||||
transportConfig: nextTransportConfig,
|
||||
updatedAt: now(),
|
||||
})
|
||||
.where(
|
||||
and(
|
||||
eq(toolConnections.id, connection.id),
|
||||
eq(toolConnections.companyId, connection.companyId),
|
||||
),
|
||||
)
|
||||
.returning();
|
||||
if (!updated) throw notFound("Tool connection not found");
|
||||
return updated;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -48,7 +48,7 @@ export function createToolProfileService(db: Db, now: () => Date) {
|
||||
eq(toolProfileEntries.profileId, profile.id),
|
||||
),
|
||||
)
|
||||
.orderBy(asc(toolProfileEntries.createdAt)),
|
||||
.orderBy(asc(toolProfileEntries.createdAt), asc(toolProfileEntries.id)),
|
||||
dbClient
|
||||
.select()
|
||||
.from(toolProfileBindings)
|
||||
@@ -387,7 +387,7 @@ export function createToolProfileService(db: Db, now: () => Date) {
|
||||
inArray(toolProfileEntries.profileId, profileIds),
|
||||
),
|
||||
)
|
||||
.orderBy(asc(toolProfileEntries.createdAt)),
|
||||
.orderBy(asc(toolProfileEntries.createdAt), asc(toolProfileEntries.id)),
|
||||
db
|
||||
.select()
|
||||
.from(toolProfileBindings)
|
||||
|
||||
@@ -1572,7 +1572,7 @@ describe("automatic retained workspace recovery", () => {
|
||||
return { ...t, queued: current, wake: claimedWake, followup, expectedIds: ids, latestAuthor: kind === "deferred-different" ? "third-user" : kind === "active" ? "second-user" : "operator" };
|
||||
}
|
||||
|
||||
async function exerciseQueuedReset(nativeDescriptor: boolean, coalesced = false, ordinary?: "active" | "deferred" | "deferred-different" | "system-deferred") {
|
||||
async function exerciseQueuedReset(nativeDescriptor: boolean, coalesced = false, ordinary?: "active" | "deferred" | "deferred-different" | "system-deferred", earlierCandidates = false) {
|
||||
const mixed = ordinary ? await ordinaryMixedQueuedFixture(ordinary) : null;
|
||||
const t = mixed ?? await queuedResetFixture(nativeDescriptor);
|
||||
const { f, original, reset, wake, queued, deps, control, execution, queueDeps, queue } = t;
|
||||
@@ -1595,12 +1595,43 @@ describe("automatic retained workspace recovery", () => {
|
||||
deps.activeRunExecutions.delete(f.runId); execution.finish(); adapterExecutionControls.delete(f.runId);
|
||||
await control.resumeRemoteStopComments(await run(f.runId));
|
||||
expect(readSystemRecovery((await action(f.actionId)).evidence)!.nextAttemptAt!).toBeLessThanOrEqual(Date.now());
|
||||
const earlier: Awaited<ReturnType<typeof seed>>[] = [];
|
||||
if (earlierCandidates) for (let index = 1; index <= 2; index++) {
|
||||
const candidate = await seed("native", false, "keep_running");
|
||||
const id = `00000000-0000-4000-8000-${String(index).padStart(12, "0")}`;
|
||||
await db.update(issueRecoveryActions).set({ id }).where(and(eq(issueRecoveryActions.id, candidate.actionId), eq(issueRecoveryActions.companyId, candidate.companyId)));
|
||||
candidate.actionId = id; candidate.input.actionId = id;
|
||||
await db.update(environmentLeases).set({ status: "active", releasedAt: null, cleanupStatus: null, metadata: candidate.lease.metadata }).where(eq(environmentLeases.id, candidate.lease.id));
|
||||
await requestWorkspaceRecovery({ ...candidate.input, idempotencyKey: `earlier-${index}`, actorId: "board" });
|
||||
earlier.push(candidate);
|
||||
}
|
||||
const ownedCompanies = [f.companyId, ...earlier.map(candidate => candidate.companyId)];
|
||||
const seededRuns = earlierCandidates ? await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(inArray(heartbeatRuns.companyId, ownedCompanies)) : [];
|
||||
if (earlierCandidates) {
|
||||
await adoptHistoricalWorkspaceRecoveries(db);
|
||||
expect((await retainedWorkspaceCandidates(db, undefined, true)).slice(0, 2).map(row => row.action.id)).toEqual(earlier.map(candidate => candidate.actionId));
|
||||
}
|
||||
let ownedRecoveryObserved = false;
|
||||
const onRecovered = vi.fn(async (current: Awaited<ReturnType<typeof run>>) => {
|
||||
if (current.id !== f.runId) return;
|
||||
if (nativeDescriptor) expect(await inspectWorkspaceSource(db, { companyId: f.companyId, issueId: f.issueId,
|
||||
runId: f.runId, actionId: f.actionId })).toMatchObject({ status: "verified", scope: "recorded_source_identity" });
|
||||
await Promise.all([control.resumeRemoteStopComments(current), control.resumeRemoteStopComments(current)]);
|
||||
ownedRecoveryObserved = true;
|
||||
});
|
||||
const repair = async () => { await Promise.all(await reconcileRetainedWorkspaces(db, f.environmentRuntime, onRecovered) ?? []); };
|
||||
const sweepSizes: number[] = [];
|
||||
let firstSweepOwned: boolean | undefined;
|
||||
const repair = async () => {
|
||||
await vi.waitFor(async () => {
|
||||
const work = await reconcileRetainedWorkspaces(db, f.environmentRuntime, onRecovered) ?? [];
|
||||
sweepSizes.push(work.length);
|
||||
await Promise.all(work);
|
||||
firstSweepOwned ??= ownedRecoveryObserved;
|
||||
expect(ownedRecoveryObserved).toBe(true);
|
||||
expect(readSystemRecovery((await action(f.actionId)).evidence)?.state).toBe("complete");
|
||||
}, { timeout: 10_000, interval: 10 });
|
||||
expect(sweepSizes.every(size => size <= 2)).toBe(true);
|
||||
};
|
||||
if (!nativeDescriptor) await repair();
|
||||
expect(await queue.startNextQueuedRunForAgent(original.agentId)).toEqual([]);
|
||||
expect(await queue.claimQueuedRun(await run(queued.id))).toBeNull();
|
||||
@@ -1629,9 +1660,16 @@ describe("automatic retained workspace recovery", () => {
|
||||
}
|
||||
expect(queueDeps.executeRun).not.toHaveBeenCalled();
|
||||
if (nativeDescriptor) await repair();
|
||||
expect(onRecovered).toHaveBeenCalled();
|
||||
expect(onRecovered).toHaveBeenCalledWith(expect.objectContaining({ id: f.runId, companyId: f.companyId }));
|
||||
expect(readSystemRecovery((await action(f.actionId)).evidence)?.state).toBe("complete");
|
||||
if (nativeDescriptor) expect(await readFile(path.join(f.host, "saved.txt"), "utf8")).toBe("finished work\n");
|
||||
if (earlierCandidates) {
|
||||
expect(sweepSizes[0]).toBe(2);
|
||||
expect(firstSweepOwned).toBe(false);
|
||||
expect(sweepSizes.length).toBeGreaterThan(1);
|
||||
expect((await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(inArray(heartbeatRuns.companyId, ownedCompanies))).map(row => row.id).sort()).toEqual(seededRuns.map(row => row.id).sort());
|
||||
for (const candidate of [f, ...earlier]) expect(await readFile(path.join(candidate.remote, "saved.txt"), "utf8")).toBe("finished work\n");
|
||||
}
|
||||
await Promise.all([control.resumeRemoteStopComments(await run(f.runId)),
|
||||
control.resumeRemoteStopComments(await run(f.runId))]);
|
||||
const [afterWake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id));
|
||||
@@ -1664,6 +1702,13 @@ describe("automatic retained workspace recovery", () => {
|
||||
{ nativeDescriptor: false, title: "does not replay a snapshot-only source without a verified native baseline" },
|
||||
])("$title", async ({ nativeDescriptor }) => exerciseQueuedReset(nativeDescriptor), 30_000);
|
||||
|
||||
it("reaches its queued snapshot-only reset after earlier recovery candidates without starting agent work", async () => {
|
||||
const previous = db;
|
||||
db = createDb(connectionString);
|
||||
try { await exerciseQueuedReset(false, false, undefined, true); }
|
||||
finally { await db.$client.end(); db = previous; }
|
||||
}, 30_000);
|
||||
|
||||
it("preserves coalesced queued user comments until real native copyback replays the original wake", async () => {
|
||||
await exerciseQueuedReset(true, true);
|
||||
}, 30_000);
|
||||
|
||||
Reference in new issue
Block a user