Merge qualified cards and routine parents into external object work

Preserve scoped reconstruction cooldown alongside keyed concurrency and the current issue authorization and deletion boundaries.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-10-31 09:35:30 -05:00
commit 5eb3d14f3b
484 files changed
+33834 -1620

No files matched your search

@@ -183,6 +183,33 @@ describe("adapter model refresh route", () => {
await unregisterTestAdapter(refreshableAdapterType);
});
it.each([
{ qualification: "pending", qualified: false, status: 422 },
{ qualification: "released", qualified: true, status: 200 },
])("keeps $qualification Kimi model selection separate from other provider catalogs", async ({ qualified, status }) => {
const { PAPERCLIP_RUNNER_ACPX_PROFILES } = await import("@paperclipai/adapter-utils");
const kimi = PAPERCLIP_RUNNER_ACPX_PROFILES.find(profile => profile.value === "kimi")! as { qualified: boolean };
const previousQualified = kimi.qualified;
const previousAdmission = process.env.PAPERCLIP_ACPX_CANDIDATE_ADMISSION;
try {
kimi.qualified = qualified;
delete process.env.PAPERCLIP_ACPX_CANDIDATE_ADMISSION;
const app = await createApp();
const res = await requestApp(app, baseUrl =>
request(baseUrl).get("/api/companies/company-1/adapters/paperclip_runner/models?provider=acpx&acpxAgent=kimi&refresh=1"),
);
expect(res.status, JSON.stringify(res.body)).toBe(status);
if (qualified) expect(res.body).toEqual([]);
else expect(res.body).toMatchObject({ code: "paperclip_runner_acpx_agent_unavailable" });
expect(mockListOpenCodeModels).not.toHaveBeenCalled();
} finally {
kimi.qualified = previousQualified;
if (previousAdmission === undefined) delete process.env.PAPERCLIP_ACPX_CANDIDATE_ADMISSION;
else process.env.PAPERCLIP_ACPX_CANDIDATE_ADMISSION = previousAdmission;
}
});
it("uses refreshModels when refresh=1 is requested", async () => {
const listModels = vi.fn(async () => [{ id: "stale-model", label: "stale-model" }]);
const refreshModels = vi.fn(async () => [{ id: "fresh-model", label: "fresh-model" }]);
@@ -31,6 +31,8 @@ vi.mock("../vendor/paperclip-runner/live/index.js", () => ({
probeAcpxGrokInstallation: vi.fn(async () => undefined),
probeAcpxPiInstallation: vi.fn(async () => undefined),
probeAcpxCursorInstallation: vi.fn(async () => undefined),
probeAcpxKimiInstallation: vi.fn(async () => undefined),
probeKimiModels: vi.fn(async () => undefined),
}));
it("advertises tool-refresh recovery for the selected legacy harness", () => {
+4 -1
View File
@@ -487,9 +487,12 @@ describe("adapter routes", () => {
default: "claude",
meta: { visibleWhen: { key: "provider", value: "acpx" } },
options: [
{ value: "claude", label: "Claude" },
{ value: "grok", label: "Grok Build" },
{ value: "claude", label: "Claude" },
{ value: "cursor", label: "Cursor" },
{ value: "copilot", label: "GitHub Copilot" },
{ value: "pi", label: "Pi" },
{ value: "kimi", label: "Kimi Code" },
],
});
expect(JSON.stringify(res.body)).not.toContain("Codex via ACPX");
@@ -1002,6 +1002,52 @@ describe("agent routes adapter validation", () => {
expect(mockAgentService.update.mock.calls.at(-1)?.[1]).toMatchObject({ adapterType: "paperclip_runner", adapterConfig: { provider: "acpx", acpxAgent: "pi", model: "" } });
});
it("keeps Kimi draft models unset through the public create route", async () => {
mockInstanceSettingsService.getExperimental.mockResolvedValue({ enableNativeRunner: true });
const originalCreate = mockAgentService.create.getMockImplementation()!;
mockAgentService.create.mockImplementation(async (...args) => ({ ...await originalCreate(...args), lifecycleHolds: ["setup"] }));
const app = await createApp();
const input = { name: "Kimi draft", adapterType: "paperclip_runner", adapterConfig: { provider: "acpx", acpxAgent: "kimi" } };
const draft = await requestApp(app, base => request(base).post("/api/companies/company-1/agents")
.send({ ...input, setupId: "00000000-0000-4000-8000-000000000092" }));
expect(draft.status, JSON.stringify(draft.body)).toBe(201);
const saved = mockAgentService.create.mock.calls.at(-1)?.[1];
expect(saved).toMatchObject(input);
expect(saved.adapterConfig.model).toBeUndefined();
expect(mockAgentService.create.mock.calls.at(-1)?.[2]).toMatchObject({ setupOnly: true, runnerResolved: true });
mockAgentService.create.mockClear();
const active = await requestApp(app, base => request(base).post("/api/companies/company-1/agents").send(input));
expect(active.status).toBe(422);
expect(mockAgentService.create).not.toHaveBeenCalled();
});
it.each([
["process", undefined, ""],
["kimi_local", "kimi-for-coding", "kimi-for-coding"],
["pi_local", "openrouter/anthropic/claude-sonnet-4.6", ""],
] as const)("preserves Kimi model selection only from the same harness: %s", async (previousAdapterType, previousModel, expectedModel) => {
mockInstanceSettingsService.getExperimental.mockResolvedValue({ enableNativeRunner: true });
mockAgentService.getById.mockResolvedValue({ ...(await mockAgentService.getById()), adapterType: previousAdapterType,
adapterConfig: previousModel === undefined ? {} : { model: previousModel },
status: "paused", lifecycleState: "paused", pauseReason: "setup", lifecycleHolds: ["setup"] });
const app = await createApp();
const response = await requestApp(app, base => request(base).patch("/api/agents/11111111-1111-4111-8111-111111111111")
.send({ adapterType: "paperclip_runner", adapterConfig: { provider: "acpx", acpxAgent: "kimi" } }));
expect(response.status, JSON.stringify(response.body)).toBe(200);
expect(mockAgentService.update.mock.calls.at(-1)?.[1]).toMatchObject({ adapterType: "paperclip_runner",
adapterConfig: { provider: "acpx", acpxAgent: "kimi", model: expectedModel } });
});
it("persists an explicit Kimi model through the public create route", async () => {
mockInstanceSettingsService.getExperimental.mockResolvedValue({ enableNativeRunner: true });
const app = await createApp();
const input = { name: "Kimi", adapterType: "paperclip_runner",
adapterConfig: { provider: "acpx", acpxAgent: "kimi", model: "kimi-for-coding" } };
const response = await requestApp(app, base => request(base).post("/api/companies/company-1/agents").send(input));
expect(response.status, JSON.stringify(response.body)).toBe(201);
expect(mockAgentService.create.mock.calls.at(-1)?.[1]).toMatchObject(input);
});
it.each([{ thinking: "high", status: 200 }, { thinking: "medium", status: 422 }, { command: "custom-pi", status: 422 }])("validates saved Pi settings during direct adapter conversion: %j", async ({ status, ...settings }) => {
const selection = await import("../services/agent-runner-selection.js");
vi.spyOn(selection, "resolveNewAgentRunnerForCompany").mockImplementation(async (_db, _companyId, input) =>
@@ -0,0 +1,405 @@
import { createHash, randomUUID } from "node:crypto";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { eq, sql } from "drizzle-orm";
import { activityLog, agentApiKeys, agents, authUsers, chatCompletionDeliveries, chatTaskHandoffs, companyMemberships, heartbeatRuns, issueComments, issueThreadInteractions, issues, projects } from "@paperclipai/db";
import type { AgentApiKeyScope } from "@paperclipai/shared";
import { createLocalAgentJwt } from "../agent-auth-jwt.js";
import { sendAgentChatMessage } from "../services/agent-chat-messages.js";
import { withConversationReadState } from "../services/agent-chat-read-state.js";
import { instanceSettingsService } from "../services/instance-settings.js";
import { issueService } from "../services/issues.js";
import { activityService } from "../services/activity.js";
import { recordAcceptedChatHandoff } from "../services/chat-completion-delivery.js";
import { conversationReplay, deliverConversationComments, undeliveredConversationComments } from "../services/agent-conversations.js";
import { PaperclipRunnerToolAuthority } from "../services/native-runtime/paperclip-runner-tool-authority.js";
import { CROSS_ISSUE_INFLUENCE_LIMIT } from "../services/cross-issue-influence-limit.js";
import { startRunnerApiTestServer } from "./helpers/runner-api-server.js";
import { recoveryPostgresSupport as support } from "./helpers/recovery-postgres.js";
(support.supported ? describe : describe.skip)("proactive agent chat messages", () => {
let server: Awaited<ReturnType<typeof startRunnerApiTestServer>>;
const originalSecret = process.env.PAPERCLIP_AGENT_JWT_SECRET;
beforeAll(async () => {
process.env.PAPERCLIP_AGENT_JWT_SECRET = randomUUID();
server = await startRunnerApiTestServer({ humanSessions: true });
await instanceSettingsService(server.db).updateExperimental({ enableAgentChat: true });
}, 60_000);
afterAll(async () => {
await server?.close();
if (originalSecret === undefined) delete process.env.PAPERCLIP_AGENT_JWT_SECRET;
else process.env.PAPERCLIP_AGENT_JWT_SECRET = originalSecret;
});
const fixture = () => server.fixture({ connectionScenario: "fresh", apiToolsEnabled: false, disableWakeOnDemand: true });
type Fixture = Awaited<ReturnType<typeof fixture>>;
const input = { body: "## Morning review\n\nThree PRs need your review.", idempotencyKey: "morning-pr-digest" };
const send = (f: Fixture, value: unknown = input) => sendAgentChatMessage(server.db, f, value);
const httpSend = (f: Fixture, value: unknown = input, keyScope: AgentApiKeyScope = { kind: "standard" }, authorizingUserId?: string) => fetch(`${server.apiUrl}/api/agents/me/chat/messages`, {
method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${createLocalAgentJwt(f.agentId, f.companyId, "paperclip_runner", f.runId, f.responsibleUserId, keyScope, authorizingUserId)}` },
body: JSON.stringify(value),
});
const apiKey = async (f: Fixture, scopeConfig: AgentApiKeyScope = { kind: "standard" }) => {
const token = `pcp_${randomUUID()}`;
const [key] = await server.db.insert(agentApiKeys).values({ companyId: f.companyId, agentId: f.agentId, name: "Chat delivery test",
responsibleUserId: f.responsibleUserId, scopeConfig, keyHash: createHash("sha256").update(token).digest("hex") }).returning();
return { ...key, token };
};
const keySend = (f: Fixture, token: string, value: unknown = input) => fetch(`${server.apiUrl}/api/agents/me/chat/messages`, {
method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${token}`, "x-paperclip-run-id": f.runId },
body: JSON.stringify(value),
});
const spendInfluence = (f: Fixture, count: number) => server.db.insert(activityLog).values(Array.from({ length: count }, (_, index) => ({
companyId: f.companyId, actorType: "agent" as const, actorId: f.agentId, agentId: f.agentId, runId: f.runId,
action: "issue.cross_issue_influence_observed", entityType: "issue", entityId: f.issueId,
details: { kind: ["comment", "update", "interaction_resolution"][index % 3] },
})));
it.each(["native", "http"])("enforces the shared run cap for %s delivery without creating an empty chat", async transport => {
const f = await fixture();
await spendInfluence(f, CROSS_ISSUE_INFLUENCE_LIMIT);
if (transport === "native") {
await expect(f.authority.execute({ tool: "send_chat_message", arguments: input, callId: randomUUID() }))
.rejects.toMatchObject({ status: 429, details: { code: "cross_issue_influence_cap_exceeded", cap: CROSS_ISSUE_INFLUENCE_LIMIT } });
} else {
const response = await httpSend(f);
expect(response.status, await response.clone().text()).toBe(429);
expect(await response.json()).toMatchObject({ details: { code: "cross_issue_influence_cap_exceeded", cap: CROSS_ISSUE_INFLUENCE_LIMIT } });
}
const snapshot = await f.snapshot();
expect(snapshot.comments).toEqual([]);
expect(snapshot.issues.filter(issue => issue.conversationAgentId)).toEqual([]);
expect(snapshot.activity).toHaveLength(CROSS_ISSUE_INFLUENCE_LIMIT);
});
it("charges a committed message once across retries, recovery and conflicting content", async () => {
const f = await fixture(), receipt = (await send(f))!;
expect((await httpSend(f)).status).toBe(200);
expect(await f.authority.recover({ tool: "send_chat_message", arguments: input, callId: randomUUID() }))
.toMatchObject({ result: { commentId: receipt.commentId, replayed: true } });
expect((await httpSend(f, { ...input, body: "Different content" })).status).toBe(409);
expect(await f.authority.recover({ tool: "send_chat_message", arguments: { ...input, idempotencyKey: "not-committed" }, callId: randomUUID() })).toBeNull();
expect((await f.snapshot()).activity.filter(row => row.action === "issue.cross_issue_influence_observed")).toHaveLength(1);
expect((await f.snapshot()).comments).toHaveLength(1);
});
it("cannot race native and HTTP delivery past the shared final slot", async () => {
const f = await fixture();
await spendInfluence(f, CROSS_ISSUE_INFLUENCE_LIMIT - 1);
const outcomes = await Promise.all([
f.authority.execute({ tool: "send_chat_message", arguments: { ...input, idempotencyKey: "native-last-slot" }, callId: randomUUID() })
.then(() => 201, error => error.status),
httpSend(f, { ...input, idempotencyKey: "http-last-slot" }).then(response => response.status),
]);
expect(outcomes.sort()).toEqual([201, 429]);
const snapshot = await f.snapshot();
expect(snapshot.activity.filter(row => row.action === "issue.cross_issue_influence_observed")).toHaveLength(CROSS_ISSUE_INFLUENCE_LIMIT);
expect(snapshot.comments).toHaveLength(1);
expect(snapshot.issues.filter(issue => issue.conversationAgentId)).toHaveLength(1);
});
it("supports standard task API keys without changing agent and run attribution", async () => {
const f = await fixture(), key = await apiKey(f);
const response = await keySend(f, key.token);
expect(response.status, await response.clone().text()).toBe(201);
expect((await f.snapshot()).activity).toContainEqual(expect.objectContaining({ action: "issue.comment_added",
actorId: f.agentId, agentId: f.agentId, runId: f.runId, responsibleUserId: f.responsibleUserId }));
});
it("preserves an external authorizer's current access intersection", async () => {
const f = await fixture();
expect((await httpSend(f, input, { kind: "standard" }, randomUUID())).status).toBe(403);
await expect(sendAgentChatMessage(server.db, { ...f, authorizingUserId: randomUUID() }, input)).rejects.toMatchObject({ status: 403 });
expect((await f.snapshot()).issues.filter(issue => issue.conversationAgentId)).toEqual([]);
expect((await f.snapshot()).comments).toEqual([]);
});
it("keeps skill-test JWT and API-key scopes confined to their own task", async () => {
const f = await fixture(), scope = { kind: "skill_test", issueId: f.issueId } as const;
const key = await apiKey(f, scope);
expect((await httpSend(f, input, scope)).status).toBe(403);
expect((await keySend(f, key.token)).status).toBe(403);
expect((await f.snapshot()).issues.filter(issue => issue.conversationAgentId)).toEqual([]);
expect((await f.snapshot()).comments).toEqual([]);
});
it.each(["admitted", "legacy", "saved-task"])("retains a persisted %s skill-test run boundary after task mode changes", async kind => {
const f = await fixture();
const contextSnapshot = kind === "admitted" ? { paperclipAgentJwtScope: { kind: "skill_test", issueId: f.issueId } }
: kind === "legacy" ? { paperclipSkillTest: { id: randomUUID() } }
: { paperclipIssue: { id: f.issueId, workMode: "skill_test" } };
await server.db.update(heartbeatRuns).set({ contextSnapshot }).where(eq(heartbeatRuns.id, f.runId));
await server.db.update(issues).set({ workMode: "skill_test" }).where(eq(issues.id, f.issueId));
await server.db.update(issues).set({ workMode: "standard" }).where(eq(issues.id, f.issueId));
expect((await httpSend(f)).status).toBe(403);
await expect(sendAgentChatMessage(server.db, f, input, { receiptOnly: true })).rejects.toThrow(/skill.test/i);
expect((await f.snapshot()).issues.filter(issue => issue.conversationAgentId)).toEqual([]);
expect((await f.snapshot()).comments).toEqual([]);
});
it("does not let task-bridge credentials publish outside their admitted task", async () => {
const f = await fixture(), key = await apiKey(f, { kind: "task_bridge", parentIssueId: f.issueId });
expect((await keySend(f, key.token)).status).toBe(403);
expect((await f.snapshot()).issues.filter(issue => issue.conversationAgentId)).toEqual([]);
});
it.each(["revoked", "narrowed", "responsible-human"])("rechecks a %s API key after waiting for the destination", async kind => {
const f = await fixture(), receipt = (await send(f))!, key = await apiKey(f);
let unlock!: () => void, ready!: (pid: number) => void;
const locked = new Promise<number>(resolve => { ready = resolve; });
const release = new Promise<void>(resolve => { unlock = resolve; });
const gate = server.db.transaction(async tx => {
const [{ pid }] = await tx.execute(sql<{ pid: number }>`select pg_backend_pid() as pid`);
await tx.select({ id: issues.id }).from(issues).where(eq(issues.id, receipt.conversationId)).for("update");
ready(pid); await release;
});
const pid = await locked;
const delivery = keySend(f, key.token, { ...input, idempotencyKey: "key-after-wait" });
try {
await vi.waitFor(async () => {
const [row] = await server.db.execute(sql<{ blocked: boolean }>`select exists(select 1 from pg_stat_activity
where datname=current_database() and ${pid}=any(pg_blocking_pids(pid))) as blocked`);
expect(row.blocked).toBe(true);
}, { timeout: 5_000, interval: 10 });
await server.db.update(agentApiKeys).set(kind === "revoked" ? { revokedAt: new Date() }
: kind === "narrowed" ? { scopeConfig: { kind: "skill_test", issueId: f.issueId } }
: { responsibleUserId: randomUUID() }).where(eq(agentApiKeys.id, key.id));
unlock(); await gate;
expect((await delivery).status).toBe(403);
expect((await f.snapshot()).comments).toHaveLength(1);
} finally { unlock(); await gate; await delivery; }
}, 30_000);
it("publishes through the native tool without generic API tools, without starting a turn or changing the source", async () => {
const f = await fixture();
expect(f.authority.definitions().map(d => d.name)).toContain("send_chat_message");
const before = await issueService(server.db).getById(f.issueId);
const receipt = await f.authority.execute({ tool: "send_chat_message", arguments: input, callId: randomUUID() }) as { commentId: string; conversationId: string; chatUrl: string };
const snapshot = await f.snapshot();
const comment = snapshot.comments.find(c => c.id === receipt.commentId)!;
expect(comment).toMatchObject({ body: input.body, authorAgentId: f.agentId, createdByRunId: f.runId, onBehalfOfUserId: f.responsibleUserId,
metadata: { chatMessage: { sourceTaskId: f.issueId, sourceRunId: f.runId } } });
expect(snapshot.issues.find(i => i.id === receipt.conversationId)).toMatchObject({ conversationAgentId: f.agentId,
conversationUserId: f.responsibleUserId, conversationState: "waiting", executionRunId: null, status: "in_review" });
expect(await issueService(server.db).getById(f.issueId)).toEqual(before);
expect(await server.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(1);
expect(snapshot.activity.filter(a => a.action === "issue.comment_added")).toHaveLength(1);
expect(await activityService(server.db).runsForIssue(f.companyId, receipt.conversationId)).toEqual([]);
expect(receipt.chatUrl).toContain(`/chats/${f.agentId}`);
expect(await withConversationReadState(server.db, f.companyId, f.responsibleUserId!, [{ id: receipt.conversationId }])).toMatchObject([{ isUnreadForMe: true }]);
expect(await f.authority.recover({ tool: "send_chat_message", arguments: input, callId: randomUUID() }))
.toMatchObject({ result: { commentId: receipt.commentId, replayed: true } });
});
it("creates once under concurrent first sends and recovers a committed response across run retries", async () => {
const f = await fixture();
expect(await f.authority.recover({ tool: "send_chat_message", arguments: input, callId: randomUUID() })).toBeNull();
expect((await f.snapshot()).issues.filter(i => i.conversationAgentId)).toHaveLength(0);
const [a, b] = await Promise.all([send(f), send(f)]);
expect(a!.commentId).toBe(b!.commentId);
expect((await f.snapshot()).comments).toHaveLength(1);
const other = await send(f, { ...input, idempotencyKey: "second" });
expect(other!.commentId).not.toBe(a!.commentId);
await expect(send(f, { ...input, body: "Different digest" })).rejects.toThrow(/different content/);
const nextRunId = randomUUID();
await server.db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, f.runId));
await server.db.insert(heartbeatRuns).values({ id: nextRunId, companyId: f.companyId, agentId: f.agentId,
status: "running", nativeIssueId: f.issueId, responsibleUserId: f.responsibleUserId });
await server.db.update(issues).set({ executionRunId: nextRunId }).where(eq(issues.id, f.issueId));
expect(await send({ ...f, runId: nextRunId })).toMatchObject({ commentId: a!.commentId, replayed: true });
expect((await f.snapshot()).comments).toHaveLength(2);
});
it("supports HTTP routine delivery during an active chat reply and after a context reset", async () => {
const f = await fixture();
await server.db.update(heartbeatRuns).set({ invocationSource: "automation", triggerDetail: "routine" }).where(eq(heartbeatRuns.id, f.runId));
const first = await httpSend(f);
expect(first.status, await first.clone().text()).toBe(201);
const receipt = await first.json();
const replyRun = randomUUID();
await server.db.insert(heartbeatRuns).values({ id: replyRun, companyId: f.companyId, agentId: f.agentId, status: "running",
nativeIssueId: receipt.conversationId, responsibleUserId: f.responsibleUserId, contextSnapshot: { issueId: receipt.conversationId, conversationSessionGeneration: 0 } });
await server.db.update(issues).set({ executionRunId: replyRun, conversationState: "active" }).where(eq(issues.id, receipt.conversationId));
const before = await issueService(server.db).getById(receipt.conversationId);
expect((await httpSend(f, { ...input, idempotencyKey: "during-reply" })).status).toBe(201);
const after = await issueService(server.db).getById(receipt.conversationId);
expect(after).toMatchObject({ executionRunId: replyRun, conversationState: before!.conversationState, status: before!.status });
await server.db.update(issues).set({ conversationSessionGeneration: 1, conversationState: "waiting", executionRunId: null }).where(eq(issues.id, receipt.conversationId));
expect((await httpSend(f, { ...input, idempotencyKey: "after-new" })).status).toBe(201);
await expect(issueService(server.db).addComment(receipt.conversationId, "Stale reply", { agentId: f.agentId, runId: replyRun })).rejects.toThrow(/earlier session/);
expect(await server.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(2);
expect((await f.snapshot()).comments).toHaveLength(3);
});
it("never enqueues a digest during recovery or the next human reply", async () => {
const f = await fixture(), svc = issueService(server.db);
const receipt = (await send(f))!;
const conversation = (await svc.getById(receipt.conversationId))!;
const enqueue = vi.fn(async (_agentId: string, _options: unknown) => null);
expect(await undeliveredConversationComments(server.db, f.companyId, conversation.id)).toEqual([]);
await deliverConversationComments(server.db, conversation, enqueue);
expect(enqueue).not.toHaveBeenCalled();
const reply = await svc.addComment(conversation.id, "Which PR should I review first?", { userId: f.responsibleUserId! }, { clientRequestId: randomUUID() });
expect((await undeliveredConversationComments(server.db, f.companyId, conversation.id)).map(comment => comment.id)).toEqual([reply.id]);
await deliverConversationComments(server.db, conversation, enqueue);
expect(enqueue).toHaveBeenCalledOnce();
expect(enqueue.mock.calls[0]?.[1]).toMatchObject({ requestedByActorType: "user", requestedByActorId: f.responsibleUserId,
contextSnapshot: { wakeCommentId: reply.id, wakeCommentIds: [reply.id] } });
expect(await conversationReplay(server.db, f.companyId, conversation.id, reply.id))
.toContain(JSON.stringify({ author: "agent", body: input.body }));
});
it.each([
{ transport: "native", replay: false }, { transport: "native", replay: true },
{ transport: "http", replay: false }, { transport: "http", replay: true },
])("serializes $transport delivery with a native chat mutation, replay=$replay", async ({ transport, replay }) => {
const f = await fixture(), receipt = (await send(f))!, chatRunId = randomUUID();
await server.db.update(agents).set({ runtimeConfig: { heartbeat: { enabled: false, wakeOnDemand: false, maxConcurrentRuns: 2 } } }).where(eq(agents.id, f.agentId));
await server.db.insert(heartbeatRuns).values({ id: chatRunId, companyId: f.companyId, agentId: f.agentId, status: "running",
runtimeMode: "native", nativeIssueId: receipt.conversationId, responsibleUserId: f.responsibleUserId,
contextSnapshot: { issueId: receipt.conversationId, conversationSessionGeneration: 0 } });
await server.db.update(issues).set({ executionRunId: chatRunId, conversationState: "active" }).where(eq(issues.id, receipt.conversationId));
const chat = new PaperclipRunnerToolAuthority(server.db, { companyId: f.companyId, agentId: f.agentId,
issueId: receipt.conversationId, runId: chatRunId, apiUrl: server.apiUrl, apiToolsEnabled: false });
let unlock!: () => void, ready!: (pid: number) => void;
const locked = new Promise<number>(resolve => { ready = resolve; });
const release = new Promise<void>(resolve => { unlock = resolve; });
const gate = server.db.transaction(async tx => {
const [{ pid }] = await tx.execute(sql<{ pid: number }>`select pg_backend_pid() as pid`);
await tx.select({ id: agents.id }).from(agents).where(eq(agents.id, f.agentId)).for("update");
ready(pid);
await release;
});
const pid = await locked;
const waitForBlocked = async (count: number) => vi.waitFor(async () => {
const [row] = await server.db.execute(sql<{ count: number }>`with recursive blocked(pid) as (
select pid from pg_stat_activity where datname = current_database() and ${pid} = any(pg_blocking_pids(pid))
union select activity.pid from pg_stat_activity activity join blocked on blocked.pid = any(pg_blocking_pids(activity.pid))
where activity.datname = current_database()
) select count(*)::int as count from blocked`);
expect(row.count).toBeGreaterThanOrEqual(count);
}, { timeout: 5000, interval: 10 });
const message = replay ? input : { ...input, idempotencyKey: "concurrent-message" };
let settled: Promise<PromiseSettledResult<unknown>[]> | undefined;
let observedDelivery: Promise<PromiseSettledResult<unknown>[]> | undefined;
try {
const delivery = transport === "native"
? f.authority.execute({ tool: "send_chat_message", arguments: message, callId: randomUUID() })
: httpSend(f, message).then(async response => {
expect(response.status, await response.clone().text()).toBe(replay ? 200 : 201);
return response.json();
});
observedDelivery = Promise.allSettled([delivery]);
await waitForBlocked(1);
const mutation = chat.execute({ tool: "write_document", callId: randomUUID(), arguments: {
key: "concurrent-note", title: "Concurrent note", body: "The chat reply stays independent.", baseRevisionId: null, idempotencyKey: "concurrent-note",
} });
settled = Promise.allSettled([delivery, mutation]);
await waitForBlocked(2);
unlock();
await gate;
const results = await settled;
expect(results.map(result => result.status), JSON.stringify(results)).toEqual(["fulfilled", "fulfilled"]);
expect(results[1]).toMatchObject({ status: "fulfilled", value: { disposition: "applied", document: { key: "concurrent-note" } } });
expect(await issueService(server.db).getById(receipt.conversationId)).toMatchObject({ conversationState: "active" });
} finally {
unlock();
await gate;
await settled;
await observedDelivery;
}
}, 30_000);
it("isolates two humans sharing an agent and two agents serving one human", async () => {
const f = await fixture();
const a = await send(f);
const userId = randomUUID();
await server.db.insert(authUsers).values({ id: userId, name: "Second human", email: `${userId}@fixture.invalid`, emailVerified: true, createdAt: new Date(), updatedAt: new Date() });
await server.db.insert(companyMemberships).values({ companyId: f.companyId, principalType: "user", principalId: userId, status: "active", membershipRole: "member" });
await server.db.update(heartbeatRuns).set({ responsibleUserId: userId }).where(eq(heartbeatRuns.id, f.runId));
const b = await send(f);
expect(b!.conversationId).not.toBe(a!.conversationId);
const [agent] = await server.db.insert(agents).values({ companyId: f.companyId, name: "Second agent", status: "active", adapterType: "process" }).returning();
const runId = randomUUID();
await server.db.insert(heartbeatRuns).values({ id: runId, companyId: f.companyId, agentId: agent.id, status: "running", issueId: f.issueId, responsibleUserId: userId });
await server.db.update(issues).set({ assigneeAgentId: agent.id, executionRunId: runId }).where(eq(issues.id, f.issueId));
const c = await send({ ...f, agentId: agent.id, runId });
expect(new Set([a!.conversationId, b!.conversationId, c!.conversationId]).size).toBe(3);
await expect(issueService(server.db).addComment(a!.conversationId, "Wrong agent", { agentId: agent.id, runId })).rejects.toThrow(/Only this conversation/);
await expect(issueService(server.db).addComment(b!.conversationId, "Bypass", { agentId: f.agentId, runId: f.runId })).rejects.toThrow(/earlier session/);
});
it("delivers to its own chat while preserving pending questions and the original handoff completion", async () => {
const f = await fixture(), svc = issueService(server.db);
const own = await svc.create(f.companyId, { title: "Own chat", assigneeAgentId: f.agentId,
conversationAgentId: f.agentId, conversationUserId: f.responsibleUserId!, conversationState: "waiting", status: "in_review" });
const [coordinator] = await server.db.insert(agents).values({ companyId: f.companyId, name: "Coordinator", status: "active", adapterType: "process" }).returning();
const origin = await svc.create(f.companyId, { title: "Origin chat", assigneeAgentId: coordinator.id,
conversationAgentId: coordinator.id, conversationUserId: f.responsibleUserId!, conversationState: "waiting", status: "in_review" });
await recordAcceptedChatHandoff(server.db, (await svc.getById(f.issueId))!, origin);
const [question] = await server.db.insert(issueThreadInteractions).values({ companyId: f.companyId, issueId: own.id,
kind: "request_confirmation", status: "pending", payload: { version: 1, prompt: "Proceed?" } }).returning();
expect(await send(f)).toMatchObject({ conversationId: own.id });
expect(await server.db.select().from(issueComments).where(eq(issueComments.issueId, origin.id))).toEqual([]);
expect(await server.db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.id, question.id))).toEqual([question]);
expect(await server.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(1);
await svc.update(f.issueId, { status: "done" });
expect(await server.db.select().from(chatCompletionDeliveries).where(eq(chatCompletionDeliveries.taskId, f.issueId)))
.toMatchObject([{ status: "pending", responseCommentId: null }]);
expect(await server.db.select().from(chatTaskHandoffs).where(eq(chatTaskHandoffs.taskId, f.issueId)))
.toMatchObject([{ conversationId: origin.id, agentId: coordinator.id }]);
});
it.each(["ask", "planning", "review", "watchdog", "stopped", "stale", "missing-human", "revoked", "private-task", "private-project", "private-ancestor", "paused-agent"])("rejects %s before creating a chat", async kind => {
const f = await fixture();
if (kind === "ask" || kind === "planning") await server.db.update(issues).set({ workMode: kind }).where(eq(issues.id, f.issueId));
if (kind === "review") await server.db.update(heartbeatRuns).set({ contextSnapshot: { issueId: f.issueId, nativeReviewInteractionId: randomUUID(), nativeReviewDecisionId: randomUUID() } }).where(eq(heartbeatRuns.id, f.runId));
if (kind === "watchdog") await server.db.update(heartbeatRuns).set({ contextSnapshot: { issueId: f.issueId, taskWatchdog: true, watchedIssueId: f.issueId } }).where(eq(heartbeatRuns.id, f.runId));
if (kind === "stopped") await server.db.update(heartbeatRuns).set({ resultJson: { executionCancellation: { state: "requested" } } }).where(eq(heartbeatRuns.id, f.runId));
if (kind === "stale") await server.db.update(issues).set({ executionRunId: null }).where(eq(issues.id, f.issueId));
if (kind === "missing-human") await server.db.update(heartbeatRuns).set({ responsibleUserId: null }).where(eq(heartbeatRuns.id, f.runId));
if (kind === "revoked") await server.db.delete(companyMemberships).where(eq(companyMemberships.principalId, f.responsibleUserId!));
if (kind === "private-task") await server.db.update(issues).set({ visibility: "private" }).where(eq(issues.id, f.issueId));
if (kind === "private-project") await server.db.update(projects).set({ visibility: "private", privacyOwnerUserId: f.responsibleUserId }).where(eq(projects.id, f.projectId));
if (kind === "private-ancestor") {
await server.db.update(issues).set({ visibility: "private" }).where(eq(issues.id, f.blockerId));
await server.db.update(issues).set({ parentId: f.blockerId, privacyParentIssueId: f.blockerId, privacyRootIssueId: f.blockerId }).where(eq(issues.id, f.issueId));
}
if (kind === "paused-agent") await server.db.update(agents).set({ status: "paused" }).where(eq(agents.id, f.agentId));
await expect(send(f)).rejects.toThrow();
expect((await f.snapshot()).comments).toHaveLength(0);
expect((await f.snapshot()).issues.filter(i => i.conversationAgentId)).toHaveLength(0);
});
it("rejects forged destinations and server-owned provenance, and respects the feature gate", async () => {
const f = await fixture();
expect((await httpSend(f, { ...input, agentId: f.agentId })).status).toBe(400);
await instanceSettingsService(server.db).updateExperimental({ enableAgentChat: false });
try { expect((await httpSend(f)).status).toBe(404); }
finally { await instanceSettingsService(server.db).updateExperimental({ enableAgentChat: true }); }
const receipt = await send(f);
const [comment] = await server.db.select().from(issueComments).where(eq(issueComments.id, receipt!.commentId));
await expect(issueService(server.db).addComment(f.issueId, "Forged source", { agentId: f.agentId, runId: f.runId }, { metadata: comment.metadata! })).rejects.toThrow(/server-owned/);
});
it("keeps unseen concurrent arrivals unread with exact comment cutoffs and monotonic receipts", async () => {
const f = await fixture();
const a = (await send(f))!, b = (await send(f, { ...input, idempotencyKey: "later" }))!;
await server.db.update(issueComments).set({ createdAt: sql`'2026-10-30 12:00:00.123456+00'::timestamptz` }).where(eq(issueComments.id, a.commentId));
await server.db.update(issueComments).set({ createdAt: sql`'2026-10-30 12:00:00.123789+00'::timestamptz` }).where(eq(issueComments.id, b.commentId));
const cookie = await server.humanSession(f.responsibleUserId!);
const mark = (id: string) => fetch(`${server.apiUrl}/api/issues/${a.conversationId}/read`, { method: "POST", headers: { Cookie: cookie, Origin: server.apiUrl, "Content-Type": "application/json" }, body: JSON.stringify({ throughCommentId: id }) });
const unread = async () => (await withConversationReadState(server.db, f.companyId, f.responsibleUserId!, [{ id: a.conversationId }]))[0].isUnreadForMe;
const response = await mark(a.commentId);
expect(response.status, await response.text()).toBe(200);
expect(await unread()).toBe(true);
expect((await mark(b.commentId)).status).toBe(200);
expect(await unread()).toBe(false);
await mark(a.commentId);
expect(await unread()).toBe(false);
const foreign = await server.fixture({ connectionScenario: "fresh" });
const foreignMessage = (await send(foreign))!;
expect((await mark(foreignMessage.commentId)).status).toBe(404);
const chats = await fetch(`${server.apiUrl}/api/companies/${f.companyId}/chats`, { headers: { Cookie: cookie } });
expect(chats.status).toBe(200);
expect(await chats.json()).toMatchObject([{ id: a.conversationId, isUnreadForMe: false }]);
});
});
@@ -29,9 +29,9 @@ import {
issues,
} from "@paperclipai/db";
import {
getEmbeddedPostgresTestSupport,
startEmbeddedPostgresTestDatabase,
} from "./helpers/embedded-postgres.js";
recoveryPostgresSupport as support,
startRecoveryPostgres,
} from "./helpers/recovery-postgres.js";
import { issueService } from "../services/issues.js";
import { documentService } from "../services/documents.js";
import { getTaskPlanContext } from "../services/task-plan-context.js";
@@ -58,16 +58,15 @@ import {
heartbeatService,
} from "../services/heartbeat.js";
const support = await getEmbeddedPostgresTestSupport();
(support.supported ? describe : describe.skip)(
"persistent agent conversations",
() => {
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let database: Awaited<ReturnType<typeof startRecoveryPostgres>>;
let db: ReturnType<typeof createDb>;
let companyId: string;
let agentId: string;
beforeAll(async () => {
database = await startEmbeddedPostgresTestDatabase(
database = await startRecoveryPostgres(
"paperclip-agent-conversations-",
);
db = createDb(database.connectionString);
@@ -27,6 +27,7 @@ import { remoteTerminationReceipt } from "../services/remote-execution-terminati
import { AgentDirectoryReuseInvalidatedError, agentDirectoryWorkingCopyService } from "../services/agent-directory-working-copies.js";
import { withDirectoryMergeLock } from "@paperclipai/adapter-utils/workspace-restore-merge";
import { heartbeatRunEvents } from "@paperclipai/db";
import { logger } from "../middleware/logger.js";
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_AGENT_DIRECTORY_TEST_DATABASE_URL");
@@ -393,6 +394,106 @@ describe("persistent agent directories", () => {
expect((await copies.get(companyId, first.runId))?.receipt?.retainedByRunId).toBeUndefined();
await copies.collectStopped({ companyId, runId: first.runId });
});
it("adopts unchanged ownership after collection retry metadata changes before claim", async () => {
const first = await run();
expect(await copies.hasChanges({ companyId, runId: first.runId })).toBe(false);
await db.update(heartbeatRuns).set({ status: "succeeded", runtimeMode: "native" }).where(eq(heartbeatRuns.id, first.runId));
const secondId = await nextRun();
const transaction = db.transaction.bind(db);
let transactions = 0;
let injected = false;
const spy = vi.spyOn(db, "transaction").mockImplementation((async (callback, options) => {
transactions++;
if (transactions === 2 && !injected) {
injected = true;
const before = (await copies.get(companyId, first.runId))!;
vi.useFakeTimers({ toFake: ["Date"] });
vi.setSystemTime(new Date(before.updatedAt.getTime() + 1000));
try { await copies.recoverStopped(); } finally { vi.useRealTimers(); }
const after = (await copies.get(companyId, first.runId))!;
expect(after.updatedAt.getTime()).toBeGreaterThan(before.updatedAt.getTime());
expect(after.nextAttemptAt).toBeInstanceOf(Date);
for (const field of ["state", "baseHash", "localRoot", "executionRoot", "processStoppedAt", "receipt", "attempts"] as const)
expect(after[field]).toEqual(before[field]);
}
return transaction(callback, options);
}) as typeof db.transaction);
try {
const adopted = await copies.adopt({ companyId, agentId, runId: secondId, previousRunId: first.runId, cwd: home });
expect(injected).toBe(true);
expect(adopted).toMatchObject({ runId: secondId, localRoot: first.localRoot, executionRoot: first.executionRoot });
expect((await copies.get(companyId, first.runId))?.receipt?.retainedByRunId).toBe(secondId);
} finally { spy.mockRestore(); vi.useRealTimers(); expect(injected).toBe(true); }
});
it.each(["materialization", "checkpoint", "stopped", "state"])("rejects changed %s authority with the same baseline before adoption", async change => {
const first = await run();
expect(await copies.hasChanges({ companyId, runId: first.runId })).toBe(false);
const secondId = await nextRun();
const warning = vi.spyOn(logger, "warn");
const privateValue = "private-materialization-/tmp/untrusted-token-do-not-log";
const transaction = db.transaction.bind(db);
let calls = 0;
let injected = false;
const spy = vi.spyOn(db, "transaction").mockImplementation((async (callback, options) => {
if (++calls === 2) {
injected = true;
const prior = (await copies.get(companyId, first.runId))!;
const values = change === "materialization" ? { receipt: { ...prior.receipt, materializationIdentity: privateValue } }
: change === "checkpoint" ? { receipt: { ...prior.receipt, checkpointState: "saved" } }
: change === "stopped" ? { processStoppedAt: new Date() } : { state: "pending_collection" };
await db.update(agentInstructionWorkingCopies).set({ ...values, updatedAt: new Date(Date.now() + 1000) })
.where(eq(agentInstructionWorkingCopies.runId, first.runId));
expect((await copies.get(companyId, first.runId))!.baseHash).toBe(prior.baseHash);
}
return transaction(callback, options);
}) as typeof db.transaction);
try {
await expect(copies.adopt({ companyId, agentId, runId: secondId, previousRunId: first.runId, cwd: home }))
.rejects.toMatchObject({ status: 409, details: { stage: "locked_adoption_authority", predecessorPresent: true,
observedAfterInitialRead: true, changedFields: expect.arrayContaining([change === "materialization" || change === "checkpoint" ? "receipt" : change === "stopped" ? "processStoppedAt" : "state"]) } });
expect(injected).toBe(true);
expect(warning).toHaveBeenCalledWith({ companyId, agentId, runId: secondId, previousRunId: first.runId,
adoptionConflict: { stage: "locked_adoption_authority", predecessorPresent: true, observedAfterInitialRead: true,
changedFields: [change === "materialization" || change === "checkpoint" ? "receipt" : change === "stopped" ? "processStoppedAt" : "state"] } },
"Agent directory adoption conflict");
expect(JSON.stringify(warning.mock.calls)).not.toContain(privateValue);
expect(await copies.get(companyId, secondId)).toBeNull();
expect((await copies.get(companyId, first.runId))?.receipt?.retainedByRunId).toBeUndefined();
expect(await fs.readFile(path.join(root, entryFile), "utf8")).toBe(initial);
} finally { spy.mockRestore(); warning.mockRestore(); }
});
it("reports closed claim predicates when retirement wins before the final owner read", async () => {
const first = await run();
expect(await copies.hasChanges({ companyId, runId: first.runId })).toBe(false);
const secondId = await nextRun();
const warning = vi.spyOn(logger, "warn");
const transaction = db.transaction.bind(db);
let injected = false;
const spy = vi.spyOn(db, "transaction").mockImplementation((async (callback, options) => {
const result = await transaction(callback, options);
if (!injected) {
injected = true;
await db.update(agentInstructionWorkingCopies).set({ state: "pending_collection" })
.where(eq(agentInstructionWorkingCopies.runId, first.runId));
}
return result;
}) as typeof db.transaction);
try {
await expect(copies.adopt({ companyId, agentId, runId: secondId, previousRunId: first.runId, cwd: home }))
.rejects.toMatchObject({ status: 409, details: { stage: "adoption_claim", observedAfterClaim: true,
predecessorPresent: true, conditions: { unchangedTurn: false, timestampMatches: true, baselineMatches: true,
localRootMatches: true, unstopped: true, successorAbsent: true, receiptMatches: true } } });
expect(warning).toHaveBeenCalledWith({ companyId, agentId, runId: secondId, previousRunId: first.runId,
adoptionConflict: { stage: "adoption_claim", observedAfterClaim: true, predecessorPresent: true,
conditions: { unchangedTurn: false, timestampMatches: true, baselineMatches: true, localRootMatches: true,
unstopped: true, successorAbsent: true, receiptMatches: true } } }, "Agent directory adoption conflict");
expect(await copies.get(companyId, secondId)).toBeNull();
expect((await copies.get(companyId, first.runId))?.receipt?.retainedByRunId).toBeUndefined();
} finally { spy.mockRestore(); warning.mockRestore(); }
});
it("allows only one stale concurrent adoption with identical updatedAt timestamps", async () => {
const fixed = new Date("2030-01-01T00:00:00.000Z");
vi.useFakeTimers({ toFake: ["Date"] });
@@ -61044,7 +61044,8 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
const beforeComments = await db
.select({ id: issueComments.id })
.from(issueComments)
.where(eq(issueComments.issueId, conversation.issueId));
.where(eq(issueComments.issueId, conversation.issueId))
.orderBy(issueComments.id);
const beforeWakes = wakeup.mock.calls.length;
const providerRuntime = runtime.endpoints.get(endpoint.id)!;
const beforeProvider = {
@@ -61201,7 +61202,8 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
db
.select({ id: issueComments.id })
.from(issueComments)
.where(eq(issueComments.issueId, conversation.issueId)),
.where(eq(issueComments.issueId, conversation.issueId))
.orderBy(issueComments.id),
).resolves.toEqual(beforeComments);
expect(wakeup.mock.calls.length).toBe(beforeWakes);
expect({
@@ -5,7 +5,7 @@ import { and, asc, eq } from "drizzle-orm";
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { agents, agentWakeupRequests, chatCompletionDeliveries as deliveries, chatTaskHandoffs as handoffs,
companies, createDb, documents, environmentLeases, heartbeatRuns, issueComments, issueRecoveryActions, issueThreadInteractions, issues } from "@paperclipai/db";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { recoveryPostgresSupport as support, startRecoveryPostgres } from "./helpers/recovery-postgres.js";
import { settleInterruptedNativeBootstrap, terminalizeLegacyExecution } from "../services/legacy-execution-recovery.js";
import { getExecutionBlocker } from "../services/execution-blocker.js";
import { settleUnrecoverableExecutions } from "../services/execution-recovery-resolution.js";
@@ -16,12 +16,11 @@ import { instanceSettingsService } from "../services/instance-settings.js";
import { chatCompletionDeliveryService, isCompletedOnboardingHandoffWake, prepareChatCompletionTurn, recordChatCompletion, recordChatHandoff } from "../services/chat-completion-delivery.js";
import { heartbeatService, shouldQueueFollowupForRunningIssueWake } from "../services/heartbeat.js";
const support = await getEmbeddedPostgresTestSupport();
(support.supported ? describe : describe.skip)("chat completion delivery", () => {
let db: ReturnType<typeof createDb>;
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let temporary: Awaited<ReturnType<typeof startRecoveryPostgres>>;
beforeAll(async () => {
temporary = await startEmbeddedPostgresTestDatabase("chat-completion-"); db = createDb(temporary.connectionString);
temporary = await startRecoveryPostgres("chat-completion-"); db = createDb(temporary.connectionString);
await instanceSettingsService(db).updateExperimental({ enableAgentChat: true });
}, 30_000);
afterAll(async () => { await temporary?.cleanup(); });
@@ -11,9 +11,8 @@ import { issueService } from "../services/issues.js";
import { documentService } from "../services/documents.js";
import { activityService } from "../services/activity.js";
import { instanceSettingsService } from "../services/instance-settings.js";
import { getEmbeddedPostgresTestSupport } from "./helpers/embedded-postgres.js";
import { recoveryPostgresSupport as support } from "./helpers/recovery-postgres.js";
const support = await getEmbeddedPostgresTestSupport();
(support.supported ? describe : describe.skip)("chat project tool handoff", () => {
let server: Awaited<ReturnType<typeof startRunnerApiTestServer>>;
const originalSecret = process.env.PAPERCLIP_AGENT_JWT_SECRET;
@@ -785,6 +785,10 @@ describe("connection management with production default agents", () => {
const created = await f.request("POST", `/companies/${f.companyId}/tools/connections`, { name: "Selected Google services", transport: "mcp_remote", authKind: "oauth", credentialPolicy: "per_user", status: "draft",
config: { url: "https://1.1.1.1/mcp", sourceTemplateKey: "google-workspace", connectionMethodKey: "paperclip-workspace" } });
expect(created, JSON.stringify(created)).toMatchObject({ status: 201 });
const handoff = await f.request("POST", `/tools/oauth/${created.body.id}/start`, { asCurrentUser: true });
expect(handoff, JSON.stringify(handoff)).toMatchObject({ status: 200, body: { expiresAt: null, requiredUserAction: { userId: f.userId } } });
const [unselected] = await server.db.select().from(toolConnections).where(eq(toolConnections.id, created.body.id));
expect(unselected.config.workspaceSetup).toBeUndefined();
const services = { gmail: "read" as const };
const started = await f.request("POST", `/tools/oauth/${created.body.id}/start`, { asCurrentUser: true, workspaceServices: services });
expect(started, JSON.stringify(started)).toMatchObject({ status: 200, body: { expiresAt: null, requiredUserAction: { userId: f.userId } } });
@@ -797,6 +801,10 @@ describe("connection management with production default agents", () => {
[f.userId]: { requestedScopes: googleWorkspaceSelectedScopes(services) },
} } } } });
expect(await server.db.select().from(toolOauthStates).where(eq(toolOauthStates.connectionId, saved.id))).toHaveLength(0);
expect(await f.request("POST", `/tools/oauth/${saved.id}/start`, { asCurrentUser: true })).toMatchObject({ status: 200, body: { authorizationUrl: started.body.authorizationUrl } });
const [handedOff] = await server.db.select().from(toolConnections).where(eq(toolConnections.id, saved.id));
expect(handedOff.config.workspaceSetup).toBeUndefined();
expect(handedOff.config.oauth).toEqual(saved.config.oauth);
});
it("discovers and invokes connection APIs through default production native bindings", async () => {
@@ -1,7 +1,7 @@
import { randomUUID } from "node:crypto";
import { and, eq } from "drizzle-orm";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { agents, assets, chatTaskHandoffs, companies, companyMemberships, createDb, documentRevisions, documents, heartbeatRuns,
import { agents, assets, chatTaskHandoffs, companies, companyMemberships, createDb, applyPendingMigrations, decisionRetention, decisionTriage, documentRevisions, documents, heartbeatRuns,
inboxDismissals, issueAttachments, issueComments, issueDocuments, issues, issueThreadInteractions } from "@paperclipai/db";
import { requestConfirmationTargetSchema, suggestedTaskDraftSchema } from "@paperclipai/shared";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
@@ -10,6 +10,7 @@ import { issueService } from "../services/issues.js";
import { instanceSettingsService } from "../services/instance-settings.js";
import { conversationDecisionReceiptsService } from "../services/conversation-decision-receipts.js";
import { attentionService } from "../services/attention.js";
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
describe("Coordinator task and artifact contracts", () => {
it("keeps a proposed plan until explicit acceptance and requires immutable attachment identity", () => {
@@ -20,13 +21,15 @@ describe("Coordinator task and artifact contracts", () => {
});
});
const support = await getEmbeddedPostgresTestSupport();
const externalDatabase = resolveExternalTestDatabaseUrl("PAPERCLIP_COORDINATOR_HANDOFF_TEST_DATABASE_URL");
const support = externalDatabase ? { supported: true } : await getEmbeddedPostgresTestSupport();
(support.supported ? describe : describe.skip)("Coordinator canonical handoffs", () => {
let db: ReturnType<typeof createDb>;
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
beforeAll(async () => {
temporary = await startEmbeddedPostgresTestDatabase("coordinator-handoff-");
db = createDb(temporary.connectionString);
if (externalDatabase) await applyPendingMigrations(externalDatabase);
else temporary = await startEmbeddedPostgresTestDatabase("coordinator-handoff-");
db = createDb(externalDatabase ?? temporary!.connectionString);
await instanceSettingsService(db).updateExperimental({ enableIsolatedWorkspaces: false, enableAgentChat: true });
}, 30_000);
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); });
@@ -194,7 +197,8 @@ const support = await getEmbeddedPostgresTestSupport();
all: true, allowUnscopedAll: true,
actor: { type: "board", userId: f.userId, source: "session", ignoreInstanceAdmin: true }, userId: f.userId,
});
expect(await f.pendingQuestions()).toEqual(canonical.items.filter(item => item.subject.id === question.id));
const withoutRank = ({ rank: _rank, ...item }: (typeof canonical.items)[number]) => item;
expect((await f.pendingQuestions()).map(withoutRank)).toEqual(canonical.items.filter(item => item.subject.id === question.id).map(withoutRank));
expect(await f.pendingQuestions()).toMatchObject([{ sourceKind: "issue_thread_interaction",
subject: { id: question.id, status: "pending" }, relatedIssue: { id: task.id }, detail: { kind: "questions" } }]);
await f.service.answerQuestions(task, question.id, { answers: [{ questionId: "channel", optionIds: ["blog"] }] }, { userId: f.userId });
@@ -202,6 +206,35 @@ const support = await getEmbeddedPostgresTestSupport();
expect((await f.receipts()).items).toEqual(expect.arrayContaining([expect.objectContaining({ sourceId: question.id, outcome: "answered" })]));
});
it("does not materialize unrelated company attention when opening task questions", async () => {
const f = await seed();
const task = await f.direct();
const question = await f.ask(task);
const [unrelated] = await db.insert(issues).values({ companyId: f.companyId, title: "Unrelated", status: "blocked" }).returning();
const unrelatedQuestion = await f.ask(unrelated);
expect((await f.pendingQuestions()).map(item => item.subject.id)).toEqual([question.id]);
const materialized = await db.select().from(decisionRetention).where(eq(decisionRetention.companyId, f.companyId));
expect(materialized.map(row => row.sourceId)).toEqual([question.id]);
expect(materialized.map(row => row.sourceId)).not.toContain(unrelatedQuestion.id);
const options = { actor: { type: "board" as const, source: "local_implicit" as const, userId: f.userId }, userId: f.userId };
expect(await attentionService(db).taskQuestions(f.companyId, [], options)).toEqual([]);
expect(await attentionService(db).taskQuestions(randomUUID(), [task.id], options)).toEqual([]);
});
it("keeps scoped questions consistent with snooze and archive policy", async () => {
const f = await seed();
const task = await f.direct();
const question = await f.ask(task);
expect(await f.pendingQuestions()).toHaveLength(1);
await db.insert(decisionTriage).values({ companyId: f.companyId, sourceKind: "issue_thread_interaction",
sourceId: question.id, snoozedUntil: new Date(Date.now() + 60_000), setByType: "user", setByUserId: f.userId });
expect(await f.pendingQuestions()).toEqual([]);
await db.delete(decisionTriage).where(eq(decisionTriage.companyId, f.companyId));
expect(await f.pendingQuestions()).toHaveLength(1);
await db.update(decisionRetention).set({ archivedAt: new Date() }).where(eq(decisionRetention.companyId, f.companyId));
expect(await f.pendingQuestions()).toEqual([]);
});
it("shows only questions from tasks handed off by this agent in the current conversation generation", async () => {
const f = await seed();
const first = await f.handoff();
@@ -383,6 +383,36 @@ describeEmbeddedPostgres("decision queue routes", () => {
.toHaveLength(1);
});
it("batches repeated seed checks and records only new memberships across concurrent reads", async () => {
const { companyId } = await seed();
const service = decisionQueueService(db);
const items = Array.from({ length: 501 }, () => ({ sourceKind: "issue_thread_interaction" as const,
sourceId: randomUUID(), issueId: null, interactionKind: "ask_user_questions" }));
await service.materializeSeedSources(companyId, items.slice(0, 2));
await Promise.all([
service.materializeSeedSources(companyId, [...items, items[0]!]),
service.materializeSeedSources(companyId, items),
]);
const memberships = await db.select().from(decisionQueueItems).where(eq(decisionQueueItems.companyId, companyId));
expect(new Set(memberships.map(item => item.sourceId))).toEqual(new Set(items.map(item => item.sourceId)));
expect(memberships).toHaveLength(items.length);
const events = await db.select().from(decisionTriageEvents).where(and(
eq(decisionTriageEvents.companyId, companyId), eq(decisionTriageEvents.action, "queue_item.seeded")));
const activity = await db.select().from(activityLog).where(and(
eq(activityLog.companyId, companyId), eq(activityLog.action, "decision_queue_item.seeded")));
expect(events).toHaveLength(items.length);
expect(activity).toHaveLength(items.length);
const queries: string[] = [];
const previousDebug = db.$client.options.debug;
try {
db.$client.options.debug = (_connection, text) => queries.push(text);
await service.materializeSeedSources(companyId, items);
} finally { db.$client.options.debug = previousDebug; }
expect(queries.filter(query => query.startsWith('insert into "decision_queue_items"')).length).toBeLessThan(10);
expect(await db.select().from(decisionTriageEvents).where(and(
eq(decisionTriageEvents.companyId, companyId), eq(decisionTriageEvents.action, "queue_item.seeded")))).toHaveLength(items.length);
});
it("records agent decide-by, preserves override history, and exposes the board override attribution", async () => {
const { companyId, agentId, issueId } = await seed();
const agent = agentActor(companyId, agentId);
@@ -264,7 +264,7 @@ describe("deferred receipt durable dispatch", () => {
for (const gate of ["budget", "agent"] as const) {
deps.budgets.getInvocationBlock.mockResolvedValue(gate === "budget"
? { scopeType: "agent", scopeId: f.agent.id, scopeName: f.agent.name, reason: "Budget blocked" } : null);
deps.getAgentInvokability.mockResolvedValue({ invokable: false, message: "Paused", reason: "paused", invalidOrgChain: false });
deps.getAgentInvokability.mockResolvedValue({ invokable: false, message: "Paused", reason: "paused", details: {}, invalidOrgChain: false });
const error = await queue.enqueueWakeup(f.agent.id, { source: "on_demand", reason: "task_retry",
requestedByActorType: "user", requestedByActorId: f.actorId, payload: { issueId: f.issue.id },
contextSnapshot: { issueId: f.issue.id, responsibleUserId: f.actorId } }, requestId).catch(cause => cause);
@@ -703,7 +703,7 @@ describe("deferred receipt durable dispatch", () => {
it("retains a moved interrupt grace deadline after an ordinary saved queue reorder", async () => {
const f = await fixture(), initialAt = new Date();
const comments = await db.insert(issueComments).values(["first", "second"].map(body => ({
companyId: f.companyId, issueId: f.issue.id, body, authorType: "user", authorUserId: f.actorId,
companyId: f.companyId, issueId: f.issue.id, body, authorType: "user" as const, authorUserId: f.actorId,
}))).returning();
const payload = { issueId: f.issue.id, queuedCommentInterrupt: { actorId: f.actorId },
_paperclipWakeContext: { wakeCommentIds: comments.map(row => row.id) } };
@@ -799,7 +799,7 @@ describe("deferred receipt durable dispatch", () => {
}
expect(resume).toHaveBeenCalledTimes(2);
});
it("reactivates a saved Retry through ordinary restore cancelled task", async () => {
it.each([false, true])("reactivates a saved Retry through ordinary restore cancelled task (held unrelated Retry: %s)", async heldFollowup => {
const f = await fixture(), actor = { actorType: "user" as const, actorId: f.actorId, userId: f.actorId };
const request = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId,
observedRunId: f.source.id, idempotencyKey: randomUUID() });
@@ -813,20 +813,71 @@ describe("deferred receipt durable dispatch", () => {
await heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !drain).ready;
await waitForReal(() => { expect(coordinator.nextWakeAt()).toBeNull(); expect(idleWorkSnapshot().active).toBe(0); }, { timeout: 5_000 });
expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!)))[0].runId).toBeNull();
await commitEligibility();
let runId: string | null = null;
await waitForReal(async () => {
const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!));
expect(wake.runId, JSON.stringify(wake)).toBeTruthy(); runId = wake.runId;
}, { timeout: 15_000 });
await waitForReal(async () => {
await heartbeat.drainActiveRunExecutions();
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!));
expect(run.status, JSON.stringify({ error: run.error, code: run.errorCode })).toBe("succeeded");
expect(run.stdoutExcerpt).toContain("receipt-proof");
}, { timeout: 15_000 });
expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${request.requestId}`)).toHaveLength(1);
await finish(f.issue.id); await quiet();
let releaseFollowup: (() => void) | undefined;
let draining: Promise<void> | undefined;
let drained = false;
const providerErrors: unknown[] = [];
try {
if (heldFollowup) {
const other = await fixture();
const release = new Promise<void>(resolve => { releaseFollowup = resolve; });
let admitted = false;
const listener = createServer(async (incoming, response) => {
try {
let body = "";
for await (const chunk of incoming) body += chunk;
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, body));
expect(run).toMatchObject({ companyId: other.companyId, agentId: other.agent.id, status: "running" });
admitted = true;
await release;
await issueService(db).update(other.issue.id, { status: "done" });
response.end("ok");
} catch (error) { providerErrors.push(error); response.statusCode = 500; response.end("failed"); }
});
extraStops.push(async () => {
const closed = new Promise<void>(resolve => listener.close(() => resolve()));
listener.closeAllConnections(); await closed;
});
await new Promise<void>(resolve => listener.listen(0, "127.0.0.1", resolve));
const port = (listener.address() as { port: number }).port;
const script = `fetch("http://127.0.0.1:${port}", {method:"POST",body:process.env.PAPERCLIP_RUN_ID}).then(async response=>{if(!response.ok)throw new Error(await response.text());console.log("held-retry-proof")}).catch(error=>{console.error(error);process.exitCode=1})`;
await db.update(agents).set({ adapterConfig: { command: process.execPath, args: ["-e", script], cwd: process.cwd() } }).where(eq(agents.id, other.agent.id));
await requestTaskRetry({ db, companyId: other.companyId, issueId: other.issue.id, actorId: other.actorId,
observedRunId: other.source.id, idempotencyKey: randomUUID() });
await waitForReal(() => expect(admitted).toBe(true), { timeout: 15_000 });
}
await commitEligibility();
let runId: string | null = null;
await waitForReal(async () => {
const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!));
expect(wake.runId, JSON.stringify(wake)).toBeTruthy(); runId = wake.runId;
}, { timeout: 15_000 });
if (heldFollowup) {
await waitForReal(async () => {
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!));
expect(run.status).toBe("succeeded"); expect(run.stdoutExcerpt).toContain("receipt-proof");
}, { timeout: 15_000 });
draining = heartbeat.drainActiveRunExecutions().then(() => { drained = true; });
await immediate();
expect(drained).toBe(false);
}
// Observe this Retry before closing the task and draining unrelated runs.
await waitForReal(async () => {
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!));
expect(run.status, JSON.stringify({ error: run.error, code: run.errorCode })).toBe("succeeded");
expect(run.stdoutExcerpt).toContain("receipt-proof");
}, { timeout: 15_000 });
expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${request.requestId}`)).toHaveLength(1);
if (heldFollowup) expect(drained).toBe(false);
releaseFollowup?.();
await finish(f.issue.id);
await draining;
expect(providerErrors).toEqual([]);
await quiet();
} finally {
releaseFollowup?.();
await draining;
}
});
it("member archive publishes committed eligibility while preserving a cancelled Retry tombstone", async () => {
const f = await fixture(), departing = `departing-${randomUUID()}`;
@@ -2,18 +2,19 @@ import { enableRunFailureReporting } from "./helpers/run-failure-reporting.js";
import { createHash, randomUUID } from "node:crypto";
import { trace } from "@opentelemetry/api";
import { InMemorySpanExporter, SimpleSpanProcessor, TracerProvider } from "@opentelemetry/sdk-trace";
import { access, chmod, mkdir, readdir, readFile, mkdtemp, rm, stat } from "node:fs/promises";
import { access, chmod, mkdir, readdir, readFile, mkdtemp, rm, stat, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { eq } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import { agents, authUsers, companies, companyMemberships, companySkills, companySkillTestRuns, createDb, heartbeatRunEvents, heartbeatRuns, issues, projects, plugins, environments } from "@paperclipai/db";
import { agents, authUsers, companies, companyMemberships, companySkills, companySkillTestRuns, createDb, executionWorkspaces, heartbeatRunEvents, heartbeatRuns, issues, projects, plugins, environments } from "@paperclipai/db";
import { companySkillService } from "../services/company-skills.js";
import { withSkillVersionCacheLock } from "../services/skill-version-cache.js";
import { registerServerAdapter, unregisterServerAdapter, type AdapterExecutionContext } from "../adapters/index.js";
import { HttpError } from "../errors.js";
import { heartbeatService } from "../services/heartbeat.js";
import { bindHeartbeatRunnerHomeMode, prepareHeartbeatExecutionConfiguration, type HeartbeatExecutionConfigurationInput } from "../services/heartbeat/execution-configuration.js";
import { buildEffectiveRunSessionConfigMetadata, resolveTaskSessionConfigFreshness } from "../services/heartbeat/workspaces.js";
import { aiRuntimeConnectionBindingSchema } from "@paperclipai/shared";
import { secretService } from "../services/secrets.js";
import { instanceSettingsService } from "../services/instance-settings.js";
@@ -24,6 +25,12 @@ import * as assignedMcp from "../services/native-runtime/assigned-mcp-tools.js";
import * as sentry from "../sentry.js";
import { waitForPendingRunFailureReports } from "../services/run-failure-report.js";
import { recoveryPostgresSupport, startRecoveryPostgres } from "./helpers/recovery-postgres.js";
import { selectHeartbeatEnvironment, type HeartbeatEnvironmentSelectionInput } from "../services/heartbeat/environment-selection.js";
import { environmentService } from "../services/environments.js";
import { executionWorkspaceService } from "../services/execution-workspaces.js";
import { buildExecutionWorkspaceAdapterConfig } from "../services/execution-workspace-policy.js";
import { buildNativeExecutionInput } from "../services/native-runtime/native-execution-input.js";
import { nativeRuntimeContextFixture } from "../services/native-runtime/runtime-context.test-fixture.js";
function deferred<T>() {
let resolve!: (value: T) => void;
@@ -35,6 +42,44 @@ const skillFailures = vi.hoisted(() => new Map<string, string>());
const skillWaits = vi.hoisted(() => new Map<string, () => Promise<void>>());
describe("Runner home mode during recovery preparation", () => {
const fingerprint = (effectiveAdapterConfig: Record<string, unknown>) => buildEffectiveRunSessionConfigMetadata({
adapterType: "paperclip_runner", effectiveAdapterConfig, agentRuntimeConfig: {}, issueOverrides: null,
workspaceConfig: {}, environment: null, environmentEnv: null, projectEnv: null, routineEnv: null, runtimeSkills: [],
});
it.each([
{ schema: "paperclip.native-execution-input.v5" },
{ schema: "paperclip.native-execution-input.v7", homeMode: "isolated" },
])("preserves the implicit isolated fingerprint through recovery and the next run: %j", async saved => {
const config = { provider: "codex", model: "fixture-model" };
// This is the effective configuration shape built by preparation, including
// its optional property when no explicit home mode was configured.
const prepare = (input: unknown) => ({ ...config, homeMode: bindHeartbeatRunnerHomeMode(config, input).homeMode });
const fresh = await fingerprint(prepare(null));
const recovered = await fingerprint(prepare(saved));
const next = await fingerprint(prepare(null));
expect(recovered.fingerprint).toBe(fresh.fingerprint);
expect(next.fingerprint).toBe(recovered.fingerprint);
expect(resolveTaskSessionConfigFreshness({ hasTaskSession: true, configuredModel: config.model,
taskSessionParams: { __paperclipConfiguredModel: config.model, __paperclipConfigFingerprint: recovered.fingerprint,
__paperclipConfigFingerprintVersion: recovered.version, __paperclipConfigCategoryFingerprints: recovered.categoryFingerprints },
configMetadata: next })).toMatchObject({ reset: false, changedCategories: [] });
expect(config).not.toHaveProperty("homeMode");
});
it("pins the admitted isolated mode but invalidates a genuinely preconfigured next run", async () => {
const config = { provider: "codex", model: "fixture-model", homeMode: "preconfigured" };
const recoveredConfig = bindHeartbeatRunnerHomeMode(config, { schema: "paperclip.native-execution-input.v5" });
expect(recoveredConfig.homeMode).toBe("isolated");
expect(bindHeartbeatRunnerHomeMode(config, null).homeMode).toBe("preconfigured");
const recovered = await fingerprint(recoveredConfig);
const next = await fingerprint(config);
expect(next.fingerprint).not.toBe(recovered.fingerprint);
expect(resolveTaskSessionConfigFreshness({ hasTaskSession: true, configuredModel: config.model,
taskSessionParams: { __paperclipConfiguredModel: config.model, __paperclipConfigFingerprint: recovered.fingerprint,
__paperclipConfigFingerprintVersion: recovered.version, __paperclipConfigCategoryFingerprints: recovered.categoryFingerprints },
configMetadata: next })).toMatchObject({ reset: true, changedCategories: ["adapterConfig"] });
});
it.each([
["preconfigured", { schema: "paperclip.native-execution-input.v5" }, "isolated"],
["preconfigured", { schema: "paperclip.native-execution-input.v7", homeMode: "isolated" }, "isolated"],
@@ -395,6 +440,118 @@ describePostgres("heartbeat execution configuration through adapter dispatch", (
expect(initial.configSnapshot).toEqual(reused.configSnapshot);
});
it("keeps shared provider workspace configuration across native recovery and the next turn", async () => {
const f = await seed();
const selectedSkills = [];
for (const name of ["Alphabetically first long skill", "B longer selected skill", "C", "D small"]) {
selectedSkills.push(await companySkillService(db).createLocalSkill(f.companyId, { name, markdown: "# Keep the saved result\n" }));
}
const instructionsRootPath = path.join(home, f.companyId);
await mkdir(instructionsRootPath);
await writeFile(path.join(instructionsRootPath, "AGENTS.md"), "Preserve the workspace across controller replacement.\n");
const config = { provider: "codex", model: "fixture-model", lifecycleMode: "warm", idleTimeoutMs: 300_000,
codexPermissionMode: "full", paperclipSkillSync: { desiredSkills: selectedSkills.map(skill => skill.key) },
instructionsRootPath, instructionsFilePath: path.join(instructionsRootPath, "AGENTS.md"),
instructionsEntryFile: "AGENTS.md", instructionsBundleMode: "managed", env: { OPENAI_API_KEY: "fixture-direct-key" } };
const [environment] = await db.insert(environments).values({ name: `Directory ${f.companyId}`, driver: "plugin",
config: { pluginKey: "test.directory", driverKey: "tasks" } }).returning();
const [agent] = await db.update(agents).set({ adapterType: "paperclip_runner", adapterConfig: config,
defaultEnvironmentId: environment.id }).where(eq(agents.id, f.agentId)).returning();
const [issue] = await db.select().from(issues).where(eq(issues.id, f.issueId));
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId,
issueId: f.issueId, scopeKind: "issue", status: "running", runtimeMode: "native", responsibleUserId: f.userId, contextSnapshot: {} }).returning();
const workspaceId = run.id;
const [workspace] = await db.insert(executionWorkspaces).values({ id: workspaceId, companyId: f.companyId,
projectId: issue.projectId, sourceIssueId: issue.id, name: "Shared task workspace", mode: "shared_workspace",
strategyType: "project_primary", providerType: "adapter_managed", cwd: `/workspaces/${workspaceId}` }).returning();
const issueSettings = { mode: "shared_workspace" as const };
const sourceConfig = (current: Record<string, unknown>) => buildExecutionWorkspaceAdapterConfig({ agentConfig: current,
projectPolicy: null, issueSettings, mode: "shared_workspace", legacyUseProjectWorkspace: null });
const selectionInput: HeartbeatEnvironmentSelectionInput = {
run, agent, task: { issueRef: { ...issue, executionWorkspaceId: null }, issueContext: issue, issueId: issue.id,
executionProjectId: issue.projectId, workspaceProjectContext: { hasWorkspace: false }, boundSourceWorkspace: null,
hasSelectedWorkspaceSource: false, taskWorkspaceIntent: null, responsibleUserId: f.userId, context: {},
freshWorkspace: null },
policy: { trustPreset: { kind: "standard", preset: "standard", boundary: null, sourcePresets: {} },
requestedExecutionWorkspaceMode: "shared_workspace", isolatedWorkspacesEnabled: true,
projectExecutionWorkspacePolicy: null, issueExecutionWorkspaceSettings: issueSettings },
config: { config, sourceWorkspaceConfig: sourceConfig(config), issueAssigneeOverrides: null },
services: { environmentsSvc: environmentService(db), executionWorkspacesSvc: executionWorkspaceService(db),
instanceSettings: instanceSettingsService(db), findSharedWorkspaceHolder: vi.fn(async () => null),
findExecutionWorkspaceHolder: vi.fn(async () => null) },
};
const prepare = async () => {
const selected = await selectHeartbeatEnvironment(db, selectionInput);
const result = await prepareHeartbeatExecutionConfiguration(db, { run, agent,
task: { issueId: issue.id, context: {}, responsibleUserId: f.userId, acceptedPlanContinuationWake: false,
acceptedPlanWakeRoutingDecision: null, taskProjectContext: null,
routineEnvContext: { routineId: null, env: null, responsibleUserId: null } },
config: { sourceWorkspaceConfig: selectionInput.config.sourceWorkspaceConfig, aiBinding: undefined },
policy: { trustPreset: selectionInput.policy.trustPreset, useIsolatedTaskDirectory: selected.useIsolatedTaskDirectory,
explicitTaskDirectory: selected.explicitTaskDirectory, requestedExecutionWorkspaceMode: selectionInput.policy.requestedExecutionWorkspaceMode },
environment: selected, services: { secretsSvc: secretService(db), companySkills: companySkillService(db),
appendRunEvent: vi.fn(), finalizeAiConnectionBusyDeferral: vi.fn() },
onFailureSecretsResolved: () => {}, onManagedAiRuntimeAcquired: () => {}, onIdentityRedactorPrepared: () => {} });
if (!result.prepared) throw new Error("Configuration was not prepared");
const metadata = await buildEffectiveRunSessionConfigMetadata({ adapterType: "paperclip_runner",
effectiveAdapterConfig: result.runtimeConfig, agentRuntimeConfig: {}, issueOverrides: null, workspaceConfig: {},
environment: null, environmentEnv: null, projectEnv: null, routineEnv: null, runtimeSkills: result.runtimeSkillEntries });
return { selected, result, metadata };
};
const first = await prepare();
expect(first.selected.useIsolatedTaskDirectory).toBe(false);
expect(first.result.runtimeConfig.workspaceStrategy).toBeUndefined();
// Preserve the pre-fix fresh name/key query order, including exact versions.
expect(first.result.runtimeConfig.paperclipSkillSync).toEqual({ desiredSkills: selectedSkills.map(skill => ({
key: skill.key, versionId: skill.currentVersionId,
})) });
// Read the actual JSONB snapshot rather than reusing the pre-persistence
// JavaScript map: PostgreSQL key order differs for these four real keys.
const [saved] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id));
run.contextSnapshot = saved.contextSnapshot;
selectionInput.task.freshWorkspace = null;
selectionInput.task.issueRef!.executionWorkspaceId = workspaceId;
selectionInput.task.issueRef!.executionWorkspacePreference = "reuse_existing";
await db.update(issues).set({ executionWorkspaceId: workspaceId, executionWorkspacePreference: "reuse_existing" }).where(eq(issues.id, issue.id));
selectionInput.run.runnerProfileJson = { nativeExecutionInput: buildNativeExecutionInput({
companyId: f.companyId, agentId: agent.id, runId: run.id, normalizedSessionId: randomUUID(), provider: "codex",
issue: { id: issue.id, identifier: "CFG-1", title: issue.title, description: null, workMode: "standard" },
taskPrompt: "Continue the admitted work", workspace: { id: workspaceId, cwd: workspace.cwd!, repoUrl: null, repoRef: null, branchName: null },
completionContract: { id: randomUUID(), sha256: `sha256:${"a".repeat(64)}`, schemaVersion: "paperclip.run-result.v1",
contract: { revision: "1", objective: "Retain the directory", criteria: [{ id: "objective", requirement: "Preserve the workspace" }] } },
runtimeContext: nativeRuntimeContextFixture(),
}) };
const recovered = await prepare();
expect(recovered.selected.nativeRecoveryExecutionWorkspaceId).toBe(workspaceId);
expect(recovered.selected.useIsolatedTaskDirectory).toBe(false);
expect(recovered.result.runtimeSkillEntries).toEqual(first.result.runtimeSkillEntries);
expect(resolveTaskSessionConfigFreshness({ hasTaskSession: true, configuredModel: config.model,
taskSessionParams: { __paperclipConfiguredModel: config.model, __paperclipConfigFingerprint: first.metadata.fingerprint,
__paperclipConfigFingerprintVersion: first.metadata.version, __paperclipConfigCategoryFingerprints: first.metadata.categoryFingerprints },
configMetadata: recovered.metadata })).toMatchObject({ reset: false, changedCategories: [] });
expect(recovered.result.runtimeConfig).toEqual(first.result.runtimeConfig);
expect(recovered.metadata.fingerprint).toBe(first.metadata.fingerprint);
selectionInput.run.runnerProfileJson = {};
run.contextSnapshot = {};
const next = await prepare();
expect(next.result.runtimeConfig).toEqual(first.result.runtimeConfig);
expect(next.metadata.fingerprint).toBe(first.metadata.fingerprint);
selectionInput.config.sourceWorkspaceConfig = sourceConfig({ ...config,
paperclipSkillSync: { desiredSkills: selectedSkills.slice(1).map(skill => skill.key) } });
const fewerSkills = await prepare();
expect(fewerSkills.result.runtimeSkillEntries).toHaveLength(3);
expect(fewerSkills.metadata.fingerprint).not.toBe(first.metadata.fingerprint);
// A real model change must still invalidate the next admission.
selectionInput.config.config = { ...config, model: "changed-model" };
selectionInput.config.sourceWorkspaceConfig = sourceConfig(selectionInput.config.config);
const changed = await prepare();
expect(changed.selected.useIsolatedTaskDirectory).toBe(false);
expect(changed.result.runtimeConfig.model).toBe("changed-model");
expect(changed.metadata.fingerprint).not.toBe(first.metadata.fingerprint);
}, 30_000);
it.each([
{ mode: "shared_workspace" as const, config: { workspaceStrategy: { type: "git_worktree", baseRef: "main" } } },
{ mode: "agent_default" as const, config: {} },
@@ -1,3 +1,4 @@
import { persistHeartbeatRunProcessMetadata } from "../services/heartbeat/run-lifecycle.js";
import { enableRunFailureReporting } from "./helpers/run-failure-reporting.js";
import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db";
import { createSystemRecovery } from "../services/system-recovery-state.js";
@@ -2418,6 +2419,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
updatedAt: new Date(0),
}).returning())[0].id }
: await seedEnvironmentLeaseFixture(fixture);
await persistHeartbeatRunProcessMetadata(db, runId, { pid: 2_000_000_000, processGroupId: null,
startedAt: new Date(0).toISOString(), targetKind: "local" });
return { ...fixture, ...lease, complete };
}
@@ -2940,6 +2943,39 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
expect(wakeup?.status).toBe("claimed");
});
it.each(["running", "failed"] as const)("holds a remote %s native PID despite a colliding live host child", async status => {
const child = spawnAliveProcess();
childProcesses.add(child);
expect(child.pid).toBeTypeOf("number");
const { companyId, agentId, runId, issueId } = await seedRunFixture({ adapterType: "process",
runtimeMode: "native", runStatus: status,
runErrorCode: status === "failed" ? "native_session_interrupted" : null,
runError: status === "failed" ? "native session timed out after 120000ms" : null });
if (status === "failed") {
await db.update(heartbeatRuns).set({ nativeIssueId: issueId, nativePhase: "retryable_failure", finishedAt: new Date() })
.where(eq(heartbeatRuns.id, runId));
await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId,
phase: "retryable_failure", nextAttemptAt: new Date(0) });
}
await persistHeartbeatRunProcessMetadata(db, runId, { pid: child.pid!, processGroupId: null,
startedAt: new Date().toISOString(), targetKind: "remote" });
const heartbeat = heartbeatService(db);
const before = await heartbeat.getRun(runId);
expect(await heartbeat.reapOrphanedRuns()).toEqual({ reaped: 0, runIds: [] });
expect(await heartbeat.getRun(runId)).toMatchObject({ status, processPid: child.pid,
processStartedAt: before!.processStartedAt, errorCode: before!.errorCode, error: before!.error });
expect(isPidAlive(child.pid!)).toBe(true);
expect(mockTerminateLocalService).not.toHaveBeenCalled();
expect(mockAdapterExecute).not.toHaveBeenCalled();
child.kill("SIGTERM");
await waitForPidExit(child.pid!);
expect(await heartbeat.reapOrphanedRuns()).toEqual({ reaped: 0, runIds: [] });
expect(await heartbeat.getRun(runId)).toMatchObject({ status, processPid: child.pid,
processStartedAt: before!.processStartedAt, errorCode: before!.errorCode, error: before!.error });
expect(await selectRecoveryRunsByAgent(agentId)).toHaveLength(1);
expect(mockAdapterExecute).not.toHaveBeenCalled();
});
it("keeps a native run active without granting legacy retry or signal authority", async () => {
const child = spawnAliveProcess();
childProcesses.add(child);
@@ -2951,6 +2987,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
processPid: child.pid ?? null,
includeIssue: false,
});
await persistHeartbeatRunProcessMetadata(db, runId, { pid: child.pid!, processGroupId: null,
startedAt: new Date().toISOString(), targetKind: "local" });
const heartbeat = heartbeatService(db);
const result = await heartbeat.reapOrphanedRuns();
@@ -3229,6 +3267,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
processPid: 999_999_999,
includeIssue: false,
});
await persistHeartbeatRunProcessMetadata(db, runId, { pid: 999_999_999, processGroupId: null,
startedAt: new Date().toISOString(), targetKind: "local" });
// The persisted run remains native even after the agent's current adapter
// changes to one that normally owns a legacy local child.
await db
@@ -33,6 +33,7 @@ vi.mock("../sentry.js", async () => {
});
import { heartbeatService } from "../services/heartbeat.ts";
import { persistHeartbeatRunProcessMetadata } from "../services/heartbeat/run-lifecycle.js";
function agentTaskRunCalls(fromIndex: number) {
return mockTelemetryClient.track.mock.calls
@@ -113,6 +114,9 @@ describeEmbeddedPostgres(
processPid: process.pid,
contextSnapshot: { issueId },
});
await persistHeartbeatRunProcessMetadata(db, runId, {
pid: process.pid, processGroupId: null, startedAt: new Date().toISOString(), targetKind: "local",
});
await db.insert(nativeRunFinalizations).values({
runId,
companyId,
@@ -2,6 +2,9 @@ import { createDb } from "@paperclipai/db";
import { executePaperclipNativeSession } from "../../services/native-runtime/native-session-executor.js";
import { dockerCommandRunner } from "./docker-command-runner.js";
// Separate TypeScript fixture import time from actual controller admission.
process.send?.({ ready: true });
process.once("message", async (input: {
connectionString: string;
container: string;
@@ -0,0 +1,432 @@
import { mkdtemp, writeFile, rm } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { spawn } from "node:child_process";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target";
import { KIMI_DISTRIBUTION_CLOSURE_SHA256, KIMI_NODE_DISTRIBUTIONS, KIMI_NODE_VERSION, KIMI_RUNTIME_VERSION } from "../../../packages/paperclip-runner/src/drivers/acpx/kimi-closure-pins.js";
import { QUALIFIED_ACPX_PROFILES } from "../../../packages/paperclip-runner/src/drivers/acpx/qualified-profiles.js";
const mocks = vi.hoisted(() => ({ command: vi.fn(), local: vi.fn(), digest: vi.fn(), ssh: vi.fn(), prepare: vi.fn(), select: vi.fn(), wire: [] as Array<{ file: string; args: string[]; stdin: boolean }> }));
vi.mock("@paperclipai/adapter-utils/execution-target", () => ({ runAdapterExecutionTargetShellCommand: mocks.command }));
vi.mock("@paperclipai/adapter-utils/ssh", () => ({ runSshCommand: mocks.ssh }));
vi.mock("../services/native-runtime/native-session-executor.js", () => ({
prepareRemoteProviderPackOnTarget: mocks.prepare, resolveRemoteProviderPackSource: mocks.select,
}));
vi.mock("node:child_process", async importOriginal => {
const actual = await importOriginal<typeof import("node:child_process")>();
return { ...actual,
execFile: (file: string, args: string[], options: object, callback: Parameters<typeof actual.execFile>[3]) => {
if (file !== "ssh") return actual.execFile(file, args, options, callback);
mocks.wire.push({ file, args, stdin: false });
return actual.execFile(process.execPath, ["-e", 'process.stdout.write("{}")'], options, callback);
},
spawn: (file: string, args: string[], options: Parameters<typeof actual.spawn>[2]) => {
if (file !== "ssh") return actual.spawn(file, args, options);
mocks.wire.push({ file, args, stdin: true });
return actual.spawn(process.execPath, ["-e", 'process.stdout.write(require("node:fs").readFileSync(0,"utf8"))'], options);
},
};
});
vi.mock("../vendor/paperclip-runner/live/index.js", async () => {
const provider = await import("../../../packages/paperclip-runner/src/drivers/acpx/kimi-metadata-probe.js");
return { probeKimiMetadata: mocks.local, kimiMetadataProbeDigest: mocks.digest,
validateKimiMetadata: provider.validateKimiMetadata, kimiMetadataProbeNodeDigest: provider.kimiMetadataProbeNodeDigest };
});
import { probeKimiExecutionTarget as rawProbeKimiExecutionTarget } from "../services/kimi-connection-probe.js";
const model = "selected-model'; $(never-execute)";
const apiKey = "selected-secret'; $(never-execute)";
const digest = `sha256:${"a".repeat(64)}`;
const remote = { kind: "remote", transport: "sandbox", remoteCwd: "/workspace" } as AdapterExecutionTarget;
const verified = () => ({ schema: "paperclip.kimi.metadata.v1", status: "verified", version: KIMI_RUNTIME_VERSION,
nodeVersion: KIMI_NODE_VERSION, profileDigest: QUALIFIED_ACPX_PROFILES.kimi.commandDigest, target: "linux-x64",
closureDigest: `sha256:${KIMI_DISTRIBUTION_CLOSURE_SHA256["linux-x64"]}`, model, contextLength: 1048576, promptSent: false });
const response = (value: unknown = verified(), exitCode = 0) => ({ stdout: JSON.stringify(value), stderr: "", exitCode, signal: null, timedOut: false });
const failed = (code: string) => ({ schema: "paperclip.kimi.metadata.v1", status: "failed", code, message: apiKey, promptSent: false });
const companyId = "candidate-company";
const pending = QUALIFIED_ACPX_PROFILES.kimi.qualificationStatus === "pending";
function probeKimiExecutionTarget(apiKey: string, target: AdapterExecutionTarget | null | undefined, model: string) {
const platform = target?.kind === "remote" ? "linux" : process.platform;
const architecture = target?.kind === "remote" ? "x64" : process.arch;
vi.stubEnv("PAPERCLIP_ACPX_CANDIDATE_ADMISSION", JSON.stringify({ schema: "paperclip.acpx-candidate-admission.v1",
companyId, agent: "kimi", model, profileDigest: QUALIFIED_ACPX_PROFILES.kimi.commandDigest,
platform, architecture, expiresAt: Date.now() + 60_000,
nativeClosureDigest: `sha256:${KIMI_DISTRIBUTION_CLOSURE_SHA256[`${platform}-${architecture}` as "darwin-arm64" | "linux-x64"]}` }));
return rawProbeKimiExecutionTarget(apiKey, target, model, { companyId });
}
beforeEach(() => {
vi.resetAllMocks();
mocks.digest.mockResolvedValue(digest);
mocks.command.mockResolvedValue(response());
mocks.ssh.mockResolvedValue({ stdout: JSON.stringify(verified()), stderr: "" });
mocks.wire.length = 0;
mocks.local.mockResolvedValue(verified());
});
afterEach(() => { vi.restoreAllMocks(); vi.unstubAllEnvs(); });
describe("Kimi setup on a selected execution target", () => {
it.each(["sandbox", "ssh"])("uses admitted fixed bytes and only explicit credentials on %s", async transport => {
vi.stubEnv("KIMI_API_KEY", "unselected-ambient-secret");
vi.stubEnv("KIMI_CODE_BASE_URL", "https://unselected.example");
const target = { ...remote, transport } as AdapterExecutionTarget;
await expect(probeKimiExecutionTarget(apiKey, target, model)).resolves.toEqual(verified());
expect(mocks.local).not.toHaveBeenCalled();
const call = transport === "ssh" ? mocks.ssh.mock.calls[0] : mocks.command.mock.calls[0];
expect(call).toBeTruthy();
const command = call![transport === "ssh" ? 1 : 2], options = call![transport === "ssh" ? 2 : 3];
if (transport === "ssh") {
expect(mocks.command).not.toHaveBeenCalled();
expect(mocks.ssh).toHaveBeenCalledOnce();
} else {
expect(mocks.command).toHaveBeenCalledOnce();
expect(call![1]).toBe(target);
}
expect(command).toContain("pack=/opt/paperclip-runner/provider-pack");
expect(command).toContain('node="$pack/provider-assets/kimi/linux-x64/runtime/node/bin/node"');
expect(command).toContain('/usr/bin/sha256sum "$entry"');
expect(command).toContain(digest.slice(7));
expect(command).toContain('/usr/bin/sha256sum "$node"');
expect(command).toContain(KIMI_NODE_DISTRIBUTIONS["linux-x64"].executableSha256);
expect(command.indexOf('/usr/bin/sha256sum "$entry"')).toBeLessThan(command.indexOf('exec /usr/bin/env -i'));
expect(command.indexOf('/usr/bin/sha256sum "$node"')).toBeLessThan(command.indexOf('exec /usr/bin/env -i'));
expect(command).toContain("unset NODE_OPTIONS NODE_PATH NODE_EXTRA_CA_CERTS");
expect(command).toContain('PAPERCLIP_ACPX_PROVIDER_PACKAGE_ROOT="$pack"');
expect(command).not.toContain(model);
expect(command).not.toContain(apiKey);
if (transport === "ssh") {
expect(options.env).toBeUndefined();
const input = JSON.parse(options.stdin);
expect(input).toMatchObject({ apiKey, model });
if (pending) expect(input.candidateAdmission).toMatchObject({ companyId, agent: "kimi", model, platform: "linux", architecture: "x64", runId: expect.stringMatching(/^kimi-metadata-/), normalizedSessionId: expect.stringMatching(/^metadata-/) });
else expect(input).not.toHaveProperty("candidateAdmission");
expect(options.timeoutMs).toBeGreaterThan(0);
expect(options.timeoutMs).toBeLessThanOrEqual(32000);
} else {
expect(options).toEqual({ cwd: "/workspace", env: { KIMI_API_KEY: apiKey, PAPERCLIP_KIMI_PROBE_MODEL: model, ...(pending ? { PAPERCLIP_ACPX_CANDIDATE_ADMISSION: expect.any(String) } : {}) }, timeoutSec: expect.any(Number), graceSec: 3 });
expect(options.timeoutSec).toBeGreaterThan(0);
expect(options.timeoutSec).toBeLessThanOrEqual(32);
}
expect(JSON.stringify(options)).not.toContain("unselected");
});
it("keeps local verification explicit and does not run a remote command", async () => {
for (const target of [undefined, null, { kind: "local" } as AdapterExecutionTarget]) {
await expect(probeKimiExecutionTarget(apiKey, target, model)).resolves.toEqual(verified());
}
expect(mocks.local).toHaveBeenCalledTimes(3);
if (pending) expect(mocks.local).toHaveBeenCalledWith(model, apiKey, expect.objectContaining({ companyId, agent: "kimi", model, platform: process.platform, architecture: process.arch, runId: expect.stringMatching(/^kimi-metadata-/), normalizedSessionId: expect.stringMatching(/^metadata-/) }));
else expect(mocks.local).toHaveBeenCalledWith(model, apiKey, undefined);
expect(mocks.command).not.toHaveBeenCalled();
expect(mocks.digest).not.toHaveBeenCalled();
mocks.local.mockResolvedValue(failed("KIMI_AUTH_REJECTED"));
const error = await probeKimiExecutionTarget(apiKey, null, model).catch(error => error);
expect(error).toMatchObject({ status: 422, details: { code: "KIMI_AUTH_REJECTED" } });
expect(error.message).not.toContain(apiKey);
});
it("rejects task-only plugin hosting before credentials leave the server", async () => {
const runnerTask = vi.fn();
await expect(probeKimiExecutionTarget(apiKey, { ...remote, runnerTask } as AdapterExecutionTarget, model))
.rejects.toMatchObject({ details: { code: "KIMI_INSTALLATION_INVALID" } });
expect(runnerTask).not.toHaveBeenCalled();
expect(mocks.local).not.toHaveBeenCalled();
expect(mocks.digest).not.toHaveBeenCalled();
expect(mocks.command).not.toHaveBeenCalled();
});
it("rejects absent or invalid key/model inputs before transmission", async () => {
vi.stubEnv("KIMI_API_KEY", apiKey);
for (const key of ["", " ", "key\nheader", "key\0header", "x".repeat(16385)]) {
await expect(probeKimiExecutionTarget(key, remote, model)).rejects.toMatchObject({ details: { code: "KIMI_AUTH_REQUIRED" } });
}
for (const selected of ["", " ", "x\0y", "x".repeat(513), "\ud800"]) {
await expect(probeKimiExecutionTarget(apiKey, remote, selected)).rejects.toMatchObject({ details: { code: "KIMI_MODEL_UNAVAILABLE" } });
}
expect(mocks.command).not.toHaveBeenCalled();
expect(mocks.local).not.toHaveBeenCalled();
});
});
describe("remote Kimi installation fallback and failure admission", () => {
it("prepares the selected current pack before testing an older SDK image", async () => {
// Actual credential-free artifacts from the failed source-16 qualification:
// the immutable SDK image carries this older helper, while the controller
// and complete selected Linux pack require the newer bytes.
const imageHelper = "9e87ba56ce4663aa36f310a7eb6224bebb7d838be479c1741c3393577f2953c7";
const selectedHelper = "22bf5125adbda95208d26762d4889de2b365a351b403ac92863f8f838f6d36fc";
const installedRoot = "/workspace/.paperclip-runtime/paperclip-runner/provider-pack";
const runner = { execute: vi.fn() };
const selected = { configuredRoot: "/controller/complete-linux-pack",
manifest: { payload: { target: { platform: "linux", architecture: "x64" } } } };
mocks.select.mockReturnValue(selected);
mocks.prepare.mockResolvedValue({ root: installedRoot, manifest: selected.manifest, source: "uploaded" });
mocks.digest.mockResolvedValue(`sha256:${selectedHelper}`);
vi.stubEnv("PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH", "/controller/complete-linux-pack");
vi.stubEnv("PAPERCLIP_RUNNER_REMOTE_BINARY_PATH", "/controller/linux-runnerd");
mocks.command.mockImplementation(async (_id, _target, command) => {
expect(command).toContain(selectedHelper);
if (command.includes("pack=/opt/paperclip-runner/provider-pack")) {
expect(imageHelper).not.toBe(selectedHelper);
return { ...response(), stdout: "", exitCode: 126 };
}
if (command.includes('pack="$HOME/.local/share/paperclip-runner/provider-pack"')) {
return { ...response(), stdout: "", exitCode: 127 };
}
expect(command).toContain(`pack='${installedRoot}'`);
return response();
});
const target = { ...remote, runner } as AdapterExecutionTarget;
await expect(probeKimiExecutionTarget(apiKey, target, model)).resolves.toEqual(verified());
expect(mocks.select).toHaveBeenCalledWith(selected.configuredRoot);
expect(mocks.prepare).toHaveBeenCalledWith({ target, runner, configuredProviderPackRoot: selected.configuredRoot, manifest: selected.manifest });
expect(mocks.command).toHaveBeenCalledOnce();
expect(mocks.prepare.mock.invocationCallOrder[0]).toBeLessThan(mocks.command.mock.invocationCallOrder[0]!);
expect(mocks.local).not.toHaveBeenCalled();
});
it.each(["unreadable-source", "wrong-platform", "missing-transport", "failed-install"])("does not transmit credentials or use the baked image after %s", async failure => {
vi.stubEnv("PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH", "/controller/selected-pack");
mocks.select.mockReturnValue({ configuredRoot: "/controller/selected-pack",
manifest: { payload: { target: { platform: failure === "wrong-platform" ? "darwin" : "linux", architecture: "x64" } } } });
if (failure === "unreadable-source") mocks.select.mockImplementation(() => { throw new Error(apiKey); });
if (failure === "failed-install") mocks.prepare.mockRejectedValue(new Error(apiKey));
const target = { ...remote, ...(failure === "missing-transport" ? {} : { runner: { execute: vi.fn() } }) } as AdapterExecutionTarget;
const error = await probeKimiExecutionTarget(apiKey, target, model).catch(error => error);
expect(error).toMatchObject({ status: 422, details: { code: "KIMI_INSTALLATION_INVALID" } });
expect(error.message).not.toContain(apiKey);
expect(mocks.command).not.toHaveBeenCalled();
expect(mocks.local).not.toHaveBeenCalled();
if (failure !== "failed-install") expect(mocks.prepare).not.toHaveBeenCalled();
});
it("uses the current qualification policy before selected-pack preparation", async () => {
vi.stubEnv("PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH", "/controller/selected-pack");
vi.stubEnv("PAPERCLIP_ACPX_CANDIDATE_ADMISSION", JSON.stringify({ schema: "paperclip.acpx-candidate-admission.v1",
companyId: "other-company", agent: "kimi", model, profileDigest: QUALIFIED_ACPX_PROFILES.kimi.commandDigest,
platform: "linux", architecture: "x64", expiresAt: Date.now() + 60_000,
nativeClosureDigest: `sha256:${KIMI_DISTRIBUTION_CLOSURE_SHA256["linux-x64"]}` }));
if (pending) {
await expect(rawProbeKimiExecutionTarget(apiKey, remote, model, { companyId }))
.rejects.toMatchObject({ details: { code: "KIMI_QUALIFICATION_REQUIRED" } });
expect(mocks.select).not.toHaveBeenCalled();
expect(mocks.prepare).not.toHaveBeenCalled();
expect(mocks.command).not.toHaveBeenCalled();
} else {
const runner = { execute: vi.fn() };
const selected = { configuredRoot: "/controller/selected-pack",
manifest: { payload: { target: { platform: "linux", architecture: "x64" } } } };
mocks.select.mockReturnValue(selected);
mocks.prepare.mockResolvedValue({ root: "/workspace/.paperclip-runtime/paperclip-runner/provider-pack", manifest: selected.manifest, source: "uploaded" });
await expect(rawProbeKimiExecutionTarget(apiKey, { ...remote, runner } as AdapterExecutionTarget, model, { companyId })).resolves.toEqual(verified());
expect(mocks.prepare).toHaveBeenCalledOnce();
expect(mocks.command.mock.calls[0]![3].env).not.toHaveProperty("PAPERCLIP_ACPX_CANDIDATE_ADMISSION");
}
});
it.each(["missing", "different-cli-bytes", "installation", "profile", "closure"])("tries only the fixed home pack after %s", async variant => {
const first = variant === "missing" ? { ...response(), stdout: "", exitCode: 127 }
: variant === "different-cli-bytes" ? { ...response(), stdout: "", exitCode: 126 }
: variant === "installation" ? response(failed("KIMI_INSTALLATION_INVALID"), 1)
: response({ ...verified(), [variant === "profile" ? "profileDigest" : "closureDigest"]: `sha256:${"0".repeat(64)}` });
mocks.command.mockResolvedValueOnce(first).mockResolvedValueOnce(response());
await expect(probeKimiExecutionTarget(apiKey, remote, model)).resolves.toEqual(verified());
expect(mocks.command).toHaveBeenCalledTimes(2);
expect(mocks.command.mock.calls[1]![2]).toContain('pack="$HOME/.local/share/paperclip-runner/provider-pack"');
expect(mocks.local).not.toHaveBeenCalled();
});
it.each(["KIMI_AUTH_REQUIRED", "KIMI_AUTH_REJECTED", "KIMI_MODEL_UNAVAILABLE", "KIMI_METADATA_TIMEOUT", "KIMI_METADATA_INVALID", "KIMI_PROVIDER_ERROR"])("keeps %s final and strips remote text", async code => {
mocks.command.mockResolvedValueOnce(response(failed(code), 1));
const error = await probeKimiExecutionTarget(apiKey, remote, model).catch(error => error);
expect(error).toMatchObject({ status: 422, details: { code } });
expect(error.message).not.toContain(apiKey);
expect(mocks.command).toHaveBeenCalledOnce();
expect(mocks.local).not.toHaveBeenCalled();
});
it.each([
{ stdout: "not json" }, { stdout: "x".repeat(65537) }, { signal: "SIGTERM" }, { timedOut: true },
{ exitCode: 1 }, { stdout: JSON.stringify({ ...verified(), promptSent: true }) },
{ stdout: JSON.stringify({ ...verified(), arbitrary: true }) },
])("never accepts malformed or interrupted success %#", alteration => {
mocks.command.mockResolvedValue({ ...response(), ...alteration });
return expect(probeKimiExecutionTarget(apiKey, remote, model)).rejects.toMatchObject({ details: { code: "KIMI_REQUEST_FAILED" } });
});
it("fails closed when no fixed remote installation matches without trying the controller", async () => {
mocks.command.mockResolvedValue({ ...response(), stdout: "", exitCode: 127 });
await expect(probeKimiExecutionTarget(apiKey, remote, model)).rejects.toMatchObject({ details: { code: "KIMI_INSTALLATION_INVALID" } });
expect(mocks.command).toHaveBeenCalledTimes(2);
expect(mocks.local).not.toHaveBeenCalled();
});
it("rejects a receipt for another model or release platform", async () => {
mocks.command.mockResolvedValue(response({ ...verified(), model: "unselected-model" }));
await expect(probeKimiExecutionTarget(apiKey, remote, model)).rejects.toMatchObject({ details: { code: "KIMI_MODEL_UNAVAILABLE" } });
expect(mocks.command).toHaveBeenCalledOnce();
mocks.command.mockResolvedValue(response({ ...verified(), target: "darwin-arm64", closureDigest: `sha256:${KIMI_DISTRIBUTION_CLOSURE_SHA256["darwin-arm64"]}` }));
await expect(probeKimiExecutionTarget(apiKey, remote, model)).rejects.toMatchObject({ details: { code: "KIMI_INSTALLATION_INVALID" } });
});
it("requires the current bundled probe before transmitting credentials", async () => {
for (const value of ["", "arbitrary-path", `sha256:${"f".repeat(63)}`]) {
mocks.digest.mockResolvedValue(value);
await expect(probeKimiExecutionTarget(apiKey, remote, model)).rejects.toMatchObject({ details: { code: "KIMI_INSTALLATION_INVALID" } });
}
mocks.digest.mockRejectedValue(new Error(`missing build ${apiKey}`));
const error = await probeKimiExecutionTarget(apiKey, remote, model).catch(error => error);
expect(error.message).not.toContain(apiKey);
expect(mocks.command).not.toHaveBeenCalled();
});
it("shares the 35 second budget across local digest and both remote locations", async () => {
let now = 1000;
vi.spyOn(Date, "now").mockImplementation(() => now);
mocks.digest.mockImplementation(async () => { now += 2000; return digest; });
mocks.command.mockImplementationOnce(async () => { now += 20_000; return { ...response(), stdout: "", exitCode: 127 }; });
await expect(probeKimiExecutionTarget(apiKey, remote, model)).resolves.toEqual(verified());
expect(mocks.command.mock.calls[0]![3].timeoutSec).toBe(30);
expect(mocks.command.mock.calls[1]![3].timeoutSec).toBe(10);
mocks.command.mockReset().mockImplementation(async () => { now += 35_001; return response(); });
await expect(probeKimiExecutionTarget(apiKey, remote, model)).rejects.toMatchObject({ details: { code: "KIMI_REQUEST_FAILED" } });
});
it("sanitizes transport errors without trying another host or local credentials", async () => {
mocks.command.mockRejectedValue(new Error(`transport ${apiKey}`));
const error = await probeKimiExecutionTarget(apiKey, remote, model).catch(error => error);
expect(error).toMatchObject({ details: { code: "KIMI_REQUEST_FAILED" } });
expect(error.message).not.toContain(apiKey);
expect(mocks.command).toHaveBeenCalledOnce();
expect(mocks.local).not.toHaveBeenCalled();
});
});
it("keeps the selected SSH key out of actual ssh argv and delivers it only over the existing stdin pipe", async () => {
const wireApiKey = "wire-private-sentinel-abc123", wireModel = "wire-model";
mocks.command.mockResolvedValue(response({ ...verified(), model: wireModel }));
mocks.ssh.mockResolvedValue({ stdout: JSON.stringify({ ...verified(), model: wireModel }), stderr: "" });
const spec = { host: "fixture.invalid", username: "fixture", port: 22, remoteWorkspacePath: "/workspace",
privateKey: null, knownHosts: null, strictHostKeyChecking: true };
const target = { kind: "remote", transport: "ssh", remoteCwd: "/workspace", spec } as AdapterExecutionTarget;
await probeKimiExecutionTarget(wireApiKey, target, wireModel);
const selected = mocks.ssh.mock.calls[0] ?? [spec, mocks.command.mock.calls[0]![2], { env: mocks.command.mock.calls[0]![3].env }];
const { runSshCommand } = await vi.importActual<typeof import("@paperclipai/adapter-utils/ssh")>("@paperclipai/adapter-utils/ssh");
const result = await runSshCommand(spec, selected[1], selected[2]);
expect(mocks.wire).toHaveLength(1);
const argv = JSON.stringify(mocks.wire[0]!.args);
for (const value of [wireApiKey, wireModel, Buffer.from(wireApiKey).toString("base64"), Buffer.from(wireApiKey).toString("hex")]) expect(argv).not.toContain(value);
expect(mocks.wire[0]!.stdin).toBe(true);
const input = JSON.parse(result.stdout);
expect(input).toMatchObject({ apiKey: wireApiKey, model: wireModel });
if (pending) expect(input.candidateAdmission).toMatchObject({ companyId, model: wireModel, platform: "linux", architecture: "x64" });
else expect(input).not.toHaveProperty("candidateAdmission");
});
it("loads the verified entry after decoding private stdin without interpreting key or model text", async () => {
await probeKimiExecutionTarget(apiKey, { ...remote, transport: "ssh" } as AdapterExecutionTarget, model);
const command = mocks.ssh.mock.calls[0]![1] as string;
const bootstrap = command.match(/ -e '([^']+)' "\$entry"/)?.[1];
expect(bootstrap).toBeTruthy();
const root = await mkdtemp(path.join(os.tmpdir(), "kimi-probe-stdin-"));
try {
const entry = path.join(root, "entry.cjs");
await writeFile(entry, 'process.stdout.write(JSON.stringify({ key:process.env.KIMI_API_KEY, model:process.env.PAPERCLIP_KIMI_PROBE_MODEL, ambient:process.env.UNSELECTED_SECRET }));');
const execute = (input: string) => new Promise<{ code: number | null; stdout: string; stderr: string }>((resolve, reject) => {
const child = spawn(process.execPath, ["-e", bootstrap!, entry], { env: {}, stdio: ["pipe", "pipe", "pipe"] });
let stdout = "", stderr = "";
child.stdout.on("data", chunk => { stdout += chunk; }); child.stderr.on("data", chunk => { stderr += chunk; });
child.on("error", reject); child.on("close", code => resolve({ code, stdout, stderr })); child.stdin.end(input);
});
const result = await execute(JSON.stringify({ apiKey, model }));
expect(result.code).toBe(0);expect(result.stderr).toBe("");expect(JSON.parse(result.stdout)).toEqual({ key:apiKey, model });
for (const bad of ["not-json-private-sentinel", JSON.stringify({ apiKey: 1, model })]) {
const result = await execute(bad);expect(result.code).toBe(1);expect(result.stdout).toBe("");expect(result.stderr).toBe("");
}
} finally { await rm(root, { recursive:true, force:true }); }
});
it("preserves SSH installation fallback and typed rejection while stripping transport text", async () => {
const target = { ...remote, transport:"ssh" } as AdapterExecutionTarget;
mocks.ssh.mockRejectedValueOnce(Object.assign(new Error(apiKey), { code:127, stdout:"" }));
await expect(probeKimiExecutionTarget(apiKey, target, model)).resolves.toEqual(verified());
expect(mocks.ssh).toHaveBeenCalledTimes(2);
expect(mocks.ssh.mock.calls[1]![1]).toContain('pack="$HOME/.local/share/paperclip-runner/provider-pack"');
for (const failure of [Object.assign(new Error(apiKey), { code:1, stdout:JSON.stringify(failed("KIMI_AUTH_REJECTED")) }),
Object.assign(new Error(apiKey), { code:"ETIMEDOUT" })]) {
mocks.ssh.mockRejectedValueOnce(failure);
const error = await probeKimiExecutionTarget(apiKey,target,model).catch(error=>error);
expect(error.details.code).toBe(failure.code===1?"KIMI_AUTH_REJECTED":"KIMI_REQUEST_FAILED");expect(error.message).not.toContain(apiKey);
}
});
it("uses trusted authority only for a pending metadata Test and ordinary no-grant execution after release", async () => {
if (!pending) {
vi.stubEnv("PAPERCLIP_ACPX_CANDIDATE_ADMISSION", "unrelated-invalid-authority");
for (const context of [{}, { companyId }, { companyId, candidateAdmission: { allowAll: true } }]) {
await expect(rawProbeKimiExecutionTarget(apiKey, remote, model, context)).resolves.toEqual(verified());
}
expect(mocks.command).toHaveBeenCalledTimes(3);
for (const call of mocks.command.mock.calls) {
expect(call[2]).not.toContain("PAPERCLIP_ACPX_CANDIDATE_ADMISSION");
expect(call[3].env).not.toHaveProperty("PAPERCLIP_ACPX_CANDIDATE_ADMISSION");
}
} else {
vi.stubEnv("PAPERCLIP_ACPX_CANDIDATE_ADMISSION", undefined);
vi.stubEnv("PAPERCLIP_RUNNER_ACPX_QUALIFICATION", JSON.stringify([{ agent: "kimi", model }]));
for (const context of [{}, { companyId }, { companyId, candidateAdmission: { allowAll: true } }]) {
await expect(rawProbeKimiExecutionTarget(apiKey, remote, model, context)).rejects.toMatchObject({ details: { code: "KIMI_QUALIFICATION_REQUIRED" } });
}
expect(mocks.local).not.toHaveBeenCalled(); expect(mocks.digest).not.toHaveBeenCalled();
expect(mocks.command).not.toHaveBeenCalled(); expect(mocks.ssh).not.toHaveBeenCalled();
await probeKimiExecutionTarget(apiKey, remote, model);
mocks.command.mockClear(); mocks.digest.mockClear();
for (const context of [{ companyId: "foreign" }, {}]) {
await expect(rawProbeKimiExecutionTarget(apiKey, remote, model, context)).rejects.toMatchObject({ details: { code: "KIMI_QUALIFICATION_REQUIRED" } });
}
const grant = JSON.parse(process.env.PAPERCLIP_ACPX_CANDIDATE_ADMISSION!);
vi.stubEnv("PAPERCLIP_ACPX_CANDIDATE_ADMISSION", JSON.stringify({ ...grant, expiresAt: 0 }));
await expect(rawProbeKimiExecutionTarget(apiKey, remote, model, { companyId })).rejects.toMatchObject({ details: { code: "KIMI_QUALIFICATION_REQUIRED" } });
expect(mocks.command).not.toHaveBeenCalled(); expect(mocks.digest).not.toHaveBeenCalled();
}
});
it("omits candidate authority entirely for released metadata Test, including the actual remote shell environment", async () => {
const path = "../vendor/paperclip-runner/index.js";
vi.doMock(path, async () => {
const actual = await vi.importActual<typeof import("../vendor/paperclip-runner/index.js")>(path);
return { ...actual, QUALIFIED_ACPX_PROFILES: { ...actual.QUALIFIED_ACPX_PROFILES,
kimi: { ...actual.QUALIFIED_ACPX_PROFILES.kimi, qualificationStatus: undefined } } };
});
vi.resetModules();
const root = await mkdtemp(os.tmpdir() + "/kimi-released-probe-");
try {
vi.stubEnv("PAPERCLIP_ACPX_CANDIDATE_ADMISSION", undefined);
const { probeKimiExecutionTarget } = await import("../services/kimi-connection-probe.js");
const { resolveKimiCandidateAdmission } = await import("../services/native-runtime/acpx-qualification.js");
expect(resolveKimiCandidateAdmission({ companyId, model }, { PAPERCLIP_ACPX_CANDIDATE_ADMISSION: "stale-invalid-candidate" })).toBeUndefined();
await probeKimiExecutionTarget(apiKey, remote, model, { companyId });
const command = mocks.command.mock.calls[0]![2] as string;
const options = mocks.command.mock.calls[0]![3];
expect(command).not.toContain("PAPERCLIP_ACPX_CANDIDATE_ADMISSION");
expect(options.env).not.toHaveProperty("PAPERCLIP_ACPX_CANDIDATE_ADMISSION");
const entry = root + "/environment.cjs";
await writeFile(entry, 'process.stdout.write(JSON.stringify({candidatePresent:Object.hasOwn(process.env,"PAPERCLIP_ACPX_CANDIDATE_ADMISSION")}));');
const suffix = command.slice(command.lastIndexOf("exec /usr/bin/env -i"));
const result = await new Promise<{code:number|null;stdout:string;stderr:string}>((resolve, reject) => {
const child = spawn("/bin/sh", ["-c", suffix], { env: { node: process.execPath, entry, pack: root,
KIMI_API_KEY: apiKey, PAPERCLIP_KIMI_PROBE_MODEL: model, PAPERCLIP_ACPX_CANDIDATE_ADMISSION: "ambient-forgery" }, stdio: ["ignore", "pipe", "pipe"] });
let stdout = "", stderr = "";
child.stdout.on("data", chunk => { stdout += chunk; }); child.stderr.on("data", chunk => { stderr += chunk; });
child.on("error", reject); child.on("close", code => resolve({ code, stdout, stderr }));
});
expect(result).toEqual({ code: 0, stdout: JSON.stringify({ candidatePresent: false }), stderr: "" });
} finally {
await rm(root, { recursive: true, force: true });
vi.doUnmock(path); vi.resetModules();
}
});
@@ -412,6 +412,15 @@ describeEmbeddedPostgres("native harness backup expiration", () => {
expect(await exists(session.current)).toBe(false);
});
it("retains incomplete native snapshot publications even after the issue closes", async () => {
const session = await seedSession({ finalization: { phase: "committed" } });
await db.update(heartbeatRuns).set({ runnerProfileJson: { nativeControllerRecovery: {
schema: "paperclip.native-controller-recovery.v1", checkpoint: null,
} } }).where(eq(heartbeatRuns.id, session.runId));
expect(await service().tick()).toMatchObject({ expired: 0, skipped: { native_recovery_incomplete: 1 } });
expect(await exists(session.current)).toBe(true);
});
it.each([
["committed", "resuming_session"],
["terminal_failure", "blocked"],
@@ -9,12 +9,14 @@ import { eq, inArray } from "drizzle-orm";
import {
activityLog,
agentWakeupRequests,
agentTaskSessions,
agents,
companies,
authUsers,
companyMemberships,
completionContracts,
createDb,
closeRegisteredClients,
executionWorkspaces,
heartbeatRunEvents,
heartbeatRuns,
@@ -78,7 +80,11 @@ vi.mock("../sentry.js", async () => {
return { ...actual, captureRunFailure: mockCaptureRunFailure };
});
import { createIsolatedTestHome } from "./helpers/isolated-test-home.js";
import { issueService } from "../services/issues.js";
import * as heartbeatWorkspaces from "../services/heartbeat/workspaces.js";
import { heartbeatService } from "../services/heartbeat.js";
import { persistHeartbeatRunProcessMetadata } from "../services/heartbeat/run-lifecycle.js";
import { instanceSettingsService } from "../services/instance-settings.js";
describe("P6-25 pre-result native session recovery", () => {
@@ -1043,13 +1049,10 @@ describe.each(["unchanged", "newer_active", "stale_idle"] as const)(
);
await once(staleProviderProcess, "spawn");
expect(staleProviderProcess.pid).toEqual(expect.any(Number));
await db
.update(heartbeatRuns)
.set({
processPid: staleProviderProcess.pid!,
processStartedAt: new Date("2026-08-09T04:00:00.000Z"),
})
.where(eq(heartbeatRuns.id, runId));
await persistHeartbeatRunProcessMetadata(db, runId, {
pid: staleProviderProcess.pid!, processGroupId: null,
startedAt: new Date("2026-08-09T04:00:00.000Z").toISOString(), targetKind: "local",
});
const factoryInputs: NativeExecutionInput[] = [];
const freshStartTurn = vi.fn<NativeSession["startTurn"]>();
const freshBackend = (
@@ -1514,4 +1517,165 @@ describe.each(["unchanged", "newer_active", "stale_idle"] as const)(
);
describe("goal-less native task-session publication", () => {
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
let db: ReturnType<typeof createDb>;
beforeAll(async () => {
const external = process.env.PAPERCLIP_TEST_DATABASE_URL;
if (external) db = createDb(external);
else {
database = await startEmbeddedPostgresTestDatabase("paperclip-native-pointer-");
db = createDb(database.connectionString);
}
}, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
afterAll(async () => {
await closeRegisteredClients(process.env.PAPERCLIP_TEST_DATABASE_URL ?? database!.connectionString);
await database?.cleanup();
});
it.each(["retained", "reset"] as const)("publishes the completed task session before provider cleanup and resumes an ordinary comment (%s)", async mode => {
const home = createIsolatedTestHome();
const previousHome = process.env.PAPERCLIP_HOME;
process.env.PAPERCLIP_HOME = home.home;
const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID();
const issuePrefix = `NPT${companyId.slice(0, 6).toUpperCase()}`;
const projectId = randomUUID(), projectWorkspaceId = randomUUID(), workProductId = randomUUID();
let providerSessionId: string | undefined;
const repoRoot = fileURLToPath(new URL("../../../", import.meta.url));
const evidenceRef = `work_product:${workProductId}`;
const inputs: NativeExecutionInput[] = [];
const configs: Record<string, unknown>[] = [];
const buildMetadata = heartbeatWorkspaces.buildEffectiveRunSessionConfigMetadata;
const configuration = vi.spyOn(heartbeatWorkspaces, "buildEffectiveRunSessionConfigMetadata").mockImplementation(input => {
configs.push(structuredClone(input.effectiveAdapterConfig));
return buildMetadata(input);
});
const openSession = vi.fn(), recoverSession = vi.fn(), startTurn = vi.fn();
let releaseClose!: () => void, notifyClose!: () => void;
const closeHeld = new Promise<void>(resolve => { releaseClose = resolve; });
const closeEntered = new Promise<void>(resolve => { notifyClose = resolve; });
let firstRunId: string | undefined;
const backendFactory = (execution: NativeExecutionInput): NativeSessionBackend => {
inputs.push(structuredClone(execution));
firstRunId ??= execution.binding.runId;
const currentRunId = execution.binding.runId;
const currentProviderSessionId = `provider-${execution.session.normalizedSessionId}`;
providerSessionId ??= currentProviderSessionId;
const turnId = `turn-${currentRunId}`;
const identity = { companyId, agentId, issueId, runId: currentRunId, sessionId: execution.session.normalizedSessionId };
const result = structuredClone(CONTROL_PLANE_CONFORMANCE_RESULT);
result.completionClaim.contractRevision = execution.completionContract.contract.revision;
result.completionClaim.criteria = execution.completionContract.contract.criteria.map(criterion => ({
...result.completionClaim.criteria[0]!, criterionId: criterion.id, evidenceRefs: [evidenceRef],
}));
result.evidence = [{ kind: "work_product", ref: evidenceRef }];
result.verification[0]!.artifactRef = evidenceRef;
let started!: () => void;
const turnStarted = new Promise<void>(resolve => { started = resolve; });
const session: NativeSession = {
identity: () => identity,
async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; },
async *events() {
await turnStarted;
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, currentRunId));
yield { schema: "paperclip.prp.event.v1", sourceEventId: `${currentRunId}:terminal`, sourceSeq: 1,
sourceInstanceId: run.runnerInstanceId!, sourceKind: "runner", runId: currentRunId,
normalizedSessionId: identity.sessionId, turnId, eventType: "turn.completed", schemaVersion: 1,
priority: 0, emittedAt: new Date().toISOString(), payload: {} } satisfies PrpEvent;
},
async startTurn(input) { startTurn(input); started(); return { turnId }; },
async result() { await turnStarted; return { result, terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, turnId }; },
async snapshot() { return { backendKind: "mock", sessionId: "driver-retained", identity, providerSessionId: currentProviderSessionId,
activeTurnId: null, pendingRuntimeRequests: [], lineage: [], cursor: "1" }; },
async close() { if (currentRunId === firstRunId) { notifyClose(); await closeHeld; } },
};
return {
async descriptor() { return { kind: "mock", name: "task-pointer-backend", version: "1",
capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },
runtimeContextCapabilities: { instructions: "native", skills: "native", mcp: "native" } }; },
async openSession(input) { openSession(input); return session; },
async recoverSession(snapshot) { recoverSession(snapshot); expect(snapshot.providerSessionId).toBe(currentProviderSessionId);
return { recovered: true, session }; },
};
};
const heartbeat = heartbeatService(db, { nativeSessionBackendFactory: backendFactory });
try {
await instanceSettingsService(db).updateExperimental({ enableNativeRunner: true, enableIsolatedWorkspaces: true });
await db.insert(authUsers).values({ id: "pointer-user", name: "Pointer user", email: "pointer@example.test", createdAt: new Date(), updatedAt: new Date() }).onConflictDoNothing();
await db.insert(companies).values({ id: companyId, name: "Native pointer", issuePrefix, status: "active", defaultResponsibleUserId: "pointer-user" });
await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: "pointer-user", status: "active", membershipRole: "member" });
await db.insert(projects).values({ id: projectId, companyId, name: "Pointer project", status: "active" });
await db.insert(projectWorkspaces).values({ id: projectWorkspaceId, companyId, projectId, name: "Pointer workspace", cwd: repoRoot, isPrimary: true });
await db.insert(agents).values({ id: agentId, companyId, name: "Pointer agent", adapterType: "paperclip_runner", status: "active",
adapterConfig: { provider: "codex", workspaceStrategy: { type: "project_primary" } },
runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 } } });
await db.insert(issues).values({ id: issueId, companyId, projectId, projectWorkspaceId, issueNumber: 1,
identifier: `${issuePrefix}-1`, title: "Deliver the saved artifact", status: "in_progress", assigneeAgentId: agentId, workMode: "standard",
// Declare the shared mode up front, as the normal profile does. The
// first workspace otherwise materializes this setting and legitimately
// changes the effective adapter strategy before the next admission.
executionWorkspaceSettings: { mode: "shared_workspace" } });
await db.insert(issueWorkProducts).values({ id: workProductId, companyId, issueId, type: "artifact", provider: "paperclip",
title: "Saved artifact", status: "ready_for_review", reviewState: "approved" });
const first = await heartbeat.wakeup(agentId, { source: "automation", triggerDetail: "system", reason: "issue_commented",
requestedByActorType: "user", requestedByActorId: "pointer-user", payload: { issueId }, contextSnapshot: { issueId, taskId: issueId, skipIssueComment: true } });
expect(first).not.toBeNull();
await closeEntered;
const [persisted] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, first!.id));
expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, first!.id))).toHaveLength(1);
expect(persisted.runnerProfileJson?.sessionCheckpoint).toMatchObject({ identity: { sessionId: persisted.nativeSessionId },
providerSessionId, semanticResult: expect.any(Object), terminal: { runTerminalState: "succeeded" } });
const pointer = await db.select().from(agentTaskSessions).where(eq(agentTaskSessions.agentId, agentId));
expect(pointer).toEqual([expect.objectContaining({ lastRunId: first!.id,
sessionParamsJson: expect.objectContaining({ sessionId: persisted.nativeSessionId }), sessionDisplayId: providerSessionId })]);
if (mode === "reset") {
expect(await heartbeat.resetRuntimeSession(agentId, { taskKey: issueId })).toMatchObject({ clearedTaskSessions: 1 });
expect(await heartbeat.listTaskSessions(agentId)).toEqual([]);
}
releaseClose();
await heartbeat.drainActiveRunExecutions();
if (mode === "reset") expect(await heartbeat.listTaskSessions(agentId)).toEqual([]);
// The normal comment route reopens a completed task before its user wake.
// Use those public services; a raw comment on a still-done task is stale.
const issuesSvc = issueService(db);
await issuesSvc.update(issueId, { status: "todo", actorUserId: "pointer-user" });
const comment = await issuesSvc.addComment(issueId, "Continue using the saved artifact.", { userId: "pointer-user" });
const second = await heartbeat.wakeup(agentId, { source: "automation", triggerDetail: "system", reason: "issue_reopened_via_comment",
requestedByActorType: "user", requestedByActorId: "pointer-user", payload: { issueId, commentId: comment.id, mutation: "comment", reopenedFrom: "done" },
contextSnapshot: { issueId, taskId: issueId, wakeCommentId: comment.id, reopenedFrom: "done", skipIssueComment: true } });
expect(second).not.toBeNull();
await heartbeat.drainActiveRunExecutions();
const after = await heartbeat.getRun(second!.id);
expect(after?.status).toBe("succeeded");
expect(inputs).toHaveLength(2);
if (mode === "retained") {
const changedKeys = [...new Set(configs.flatMap(config => Object.keys(config)))].filter(key =>
JSON.stringify(configs[0]?.[key]) !== JSON.stringify(configs.at(-1)?.[key]));
expect(changedKeys).toEqual([]);
expect(after).toMatchObject({ nativeSessionId: persisted.nativeSessionId, sessionIdAfter: providerSessionId });
expect(openSession).toHaveBeenCalledOnce();
expect(recoverSession).toHaveBeenCalledOnce();
} else {
expect(after?.nativeSessionId).not.toBe(persisted.nativeSessionId);
expect(after?.sessionIdAfter).not.toBe(providerSessionId);
expect(openSession).toHaveBeenCalledTimes(2);
expect(recoverSession).not.toHaveBeenCalled();
}
expect(startTurn).toHaveBeenCalledTimes(2);
expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, second!.id))).toHaveLength(1);
} finally {
releaseClose();
try {
await heartbeat.drainActiveRunExecutions();
} finally {
configuration.mockRestore();
if (previousHome === undefined) delete process.env.PAPERCLIP_HOME;
else process.env.PAPERCLIP_HOME = previousHome;
home.cleanup();
}
}
}, 30_000);
});
enableRunFailureReporting();
@@ -23,6 +23,7 @@ const apiPrefixes: Record<string, string> = {
"agents.ts": "/api",
"agent-project-access.ts": "/api",
"agent-commentary.ts": "/api",
"agent-chat-messages.ts": "/api",
"agent-avatars.ts": "/api",
"agent-profile-avatar.ts": "/api",
"announcements.ts": "/api",
@@ -10,7 +10,7 @@ afterEach(async () => { await coordinator?.stop(); vi.useRealTimers(); });
it("preserves a subsecond replacement deadline when the owner does not require pass spacing", async () => {
vi.useFakeTimers();
const onError = vi.fn();
const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) });
const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []), transaction: vi.fn(async () => undefined) });
coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => true, onError });
const now = Date.now();
let dueAt = now + 60_000;
@@ -38,6 +38,263 @@ it("preserves a subsecond replacement deadline when the owner does not require p
expect(onError).not.toHaveBeenCalled();
});
type Entry = { id: string; companyId: string; issueId: string; dueAt: number };
function concurrentFixture(entries: Entry[], limit = 4, concurrent = true) {
vi.useFakeTimers();
const onError = vi.fn();
const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []), transaction: vi.fn(async () => undefined) });
let canRun = true;
let releaseAll = false;
const releases = new Map<string, () => void>();
const signals: AbortSignal[] = [];
const started: string[] = [];
const active = new Set<string>();
const completed = new Set<string>();
let maximumActive = 0;
const check = vi.fn(async (entry: Entry, signal?: AbortSignal) => {
started.push(entry.id);
signals.push(signal!);
active.add(entry.id);
maximumActive = Math.max(maximumActive, active.size);
if (!releaseAll) await new Promise<void>(resolve => releases.set(entry.id, resolve));
active.delete(entry.id);
completed.add(entry.id);
return null;
});
const load = vi.fn(async () => entries.filter(entry => !completed.has(entry.id)));
coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => canRun, onError });
const options = {
queue: DELIVERY_QUEUES.nativeResponsePresentation,
concurrentChecks: concurrent ? { limit, key: (entry: Entry) => `${entry.companyId}:${entry.issueId}` } : undefined,
decodeScope: () => undefined,
matches: () => true,
load,
check,
};
const worker = registerScopedDeadlineWork(coordinator, options, () => canRun);
return {
worker, check, load, started, active, signals, onError, entries,
maximumActive: () => maximumActive,
pause: () => { canRun = false; },
resume: () => { canRun = true; worker.wake(); },
release: (id: string) => releases.get(id)?.(),
releaseAll: () => { releaseAll = true; for (const resolve of releases.values()) resolve(); },
};
}
const entry = (id: string, issueId = id, companyId = "company"): Entry => ({ id, issueId, companyId, dueAt: 0 });
it("checks independent tasks concurrently with a bounded number of active checks", async () => {
const f = concurrentFixture(Array.from({ length: 6 }, (_, index) => entry(String(index))));
try {
await vi.waitFor(() => expect(f.started).toHaveLength(4), { timeout: 100 });
expect(f.maximumActive()).toBe(4);
f.release("1");
await vi.waitFor(() => expect(f.started).toHaveLength(5), { timeout: 100 });
expect(f.active.has("0")).toBe(true);
expect(f.maximumActive()).toBe(4);
} finally { f.releaseAll(); await f.worker.ready; }
expect(f.started).toHaveLength(6);
expect(new Set(f.started).size).toBe(6);
expect(coordinator!.nextWakeAt()).toBeNull();
await vi.advanceTimersByTimeAsync(60_000);
expect(f.load).toHaveBeenCalledOnce();
expect(f.check).toHaveBeenCalledTimes(6);
f.entries.push(entry("fresh"));
f.worker.invalidate();
await vi.advanceTimersByTimeAsync(1);
expect(f.load).toHaveBeenCalledTimes(2);
expect(f.check).toHaveBeenCalledTimes(7);
expect(f.started.at(-1)).toBe("fresh");
expect(coordinator!.nextWakeAt()).toBeNull();
expect(f.onError).not.toHaveBeenCalled();
});
it("serializes each task in candidate order while other tasks and companies progress", async () => {
const f = concurrentFixture([
entry("1", "shared"), entry("2", "shared"), entry("3", "other"), entry("4", "shared", "other-company"),
]);
try {
await vi.waitFor(() => expect(new Set(f.started)).toEqual(new Set(["1", "3", "4"])), { timeout: 100 });
f.release("3");
await vi.waitFor(() => expect(f.active.has("3")).toBe(false), { timeout: 100 });
expect(f.started).not.toContain("2");
f.release("1");
await vi.waitFor(() => expect(f.started).toHaveLength(4), { timeout: 100 });
expect(f.started.filter(id => id === "1" || id === "2")).toEqual(["1", "2"]);
expect(f.active.has("1")).toBe(false);
} finally { f.releaseAll(); await f.worker.ready; }
expect(f.maximumActive()).toBe(3);
expect(f.onError).not.toHaveBeenCalled();
});
it("preserves global candidate order and serial dispatch unless concurrency is enabled", async () => {
const f = concurrentFixture([entry("1", "shared"), entry("2", "other"), entry("3", "shared")], 4, false);
try {
await vi.waitFor(() => expect(f.started).toEqual(["1"]), { timeout: 100 });
f.release("1");
await vi.waitFor(() => expect(f.started).toEqual(["1", "2"]), { timeout: 100 });
} finally { f.releaseAll(); await f.worker.ready; }
expect(f.started).toEqual(["1", "2", "3"]);
expect(f.maximumActive()).toBe(1);
});
it("retains unstarted checks through drain and resumes without idle discovery reads", async () => {
const f = concurrentFixture(Array.from({ length: 6 }, (_, index) => entry(String(index))));
try {
await vi.waitFor(() => expect(f.started).toHaveLength(4), { timeout: 100 });
f.pause();
} finally { f.releaseAll(); await f.worker.ready; }
expect(f.started).toHaveLength(4);
await vi.advanceTimersByTimeAsync(60_000);
expect(f.load).toHaveBeenCalledOnce();
expect(f.started).toHaveLength(4);
f.resume();
await vi.advanceTimersByTimeAsync(1);
expect(f.started).toHaveLength(6);
expect(coordinator!.nextWakeAt()).toBeNull();
expect(f.onError).not.toHaveBeenCalled();
});
it("stops starting checks and awaits every admitted check during shutdown", async () => {
const f = concurrentFixture(Array.from({ length: 6 }, (_, index) => entry(String(index))));
let stopped = false;
let stop: Promise<void> | undefined;
try {
await vi.waitFor(() => expect(f.started).toHaveLength(4), { timeout: 100 });
stop = coordinator!.stop().then(() => { stopped = true; });
expect(f.signals.every(signal => signal.aborted)).toBe(true);
expect(stopped).toBe(false);
f.release("0");
await vi.waitFor(() => expect(f.active.has("0")).toBe(false), { timeout: 100 });
expect(stopped).toBe(false);
expect(f.started).toHaveLength(4);
} finally { f.releaseAll(); await f.worker.ready; await stop; }
expect(stopped).toBe(true);
expect(f.started).toHaveLength(4);
});
it("keeps failures isolated and retries their deadline without repeating successful checks", async () => {
vi.useFakeTimers();
const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []), transaction: vi.fn(async () => undefined) });
const onError = vi.fn();
coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => true, onError });
const entries = [entry("1", "shared"), entry("2", "shared"), entry("3")];
const load = vi.fn(async () => entries);
const check = vi.fn(async (_entry: Entry) => null).mockRejectedValueOnce(new Error("temporary failure"));
const options = {
queue: DELIVERY_QUEUES.nativeResponsePresentation,
concurrentChecks: { limit: 4, key: (entry: Entry) => `${entry.companyId}:${entry.issueId}` },
decodeScope: () => undefined, matches: () => true, load, check,
};
const worker = registerScopedDeadlineWork(coordinator, options, () => true);
await worker.ready;
expect(check.mock.calls.map(([entry]) => entry.id).sort()).toEqual(["1", "2", "3"]);
expect(onError).toHaveBeenCalledOnce();
expect(onError.mock.calls[0][0]).toBeInstanceOf(AggregateError);
await vi.advanceTimersByTimeAsync(29_999);
expect(check).toHaveBeenCalledTimes(3);
await vi.advanceTimersByTimeAsync(1);
expect(check.mock.calls.map(([entry]) => entry.id)).toEqual(["1", "3", "2", "1"]);
expect(load).toHaveBeenCalledOnce();
expect(coordinator.nextWakeAt()).toBeNull();
});
it("joins one beforePass across keyed parallel checks while retaining per-task order", async () => {
vi.useFakeTimers();
const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []), transaction: vi.fn(async () => undefined) });
const onError = vi.fn(), beforePass = vi.fn(async () => {});
const started: string[] = [], completed = new Set<string>(), releases = new Map<string, () => void>();
let releaseAll = false;
const entries = [entry("first", "shared"), entry("second", "shared"), entry("independent", "other")];
const load = vi.fn(async () => entries.filter(row => !completed.has(row.id)));
coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => true, onError });
const worker = registerScopedDeadlineWork(coordinator, {
queue: DELIVERY_QUEUES.nativeResponsePresentation, minimumPassIntervalMs: 0,
concurrentChecks: { limit: 2, key: (row: Entry) => `${row.companyId}:${row.issueId}` },
reconstructionIntervalMs: 30_000, beforePass,
decodeScope: hint => hint === "*" ? undefined : hint,
matches: (row, companyId?: string) => companyId === undefined || row.companyId === companyId,
load,
check: async row => {
started.push(row.id);
if (!releaseAll) await new Promise<void>(resolve => releases.set(row.id, resolve));
completed.add(row.id);
return null;
},
}, () => true);
try {
await vi.advanceTimersByTimeAsync(0);
expect(started).toEqual(["first", "independent"]);
expect(beforePass).toHaveBeenCalledOnce();
worker.invalidate("company");
await vi.advanceTimersByTimeAsync(0);
expect(beforePass).toHaveBeenCalledOnce();
releases.get("independent")!();
await vi.advanceTimersByTimeAsync(0);
expect(started).not.toContain("second");
releases.get("first")!();
await vi.advanceTimersByTimeAsync(0);
expect(started).toEqual(["first", "independent", "second"]);
expect(beforePass).toHaveBeenCalledOnce();
} finally {
releaseAll = true; for (const release of releases.values()) release(); await worker.ready;
}
await vi.advanceTimersByTimeAsync(1);
expect(beforePass).toHaveBeenCalledTimes(2);
expect(load).toHaveBeenCalledTimes(2);
expect(coordinator.nextWakeAt()).toBeNull();
await vi.advanceTimersByTimeAsync(120_000);
expect(load).toHaveBeenCalledTimes(2); expect(beforePass).toHaveBeenCalledTimes(2);
expect(onError).not.toHaveBeenCalled();
});
it("keeps an exact deadline and late reconstruction debt with concurrency enabled across drain", async () => {
vi.useFakeTimers();
const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []), transaction: vi.fn(async () => undefined) });
const onError = vi.fn(), beforePass = vi.fn(async () => {});
let canRun = true, completed = false, entered!: () => void, release!: () => void;
const loading = new Promise<void>(resolve => { entered = resolve; });
const held = new Promise<void>(resolve => { release = resolve; });
const now = Date.now(), dueAt = now + 5_000;
const candidate = { ...entry("saved", "task"), dueAt };
const load = vi.fn(async () => { if (load.mock.calls.length === 2) { entered(); await held; } return completed ? [] : [candidate]; });
const check = vi.fn(async () => { completed = true; return null; });
coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => canRun, onError });
const worker = registerScopedDeadlineWork(coordinator, {
queue: DELIVERY_QUEUES.nativeResponsePresentation, minimumPassIntervalMs: 0,
concurrentChecks: { limit: 2, key: (row: Entry) => row.issueId },
reconstructionIntervalMs: 30_000, beforePass,
decodeScope: hint => hint === "*" ? undefined : hint,
matches: (row, companyId?: string) => companyId === undefined || row.companyId === companyId,
load, check,
}, () => canRun);
await worker.ready;
worker.invalidate("company");
try {
await vi.advanceTimersByTimeAsync(1); await loading;
worker.invalidate("company");
} finally { release(); }
await vi.advanceTimersByTimeAsync(0);
expect(load).toHaveBeenCalledTimes(2);
expect(coordinator.nextWakeAt()).toBe(dueAt);
await vi.advanceTimersByTimeAsync(4_999);
expect(check).toHaveBeenCalledOnce(); expect(load).toHaveBeenCalledTimes(2);
expect(coordinator.nextWakeAt()).toBe(now + 30_001);
canRun = false; worker.wake();
await vi.advanceTimersByTimeAsync(20_000);
expect(load).toHaveBeenCalledTimes(2); expect(check).toHaveBeenCalledOnce();
canRun = true; worker.wake();
await vi.advanceTimersByTimeAsync(5_001);
expect(load).toHaveBeenCalledTimes(3); expect(check).toHaveBeenCalledOnce();
expect(beforePass).toHaveBeenCalledTimes(4);
expect(coordinator.nextWakeAt()).toBeNull();
await vi.advanceTimersByTimeAsync(120_000);
expect(load).toHaveBeenCalledTimes(3); expect(check).toHaveBeenCalledOnce();
expect(onError).not.toHaveBeenCalled();
});
it("coalesces broad reconstruction without delaying an exact object hint or saved deadline", async () => {
vi.useFakeTimers();
@@ -10,7 +10,7 @@ import { registerSkillVersionRetentionWork } from "../services/skill-version-ret
import { notifySkillVersionRetention } from "../services/skill-version-retention-notifications.js";
import { notifyInstructionCollection } from "../services/instruction-collection-notifications.js";
import { notifyNativeEnvironmentSettlement } from "../services/native-environment-settlement-notifications.js";
import { idleWorkSnapshot } from "../services/task-admission.js";
import { idleWorkSnapshot, trackIdleWork } from "../services/task-admission.js";
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
import { applyPendingMigrations } from "@paperclipai/db";
import { touchSkillVersionCache, lastUsedAt, skillVersionCachePaths } from "../services/skill-version-cache.js";
@@ -158,9 +158,15 @@ describeEmbeddedPostgres("skill version snapshot retention", () => {
return dir;
}
async function startRetention(graceMs = 20_000, service = skillVersionRetentionService(db, { graceMs }), clock: { shouldAdvanceTime?: boolean; now?: number } = {}) {
async function startRetention(
graceMs = 20_000,
service = skillVersionRetentionService(db, { graceMs }),
clock: boolean | { shouldAdvanceTime?: boolean; now?: number } = true,
) {
const clockOptions = typeof clock === "boolean" ? { shouldAdvanceTime: clock } : clock;
const advanceAutomatically = clockOptions.shouldAdvanceTime ?? true;
await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select 1`)));
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout", "setInterval", "clearInterval"], shouldAdvanceTime: true, ...clock });
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout", "setInterval", "clearInterval"], shouldAdvanceTime: advanceAutomatically, ...clockOptions });
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => admitted, canReconcile: () => true,
onError: error => { throw error; } });
const register = coordinator.register.bind(coordinator);
@@ -170,7 +176,8 @@ describeEmbeddedPostgres("skill version snapshot retention", () => {
return pass;
} });
worker = registerSkillVersionRetentionWork(coordinator, service, () => admitted);
await worker.ready;
if (advanceAutomatically) await worker.ready;
else await waitForIo(worker.ready);
return service;
}
async function settled(deadline: number | null = null) {
@@ -180,6 +187,32 @@ describeEmbeddedPostgres("skill version snapshot retention", () => {
expect(coordinator!.nextWakeAt()).toBe(deadline);
}, { timeout: 10_000 });
}
async function waitForIo<T>(operation: Promise<T>) {
let complete = false;
void operation.then(() => { complete = true; }, () => { complete = true; });
const deadline = process.hrtime.bigint() + 10_000_000_000n;
while (!complete) {
if (process.hrtime.bigint() >= deadline) throw new Error("Retention I/O did not settle within 10000ms");
await vi.advanceTimersByTimeAsync(0);
await realDelay(0);
}
return operation;
}
async function settledAtDeadlines() {
const deadline = process.hrtime.bigint() + 10_000_000_000n;
for (;;) {
if (process.hrtime.bigint() >= deadline) throw new Error("Retention work did not settle within 10000ms");
if (idleWorkSnapshot().active > 0) {
if (passes.size) await waitForIo(Promise.all([...passes]));
await vi.advanceTimersByTimeAsync(0);
await realDelay(0);
continue;
}
const next = coordinator!.nextWakeAt();
if (next === null) return;
await vi.advanceTimersByTimeAsync(Math.max(0, next - Date.now()));
}
}
async function quiet(ms = 3 * HOUR_MS) {
const reads = vi.spyOn(db, "select"), transactions = vi.spyOn(db, "transaction"), directories = vi.spyOn(fs, "opendir");
await vi.advanceTimersByTimeAsync(ms);
@@ -581,6 +614,30 @@ describeEmbeddedPostgres("skill version snapshot retention", () => {
expect(await exists(directory)).toBe(false); await quiet();
});
it("does not advance retention deadlines while owned directory I/O is pending", async () => {
await startRetention(20_000, undefined, false);
const entered = Promise.withResolvers<void>(), release = Promise.withResolvers<void>();
const reading = trackIdleWork((async () => {
const directory = await fs.opendir(paperclipHome!);
try { entered.resolve(); await release.promise; }
finally { await directory.close(); }
})());
const advance = vi.spyOn(vi, "advanceTimersByTimeAsync");
let settling: Promise<void> | undefined;
try {
await Promise.race([entered.promise, reading]);
const heldAt = Date.now();
settling = settledAtDeadlines();
await realDelay(0);
expect(advance.mock.calls.every(([ms]) => ms === 0)).toBe(true);
expect(Date.now()).toBe(heldAt);
} finally {
release.resolve();
try { await reading; await settling; }
finally { advance.mockRestore(); }
}
});
it("keeps directory pages moving while another company's grace deadline is due", async () => {
const dueCompany = await createCompany(), pagedCompany = await createCompany();
const dueSkill = await createSkill(dueCompany, 2), pagedSkill = await createSkill(pagedCompany, 1);
@@ -595,26 +652,12 @@ describeEmbeddedPostgres("skill version snapshot retention", () => {
return prune(id, options);
};
const expiring = await writeSnapshot(dueCompany, dueSkill.skillId, dueSkill.versionIds[0]!, 0);
const completeIo = async (work: Promise<unknown>) => {
let done = false;
void work.then(() => { done = true; }, () => { done = true; });
while (!done) {
if (vi.isFakeTimers()) await vi.advanceTimersByTimeAsync(0);
await realDelay(0);
}
await work;
};
const accessedAt = (await lastUsedAt(skillVersionCachePaths(dueCompany, dueSkill.skillId, dueSkill.versionIds[0]!).access))!;
await completeIo(startRetention(5000, service, { shouldAdvanceTime: false, now: accessedAt }));
await startRetention(5000, service, { shouldAdvanceTime: false, now: accessedAt });
expect(await exists(expiring)).toBe(true);
// Keep virtual time fixed during real I/O; measure scheduler fairness,
// independent of filesystem latency and CI CPU scheduling.
while (coordinator!.nextWakeAt() !== null) {
await vi.advanceTimersByTimeAsync(Math.max(0, coordinator!.nextWakeAt()! - Date.now()));
await completeIo(Promise.all([...passes]));
await realDelay(0);
expect(idleWorkSnapshot().active).toBe(0);
}
await settledAtDeadlines();
expect(pageTimes.length).toBeGreaterThan(5);
expect(Math.max(...pageTimes.slice(1).map((at, index) => at - pageTimes[index]!))).toBeLessThan(3000);
expect(await exists(expiring)).toBe(false); await quiet();
+66 -1
View File
@@ -2,7 +2,7 @@ import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database
import { randomUUID } from "node:crypto";
import express from "express";
import request from "supertest";
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import { eq } from "drizzle-orm";
import {
activityLog,
@@ -638,6 +638,71 @@ describeEmbeddedPostgres("status card routes", () => {
expect(refreshIssue).toMatchObject({ status: "cancelled" });
});
it("keeps the current refresh writable after an older revision loses its claim", async () => {
const company = await seedCompany();
const summarizer = await seedSummarizer(company.id);
const service = statusCardService(db);
const realIssues = issueService(db);
await realIssues.create(company.id, { title: "Watched blocker", status: "blocked", priority: "medium" });
const card = await service.create(company.id, {
interestPrompt: "Revision race", titlePinned: false,
refreshPolicy: { ...defaultStatusCardRefreshPolicy, mode: "interval", intervalMinutes: 15 },
}, { agentId: null, userId: "board-user" });
await db.update(statusCards).set({
state: "active", fingerprint: {},
queries: [{ scope: "issues", status: ["blocked"], sort: "updated", limit: 20, offset: 0 }] as typeof card.queries,
}).where(eq(statusCards.id, card.id));
const oldGate = Promise.withResolvers<void>();
const currentGate = Promise.withResolvers<void>();
let oldIssueId: string | undefined;
let currentIssueId: string | undefined;
const blockedIssues = (which: "old" | "current") => ({
...realIssues,
update: async (...args: Parameters<typeof realIssues.update>) => {
const result = await realIssues.update(...args);
if (args[1].description) {
if (which === "old") { oldIssueId = args[0]; await oldGate.promise; }
else { currentIssueId = args[0]; await currentGate.promise; }
}
return result;
},
});
const oldRefresh = statusCardService(db, { issuesSvc: blockedIssues("old") })
.requestRefresh(card.id, { trigger: "interval" });
let currentRefresh: ReturnType<typeof service.requestRefresh> | undefined;
try {
await vi.waitFor(() => expect(oldIssueId).toBeTruthy());
// Ordinary policy editing changes the card revision without changing the
// watched fingerprint. Both refreshes have created their task before the
// old loser cleans up; the current refresh has not yet claimed the card.
await service.update((await service.getById(card.id))!, {
refreshPolicy: defaultStatusCardRefreshPolicy,
}, { agentId: null, userId: "board-user" });
currentRefresh = statusCardService(db, { issuesSvc: blockedIssues("current") })
.requestRefresh(card.id, { trigger: "manual" });
await vi.waitFor(() => expect(currentIssueId).toBeTruthy());
oldGate.resolve();
await oldRefresh;
currentGate.resolve();
const current = await currentRefresh;
const savedIssue = await realIssues.getById(current.generatingIssue!.id);
expect(savedIssue).toMatchObject({ status: "todo" });
expect(current.generatingIssue!.id).not.toBe(oldIssueId);
const run = await seedRun(company.id, summarizer.id);
await db.update(issues).set({ checkoutRunId: run.id }).where(eq(issues.id, savedIssue!.id));
await service.writeSummary(card.id, {
generationIssueId: savedIssue!.id, markdown: "Current revision summary",
changeSummary: "Summarized the current revision", model: "test",
}, { agentId: summarizer.id, runId: run.id });
expect(await service.getById(card.id)).toMatchObject({ generatingIssueId: null });
expect(await realIssues.getById(oldIssueId!)).toMatchObject({ status: "cancelled" });
} finally {
oldGate.resolve(); currentGate.resolve();
await Promise.allSettled([oldRefresh, ...(currentRefresh ? [currentRefresh] : [])]);
}
});
it("finalizes cancelled generation tasks as failed ledger entries", async () => {
const company = await seedCompany();
await enableStatusCards();
@@ -83,6 +83,10 @@ import {
} from "../services/tool-access.js";
import { accessService } from "../services/access.js";
import { instanceSettingsService } from "../services/instance-settings.js";
import { createOAuthAuthorizationStart } from "../services/tool-access/oauth/authorization-start.js";
import { createOAuthClientRegistration } from "../services/tool-access/oauth/client-registration.js";
import { createPersonalConnectionIdentity } from "../services/tool-access/connections/personal-identity.js";
import { createConnectionSetupPolicy } from "../services/tool-access/connections/setup-policy.js";
import { toolAccessPolicyService } from "../services/tool-access-policy.js";
import { secretService } from "../services/secrets.js";
import {
@@ -2620,6 +2624,224 @@ describeEmbeddedPostgres("tool access service", () => {
expect(reconnected.connection.agentInstructions).toEqual(settings);
});
it.each(["managed", "customer"] as const)("preserves a %s Workspace draft's selection after declined and expired authorization", async (mode) => {
const company = await createCompany(db);
const userId = `workspace-draft-${randomUUID()}`;
await grantBoardUser(db, company.id, userId, [], "owner");
const actor = { actorType: "user" as const, actorId: userId };
const connector = fakeGoogleWorkspaceConnector(company.id, userId, "workspace.all");
const service = createTestToolAccessService(db, { paperclipCloudConnector: connector });
const connected = await service.connectGalleryApp(company.id, {
galleryKey: "google-workspace", grantKind: "user",
connectionMethodKey: mode === "managed" ? "paperclip-workspace" : "customer-workspace-oauth",
...(mode === "customer" ? { oauthClient: { clientId: "fixture-client", clientSecret: "fixture-secret" } } : {}),
}, actor);
const redirectUri = "https://paperclip.example.test/api/tools/oauth/callback";
const [unstarted] = await db.select().from(toolConnections).where(eq(toolConnections.id, connected.connectionId));
const configWithProposals = { ...unstarted.config, oauth: { ...(unstarted.config.oauth as Record<string, unknown>), pendingWorkspaceServiceRequests: {
[userId]: { requestedScopes: googleWorkspaceSelectedScopes({ drive: "write" }), requestedAt: "2026-10-26T10:00:00Z" },
other: { requestedScopes: googleWorkspaceSelectedScopes({ calendar: "write" }), requestedAt: "2026-10-26T10:00:00Z" },
} } };
await db.update(toolConnections).set({ config: configWithProposals, transportConfig: configWithProposals }).where(eq(toolConnections.id, connected.connectionId));
const started = await service.startOAuth(company.id, connected.connectionId, {
redirectUri, actor, workspaceServices: { gmail: "read" },
});
const state = new URL(started.authorizationUrl).searchParams.get("state")!;
if (mode === "managed") {
await expect(service.completePaperclipCloudConnectorCallback({ state, error: "access_denied", actor })).rejects.toMatchObject({ status: 400 });
} else {
await expect(service.completeOAuthCallback({ state, error: "access_denied", redirectUri, actor })).rejects.toMatchObject({ status: 400 });
}
const draft = await service.getConnection(connected.connectionId);
expect(draft.status).toBe("draft");
expect(draft.config.workspaceSetup).toEqual({
version: 1, companyId: company.id, connectionId: draft.id, actorType: "user", actorId: userId,
grantKind: "user", subjectId: userId,
requestedScopes: ["https://www.googleapis.com/auth/gmail.readonly"],
});
expect((draft.config.oauth as typeof configWithProposals.oauth).pendingWorkspaceServiceRequests).toEqual({ other: configWithProposals.oauth.pendingWorkspaceServiceRequests.other });
const retained = await service.connectGalleryApp(company.id, {
galleryKey: "google-workspace", resumeConnectionId: draft.id,
connectionMethodKey: mode === "managed" ? "paperclip-workspace" : "customer-workspace-oauth",
...(mode === "customer" ? { oauthClient: { clientId: "fixture-client" } } : {}),
}, actor);
expect(retained.connectionId).toBe(draft.id);
const resumed = await service.startOAuth(company.id, retained.connectionId, { redirectUri, actor });
const [resumedState] = await db.select().from(toolOauthStates).where(eq(toolOauthStates.state, new URL(resumed.authorizationUrl).searchParams.get("state")!));
expect(resumedState.requestedScopes).toEqual(["https://www.googleapis.com/auth/gmail.readonly"]);
await db.update(toolOauthStates).set({ expiresAt: new Date(Date.now() - 1000) }).where(eq(toolOauthStates.state, resumedState.state));
if (mode === "managed") {
await expect(service.completePaperclipCloudConnectorCallback({ state: resumedState.state, claimId: "expired", actor })).rejects.toMatchObject({ status: 400 });
} else {
await expect(service.completeOAuthCallback({ state: resumedState.state, code: "expired", redirectUri, actor })).rejects.toMatchObject({ status: 400 });
}
const coldService = createTestToolAccessService(db, { paperclipCloudConnector: connector });
const coldResume = await coldService.startOAuth(company.id, draft.id, { redirectUri, actor });
const [coldState] = await db.select().from(toolOauthStates).where(eq(toolOauthStates.state, new URL(coldResume.authorizationUrl).searchParams.get("state")!));
expect(coldState.requestedScopes).toEqual(["https://www.googleapis.com/auth/gmail.readonly"]);
});
it.each(["bound", "first-use"] as const)("preserves the latest Workspace draft choice across overlapping %s customer starts", async (bindingMode) => {
const company = await createCompany(db);
const userId = `workspace-overlap-${randomUUID()}`;
await grantBoardUser(db, company.id, userId, [], "owner");
const actor = { actorType: "user" as const, actorId: userId };
const service = createTestToolAccessService(db);
const connected = await service.connectGalleryApp(company.id, {
galleryKey: "google-workspace", connectionMethodKey: "customer-workspace-oauth", grantKind: "user",
oauthClient: { clientId: "fixture-client", clientSecret: "fixture-secret" },
}, actor);
const redirectUri = "https://paperclip.example.test/api/tools/oauth/callback";
const endpoints = {
provider: "google", authorizationUrl: "https://accounts.google.com/o/oauth2/v2/auth",
tokenUrl: "https://oauth2.googleapis.com/token", registrationUrl: null,
scopes: googleWorkspaceSelectedScopes(GOOGLE_WORKSPACE_DEFAULT_SELECTION),
offlineAccessSupported: true, issuer: "https://accounts.google.com", resource: null,
};
const getConnectionRow = async () => {
const [row] = await db.select().from(toolConnections).where(eq(toolConnections.id, connected.connectionId));
return row;
};
const initial = await getConnectionRow();
if (bindingMode === "bound") {
const config = { ...initial.config, oauth: { ...(initial.config.oauth as Record<string, unknown>),
clientRedirectUri: redirectUri, clientIssuer: endpoints.issuer, clientResource: null, clientCompanyId: company.id,
} };
await db.update(toolConnections).set({ config, transportConfig: config }).where(eq(toolConnections.id, initial.id));
} else {
expect((initial.config.oauth as Record<string, unknown>).clientRedirectUri).toBeUndefined();
}
const observedClientRows: typeof initial[] = [];
const ensureOAuthClient = createOAuthClientRegistration(db, {
now: () => new Date(), options: {}, getConnectionRow,
requestOAuthClientRegistration: vi.fn(),
store: {
createOrRotateOAuthSecret: vi.fn(), syncCredentialBindings: vi.fn(),
oauthClientForConnection: async (row) => {
observedClientRows.push(row);
return { clientId: "fixture-client", clientSecret: "fixture-secret", clientIdEnv: null, clientSecretEnv: null };
},
},
});
let releaseA!: () => void;
let reachedA!: () => void;
const pausedA = new Promise<void>(resolve => { reachedA = resolve; });
const resumeA = new Promise<void>(resolve => { releaseA = resolve; });
let discoveries = 0;
const makeStart = () => createOAuthAuthorizationStart(db, {
now: () => new Date(), getConnectionRow, ensureOAuthClient,
setupPolicy: createConnectionSetupPolicy(db), fixedPersonalIdentityForReconnect: createPersonalConnectionIdentity(db),
currentCloudConnector: () => null, vercelConnect: null,
oauthEndpointsForConnection: async () => {
if (++discoveries === 1) { reachedA(); await resumeA; }
return endpoints;
},
assertOAuthEndpointUrl: (_kind, url) => String(url),
});
const start = makeStart();
const first = start(company.id, initial.id, { redirectUri, actor, subjectUserId: userId, workspaceServices: { gmail: "write" } });
let startedA;
let startedB;
let afterB;
try {
await pausedA;
startedB = await start(company.id, initial.id, { redirectUri, actor, subjectUserId: userId, workspaceServices: { gmail: "read" } });
afterB = await getConnectionRow();
expect((afterB.config.oauth as Record<string, unknown>).clientRedirectUri).toBe(redirectUri);
const pending = { anotherUser: { requestedScopes: googleWorkspaceSelectedScopes({ calendar: "read" }), requestedAt: "2026-10-30T16:00:00Z" } };
await db.update(toolConnections).set({
config: { ...afterB.config, concurrentSetting: "config-b", oauth: { ...(afterB.config.oauth as Record<string, unknown>), pendingWorkspaceServiceRequests: pending } },
transportConfig: { ...afterB.transportConfig, transportOnlySetting: "transport-b", oauth: { ...(afterB.transportConfig.oauth as Record<string, unknown>), pendingWorkspaceServiceRequests: pending } },
}).where(eq(toolConnections.id, initial.id));
} finally {
releaseA();
startedA = await first;
}
const writeScopes = googleWorkspaceSelectedScopes({ gmail: "write" });
const readScopes = googleWorkspaceSelectedScopes({ gmail: "read" });
const expectedBinding = { version: 1, companyId: company.id, connectionId: initial.id, actorType: "user", actorId: userId,
grantKind: "user", subjectId: userId, requestedScopes: readScopes };
if (bindingMode === "first-use") {
expect(observedClientRows).toHaveLength(2);
expect((observedClientRows[1].config.oauth as Record<string, unknown>).clientRedirectUri).toBe(redirectUri);
expect(observedClientRows[1].config.workspaceSetup).toEqual(expectedBinding);
}
const latest = await getConnectionRow();
expect(afterB!.config.workspaceSetup).toEqual(expectedBinding);
expect(latest.config.workspaceSetup).toEqual(expectedBinding);
expect(latest.transportConfig.workspaceSetup).toEqual(expectedBinding);
expect(latest.config.concurrentSetting).toBe("config-b");
expect(latest.transportConfig.transportOnlySetting).toBe("transport-b");
for (const config of [latest.config, latest.transportConfig]) {
expect((config.oauth as Record<string, unknown>).pendingWorkspaceServiceRequests).toEqual({ anotherUser: {
requestedScopes: googleWorkspaceSelectedScopes({ calendar: "read" }), requestedAt: "2026-10-30T16:00:00Z",
} });
}
for (const [started, expected] of [[startedA!, writeScopes], [startedB!, readScopes]] as const) {
const url = new URL(started.authorizationUrl);
expect(url.searchParams.get("scope")?.split(" ")).toEqual(expected);
const [state] = await db.select().from(toolOauthStates).where(eq(toolOauthStates.state, url.searchParams.get("state")!));
expect(state.requestedScopes).toEqual(expected);
}
const cold = await makeStart()(company.id, initial.id, { redirectUri, actor, subjectUserId: userId });
const coldUrl = new URL(cold.authorizationUrl);
expect(coldUrl.searchParams.get("scope")?.split(" ")).toEqual(readScopes);
const [coldState] = await db.select().from(toolOauthStates).where(eq(toolOauthStates.state, coldUrl.searchParams.get("state")!));
expect(coldState.requestedScopes).toEqual(readScopes);
});
it.each((["clientId", "clientSecretRef", "clientIssuer", "clientResource", "clientRedirectUri", "issuer", "resource"] as const)
.flatMap(field => (["discovery", "client-resolution"] as const).map(stage => [field, stage] as const)))(
"rejects a concurrent %s change during Workspace %s", async (field, stage) => {
const company = await createCompany(db);
const actor = { actorType: "user" as const, actorId: "board" };
const service = createTestToolAccessService(db);
const connected = await service.connectGalleryApp(company.id, {
galleryKey: "google-workspace", connectionMethodKey: "customer-workspace-oauth", grantKind: "organization",
oauthClient: { clientId: "fixture-client", clientSecret: "fixture-secret" },
}, actor);
const getConnectionRow = async () => {
const [row] = await db.select().from(toolConnections).where(eq(toolConnections.id, connected.connectionId));
return row;
};
const endpoints = { provider: "google", authorizationUrl: "https://accounts.google.com/o/oauth2/v2/auth",
tokenUrl: "https://oauth2.googleapis.com/token", scopes: googleWorkspaceSelectedScopes({ gmail: "read" }),
issuer: "https://accounts.google.com", resource: null };
const ensureClient = createOAuthClientRegistration(db, {
now: () => new Date(), options: {}, getConnectionRow, requestOAuthClientRegistration: vi.fn(),
store: { createOrRotateOAuthSecret: vi.fn(), syncCredentialBindings: vi.fn(),
oauthClientForConnection: async () => ({ clientId: "fixture-client", clientSecret: "fixture-secret", clientIdEnv: null, clientSecretEnv: null }) },
});
let changed: Awaited<ReturnType<typeof getConnectionRow>>;
const mutateClient = async () => {
const row = await getConnectionRow();
const updates = field === "clientSecretRef"
? { credentialSecretRefs: row.credentialSecretRefs.map(ref => ref.configPath === "oauth.client_secret" ? { ...ref, versionSelector: 1 } : ref) }
: {
config: { ...row.config, oauth: { ...(row.config.oauth as Record<string, unknown>), [field]: `changed-${field}` } },
transportConfig: { ...row.transportConfig, oauth: { ...(row.transportConfig.oauth as Record<string, unknown>), [field]: `changed-${field}` } },
};
if (field === "clientSecretRef") expect(updates.credentialSecretRefs).not.toEqual(row.credentialSecretRefs);
[changed] = await db.update(toolConnections).set(updates).where(eq(toolConnections.id, row.id)).returning();
};
const start = createOAuthAuthorizationStart(db, {
now: () => new Date(), getConnectionRow,
setupPolicy: createConnectionSetupPolicy(db), fixedPersonalIdentityForReconnect: createPersonalConnectionIdentity(db),
currentCloudConnector: () => null, vercelConnect: null,
oauthEndpointsForConnection: async () => { if (stage === "discovery") await mutateClient(); return endpoints; },
ensureOAuthClient: async input => { const resolved = await ensureClient(input); if (stage === "client-resolution") await mutateClient(); return resolved; },
assertOAuthEndpointUrl: (_kind, url) => String(url),
});
await expect(start(company.id, connected.connectionId, {
redirectUri: "https://paperclip.example.test/api/tools/oauth/callback", actor, workspaceServices: { gmail: "read" },
})).rejects.toMatchObject({ status: 409 });
const retained = await getConnectionRow();
expect(retained.config).toEqual(changed!.config);
expect(retained.transportConfig).toEqual(changed!.transportConfig);
expect(retained.credentialSecretRefs).toEqual(changed!.credentialSecretRefs);
},
);
it.each(["user", "organization"] as const)("preserves the %s Workspace request when account-page reconnect omits a selection", async (kind) => {
const company = await createCompany(db);
const service = createTestToolAccessService(db);
@@ -2653,6 +2875,69 @@ describeEmbeddedPostgres("tool access service", () => {
expect(new URL(selected.authorizationUrl).searchParams.get("scope")?.split(" ")).toEqual(requestedScopes);
});
it("requires explicit selection for historical Workspace drafts and a different setup owner", async () => {
const company = await createCompany(db);
const userId = `workspace-owner-${randomUUID()}`;
const otherUserId = `workspace-owner-${randomUUID()}`;
await grantBoardUser(db, company.id, userId, [], "owner");
await grantBoardUser(db, company.id, otherUserId, [], "owner");
const actor = { actorType: "user" as const, actorId: userId };
const connector = fakeGoogleWorkspaceConnector(company.id, userId, "workspace.all");
const service = createTestToolAccessService(db, { paperclipCloudConnector: connector });
const connected = await service.connectGalleryApp(company.id, {
galleryKey: "google-workspace", connectionMethodKey: "paperclip-workspace", grantKind: "organization",
}, actor);
const redirectUri = "https://paperclip.example.test/api/tools/oauth/callback";
await expect(service.startOAuth(company.id, connected.connectionId, { redirectUri, actor })).rejects.toMatchObject({ status: 400 });
await service.startOAuth(company.id, connected.connectionId, { redirectUri, actor, workspaceServices: { gmail: "read" } });
await expect(service.startOAuth(company.id, connected.connectionId, {
redirectUri, actor: { actorType: "user", actorId: otherUserId },
})).rejects.toMatchObject({ status: 400 });
expect((await service.getConnection(connected.connectionId)).config.workspaceSetup).toMatchObject({ actorId: userId });
});
it.each(["credentialPolicy", "credentialSource", "authKind", "transport", "createdByUserId", "sourceTemplateKey", "connectionMethodKey"] as const)("rejects a Workspace draft whose %s changes during authorization", async (field) => {
const company = await createCompany(db);
const actor = { actorType: "user" as const, actorId: "board" };
const service = createTestToolAccessService(db);
const connected = await service.connectGalleryApp(company.id, {
galleryKey: "google-workspace", connectionMethodKey: "customer-workspace-oauth", grantKind: "organization",
oauthClient: { clientId: "fixture-client", clientSecret: "fixture-secret" },
}, actor);
const [original] = await db.select().from(toolConnections).where(eq(toolConnections.id, connected.connectionId));
const changed: Partial<typeof toolConnections.$inferInsert> = {};
if (field === "credentialPolicy") changed.credentialPolicy = "per_user";
else if (field === "credentialSource") {
changed.credentialSource = "vercel_connect";
changed.credentialRefs = [];
changed.credentialSecretRefs = [];
changed.externalCredential = { provider: "vercel_connect", connectorId: "fixture", connectorUid: "fixture", service: "google", principalMode: "user" };
}
else if (field === "authKind") changed.authKind = "none";
else if (field === "transport") changed.transport = "rest_api";
else if (field === "createdByUserId") changed.createdByUserId = "another-owner";
else changed.config = { ...original.config, [field]: "different" };
const start = createOAuthAuthorizationStart(db, {
now: () => new Date(),
getConnectionRow: async () => {
await db.update(toolConnections).set(changed).where(eq(toolConnections.id, original.id));
return original;
},
setupPolicy: createConnectionSetupPolicy(db),
fixedPersonalIdentityForReconnect: createPersonalConnectionIdentity(db),
currentCloudConnector: () => null,
vercelConnect: null,
ensureOAuthClient: vi.fn(),
oauthEndpointsForConnection: vi.fn(),
assertOAuthEndpointUrl: vi.fn(),
});
await expect(start(company.id, original.id, {
redirectUri: "https://paperclip.example.test/api/tools/oauth/callback", actor, workspaceServices: { gmail: "read" },
})).rejects.toMatchObject({ status: 409 });
expect((await service.getConnection(original.id)).config.workspaceSetup).toBeUndefined();
expect(await db.select().from(toolOauthStates).where(eq(toolOauthStates.connectionId, original.id))).toHaveLength(0);
});
it("binds selected Workspace service scopes to OAuth state and rejects other providers", async () => {
const company = await createCompany(db);
const service = createTestToolAccessService(db);
@@ -2686,7 +2971,7 @@ describeEmbeddedPostgres("tool access service", () => {
oauthClient: { clientId: "test-workspace-client", clientSecret: "test-workspace-secret" },
}, actor);
const redirectUri = "https://paperclip.example.test/api/tools/oauth/callback";
const started = await service.startOAuth(company.id, connected.connectionId, { redirectUri, actor });
const started = await service.startOAuth(company.id, connected.connectionId, { redirectUri, actor, workspaceServices: GOOGLE_WORKSPACE_DEFAULT_SELECTION });
const authorization = new URL(started.authorizationUrl);
expect(authorization.searchParams.get("scope")?.split(" ")).toEqual(googleWorkspaceSelectedScopes(GOOGLE_WORKSPACE_DEFAULT_SELECTION));
expect(authorization.searchParams.get("enable_granular_consent")).toBe("true");
@@ -2742,7 +3027,9 @@ describeEmbeddedPostgres("tool access service", () => {
const redirectUri = `https://paperclip.example/api/tools/oauth/${mode === "managed" ? "cloud-connector/" : ""}callback`;
for (const reconnect of [false, true]) {
if (reconnect) await service.connectGalleryApp(company.id, { ...input, reconnectConnectionId: connected.connectionId }, actor);
const started = await service.startOAuth(company.id, connected.connectionId, { redirectUri, actor });
const started = await service.startOAuth(company.id, connected.connectionId, { redirectUri, actor,
...(!reconnect ? { workspaceServices: GOOGLE_WORKSPACE_DEFAULT_SELECTION } : {}),
});
const state = new URL(started.authorizationUrl).searchParams.get("state")!;
const completed = mode === "managed"
? await service.completePaperclipCloudConnectorCallback({ state, claimId: "workspace-no-agents", actor })
@@ -7629,6 +7916,7 @@ describeEmbeddedPostgres("tool access service", () => {
expect(workspaceApp.methods.map((method: { key: string }) => method.key)).toContain(methodKey);
const connected = await request(app).post(`/api/companies/${company.id}/tools/apps/connect`).send({
galleryKey: slug, connectionMethodKey: methodKey, grantKind: "user", name: `Personal ${slug}`,
...(profile === "workspace.all" ? { workspaceServices: GOOGLE_WORKSPACE_DEFAULT_SELECTION } : {}),
});
expect(connected.status).toBe(201);
expect(connected.body.connection).toMatchObject({ credentialPolicy: "per_user", ownership: "platform_shared" });
@@ -7691,6 +7979,7 @@ describeEmbeddedPostgres("tool access service", () => {
.map((method: { key: string }) => method.key)).toContain(methodKey);
const result = await request(app).post(`/api/companies/${company.id}/tools/apps/connect`).send({
galleryKey: slug, connectionMethodKey: methodKey, grantKind: "user", name: `Cloud ${slug}`,
...(profile === "workspace.all" ? { workspaceServices: GOOGLE_WORKSPACE_DEFAULT_SELECTION } : {}),
});
expect(result.status, JSON.stringify(result.body)).toBe(201);
expect(result.body.auth.startUrl).toBe("https://my.paperclip.app/connections/confirm?id=test-workspace-session");
@@ -8297,6 +8586,7 @@ describeEmbeddedPostgres("tool access service", () => {
}, actor);
const started = await service.startOAuth(company.id, connected.connectionId, {
redirectUri: "https://paperclip.example/api/tools/oauth/cloud-connector/callback", actor,
workspaceServices: GOOGLE_WORKSPACE_DEFAULT_SELECTION,
});
const completed = await service.completePaperclipCloudConnectorCallback({
state: new URL(started.authorizationUrl).searchParams.get("state")!, claimId: "workspace-claim", actor,
@@ -8364,7 +8654,7 @@ describeEmbeddedPostgres("tool access service", () => {
...(mode === "customer" ? { oauthClient: { clientId: "workspace-client", clientSecret: "workspace-client-secret" } } : {}),
}, actor);
const redirectUri = mode === "managed" ? "https://paperclip.example/api/tools/oauth/cloud-connector/callback" : "https://paperclip.example/api/tools/oauth/callback";
const started = await service.startOAuth(company.id, connected.connectionId, { redirectUri, actor });
const started = await service.startOAuth(company.id, connected.connectionId, { redirectUri, actor, workspaceServices: GOOGLE_WORKSPACE_DEFAULT_SELECTION });
const state = new URL(started.authorizationUrl).searchParams.get("state")!;
const completed = mode === "managed"
? await service.completePaperclipCloudConnectorCallback({ state, claimId: "fixture", actor })
@@ -0,0 +1,111 @@
import { randomUUID } from "node:crypto";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { eq, sql } from "drizzle-orm";
import {
companies,
createDb,
toolApplications,
toolCatalogEntries,
toolConnections,
toolProfileEntries,
toolProfiles,
} from "@paperclipai/db";
import {
getEmbeddedPostgresTestSupport,
startEmbeddedPostgresTestDatabase,
} from "./helpers/embedded-postgres.js";
import { createToolProfileService } from "../services/tool-access/profiles/service.js";
const support = await getEmbeddedPostgresTestSupport();
const describeDatabase = support.supported ? describe : describe.skip;
describeDatabase("tool profile entry order", () => {
let db: ReturnType<typeof createDb>;
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
beforeAll(async () => {
database = await startEmbeddedPostgresTestDatabase("paperclip-profile-entry-order-");
db = createDb(database.connectionString, { maxConnections: 1 });
}, 20_000);
afterAll(async () => {
await db?.$client.end({ timeout: 1 });
await database?.cleanup();
});
it("keeps list and detail entries stable across tied timestamps and physical row order", async () => {
const [company, foreign] = await db.insert(companies).values([
{ name: "Entry order", issuePrefix: `EO${randomUUID().slice(0, 6)}` },
{ name: "Other entry order", issuePrefix: `EF${randomUUID().slice(0, 6)}` },
]).returning();
const [profile, foreignProfile] = await db.insert(toolProfiles).values([
{ companyId: company.id, profileKey: "ordered", name: "Ordered" },
{ companyId: foreign.id, profileKey: "foreign", name: "Foreign" },
]).returning();
const [application] = await db.insert(toolApplications).values({
companyId: company.id, name: "Order fixture", type: "mcp_http", status: "active",
}).returning();
const [connection] = await db.insert(toolConnections).values({
companyId: company.id, applicationId: application.id, name: "Order fixture",
uid: `fixture/${randomUUID()}`, transport: "mcp_remote", status: "active",
enabled: true, config: {},
}).returning();
await db.insert(toolCatalogEntries).values(["allowed", "blocked"].map(toolName => ({
companyId: company.id, applicationId: application.id, connectionId: connection.id,
name: toolName, toolName, riskLevel: "read" as const, status: "active" as const,
versionHash: randomUUID(), schemaHash: randomUUID(),
})));
// Batch inserts use now() for every row. No entry ordinal exists, and an
// exclusion wins regardless of where it appears among matching includes.
const tiedAt = new Date("2026-01-02T00:00:00Z");
const tied = Array.from({ length: 30 }, (_, index) => ({
id: `00000000-0000-4000-8000-${String(index + 1).padStart(12, "0")}`,
companyId: company.id, profileId: profile.id, selectorType: "tool_name" as const,
effect: index === 29 ? "exclude" as const : "include" as const,
toolName: index === 0 ? "allowed" : index === 1 || index === 29 ? "blocked" : `other_${index}`,
createdAt: tiedAt,
}));
const earlier = {
...tied[0], id: "ffffffff-ffff-4fff-8fff-ffffffffffff", toolName: "earlier",
createdAt: new Date("2026-01-01T00:00:00Z"),
};
await db.insert(toolProfileEntries).values([
...tied.toReversed(), earlier,
{ companyId: foreign.id, profileId: foreignProfile.id,
selectorType: "tool_name", toolName: "foreign" },
]);
// Change only heap layout in this disposable database. Force sequential
// scans so both reads exercise the different layouts, not one index order.
await db.execute(sql`SET enable_indexscan = off`);
await db.execute(sql`SET enable_bitmapscan = off`);
await db.execute(sql`CREATE INDEX entry_order_fixture_desc ON tool_profile_entries (id DESC)`);
const service = createToolProfileService(db, () => new Date());
const snapshots = [];
for (const index of ["tool_profile_entries_pkey", "entry_order_fixture_desc"]) {
await db.execute(sql`CLUSTER tool_profile_entries USING ${sql.identifier(index)}`);
const physical = await db.execute<{ id: string }>(sql`
SELECT id FROM tool_profile_entries WHERE profile_id = ${profile.id} ORDER BY ctid
`);
const [listed] = await service.listProfiles(company.id);
const detail = await service.getProfile(profile.id, company.id);
snapshots.push({ physical: physical.map(row => row.id), listed, detail });
}
expect(snapshots[0].physical).not.toEqual(snapshots[1].physical);
const expectedIds = [earlier.id, ...tied.map(entry => entry.id)];
for (const { listed, detail } of snapshots) {
expect(listed.entries.map(entry => entry.id)).toEqual(expectedIds);
expect(detail).toEqual(listed);
expect(listed.summary).toMatchObject({ allowedToolCount: 1, excludedToolCount: 1, totalToolCount: 2 });
}
expect(snapshots[1].listed).toEqual(snapshots[0].listed);
expect(await service.listProfiles(foreign.id)).toMatchObject([{ id: foreignProfile.id }]);
await expect(service.getProfile(profile.id, foreign.id)).rejects.toThrow("Tool profile not found");
// A real entry change must remain visible to configuration comparison.
await db.update(toolProfileEntries).set({ effect: "include" }).where(eq(toolProfileEntries.id, tied[29].id));
const changed = await service.getProfile(profile.id, company.id);
expect(changed.entries).not.toEqual(snapshots[1].detail.entries);
expect(changed.summary).toMatchObject({ allowedToolCount: 2, excludedToolCount: 0 });
});
});
+66 -2
View File
@@ -3,18 +3,25 @@ import { assertValidAdapterLoginCapability } from "@paperclipai/adapter-utils";
import { listServerAdapters, requireServerAdapter } from "./registry.js";
import * as executionTarget from "@paperclipai/adapter-utils/execution-target";
import { BUILTIN_ADAPTER_TYPES } from "./builtin-adapter-types.js";
import { QUALIFIED_ACPX_PROFILES, KIMI_DISTRIBUTION_CLOSURE_SHA256 } from "../vendor/paperclip-runner/index.js";
const { probeInstallation, probeGrokInstallation, probePiInstallation, verifyApiKey } = vi.hoisted(() => ({
const { probeInstallation, probeGrokInstallation, probePiInstallation, probeKimiInstallation, probeKimiModels, probeKimiRemote, verifyApiKey } = vi.hoisted(() => ({
verifyApiKey: vi.fn(),
probeInstallation: vi.fn(),
probeGrokInstallation: vi.fn(),
probePiInstallation: vi.fn(),
probeKimiInstallation: vi.fn(),
probeKimiModels: vi.fn(),
probeKimiRemote: vi.fn(),
}));
vi.mock("../services/ai-api-key-test.js", () => ({ validateAiApiKey: verifyApiKey }));
vi.mock("../services/kimi-connection-probe.js", () => ({ probeKimiExecutionTarget: probeKimiRemote }));
vi.mock("../vendor/paperclip-runner/live/index.js", () => ({
probeAcpxClaudeInstallation: probeInstallation,
probeAcpxGrokInstallation: probeGrokInstallation,
probeAcpxPiInstallation: probePiInstallation,
probeAcpxKimiInstallation: probeKimiInstallation,
probeKimiModels,
probeAcpxCursorInstallation: vi.fn(async () => undefined),
}));
@@ -102,8 +109,11 @@ describe("native ACPX environment checks", () => {
probeInstallation.mockReset().mockResolvedValue(undefined);
probeGrokInstallation.mockReset().mockResolvedValue(undefined);
probePiInstallation.mockReset().mockResolvedValue(undefined);
probeKimiInstallation.mockReset().mockResolvedValue(undefined);
probeKimiModels.mockReset().mockResolvedValue(undefined);
probeKimiRemote.mockReset().mockResolvedValue(undefined);
});
afterEach(() => vi.restoreAllMocks());
afterEach(() => { vi.restoreAllMocks(); vi.unstubAllEnvs(); });
const context = {
companyId: "company-test",
@@ -111,6 +121,53 @@ describe("native ACPX environment checks", () => {
config: { provider: "acpx", acpxAgent: "claude", model: "claude-sonnet-5" },
};
it("requires an explicit Kimi credential and probes its own installed runtime during qualification", async () => {
const model = "explicit-kimi-model";
authorizeKimi(model);
const config = { provider: "acpx", acpxAgent: "kimi", model };
const missing = await requireServerAdapter("paperclip_runner").testEnvironment!({ ...context, config });
expect(missing.status).toBe("fail");
expect(missing.checks[0].code).toBe("KIMI_AUTH_REQUIRED");
expect(probeKimiInstallation).not.toHaveBeenCalled();
const ready = await requireServerAdapter("paperclip_runner").testEnvironment!({ ...context, config: { ...config, env: { KIMI_API_KEY: "fixture-key" } } });
expect(ready.status).toBe("pass");
expect(probeKimiRemote).toHaveBeenCalledExactlyOnceWith("fixture-key", undefined, model, { companyId: context.companyId });
expect(probeKimiInstallation).not.toHaveBeenCalled();
expect(probeKimiModels).not.toHaveBeenCalled();
expect(probeInstallation).not.toHaveBeenCalled();
expect(verifyApiKey).not.toHaveBeenCalled();
expect(ready.checks[0]).toMatchObject({ code: "kimi_metadata_verified", message: expect.stringContaining("No model prompt was sent") });
});
it("keeps a rejected Kimi catalog credential out of the ready lifecycle", async () => {
const model = "explicit-kimi-model";
authorizeKimi(model);
probeKimiRemote.mockRejectedValue(Object.assign(new Error("The Kimi Code credential was rejected by the model catalog."), { code: "KIMI_AUTH_REQUIRED" }));
const result = await requireServerAdapter("paperclip_runner").testEnvironment!({ ...context,
config: { provider: "acpx", acpxAgent: "kimi", model, env: { KIMI_API_KEY: "fixture-key" } },
});
expect(result.status).toBe("fail");
expect(result.checks[0].message).toContain("credential was rejected");
});
it("requires the selected remote Kimi runtime and authenticated catalog instead of a host installation", async () => {
const model = "explicit-kimi-model";
authorizeKimi(model);
const runner = { execute: vi.fn() };
const target = { kind: "remote" as const, transport: "sandbox" as const, remoteCwd: "/workspace", providerKey: "test", runner };
const result = await requireServerAdapter("paperclip_runner").testEnvironment!({ ...context,
config: { provider: "acpx", acpxAgent: "kimi", model, env: { KIMI_API_KEY: "fixture-key" } },
executionTarget: target,
});
expect(result.status).toBe("pass");
expect(result.checks[0]).toMatchObject({ code: "kimi_metadata_verified" });
expect(probeKimiRemote).toHaveBeenCalledExactlyOnceWith("fixture-key", target, model, { companyId: context.companyId });
expect(runner.execute).not.toHaveBeenCalled();
expect(probeKimiInstallation).not.toHaveBeenCalled();
expect(probeKimiModels).not.toHaveBeenCalled();
});
it("verifies a native Claude key without the global CLI or a model prompt", async () => {
const result = await requireServerAdapter("paperclip_runner").testEnvironment!({ ...context,
config: { ...context.config, env: { ANTHROPIC_API_KEY: "test-key" } },
@@ -256,3 +313,10 @@ describe("native ACPX environment checks", () => {
expect(result.checks[0].code).toBe("acpx_runtime_unavailable");
});
});
function authorizeKimi(model: string) {
vi.stubEnv("PAPERCLIP_ACPX_CANDIDATE_ADMISSION", JSON.stringify({ schema: "paperclip.acpx-candidate-admission.v1",
companyId: "company-test", agent: "kimi", model, profileDigest: QUALIFIED_ACPX_PROFILES.kimi.commandDigest,
platform: "linux", architecture: "x64", nativeClosureDigest: `sha256:${KIMI_DISTRIBUTION_CLOSURE_SHA256["linux-x64"]}`,
expiresAt: Date.now() + 60_000 }));
}
+18 -3
View File
@@ -9,6 +9,7 @@ import {
buildSandboxNpmInstallCommand,
getAdapterSessionManagement,
PAPERCLIP_RUNNER_PERMISSION_CAPABILITIES,
PAPERCLIP_RUNNER_ACPX_PROFILES,
} from "@paperclipai/adapter-utils";
import type { AdapterLoginCapability } from "@paperclipai/adapter-utils";
import { runAdapterExecutionTargetShellCommand } from "@paperclipai/adapter-utils/execution-target";
@@ -469,8 +470,22 @@ const paperclipRunnerAdapter: ServerAdapterModule = {
checks: [{ code: "copilot_metadata_verified", level: "info" as const,
message: "The verified Copilot runtime authenticated this account and accepted the selected model. No model prompt was sent." }] };
}
if (profile.acpxAgent !== "claude" && profile.acpxAgent !== "grok" && profile.acpxAgent !== "cursor" && profile.acpxAgent !== "pi") throw new Error("Select Codex to use the native Codex runner.");
if (profile.acpxAgent !== "claude" && profile.acpxAgent !== "grok" && profile.acpxAgent !== "cursor" && profile.acpxAgent !== "pi" && profile.acpxAgent !== "kimi") throw new Error("Select Codex to use the native Codex runner.");
if (profile.acpxAgent === "kimi") {
const environment = context.config.env as Record<string, unknown> | undefined;
if (typeof environment?.KIMI_API_KEY !== "string" || !environment.KIMI_API_KEY.trim()) {
return { adapterType: "paperclip_runner", status: "fail" as const, testedAt: new Date().toISOString(),
checks: [{ code: "KIMI_AUTH_REQUIRED", level: "error" as const, message: "Bind a company secret to KIMI_API_KEY in the agent environment." }] };
}
}
const target = context.executionTarget;
if (profile.acpxAgent === "kimi") {
const { probeKimiExecutionTarget } = await import("../services/kimi-connection-probe.js");
await probeKimiExecutionTarget((context.config.env as Record<string, string>).KIMI_API_KEY, target, profile.model, { companyId: context.companyId });
return { adapterType: "paperclip_runner", status: "pass" as const, testedAt: new Date().toISOString(),
checks: [{ code: "kimi_metadata_verified", level: "info" as const,
message: "The selected environment verified the pinned Kimi Code runtime. Its Code model catalog accepted the credential and lists the exact selected model. No model prompt was sent." }] };
}
if (target?.kind === "remote") {
const probe = await runAdapterExecutionTargetShellCommand(
`acpx-platform-${crypto.randomUUID()}`, target, "uname -s && uname -m",
@@ -478,7 +493,7 @@ const paperclipRunnerAdapter: ServerAdapterModule = {
);
if (probe.timedOut || probe.exitCode !== 0) throw new Error("Could not verify the remote ACPX runner platform.");
const [os, arch] = probe.stdout.trim().split(/\s+/);
if (!((os === "Linux" && (arch === "x86_64" || (profile.acpxAgent === "claude" && ["aarch64", "arm64"].includes(arch)))) || (os === "Darwin" && (arch === "arm64" || (profile.acpxAgent !== "grok" && arch === "x86_64"))))) {
if (!((os === "Linux" && (arch === "x86_64" || (profile.acpxAgent === "claude" && ["aarch64", "arm64"].includes(arch)))) || (os === "Darwin" && (arch === "arm64" || (!["grok"].includes(profile.acpxAgent) && arch === "x86_64"))))) {
throw new Error(`ACPX ${profile.acpxAgent} requires a qualified Linux ${profile.acpxAgent === "claude" ? "x64/ARM64" : "x64"} or macOS architecture.`);
}
return verifyNativeClaudeApiKey(context.config, {
@@ -615,7 +630,7 @@ const paperclipRunnerAdapter: ServerAdapterModule = {
},
{
key: "acpxAgent", label: "ACP agent", type: "select" as const, default: "claude",
options: [{ value: "claude", label: "Claude" }, { value: "grok", label: "Grok Build" }, { value: "copilot", label: "GitHub Copilot" }],
options: PAPERCLIP_RUNNER_ACPX_PROFILES.filter(profile => profile.qualified).map(profile => ({ value: profile.value, label: profile.label })),
meta: { visibleWhen: { key: "provider", value: "acpx" } },
},
{
+2
View File
@@ -127,6 +127,7 @@ import {
import { smokeLabRoutes } from "./routes/smoke-lab.js";
import { costRoutes } from "./routes/costs.js";
import { agentCommentaryRoutes } from "./routes/agent-commentary.js";
import { agentChatMessageRoutes } from "./routes/agent-chat-messages.js";
import { activityRoutes } from "./routes/activity.js";
import { dashboardRoutes } from "./routes/dashboard.js";
import { attentionRoutes } from "./routes/attention.js";
@@ -672,6 +673,7 @@ export async function createApp(
api.use(companySkillPolicyRoutes(db));
api.use(inboxAgentPolicyRoutes(db));
api.use(agentCommentaryRoutes(db));
api.use(agentChatMessageRoutes(db));
api.use(builtInAgentRoutes(db));
api.use(summarySlotRoutes(db));
api.use(statusCardRoutes(db));
+22
View File
@@ -0,0 +1,22 @@
import { Router } from "express";
import type { Db } from "@paperclipai/db";
import { forbidden } from "../errors.js";
import { sendAgentChatMessage } from "../services/agent-chat-messages.js";
import { assertCompanyAccess } from "./authz.js";
export function agentChatMessageRoutes(db: Db) {
const router = Router();
router.post("/agents/me/chat/messages", async (req, res) => {
if (req.actor.type !== "agent" || !req.actor.agentId || !req.actor.companyId || !req.actor.runId) {
throw forbidden("An authenticated task run is required");
}
assertCompanyAccess(req, req.actor.companyId);
const receipt = await sendAgentChatMessage(db, {
companyId: req.actor.companyId, agentId: req.actor.agentId, runId: req.actor.runId, agentApiKeyId: req.actor.keyId,
keyScope: req.actor.keyScope, onBehalfOfUserId: req.actor.onBehalfOfUserId,
authorizingUserId: req.actor.authorizingUserId, source: req.actor.source,
}, req.body);
res.status(receipt!.replayed ? 200 : 201).json(receipt);
});
return router;
}
+14 -5
View File
@@ -41,7 +41,7 @@ import { toolConnections } from "@paperclipai/db";
import { aiConnectionService } from "../services/ai-connections.js";
import { probeCopilotConnection, probeCopilotExecutionTarget } from "../services/copilot-connection-probe.js";
import { antigravityConnectionModels } from "../services/antigravity-connection-models.js";
import { resolveAntigravityCandidateAdmission } from "../services/native-runtime/acpx-qualification.js";
import { resolveAntigravityCandidateAdmission, resolveKimiCandidateAdmission } from "../services/native-runtime/acpx-qualification.js";
import { defaultAiConnectionForHire } from "../services/agent-ai-connection-default.js";
import { assertAiConnectionCreateAccess, canInstallSharedAiConnectionForNewAgent, responsibleUserForAiRequest } from "./ai-connections.js";
import { isAiConnectionCompatible } from "@paperclipai/shared";
@@ -1874,6 +1874,11 @@ export function agentRoutes(
try { antigravityCandidate = resolveAntigravityCandidateAdmission({ companyId, model: selected.model }); }
catch { throw unprocessable("This Google Antigravity candidate requires the exact enabled model and a current operator authorization.", { code: "paperclip_runner_antigravity_candidate_invalid" }); }
}
let kimiCandidate: ReturnType<typeof resolveKimiCandidateAdmission>;
if (companyId && selected?.provider === "acpx" && selected.acpxAgent === "kimi" && typeof selected.model === "string") {
try { kimiCandidate = resolveKimiCandidateAdmission({ companyId, model: selected.model }); }
catch { throw unprocessable("This Kimi candidate requires the exact enabled model and a current operator authorization.", { code: "paperclip_runner_kimi_candidate_invalid" }); }
}
if (asRecord(config)?.provider === "openai_dot") {
if (experimental.enableOpenAiDot !== true) throw unprocessable(
"OpenAI Dot is experimental and disabled on this instance.",
@@ -1886,7 +1891,7 @@ export function agentRoutes(
} else if (!([undefined, "codex", "opencode"].includes(asRecord(config)?.provider as string | undefined)) &&
!(asRecord(config)?.provider === "acpx" && asRecord(config)?.acpxAgent === "copilot") &&
!["claude_local", "pi_local", "grok_local"].includes(agentHarnessType("paperclip_runner", asRecord(config) ?? {})) &&
experimental.enableNativeRunner !== true && !antigravityCandidate) {
experimental.enableNativeRunner !== true && !antigravityCandidate && !kimiCandidate) {
throw unprocessable(
"Paperclip Runner is experimental and disabled on this instance.",
{ code: "paperclip_runner_rollout_disabled" },
@@ -1961,9 +1966,9 @@ export function agentRoutes(
throw unprocessable("Select a Paperclip Runner provider before converting this agent.");
}
const next = { ...defaults, ...input.nextAdapterConfig };
if (next.provider === "acpx" && next.acpxAgent === "pi") {
if (next.provider === "acpx" && (next.acpxAgent === "pi" || next.acpxAgent === "kimi")) {
next.model = input.nextAdapterConfig.model
?? (input.previousAdapterType === "pi_local" ? input.previousAdapterConfig.model : "");
?? (input.previousAdapterType === `${next.acpxAgent}_local` ? input.previousAdapterConfig.model : "");
} else if (!asNonEmptyString(next.model)) next.model = defaults.model;
return normalizeLegacyRunnerProvider(next);
}
@@ -2200,7 +2205,7 @@ export function agentRoutes(
);
const incompleteModelSetup = input.setupOnly && input.adapterType === "paperclip_runner"
&& (normalizedAdapterConfig.provider === "opencode"
|| (normalizedAdapterConfig.provider === "acpx" && normalizedAdapterConfig.acpxAgent === "pi"))
|| (normalizedAdapterConfig.provider === "acpx" && ["pi", "kimi"].includes(String(normalizedAdapterConfig.acpxAgent))))
&& (normalizedAdapterConfig.model === undefined || normalizedAdapterConfig.model === "");
// Inactive drafts can precede model selection. Test/hire always validate the complete profile.
if (!incompleteModelSetup) await assertAdapterConfigConstraints(
@@ -3076,6 +3081,10 @@ export function agentRoutes(
return;
}
const provider = asNonEmptyString(req.query.provider);
if (type === "paperclip_runner" && provider === "acpx" && req.query.acpxAgent === "kimi") {
res.json([]);
return;
}
if (type === "paperclip_runner" && provider === "acpx" && req.query.acpxAgent === "antigravity") {
if (environment && environment.driver !== "local") throw unprocessable("Google Antigravity account model discovery requires the Paperclip host.");
let binding: AiConnectionBinding = { provider: "google", method: "subscription", mode: "responsible_user" };
+20 -4
View File
@@ -11,6 +11,8 @@ import { issueSummariesQuerySchema, type IssueSummary } from "@paperclipai/share
import { listIssueSummaries } from "../services/issue-summaries.js";
import { assertNoCaseHandoffMutation } from "../services/heartbeat/case-handoff-lifecycle.js";
import { assertLegacyRunTaskOwnership } from "../legacy-run-task-ownership.js";
import { withConversationReadState } from "../services/agent-chat-read-state.js";
import { markIssueReadSchema } from "@paperclipai/shared";
import { enqueueFastResponse } from "../services/fast-responses.js";
import { hasRequiredWorkspaceRecovery, LEGACY_WORKSPACE_RECOVERY_SCHEMA } from "../services/workspace-restore-recovery-state.js";
import { hasUnverifiedNativeWorkspaceFinalizationOwner } from "../services/native-runtime/native-workspace-finalization-recovery-state.js";
@@ -12107,11 +12109,22 @@ export function issueRoutes(
res.status(403).json({ error: "Board user context required" });
return;
}
const input = markIssueReadSchema.parse(req.body ?? {});
let readAt: Date | SQL = new Date();
if (input.throughCommentId) {
if (!issue.conversationAgentId || issue.conversationUserId !== req.actor.userId) throw forbidden("Only the chat owner can mark a conversation message read");
const [comment] = await db.select({ createdAt: sql<string>`${issueComments.createdAt}::text` }).from(issueComments).where(and(
eq(issueComments.id, input.throughCommentId), eq(issueComments.companyId, issue.companyId), eq(issueComments.issueId, issue.id),
isNull(issueComments.deletedAt),
));
if (!comment) throw notFound("Chat message not found");
readAt = sql`${comment.createdAt}::timestamptz`;
}
const readState = await svc.markRead(
issue.companyId,
issue.id,
req.actor.userId,
new Date(),
readAt,
);
const actor = getActorInfo(req);
await logActivity(db, {
@@ -18546,7 +18559,7 @@ export function issueRoutes(
if (req.actor.type !== "board" || !req.actor.userId) throw forbidden("Board user access required");
if (!(await instanceSettings.getExperimental()).enableAgentChat) throw notFound("Agent Chat is disabled");
const conversations = await svc.listConversations(companyId, req.actor.userId);
res.json(await filterIssuesForActor(req, conversations));
res.json(await withConversationReadState(db, companyId, req.actor.userId, await filterIssuesForActor(req, conversations)));
});
// GET stays read-only. POST resolves the single chat on explicit add or first send/upload.
@@ -18562,7 +18575,10 @@ export function issueRoutes(
const agent = resolved.agent;
const existing = await svc.getConversation(companyId, agent.id, req.actor.userId);
if (existing && !(await assertIssueReadAllowed(req, res, existing))) return;
if (existing || method === "get") { res.json(existing); return; }
if (existing || method === "get") {
res.json(existing ? (await withConversationReadState(db, companyId, req.actor.userId, [existing]))[0] : null);
return;
}
const issue = await svc.create(companyId, {
title: `Chat with ${agent.name}`, assigneeAgentId: agent.id,
conversationAgentId: agent.id, conversationUserId: req.actor.userId,
@@ -18571,7 +18587,7 @@ export function issueRoutes(
await logActivity(db, { companyId, actorType: "user", actorId: req.actor.userId,
action: "issue.conversation_opened", entityType: "issue", entityId: issue.id,
details: { agentId: agent.id } });
res.json(issue);
res.json((await withConversationReadState(db, companyId, req.actor.userId, [issue]))[0]);
});
}
+13 -1
View File
@@ -38,6 +38,8 @@ import {
slackSearchConfigSchema,
// Agent
submitAgentCommentarySchema,
sendChatMessageSchema,
markIssueReadSchema,
agentCommentaryQuerySchema,
AGENT_PALETTE_IDS,
AGENT_AVATAR_SIZES,
@@ -1670,6 +1672,7 @@ function resolveOperationAuthLevel(
|| /^\/api\/companies\/\{companyId\}\/agents\/\{agentId\}\/grok-binding(?:\/(?:routine|verify|retry|stop|recover))?$/.test(path)) return "board";
if (PUBLIC_OPERATIONS.has(key)) return "public";
if (key === "POST /api/companies/{companyId}/agent-commentary") return "agent_heartbeat";
if (key === "POST /api/agents/me/chat/messages") return "agent_heartbeat";
if (method.toUpperCase() === "GET" && path.startsWith("/api/companies/{companyId}/agent-commentary")) return "board";
if (key === "POST /api/mcp/project-tools" || key === "POST /api/companies/{companyId}/slack/tasks/{issueId}/tools") return "agent_run";
if (RUNTIME_TOOLS_OPERATIONS.has(key)) return "runtime_tools";
@@ -4736,7 +4739,7 @@ registry.registerPath({
path: "/api/issues/{id}/read",
tags: ["issues"],
summary: "Mark an issue as read",
request: { params: z.object({ id: z.string() }) },
request: { params: z.object({ id: z.string() }), body: { ...jsonBody(markIssueReadSchema), required: false } },
responses: { 200: r.ok(), 401: r.unauthorized },
});
@@ -5760,6 +5763,15 @@ registry.registerPath({
responses: { 200: r.ok(), 401: r.unauthorized, 403: r.forbidden, 404: r.notFound },
});
registry.registerPath({
method: "post", path: "/api/agents/me/chat/messages", tags: ["agents"],
summary: "Send a task message to the agent's own chat",
description: "Requires the assigned agent's active standard task run, standard credentials and an active responsible human. Resolves or creates that agent's conversation automatically. Company visibility, task permissions, Stop and the shared cross-issue run cap apply. Equivalent retries do not spend another slot. Does not wake the chat agent or change task status.",
request: { body: jsonBody(sendChatMessageSchema) },
responses: { 200: r.ok(), 201: { ...r.ok(), description: "Message saved" }, 400: r.badRequest,
401: r.unauthorized, 403: r.forbidden, 404: r.notFound, 409: r.conflict, 429: r.tooManyRequests },
});
const agentCommentaryAcknowledgementSchema = z.object({
id: z.string().uuid(),
kind: z.enum(["complaint", "suggestion"]),
@@ -229,7 +229,9 @@ describe("run failure diagnostics", () => {
const result = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics(run({ errorCode: "adapter_failed" }), { error: thrown }));
expect(result.execution).toEqual({ remoteRunnerRecoveryStage: "process_verification", remoteRunnerRecoveryReason: "process_not_running" });
expect(result.exceptions).toMatchObject([{ code: "runner_remote_process_identity_unavailable" }]);
expect(JSON.stringify(result)).not.toContain("private");
expect(JSON.stringify(result)).not.toContain("private-path");
expect(result.exceptions[0]).not.toHaveProperty("path");
expect(result.exceptions[0]).not.toHaveProperty("pid");
const transport = attachRemoteRunnerRecoveryDiagnostic(new Error("socket closed"),
{ stage: "identity_marker", reason: "probe_transport_failed" });
+4
View File
@@ -480,6 +480,10 @@ export function activityService(db: Db) {
.where(
and(
eq(heartbeatRuns.companyId, companyId),
sql`(not exists (select 1 from ${issues} where ${issues.id} = ${issueId}
and ${issues.companyId} = ${companyId} and ${issues.conversationAgentId} is not null)
or coalesce(${heartbeatRuns.nativeIssueId}::text, ${heartbeatRuns.issueId}::text,
${heartbeatRuns.contextSnapshot} ->> 'issueId', ${heartbeatRuns.contextSnapshot} ->> 'taskId') = ${issueId})`,
...(selectedRunIds ? [inArray(heartbeatRuns.id, selectedRunIds)] : []),
or(
eq(heartbeatRuns.issueId, issueId),
+193
View File
@@ -0,0 +1,193 @@
import { createHash } from "node:crypto";
import { and, eq, isNull, sql } from "drizzle-orm";
import { agentApiKeys, agents, companies, companyMemberships, heartbeatRuns, issueComments, issues, nativeRunResults, type Db } from "@paperclipai/db";
import { agentApiKeyScopeSchema, isUuidLike, sendChatMessageSchema, type AgentApiKeyScope } from "@paperclipai/shared";
import { badRequest, conflict, forbidden, notFound, tooManyRequests } from "../errors.js";
import { agentRunWritesRevoked, assertAgentRunWriteAllowed } from "../agent-run-cancellation.js";
import { authorizationService, canActorReadIssuePrivacy, type AuthorizationActor } from "./authorization.js";
import { persistActivity, publishActivity, type ActivityPublication } from "./activity-log.js";
import { instanceSettingsService } from "./instance-settings.js";
import { issueService } from "./issues.js";
import { issueTreeControlService } from "./issue-tree-control.js";
import { resolveActorSourceTrustForIssue } from "./source-trust.js";
import { resolveTaskWatchdogMutationScope } from "./task-watchdog-scope.js";
import { readNativeReviewAssignmentContext } from "./native-runtime/native-review-participant.js";
import { nativeRunKeyScope } from "./native-runtime/native-skill-test-scope.js";
import { crossIssueInfluenceLimitError, observeCrossIssueInfluence } from "./cross-issue-influence-limit.js";
type Actor = { companyId: string; agentId: string; runId: string; agentApiKeyId?: string | null;
keyScope?: AgentApiKeyScope | null; onBehalfOfUserId?: string | null;
authorizingUserId?: string | null; source?: AuthorizationActor["source"] };
export interface ChatMessageReceipt {
commentId: string;
conversationId: string;
chatUrl: string;
sourceTaskId: string;
replayed: boolean;
}
function runTask(run: typeof heartbeatRuns.$inferSelect) {
return run.nativeIssueId ?? run.issueId ?? run.contextSnapshot?.issueId ?? run.contextSnapshot?.taskId;
}
export async function sendAgentChatMessage(
db: Db,
actor: Actor,
value: unknown,
options: { authorizeNative?: (tx: Db) => Promise<unknown>; receiptOnly?: boolean } = {},
): Promise<ChatMessageReceipt | null> {
const parsed = sendChatMessageSchema.safeParse(value);
if (!parsed.success) throw badRequest("Chat messages require body and idempotencyKey only", parsed.error.issues);
if (![actor.companyId, actor.agentId, actor.runId].every(isUuidLike)) throw forbidden("An authenticated task run is required");
const credentialScope = agentApiKeyScopeSchema.safeParse(actor.keyScope ?? { kind: "standard" });
if (!credentialScope.success) throw forbidden("The agent credential scope is invalid");
if (credentialScope.data.kind !== "standard") throw forbidden("Scoped test and task bridge credentials cannot publish agent chat messages");
const input = parsed.data;
const contentHash = createHash("sha256").update(input.body).digest("hex");
const publications: ActivityPublication[] = [];
const result = await db.transaction(async transaction => {
const tx = transaction as unknown as Db;
await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${`issue-privacy-tree:${actor.companyId}`}, 0))`);
const runWhere = and(eq(heartbeatRuns.id, actor.runId), eq(heartbeatRuns.companyId, actor.companyId), eq(heartbeatRuns.agentId, actor.agentId));
const [initial] = await tx.select().from(heartbeatRuns).where(runWhere);
const sourceId = initial && runTask(initial);
if (typeof sourceId !== "string" || !isUuidLike(sourceId)) throw forbidden("Chat delivery requires a task-bound run");
const sourceQuery = () => tx.select().from(issues).where(and(eq(issues.id, sourceId), eq(issues.companyId, actor.companyId)));
const [initialSource] = await sourceQuery();
if (!initialSource) throw forbidden("This run no longer owns the source task");
let source = initialSource;
const contextQuery = () => tx.select({ run: heartbeatRuns, agent: agents }).from(heartbeatRuns)
.innerJoin(agents, and(eq(agents.id, heartbeatRuns.agentId), eq(agents.companyId, heartbeatRuns.companyId)))
.where(runWhere);
const validateContext = (context: { run: typeof heartbeatRuns.$inferSelect; agent: typeof agents.$inferSelect } | undefined) => {
if (!context || context.run.status !== "running" || agentRunWritesRevoked(context.run)
|| runTask(context.run) !== sourceId || source.assigneeAgentId !== actor.agentId
|| source.executionRunId !== actor.runId || (source.checkoutRunId && source.checkoutRunId !== actor.runId)
|| ["paused", "terminated", "pending_approval", "error"].includes(context.agent.status)) {
throw forbidden("This run no longer owns the source task");
}
if (source.conversationAgentId) throw forbidden("Reply normally in this conversation; send_chat_message is for execution tasks");
if (nativeRunKeyScope({ contextSnapshot: context.run.contextSnapshot, issueId: source.id, workMode: source.workMode }).kind === "skill_test") {
throw forbidden("Skill-test runs cannot publish agent chat messages");
}
if (source.workMode !== "standard" || ["done", "cancelled"].includes(source.status)
|| readNativeReviewAssignmentContext(context.run.contextSnapshot)) {
throw forbidden("Chat delivery requires an ordinary task in standard execution mode");
}
return context;
};
const [initialContext] = await contextQuery();
const context = validateContext(initialContext);
if (!(await instanceSettingsService(tx).getExperimental()).enableAgentChat) throw notFound("Agent Chat is disabled");
const userId = context.run.responsibleUserId;
if (!userId) throw forbidden("This task has no responsible human to receive the message");
if (actor.onBehalfOfUserId !== undefined && actor.onBehalfOfUserId !== userId) throw forbidden("The authenticated responsible human no longer matches this run");
const assertCurrentKey = async (lock: boolean) => {
if (!actor.agentApiKeyId) return;
const query = tx.select().from(agentApiKeys).where(and(eq(agentApiKeys.id, actor.agentApiKeyId),
eq(agentApiKeys.companyId, actor.companyId), eq(agentApiKeys.agentId, actor.agentId), isNull(agentApiKeys.revokedAt)));
const [key] = lock ? await query.for("share") : await query;
const currentScope = key && agentApiKeyScopeSchema.safeParse(key.scopeConfig ?? { kind: "standard" });
if (!key || !currentScope?.success || JSON.stringify(currentScope.data) !== JSON.stringify(credentialScope.data)
|| (key.responsibleUserId?.trim() || null) !== (actor.onBehalfOfUserId ?? userId)) {
throw forbidden("The agent key was revoked or its authority changed");
}
};
await assertCurrentKey(false);
const [membership] = await tx.select({ id: companyMemberships.id }).from(companyMemberships).where(and(
eq(companyMemberships.companyId, actor.companyId), eq(companyMemberships.principalType, "user"),
eq(companyMemberships.principalId, userId), eq(companyMemberships.status, "active"),
)).for("share");
if (!membership) throw forbidden("The responsible human is no longer an active company member");
const authActor: AuthorizationActor = { type: "agent", ...actor, keyId: actor.agentApiKeyId, keyScope: credentialScope.data,
source: actor.source ?? "agent_jwt", onBehalfOfUserId: actor.onBehalfOfUserId ?? userId };
const authorize = async (issueId: string, action: "issue:read" | "issue:comment") => {
const decision = await authorizationService(tx).decide({ actor: authActor, action,
resource: { type: "issue", companyId: actor.companyId, issueId } });
if (!decision.allowed) throw forbidden(decision.explanation);
};
await authorize(source.id, "issue:read");
if (!(await canActorReadIssuePrivacy(tx, { type: "none" }, source))) {
throw forbidden("Restricted task content cannot be sent to a company-visible chat");
}
const svc = issueService(tx);
let conversation = await svc.getConversation(actor.companyId, actor.agentId, userId);
if (!conversation && !options.receiptOnly) {
conversation = await svc.create(actor.companyId, {
title: `Chat with ${context.agent.name}`, assigneeAgentId: actor.agentId,
conversationAgentId: actor.agentId, conversationUserId: userId,
conversationState: "waiting", status: "in_review", createdByAgentId: actor.agentId,
responsibleUserId: userId, trustExplicitResponsibleUserId: true,
});
const opened = await persistActivity(tx, { companyId: actor.companyId, actorType: "agent", actorId: actor.agentId,
agentId: actor.agentId, runId: actor.runId, action: "issue.conversation_opened", entityType: "issue", entityId: conversation.id,
details: { agentId: actor.agentId, conversationUserId: userId } });
publications.push(opened.publication);
}
if (conversation) await tx.select({ id: issues.id }).from(issues).where(and(
eq(issues.id, conversation.id), eq(issues.companyId, actor.companyId),
)).for("update");
const [lockedSource] = await sourceQuery().for("update");
if (!lockedSource) throw forbidden("This run no longer owns the source task");
source = lockedSource;
if (options.authorizeNative) await options.authorizeNative(tx);
const [lockedContext] = await contextQuery().for("update");
const current = validateContext(lockedContext);
if (current.run.responsibleUserId !== userId) throw forbidden("The responsible human changed before this message was saved");
await assertCurrentKey(true);
if ((await resolveTaskWatchdogMutationScope(tx, { ...actor, type: "agent" })).kind !== "none") {
throw forbidden("Chat delivery requires an ordinary task in standard execution mode");
}
await assertAgentRunWriteAllowed(tx, actor.companyId, actor);
if (await issueTreeControlService(tx).getActivePauseHoldGate(actor.companyId, source.id)) throw forbidden("This task is paused");
const [finished] = await tx.select({ id: nativeRunResults.id }).from(nativeRunResults).where(and(
eq(nativeRunResults.companyId, actor.companyId), eq(nativeRunResults.runId, actor.runId), eq(nativeRunResults.schemaStatus, "accepted"),
)).limit(1);
if (finished) throw forbidden("Send the message before finishing the run");
if (!(await instanceSettingsService(tx).getExperimental()).enableAgentChat) throw notFound("Agent Chat is disabled");
await authorize(source.id, "issue:read");
if (!conversation) return null;
await authorize(conversation.id, "issue:comment");
const recipientDecision = await authorizationService(tx).decide({ actor: { type: "board", userId, source: "session", ignoreInstanceAdmin: true },
action: "issue:read", resource: { type: "issue", companyId: actor.companyId, issueId: conversation.id } });
if (!recipientDecision.allowed) throw forbidden("The responsible human cannot access this chat");
const requestId = `chat:${source.id}:${createHash("sha256").update(input.idempotencyKey).digest("hex")}`;
const [prior] = await tx.select().from(issueComments).where(and(eq(issueComments.companyId, actor.companyId),
eq(issueComments.issueId, conversation.id), eq(issueComments.authorAgentId, actor.agentId), eq(issueComments.clientRequestId, requestId)));
const [company] = await tx.select({ prefix: companies.issuePrefix }).from(companies).where(eq(companies.id, actor.companyId));
const receipt = (commentId: string, replayed: boolean): ChatMessageReceipt => ({ commentId, conversationId: conversation!.id,
chatUrl: `/${company.prefix}/chats/${actor.agentId}`, sourceTaskId: source.id, replayed });
if (prior) {
if (prior.metadata?.chatMessage?.contentHash !== contentHash) {
if (options.receiptOnly) return null;
throw conflict("Chat message retry key was already used with different content");
}
return receipt(prior.id, true);
}
if (options.receiptOnly) return null;
const influence = await observeCrossIssueInfluence(tx, { companyId: actor.companyId, agentId: actor.agentId,
runId: actor.runId, responsibleUserId: userId, targetIssueId: conversation.id,
targetIssueIdentifier: conversation.identifier, kind: "comment" });
if (influence && !influence.allowed) {
const denial = crossIssueInfluenceLimitError(influence, { actorLabel: current.agent.name,
assigneeLabel: current.agent.name, issueIdentifier: conversation.identifier });
throw tooManyRequests(denial.error, denial.details);
}
const sourceTrust = await resolveActorSourceTrustForIssue({ db: tx, issue: source,
actor: { actorType: "agent", actorId: actor.agentId, agentId: actor.agentId, runId: actor.runId } });
const comment = await svc.addComment(conversation.id, input.body, { agentId: actor.agentId, runId: actor.runId }, {
clientRequestId: requestId, sourceTrust, proactiveChatMessage: { sourceTaskId: source.id, userId },
metadata: { version: 1, chatMessage: { sourceTaskId: source.id, sourceTaskIdentifier: source.identifier!, sourceRunId: actor.runId, contentHash },
sections: [{ rows: [{ type: "issue_link", issueId: source.id, identifier: source.identifier, label: "Source task" }] }] },
}, tx);
const activity = await persistActivity(tx, { companyId: actor.companyId, actorType: "agent", actorId: actor.agentId,
agentId: actor.agentId, runId: actor.runId, agentApiKeyId: actor.agentApiKeyId,
action: "issue.comment_added", entityType: "issue", entityId: conversation.id,
details: { commentId: comment.id, identifier: conversation.identifier, conversationUserId: userId,
conversationAgentId: actor.agentId, sourceTaskId: source.id } });
publications.push(activity.publication);
return receipt(comment.id, false);
});
for (const publication of publications) publishActivity(publication);
return result;
}
@@ -0,0 +1,21 @@
import { and, eq, inArray, sql } from "drizzle-orm";
import { issueComments, issueReadStates, type Db } from "@paperclipai/db";
export async function withConversationReadState<T extends { id: string }>(db: Db, companyId: string, userId: string, conversations: T[]) {
if (!conversations.length) return [] as Array<T & { isUnreadForMe: boolean; lastExternalCommentAt: Date | null }>;
const rows = await db.select({
issueId: issueComments.issueId,
latest: sql<Date>`max(${issueComments.createdAt})`,
unread: sql<boolean>`bool_or(${issueComments.createdAt} > coalesce(${issueReadStates.lastReadAt}, to_timestamp(0)))`,
}).from(issueComments).leftJoin(issueReadStates, and(
eq(issueReadStates.companyId, companyId), eq(issueReadStates.issueId, issueComments.issueId), eq(issueReadStates.userId, userId),
)).where(and(eq(issueComments.companyId, companyId), inArray(issueComments.issueId, conversations.map(chat => chat.id)),
sql`${issueComments.deletedAt} is null`, sql`${issueComments.authorAgentId} is not null`,
sql`${issueComments.authorType} is distinct from 'system'`,
sql`${issueComments.presentation} ->> 'kind' is distinct from 'system_notice'`,
eq(issueComments.origin, "comment"),
)).groupBy(issueComments.issueId);
const byId = new Map(rows.map(row => [row.issueId, row]));
return conversations.map(chat => ({ ...chat, isUnreadForMe: byId.get(chat.id)?.unread ?? false,
lastExternalCommentAt: byId.get(chat.id)?.latest ?? null }));
}
@@ -530,6 +530,9 @@ export async function undeliveredConversationComments(
eq(issueComments.companyId, companyId),
eq(issueComments.issueId, issueId),
isNull(issueComments.deletedAt),
sql`${issueComments.authorUserId} is not null`,
isNull(issueComments.authorAgentId),
eq(issueComments.origin, "comment"),
sql`${issueComments.clientRequestId} is not null`,
sql`not exists (select 1 from ${agentWakeupRequests} where ${agentWakeupRequests.companyId} = ${companyId}
and ${agentWakeupRequests.idempotencyKey} = 'conversation-comment:' || ${issueComments.id}::text)`,
@@ -1,6 +1,7 @@
import { notifyInstructionCollection } from "./instruction-collection-notifications.js";
import fs from "node:fs/promises";
import path from "node:path";
import { isDeepStrictEqual } from "node:util";
import { and, eq, inArray, isNull, sql } from "drizzle-orm";
import { agents, environmentLeases, environments, agentInstructionWorkingCopies as copies, type Db } from "@paperclipai/db";
import { syncDirectoryToSsh, restoreWorkspaceFromSshExecution } from "@paperclipai/adapter-utils/ssh";
@@ -20,6 +21,11 @@ import { cachedAgentFileManifest, captureAgentFileCheckpoint, checkpointBaseline
import { logger } from "../middleware/logger.js";
type Copy = typeof copies.$inferSelect;
function adoptionAuthority(row: Copy) {
const { updatedAt: _updatedAt, nextAttemptAt: _nextAttemptAt, ...authority } = row;
return authority;
}
export class AgentDirectoryReuseInvalidatedError extends Error {}
const completed = new Set(["saved", "unchanged", "resolved", "unavailable"]);
const transports = new Map<string, PreparedAdapterExecutionTargetRuntime>();
@@ -291,10 +297,22 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
// adopt_remote constructs a collector on the successor target without
// uploading the stale host mirror over the retained remote directory.
const replacement = remote ? await transport({ ...prior, runId: input.runId }, remote, true) : undefined;
const expectedPrior = prior;
let adopted: Copy | null;
try { adopted = await store.locked(input.companyId, input.agentId, bound, false, async (tx, _agent, canonical) => {
if (remote && !validLeases(await tx.select().from(environmentLeases).where(and(
eq(environmentLeases.companyId, input.companyId), inArray(environmentLeases.id, leaseIds))).for("update"))) return null;
const [lockedPrior] = await tx.select().from(copies).where(and(eq(copies.companyId, input.companyId),
eq(copies.runId, expectedPrior.runId), eq(copies.agentId, input.agentId))).for("update");
if (!lockedPrior || !isDeepStrictEqual(adoptionAuthority(lockedPrior), adoptionAuthority(expectedPrior))) {
const fields = ["state", "receipt", "baseHash", "localRoot", "executionRoot", "location", "processStoppedAt", "responsibleUserId", "entryFile", "attempts", "candidateHash", "errorCode", "errorMessage", "baseRevisionId", "candidateBase64", "createdAt"] as const;
const diagnostic = { stage: "locked_adoption_authority", predecessorPresent: Boolean(lockedPrior), observedAfterInitialRead: true,
changedFields: lockedPrior ? fields.filter(field => !isDeepStrictEqual(lockedPrior[field], expectedPrior[field])) : [] };
logger.warn({ companyId: input.companyId, agentId: input.agentId, runId: input.runId, previousRunId: input.previousRunId,
adoptionConflict: diagnostic }, "Agent directory adoption conflict");
throw conflict("Agent directory ownership changed", diagnostic);
}
prior = lockedPrior;
const current = await inspectAgentDirectory(canonical);
const canonicalChanged = agentDirectoryBaselineDigest(current.snapshot) !== agentDirectoryBaselineDigest(baseline(prior));
if (canonicalChanged && !input.allowRetirementHandoff) return null;
@@ -304,7 +322,18 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
.where(and(eq(copies.companyId, input.companyId), eq(copies.runId, prior.runId), eq(copies.agentId, input.agentId), eq(copies.state, "unchanged_turn"),
eq(copies.updatedAt, prior.updatedAt), eq(copies.baseHash, prior.baseHash), eq(copies.localRoot, prior.localRoot),
isNull(copies.processStoppedAt), sql`${copies.receipt}->>'retainedByRunId' IS NULL`)).returning();
if (!claimed) throw conflict("Agent directory ownership changed");
if (!claimed) {
const [conditions] = await tx.select({ unchangedTurn: eq(copies.state, "unchanged_turn"),
timestampMatches: eq(copies.updatedAt, prior.updatedAt), baselineMatches: eq(copies.baseHash, prior.baseHash),
localRootMatches: eq(copies.localRoot, prior.localRoot), unstopped: isNull(copies.processStoppedAt),
successorAbsent: sql<boolean>`${copies.receipt}->>'retainedByRunId' IS NULL`,
receiptMatches: sql<boolean>`${copies.receipt} IS NOT DISTINCT FROM ${JSON.stringify(prior.receipt)}::jsonb`,
}).from(copies).where(and(eq(copies.companyId, input.companyId), eq(copies.runId, prior.runId), eq(copies.agentId, input.agentId)));
const diagnostic = { stage: "adoption_claim", observedAfterClaim: true, predecessorPresent: Boolean(conditions), conditions: conditions ?? null };
logger.warn({ companyId: input.companyId, agentId: input.agentId, runId: input.runId, previousRunId: input.previousRunId,
adoptionConflict: diagnostic }, "Agent directory adoption conflict");
throw conflict("Agent directory ownership changed", diagnostic);
}
const { createdAt: _createdAt, updatedAt: _updatedAt, ...values } = prior;
const [next] = await tx.insert(copies).values({ ...values, runId: input.runId, responsibleUserId: bound.onBehalfOfUserId!, state: "prepared",
receipt: { ...prior.receipt, materializationRunId: prior.receipt?.materializationRunId ?? prior.runId,
+16 -8
View File
@@ -1080,8 +1080,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
Math.max(Math.trunc(serviceOptions.openDecisionLimit ?? OPEN_DECISION_DEFAULT_LIMIT), 1),
OPEN_DECISION_MAX_LIMIT,
);
const service = {
list: async (companyId: string, options: AttentionListOptions = {}): Promise<AttentionFeed> => {
async function list(companyId: string, options: AttentionListOptions = {}, questionIssueIds?: readonly string[]): Promise<AttentionFeed> {
if (options.all && !options.queue && !options.allowUnscopedAll) {
throw badRequest("all requires a queue filter");
}
@@ -1105,7 +1104,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
collected.push({ ...item, dismissal });
};
if (!options.excludeSources?.includes("approval")) {
if (!questionIssueIds && !options.excludeSources?.includes("approval")) {
const pendingApprovals = await db
.select({
id: approvals.id,
@@ -1196,6 +1195,8 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
.where(and(
eq(issueThreadInteractions.companyId, companyId),
inArray(issueThreadInteractions.status, [...PENDING_INTERACTION_STATUSES]),
questionIssueIds ? inArray(issueThreadInteractions.issueId, [...questionIssueIds]) : undefined,
questionIssueIds ? eq(issueThreadInteractions.kind, "ask_user_questions") : undefined,
))
.orderBy(desc(issueThreadInteractions.updatedAt), desc(issueThreadInteractions.id));
// Addressee invokability needs the org graph; the audience line also needs
@@ -1285,6 +1286,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
}));
}
if (!questionIssueIds) {
const openDecisionQuery = db.select({
id: decisions.id,
bundleId: decisions.bundleId,
@@ -1865,6 +1867,8 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
}));
}
}
const deduped = new Map<string, AttentionItem>();
for (const item of collected) {
const current = deduped.get(item.dedupKey);
@@ -1972,18 +1976,22 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
}
}
return { ...summary, nextCursor, items };
},
};
}
return {
...service,
list: (companyId: string, options: AttentionListOptions = {}) => list(companyId, options),
taskQuestions: async (companyId: string, issueIds: readonly string[], options: { actor: AuthorizationActor; userId: string }): Promise<AttentionItem[]> => {
if (issueIds.length === 0) return [];
const feed = await list(companyId, { ...options, all: true, allowUnscopedAll: true }, issueIds);
return feed.items;
},
summary: async (companyId: string, options: Pick<AttentionListOptions, "userId" | "actor"> = {}): Promise<AttentionSummary> => {
const { items: _items, nextCursor: _cursor, ...summary } = await service.list(companyId, {
const { items: _items, nextCursor: _cursor, ...summary } = await list(companyId, {
...options, all: true, allowUnscopedAll: true, summaryOnly: true,
});
return summary;
},
taskIndex: async (companyId: string, options: Pick<AttentionListOptions, "userId" | "actor"> = {}): Promise<AttentionTaskIndex> => {
const { items, nextCursor: _cursor, ...summary } = await service.list(companyId, {
const { items, nextCursor: _cursor, ...summary } = await list(companyId, {
...options, all: true, allowUnscopedAll: true, summaryOnly: true,
});
return { ...summary, taskIds: [...new Set(items.flatMap(item => item.relatedIssue
+7 -2
View File
@@ -6150,9 +6150,14 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo
if (!versionId || (selectedKeys && !selectedKeys.has(key))) continue;
const [version] = await db.select({ skillId: companySkillVersions.companySkillId }).from(companySkillVersions)
.where(and(eq(companySkillVersions.companyId, companyId), eq(companySkillVersions.id, versionId)));
skills = skills.filter(skill => skill.key !== key);
// A saved JSONB selection map has a different key order from the fresh
// name/key query. Replace existing rows in place so pinning versions does
// not reorder the server-produced preference (and its session fingerprint).
const index = skills.findIndex(skill => skill.key === key);
if (index !== -1) skills.splice(index, 1);
const owner = version ? await getById(companyId, version.skillId) : null;
if (owner) skills.push({ ...owner, key, slug: owner.key === key ? owner.slug : key.split("/").at(-1)! });
if (owner) skills.splice(index === -1 ? skills.length : index, 0,
{ ...owner, key, slug: owner.key === key ? owner.slug : key.split("/").at(-1)! });
}
await packages.prefetch(companyId, skills.flatMap(skill => {
const id = options.versionSelections?.get(skill.key) ?? skill.currentVersionId;
@@ -56,17 +56,7 @@ export function conversationDecisionReceiptsService(db: Db) {
const taskIds = new Set([...handoffIds, ...directIds].filter(id => id !== conversationId));
if (taskIds.size === 0) return [];
// One on-demand canonical company snapshot, not one query per task. The
// attention owner applies current audience, privacy, dismissal, snooze and
// archive policy, including its existing queue/retention bookkeeping.
// Cost grows with the company feed, not only the returned task questions;
// there is currently no task-scoped attention API. Do not poll this read.
const feed = await attentionService(db).list(companyId, {
all: true, allowUnscopedAll: true, actor: options.actor, userId: options.userId,
});
return feed.items.filter(item => item.sourceKind === "issue_thread_interaction"
&& item.detail?.kind === "questions"
&& item.relatedIssue !== null && taskIds.has(item.relatedIssue.id));
return attentionService(db).taskQuestions(companyId, [...taskIds], options);
},
async listCreatedTasks(companyId: string, conversationId: string, options: {
actor: AuthorizationActor; userId: string; cursor?: string; limit?: number;
+22 -20
View File
@@ -456,34 +456,36 @@ export function decisionQueueService(db: Db) {
});
}
let insertedAnyItem = false;
for (const item of matchingItems) {
const inserted = await txDb.insert(decisionQueueItems).values({
for (let offset = 0; offset < matchingItems.length; offset += 500) {
const inserted = await txDb.insert(decisionQueueItems).values(matchingItems.slice(offset, offset + 500).map(item => ({
companyId,
queueId: queue.id,
sourceKind: item.sourceKind,
sourceId: item.sourceId,
addedByType: "system",
}).onConflictDoNothing({
}))).onConflictDoNothing({
target: [decisionQueueItems.queueId, decisionQueueItems.sourceKind, decisionQueueItems.sourceId],
}).returning();
if (!inserted[0]) continue;
if (inserted.length === 0) continue;
insertedAnyItem = true;
await txDb.insert(decisionTriageEvents).values({
companyId,
queueId: queue.id,
sourceKind: item.sourceKind,
sourceId: item.sourceId,
action: "queue_item.seeded",
...eventActorColumns(SYSTEM_ACTOR),
details: { seedKey: seed.key },
});
await recordActivity(txDb, SYSTEM_ACTOR, {
companyId,
action: "decision_queue_item.seeded",
entityType: "decision_queue",
entityId: queue.id,
details: { sourceKind: item.sourceKind, sourceId: item.sourceId, seedKey: seed.key },
});
for (const item of inserted) {
await txDb.insert(decisionTriageEvents).values({
companyId,
queueId: queue.id,
sourceKind: item.sourceKind,
sourceId: item.sourceId,
action: "queue_item.seeded",
...eventActorColumns(SYSTEM_ACTOR),
details: { seedKey: seed.key },
});
await recordActivity(txDb, SYSTEM_ACTOR, {
companyId,
action: "decision_queue_item.seeded",
entityType: "decision_queue",
entityId: queue.id,
details: { sourceKind: item.sourceKind, sourceId: item.sourceId, seedKey: seed.key },
});
}
}
if (insertedAnyItem) {
await txDb.update(decisionQueues).set({ updatedAt: new Date() })
+25 -2
View File
@@ -2,6 +2,7 @@ import { registerDeferredReceiptWork } from "./heartbeat/deferred-receipt-work.j
import { reconcileTaskChatRetryReceipts } from "./task-retry.js";
import { registerIssueRecoveryWork } from "./recovery/work.js";
import { registerRunDispatchWork } from "./heartbeat/dispatch-work.js";
import { NATIVE_RETAINED_OWNER_WAIT } from "./native-runtime/native-controller-recovery.js";
import { externalConversationSettlementWork } from "./external-conversation-settlement-work.js";
import { nativeRecoveryNoticeWork } from "./native-runtime/native-recovery-notice-work.js";
import { nativeRunProjectionWork } from "./native-runtime/native-run-projection-work.js";
@@ -610,6 +611,9 @@ const nativeSessionResumeDispatchTimers = new Map<
string,
ReturnType<typeof setTimeout>
>();
// Routes and the scheduler can drain/resume through different service facades.
// Rearm through the original owner so its database and dispatch mode survive.
const nativeRecoveryDeadlines = new Map<string, { rearm: () => void }>();
// Shared with HTTP admission so an idle hold fences work before inspection.
export { computeTaskDrain, applyTaskDrain, startTaskDrain, stopTaskDrain } from "./task-admission.js";
@@ -1339,6 +1343,9 @@ export function heartbeatService(
}
function resumeQueuedRunsAfterDrain(...args: Parameters<typeof heartbeatQueue.resumeQueuedRunsAfterDrain>) {
for (const pending of nativeRecoveryDeadlines.values()) {
pending.rearm();
}
const promise = heartbeatQueue.resumeQueuedRunsAfterDrain(...args);
activeWakeupPromises.add(promise);
void promise.then(
@@ -1708,10 +1715,15 @@ export function heartbeatService(
}
}
// Native ownership waits survive a temporary drain without adding a polling
// loop. A process restart rediscovers the same deadlines from coordinator rows.
function scheduleNativeSessionResumeDispatch(
runId: string,
nextAttemptAt: Date,
mode?: "restart",
) {
const pending = { rearm: () => scheduleNativeSessionResumeDispatch(runId, nextAttemptAt, mode) };
nativeRecoveryDeadlines.set(runId, pending);
const prior = nativeSessionResumeDispatchTimers.get(runId);
if (prior) clearTimeout(prior);
const delayMs = Math.max(0, nextAttemptAt.getTime() - Date.now());
@@ -1720,6 +1732,12 @@ export function heartbeatService(
nativeSessionResumeDispatchTimers.delete(runId);
void (async () => {
if ((await getSchedulingSuppression()).suppressed) return;
if (nativeRecoveryDeadlines.get(runId) !== pending) return;
nativeRecoveryDeadlines.delete(runId);
if (mode === "restart") {
await recoverNativeRunsAfterRestart(new Date(), [runId]);
return;
}
await dispatchNativeSessionResumptions({
db,
runnerInstanceId:
@@ -1741,6 +1759,9 @@ export function heartbeatService(
},
});
})().catch((error) => {
if (!nativeRecoveryDeadlines.has(runId)) {
scheduleNativeSessionResumeDispatch(runId, new Date(Date.now() + 30_000), mode);
}
logger.error(
{ err: error, runId },
"failed to dispatch scheduled native session resume",
@@ -3531,7 +3552,8 @@ export function heartbeatService(
err.original instanceof Error
? err.original.message
: String(err.original ?? "");
const retryReasonCode = /native_finalization_missing/i.test(
const waitingForRetainedOwner = retryMessage === NATIVE_RETAINED_OWNER_WAIT;
const retryReasonCode = waitingForRetainedOwner ? NATIVE_RETAINED_OWNER_WAIT : /native_finalization_missing/i.test(
retryMessage,
)
? "semantic_result_missing"
@@ -3551,7 +3573,8 @@ export function heartbeatService(
stream: "system",
level: "warn",
message:
retryReasonCode === "semantic_result_missing"
waitingForRetainedOwner ? "native continuation waiting for the retained controller lease"
: retryReasonCode === "semantic_result_missing"
? "provider turn completed without a semantic result; same-run disposition recovery persisted"
: "native session transport interrupted; same-run resume persisted",
payload: {
@@ -49,7 +49,11 @@ export function bindHeartbeatRunnerHomeMode(config: Record<string, unknown>, per
const saved = parseObject(persistedExecution);
if (Object.keys(saved).length === 0) return config;
// Execution schemas before v7 always used isolated homes.
return { ...config, homeMode: saved.homeMode ?? "isolated" };
const homeMode = saved.homeMode ?? "isolated";
// Omission already means isolated. Recovery must not add a fingerprinted
// configuration property merely to restate that unchanged default.
if (homeMode === "isolated" && config.homeMode === undefined) return config;
return { ...config, homeMode };
}
export interface HeartbeatExecutionConfigurationInput {
@@ -0,0 +1,102 @@
import type { Db } from "@paperclipai/db";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { heartbeatService, startTaskDrain, stopTaskDrain } from "../heartbeat.js";
import type { HeartbeatRecoveryDependencies } from "./recovery.js";
const hooks = vi.hoisted(() => ({
schedule: undefined as HeartbeatRecoveryDependencies["scheduleNativeSessionResumeDispatch"] | undefined,
restart: vi.fn(async () => []),
resume: vi.fn(async () => []),
queue: vi.fn(async () => undefined),
}));
vi.mock("./recovery.js", async original => {
const module = await original<typeof import("./recovery.js")>();
return { ...module, createHeartbeatRecovery: (db: Db, dependencies: HeartbeatRecoveryDependencies) => {
hooks.schedule = dependencies.scheduleNativeSessionResumeDispatch;
return { ...module.createHeartbeatRecovery(db, dependencies), recoverNativeRunsAfterRestart: hooks.restart };
} };
});
vi.mock("./queue.js", async original => {
const module = await original<typeof import("./queue.js")>();
return { ...module, createHeartbeatQueue: (...args: Parameters<typeof module.createHeartbeatQueue>) => ({
...module.createHeartbeatQueue(...args), resumeQueuedRunsAfterDrain: hooks.queue,
}) };
});
vi.mock("../native-runtime/index.js", async original => ({
...await original<typeof import("../native-runtime/index.js")>(),
dispatchNativeSessionResumptions: hooks.resume,
}));
describe("native deadline drain and resume", () => {
let heartbeat: ReturnType<typeof heartbeatService>;
beforeEach(() => {
vi.clearAllMocks();
vi.useFakeTimers();
stopTaskDrain();
const db = new Proxy({}, { get: () => { throw new Error("Unexpected deadline database access"); } }) as Db;
heartbeat = heartbeatService(db, { runtimeEnv: {} });
});
afterEach(async () => {
stopTaskDrain();
await heartbeat?.resumeQueuedRunsAfterDrain();
await vi.runOnlyPendingTimersAsync();
await heartbeat?.drainActiveRunExecutions();
vi.useRealTimers();
});
it.each([undefined, "restart"] as const)("retains an overdue %s wait through drain with its original dispatch mode", async mode => {
const deadline = new Date(Date.now() + 100);
startTaskDrain({});
hooks.schedule!("waiting-run", deadline, mode);
await vi.advanceTimersByTimeAsync(100);
expect(hooks.restart).not.toHaveBeenCalled();
expect(hooks.resume).not.toHaveBeenCalled();
stopTaskDrain();
await heartbeat.resumeQueuedRunsAfterDrain();
await vi.advanceTimersByTimeAsync(0);
const called = mode === "restart" ? hooks.restart : hooks.resume;
const other = mode === "restart" ? hooks.resume : hooks.restart;
expect(called).toHaveBeenCalledOnce();
expect(other).not.toHaveBeenCalled();
await heartbeat.resumeQueuedRunsAfterDrain();
await vi.advanceTimersByTimeAsync(60_000);
expect(called).toHaveBeenCalledOnce();
expect(vi.getTimerCount()).toBe(0);
});
it("preserves a future follow-up deadline when draining cancels its timer", async () => {
const deadline = new Date(Date.now() + 100);
hooks.schedule!("future-run", deadline);
await heartbeat.drainActiveRunExecutions();
await heartbeat.resumeQueuedRunsAfterDrain();
await vi.advanceTimersByTimeAsync(99);
expect(hooks.resume).not.toHaveBeenCalled();
await vi.advanceTimersByTimeAsync(1);
expect(hooks.resume).toHaveBeenCalledOnce();
expect(hooks.restart).not.toHaveBeenCalled();
});
it("rearms the original owner when another service facade ends the drain", async () => {
hooks.schedule!("other-facade-run", new Date(Date.now() + 100));
const db = new Proxy({}, { get: () => { throw new Error("Unexpected replacement facade database access"); } }) as Db;
const other = heartbeatService(db, { runtimeEnv: {} });
await other.drainActiveRunExecutions();
await vi.advanceTimersByTimeAsync(100);
await other.resumeQueuedRunsAfterDrain();
await vi.advanceTimersByTimeAsync(0);
expect(hooks.resume).toHaveBeenCalledOnce();
expect(hooks.restart).not.toHaveBeenCalled();
});
it("retries a failed follow-up obligation without changing its dispatch mode", async () => {
hooks.resume.mockRejectedValueOnce(new Error("temporary database failure"));
hooks.schedule!("failed-follow-up", new Date());
await vi.advanceTimersByTimeAsync(0);
expect(hooks.resume).toHaveBeenCalledOnce();
await vi.advanceTimersByTimeAsync(30_000);
expect(hooks.resume).toHaveBeenCalledTimes(2);
expect(hooks.restart).not.toHaveBeenCalled();
expect(vi.getTimerCount()).toBe(0);
});
});
+317 -5
View File
@@ -1,6 +1,8 @@
import { randomUUID } from "node:crypto";
import { spawn } from "node:child_process";
import { once } from "node:events";
import type { Db } from "@paperclipai/db";
import { activityLog, agents, applyPendingMigrations, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, environments, environmentLeases, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, nativeRunnerCheckpoints } from "@paperclipai/db";
import { activityLog, agents, applyPendingMigrations, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, environments, environmentLeases, heartbeatRunEvents, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, nativeRunnerCheckpoints } from "@paperclipai/db";
import { eq } from "drizzle-orm";
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
@@ -19,7 +21,10 @@ import { logger } from "../../middleware/logger.js";
import { listExpiredNativeRunnerAuthorities } from "../native-runtime/native-runner-checkpoint.js";
import { DELIVERY_QUEUES, subscribeDeliveryWork } from "../delivery-work-notifications.js";
import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js";
import { hasRemoteTerminationReceipt } from "../remote-execution-termination.js";
import { hasRemoteTerminationReceipt, remoteTerminationReceipt } from "../remote-execution-termination.js";
import { persistHeartbeatRunProcessMetadata } from "./run-lifecycle.js";
import { appendHeartbeatRunEvent } from "../heartbeat-run-events.js";
import { hasNativeLocalProcessStop, PROCESS_START_REQUESTED, releaseSettledNativeRunProcess } from "../native-local-process-stop.js";
type Run = typeof heartbeatRuns.$inferSelect;
@@ -202,6 +207,302 @@ describePostgres("heartbeat recovery module ownership", () => {
return (await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, issueId, invocationSource: "automation", status: "running", runtimeMode: "legacy" }).returning())[0];
}
async function ownedProcess() {
const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)", `paperclip-recovery-${randomUUID()}`], {
stdio: "ignore", detached: true, env: { PATH: process.env.PATH },
});
await once(child, "spawn");
return {
pid: child.pid!,
async stop() {
if (child.exitCode !== null || child.signalCode !== null) return;
const exited = once(child, "exit");
child.kill("SIGTERM");
await exited;
},
alive() { return process.kill(child.pid!, 0); },
};
}
async function nativeProcessFixture(pid: number, targetKind: "local" | "remote", status: "running" | "failed") {
const run = await seedRun();
await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: run.issueId, status,
nativePhase: status === "failed" ? "retryable_failure" : "running",
errorCode: status === "failed" ? "native_session_interrupted" : null,
error: status === "failed" ? "native session timed out after 120000ms" : null,
finishedAt: status === "failed" ? new Date() : null,
}).where(eq(heartbeatRuns.id, run.id));
await db.insert(nativeRunFinalizations).values({ runId: run.id, companyId: run.companyId, issueId: run.issueId!,
phase: status === "failed" ? "retryable_failure" : "running", nextAttemptAt: new Date(Date.now() - 1) });
await persistHeartbeatRunProcessMetadata(db, run.id, { pid, processGroupId: pid,
startedAt: new Date().toISOString(), targetKind });
const deps = callbacks(db);
deps.setRunStatusFromLive.mockImplementation(async (id, nextStatus, fromStatuses, patch) => {
const [current] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id));
if (!current || !fromStatuses.includes(current.status)) return { updated: false, run: current ?? null };
const [updated] = await db.update(heartbeatRuns).set({ ...patch, status: nextStatus })
.where(eq(heartbeatRuns.id, id)).returning();
return { updated: true, run: updated };
});
return {
run, deps,
current: async () => (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0],
async cleanup() {
await Promise.all([...deps.activeRunExecutionPromises]);
await db.delete(environmentLeases).where(eq(environmentLeases.heartbeatRunId, run.id));
await db.delete(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id));
await db.delete(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, run.id));
await db.delete(heartbeatRuns).where(eq(heartbeatRuns.id, run.id));
},
};
}
it.each(["failed", "running"] as const)("does not attribute a remote %s run's PID to a live host process", async status => {
const child = await ownedProcess();
const f = await nativeProcessFixture(child.pid, "remote", status);
try {
await db.update(agents).set({ adapterType: "process" }).where(eq(agents.id, f.run.agentId));
const before = await f.current();
expect(await createHeartbeatRecovery(db, f.deps).reapOrphanedRuns()).toEqual({ reaped: 0, runIds: [] });
expect(await f.current()).toMatchObject({ status, errorCode: before.errorCode, error: before.error,
processPid: child.pid, processGroupId: child.pid, processStartedAt: before.processStartedAt });
expect(f.deps.setRunStatusFromLive).not.toHaveBeenCalled();
expect(f.deps.executeRun).not.toHaveBeenCalled();
expect(f.deps.terminateHeartbeatRunProcess).not.toHaveBeenCalled();
expect(await hasNativeLocalProcessStop(db, f.run.companyId, f.run.id)).toBe(false);
expect(child.alive()).toBe(true);
} finally { try { await f.cleanup(); } finally { await child.stop(); } }
});
async function remoteRetryFixture(pid: number, target: "workspace" | "plugin_task" = "workspace") {
const fixture = await nativeProcessFixture(pid, "remote", "failed");
const sessionId = randomUUID();
const [lease] = await db.insert(environmentLeases).values({ companyId: fixture.run.companyId,
heartbeatRunId: fixture.run.id, issueId: fixture.run.issueId, status: "active", provider: "daytona",
providerLeaseId: randomUUID(), metadata: target === "plugin_task" ? { runnerTaskLease: true } : {},
}).returning();
await db.update(agents).set({ adapterType: "paperclip_runner" }).where(eq(agents.id, fixture.run.agentId));
await db.update(issues).set({ executionRunId: fixture.run.id }).where(eq(issues.id, fixture.run.issueId!));
await db.update(heartbeatRuns).set({ nativeSessionId: sessionId, runnerProfileJson: {
sessionCheckpoint: { identity: { runId: fixture.run.id, companyId: fixture.run.companyId,
issueId: fixture.run.issueId, agentId: fixture.run.agentId, sessionId }, providerSessionId: "existing-provider" },
...(target === "workspace" ? { nativeWorkspaceSync: { schema: "paperclip.native-workspace-sync/v3", state: "prepared",
workspaceId: randomUUID(), leaseId: lease.id, providerLeaseId: lease.providerLeaseId, remoteCwd: "/work",
descriptorSha256: "a".repeat(64), baselineSha256: "b".repeat(64), finalHostSha256: null,
resourceDisposition: "keep_running" } } : {}),
} }).where(eq(heartbeatRuns.id, fixture.run.id));
await db.insert(nativeRunnerCheckpoints).values({ runId: fixture.run.id, companyId: fixture.run.companyId,
issueId: fixture.run.issueId!, part: "opaque-checkpoint-record", material: { fixture: true } });
fixture.deps.getRun.mockImplementation(async id => (await db.select().from(heartbeatRuns)
.where(eq(heartbeatRuns.id, id)))[0] ?? null);
native.claims.mockImplementation(claimNativeRestartRecoveries);
return { ...fixture, lease };
}
it.each(["failed", "running"] as const)("routes a due remote %s retry through verified recovery without local PID inspection", async status => {
const child = await ownedProcess();
const f = await remoteRetryFixture(child.pid);
const originalKill = process.kill.bind(process);
const kill = vi.spyOn(process, "kill").mockImplementation(originalKill);
try {
await db.update(heartbeatRuns).set({ status }).where(eq(heartbeatRuns.id, f.run.id));
await db.update(nativeRunFinalizations).set({ nextAttemptAt: null }).where(eq(nativeRunFinalizations.runId, f.run.id));
const recovery = createHeartbeatRecovery(db, f.deps);
await recovery.reapOrphanedRuns();
await Promise.all([...f.deps.activeRunExecutionPromises]);
expect(f.deps.executeRun).toHaveBeenCalledExactlyOnceWith(f.run.id, expect.objectContaining({
nativeRestartRecovery: expect.objectContaining({ kind: "reattach_remote_runner", runId: f.run.id,
remote: { providerLeaseId: f.lease.providerLeaseId, remoteCwd: "/work" } }),
}));
expect(native.claims).toHaveBeenCalledWith(expect.objectContaining({ runIds: [f.run.id] }));
expect(await f.current()).toMatchObject({ status: "running", processPid: child.pid, processGroupId: child.pid });
expect(kill.mock.calls.filter(([pid]) => Math.abs(pid as number) === child.pid)).toEqual([]);
expect(await hasNativeLocalProcessStop(db, f.run.companyId, f.run.id)).toBe(false);
await recovery.reapOrphanedRuns();
expect(f.deps.executeRun).toHaveBeenCalledOnce();
expect(f.deps.terminateHeartbeatRunProcess).not.toHaveBeenCalled();
kill.mockRestore();
expect(child.alive()).toBe(true);
} finally { kill.mockRestore(); try { await f.cleanup(); } finally { await child.stop(); } }
});
it("routes a due plugin-task retry through its existing remote lease", async () => {
const child = await ownedProcess();
const f = await remoteRetryFixture(child.pid, "plugin_task");
try {
await createHeartbeatRecovery(db, f.deps).reapOrphanedRuns();
await Promise.all([...f.deps.activeRunExecutionPromises]);
expect(f.deps.executeRun).toHaveBeenCalledExactlyOnceWith(f.run.id, expect.objectContaining({
nativeRestartRecovery: expect.objectContaining({ kind: "reattach_remote_runner", runId: f.run.id,
remote: { kind: "plugin_task", leaseId: f.lease.id } }),
}));
expect(await f.current()).toMatchObject({ processPid: child.pid, processGroupId: child.pid });
expect(await hasNativeLocalProcessStop(db, f.run.companyId, f.run.id)).toBe(false);
expect(child.alive()).toBe(true);
} finally { try { await f.cleanup(); } finally { await child.stop(); } }
});
it.each(["future_retry", "live_owner", "missing_checkpoint", "active_execution"] as const)(
"does not execute a remote retry without current authority: %s", async condition => {
const child = await ownedProcess();
const f = await remoteRetryFixture(child.pid);
try {
if (condition === "future_retry") await db.update(nativeRunFinalizations)
.set({ nextAttemptAt: new Date(Date.now() + 60_000) }).where(eq(nativeRunFinalizations.runId, f.run.id));
if (condition === "live_owner") await db.update(nativeRunFinalizations)
.set({ leaseOwner: "other-controller", leaseExpiresAt: new Date(Date.now() + 60_000) })
.where(eq(nativeRunFinalizations.runId, f.run.id));
if (condition === "missing_checkpoint") {
const run = await f.current();
await db.update(heartbeatRuns).set({ runnerProfileJson: { ...run.runnerProfileJson, sessionCheckpoint: null } })
.where(eq(heartbeatRuns.id, f.run.id));
}
if (condition === "active_execution") f.deps.activeRunExecutions.add(f.run.id);
await createHeartbeatRecovery(db, f.deps).reapOrphanedRuns();
expect(f.deps.executeRun).not.toHaveBeenCalled();
expect(f.deps.terminateHeartbeatRunProcess).not.toHaveBeenCalled();
expect(await f.current()).toMatchObject({ processPid: child.pid, processGroupId: child.pid });
if (condition === "future_retry") expect(native.claims).not.toHaveBeenCalled();
if (condition === "missing_checkpoint") {
const [coordinator] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.run.id));
expect(coordinator).toMatchObject({ phase: "terminal_failure", recoveryState: "blocked",
failureCode: "native_restart_recovery_blocked" });
}
expect(child.alive()).toBe(true);
} finally { f.deps.activeRunExecutions.delete(f.run.id); try { await f.cleanup(); } finally { await child.stop(); } }
},
);
it.each(["unknown", "remote"] as const)("exposes expired observed %s ownership without attributing host liveness", async target => {
const child = await ownedProcess();
const f = await nativeProcessFixture(child.pid, "remote", "running");
const originalKill = process.kill.bind(process);
let restoreKill: (() => void) | undefined;
try {
if (target === "unknown") await db.delete(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, f.run.id));
await db.update(heartbeatRuns).set({ nativePhase: null })
.where(eq(heartbeatRuns.id, f.run.id));
await db.update(nativeRunFinalizations).set({ phase: "observed", attempt: 2,
leaseOwner: "prior-native-owner", leaseExpiresAt: new Date(0), nextAttemptAt: null })
.where(eq(nativeRunFinalizations.runId, f.run.id));
const before = await f.current();
const coordinatorBefore = await db.select().from(nativeRunFinalizations)
.where(eq(nativeRunFinalizations.runId, f.run.id));
const kill = vi.spyOn(process, "kill").mockImplementation(originalKill);
restoreKill = () => kill.mockRestore();
expect(await createHeartbeatRecovery(db, f.deps).reapOrphanedRuns()).toEqual({ reaped: 0, runIds: [] });
expect(kill.mock.calls.filter(([pid]) => Math.abs(pid as number) === child.pid)).toEqual([]);
expect(await f.current()).toMatchObject({ status: "running", nativePhase: null,
processPid: child.pid, processGroupId: child.pid, processStartedAt: before.processStartedAt,
errorCode: NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE });
expect(f.deps.appendRunEvent).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({
payload: { reason: "observed_owner_unverified" },
}));
expect(await db.select().from(nativeRunFinalizations)
.where(eq(nativeRunFinalizations.runId, f.run.id))).toEqual(coordinatorBefore);
expect(f.deps.executeRun).not.toHaveBeenCalled();
expect(f.deps.terminateHeartbeatRunProcess).not.toHaveBeenCalled();
kill.mockRestore(); restoreKill = undefined;
expect(child.alive()).toBe(true);
} finally { restoreKill?.(); try { await f.cleanup(); } finally { await child.stop(); } }
});
it.each(["failed", "running"] as const)("protects a verified local %s run's live process", async status => {
const child = await ownedProcess();
const f = await nativeProcessFixture(child.pid, "local", status);
try {
await db.update(agents).set({ adapterType: "paperclip_runner", adapterConfig: { driver: "daytona" } })
.where(eq(agents.id, f.run.agentId));
await createHeartbeatRecovery(db, f.deps).reapOrphanedRuns();
expect(await f.current()).toMatchObject({ status, processPid: child.pid, processGroupId: child.pid,
errorCode: NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE });
expect(f.deps.appendRunEvent).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({
payload: { reason: "live_process_identifier", processPidAlive: true, processGroupAlive: true },
}));
expect(f.deps.executeRun).not.toHaveBeenCalled();
expect(f.deps.terminateHeartbeatRunProcess).not.toHaveBeenCalled();
expect(child.alive()).toBe(true);
} finally { try { await f.cleanup(); } finally { await child.stop(); } }
});
it.each(["missing", "new_launch", "wrong_birth", "foreign_host", "wrong_agent", "wrong_pid", "wrong_group", "native_source", "remote_workspace", "remote_lease"] as const)(
"holds a native PID without local authority: %s", async evidence => {
const child = await ownedProcess();
await child.stop();
const f = await nativeProcessFixture(child.pid, "local", "failed");
try {
if (evidence === "missing") await db.delete(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, f.run.id));
if (evidence === "new_launch") await appendHeartbeatRunEvent(db, { companyId: f.run.companyId,
runId: f.run.id, agentId: f.run.agentId, eventType: PROCESS_START_REQUESTED });
if (["wrong_birth", "foreign_host", "wrong_agent", "wrong_pid", "wrong_group", "native_source"].includes(evidence)) {
const [event] = await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, f.run.id));
const foreignAgentId = randomUUID();
if (evidence === "wrong_agent") await db.insert(agents).values({ id: foreignAgentId,
companyId: f.run.companyId, name: "Other process owner", role: "engineer", adapterType: "process" });
await db.update(heartbeatRunEvents).set({
...(evidence === "wrong_agent" ? { agentId: foreignAgentId } : {}),
...(evidence === "native_source" ? { sourceEventId: "provider-event" } : {}),
payload: { ...event.payload, ...(evidence === "wrong_birth"
? { processStartedAt: new Date(0).toISOString() } : evidence === "foreign_host" ? { hostname: "foreign-host" }
: evidence === "wrong_pid" ? { processPid: child.pid + 1 }
: evidence === "wrong_group" ? { processGroupId: child.pid + 1 } : {}) },
}).where(eq(heartbeatRunEvents.id, event.id));
}
if (evidence === "remote_workspace") await db.update(heartbeatRuns).set({
runnerProfileJson: { nativeWorkspaceSync: { schema: "paperclip.native-workspace-sync/v3", state: "prepared",
workspaceId: randomUUID(), leaseId: randomUUID(), providerLeaseId: randomUUID(), remoteCwd: "/work",
descriptorSha256: "a".repeat(64), baselineSha256: "b".repeat(64), finalHostSha256: null, resourceDisposition: "destroy" } },
}).where(eq(heartbeatRuns.id, f.run.id));
if (evidence === "remote_lease") await db.insert(environmentLeases).values({ companyId: f.run.companyId,
heartbeatRunId: f.run.id, issueId: f.run.issueId, status: "active", provider: "daytona", providerLeaseId: randomUUID() });
const before = await f.current();
await createHeartbeatRecovery(db, f.deps).reapOrphanedRuns();
expect(await f.current()).toMatchObject({ processPid: child.pid, processGroupId: child.pid,
processStartedAt: before.processStartedAt, errorCode: "native_session_interrupted" });
expect(f.deps.executeRun).not.toHaveBeenCalled();
expect(f.deps.terminateHeartbeatRunProcess).not.toHaveBeenCalled();
expect(await hasNativeLocalProcessStop(db, f.run.companyId, f.run.id)).toBe(false);
} finally { await f.cleanup(); }
},
);
it("clears a verified stopped local retry identity and records its stop", async () => {
const child = await ownedProcess();
await child.stop();
const f = await nativeProcessFixture(child.pid, "local", "failed");
try {
await createHeartbeatRecovery(db, f.deps).reapOrphanedRuns();
expect(await f.current()).toMatchObject({ processPid: null, processGroupId: null, processStartedAt: null });
expect(await hasNativeLocalProcessStop(db, f.run.companyId, f.run.id)).toBe(true);
expect(f.deps.terminateHeartbeatRunProcess).not.toHaveBeenCalled();
} finally { await f.cleanup(); }
});
it("retires a remote identity only with an exact authenticated termination receipt", async () => {
const child = await ownedProcess();
const f = await nativeProcessFixture(child.pid, "remote", "failed");
try {
await db.update(heartbeatRuns).set({ runnerProfileJson: { sessionCheckpoint: { fixture: true } } })
.where(eq(heartbeatRuns.id, f.run.id));
await db.update(nativeRunFinalizations).set({ phase: "terminal_failure" })
.where(eq(nativeRunFinalizations.runId, f.run.id));
const [lease] = await db.insert(environmentLeases).values({ companyId: f.run.companyId,
heartbeatRunId: f.run.id, issueId: f.run.issueId, status: "active", provider: "daytona", providerLeaseId: randomUUID() }).returning();
expect(await releaseSettledNativeRunProcess(db, f.run.id)).toBe("pending");
const receipt = remoteTerminationReceipt(lease, { state: "destroyed", providerLeaseId: lease.providerLeaseId })!;
await db.update(environmentLeases).set({ status: "released", cleanupStatus: "success", releasedAt: new Date(),
metadata: { remoteExecutionTermination: { ...receipt, runId: randomUUID() } } }).where(eq(environmentLeases.id, lease.id));
expect(await releaseSettledNativeRunProcess(db, f.run.id)).toBe("pending");
expect(await f.current()).toMatchObject({ processPid: child.pid, processGroupId: child.pid });
await db.update(environmentLeases).set({ metadata: { remoteExecutionTermination: receipt } })
.where(eq(environmentLeases.id, lease.id));
expect(await releaseSettledNativeRunProcess(db, f.run.id)).toBe("released");
expect(await f.current()).toMatchObject({ processPid: null, processGroupId: null, processStartedAt: null });
expect(await hasNativeLocalProcessStop(db, f.run.companyId, f.run.id)).toBe(false);
expect(child.alive()).toBe(true);
} finally { try { await f.cleanup(); } finally { await child.stop(); } }
});
it.each(["ephemeral", "reuse_by_environment", "live_owner", "live_executor", "mismatched_source", "finalizing"] as const)(
"reconstructs missing native preservation during orphan cleanup without replay: %s", async state => {
const run = await seedRun();
@@ -857,11 +1158,12 @@ describePostgres("heartbeat recovery module ownership", () => {
await Promise.all([...deps.activeRunExecutionPromises]);
});
it("rearms the persisted native retry deadline through the service callback", async () => {
it.each(["future", "overdue-owner"])("rearms the persisted native retry deadline through the service callback: %s", async scenario => {
const run = await seedRun();
const nextAttemptAt = new Date(Date.now() + 60_000);
const nextAttemptAt = new Date(Date.now() + (scenario === "future" ? 60_000 : -1_000));
await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: run.issueId, status: "failed" }).where(eq(heartbeatRuns.id, run.id));
await db.insert(nativeRunFinalizations).values({ runId: run.id, companyId: run.companyId, issueId: run.issueId!, phase: "retryable_failure", nextAttemptAt });
await db.insert(nativeRunFinalizations).values({ runId: run.id, companyId: run.companyId, issueId: run.issueId!, phase: "retryable_failure", nextAttemptAt,
failureCode: scenario === "overdue-owner" ? "native_controller_recovery_waiting_for_owner" : null });
const deps = callbacks(db);
const result = await createHeartbeatRecovery(db, deps).recoverNativeRunsAfterRestart();
expect(result.scheduledRetryRunIds).toEqual([run.id]);
@@ -872,6 +1174,16 @@ describePostgres("heartbeat recovery module ownership", () => {
await db.delete(heartbeatRuns).where(eq(heartbeatRuns.id, run.id));
});
it("retains a live remote owner's actual lease deadline without dispatching competing work", async () => {
const deps = callbacks(db);
const runId = randomUUID(), retryAt = new Date(Date.now() + 60_000);
native.claims.mockResolvedValue([{ kind: "awaiting_evidence", runId, reason: "remote_controller_lease_live", retryAt }]);
const result = await createHeartbeatRecovery(db, deps).recoverNativeRunsAfterRestart();
expect(result.awaitingEvidenceRunIds).toEqual([runId]);
expect(deps.scheduleNativeSessionResumeDispatch).toHaveBeenCalledWith(runId, retryAt, "restart");
expect(deps.executeRun).not.toHaveBeenCalled();
});
it("retains shutdown ownership until background native cleanup physically settles", async () => {
const deps = callbacks(db);
let settle!: () => void;
+66 -14
View File
@@ -1,3 +1,4 @@
import { NATIVE_RETAINED_OWNER_WAIT } from "../native-runtime/native-controller-recovery.js";
import { nativeEnvironmentSettlementDisposition, nativeEnvironmentSettlementWork } from "../native-runtime/native-environment-settlement-work.js";
import { nativeWorkspaceFinalizationWork } from "../native-runtime/native-workspace-finalization-work.js";
import { attemptNativeWorkspaceFinalization } from "../native-runtime/native-workspace-finalizer.js";
@@ -18,7 +19,7 @@ import {
hasStopOnlyCleanup,
settleStopOnlyCleanup,
} from "../sandbox-stop-and-retain.js";
import { discoverSettledNativeRunProcesses, recordNativeLocalProcessStop, retryPendingSettledNativeRunProcesses } from "../native-local-process-stop.js";
import { discoverSettledNativeRunProcesses, PROCESS_IDENTITY_RECORDED, PROCESS_START_REQUESTED, recordNativeLocalProcessStop, retryPendingSettledNativeRunProcesses } from "../native-local-process-stop.js";
import { pendingCaseHandoffForRun } from "./case-handoff-lifecycle.js";
import {
legacyControllerBootId,
@@ -30,9 +31,11 @@ import { runUsedConversationAdapter } from "../conversation-continuation.js";
import { legacyExecutionNeedsReconciliationWithEvidence, terminalizeLegacyExecution } from "../legacy-execution-recovery.js";
import { reconcileRetainedWorkspaces } from "../workspace-recovery.js";
import { randomUUID } from "node:crypto";
import os from "node:os";
import {
and,
asc,
desc,
eq,
gt,
inArray,
@@ -46,6 +49,7 @@ import type { Db } from "@paperclipai/db";
import {
agents,
environmentLeases,
heartbeatRunEvents,
heartbeatRuns,
issues,
nativeRunFinalizations,
@@ -123,7 +127,7 @@ export interface HeartbeatRecoveryDependencies extends Pick<HeartbeatRetryDepend
nativeRestartRecovery?: NativeRestartRecoveryClaim;
onNativeExecutionLeaseAcquired?: () => Promise<void>;
}) => Promise<void>;
scheduleNativeSessionResumeDispatch: (runId: string, nextAttemptAt: Date) => void;
scheduleNativeSessionResumeDispatch: (runId: string, nextAttemptAt: Date, mode?: "restart") => void;
cancelHeartbeatNativeRun: (input: {
db: Db; runId: string; reason: string; runtimeMode: string | null;
}) => Promise<unknown>;
@@ -953,7 +957,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
inArray(heartbeatRuns.status, ["running", "failed"]),
isNull(nativeRunFinalizations.resultId),
eq(nativeRunFinalizations.phase, "retryable_failure"),
gt(nativeRunFinalizations.nextAttemptAt, now),
or(gt(nativeRunFinalizations.nextAttemptAt, now), eq(nativeRunFinalizations.failureCode, NATIVE_RETAINED_OWNER_WAIT)),
),
);
for (const scheduled of scheduledNativeRetries) {
@@ -1001,6 +1005,9 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
disposition.kind === "bootstrap_incomplete",
);
for (const disposition of dispositions) {
if (disposition.kind === "awaiting_evidence" && disposition.retryAt) {
scheduleNativeSessionResumeDispatch(disposition.runId, disposition.retryAt, "restart");
}
const run = await getRun(disposition.runId);
if (run) {
const isClaim =
@@ -1836,6 +1843,29 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
return { swept: rows.length, destroyed, capped };
}
async function nativeProcessIsLocal(run: HeartbeatRun) {
if (parseObject(run.runnerProfileJson).nativeWorkspaceSync != null) return false;
const [identity] = await db.select({
eventType: heartbeatRunEvents.eventType,
agentId: heartbeatRunEvents.agentId,
payload: heartbeatRunEvents.payload,
}).from(heartbeatRunEvents).where(and(
eq(heartbeatRunEvents.companyId, run.companyId),
eq(heartbeatRunEvents.runId, run.id),
isNull(heartbeatRunEvents.sourceEventId),
inArray(heartbeatRunEvents.eventType, [PROCESS_START_REQUESTED, PROCESS_IDENTITY_RECORDED]),
)).orderBy(desc(heartbeatRunEvents.seq)).limit(1);
const recorded = identity?.payload;
if (identity?.eventType !== PROCESS_IDENTITY_RECORDED || identity.agentId !== run.agentId ||
recorded?.targetKind !== "local" || recorded.hostname !== os.hostname() ||
recorded.processPid !== run.processPid || recorded.processGroupId !== run.processGroupId ||
!run.processStartedAt || recorded.processStartedAt !== run.processStartedAt.toISOString()) return false;
const leases = await db.select({ provider: environmentLeases.provider }).from(environmentLeases).where(and(
eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id),
));
return leases.every(lease => lease.provider === "local");
}
async function markNativeOwnershipUnverified(
run: typeof heartbeatRuns.$inferSelect,
evidence: {
@@ -2030,13 +2060,12 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
);
});
// A retryable native run can retain process identifiers from the failed
// attempt. Inspect them before the recovery claim: a live identifier is
// unowned and blocks recovery, while identifiers that are all dead can be
// cleared with a compare-and-set so the explicit retryable failure becomes
// claimable in this same sweep.
// Retryable runs can retain process identifiers from the failed attempt.
// Only verified local identities may be checked and cleared on this host.
// Due remote retries instead re-enter the existing restart-recovery owner,
// which verifies the saved checkpoint, controller lease, and remote target.
const retryableNativeProcesses = await db
.select({ run: heartbeatRuns })
.select({ run: heartbeatRuns, nextAttemptAt: nativeRunFinalizations.nextAttemptAt })
.from(heartbeatRuns)
.innerJoin(
nativeRunFinalizations,
@@ -2051,12 +2080,27 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
),
);
const claimableNativeRunIds = new Set<string>();
for (const { run } of retryableNativeProcesses) {
const remoteRetryRunIds: string[] = [];
for (const { run, nextAttemptAt } of retryableNativeProcesses) {
if (isNativeRunnerOwnershipHeld(run)) continue;
if (!run.processPid && !run.processGroupId) {
claimableNativeRunIds.add(run.id);
continue;
}
if (!await nativeProcessIsLocal(run)) {
if (nextAttemptAt && nextAttemptAt > now) continue;
if (activeRunExecutions.has(run.id) || runningProcesses.has(run.id)) continue;
if (parseObject(run.runnerProfileJson).nativeWorkspaceSync != null) {
remoteRetryRunIds.push(run.id);
} else {
const [taskLease] = await db.select({ id: environmentLeases.id }).from(environmentLeases).where(and(
eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id),
sql`${environmentLeases.metadata}->>'runnerTaskLease' = 'true'`,
)).limit(1);
if (taskLease) remoteRetryRunIds.push(run.id);
}
continue;
}
const processPidAlive =
!!run.processPid && isProcessAlive(run.processPid);
const processGroupAlive =
@@ -2082,6 +2126,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
and(
eq(heartbeatRuns.id, run.id),
eq(heartbeatRuns.runtimeMode, "native"),
eq(heartbeatRuns.nextEventSeq, run.nextEventSeq),
run.processPid === null
? isNull(heartbeatRuns.processPid)
: eq(heartbeatRuns.processPid, run.processPid),
@@ -2101,6 +2146,10 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
if (cleared) claimableNativeRunIds.add(cleared.id);
}
const remoteRetryClaims = remoteRetryRunIds.length
? (await recoverNativeRunsAfterRestart(now, remoteRetryRunIds)).claims
: [];
const executingRun = (runId: string) => activeRunExecutions.has(runId) || runningProcesses.has(runId);
if (!settledProcessesDiscovered) {
settledProcessesDiscovered = await discoverSettledNativeRunProcesses(db, { skip: executingRun }).then(() => true, (error) => {
@@ -2148,7 +2197,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
return [];
});
const resumedRunIds = new Set(
nativeResumeClaims.map((claim) => claim.runId),
[...nativeResumeClaims, ...remoteRetryClaims].map((claim) => claim.runId),
);
// A terminal issue transition writes this intent in the same transaction
@@ -2300,10 +2349,12 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
// preserves its explicit physical-copyback hold. Keep its source lease.
if (nativeWorkspaceFinalizationPending) continue;
const nativeRun = run.runtimeMode === "native";
const nativeLocalProcess = nativeRun && !!(run.processPid || run.processGroupId) &&
await nativeProcessIsLocal(run);
const nativeProcessPidAlive =
nativeRun && !!run.processPid && isProcessAlive(run.processPid);
nativeLocalProcess && !!run.processPid && isProcessAlive(run.processPid);
const nativeProcessGroupAlive =
nativeRun &&
nativeLocalProcess &&
!!run.processGroupId &&
isProcessGroupAlive(run.processGroupId);
const coordinatorOwnedByCurrentController =
@@ -2360,6 +2411,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
});
continue;
}
if (nativeRun && !locallyTracked && (run.processPid || run.processGroupId) && !nativeLocalProcess) continue;
if (resumedRunIds.has(run.id)) continue;
if (locallyTracked) continue;
if (await hasLiveLegacyController(db, run)) continue;
@@ -2379,7 +2431,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
// for a read-only liveness check so a lost in-memory handle cannot cause
// overlapping provider/tool execution while that child is still alive.
const checksPersistedChildLiveness =
currentAdapterTracksLocalChild || run.runtimeMode === "native";
nativeRun ? nativeLocalProcess : currentAdapterTracksLocalChild;
const processPidAlive =
checksPersistedChildLiveness &&
run.processPid &&
@@ -309,9 +309,16 @@ describe.skipIf(!support.supported && !process.env.PAPERCLIP_TEST_DATABASE_URL)(
it("retains native completion metadata when reconciliation already committed the same terminal status", async () => {
const finishedAt = new Date("2026-10-09T12:00:00Z");
const f = await fixture({ runtimeMode: "native", status: "succeeded", finishedAt, resultJson: { durable: "accepted" }, usageJson: { durableReceipt: "native" } });
f.input.adapterResult.nativeFinalization = { schema: "paperclip.native-finalization.v1", runtimeMode: "native", runId: f.run.id, companyId: f.run.companyId, issueId: "unused", result: {}, terminal: { schema: "paperclip.prp.terminal.v1", turnTerminalState: "completed", runTerminalState: "succeeded", reportedWorkDisposition: "done" }, turnId: null, sourceInstanceId: "test", normalizedSessionId: "native-session", providerSessionId: null, driverKind: "test", driverVersion: "1", nativeEventCount: 1, highestContiguousSourceSeq: 1, workspaceFinalizeStatus: "succeeded" };
await db.update(agents).set({ adapterType: "paperclip_runner" }).where(eq(agents.id, f.agent.id));
f.agent.adapterType = "paperclip_runner";
f.input.sessionCodec = getAdapterSessionCodec(f.agent.adapterType);
const nativeSessionId = randomUUID(), providerSessionId = randomUUID();
f.input.adapterResult.sessionId = nativeSessionId;
delete f.input.adapterResult.sessionParams;
f.input.adapterResult.sessionDisplayId = providerSessionId;
f.input.adapterResult.nativeFinalization = { schema: "paperclip.native-finalization.v1", runtimeMode: "native", runId: f.run.id, companyId: f.run.companyId, issueId: "unused", result: {}, terminal: { schema: "paperclip.prp.terminal.v1", turnTerminalState: "completed", runTerminalState: "succeeded", reportedWorkDisposition: "done" }, turnId: null, sourceInstanceId: "test", normalizedSessionId: nativeSessionId, providerSessionId, driverKind: "test", driverVersion: "1", nativeEventCount: 1, highestContiguousSourceSeq: 1, workspaceFinalizeStatus: "succeeded" };
await f.completion.completeRun(f.input);
expect(await read(f.run.id)).toMatchObject({ status: "succeeded", finishedAt, sessionIdAfter: "provider-session", resultJson: { durable: "accepted" }, usageJson: { durableReceipt: "native", inputTokens: 20 } });
expect(await read(f.run.id)).toMatchObject({ status: "succeeded", finishedAt, sessionIdAfter: providerSessionId, resultJson: { durable: "accepted" }, usageJson: { durableReceipt: "native", inputTokens: 20 } });
expect(f.deps.updateRuntimeState).toHaveBeenCalledOnce();
});
@@ -1,4 +1,5 @@
import { preserveAcceptedRunUsageProjectionSql } from "../usage-receipts.js";
import { nativeTaskSessionPublicationHandled } from "../native-runtime/native-task-session-publication.js";
import { notifyChatRunMilestones } from "../chat-run-milestone-notifications.js";
import { measureRunPhase } from "../execution-preparation-timing.js";
import { PROJECT_ACCESS_DENIED, ProjectAccessDeniedError } from "../project-access-denial.js";
@@ -489,6 +490,13 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC
normalizedUsage,
);
if (!taskKey) return;
if (nativeTaskSessionPublicationHandled(finalizedRun.runnerProfileJson, {
runId: finalizedRun.id,
companyId: finalizedRun.companyId,
agentId: agent.id,
taskKey,
adapterType: agent.adapterType,
})) return;
if (
adapterResult.clearSession ||
(!nextSessionState.params && !nextSessionState.displayId)
@@ -1,3 +1,5 @@
import { NativeControllerRecoveryWaitingError } from "../native-runtime/native-controller-recovery.js";
import { createNativeUsageDelivery } from "../native-runtime/native-usage-delivery.js";
import type { PrpEvent } from "../../vendor/paperclip-runner/index.js";
import { randomUUID } from "node:crypto";
@@ -10,7 +12,7 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vites
import { trace } from "@opentelemetry/api";
import { InMemorySpanExporter, SimpleSpanProcessor, TracerProvider } from "@opentelemetry/sdk-trace";
import { createRunPreparationTiming, measureRunPhase, measureWorkspaceSyncPhase, withRunExecutionTiming } from "../execution-preparation-timing.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
import { recoveryPostgresSupport as support, startRecoveryPostgres } from "../../__tests__/helpers/recovery-postgres.js";
import * as pluginWorkspaces from "../plugin-workspaces.js";
import * as legacyRecovery from "../legacy-execution-recovery.js";
import { createAgentIdentityRedactor } from "../agent-identity-redaction.js";
@@ -43,15 +45,14 @@ vi.mock("./run-preparation.js", async original => ({
revokeHeartbeatRunGatewayTokens: vi.fn(async () => {}),
}));
const support = await getEmbeddedPostgresTestSupport();
describe.skipIf(!support.supported)("heartbeat runtime execution boundary", () => {
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let database: Awaited<ReturnType<typeof startRecoveryPostgres>>;
let db: Db;
let home: string;
beforeAll(async () => {
home = await mkdtemp(join(tmpdir(), "paperclip-runtime-execution-"));
vi.stubEnv("PAPERCLIP_HOME", home);
database = await startEmbeddedPostgresTestDatabase("paperclip-runtime-execution");
database = await startRecoveryPostgres("paperclip-runtime-execution");
db = createDb(database.connectionString);
await db.execute(sql`set client_min_messages = warning`);
}, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
@@ -283,6 +284,18 @@ describe.skipIf(!support.supported)("heartbeat runtime execution boundary", () =
expectCleanup(input);
});
it("preserves the retained sandbox while native admission waits for its current owner", async () => {
const { input } = await fixture(true);
const error = new NativeControllerRecoveryWaitingError(new Date(Date.now() + 60_000));
vi.mocked(executePaperclipNativeSession).mockRejectedValueOnce(error);
await expect(executeHeartbeatRuntime(db, input)).rejects.toMatchObject({
name: "NativeSessionResumeScheduledError", original: error,
});
expect(input.effects.onNativeSessionResumeScheduled).toHaveBeenCalledOnce();
expect(input.effects.onProviderResourceDisposition).not.toHaveBeenCalledWith("stop_and_retain");
expect(await finalizations()).toHaveLength(0);
});
it("records native restoration inside the workspace finalization operation", async () => {
const { input, run } = await fixture(true);
const endpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
@@ -1,3 +1,5 @@
import { NativeControllerRecoveryWaitingError } from "../native-runtime/native-controller-recovery.js";
import { usesNativeUsageDelivery } from "../native-runtime/native-usage-accounting.js";
import { DELIVERY_QUEUES } from "../delivery-work-notifications.js";
import { writeExecutionControl } from "../execution-control-notifications.js";
@@ -559,6 +561,20 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
sessionGoalControl.requestId,
));
if (completedGoalControl) sessionGoalControl = null;
// Goal checkpoints already publish their task session. Goal-less turns
// need the same durable pointer before cleanup can delay host completion.
const taskSessionPublication = taskKey && !sessionGoalControl &&
context.resumeSessionGoalHeartbeat !== true && !completedGoalControl
? {
taskKey,
adapterType: agent.adapterType,
sessionParamsJson: attachPaperclipSessionMetadataToSessionParams(
{ ...runtimeSessionParamsForAdapter, cwd: executionWorkspace.cwd },
configuredModel,
sessionConfigMetadata,
),
}
: undefined;
const nativeDispatchAtMs = Date.now();
const runCreatedAtMs = run.createdAt.getTime();
const runStartedAtMs = (run.startedAt ?? run.createdAt).getTime();
@@ -608,6 +624,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
resumeSessionGoalHeartbeat:
context.resumeSessionGoalHeartbeat === true ||
completedGoalControl,
taskSessionPublication,
onGoalCheckpoint: async (snapshot) => {
if (!taskKey) return;
const params =
@@ -1026,6 +1043,10 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
preserveRunGatewayTokens = true;
throw adapterErr;
}
if (adapterErr instanceof NativeControllerRecoveryWaitingError) {
input.effects.onNativeSessionResumeScheduled();
throw new NativeSessionResumeScheduledError(adapterErr);
}
if (adapterErr instanceof NativeRunnerOwnershipUnverifiedError) {
input.effects.onNativeOwnershipHeld();
throw adapterErr;
@@ -4,6 +4,7 @@ import { inspectNativeRunnerUpgradeHeader } from "../native-runtime/native-runne
import { readNativeWorkspaceSyncReference } from "../native-runtime/native-workspace-sync.js";
import { REMOTE_RUNNER_RECONNECT_GRACE_MS } from "../native-runtime/native-runner-recovery-contract.js";
import { currentNativeControllerIdentity } from "../native-runtime/native-restart-recovery.js";
import { readNativeControllerRecoveryInventory } from "../native-runtime/native-controller-recovery-inventory.js";
const MAX_UPGRADE_RUNS = 1_000;
type Counters = { activeRuns: number; pendingWakes: number; draining: boolean };
@@ -60,8 +61,16 @@ export async function readUpgradeReadiness(db: Db, input: {
&& identity.runnerInstanceId === row.runnerId && identity.normalizedSessionId === row.sessionId
&& identity.environmentLeaseId === (row.pluginLeaseId ?? workspace?.workspaceId)) };
});
return summarizeUpgradeReadiness({ rows: candidates, before, after: input.counters(),
// Additive evidence does not strengthen or redefine the existing ready flag.
const retained = await readNativeControllerRecoveryInventory(db).catch(() => ({
version: 1 as const, observedAt: new Date().toISOString(), coverage: "unknown" as const,
reasons: { inventory_unavailable: 1 }, fullPayloadsValidated: false as const,
}));
const summary = summarizeUpgradeReadiness({ rows: candidates, before, after: input.counters(),
activeRunIds: [...new Set([...beforeIds, ...input.activeRunIds])], controllerBootId: owner.bootId });
return { ...summary, nativeControllerRecovery: { activeExecutionAdoption: {
recoverableRuns: summary.recoverableRuns, blockingRuns: summary.blockingRuns, truncated: summary.truncated,
}, retained } };
}
export function summarizeUpgradeReadiness(input: {
+26 -7
View File
@@ -38,7 +38,7 @@ import { requestRestorePointDeletion } from "./git-restore-point-ledger.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
import { recordChatHandoff, recordAcceptedChatHandoff, lockAcceptedChatHandoffSource, recordChatCompletion, existingChatCompletionReply, acknowledgeChatCompletionReply } from "./chat-completion-delivery.js";
import { mirrorSlackBoardComment, slackBoardReplyBindings } from "./slack-board-messages.js";
import { assertAgentRunWriteAllowed } from "../agent-run-cancellation.js";
import { agentRunWritesRevoked, assertAgentRunWriteAllowed } from "../agent-run-cancellation.js";
import { assertLegacyRunTaskOwnership } from "../legacy-run-task-ownership.js";
import { retryIdempotentDatabaseOperation } from "../database-retry.js";
import { externalConversationStateSql, nonIdleSlackIssueCondition, resumeSlackConversation } from "./slack-conversation-state.js";
@@ -152,7 +152,7 @@ import {
isUuidLike,
normalizeIssueIdentifier as normalizeIssueReferenceIdentifier,
} from "@paperclipai/shared";
import { conflict, HttpError, notFound, unprocessable } from "../errors.js";
import { conflict, forbidden, HttpError, notFound, unprocessable } from "../errors.js";
import { isForeignKeyViolation } from "../db-errors.js";
import { logger } from "../middleware/logger.js";
import { parseObject } from "../adapters/utils.js";
@@ -1694,6 +1694,7 @@ function withAgentCommentAuthorizationMetadata(
const reason = authorizationReason?.trim() || "internal_agent_write";
return {
version: 1,
...(metadata?.chatMessage ? { chatMessage: metadata.chatMessage } : {}),
...(metadata?.sourceRunId !== undefined
? { sourceRunId: metadata.sourceRunId }
: {}),
@@ -8652,7 +8653,7 @@ export function issueService(db: Db) {
companyId: string,
issueId: string,
userId: string,
readAt: Date = new Date(),
readAt: Date | SQL = new Date(),
) => {
const now = new Date();
const [row] = await db
@@ -8671,7 +8672,7 @@ export function issueService(db: Db) {
issueReadStates.userId,
],
set: {
lastReadAt: readAt,
lastReadAt: sql`greatest(${issueReadStates.lastReadAt}, excluded.last_read_at)`,
updatedAt: now,
},
})
@@ -12933,6 +12934,7 @@ export function issueService(db: Db) {
sourceTrust?: typeof issueComments.$inferInsert.sourceTrust;
createdAt?: Date | string | null;
clientRequestId?: string;
proactiveChatMessage?: { sourceTaskId: string; userId: string };
/** Server-only: authenticated Paperclip messages also belong in the Slack thread. */
mirrorToSlack?: boolean;
},
@@ -12989,11 +12991,28 @@ export function issueService(db: Db) {
return existing;
}
}
if (options?.metadata?.chatMessage && !options.proactiveChatMessage) {
throw forbidden("Chat message provenance is server-owned");
}
if (issue.conversationAgentId && actor.agentId && issue.conversationAgentId !== actor.agentId) {
throw forbidden("Only this conversation's agent can post agent messages");
}
if (issue.conversationAgentId && actor.agentId && !actor.runId) {
throw forbidden("Agent chat messages require an authenticated run");
}
if (issue.conversationAgentId && actor.runId) {
const [run] = await dbOrTx.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, actor.runId));
const [current] = await dbOrTx.select().from(issues).where(eq(issues.id, issueId));
if (run?.status === "cancelled") throw conflict("This conversation turn was cancelled; it cannot post a reply");
if (run?.contextSnapshot?.conversationSessionGeneration !== current.conversationSessionGeneration) {
if (agentRunWritesRevoked(run)) throw conflict("This conversation turn was cancelled; it cannot post a reply");
const runTaskId = run?.nativeIssueId ?? run?.issueId ?? run?.contextSnapshot?.issueId ?? run?.contextSnapshot?.taskId;
if (options?.proactiveChatMessage) {
if (run?.status !== "running" || run.agentId !== actor.agentId || run.companyId !== issue.companyId
|| runTaskId !== options.proactiveChatMessage.sourceTaskId || runTaskId === issueId
|| current.conversationUserId !== options.proactiveChatMessage.userId
|| run.responsibleUserId !== current.conversationUserId) {
throw forbidden("Invalid background chat message authority");
}
} else if (runTaskId !== issueId || run?.contextSnapshot?.conversationSessionGeneration !== current.conversationSessionGeneration) {
throw conflict("Conversation session changed; this reply belongs to an earlier session");
}
}
@@ -13128,7 +13147,7 @@ export function issueService(db: Db) {
: (options?.metadata ?? null),
);
let comment: typeof issueComments.$inferSelect | null = null;
if (createdByRunId) {
if (createdByRunId && !options?.proactiveChatMessage) {
const existing = await dbOrTx
.select()
.from(issueComments)
@@ -0,0 +1,120 @@
import { randomUUID } from "node:crypto";
import { runAdapterExecutionTargetShellCommand, type AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target";
import { kimiMetadataProbeDigest, kimiMetadataProbeNodeDigest, probeKimiMetadata, validateKimiMetadata } from "../vendor/paperclip-runner/live/index.js";
import { unprocessable } from "../errors.js";
import { createSshCommandManagedRuntimeRunner, runSshCommand } from "@paperclipai/adapter-utils/ssh";
import { QUALIFIED_ACPX_PROFILES } from "../vendor/paperclip-runner/index.js";
import { resolveKimiCandidateAdmission } from "./native-runtime/acpx-qualification.js";
const REQUEST_FAILED = () => unprocessable("Kimi Code metadata verification failed.", { code: "KIMI_REQUEST_FAILED" });
const INSTALLATION_INVALID = () => unprocessable("Install the current verified Kimi Code runtime in this execution environment.", { code: "KIMI_INSTALLATION_INVALID" });
const SSH_INPUT_PROBE = 'try { const input = JSON.parse(require("node:fs").readFileSync(0, "utf8")); if (typeof input.apiKey !== "string" || typeof input.model !== "string") process.exit(1); process.env.KIMI_API_KEY = input.apiKey; process.env.PAPERCLIP_KIMI_PROBE_MODEL = input.model; if (input.candidateAdmission !== undefined) process.env.PAPERCLIP_ACPX_CANDIDATE_ADMISSION = JSON.stringify(input.candidateAdmission); require(process.argv[1]); } catch { process.exit(1); }';
export async function probeKimiExecutionTarget(apiKey: string, target: AdapterExecutionTarget | null | undefined, model: string, context: { companyId?: string } = {}) {
if (typeof apiKey !== "string" || !apiKey.trim() || /[\0\r\n]/.test(apiKey) || Buffer.byteLength(apiKey) > 16 * 1024) {
throw unprocessable("Bind a valid company secret to KIMI_API_KEY.", { code: "KIMI_AUTH_REQUIRED" });
}
if (typeof model !== "string" || !model.trim() || model.includes("\0") || Buffer.byteLength(model) > 512 || Buffer.from(model).toString("utf8") !== model) {
throw unprocessable("Select the exact Kimi Code model.", { code: "KIMI_MODEL_UNAVAILABLE" });
}
let candidateAdmission: Parameters<typeof probeKimiMetadata>[2];
if (QUALIFIED_ACPX_PROFILES.kimi.qualificationStatus === "pending") {
try {
const admission = resolveKimiCandidateAdmission({ companyId: context.companyId, model,
target: target?.kind === "remote" ? { platform: "linux", architecture: "x64" } : { platform: process.platform, architecture: process.arch } });
if (!admission) throw new Error("Missing candidate admission");
const probeId = `kimi-metadata-${randomUUID()}`;
candidateAdmission = { ...admission, runId: probeId, normalizedSessionId: `metadata-${randomUUID()}` };
} catch {
throw unprocessable("Kimi Code qualification requires current trusted operator admission for this company and execution target.", { code: "KIMI_QUALIFICATION_REQUIRED" });
}
}
if (!target || target.kind === "local") {
const result = validateKimiMetadata(await probeKimiMetadata(model, apiKey, candidateAdmission), model);
if (result.status !== "verified") throw unprocessable(result.message, { code: result.code });
return result;
}
if (target.transport === "sandbox" && target.runnerTask) {
throw unprocessable("Kimi Code metadata verification is unavailable in this hosting environment.", { code: "KIMI_INSTALLATION_INVALID" });
}
let locations = ["/opt/paperclip-runner/provider-pack", '"$HOME/.local/share/paperclip-runner/provider-pack"'];
const selectedPackRoot = process.env.PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH?.trim();
if (selectedPackRoot) {
try {
// Test uses the same complete-pack installer as native execution. It
// needs no native run/session or DB context and opens no model session.
const { prepareRemoteProviderPackOnTarget, resolveRemoteProviderPackSource } = await import("./native-runtime/native-session-executor.js");
const selected = resolveRemoteProviderPackSource(selectedPackRoot);
if (selected.manifest.payload.target.platform !== "linux" || selected.manifest.payload.target.architecture !== "x64") throw INSTALLATION_INVALID();
const runner = target.transport === "ssh" ? createSshCommandManagedRuntimeRunner({ spec: target.spec, defaultCwd: target.remoteCwd }) : target.runner;
if (!runner) throw INSTALLATION_INVALID();
const prepared = await prepareRemoteProviderPackOnTarget({ target, runner,
configuredProviderPackRoot: selected.configuredRoot, manifest: selected.manifest });
locations = [`'${prepared.root.replaceAll("'", "'\\''")}'`];
} catch { throw INSTALLATION_INVALID(); }
}
// Preparation retains the normal installer's bounded stages. The existing
// metadata/HTTP budget begins only after that preparation has completed.
const deadlineAt = Date.now() + 35_000;
let digest: string;
try { digest = await kimiMetadataProbeDigest(); }
catch { throw INSTALLATION_INVALID(); }
const nodeDigest = kimiMetadataProbeNodeDigest();
if (!/^sha256:[a-f0-9]{64}$/.test(digest) || !/^sha256:[a-f0-9]{64}$/.test(nodeDigest)) throw INSTALLATION_INVALID();
for (const location of locations) {
const remainingMs = deadlineAt - Date.now();
if (remainingMs <= 3_000) throw REQUEST_FAILED();
const command = [
`pack=${location}`,
'entry="$pack/dist/cli/kimi-metadata-probe.cjs"; node="$pack/provider-assets/kimi/linux-x64/runtime/node/bin/node"',
'[ -f "$pack/provider-pack.json" ] && [ -f "$entry" ] && [ ! -L "$entry" ] && [ -x "$node" ] && [ ! -L "$node" ] || exit 127',
'unset NODE_OPTIONS NODE_PATH NODE_EXTRA_CA_CERTS NODE_USE_ENV_PROXY NODE_TLS_REJECT_UNAUTHORIZED OPENSSL_CONF SSL_CERT_FILE SSL_CERT_DIR HTTP_PROXY HTTPS_PROXY ALL_PROXY NO_PROXY http_proxy https_proxy all_proxy no_proxy LD_PRELOAD LD_LIBRARY_PATH LD_AUDIT GLIBC_TUNABLES DYLD_INSERT_LIBRARIES DYLD_LIBRARY_PATH DYLD_FRAMEWORK_PATH',
'PATH=/usr/bin:/bin; export PATH',
'actual=$(/usr/bin/env -i /usr/bin/sha256sum "$entry" 2>/dev/null) || exit 126',
`actual=\${actual%% *}; [ "$actual" = "${digest.slice(7)}" ] || exit 126`,
'actual=$(/usr/bin/env -i /usr/bin/sha256sum "$node" 2>/dev/null) || exit 126',
`actual=\${actual%% *}; [ "$actual" = "${nodeDigest.slice(7)}" ] || exit 126`,
target.transport === "ssh"
? `exec /usr/bin/env -i PAPERCLIP_ACPX_PROVIDER_PACKAGE_ROOT="$pack" PAPERCLIP_ACPX_PROVIDER_PACKAGE_MANIFEST="$pack/package.json" "$node" -e '${SSH_INPUT_PROBE}' "$entry"`
: ['exec /usr/bin/env -i KIMI_API_KEY="$KIMI_API_KEY" PAPERCLIP_KIMI_PROBE_MODEL="$PAPERCLIP_KIMI_PROBE_MODEL"',
...(candidateAdmission ? ['PAPERCLIP_ACPX_CANDIDATE_ADMISSION="$PAPERCLIP_ACPX_CANDIDATE_ADMISSION"'] : []),
'PAPERCLIP_ACPX_PROVIDER_PACKAGE_ROOT="$pack" PAPERCLIP_ACPX_PROVIDER_PACKAGE_MANIFEST="$pack/package.json" "$node" "$entry"'].join(" "),
].join("; ");
let response: Pick<Awaited<ReturnType<typeof runAdapterExecutionTargetShellCommand>>, "stdout" | "exitCode" | "signal" | "timedOut">;
try {
if (target.transport === "ssh") {
try {
const result = await runSshCommand(target.spec, command, {
stdin: JSON.stringify({ apiKey, model, ...(candidateAdmission ? { candidateAdmission } : {}) }), timeoutMs: Math.max(1, Math.floor((remainingMs - 3_000) / 1000)) * 1000,
});
response = { stdout: result.stdout, exitCode: 0, signal: null, timedOut: false };
} catch (error) {
const failure = error as { code?: unknown; stdout?: unknown; signal?: string | null };
if (typeof failure.code !== "number") throw error;
response = { stdout: typeof failure.stdout === "string" ? failure.stdout : "", exitCode: failure.code,
signal: failure.signal ?? null, timedOut: false };
}
} else {
response = await runAdapterExecutionTargetShellCommand(`kimi-metadata-${randomUUID()}`, target, command, {
cwd: target.remoteCwd, env: { KIMI_API_KEY: apiKey, PAPERCLIP_KIMI_PROBE_MODEL: model,
...(candidateAdmission ? { PAPERCLIP_ACPX_CANDIDATE_ADMISSION: JSON.stringify(candidateAdmission) } : {}) },
timeoutSec: Math.max(1, Math.floor((remainingMs - 3_000) / 1000)), graceSec: 3,
});
}
} catch { throw REQUEST_FAILED(); }
if (Date.now() > deadlineAt || response.timedOut || response.signal || Buffer.byteLength(response.stdout) > 64 * 1024) throw REQUEST_FAILED();
if ([126, 127].includes(response.exitCode ?? -1)) continue;
let payload: unknown;
try { payload = JSON.parse(response.stdout.trim()); } catch { throw REQUEST_FAILED(); }
const result = validateKimiMetadata(payload, model);
if (result.status !== "verified") {
if (result.code === "KIMI_INSTALLATION_INVALID") continue;
throw unprocessable(result.message, { code: result.code });
}
if (response.exitCode !== 0) throw REQUEST_FAILED();
if (result.target !== "linux-x64") throw INSTALLATION_INVALID();
return result;
}
throw INSTALLATION_INVALID();
}
@@ -1,6 +1,7 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { readFileSync } from "node:fs";
import { ACPX_QUALIFICATION_ENV, resolveAcpxQualification } from "./acpx-qualification.js";
import { ACPX_QUALIFICATION_ENV, ANTIGRAVITY_CANDIDATE_ADMISSION_ENV, resolveAcpxQualification, resolveKimiCandidateAdmission } from "./acpx-qualification.js";
import { QUALIFIED_ACPX_PROFILES, KIMI_DISTRIBUTION_CLOSURE_SHA256 } from "../../vendor/paperclip-runner/index.js";
import type { NativeExecutionInput } from "../../vendor/paperclip-runner/index.js";
import { resolvePaperclipRunnerProviderProfile, resolvePaperclipRunnerNativeProviderInput } from "./provider-profile.js";
@@ -9,6 +10,20 @@ const authorize = (value: unknown) => ({ [ACPX_QUALIFICATION_ENV]: JSON.stringif
describe("host ACPX qualification admission", () => {
afterEach(() => vi.unstubAllEnvs());
it.each([undefined, "malformed", JSON.stringify(kimiAdmission("linux", "x64"))])("admits released Kimi without candidate authority (%s)", authority => {
vi.stubEnv(ANTIGRAVITY_CANDIDATE_ADMISSION_ENV, authority);
vi.stubEnv(ACPX_QUALIFICATION_ENV, "malformed-unrelated-authority");
const config = { provider: "acpx", acpxAgent: "kimi", model: "kimi-for-coding" };
expect(QUALIFIED_ACPX_PROFILES.kimi.agentProfileVersion).toBe(17);
expect(QUALIFIED_ACPX_PROFILES.kimi.qualificationStatus).not.toBe("pending");
expect(resolveKimiCandidateAdmission({ companyId: "candidate-company", model: config.model })).toBeUndefined();
expect(resolvePaperclipRunnerProviderProfile(config, { companyId: "candidate-company" })).toMatchObject(config);
expect(resolvePaperclipRunnerNativeProviderInput({ companyId: "candidate-company", backend: "acpx_runtime", adapterConfig: config }))
.toMatchObject({ ...config, acpxPermissionMode: "approve-all" });
expect(() => resolvePaperclipRunnerProviderProfile({ ...config, model: "" }))
.toThrow(expect.objectContaining({ code: "paperclip_runner_acpx_model_required" }));
});
it.each([undefined, "malformed", JSON.stringify([{ agent: "antigravity", model: "exact-model" }])])(
"keeps pending Antigravity closed despite host authorization (%s)", host => {
vi.stubEnv(ACPX_QUALIFICATION_ENV, host);
@@ -97,3 +112,9 @@ it.each([undefined, "off", "low", "high", "max"] as const)("validates saved Pi t
expect(resolvePaperclipRunnerNativeProviderInput({ backend: "acpx_runtime", adapterConfig })).toMatchObject({ piThinkingLevel: piThinkingLevel ?? "low" });
for (const alias of ["medium", "minimal", "xhigh"]) expect(() => resolvePaperclipRunnerProviderProfile({ ...adapterConfig, piThinkingLevel: alias })).toThrow(expect.objectContaining({ code: "paperclip_runner_pi_thinking_invalid" }));
});
function kimiAdmission(platform: "darwin" | "linux", architecture: "arm64" | "x64") {
return { schema: "paperclip.acpx-candidate-admission.v1", companyId: "candidate-company", agent: "kimi", model: "kimi-for-coding",
platform, architecture, expiresAt: Date.now() + 60_000, profileDigest: QUALIFIED_ACPX_PROFILES.kimi.commandDigest,
nativeClosureDigest: `sha256:${KIMI_DISTRIBUTION_CLOSURE_SHA256[`${platform}-${architecture}` as "darwin-arm64" | "linux-x64"]}` };
}
@@ -1,5 +1,5 @@
import { PAPERCLIP_RUNNER_ACPX_PROFILES } from "@paperclipai/adapter-utils";
import { parseAcpxCandidateAdmission, type AcpxCandidateAdmission, type NativeExecutionInput } from "../../vendor/paperclip-runner/index.js";
import { parseAcpxCandidateAdmission, QUALIFIED_ACPX_PROFILES, type AcpxCandidateAdmission, type NativeExecutionInput } from "../../vendor/paperclip-runner/index.js";
/** Trusted operator startup authority, separate from saved agent configuration. */
export const ANTIGRAVITY_CANDIDATE_ADMISSION_ENV = "PAPERCLIP_ACPX_CANDIDATE_ADMISSION";
@@ -23,9 +23,31 @@ export function resolveAntigravityCandidateAdmission(
});
}
export function resolveKimiCandidateAdmission(
input: { companyId?: string; model?: string; target?: { platform: string; architecture: string } },
hostEnvironment: NodeJS.ProcessEnv = process.env,
): AcpxCandidateAdmission | undefined {
if (QUALIFIED_ACPX_PROFILES.kimi.qualificationStatus !== "pending") return undefined;
const encoded = hostEnvironment[ANTIGRAVITY_CANDIDATE_ADMISSION_ENV];
if (!encoded || !input.companyId) return undefined;
let value: unknown;
try { value = JSON.parse(encoded); } catch { throw new Error("Invalid Kimi candidate admission"); }
if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("Invalid Kimi candidate admission");
const record = value as Record<string, unknown>;
if (record.companyId !== input.companyId || record.agent !== "kimi") return undefined;
const admission = parseAcpxCandidateAdmission(value, {
companyId: input.companyId, agent: "kimi",
model: input.model ?? (typeof record.model === "string" ? record.model : ""),
});
if (input.target && (admission.platform !== input.target.platform || admission.architecture !== input.target.architecture)) {
throw new Error("Kimi candidate admission does not match the execution target");
}
return admission;
}
/** Operator-only admission for an exact candidate/model during qualification. */
export const ACPX_QUALIFICATION_ENV = "PAPERCLIP_RUNNER_ACPX_QUALIFICATION";
export type AcpxQualificationCandidate = "cursor" | "copilot" | "pi";
export type AcpxQualificationCandidate = "cursor" | "copilot" | "pi" | "kimi";
export function resolveAcpxQualification(
provider: NativeExecutionInput["provider"] | { kind: "acpx"; agent: string; model: string },
@@ -45,14 +67,14 @@ export function resolveAcpxQualification(
if (!encoded) return undefined;
let entries: unknown;
try { entries = JSON.parse(encoded); } catch { throw new Error("Invalid ACPX qualification authorization"); }
if (!Array.isArray(entries) || entries.length < 1 || entries.length > 3) {
if (!Array.isArray(entries) || entries.length < 1 || entries.length > 4) {
throw new Error("Invalid ACPX qualification authorization");
}
const agents = new Set<string>();
for (const entry of entries) {
if (!entry || typeof entry !== "object" || Array.isArray(entry)
|| Object.keys(entry).sort().join(",") !== "agent,model"
|| !["cursor", "copilot", "pi"].includes(entry.agent)
|| !["cursor", "copilot", "pi", "kimi"].includes(entry.agent)
|| typeof entry.model !== "string" || !entry.model.trim() || entry.model !== entry.model.trim()
|| agents.has(entry.agent)) throw new Error("Invalid ACPX qualification authorization");
agents.add(entry.agent);
@@ -0,0 +1,149 @@
import { randomUUID } from "node:crypto";
import { and, eq } from "drizzle-orm";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { agents, authUsers, companies, completionContracts, createDb, heartbeatRunEvents, heartbeatRuns, issueComments, issues,
nativeRunFinalizations, nativeRunResults, type Db } from "@paperclipai/db";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js";
import { NativeRunCoordinatorStore } from "./native-run-coordinator-store.js";
import { finalizeNativeRun, recoverCommittedNativeChatResponse } from "./native-run-finalizer.js";
import { nativeResponsePresentationWork } from "./native-response-presentation-work.js";
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_AGENT_CHAT_RESPONSE_TEST_DATABASE_URL");
const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport();
describe.skipIf(!support.supported)("committed Agent Chat reply recovery", () => {
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
let db: Db;
beforeAll(async () => {
if (!external) database = await startEmbeddedPostgresTestDatabase("agent-chat-response-");
db = createDb(external ?? database!.connectionString);
}, 60_000);
afterAll(async () => { await db?.$client.end(); await database?.cleanup(); });
async function fixture(conversation = true) {
const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID();
const sessionId = randomUUID(), runnerId = randomUUID(), userId = `fixture-${randomUUID()}`, wakeId = randomUUID();
await db.insert(authUsers).values({ id: userId, name: "Conversation owner", email: `${userId}@example.test`,
createdAt: new Date("2020-01-01T00:00:00Z"), updatedAt: new Date("2020-01-01T00:00:00Z") });
await db.insert(companies).values({ id: companyId, name: "Reply recovery fixture", issuePrefix: companyId.slice(0, 8) });
await db.insert(agents).values({ id: agentId, companyId, name: "Conversation agent", adapterType: "paperclip_runner" });
await db.insert(issues).values({ id: issueId, companyId, title: "Conversation reply", status: "in_progress", assigneeAgentId: agentId,
...(conversation ? { conversationAgentId: agentId, conversationUserId: userId, conversationState: "active", conversationSessionGeneration: 0 } : {}) });
await db.insert(issueComments).values({ id: wakeId, companyId, issueId, authorUserId: userId, body: "Hire one teammate and tell me when ready.",
createdAt: new Date("2020-01-01T00:00:00Z") });
const [contract] = await db.insert(completionContracts).values({ companyId, issueId, revision: 1,
schemaVersion: "paperclip.completion-contract.v1", policyVersion: "test", risk: "standard", completionAuthority: "server_arbiter",
incompleteCriteriaPolicy: "preserve_non_terminal", contractJson: { revision: "test", objective: "Answer the user", criteria: [] },
canonicalSha256: "fixture-conversation-contract", createdByActorType: "system", createdByActorId: "test" }).returning();
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: issueId, nativeSessionId: sessionId,
runnerInstanceId: runnerId, completionContractId: contract.id, completionContractSha256: contract.canonicalSha256,
status: "running", runtimeMode: "native", responsibleUserId: userId,
contextSnapshot: { issueId, source: "issue.comment", wakeCommentId: wakeId, ...(conversation ? { conversationSessionGeneration: 0 } : {}) } });
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId));
const port = new NativeRunCoordinatorStore(db, { companyId, issueId, runId, agentId, normalizedSessionId: sessionId,
runnerSourceInstanceId: runnerId, completionContractId: contract.id, completionContractSha256: contract.canonicalSha256,
completionContractRevision: "test", completionContractCriterionIds: [] });
const reply = "The teammate is hired and ready for your next instruction.";
const prpEvent = { runId, normalizedSessionId: sessionId, turnId: "hire-turn", sourceEventId: "fixture-final-answer",
payload: { kind: "agentMessage", channel: "final", providerPhase: "final", text: reply } };
await db.insert(heartbeatRunEvents).values({ companyId, runId, agentId, seq: 170, eventType: "item.completed", payload: { prpEvent } });
await port.completeRun({ result: { schema: "paperclip.run_result.v1", reportedWorkDisposition: "done",
summary: "A completed hire, with its final message preserved separately.", completionClaim: { contractRevision: "test",
objectiveSatisfied: true, criteria: [], remainingWork: [] }, evidence: [], verification: [], attentionRequests: [], artifacts: [] },
terminal: { schema: "paperclip.prp.terminal.v1", runTerminalState: "succeeded", turnTerminalState: "completed", reportedWorkDisposition: "done" }, turnId: "hire-turn" });
await finalizeNativeRun({ db, runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: false });
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date("2020-01-01T00:02:00Z") }).where(eq(heartbeatRuns.id, runId));
expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)))[0].phase).toBe("committed");
const input = { companyId, issueId, runId };
return { ...input, input, agentId, userId, wakeId, prpEvent, reply };
}
const comments = (runId: string) => db.select().from(issueComments).where(eq(issueComments.createdByRunId, runId));
const issue = async (issueId: string) => (await db.select().from(issues).where(eq(issues.id, issueId)))[0];
it("finds a committed final Agent Chat turn, persists the exact reply once and only then settles waiting", async () => {
const f = await fixture();
expect(await comments(f.runId)).toEqual([]);
expect(await issue(f.issueId)).toMatchObject({ status: "in_progress", conversationState: "active" });
const work = nativeResponsePresentationWork(db), entries = await work.load(f.input);
expect(entries).toEqual([expect.objectContaining({ id: f.runId })]);
await expect(work.check(entries[0])).resolves.toBeNull();
expect(await comments(f.runId)).toEqual([expect.objectContaining({ body: f.reply, authorAgentId: f.agentId, createdByRunId: f.runId })]);
expect(await issue(f.issueId)).toMatchObject({ status: "in_review", conversationState: "waiting" });
expect(await work.load(f.input)).toEqual([]);
await expect(recoverCommittedNativeChatResponse(db, f.input)).resolves.toBe(false);
expect(await comments(f.runId)).toHaveLength(1);
});
it("keeps a later user message actionable after recovering the earlier exact reply", async () => {
const f = await fixture();
await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId, authorUserId: f.userId, body: "Another instruction",
createdAt: new Date("2020-01-01T00:01:00Z") });
await expect(recoverCommittedNativeChatResponse(db, f.input)).resolves.toBe(true);
expect(await comments(f.runId)).toHaveLength(1);
expect(await issue(f.issueId)).toMatchObject({ status: "in_progress", conversationState: "active" });
});
it("does not duplicate a reply under concurrent recovery", async () => {
const f = await fixture();
expect((await Promise.all([recoverCommittedNativeChatResponse(db, f.input), recoverCommittedNativeChatResponse(db, f.input)])).sort())
.toEqual([false, true]);
expect(await comments(f.runId)).toHaveLength(1);
});
it.each(["skip", "generation", "agent", "user", "execution", "waiting", "turn", "event-run", "session", "channel", "kind", "digest", "partial"])(
"keeps %s mismatches unchanged instead of publishing an unauthorized or incomplete reply", async fault => {
const f = await fixture();
if (["skip", "generation"].includes(fault)) await db.update(heartbeatRuns).set({ contextSnapshot: { issueId: f.issueId,
source: "issue.comment", wakeCommentId: f.wakeId, conversationSessionGeneration: fault === "generation" ? 1 : 0,
...(fault === "skip" ? { skipIssueComment: true } : {}) } }).where(eq(heartbeatRuns.id, f.runId));
if (fault === "user") await db.update(heartbeatRuns).set({ responsibleUserId: "another-user" }).where(eq(heartbeatRuns.id, f.runId));
if (fault === "agent") {
const other = randomUUID(); await db.insert(agents).values({ id: other, companyId: f.companyId, name: "Other agent" });
await db.update(issues).set({ conversationAgentId: other, assigneeAgentId: other }).where(eq(issues.id, f.issueId));
}
if (fault === "execution") {
const other = randomUUID(); await db.insert(heartbeatRuns).values({ id: other, companyId: f.companyId, agentId: f.agentId });
await db.update(issues).set({ executionRunId: other }).where(eq(issues.id, f.issueId));
}
if (fault === "waiting") await db.update(issues).set({ executionRunId: null, status: "in_review", conversationState: "waiting" }).where(eq(issues.id, f.issueId));
if (["turn", "event-run", "session", "channel", "kind"].includes(fault)) {
const event = structuredClone(f.prpEvent);
if (fault === "turn") event.turnId = "foreign-turn";
if (fault === "event-run") event.runId = randomUUID();
if (fault === "session") event.normalizedSessionId = randomUUID();
if (fault === "channel") event.payload.channel = "analysis";
if (fault === "kind") event.payload.kind = "reasoning";
await db.update(heartbeatRunEvents).set({ payload: { prpEvent: event } }).where(and(eq(heartbeatRunEvents.runId, f.runId), eq(heartbeatRunEvents.seq, 170)));
}
if (fault === "digest") await db.update(nativeRunResults).set({ canonicalSha256: "invalid-digest" }).where(eq(nativeRunResults.runId, f.runId));
if (fault === "partial") await db.update(nativeRunFinalizations).set({ phase: "copying_back" }).where(eq(nativeRunFinalizations.runId, f.runId));
const before = await issue(f.issueId);
await expect(recoverCommittedNativeChatResponse(db, f.input)).resolves.toBe(false);
expect(await comments(f.runId)).toEqual([]); expect(await issue(f.issueId)).toEqual(before);
},
);
it.each([false, true])("preserves a selected reply without replacing or resurrecting it (deleted=%s)", async deleted => {
const f = await fixture();
const [saved] = await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId, authorAgentId: f.agentId,
createdByRunId: f.runId, body: "Already selected", ...(deleted ? { deletedAt: new Date("2020-01-01T00:03:00Z") } : {}) }).returning();
await db.update(heartbeatRuns).set({ resultJson: { presentationDecision: { commentId: saved.id, commentAction: "create" } } }).where(eq(heartbeatRuns.id, f.runId));
await expect(recoverCommittedNativeChatResponse(db, f.input)).resolves.toBe(false);
expect(await comments(f.runId)).toEqual([saved]);
});
it("never crosses company scope and keeps ordinary internal task response recovery intact", async () => {
const f = await fixture(false);
await expect(recoverCommittedNativeChatResponse(db, { ...f.input, companyId: randomUUID() })).resolves.toBe(false);
const work = nativeResponsePresentationWork(db), entries = await work.load(f.input);
expect(entries).toHaveLength(1); await expect(work.check(entries[0])).resolves.toBeNull();
expect(await comments(f.runId)).toEqual([expect.objectContaining({ body: f.reply })]);
expect((await issue(f.issueId)).conversationAgentId).toBeNull();
});
it("rolls back a partial reply write without changing the waiting state or consuming recovery", async () => {
const f = await fixture(), abort = new Error("fixture-commit-aborted");
const failing = new Proxy(db, { get(target, key) { return key === "transaction"
? (callback: Parameters<Db["transaction"]>[0]) => db.transaction(async tx => { await callback(tx); throw abort; })
: Reflect.get(target, key); } });
await expect(recoverCommittedNativeChatResponse(failing, f.input)).rejects.toBe(abort);
expect(await comments(f.runId)).toEqual([]);
expect(await issue(f.issueId)).toMatchObject({ status: "in_progress", conversationState: "active" });
await expect(recoverCommittedNativeChatResponse(db, f.input)).resolves.toBe(true);
expect(await comments(f.runId)).toHaveLength(1);
});
});
@@ -0,0 +1,61 @@
import type { Db } from "@paperclipai/db";
import { sql } from "drizzle-orm";
import { inspectNativeRunnerUpgradeHeader } from "./native-runner-checkpoint.js";
/** An explicit, paginated diagnostic. Never called by an idle/background scan.
* This inventories selected retained task-session pointers, not all history.
* Header evidence is preliminary: full snapshot/journal validation still occurs
* only when ordinary admission selects its exact continuation source. */
export async function readNativeControllerRecoveryInventory(db: Db, input: { afterId?: string; limit?: number } = {}) {
if (input.limit !== undefined && !Number.isFinite(input.limit)) throw new Error("native_recovery_inventory_limit_invalid");
const limit = Math.min(128, Math.max(1, Math.floor(input.limit ?? 128)));
const rows = await db.transaction(async tx => {
await tx.execute(sql`set local statement_timeout = '2s'`);
return tx.execute<{
id: string; runId: string | null; companyId: string; issueId: string | null; sessionId: string | null;
runnerId: string | null; metadata: Record<string, unknown> | null; material: Record<string, unknown> | null;
pending: boolean;
}>(sql`with selected as (
select id, company_id, agent_id, last_run_id from agent_task_sessions
where adapter_type = 'paperclip_runner' ${input.afterId ? sql`and id > ${input.afterId}::uuid` : sql``}
order by id limit ${limit + 1}
) select s.id, s.company_id as "companyId", r.id as "runId", r.native_issue_id as "issueId",
r.native_session_id as "sessionId", r.runner_instance_id as "runnerId",
case when pg_column_size(r.runner_profile_json->'nativeControllerRecovery') <= 4096
then r.runner_profile_json->'nativeControllerRecovery' else null end as metadata,
case when pg_column_size(c.material) <= 262144 and sum(pg_column_size(c.material)) over () <= 8388608
then c.material else null end as material,
(r.finished_at is null or r.status not in ('succeeded', 'failed', 'cancelled', 'interrupted', 'timed_out')
or f.run_id is null or f.lease_owner is not null or f.recovery_state is not null
or f.phase not in ('committed', 'terminal_failure')) is true as pending
from selected s left join heartbeat_runs r on r.id = s.last_run_id and r.company_id = s.company_id and r.agent_id = s.agent_id
left join native_run_finalizations f on f.run_id = r.id and f.company_id = s.company_id
left join native_runner_checkpoints c on c.run_id = r.id and c.company_id = s.company_id
and c.issue_id = r.native_issue_id and c.part = 'header'
order by s.id`);
}, { accessMode: "read only" });
const truncated = rows.length > limit;
const page = rows.slice(0, limit);
const reasons: Record<string, number> = {};
let selectedRetainedEvidence = 0, outstandingObligations = 0;
for (const row of page) {
let reason: string | null = null;
if (row.pending) { outstandingObligations++; reason = "persistence_or_handoff_pending"; }
else if (!row.runId || !row.issueId || !row.sessionId) reason = "selected_binding_missing";
else if (row.metadata?.schema !== "paperclip.native-controller-recovery.v1") reason = "metadata_unsupported_or_missing";
else {
const checkpoint = row.metadata.checkpoint as Record<string, unknown> | null;
const header = row.material ? inspectNativeRunnerUpgradeHeader({ companyId: row.companyId, issueId: row.issueId, runId: row.runId }, row.material) : null;
if (!checkpoint || checkpoint.runId !== row.runId || checkpoint.runnerInstanceId !== row.runnerId
|| checkpoint.normalizedSessionId !== row.sessionId || !header
|| header.runnerInstanceId !== row.runnerId || header.normalizedSessionId !== row.sessionId) reason = "checkpoint_binding_missing";
else selectedRetainedEvidence++;
}
if (reason) reasons[reason] = (reasons[reason] ?? 0) + 1;
}
return { version: 1 as const, observedAt: new Date().toISOString(), inspected: page.length,
coverage: truncated || input.afterId ? "partial" as const : "selected_task_session_pointers" as const,
truncated, nextAfterId: truncated ? page.at(-1)!.id : null,
selectedRetainedEvidence, outstandingObligations, reasons,
fullPayloadsValidated: false as const };
}
@@ -0,0 +1,165 @@
import { randomUUID } from "node:crypto";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { eq } from "drizzle-orm";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { agentTaskSessions, agents, companies, createDb, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults } from "@paperclipai/db";
import { DurablePrpControlPlane, type PersistedNativeSession } from "../../vendor/paperclip-runner/index.js";
import { startRecoveryPostgres, recoveryPostgresSupport } from "../../__tests__/helpers/recovery-postgres.js";
import { createIsolatedTestHome } from "../../__tests__/helpers/isolated-test-home.js";
import { prepareNativeHeartbeatRun } from "./prepare-native-run.js";
import { nativeRunnerCheckpointWriter } from "./native-runner-checkpoint.js";
import { createNativeControllerRecovery, deferNativeControllerRecovery, importNativeControllerRecovery, NATIVE_RETAINED_OWNER_WAIT, resolveNativeControllerRecovery } from "./native-controller-recovery.js";
import { claimNativeSessionResumptions } from "./native-finalization-reconciler.js";
import { claimNativeRestartRecoveries } from "./native-restart-recovery.js";
import { readNativeControllerRecoveryInventory } from "./native-controller-recovery-inventory.js";
describe.skipIf(!recoveryPostgresSupport.supported)("native controller durable discovery and import", () => {
let database: Awaited<ReturnType<typeof startRecoveryPostgres>>;
let db: ReturnType<typeof createDb>;
let home: ReturnType<typeof createIsolatedTestHome>;
let root: string;
const previousHome = process.env.PAPERCLIP_HOME;
const companyId = randomUUID(), agentId = randomUUID();
const compatibility = { configDigest: `sha256:${"a".repeat(64)}`, processConfigDigest: null };
beforeAll(async () => {
home = createIsolatedTestHome();
process.env.PAPERCLIP_HOME = home.home;
root = await mkdtemp(join(tmpdir(), "native-controller-discovery-"));
database = await startRecoveryPostgres("native-controller-discovery-");
db = createDb(database.connectionString);
await db.insert(companies).values({ id: companyId, name: "Controller discovery", issuePrefix: "NCD" });
await db.insert(agents).values({ id: agentId, companyId, name: "Runner", adapterType: "paperclip_runner", adapterConfig: { provider: "codex" } });
}, 60_000);
afterAll(async () => {
await database?.cleanup();
if (root) await rm(root, { recursive: true, force: true });
home?.cleanup();
if (previousHome === undefined) delete process.env.PAPERCLIP_HOME;
else process.env.PAPERCLIP_HOME = previousHome;
});
async function seed() {
const issueId = randomUUID(), runId = randomUUID();
await db.insert(issues).values({ id: issueId, companyId, title: "Continue retained execution", assigneeAgentId: agentId, status: "in_progress" });
const [run] = await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running", runtimeMode: "native", nativeIssueId: issueId }).returning();
const native = await prepareNativeHeartbeatRun({ db, run, issue: { id: issueId, title: "Continue retained execution", description: null, reviewPolicy: null }, environmentLeaseId: randomUUID() });
const owner = { companyId, issueId, runId, authorityRunId: runId, bootId: randomUUID(), generation: 1, leaseOwner: randomUUID() };
await db.insert(nativeRunFinalizations).values({ companyId, issueId, runId, phase: "observed", controllerBootId: owner.bootId,
controllerGeneration: 1, leaseOwner: owner.leaseOwner, leaseExpiresAt: new Date(Date.now() + 60_000) });
const core = new DurablePrpControlPlane({ stateDirectory: join(root, runId),
identity: { runId, normalizedSessionId: native.normalizedSessionId, runnerInstanceId: native.runnerInstanceId,
environmentLeaseId: native.environmentLeaseId, turnId: native.turnId, itemId: native.itemId },
expectedRunnerVersion: "discovery-fixture", expectedRunnerDigest: `sha256:${"a".repeat(64)}` });
core.setCheckpointPersistence(nativeRunnerCheckpointWriter(db, owner));
await core.flushCheckpoint();
await core.stop();
const snapshot: PersistedNativeSession = { backendKind: "mock", sessionId: native.normalizedSessionId,
identity: { companyId, agentId, issueId, runId, sessionId: native.normalizedSessionId } };
const [prepared] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
const [result] = await db.insert(nativeRunResults).values({ companyId, issueId, runId,
completionContractId: prepared.completionContractId!, serverFingerprint: "fixture", schemaStatus: "accepted",
resultJson: {}, canonicalSha256: "fixture" }).returning();
await db.update(nativeRunFinalizations).set({ phase: "committed", resultId: result.id, leaseOwner: null, leaseExpiresAt: null }).where(eq(nativeRunFinalizations.runId, runId));
const profile = { ...prepared.runnerProfileJson, sessionCheckpoint: snapshot };
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date(), runnerProfileJson: profile }).where(eq(heartbeatRuns.id, runId));
const scope = { companyId, agentId, issueId, runId, normalizedSessionId: native.normalizedSessionId };
return { scope, native, profile, snapshot, owner,
supervisor: { schema: "paperclip.native-session-supervisor.v1", ...compatibility, snapshot } };
}
it("imports a retained source once, rejects stale input, and resolves it without any local projection", async () => {
const fixture = await seed();
const input = { ...fixture.scope, expectedConfigDigest: compatibility.configDigest, supervisor: fixture.supervisor };
expect(await importNativeControllerRecovery(db, { ...input, supervisor: { ...input.supervisor, snapshot: { ...fixture.snapshot, providerSessionId: "stale" } } }))
.toEqual({ kind: "missing_evidence", reason: "stale_supervisor_snapshot" });
expect(await importNativeControllerRecovery(db, input)).toEqual({ kind: "imported" });
expect(await importNativeControllerRecovery(db, input)).toEqual({ kind: "unchanged" });
await rm(join(root, fixture.scope.runId), { recursive: true });
const currentRunId = randomUUID();
await db.insert(heartbeatRuns).values({ id: currentRunId, companyId, agentId, nativeIssueId: fixture.scope.issueId,
nativeSessionId: fixture.scope.normalizedSessionId, status: "running", runtimeMode: "native" });
const recovered = await resolveNativeControllerRecovery(db, { ...fixture.scope, currentRunId });
expect(recovered.kind).toBe("recoverable");
if (recovered.kind !== "recoverable") throw new Error(JSON.stringify(recovered));
expect(recovered.snapshot).toEqual(fixture.snapshot);
expect(recovered.checkpoint.recoveryEvidence.runnerInstanceId).toBe(fixture.native.runnerInstanceId);
const deadline = new Date(Date.now() + 60_000);
await db.update(nativeRunFinalizations).set({ leaseOwner: "other-controller", leaseExpiresAt: deadline }).where(eq(nativeRunFinalizations.runId, fixture.scope.runId));
expect(await importNativeControllerRecovery(db, input)).toEqual({ kind: "waiting_for_owner", deadline });
expect(await resolveNativeControllerRecovery(db, { ...fixture.scope, currentRunId })).toEqual({ kind: "waiting_for_owner", deadline });
});
it("holds mismatched snapshots and runner bindings instead of falling back to fresh execution", async () => {
const fixture = await seed(), currentRunId = randomUUID();
const metadata = createNativeControllerRecovery(fixture.snapshot, compatibility, { runId: fixture.scope.runId,
runnerInstanceId: fixture.native.runnerInstanceId, normalizedSessionId: fixture.scope.normalizedSessionId });
await db.insert(heartbeatRuns).values({ id: currentRunId, companyId, agentId, nativeIssueId: fixture.scope.issueId,
nativeSessionId: fixture.scope.normalizedSessionId, status: "running", runtimeMode: "native" });
for (const envelope of [{ ...metadata, snapshotSha256: "wrong" }, { ...metadata, checkpoint: { ...metadata.checkpoint!, runnerInstanceId: randomUUID() } }]) {
await db.update(heartbeatRuns).set({ runnerProfileJson: { ...fixture.profile, nativeControllerRecovery: envelope } }).where(eq(heartbeatRuns.id, fixture.scope.runId));
expect(await resolveNativeControllerRecovery(db, { ...fixture.scope, currentRunId }))
.toMatchObject({ kind: "missing_evidence", reason: "snapshot_digest_or_checkpoint_binding" });
}
});
it("keeps an unstarted follow-up on the same run until the retained owner's deadline", async () => {
const fixture = await seed(), runId = randomUUID();
const scope = { ...fixture.scope, runId };
const deadline = new Date(Date.now() + 60_000);
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: scope.issueId,
nativeSessionId: scope.normalizedSessionId, status: "running", runtimeMode: "native",
// The admission boundary has already stored its input; the deferral
// and resumption owners never execute this transport-independent fixture.
runnerProfileJson: { nativeExecutionInput: { binding: scope } } });
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, scope.issueId));
await db.insert(nativeRunFinalizations).values({ companyId, issueId: scope.issueId, runId, phase: "observed" });
await deferNativeControllerRecovery(db, { ...scope, retryAt: deadline });
const read = () => db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId));
expect((await read())[0]).toMatchObject({ phase: "retryable_failure", attempt: 0, leaseOwner: null,
nextAttemptAt: deadline, failureCode: NATIVE_RETAINED_OWNER_WAIT });
expect(await claimNativeSessionResumptions({ db, runnerInstanceId: "successor", runIds: [runId] })).toEqual([]);
// Even after expiry, restart adoption cannot reinterpret this pre-provider
// wait as a crashed provider. The ordinary resumption owner consumes it.
expect(await claimNativeRestartRecoveries({ db, runIds: [runId], restartKind: "hard", now: deadline })).toEqual([]);
const claims = await claimNativeSessionResumptions({ db, runnerInstanceId: "successor", runIds: [runId], now: deadline });
expect(claims).toHaveLength(1);
const nextDeadline = new Date(deadline.getTime() + 60_000);
await deferNativeControllerRecovery(db, { ...scope, leaseOwner: claims[0].leaseOwner, retryAt: nextDeadline });
expect((await read())[0]).toMatchObject({ attempt: 0, leaseOwner: null, nextAttemptAt: nextDeadline });
await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, runId));
expect(await claimNativeSessionResumptions({ db, runnerInstanceId: "successor", runIds: [runId], now: nextDeadline })).toEqual([]);
});
it.each(["live-owner", "expired-owner", "provider-attempt", "stopped", "operator-stop"])("does not publish an admission wait after authority changes: %s", async scenario => {
const fixture = await seed(), runId = randomUUID();
const scope = { ...fixture.scope, runId };
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: scope.issueId,
status: scenario === "stopped" ? "cancelled" : "running", runtimeMode: "native",
resultJson: scenario === "operator-stop" ? { cancellation: { source: "operator", expected: true,
initiator: { type: "user" }, reason: "Stop", recordedAt: new Date().toISOString() } } : null,
runnerProfileJson: { nativeExecutionInput: { binding: scope } } });
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, scope.issueId));
await db.insert(nativeRunFinalizations).values({ companyId, issueId: scope.issueId, runId, phase: "observed",
attempt: scenario === "provider-attempt" ? 1 : 0,
leaseOwner: scenario.endsWith("-owner") ? "other-controller" : null,
leaseExpiresAt: scenario === "expired-owner" ? new Date(Date.now() - 1_000) : null });
await expect(deferNativeControllerRecovery(db, { ...scope,
leaseOwner: scenario === "expired-owner" ? "other-controller" : undefined,
retryAt: new Date(Date.now() + 60_000) }))
.rejects.toThrow("wait_authority_changed");
expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)))[0].phase).toBe("observed");
});
it("reports bounded retained-pointer coverage without loading historical payloads", async () => {
const fixture = await seed();
await importNativeControllerRecovery(db, { ...fixture.scope, expectedConfigDigest: compatibility.configDigest, supervisor: fixture.supervisor });
await db.insert(agentTaskSessions).values([0, 1].map(index => ({ companyId, agentId, taskKey: `${fixture.scope.issueId}-${index}`,
adapterType: "paperclip_runner", lastRunId: fixture.scope.runId })));
const first = await readNativeControllerRecoveryInventory(db, { limit: 1 });
expect(first).toMatchObject({ inspected: 1, coverage: "partial", truncated: true, selectedRetainedEvidence: 1, fullPayloadsValidated: false });
const next = await readNativeControllerRecoveryInventory(db, { limit: 1, afterId: first.nextAfterId! });
expect(next).toMatchObject({ inspected: 1, coverage: "partial", truncated: false, selectedRetainedEvidence: 1 });
});
});
@@ -0,0 +1,43 @@
import { describe, expect, it } from "vitest";
import { createNativeControllerRecovery, readNativeControllerRecovery } from "./native-controller-recovery.js";
import { nativeSha256 } from "./canonical.js";
import { redactRegisteredSecretValues } from "../run-secret-redaction.js";
import type { PersistedNativeSession } from "../../vendor/paperclip-runner/index.js";
const snapshot: PersistedNativeSession = { backendKind: "mock", sessionId: "provider",
identity: { companyId: "company", agentId: "agent", issueId: "issue", runId: "run", sessionId: "session" } };
const compatibility = { configDigest: `sha256:${"a".repeat(64)}`, processConfigDigest: null };
const binding = { runId: "run", runnerInstanceId: "runner", normalizedSessionId: "session" };
describe("native controller recovery envelope", () => {
it("binds the stored snapshot, including redaction, without serializing its contents twice", () => {
const stored = { ...snapshot, providerSessionId: "[REDACTED]" };
const envelope = createNativeControllerRecovery(stored, compatibility, binding);
expect(envelope.snapshotSha256).toBe(nativeSha256(stored));
expect(readNativeControllerRecovery(envelope, JSON.parse(JSON.stringify(stored)))).toEqual(envelope);
expect(readNativeControllerRecovery(envelope, { ...stored, providerSessionId: "secret-before-redaction" })).toBeNull();
expect(JSON.stringify(envelope)).not.toContain("[REDACTED]");
expect(Buffer.byteLength(JSON.stringify(envelope))).toBeLessThan(4096);
});
it("does not certify a missing PRP publication or unknown format", () => {
const pending = createNativeControllerRecovery(snapshot, compatibility, null);
expect(readNativeControllerRecovery(pending, snapshot)).toBeNull();
const published = createNativeControllerRecovery(snapshot, compatibility, binding);
expect(readNativeControllerRecovery({ ...published, schema: "future" }, snapshot)).toBeNull();
expect(readNativeControllerRecovery({ ...published, processConfigDigest: "unverified" }, snapshot)).toBeNull();
});
it("enforces the metadata byte bound on both write and read", () => {
expect(() => createNativeControllerRecovery(snapshot, { ...compatibility, configDigest: "x".repeat(4096) }, binding))
.toThrow("metadata_too_large");
expect(readNativeControllerRecovery({ ...createNativeControllerRecovery(snapshot, compatibility, binding), extra: "x".repeat(4096) }, snapshot))
.toBeNull();
});
it("keeps compatibility metadata out of public run projections", () => {
expect(redactRegisteredSecretValues({ runnerProfileJson: { preserved: true,
nativeControllerRecovery: createNativeControllerRecovery(snapshot, compatibility, binding) } }, []))
.toEqual({ runnerProfileJson: { preserved: true } });
});
});
@@ -0,0 +1,270 @@
import { and, eq, ne, sql } from "drizzle-orm";
import { environmentLeases, heartbeatRuns, issues, nativeRunFinalizations, nativeRunnerCheckpoints, type Db } from "@paperclipai/db";
import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target";
import type { DurablePrpCheckpoint, PersistedNativeSession } from "../../vendor/paperclip-runner/index.js";
import { notifyNativeFinalizationWork } from "../native-finalization-notifications.js";
import { readRunCancellation } from "../run-cancellation.js";
import { nativeSha256 } from "./canonical.js";
import { findNativeSessionResumeRun, isNativeSessionId, isUnusedNativeSessionBootstrap, nativeSessionProviderEvidence } from "./native-session-resume.js";
import { loadNativeRunnerCheckpoint } from "./native-runner-checkpoint.js";
import { redactRunnerApiPayloads } from "./runner-api-log-redaction.js";
export const NATIVE_CONTROLLER_RECOVERY_MAX_BYTES = 4 * 1024;
/** Private run-profile metadata. No credentials or controller-local paths. */
export interface NativeControllerRecovery {
schema: "paperclip.native-controller-recovery.v1";
snapshotSha256: string;
configDigest: string;
processConfigDigest: string | null;
checkpoint: {
runId: string;
runnerInstanceId: string;
normalizedSessionId: string;
} | null;
}
export type NativeControllerCompatibility = Pick<NativeControllerRecovery, "configDigest" | "processConfigDigest">;
export function createNativeControllerRecovery(
snapshot: PersistedNativeSession,
compatibility: NativeControllerCompatibility,
checkpoint: NativeControllerRecovery["checkpoint"],
): NativeControllerRecovery {
const value: NativeControllerRecovery = {
schema: "paperclip.native-controller-recovery.v1",
snapshotSha256: nativeSha256(snapshot),
configDigest: compatibility.configDigest,
processConfigDigest: compatibility.processConfigDigest,
checkpoint,
};
if (Buffer.byteLength(JSON.stringify(value)) > NATIVE_CONTROLLER_RECOVERY_MAX_BYTES) {
throw new Error("native_controller_recovery_metadata_too_large");
}
return value;
}
function record(value: unknown): Record<string, unknown> {
return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
}
export function readNativeControllerRecovery(value: unknown, snapshot: unknown): NativeControllerRecovery | null {
const envelope = record(value), checkpoint = record(envelope.checkpoint);
if (Buffer.byteLength(JSON.stringify(value) ?? "null") > NATIVE_CONTROLLER_RECOVERY_MAX_BYTES
|| envelope.schema !== "paperclip.native-controller-recovery.v1"
|| typeof envelope.snapshotSha256 !== "string" || envelope.snapshotSha256 !== nativeSha256(snapshot)
|| typeof envelope.configDigest !== "string" || !/^sha256:[a-f0-9]{64}$/.test(envelope.configDigest)
|| !(envelope.processConfigDigest === null || (typeof envelope.processConfigDigest === "string" && /^[a-f0-9]{64}$/.test(envelope.processConfigDigest)))
|| ![checkpoint.runId, checkpoint.runnerInstanceId, checkpoint.normalizedSessionId].every(v => typeof v === "string" && v.length > 0)) return null;
return envelope as unknown as NativeControllerRecovery;
}
export type NativeControllerRecoveryOutcome =
| { kind: "recoverable"; snapshot: PersistedNativeSession; metadata: NativeControllerRecovery; checkpoint: DurablePrpCheckpoint }
| { kind: "waiting_for_owner"; deadline: Date }
| { kind: "missing_evidence" | "unsupported" | "dependency_unavailable"; reason: string; cause?: unknown }
| { kind: "no_prior_execution" };
export const NATIVE_RETAINED_OWNER_WAIT = "native_controller_recovery_waiting_for_owner";
export class NativeControllerRecoveryWaitingError extends Error {
constructor(readonly retryAt: Date) {
super(NATIVE_RETAINED_OWNER_WAIT);
this.name = "NativeControllerRecoveryWaitingError";
}
}
/** Admission has not crossed the provider boundary. Retain this same business
* run on the existing retry owner; never turn an ownership wait into a failure
* or invent a new provider attempt. The usual resumption gates recheck Stop,
* permissions and task ownership when this deadline arrives. */
export async function deferNativeControllerRecovery(db: Db, input: {
companyId: string; agentId: string; issueId: string; runId: string;
leaseOwner?: string; retryAt: Date;
}): Promise<void> {
await db.transaction(async tx => {
const [task] = await tx.select({ executionRunId: issues.executionRunId }).from(issues)
.where(and(eq(issues.companyId, input.companyId), eq(issues.id, input.issueId))).for("update");
const [owner] = await tx.select().from(nativeRunFinalizations).where(and(
eq(nativeRunFinalizations.companyId, input.companyId), eq(nativeRunFinalizations.issueId, input.issueId),
eq(nativeRunFinalizations.runId, input.runId))).for("update");
const [run] = await tx.select().from(heartbeatRuns).where(and(
eq(heartbeatRuns.id, input.runId), eq(heartbeatRuns.companyId, input.companyId),
eq(heartbeatRuns.agentId, input.agentId), eq(heartbeatRuns.nativeIssueId, input.issueId))).for("update");
const [authority] = await tx.select({ runId: nativeRunnerCheckpoints.runId }).from(nativeRunnerCheckpoints)
.where(and(eq(nativeRunnerCheckpoints.companyId, input.companyId), eq(nativeRunnerCheckpoints.runId, input.runId))).limit(1);
const progress = await nativeSessionProviderEvidence(tx as unknown as Db, [input.runId]);
const cancellation = record(run?.resultJson?.nativeCancellation);
const stop = readRunCancellation(run?.resultJson);
const [clock] = await tx.execute<{ now: Date }>(sql`select clock_timestamp() as now`);
if (!run || run.runtimeMode !== "native" || run.status !== "running" || task?.executionRunId !== input.runId
|| !owner || owner.phase !== "observed" || owner.resultId || owner.attempt !== 0
|| owner.controllerGeneration !== 0 || owner.leaseOwner !== (input.leaseOwner ?? null)
|| (input.leaseOwner && (!owner.leaseExpiresAt || owner.leaseExpiresAt <= new Date(clock.now)))
|| run.processPid || run.processGroupId || run.processStartedAt || authority || progress.size
|| !record(run.runnerProfileJson).nativeExecutionInput || run.resultJson?.startupCancellation
|| (stop?.source === "operator" && stop.expected)
|| (["run", "issue"].includes(String(cancellation.scope)) && ["pending", "acknowledged"].includes(String(cancellation.dispatchState)))) {
throw new Error("native_controller_recovery_wait_authority_changed");
}
await notifyNativeFinalizationWork(tx, input);
await tx.update(nativeRunFinalizations).set({
phase: "retryable_failure", leaseOwner: null, leaseExpiresAt: null, controlDeadlineAt: null,
nextAttemptAt: input.retryAt, recoveryState: "resuming_session", failureCode: NATIVE_RETAINED_OWNER_WAIT,
failureDetail: { originalFailureCode: NATIVE_RETAINED_OWNER_WAIT,
message: "Waiting for the retained native controller owner to release its lease.",
recoveryMode: record(run.runnerProfileJson).sessionCheckpoint ? "same_session_resume" : "bootstrap_retry",
providerSessionEstablished: false, providerEventsExist: false },
updatedAt: new Date(),
}).where(eq(nativeRunFinalizations.runId, input.runId));
});
}
/** Bind a read of retained Runner lifecycle to the surviving sandbox. The
* ordinary admission owner still checks current permissions before execution. */
export async function assertNativeRetainedSandboxBinding(db: Db, input: {
companyId: string; issueId: string; runId: string;
target: Extract<AdapterExecutionTarget, { kind: "remote" }>;
}): Promise<void> {
const { target } = input;
if (target.transport !== "sandbox" || target.sandboxLeaseAcquisition?.outcome !== "resumed"
|| !target.providerKey || !target.environmentId || !target.leaseId) {
throw new Error("native_controller_recovery_dependency_unavailable:retained_sandbox");
}
const [owner] = await db.select().from(nativeRunFinalizations).where(and(
eq(nativeRunFinalizations.companyId, input.companyId), eq(nativeRunFinalizations.issueId, input.issueId),
eq(nativeRunFinalizations.runId, input.runId)));
const leases = await db.select({ metadata: environmentLeases.metadata, status: environmentLeases.status }).from(environmentLeases).where(and(
eq(environmentLeases.companyId, input.companyId), eq(environmentLeases.issueId, input.issueId),
eq(environmentLeases.heartbeatRunId, input.runId), eq(environmentLeases.environmentId, target.environmentId),
eq(environmentLeases.provider, target.providerKey), eq(environmentLeases.providerLeaseId, target.sandboxLeaseAcquisition.providerLeaseId)));
if (!owner || owner.leaseOwner || owner.phase !== "committed" || !owner.resultId || owner.recoveryState
|| leases.length !== 1 || leases[0].metadata?.remoteCwd !== target.remoteCwd
|| !["active", "released", "expired"].includes(leases[0].status)) {
throw new Error("native_controller_recovery_missing_evidence:retained_sandbox_binding");
}
}
/** Only the existing bounded selector chooses history. Never search backwards
* again when its selected record is incomplete or incompatible. This operation
* reads evidence; it grants neither command nor acknowledgement authority. */
export async function resolveNativeControllerRecovery(db: Db, input: {
companyId: string; agentId: string; issueId: string; normalizedSessionId: string; currentRunId: string;
}): Promise<NativeControllerRecoveryOutcome> {
if (!isNativeSessionId(input.normalizedSessionId)) return { kind: "missing_evidence", reason: "session_binding" };
const [current] = await db.select({ createdAt: heartbeatRuns.createdAt, processPid: heartbeatRuns.processPid,
processGroupId: heartbeatRuns.processGroupId, processStartedAt: heartbeatRuns.processStartedAt,
runnerProfileJson: heartbeatRuns.runnerProfileJson }).from(heartbeatRuns).where(and(
eq(heartbeatRuns.id, input.currentRunId), eq(heartbeatRuns.companyId, input.companyId),
eq(heartbeatRuns.agentId, input.agentId), eq(heartbeatRuns.nativeIssueId, input.issueId),
eq(heartbeatRuns.nativeSessionId, input.normalizedSessionId)));
if (!current) return { kind: "missing_evidence", reason: "current_binding" };
const source = await findNativeSessionResumeRun(db, { ...input, beforeCreatedAt: current.createdAt, includeProgressBarrier: true });
if (!source) {
// An exhausted candidate window is unknown history, not fresh-bootstrap
// proof. Read only IDs, and retain the selector's existing 32-row boundary.
const prior = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(
ne(heartbeatRuns.id, input.currentRunId), eq(heartbeatRuns.companyId, input.companyId),
eq(heartbeatRuns.agentId, input.agentId), eq(heartbeatRuns.nativeIssueId, input.issueId),
eq(heartbeatRuns.nativeSessionId, input.normalizedSessionId))).limit(33);
if (prior.length > 32) return { kind: "missing_evidence", reason: "candidate_window_exhausted" };
const progress = await nativeSessionProviderEvidence(db, [input.currentRunId]);
if (!isUnusedNativeSessionBootstrap(current, progress.has(input.currentRunId))) {
return { kind: "missing_evidence", reason: "current_execution_progress" };
}
// The normal admission and remote untouched-session claim still execute.
return { kind: "no_prior_execution" };
}
const [owner] = await db.select({ leaseOwner: nativeRunFinalizations.leaseOwner, leaseExpiresAt: nativeRunFinalizations.leaseExpiresAt }).from(nativeRunFinalizations)
.where(and(eq(nativeRunFinalizations.runId, source.id), eq(nativeRunFinalizations.companyId, input.companyId),
eq(nativeRunFinalizations.issueId, input.issueId)));
if (owner?.leaseOwner && owner.leaseExpiresAt && owner.leaseExpiresAt.getTime() > Date.now()) {
return { kind: "waiting_for_owner", deadline: owner.leaseExpiresAt };
}
if (source.runtimeMode !== "native" || !["succeeded", "failed", "cancelled", "interrupted", "timed_out"].includes(source.status)
|| source.createdAt > current.createdAt) return { kind: "missing_evidence", reason: "newer_progress_barrier" };
const profile = record(source.runnerProfileJson), snapshot = record(profile.sessionCheckpoint);
const identity = record(snapshot.identity);
if (identity.companyId !== input.companyId || identity.agentId !== input.agentId
|| identity.issueId !== input.issueId || identity.runId !== source.id || identity.sessionId !== input.normalizedSessionId) {
return { kind: "missing_evidence", reason: "snapshot_binding" };
}
if (record(profile.nativeControllerRecovery).schema !== "paperclip.native-controller-recovery.v1") {
return { kind: "unsupported", reason: "metadata_format" };
}
const metadata = readNativeControllerRecovery(profile.nativeControllerRecovery, snapshot);
if (!metadata?.checkpoint || metadata.checkpoint.runId !== source.id
|| metadata.checkpoint.normalizedSessionId !== input.normalizedSessionId
|| metadata.checkpoint.runnerInstanceId !== source.runnerInstanceId) {
return { kind: "missing_evidence", reason: "snapshot_digest_or_checkpoint_binding" };
}
let checkpoint: DurablePrpCheckpoint | null;
try {
checkpoint = await loadNativeRunnerCheckpoint(db, { companyId: input.companyId, issueId: input.issueId, runId: source.id });
} catch (cause) {
return { kind: "dependency_unavailable", reason: "checkpoint_authority_unreadable", cause };
}
if (!checkpoint || checkpoint.recoveryEvidence.runnerInstanceId !== metadata.checkpoint.runnerInstanceId
|| checkpoint.recoveryEvidence.normalizedSessionId !== input.normalizedSessionId) {
return { kind: "missing_evidence", reason: "checkpoint_binding" };
}
return { kind: "recoverable", snapshot: snapshot as unknown as PersistedNativeSession, metadata, checkpoint };
}
/** Bounded internal migration of ONE retained source while its disk is available.
* The migration owner supplies compatibility computed from the admitted input.
* This never repairs/replaces the saved snapshot or claims a live execution. */
export async function importNativeControllerRecovery(db: Db, input: {
companyId: string; agentId: string; issueId: string; runId: string; normalizedSessionId: string;
expectedConfigDigest: string;
supervisor: unknown;
}): Promise<{ kind: "imported" | "unchanged" } | Exclude<NativeControllerRecoveryOutcome, { kind: "recoverable" } | { kind: "no_prior_execution" }>> {
const supervisor = record(input.supervisor);
if (supervisor.schema !== "paperclip.native-session-supervisor.v1" || supervisor.configDigest !== input.expectedConfigDigest) {
return { kind: "unsupported", reason: "supervisor_compatibility" };
}
return db.transaction(async tx => {
const [task] = await tx.select({ executionRunId: issues.executionRunId }).from(issues)
.where(and(eq(issues.companyId, input.companyId), eq(issues.id, input.issueId))).for("share");
const [owner] = await tx.select().from(nativeRunFinalizations).where(and(
eq(nativeRunFinalizations.companyId, input.companyId), eq(nativeRunFinalizations.issueId, input.issueId),
eq(nativeRunFinalizations.runId, input.runId))).for("update");
if (!task || !owner) return { kind: "missing_evidence" as const, reason: "owner_binding" };
if (owner.leaseOwner || task.executionRunId) {
return owner.leaseExpiresAt && owner.leaseExpiresAt > new Date()
? { kind: "waiting_for_owner" as const, deadline: owner.leaseExpiresAt }
: { kind: "missing_evidence" as const, reason: "normal_ownership_recovery_required" };
}
if (owner.phase !== "committed" || !owner.resultId || owner.recoveryState) {
return { kind: "missing_evidence" as const, reason: "pending_finalization" };
}
const [run] = await tx.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, input.runId),
eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.agentId, input.agentId),
eq(heartbeatRuns.nativeIssueId, input.issueId), eq(heartbeatRuns.nativeSessionId, input.normalizedSessionId))).for("update");
const profile = record(run?.runnerProfileJson), snapshot = record(profile.sessionCheckpoint), identity = record(snapshot.identity);
if (run?.runtimeMode !== "native" || run.status !== "succeeded" || !run.finishedAt
|| identity.runId !== input.runId || identity.companyId !== input.companyId || identity.issueId !== input.issueId
|| identity.agentId !== input.agentId || identity.sessionId !== input.normalizedSessionId
|| nativeSha256(snapshot) !== nativeSha256(redactRunnerApiPayloads(supervisor.snapshot))) {
return { kind: "missing_evidence" as const, reason: "stale_supervisor_snapshot" };
}
const checkpoint = await loadNativeRunnerCheckpoint(tx, input);
if (!checkpoint || checkpoint.recoveryEvidence.normalizedSessionId !== input.normalizedSessionId
|| checkpoint.recoveryEvidence.runnerInstanceId !== run.runnerInstanceId) {
return { kind: "missing_evidence" as const, reason: "checkpoint_binding" };
}
const metadata = createNativeControllerRecovery(snapshot as unknown as PersistedNativeSession, {
configDigest: input.expectedConfigDigest,
processConfigDigest: typeof supervisor.processConfigDigest === "string" ? supervisor.processConfigDigest : null,
}, { runId: input.runId, runnerInstanceId: checkpoint.recoveryEvidence.runnerInstanceId, normalizedSessionId: input.normalizedSessionId });
if (!readNativeControllerRecovery(metadata, snapshot)) return { kind: "unsupported" as const, reason: "metadata_format" };
if (profile.nativeControllerRecovery) {
return nativeSha256(profile.nativeControllerRecovery) === nativeSha256(metadata)
? { kind: "unchanged" as const }
: { kind: "missing_evidence" as const, reason: "existing_metadata_conflict" };
}
await tx.update(heartbeatRuns).set({ runnerProfileJson: { ...profile, nativeControllerRecovery: metadata }, updatedAt: new Date() })
.where(eq(heartbeatRuns.id, input.runId));
return { kind: "imported" as const };
});
}
@@ -181,6 +181,8 @@ async function sessionRetentionBlocker(
id: heartbeatRuns.id,
status: heartbeatRuns.status,
finishedAt: heartbeatRuns.finishedAt,
incompleteRecovery: sql<boolean>`(${heartbeatRuns.runnerProfileJson}->'nativeControllerRecovery'->>'schema' = 'paperclip.native-controller-recovery.v1'
and ${heartbeatRuns.runnerProfileJson}->'nativeControllerRecovery'->'checkpoint' = 'null'::jsonb) is true`,
issueId: sql<string | null>`coalesce(${heartbeatRuns.nativeIssueId}, ${heartbeatRuns.issueId})`,
})
.from(heartbeatRuns)
@@ -194,6 +196,7 @@ async function sessionRetentionBlocker(
if (runs.some((run) => LIVE_RUN_STATUSES.includes(run.status) || !run.finishedAt)) {
return { reason: "live_run" };
}
if (runs.some(run => run.incompleteRecovery)) return { reason: "native_recovery_incomplete" };
if (input.checkpoint && runs.some(run => run.finishedAt! >= input.closedBefore)) return { reason: "recent_run" };
const issueIds = [...new Set(runs.map((run) => run.issueId).filter((id): id is string => Boolean(id)))];
if (issueIds.length === 0) return { reason: "issue_unknown" };
@@ -0,0 +1,92 @@
import { randomUUID } from "node:crypto";
import { PgDialect } from "drizzle-orm/pg-core";
import type { SQL } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { DurableRecoveryIdentity, PrpEvent } from "../../vendor/paperclip-runner/index.js";
import { buildNativeExecutionInput } from "./native-execution-input.js";
import { nativeRuntimeContextFixture } from "./runtime-context.test-fixture.js";
import { createNativeRecoveryUsageDelivery } from "./native-recovery-usage-delivery.js";
const accounting = vi.hoisted(() => ({ open: vi.fn(), capture: vi.fn() }));
vi.mock("./native-usage-delivery.js", () => ({ createNativeUsageContinuationDelivery: accounting.open }));
beforeEach(() => { vi.clearAllMocks(); accounting.capture.mockResolvedValue(undefined); accounting.open.mockResolvedValue({ capture: accounting.capture }); });
function fixture() {
const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), sessionId = randomUUID();
const source: DurableRecoveryIdentity = { runId: randomUUID(), runnerInstanceId: randomUUID(), environmentLeaseId: randomUUID(),
normalizedSessionId: sessionId, turnId: randomUUID(), itemId: randomUUID() };
const build = (runId: string) => buildNativeExecutionInput({ companyId, agentId, runId, normalizedSessionId: sessionId,
provider: "codex", issue: { id: issueId, identifier: "REC-1", title: "Continue", description: null, workMode: "standard" },
taskPrompt: "Continue once", workspace: { id: randomUUID(), cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null },
completionContract: { id: randomUUID(), sha256: `sha256:${"a".repeat(64)}`, schemaVersion: "paperclip.run-result.v1",
contract: { revision: "1", objective: "Continue", criteria: [{ id: "objective", requirement: "Preserve the result" }] } },
runtimeContext: nativeRuntimeContextFixture() });
const execution = build(randomUUID()), previous = build(source.runId);
const row = { run: { status: "succeeded", finishedAt: new Date() as Date | null, runtimeMode: "native", runnerInstanceId: source.runnerInstanceId,
nativeSessionId: sessionId, runnerProfileJson: { nativeExecutionInput: previous } },
owner: { phase: "committed", resultId: randomUUID() as string | null, leaseOwner: null as string | null, companyId, issueId } };
const where = vi.fn(async (_predicate: SQL) => [row]);
const select = vi.fn(() => ({ from: () => ({ innerJoin: () => ({ where }) }) }));
const db = { select } as unknown as Db;
const matchesExecution = vi.fn(() => true);
const deliver = () => createNativeRecoveryUsageDelivery({ db, execution, sourceIdentity: { ...source }, matchesExecution });
const event: PrpEvent = { schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", sourceInstanceId: source.runnerInstanceId,
sourceEventId: "source-usage", sourceSeq: 1, runId: source.runId, normalizedSessionId: sessionId, turnId: source.turnId,
eventType: "usage.reported", priority: 0, emittedAt: new Date().toISOString(), payload: {} };
return { source, execution, previous, row, select, where, matchesExecution, deliver, event };
}
describe("source accounting recovery authority", () => {
it("pins the relational source tuple and hydrates once without binding target accounting", async () => {
const f = fixture(), deliver = f.deliver();
await deliver(f.event, f.source);
await deliver({ ...f.event, sourceEventId: "source-terminal", sourceSeq: 2, eventType: "turn.completed" }, f.source);
expect(f.select).toHaveBeenCalledOnce();
expect(new PgDialect().sqlToQuery(f.where.mock.calls[0][0]).params).toEqual([
f.source.runId, f.execution.binding.companyId, f.execution.binding.agentId, f.execution.binding.issueId,
]);
expect(f.matchesExecution).toHaveBeenCalledWith(f.previous);
expect(accounting.open).toHaveBeenCalledExactlyOnceWith(expect.anything(), {
companyId: f.execution.binding.companyId, runId: f.source.runId, runnerInstanceId: f.source.runnerInstanceId, provider: f.previous.provider,
});
expect(accounting.capture).toHaveBeenCalledTimes(2);
expect(accounting.capture).toHaveBeenCalledWith(f.event);
});
it.each(["runId", "runnerInstanceId", "environmentLeaseId", "normalizedSessionId", "turnId", "itemId"] as const)(
"rejects a changed pinned %s before source hydration", async key => {
const f = fixture();
await expect(f.deliver()(f.event, { ...f.source, [key]: randomUUID() })).rejects.toThrow("native_recovery_usage_source_unproven");
expect(f.select).not.toHaveBeenCalled();
expect(accounting.open).not.toHaveBeenCalled();
});
it.each(["runId", "sourceInstanceId", "normalizedSessionId"] as const)("rejects mismatched event %s before source hydration", async key => {
const f = fixture();
await expect(f.deliver()({ ...f.event, [key]: randomUUID() }, f.source)).rejects.toThrow("native_recovery_usage_source_unproven");
expect(f.select).not.toHaveBeenCalled();
});
it.each(["nonterminal", "unfinished", "uncommitted", "no-result", "held", "company", "issue", "scope"])(
"keeps an unproven source held: %s", async change => {
const f = fixture();
if (change === "nonterminal") f.row.run.status = "running";
if (change === "unfinished") f.row.run.finishedAt = null;
if (change === "uncommitted") f.row.owner.phase = "observed";
if (change === "no-result") f.row.owner.resultId = null;
if (change === "held") f.row.owner.leaseOwner = "live-owner";
if (change === "company") f.row.owner.companyId = randomUUID();
if (change === "issue") f.row.owner.issueId = randomUUID();
if (change === "scope") f.matchesExecution.mockReturnValue(false);
await expect(f.deliver()(f.event, f.source)).rejects.toThrow("native_recovery_usage_source_unproven");
expect(accounting.open).not.toHaveBeenCalled();
expect(accounting.capture).not.toHaveBeenCalled();
});
it("keeps a durable source receipt failure unacknowledged", async () => {
const f = fixture();
accounting.capture.mockRejectedValue(new Error("native_usage_after_finalization_requires_reconciliation"));
await expect(f.deliver()(f.event, f.source)).rejects.toThrow("native_usage_after_finalization_requires_reconciliation");
});
});
@@ -0,0 +1,47 @@
import { and, eq } from "drizzle-orm";
import { heartbeatRuns, nativeRunFinalizations, type Db } from "@paperclipai/db";
import { parseNativeExecutionInput, type DurableRecoveryIdentity, type NativeExecutionInput, type PrpEvent } from "../../vendor/paperclip-runner/index.js";
import { createNativeUsageContinuationDelivery } from "./native-usage-delivery.js";
const identityKeys = ["runId", "runnerInstanceId", "environmentLeaseId", "normalizedSessionId", "turnId", "itemId"] as const;
/** A cold handoff may discover accounting that belongs to its completed source.
* Hydrate that producer once; never route its receipts through the new run. */
export function createNativeRecoveryUsageDelivery(input: {
db: Db;
execution: NativeExecutionInput;
sourceIdentity: Record<string, unknown>;
matchesExecution(previous: NativeExecutionInput): boolean;
}) {
let delivery: ReturnType<typeof createNativeUsageContinuationDelivery> | undefined;
const deny = (): never => { throw new Error("native_recovery_usage_source_unproven"); };
const open = async (source: DurableRecoveryIdentity) => {
const { binding } = input.execution;
const [prior] = await input.db.select({ run: heartbeatRuns, owner: nativeRunFinalizations })
.from(heartbeatRuns).innerJoin(nativeRunFinalizations, eq(nativeRunFinalizations.runId, heartbeatRuns.id))
.where(and(eq(heartbeatRuns.id, source.runId), eq(heartbeatRuns.companyId, binding.companyId),
eq(heartbeatRuns.agentId, binding.agentId), eq(heartbeatRuns.nativeIssueId, binding.issueId)));
if (!prior || prior.run.status !== "succeeded" || prior.run.finishedAt === null ||
prior.run.runtimeMode !== "native" || prior.run.runnerInstanceId !== source.runnerInstanceId ||
prior.run.nativeSessionId !== source.normalizedSessionId || prior.owner.phase !== "committed" ||
!prior.owner.resultId || prior.owner.leaseOwner || prior.owner.companyId !== binding.companyId ||
prior.owner.issueId !== binding.issueId) return deny();
const previous = parseNativeExecutionInput(prior.run.runnerProfileJson?.nativeExecutionInput);
if (previous.binding.companyId !== binding.companyId || previous.binding.agentId !== binding.agentId ||
previous.binding.issueId !== binding.issueId || previous.binding.runId !== source.runId ||
!input.matchesExecution(previous)) return deny();
return createNativeUsageContinuationDelivery(input.db, {
companyId: binding.companyId, runId: source.runId,
runnerInstanceId: source.runnerInstanceId, provider: previous.provider,
});
};
return async (event: PrpEvent, source: DurableRecoveryIdentity) => {
if (source.runId === input.execution.binding.runId ||
identityKeys.some(key => typeof input.sourceIdentity[key] !== "string" ||
!input.sourceIdentity[key] || source[key] !== input.sourceIdentity[key]) ||
event.runId !== source.runId || event.sourceInstanceId !== source.runnerInstanceId ||
event.normalizedSessionId !== source.normalizedSessionId) return deny();
delivery ??= open(source);
await (await delivery).capture(event);
};
}
@@ -1,6 +1,6 @@
import { QueryBuilder } from "drizzle-orm/pg-core";
import { and, asc, eq, gt, inArray, isNotNull, isNull, notExists, or, sql } from "drizzle-orm";
import { heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults, statusDecisions, statusDecisionEffects } from "@paperclipai/db";
import { and, asc, eq, exists, gt, inArray, isNotNull, isNull, notExists, or, sql } from "drizzle-orm";
import { heartbeatRunEvents, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults, statusDecisions, statusDecisionEffects } from "@paperclipai/db";
import type { NativeResponsePresentationScope } from "../native-response-presentation-notifications.js";
export function nativeResponsePresentationCandidates(scope?: NativeResponsePresentationScope, after?: string, runId?: string) {
@@ -30,7 +30,24 @@ export function nativeResponsePresentationCandidates(scope?: NativeResponsePrese
sql`${nativeRunResults.resultJson} #>> '{terminal,turnTerminalState}' = 'completed'`,
or(
and(eq(heartbeatRuns.status, "succeeded"),
sql`(coalesce(${issues.conversationAgentId}::text, '') = '' or coalesce(${issues.conversationUserId}, '') = '')`,
or(sql`(coalesce(${issues.conversationAgentId}::text, '') = '' or coalesce(${issues.conversationUserId}, '') = '')`,
and(eq(statusDecisions.reasonCode, "conversation_turn_finished"),
eq(issues.conversationAgentId, heartbeatRuns.agentId), eq(issues.conversationUserId, heartbeatRuns.responsibleUserId),
eq(issues.status, "in_progress"), eq(issues.conversationState, "active"),
or(isNull(issues.executionRunId), eq(issues.executionRunId, heartbeatRuns.id)),
sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issues.id}::text`,
sql`jsonb_typeof(${heartbeatRuns.contextSnapshot}->'conversationSessionGeneration') = 'number'`,
sql`${heartbeatRuns.contextSnapshot}->>'conversationSessionGeneration' = ${issues.conversationSessionGeneration}::text`,
exists(db.select({ id: heartbeatRunEvents.id }).from(heartbeatRunEvents).where(and(
eq(heartbeatRunEvents.companyId, heartbeatRuns.companyId), eq(heartbeatRunEvents.runId, heartbeatRuns.id),
eq(heartbeatRunEvents.agentId, heartbeatRuns.agentId), eq(heartbeatRunEvents.eventType, "item.completed"),
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,runId}' = ${heartbeatRuns.id}::text`,
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,normalizedSessionId}' = ${heartbeatRuns.nativeSessionId}::text`,
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,turnId}' = ${nativeRunResults.turnId}`,
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,payload,kind}' = 'agentMessage'`,
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,payload,channel}' = 'final'`,
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,payload,text}' ~ '[^[:space:]]'`,
))))),
sql`${nativeRunResults.resultJson} #>> '{result,reportedWorkDisposition}' = 'done'`,
sql`${nativeRunResults.resultJson} #>> '{terminal,reportedWorkDisposition}' = 'done'`),
and(sql`${nativeRunResults.resultJson} #>> '{result,reportedWorkDisposition}' = 'yielded'`,
@@ -56,6 +56,7 @@ export function nativeResponsePresentationWork(db: Db) {
function register(coordinator: ReturnType<typeof createDeliveryWorkCoordinator>, canRun: () => boolean) {
const worker = registerScopedDeadlineWork(coordinator, {
queue: DELIVERY_QUEUES.nativeResponsePresentation,
concurrentChecks: { limit: 4, key: (entry: Entry) => `${entry.companyId}:${entry.issueId}` },
decodeScope: (hint: string): NativeResponsePresentationScope | undefined => hint === "*" ? undefined : JSON.parse(hint),
matches: (entry: Entry, scope?: NativeResponsePresentationScope) => !scope || entry.companyId === scope.companyId && (!scope.issueId || entry.issueId === scope.issueId) && (!scope.runId || entry.id === scope.runId) && (!scope.agentId || entry.agentId === scope.agentId),
load: (scope, signal) => load(scope, signal, canRun),
@@ -8,6 +8,18 @@ import {
} from "./native-restart-recovery.js";
describe("native restart recovery classification", () => {
it.each([false, true])("recognizes verified staged remote evidence with target snapshot=%s", hasCheckpoint => {
const evidence = { remoteSandbox: true, runnerPidAlive: false, runnerGroupAlive: false,
processStartMatches: false, hasCheckpoint, hasProviderEvidence: false };
expect(classifyNativeRunnerRecoveryEvidence(evidence).claimKind).toBeNull();
expect(classifyNativeRunnerRecoveryEvidence({ ...evidence, stagedAttachmentVerified: true })).toEqual({
claimKind: "reattach_remote_runner", reason: "staged_remote_attachment_requires_sandbox_verification",
});
expect(classifyNativeRunnerRecoveryEvidence({ ...evidence, stagedAttachmentVerified: true,
checkpointFailed: true }).claimKind).toBeNull();
expect(classifyNativeRunnerRecoveryEvidence({ ...evidence, remoteSandbox: false,
stagedAttachmentVerified: true }).claimKind).toBe(hasCheckpoint ? null : "bootstrap_incomplete");
});
it.each([true, false])("uses the remote fenced lease instead of the old container PID (live=%s)", async live => {
const now = new Date();
const probe = vi.fn(() => { throw new Error("must not probe another container's PID"); });
@@ -1,3 +1,4 @@
import { NATIVE_RETAINED_OWNER_WAIT } from "./native-controller-recovery.js";
import { notifyNativeEnvironmentSettlement } from "../native-environment-settlement-notifications.js";
import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
import { notifyNativeStatusReassessment } from "../native-status-reassessment-notifications.js";
@@ -6,7 +7,8 @@ import { notifyCancelledWorkspacePreservation } from "../cancelled-workspace-pre
import { notifyInstructionCollection } from "../instruction-collection-notifications.js";
import { notifyExecutionRunWork } from "../execution-control-notifications.js";
import { preserveNativeFailedWorkspace } from "./native-failed-workspace-recovery.js";
import { hasNativeRunnerCheckpoint } from "./native-runner-checkpoint.js";
import { hasNativeRunnerCheckpoint, loadNativeRunnerCheckpointHeader } from "./native-runner-checkpoint.js";
import { resolveNativeStagedAttachmentRecovery, type NativeStagedAttachmentRecovery } from "./native-staged-attachment-recovery.js";
import { readNativeWorkspaceSyncReference } from "./native-workspace-sync.js";
import { activeNativeExecutionRunIds, NATIVE_PREPARATION_FENCING_VERSION } from "./native-active-runs.js";
import {
@@ -15,7 +17,7 @@ import {
recordNativeLocalProcessStop,
} from "../native-local-process-stop.js";
import { randomUUID } from "node:crypto";
import { and, desc, eq, inArray, isNotNull, isNull, lte, or, sql } from "drizzle-orm";
import { and, desc, eq, inArray, isNotNull, isNull, lte, ne, or, sql } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import {
agents,
@@ -68,6 +70,8 @@ export type NativeRestartRecoveryClaim =
recoveryRequestId: string | null;
remote: { kind?: "sandbox"; providerLeaseId: string; remoteCwd: string }
| { kind: "plugin_task"; leaseId: string };
/** Discovery evidence only; executor revalidates the encrypted header. */
stagedAttachment?: Pick<NativeStagedAttachmentRecovery, "sourceIdentity" | "targetIdentity" | "templateDigest">;
}
| {
kind: "resume_dead_runner";
@@ -94,6 +98,7 @@ export type NativeRestartRecoveryDisposition =
kind: "awaiting_evidence" | "blocked" | "already_finalized";
runId: string;
reason: string;
retryAt?: Date;
};
export function nextNativeProviderAttempt(
@@ -320,6 +325,7 @@ export function classifyNativeRunnerRecoveryEvidence(input: {
checkpointIdentityMatches?: boolean;
hasProviderEvidence: boolean;
checkpointFailed?: boolean;
stagedAttachmentVerified?: boolean;
providerAttempt?: number;
}): {
claimKind: NativeRestartRecoveryClaim["kind"] | null;
@@ -334,6 +340,9 @@ export function classifyNativeRunnerRecoveryEvidence(input: {
// inspection; the executor must prove the original lease and authenticate
// the surviving runner before it can resume delivery.
if (input.remoteSandbox) {
if (input.stagedAttachmentVerified === true) {
return { claimKind: "reattach_remote_runner", reason: "staged_remote_attachment_requires_sandbox_verification" };
}
if (input.unclaimedBootstrap === true && !input.hasCheckpoint && !input.hasProviderEvidence) {
return { claimKind: "bootstrap_incomplete", reason: "remote_execution_never_claimed" };
}
@@ -514,6 +523,9 @@ export async function claimNativeRestartRecoveries(input: {
eq(agents.adapterType, "paperclip_runner"),
inArray(heartbeatRuns.status, ["running", "failed"]),
isNull(nativeRunFinalizations.resultId),
// A pre-provider retained-owner wait belongs to normal same-run
// resumption, including when its deadline elapsed while we were down.
or(isNull(nativeRunFinalizations.failureCode), ne(nativeRunFinalizations.failureCode, NATIVE_RETAINED_OWNER_WAIT)),
or(
eq(nativeRunFinalizations.phase, "observed"),
and(
@@ -668,9 +680,21 @@ export async function claimNativeRestartRecoveries(input: {
kind: "awaiting_evidence",
runId: row.run.id,
reason: takeover.reason,
...(durableAuthority && row.coordinator.leaseExpiresAt ? { retryAt: row.coordinator.leaseExpiresAt } : {}),
} as const;
}
// Only the classic remote path stages attachments. After the owner is
// eligible for takeover, inspect its bounded header even if activation
// already published a runtime snapshot. Never scan command/event payloads.
const stagedHeader = remoteWorkspace && !pluginTask && durableAuthority
? await loadNativeRunnerCheckpointHeader(tx, {
companyId: row.run.companyId, issueId: row.coordinator.issueId, runId: row.run.id,
}) : null;
const stagedAttachment = stagedHeader ? await resolveNativeStagedAttachmentRecovery({
db: tx, currentRun: row.run, header: stagedHeader,
}) : null;
const runnerPidAlive =
!remoteProcess && processIsAlive(row.run.processPid);
const runnerGroupAlive =
@@ -859,6 +883,7 @@ export async function claimNativeRestartRecoveries(input: {
knownProviderProcessIdentityAmbiguous:
providerProcesses.ambiguousLivePids.length > 0,
hasCheckpoint,
stagedAttachmentVerified: stagedAttachment !== null,
checkpointIdentityMatches:
checkpointIdentityMatches &&
(!remoteSandbox || remoteWorkspace !== null),
@@ -1109,6 +1134,10 @@ export async function claimNativeRestartRecoveries(input: {
return {
kind: claimKind,
...common,
...(stagedAttachment ? { stagedAttachment: {
sourceIdentity: stagedAttachment.sourceIdentity, targetIdentity: stagedAttachment.targetIdentity,
templateDigest: stagedAttachment.templateDigest,
} } : {}),
remote: pluginTask ? { kind: "plugin_task", leaseId: pluginTask.id } : {
providerLeaseId: remoteWorkspace!.providerLeaseId,
remoteCwd: remoteWorkspace!.remoteCwd,
@@ -29,7 +29,7 @@ import { readPersistedNativeFailureEvidence } from "./native-provider-failure-ev
import { dismissAutomaticCompletionReviews } from "./automatic-completion-reviews.js";
import { findSupersededExternalReviewRequests } from "./superseded-external-review-requests.js";
import { getNativeReviewAssignment, readNativeReviewAssignmentContext } from "./native-review-participant.js";
import { conversationNativeDecision, isConversation } from "../agent-conversations.js";
import { conversationNativeDecision, isConversation, settleConversationTurnForRun } from "../agent-conversations.js";
import { issueTreeControlService } from "../issue-tree-control.js";
import { randomUUID } from "node:crypto";
import { preserveNativeWorkspaceExportLease } from "./native-workspace-export-resume.js";
@@ -747,6 +747,7 @@ async function recoverCommittedNativeChatResponseInScope(
): Promise<boolean> {
let agentId: string | null = null;
let materialized = false;
let conversationReply = false;
const entered = beginRunPhase("response.transaction.entry");
try {
materialized = await measureRunPhase("response.transaction", () => db.transaction(async (transaction) => {
@@ -789,6 +790,7 @@ async function recoverCommittedNativeChatResponseInScope(
id: heartbeatRuns.id,
companyId: heartbeatRuns.companyId,
agentId: heartbeatRuns.agentId,
responsibleUserId: heartbeatRuns.responsibleUserId,
nativeIssueId: heartbeatRuns.nativeIssueId,
nativeSessionId: heartbeatRuns.nativeSessionId,
completionContractId: heartbeatRuns.completionContractId,
@@ -803,6 +805,8 @@ async function recoverCommittedNativeChatResponseInScope(
contextSnapshot: sql<Record<string, unknown> | null>`case when ${heartbeatRuns.contextSnapshot} is null then null
else jsonb_build_object(
'skipIssueComment', ${heartbeatRuns.contextSnapshot}->'skipIssueComment',
'issueId', ${heartbeatRuns.contextSnapshot}->'issueId',
'conversationSessionGeneration', ${heartbeatRuns.contextSnapshot}->'conversationSessionGeneration',
'source', ${heartbeatRuns.contextSnapshot}->'source',
'externalChatContinuation', ${heartbeatRuns.contextSnapshot}->'externalChatContinuation',
'paperclipWake', jsonb_build_object('externalInteractionContinuation',
@@ -909,12 +913,16 @@ async function recoverCommittedNativeChatResponseInScope(
agentId = run.agentId;
return true;
}
// Copyback can be owned by reconciliation before the live heartbeat
// reaches its presentation step. Recover a completed internal task's
// exact final reply from the same accepted turn, without rerunning work
// or granting external-chat publication authority.
const conversation = isConversation(issue);
const currentConversation = conversation && decision.reasonCode === "conversation_turn_finished"
&& issue.conversationAgentId === run.agentId && issue.conversationUserId === run.responsibleUserId
&& issue.status === "in_progress" && issue.conversationState === "active"
&& (!issue.executionRunId || issue.executionRunId === run.id)
&& run.contextSnapshot?.issueId === issue.id
&& typeof run.contextSnapshot.conversationSessionGeneration === "number"
&& run.contextSnapshot.conversationSessionGeneration === issue.conversationSessionGeneration;
if (
!isConversation(issue) &&
(!conversation || currentConversation) &&
run.status === "succeeded" &&
record(run.contextSnapshot).skipIssueComment !== true &&
!isExternalChatPresentationContext(run.contextSnapshot) &&
@@ -929,6 +937,9 @@ async function recoverCommittedNativeChatResponseInScope(
eq(heartbeatRunEvents.companyId, input.companyId),
eq(heartbeatRunEvents.runId, run.id),
eq(heartbeatRunEvents.eventType, "item.completed"),
conversation ? and(eq(heartbeatRunEvents.agentId, run.agentId),
sql`${heartbeatRunEvents.payload}->'prpEvent'->>'runId' = ${run.id}`,
sql`${heartbeatRunEvents.payload}->'prpEvent'->>'normalizedSessionId' = ${run.nativeSessionId}`) : undefined,
sql`${heartbeatRunEvents.payload}->'prpEvent'->>'turnId' = ${accepted.turnId}`,
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,payload,kind}' = 'agentMessage'`,
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,payload,channel}' = 'final'`,
@@ -963,6 +974,7 @@ async function recoverCommittedNativeChatResponseInScope(
})}::jsonb`, updatedAt: new Date(),
}).where(eq(heartbeatRuns.id, run.id));
agentId = run.agentId;
conversationReply = conversation;
return true;
}
if (
@@ -1039,12 +1051,14 @@ async function recoverCommittedNativeChatResponseInScope(
} finally {
entered();
}
if (materialized && agentId)
if (materialized && agentId) {
if (conversationReply) await settleConversationTurnForRun(db, input.companyId, input.runId);
publishChatPublicationCommitSignal({
...input,
agentId,
eventType: "run.presentation.resolved",
});
}
return materialized;
}
@@ -5,7 +5,7 @@ import { createLocalEncryptedCipher } from "../../secrets/local-encrypted-provid
import { legacyControllerBootId } from "../legacy-controller-lease.js";
type Binding = { companyId: string; issueId: string; runId: string };
type Owner = Binding & { bootId: string; leaseOwner: string; generation: number; authorityRunId: string };
export type NativeRunnerCheckpointOwner = Binding & { bootId: string; leaseOwner: string; generation: number; authorityRunId: string };
function encryptParts(binding: Binding, records: ReadonlyMap<string, string>) {
if (!records.size) return [];
const cipher = createLocalEncryptedCipher("write");
@@ -25,49 +25,79 @@ function decryptPart(cipher: ReturnType<typeof createLocalEncryptedCipher>, bind
return envelope.contents;
}
export function nativeRunnerCheckpointWriter(db: Db, owner: Owner) {
type CheckpointTransaction = Parameters<Parameters<Db["transaction"]>[0]>[0];
/** A dependent run-lock query can consume this CTE to preserve coordinator-first
* lock order without adding a second database round trip. */
export function nativeRunnerCheckpointOwnerLock(tx: CheckpointTransaction, owner: NativeRunnerCheckpointOwner) {
return tx.$with("checkpoint_owner").as(tx.select({
leaseOwner: nativeRunFinalizations.leaseOwner,
leaseExpiresAt: nativeRunFinalizations.leaseExpiresAt,
resultId: nativeRunFinalizations.resultId,
}).from(nativeRunFinalizations).where(and(
eq(nativeRunFinalizations.companyId, owner.companyId), eq(nativeRunFinalizations.issueId, owner.issueId),
eq(nativeRunFinalizations.runId, owner.authorityRunId), eq(nativeRunFinalizations.controllerBootId, owner.bootId),
eq(nativeRunFinalizations.controllerGeneration, owner.generation))).for("update"));
}
/** Locks the coordinator before callers lock a run. A caller that subsequently
* waits for another lock must recheck the fence before writing. */
export async function lockNativeRunnerCheckpointOwner(
tx: CheckpointTransaction, owner: NativeRunnerCheckpointOwner,
) {
// A locking CTE acquires ownership before the outer projection samples
// wall time. Sampling in the locking SELECT can precede its lock wait.
// Keep both steps in one statement without another database round trip.
const locked = nativeRunnerCheckpointOwnerLock(tx, owner);
const [owned] = await tx.with(locked).select({ leaseOwner: locked.leaseOwner,
leaseExpiresAt: locked.leaseExpiresAt, resultId: locked.resultId, now: sql<Date>`clock_timestamp()` }).from(locked);
if (!owned) throw new Error("native_runner_checkpoint_owner_lost");
return owned;
}
export async function validateNativeRunnerCheckpointOwner(
tx: CheckpointTransaction, owner: NativeRunnerCheckpointOwner,
owned: Awaited<ReturnType<typeof lockNativeRunnerCheckpointOwner>>, newWork: boolean,
): Promise<void> {
const current = owned;
const activeOwner = current.leaseOwner === owner.leaseOwner
&& current.leaseExpiresAt !== null && current.leaseExpiresAt > new Date(owned.now);
if (!activeOwner) {
const [task] = await tx.select({ executionRunId: issues.executionRunId }).from(issues)
.where(and(eq(issues.companyId, owner.companyId), eq(issues.id, owner.issueId)));
const [successor] = task?.executionRunId && task.executionRunId !== owner.runId
? await tx.select({ runId: nativeRunFinalizations.runId }).from(nativeRunFinalizations).where(and(
eq(nativeRunFinalizations.runId, task.executionRunId), eq(nativeRunFinalizations.companyId, owner.companyId),
eq(nativeRunFinalizations.issueId, owner.issueId), eq(nativeRunFinalizations.controllerBootId, owner.bootId),
gt(nativeRunFinalizations.leaseExpiresAt, sql`clock_timestamp()`))) : [];
const [preparingSuccessor] = !successor && task?.executionRunId && task.executionRunId !== owner.runId
? await tx.select({ runId: heartbeatRuns.id }).from(heartbeatRuns).where(and(
eq(heartbeatRuns.id, task.executionRunId), eq(heartbeatRuns.companyId, owner.companyId),
eq(heartbeatRuns.issueId, owner.issueId), eq(heartbeatRuns.runtimeMode, "legacy"),
eq(heartbeatRuns.status, "running"), eq(heartbeatRuns.executionStage, "preparing"),
eq(heartbeatRuns.controllerBootId, legacyControllerBootId),
gt(heartbeatRuns.controllerLeaseExpiresAt, sql`clock_timestamp()`),
notExists(tx.select({ runId: nativeRunFinalizations.runId }).from(nativeRunFinalizations)
.where(eq(nativeRunFinalizations.runId, heartbeatRuns.id))))) : [];
if (!current.resultId || newWork || !task || (task.executionRunId && task.executionRunId !== owner.runId && !successor && !preparingSuccessor)) {
throw new Error("native_runner_checkpoint_owner_lost");
}
}
}
export async function assertNativeRunnerCheckpointOwner(tx: CheckpointTransaction, owner: NativeRunnerCheckpointOwner, newWork: boolean) {
const locked = await lockNativeRunnerCheckpointOwner(tx, owner);
await validateNativeRunnerCheckpointOwner(tx, owner, locked, newWork);
}
export function nativeRunnerCheckpointWriter(db: Db, owner: NativeRunnerCheckpointOwner) {
const persistenceDb = checkpointDb(db);
let previousWork = new Set<string>();
return async (checkpoint: DurablePrpCheckpoint): Promise<void> => {
if (checkpoint.runId !== owner.runId) throw new Error("native_runner_checkpoint_binding_mismatch");
const changed = encryptParts(owner, checkpoint.changedRecords);
await persistenceDb.transaction(async tx => {
// A locking CTE acquires ownership before the outer projection samples
// wall time. Sampling in the locking SELECT can precede its lock wait.
// Keep both steps in one statement without another database round trip.
const locked = tx.$with("checkpoint_owner").as(tx.select().from(nativeRunFinalizations)
.where(and(eq(nativeRunFinalizations.companyId, owner.companyId),
eq(nativeRunFinalizations.issueId, owner.issueId), eq(nativeRunFinalizations.runId, owner.authorityRunId),
eq(nativeRunFinalizations.controllerBootId, owner.bootId),
eq(nativeRunFinalizations.controllerGeneration, owner.generation))).for("update"));
const [owned] = await tx.with(locked).select({ leaseOwner: locked.leaseOwner,
leaseExpiresAt: locked.leaseExpiresAt, resultId: locked.resultId, now: sql<Date>`clock_timestamp()` }).from(locked);
if (!owned) throw new Error("native_runner_checkpoint_owner_lost");
const current = owned;
const activeOwner = current.leaseOwner === owner.leaseOwner
&& current.leaseExpiresAt !== null && current.leaseExpiresAt > new Date(owned.now);
if (!activeOwner) {
const newWork = checkpoint.workCommandIds.some(id => !previousWork.has(id));
const [task] = await tx.select({ executionRunId: issues.executionRunId }).from(issues)
.where(and(eq(issues.companyId, owner.companyId), eq(issues.id, owner.issueId)));
const [successor] = task?.executionRunId && task.executionRunId !== owner.runId
? await tx.select({ runId: nativeRunFinalizations.runId }).from(nativeRunFinalizations).where(and(
eq(nativeRunFinalizations.runId, task.executionRunId), eq(nativeRunFinalizations.companyId, owner.companyId),
eq(nativeRunFinalizations.issueId, owner.issueId), eq(nativeRunFinalizations.controllerBootId, owner.bootId),
gt(nativeRunFinalizations.leaseExpiresAt, sql`clock_timestamp()`))) : [];
const [preparingSuccessor] = !successor && task?.executionRunId && task.executionRunId !== owner.runId
? await tx.select({ runId: heartbeatRuns.id }).from(heartbeatRuns).where(and(
eq(heartbeatRuns.id, task.executionRunId), eq(heartbeatRuns.companyId, owner.companyId),
eq(heartbeatRuns.issueId, owner.issueId), eq(heartbeatRuns.runtimeMode, "legacy"),
eq(heartbeatRuns.status, "running"), eq(heartbeatRuns.executionStage, "preparing"),
eq(heartbeatRuns.controllerBootId, legacyControllerBootId),
gt(heartbeatRuns.controllerLeaseExpiresAt, sql`clock_timestamp()`),
notExists(tx.select({ runId: nativeRunFinalizations.runId }).from(nativeRunFinalizations)
.where(eq(nativeRunFinalizations.runId, heartbeatRuns.id))))) : [];
if (!current.resultId || newWork || !task || (task.executionRunId && task.executionRunId !== owner.runId && !successor && !preparingSuccessor)) {
throw new Error("native_runner_checkpoint_owner_lost");
}
}
await assertNativeRunnerCheckpointOwner(tx, owner, checkpoint.workCommandIds.some(id => !previousWork.has(id)));
if (checkpoint.predecessor) {
const [prior] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(
eq(heartbeatRuns.id, checkpoint.predecessor.runId), eq(heartbeatRuns.companyId, owner.companyId),
@@ -150,7 +180,17 @@ export async function registerNativeRunnerCheckpoint(input: {
await input.authority.flushCheckpoint();
}
export async function loadNativeRunnerCheckpoint(db: Db, binding: Binding): Promise<DurablePrpCheckpoint | null> {
/** Exact-run bounded metadata read; never loads command/event payload records. */
export async function loadNativeRunnerCheckpointHeader(db: Pick<Db, "select">, binding: Binding): Promise<string | null> {
const [row] = await db.select({ material: nativeRunnerCheckpoints.material }).from(nativeRunnerCheckpoints).where(and(
eq(nativeRunnerCheckpoints.companyId, binding.companyId), eq(nativeRunnerCheckpoints.issueId, binding.issueId),
eq(nativeRunnerCheckpoints.runId, binding.runId), eq(nativeRunnerCheckpoints.part, "header"))).limit(1);
if (!row) return null;
try { return decryptPart(createLocalEncryptedCipher("read"), binding, "header", row.material); }
catch { return null; }
}
export async function loadNativeRunnerCheckpoint(db: Pick<Db, "select">, binding: Binding): Promise<DurablePrpCheckpoint | null> {
const rows = await db.select().from(nativeRunnerCheckpoints).where(and(
eq(nativeRunnerCheckpoints.companyId, binding.companyId), eq(nativeRunnerCheckpoints.issueId, binding.issueId),
eq(nativeRunnerCheckpoints.runId, binding.runId)));
@@ -6,9 +6,12 @@ import { prepareNativeRemoteRunner, createNativeRunnerExecutionDriver } from "./
const transport = vi.hoisted(() => ({
register: vi.fn(),
connect: vi.fn(),
checkpoint: vi.fn(),
checkpointRegistration: vi.fn(),
}));
vi.mock("../../realtime/runner-prp-ws.js", () => ({ registerRunnerPrpAuthority: transport.register }));
vi.mock("../../realtime/runner-prp-outbound.js", () => ({ connectRunnerPrpIngress: transport.connect }));
vi.mock("./native-runner-checkpoint.js", () => ({ nativeRunnerCheckpointRegistration: transport.checkpointRegistration }));
const binding = { companyId: "company", issueId: "issue", agentId: "agent", runId: "run-a" };
const authority = {} as Parameters<NonNullable<ReturnType<typeof createNativeRunnerExecutionDriver>["controlPlaneRegistration"]>>[0];
@@ -30,9 +33,41 @@ beforeEach(() => {
vi.clearAllMocks();
transport.register.mockResolvedValue({ connectUrl: "ws://127.0.0.1/core", release: vi.fn() });
transport.connect.mockReturnValue({ ready: Promise.resolve(), close: vi.fn(async () => {}) });
transport.checkpointRegistration.mockReturnValue(transport.checkpoint);
transport.checkpoint.mockResolvedValue(undefined);
});
describe("shared Runner execution driver", () => {
it("installs both staged checkpoint writers under the business lease without acquiring endpoints", async () => {
const prepare = vi.fn();
const persistence = { db: {} as never, bootId: "boot-c", authorityRunId: "run-c" };
const driver = createNativeRunnerExecutionDriver({ target: sandbox(), binding: { ...binding, runId: "run-c" },
controllerPersistence: persistence, prepareRemote: prepare });
const sourceIdentity = { runnerInstanceId: "runner", environmentLeaseId: "lease", normalizedSessionId: "session",
runId: "run-a", turnId: "turn-a", itemId: "item-a" };
const targetIdentity = { ...sourceIdentity, runId: "run-c", turnId: "turn-c", itemId: "item-c" };
await driver.onStagedRunAttachment!(authority, { sourceIdentity, targetIdentity, templateDigest: `sha256:${"a".repeat(64)}` });
expect(transport.checkpointRegistration).toHaveBeenCalledExactlyOnceWith({ ...persistence,
binding: { companyId: binding.companyId, issueId: binding.issueId, runId: "run-c" } });
expect(transport.checkpoint.mock.calls).toEqual([[authority, sourceIdentity], [authority, targetIdentity]]);
expect(prepare).not.toHaveBeenCalled();
expect(transport.register).not.toHaveBeenCalled();
expect(transport.connect).not.toHaveBeenCalled();
});
it("installs the target writer first when restoring an already persisted stage", async () => {
const sourceIdentity = { runnerInstanceId: "runner", environmentLeaseId: "lease", normalizedSessionId: "session",
runId: "run-a", turnId: "turn-a", itemId: "item-a" };
const targetIdentity = { ...sourceIdentity, runId: "run-c", turnId: "turn-c", itemId: "item-c" };
const restored = { store: { state: { stagedRunAttachment: { sourceIdentity, targetIdentity } } } } as unknown as typeof authority;
const driver = createNativeRunnerExecutionDriver({ target: sandbox(), binding: { ...binding, runId: "run-c" },
controllerPersistence: { db: {} as never, bootId: "boot-c", authorityRunId: "run-c" } });
await driver.onStagedRunAttachment!(restored, { sourceIdentity, targetIdentity, templateDigest: `sha256:${"a".repeat(64)}` });
expect(transport.checkpoint.mock.calls).toEqual([[restored, targetIdentity], [restored, sourceIdentity]]);
expect(transport.register).not.toHaveBeenCalled();
expect(transport.connect).not.toHaveBeenCalled();
});
it("keeps local execution on the registered loopback route", async () => {
const driver = createNativeRunnerExecutionDriver({ target: { kind: "local" }, binding });
const connection = await driver.controlPlaneRegistration!(authority);
@@ -92,6 +92,20 @@ export async function prepareNativeRemoteRunner(input: RemotePreparation) {
export function createNativeRunnerExecutionDriver(input: NativeRunnerExecutionDriverInput): RunnerExecutionDriver {
return {
...input.process,
onStagedRunAttachment: input.controllerPersistence && input.binding.issueId
? async (authority, proof) => {
// Both journals use the current business lease. Install persistence
// before staging; endpoint acquisition still belongs to attachment.
const register = nativeRunnerCheckpointRegistration({ ...input.controllerPersistence!,
binding: { companyId: input.binding.companyId, issueId: input.binding.issueId!, runId: input.binding.runId },
});
// A restored stage already publishes under the target. Its writer
// must exist before registration flushes that restored revision.
const identities = authority.store?.state.stagedRunAttachment
? [proof.targetIdentity, proof.sourceIdentity] : [proof.sourceIdentity, proof.targetIdentity];
for (const identity of identities) await register(authority, identity);
}
: undefined,
controlPlaneRegistration: async (authority, attachmentIdentity) => {
await input.beforeConnect?.(attachmentIdentity, authority);
if (input.controllerPersistence && input.binding.issueId) {
@@ -60,6 +60,7 @@ import { readProcessStartedAt } from "../hot-restart.js";
import { resolvePaperclipInstanceRoot } from "../../home-paths.js";
import { prepareNativeHeartbeatRun } from "./prepare-native-run.js";
import { buildNativeExecutionInput } from "./native-execution-input.js";
import { buildNativeExecutionWithCheckpoint, findNativeSessionResumeRun, nativeToolContractFingerprintForTarget } from "./native-session-resume.js";
import { nativeRuntimeContextFixture } from "./runtime-context.test-fixture.js";
import { materializeAsset } from "./runtime-context.js";
import { createRunnerTestTlsTunnel, dockerCommandRunner } from "../../__tests__/helpers/docker-command-runner.js";
@@ -321,8 +322,10 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
};
}
it.skipIf(!process.env.PAPERCLIP_TEST_RUNNER_CONTAINER).each(["warm", "hard", "legacy", "changed", "same-server"])("preserves remote warm continuity across completed turns: %s", async (scenario) => {
it.skipIf(!process.env.PAPERCLIP_TEST_RUNNER_CONTAINER).each(["warm", "hard", "legacy", "changed", "same-server", "diskless", "diskless-per-turn", "diskless-per-turn-read-retry"])("preserves remote warm continuity across completed turns: %s", async (scenario) => {
const hardRestart = !["warm", "same-server"].includes(scenario);
const diskless = scenario.startsWith("diskless");
const perTurn = scenario.startsWith("diskless-per-turn");
const container = process.env.PAPERCLIP_TEST_RUNNER_CONTAINER!;
const tunnel = await createRunnerTestTlsTunnel({ port: Number(process.env.PAPERCLIP_TEST_RUNNER_PORT),
key: process.env.PAPERCLIP_TEST_RUNNER_TLS_KEY!, cert: process.env.NODE_EXTRA_CA_CERTS! });
@@ -346,12 +349,13 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
taskPrompt: "Complete the first turn.",
workspace: { id: randomUUID(), cwd: workspace, repoUrl: null, repoRef: null, branchName: null },
normalizedSessionId: fixture.native.normalizedSessionId,
lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 },
lifecyclePolicy: perTurn ? { mode: "per_turn", idleTimeoutMs: null } : { mode: "warm", idleTimeoutMs: 60_000 },
runtimeContext: context,
completionContract: await boundContract(fixture.runId),
});
await fixture.db.update(heartbeatRuns).set({ runnerProfileJson: {
...fixture.preparedProfile, nativeExecutionInput: execution,
nativeToolContractFingerprint: nativeToolContractFingerprintForTarget("remote", execution.provider.kind),
} }).where(eq(heartbeatRuns.id, fixture.runId));
const remoteCwd = `/tmp/paperclip-continuity-${fixture.issueId}`;
expect((await runner.execute({ command: "mkdir", args: ["-p", remoteCwd] })).exitCode).toBe(0);
@@ -385,6 +389,26 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
onLog: async (_stream: "stdout" | "stderr", text: string) => { logs.push(text); },
};
let controller: ReturnType<typeof spawn> | undefined;
const executeInController = (executionOptions: Parameters<typeof executePaperclipNativeSession>[0]) => new Promise<{ exitCode: number | null }>((resolveResult, reject) => {
controller = spawn(process.execPath, ["--import", createRequire(import.meta.url).resolve("tsx"), resolve("server/src/__tests__/helpers/remote-warm-controller.test-worker.ts")],
{ stdio: ["ignore", "pipe", "pipe", "ipc"], env: process.env });
let timer = setTimeout(() => { controller?.kill("SIGKILL"); reject(new Error("Controller fixture imports exceeded 120 seconds")); }, 120_000);
controller.stdout?.on("data", data => logs.push(String(data)));
controller.stderr?.on("data", data => logs.push(String(data)));
controller.on("error", error => { clearTimeout(timer); reject(error); });
controller.on("exit", (code, signal) => { clearTimeout(timer); reject(new Error(`Warm controller exited: ${code}/${signal}: ${logs.join("").slice(-3_000)}`)); });
controller.on("message", (message: { ready?: boolean; log?: { text: string }; result?: { exitCode: number | null }; error?: string }) => {
if (message.ready) {
clearTimeout(timer);
timer = setTimeout(() => { controller?.kill("SIGKILL"); reject(new Error(`Warm controller admission exceeded 60 seconds: ${logs.join("").slice(-3_000)}`)); }, 60_000);
controller!.send?.({ connectionString: temporary.connectionString, container, ingressUrl: `wss://127.0.0.1:${tunnel.port}`,
options: { ...executionOptions, db: undefined, onLog: undefined, runnerExecutionTarget: { ...executionOptions.runnerExecutionTarget, runner: undefined } } });
}
if (message.log) logs.push(message.log.text);
if (message.error) { clearTimeout(timer); reject(new Error(message.error)); }
if (message.result) { clearTimeout(timer); resolveResult(message.result); }
});
});
try {
const firstOptions = { ...options, execution, runnerExecutionTarget: target };
const executeFirst = async () => {
@@ -454,28 +478,13 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
expect(nextProcess.stdout).toBe(priorProcess.stdout);
return result;
};
const first = hardRestart ? await new Promise<{ exitCode: number | null }>((resolveResult, reject) => {
controller = spawn(process.execPath, ["--import", createRequire(import.meta.url).resolve("tsx"), resolve("server/src/__tests__/helpers/remote-warm-controller.test-worker.ts")],
{ stdio: ["ignore", "pipe", "pipe", "ipc"], env: process.env });
const timer = setTimeout(() => { controller?.kill("SIGKILL"); reject(new Error("Warm controller startup exceeded 60 seconds")); }, 60_000);
controller.stdout?.on("data", data => logs.push(String(data)));
controller.stderr?.on("data", data => logs.push(String(data)));
controller.on("error", error => { clearTimeout(timer); reject(error); });
controller.on("exit", (code, signal) => { clearTimeout(timer); reject(new Error(`Warm controller exited: ${code}/${signal}: ${logs.join("").slice(-3_000)}`)); });
controller.on("message", (message: { log?: { text: string }; result?: { exitCode: number | null }; error?: string }) => {
if (message.log) logs.push(message.log.text);
if (message.error) { clearTimeout(timer); reject(new Error(message.error)); }
if (message.result) { clearTimeout(timer); resolveResult(message.result); }
});
controller.send?.({ connectionString: temporary.connectionString, container, ingressUrl: `wss://127.0.0.1:${tunnel.port}`,
options: { ...firstOptions, db: undefined, onLog: undefined, runnerExecutionTarget: { ...target, runner: undefined } } });
}) : await executeFirst();
const first = hardRestart ? await executeInController(firstOptions) : await executeFirst();
expect(first.exitCode).toBe(0);
await finalizeNativeRun({ db: fixture.db, runId: fixture.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
const snapshot = await runner.execute({ command: "sh", args: ["-c", `cat '${remoteCwd}'/.paperclip-runtime/paperclip-runner/sessions/*/runner/runner-state.json`] });
expect(snapshot.exitCode).toBe(0);
const saved = JSON.parse(snapshot.stdout);
expect(saved.lifecycle).toBe("ready");
expect(saved.lifecycle).toBe(perTurn ? "suspended" : "ready");
const processBefore = await runner.execute({ command: "sh", args: ["-c", `cat '${remoteCwd}'/.paperclip-runtime/paperclip-runner/sessions/*/runner/runner-process.identity`] });
expect(processBefore.exitCode).toBe(0);
const historyBefore = await runner.execute({ command: "sh", args: ["-c", `cat '${remoteCwd}'/.paperclip-runtime/paperclip-runner/sessions/*/filesystem/codex-home/fake-codex-state.json`] });
@@ -502,12 +511,11 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
runnerExecutionTarget: { ...target, leaseId: nextLeaseId, sandboxLeaseAcquisition: { outcome: "resumed" as const, providerLeaseId: container } },
};
if (controller) {
await expect(executePaperclipNativeSession(nextOptions)).rejects.toThrow("prior_owner_terminal_state_indeterminate");
const exited = new Promise<void>(resolveExit => controller!.once("exit", () => resolveExit()));
controller.kill("SIGKILL");
(controller as ReturnType<typeof spawn>).kill("SIGKILL");
await exited;
controller = undefined;
await expect(executePaperclipNativeSession({ ...nextOptions,
if (!perTurn) await expect(executePaperclipNativeSession({ ...nextOptions,
runnerExecutionTarget: { ...nextOptions.runnerExecutionTarget,
sandboxLeaseAcquisition: { outcome: "replacement", providerLeaseId: "unrelated-sandbox" } },
})).rejects.toThrow("prior_owner_terminal_state_indeterminate");
@@ -526,10 +534,43 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
}
expect(changed).toBe(1);
}
const [nextPrepared] = await fixture.db.select({ profile: heartbeatRuns.runnerProfileJson }).from(heartbeatRuns).where(eq(heartbeatRuns.id, nextRunId));
await fixture.db.update(heartbeatRuns).set({ runnerProfileJson: { ...nextPrepared.profile,
nativeExecutionInput: nextOptions.execution } }).where(eq(heartbeatRuns.id, nextRunId));
const second = await executePaperclipNativeSession(nextOptions);
if (diskless) {
// Preserve sandbox/provider storage and workspace dependencies. Delete
// both native controller projections; admission must discover the source.
await rm(resolve(resolvePaperclipInstanceRoot(), "runtime", "paperclip-runner"), { recursive: true, force: true });
}
// Exercise the existing admission selector/rebinder, which supplies per-turn
// runtime snapshots. It does not restore PRP or supervisor files.
const admitContinuation = async (current: typeof execution) => {
const [row] = await fixture.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, current.binding.runId));
const selected = await findNativeSessionResumeRun(fixture.db, { ...current.binding, currentRunId: row.id,
normalizedSessionId: fixture.native.normalizedSessionId, beforeCreatedAt: row.createdAt });
const admission = buildNativeExecutionWithCheckpoint({ previousRun: selected,
normalizedSessionId: fixture.native.normalizedSessionId, executionTargetKind: "remote", buildExecution: () => current });
if (perTurn) expect(admission.checkpoint).not.toBeNull();
await fixture.db.update(heartbeatRuns).set({ runnerProfileJson: { ...row.runnerProfileJson,
nativeExecutionInput: current, nativeToolContractFingerprint: nativeToolContractFingerprintForTarget("remote", current.provider.kind),
...(perTurn ? { sessionCheckpoint: admission.checkpoint } : {}) } }).where(eq(heartbeatRuns.id, row.id));
};
await admitContinuation(nextOptions.execution);
if (scenario === "diskless-per-turn-read-retry") {
let rejectedReads = 0;
await expect(executePaperclipNativeSession({ ...nextOptions,
runnerExecutionTarget: { ...nextOptions.runnerExecutionTarget, runner: { ...runner,
execute: async request => {
if (request.args?.some(arg => arg.includes("base64 <") && arg.includes("runner-state.json"))) {
rejectedReads++;
throw new Error("fixture_remote_state_timeout");
}
return runner.execute(request);
},
} },
})).rejects.toThrow("fixture_remote_state_timeout");
expect(rejectedReads).toBe(1);
// Retry with the partial PRP projection left by the failed read. No
// fixture writes a suspended receipt or changes the surviving Runner.
}
const second = diskless ? await executeInController(nextOptions) : await executePaperclipNativeSession(nextOptions);
expect(second.exitCode).toBe(0);
const historyAfter = await runner.execute({ command: "sh", args: ["-c", `cat '${remoteCwd}'/.paperclip-runtime/paperclip-runner/sessions/*/filesystem/codex-home/fake-codex-state.json`] });
const providerAfter = JSON.parse(historyAfter.stdout);
@@ -540,12 +581,19 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
expect(calls.stdout.split("\n").filter(line => line === "thread/start")).toHaveLength(1);
const providerStarts = calls.stdout.split("\n").filter(line => line === "process-start").length;
expect(providerStarts).toBeGreaterThanOrEqual(1);
expect(providerStarts).toBeLessThanOrEqual(["legacy", "changed"].includes(scenario) ? 3 : hardRestart ? 2 : 1);
expect(providerStarts).toBeLessThanOrEqual(perTurn || ["legacy", "changed"].includes(scenario) ? 3 : hardRestart ? 2 : 1);
const processAfter = await runner.execute({ command: "sh", args: ["-c", `cat '${remoteCwd}'/.paperclip-runtime/paperclip-runner/sessions/*/runner/runner-process.identity`] });
expect(processAfter.exitCode).toBe(0);
if (["legacy", "changed"].includes(scenario)) expect(processAfter.stdout).not.toBe(processBefore.stdout);
if (perTurn || ["legacy", "changed"].includes(scenario)) expect(processAfter.stdout).not.toBe(processBefore.stdout);
else expect(processAfter.stdout).toBe(processBefore.stdout);
await finalizeNativeRun({ db: fixture.db, runId: nextRunId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
if (diskless && controller) {
const exited = new Promise<void>(resolveExit => controller!.once("exit", () => resolveExit()));
(controller as ReturnType<typeof spawn>).kill("SIGKILL");
await exited;
controller = undefined;
await rm(resolve(resolvePaperclipInstanceRoot(), "runtime", "paperclip-runner"), { recursive: true, force: true });
}
const thirdRunId = randomUUID();
const thirdLeaseId = randomUUID();
const [thirdRun] = await fixture.db.insert(heartbeatRuns).values({
@@ -562,9 +610,11 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
issue: { id: fixture.issueId, title: "Continue the same conversation", description: null, reviewPolicy: null },
environmentLeaseId: thirdLeaseId });
await fixture.db.insert(nativeRunFinalizations).values({ runId: thirdRunId, companyId, issueId: fixture.issueId, phase: "observed" });
const thirdExecution = { ...nextOptions.execution, completionContract: await boundContract(thirdRunId),
binding: { ...execution.binding, runId: thirdRunId }, task: { ...execution.task, prompt: "Complete the third conversation turn." } };
await admitContinuation(thirdExecution);
const third = await executePaperclipNativeSession({ ...nextOptions, runnerInstanceId: thirdNative.runnerInstanceId,
execution: { ...nextOptions.execution, completionContract: await boundContract(thirdRunId),
binding: { ...execution.binding, runId: thirdRunId }, task: { ...execution.task, prompt: "Complete the third conversation turn." } },
execution: thirdExecution,
runnerExecutionTarget: { ...nextOptions.runnerExecutionTarget, leaseId: thirdLeaseId },
});
expect(third.exitCode).toBe(0);
@@ -573,10 +623,11 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
const finalCalls = await runner.execute({ command: "cat", args: [`${remoteCwd}/provider-calls.log`] });
expect(finalCalls.stdout.split("\n").filter(line => line === "turn/start")).toHaveLength(3);
expect(finalCalls.stdout.split("\n").filter(line => line === "thread/start")).toHaveLength(1);
expect(finalCalls.stdout.split("\n").filter(line => line === "process-start")).toHaveLength(providerStarts);
expect(finalCalls.stdout.split("\n").filter(line => line === "process-start")).toHaveLength(providerStarts + (perTurn ? 1 : 0));
const finalProcess = await runner.execute({ command: "sh", args: ["-c", `cat '${remoteCwd}'/.paperclip-runtime/paperclip-runner/sessions/*/runner/runner-process.identity`] });
expect(finalProcess.stdout).toBe(processAfter.stdout);
expect(await closeIdleWarmNativeSessionsForRestart()).toEqual({ closed: 1, busy: 0, failed: 0 });
if (perTurn) expect(finalProcess.stdout).not.toBe(processAfter.stdout);
else expect(finalProcess.stdout).toBe(processAfter.stdout);
expect(await closeIdleWarmNativeSessionsForRestart()).toEqual({ closed: perTurn ? 0 : 1, busy: 0, failed: 0 });
const suspended = await runner.execute({ command: "sh", args: ["-c", `cat '${remoteCwd}'/.paperclip-runtime/paperclip-runner/sessions/*/runner/runner-state.json`] });
expect(suspended.exitCode).toBe(0);
expect(JSON.parse(suspended.stdout).lifecycle).toBe("suspended");
@@ -588,7 +639,7 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
await closeIdleWarmNativeSessionsForRestart();
await tunnel.close();
}
}, 180_000);
}, 360_000);
async function persistRestartEvidence(input: {
fixture: Awaited<ReturnType<typeof seedRun>>;
@@ -1277,10 +1328,11 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
} finally { await authority.stop(); }
});
it.each(["idle", "preparing_successor", "unclaimed_successor", "claimed_successor", "expired_preparation", "dispatching_successor", "foreign_preparation", "foreign_native_preparation", "wrong_issue", "new_work", "replaced_controller"] as const)(
"settles a completed warm checkpoint during early retirement without granting work: %s",
async scenario => {
const fixture = await seedRun(`WARM-EARLY-RETIREMENT-${scenario}`);
it.each((["idle", "preparing_successor", "unclaimed_successor", "claimed_successor", "expired_preparation", "dispatching_successor", "foreign_preparation", "foreign_native_preparation", "wrong_issue", "new_work", "replaced_controller", "unaccepted"] as const).flatMap(scenario =>
(["runner.suspend", "turn.stop"] as const).map(command => ({ scenario, command }))))(
"settles a completed warm checkpoint during early retirement without granting work: $scenario/$command",
async ({ scenario, command }) => {
const fixture = await seedRun(`WARM-EARLY-RETIREMENT-${scenario}-${command}`);
const binding = { companyId, issueId: fixture.issueId, runId: fixture.runId };
const identity = transportOptions(fixture, runtimeRoot).prpIdentity;
const authority = new DurablePrpControlPlane({
@@ -1305,7 +1357,7 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
});
await fixture.db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() })
.where(eq(heartbeatRuns.id, fixture.runId));
await fixture.db.update(nativeRunFinalizations).set({ phase: "committed", resultId,
await fixture.db.update(nativeRunFinalizations).set({ phase: "committed", resultId: scenario === "unaccepted" ? null : resultId,
leaseOwner: null, leaseExpiresAt: null,
...(scenario === "replaced_controller" ? { controllerBootId: randomUUID(), controllerGeneration: 2 } : {}),
}).where(eq(nativeRunFinalizations.runId, fixture.runId));
@@ -1314,7 +1366,7 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
if (scenario !== "idle") {
nextRunId = randomUUID();
const preparing = scenario !== "claimed_successor" && scenario !== "unclaimed_successor";
const otherIssue = scenario === "wrong_issue" ? (await seedRun("WARM-OTHER-ISSUE")).issueId : fixture.issueId;
const otherIssue = scenario === "wrong_issue" ? (await seedRun(`WARM-OTHER-ISSUE-${command}`)).issueId : fixture.issueId;
await fixture.db.insert(heartbeatRuns).values({ id: nextRunId, companyId, agentId,
issueId: otherIssue, scopeKind: "issue", contextSnapshot: { issueId: otherIssue },
...(preparing ? { runtimeMode: "legacy", executionStage: scenario === "dispatching_successor" ? "dispatching" : "preparing",
@@ -1331,7 +1383,7 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
}
await fixture.db.update(issues).set({ executionRunId: nextRunId }).where(eq(issues.id, fixture.issueId));
const before = (await loadNativeRunnerCheckpoint(fixture.db, binding))!;
authority.queueCommand(scenario === "new_work" ? "turn.start" : "runner.suspend", {}, `early-retirement-${scenario}`);
authority.queueCommand(scenario === "new_work" ? "turn.start" : command, {}, `early-retirement-${scenario}`);
if (!["idle", "preparing_successor", "claimed_successor"].includes(scenario)) {
await expect(authority.flushCheckpoint()).rejects.toThrow("native_runner_checkpoint_owner_lost");
expect((await loadNativeRunnerCheckpoint(fixture.db, binding))?.records).toEqual(before.records);
@@ -1342,7 +1394,7 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
saved.restoreSessionDirectory(restoredRoot);
const restored = JSON.parse(await readFile(resolve(restoredRoot, "control-plane", "control-plane-state.json"), "utf8"));
expect(restored.identity).toEqual(identity);
expect(restored.commands).toContainEqual(expect.objectContaining({ type: "runner.suspend", status: "pending" }));
expect(restored.commands).toContainEqual(expect.objectContaining({ type: command, status: "pending" }));
}
const [task] = await fixture.db.select({ executionRunId: issues.executionRunId }).from(issues)
.where(eq(issues.id, fixture.issueId));
@@ -1351,6 +1403,104 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
},
);
realProcessIt("durably retires an accepted warm Runner while its same-controller successor prepares", async () => {
const fixture = await seedRun("REAL-WARM-RETIREMENT");
const binding = { companyId, issueId: fixture.issueId, runId: fixture.runId };
const root = resolve(runtimeRoot, fixture.runId, "real-retirement");
const options = transportOptions(fixture, root);
const retirementController = await currentNativeControllerIdentity();
const diagnostics: string[] = [], checkpointFailures: unknown[] = [];
let authority!: DurablePrpControlPlane;
await fixture.db.update(nativeRunFinalizations).set({ controllerBootId: retirementController.bootId,
controllerGeneration: 1, leaseOwner: "real-retirement-owner", leaseExpiresAt: new Date(Date.now() + 60_000),
}).where(eq(nativeRunFinalizations.runId, fixture.runId));
const transport = createRunnerdCodexTransport({ ...options,
codexArgs: [...options.codexArgs, "--linger-after-turn-start"],
onDiagnostic: message => diagnostics.push(message),
controlPlaneRegistration: async core => {
authority = core;
const write = nativeRunnerCheckpointWriter(fixture.db, { ...binding, authorityRunId: fixture.runId,
bootId: retirementController.bootId, generation: 1, leaseOwner: "real-retirement-owner" });
core.setCheckpointPersistence(async checkpoint => {
try { await write(checkpoint); }
catch (error) { checkpointFailures.push(error); throw error; }
});
return registerRunnerPrpAuthority({ ...binding, authority: core });
},
});
let runnerPid: number | null = null, providerPid: number | null = null;
try {
await transport.transport.request("thread/start", { cwd: root, dynamicTools: [] });
await transport.transport.request("turn/start", { input: [{ type: "text", text: "Keep the provider alive after accepting its result." }] });
runnerPid = transport.evidence().runnerPid;
providerPid = transport.evidence().providerPid;
expect(runnerPid).toEqual(expect.any(Number));
expect(providerPid).toEqual(expect.any(Number));
await waitForCondition("active real provider before accepted-result retirement", () =>
authority.store.state.committedEvents.some(event => event.eventType === "turn.started"));
await authority.flushCheckpoint();
const [run] = await fixture.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId));
const resultId = randomUUID();
await fixture.db.insert(nativeRunResults).values({ id: resultId, ...binding, turnId: options.prpIdentity.turnId,
completionContractId: run.completionContractId!, serverFingerprint: "real-retirement-accepted",
schemaStatus: "accepted", canonicalSha256: "a".repeat(64),
resultJson: { schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", summary: "Accepted before provider cleanup." },
});
await fixture.db.update(nativeRunFinalizations).set({ phase: "committed", resultId, leaseOwner: null, leaseExpiresAt: null })
.where(eq(nativeRunFinalizations.runId, fixture.runId));
await fixture.db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() })
.where(eq(heartbeatRuns.id, fixture.runId));
const nextRunId = randomUUID();
const [nextRun] = await fixture.db.insert(heartbeatRuns).values({ id: nextRunId, companyId, agentId,
issueId: fixture.issueId, nativeIssueId: fixture.issueId, scopeKind: "issue", status: "running", runtimeMode: "native",
invocationSource: "assignment", contextSnapshot: { issueId: fixture.issueId } }).returning();
const next = await prepareNativeHeartbeatRun({ db: fixture.db, run: nextRun,
issue: { id: fixture.issueId, title: "Next conversation", description: null }, environmentLeaseId: randomUUID() });
await fixture.db.insert(nativeRunFinalizations).values({ runId: nextRunId, companyId, issueId: fixture.issueId, phase: "observed" });
await fixture.db.update(issues).set({ executionRunId: nextRunId }).where(eq(issues.id, fixture.issueId));
const [contract] = await fixture.db.select().from(completionContracts).where(eq(completionContracts.id, run.completionContractId!));
const nextExecution = buildNativeExecutionInput({ companyId, agentId, runId: nextRunId,
issue: { id: fixture.issueId, identifier: "NRR-NATIVE-RETIREMENT", title: "Next conversation", description: null, workMode: "standard" },
taskPrompt: "Start only after the previous provider durably suspends.",
workspace: { id: randomUUID(), cwd: root, repoUrl: null, repoRef: null, branchName: null },
normalizedSessionId: next.normalizedSessionId, runtimeContext: nativeRuntimeContextFixture(),
completionContract: { id: contract.id, sha256: contract.canonicalSha256, schemaVersion: contract.schemaVersion, contract: contract.contractJson as never },
});
const fixtureStoppedAfterRealClaim = new Error("fixture stopped before next provider dispatch");
await expect(executePaperclipNativeSession({ db: fixture.db, execution: nextExecution, runnerInstanceId: next.runnerInstanceId,
onExecutionLeaseAcquired: async () => {
const [claimed] = await fixture.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, nextRunId));
expect(claimed.controllerBootId).toBe(retirementController.bootId);
expect(claimed.leaseOwner).toEqual(expect.any(String));
expect(claimed.leaseExpiresAt!.getTime()).toBeGreaterThan(Date.now());
await expect(transport.transport.close(), diagnostics.join("\n")).resolves.toBeUndefined();
throw fixtureStoppedAfterRealClaim;
},
})).rejects.toBe(fixtureStoppedAfterRealClaim);
expect(checkpointFailures).toEqual([]);
const state = JSON.parse(await readFile(resolve(root, "runner", "runner-state.json"), "utf8"));
expect(state).toMatchObject({ lifecycle: "suspended", runId: fixture.runId,
normalizedSessionId: options.prpIdentity.normalizedSessionId, runnerInstanceId: options.prpIdentity.runnerInstanceId });
const saved = (await loadNativeRunnerCheckpoint(fixture.db, binding))!;
const commands = [...saved.records].filter(([key]) => key.startsWith("command:")).map(([, value]) => JSON.parse(value));
for (const type of ["turn.stop", "runner.drain", "runner.suspend"]) {
expect(commands, diagnostics.join("\n")).toContainEqual(expect.objectContaining({ type, status: "completed" }));
}
expect(commands.filter(command => command.type === "turn.start")).toHaveLength(1);
const [task] = await fixture.db.select().from(issues).where(eq(issues.id, fixture.issueId));
expect(task.executionRunId).toBe(nextRunId);
} catch (error) {
throw new Error(`${String(error)}; owned retirement diagnostics ${JSON.stringify({
checkpointFailures: checkpointFailures.map(String), diagnostics,
commands: authority?.store.state.commands.map(({ type, status }) => ({ type, status })),
})}`);
} finally {
await transport.transport.close().catch(() => undefined);
if (providerPid && processAlive(providerPid)) process.kill(providerPid, "SIGKILL");
await stopOwnedProcessGroup(runnerPid, root).catch(() => undefined);
}
}, 45_000);
it("applies checkpoint deltas in Postgres without rewriting authority and removes only evicted events", async () => {
const fixture = await seedRun("CHECKPOINT-DELTA");
const root = resolve(runtimeRoot, fixture.runId);
@@ -1463,8 +1613,8 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
realProcessIt("keeps checkpoint SQL independent of streamed output volume with a real Postgres writer", async () => {
const samples: Array<{ chunks: number; checkpointWrites: number; transactions: number;
changedRows: number; checkpointBytes: number; walBytes: number; elapsedMs: number; idleSql: number;
streamSql: number; streamTransactions: number; streamChangedRows: number }> = [];
for (const chunks of [8, 512]) {
streamSql: number; streamTransactions: number; streamChangedRows: number; renewalSql: number }> = [];
for (const chunks of [8, 512, 8192]) {
const fixture = await seedRun(`CHECKPOINT-COST-${chunks}`);
const root = resolve(runtimeRoot, fixture.runId);
const releaseOutput = resolve(root, "release-output");
@@ -1472,19 +1622,31 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
const owner = { ...binding, authorityRunId: fixture.runId, bootId: successor.bootId,
generation: 1, leaseOwner: "cost-controller" };
await fixture.db.update(nativeRunFinalizations).set({ controllerBootId: owner.bootId,
controllerGeneration: 1, leaseOwner: owner.leaseOwner, leaseExpiresAt: new Date(Date.now() + 120_000),
controllerPid: successor.pid, controllerProcessStartedAt: successor.processStartedAt,
controllerGeneration: 1, leaseOwner: owner.leaseOwner, leaseExpiresAt: new Date(Date.now() + 60_000),
}).where(eq(nativeRunFinalizations.runId, fixture.runId));
let authority!: DurablePrpControlPlane;
let measuring = false;
let transactions = 0, changedRows = 0, checkpointBytes = 0;
const statements: string[] = [];
const capturedClients = [fixture.db.$client, checkpointDb(fixture.db).$client];
const renewalStatements: string[] = [];
let leaseLost = false;
const renewal = startNativeSessionExecutionLeaseRenewal({ db: fixture.db, ...binding, leaseOwner: owner.leaseOwner,
attempt: 0, controller: successor, remote: true, onLost: async () => { leaseLost = true; } });
const capturedClients = [fixture.db.$client, checkpointDb(fixture.db).$client, livenessDb(fixture.db).$client];
const priorDebug = capturedClients.map(client => client.options.debug);
// Capture statements only, never SQL parameters (encrypted state or secrets).
for (const client of capturedClients) {
client.options.debug = (_connection, query) => { if (measuring) statements.push(query); };
}
capturedClients.forEach((client, index) => {
client.options.debug = (_connection, query) => {
if (!measuring) return;
// Renewals are existing liveness traffic, reported separately. A
// checkpoint query on any pool still counts against the zero oracle.
if (index === 2 && !query.includes("native_runner_checkpoints")) renewalStatements.push(query);
else statements.push(query);
};
});
const write = nativeRunnerCheckpointWriter(fixture.db, owner);
let costFailure: unknown;
const transport = createRunnerdCodexTransport({ ...transportOptions(fixture, root),
codexArgs: ["--state-file", resolve(root, "provider.json"), "--linger-after-turn-start",
"--held-output", "--held-output-count", String(chunks), "--held-output-release-file", releaseOutput],
@@ -1516,7 +1678,7 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
const beforeStream = { sql: statements.length, transactions, changedRows };
await writeFile(releaseOutput, "release\n", { flag: "wx" });
await waitForCondition(`the final scripted text chunk of ${chunks}`, () => authority.store.state.committedEvents
.some(event => event.eventType === "item.delta" && JSON.stringify(event.envelope).includes(`still computing ${chunks - 1} `)), 60_000)
.some(event => event.eventType === "item.delta" && JSON.stringify(event.envelope).includes(`still computing ${chunks - 1} `)), chunks > 4096 ? 480_000 : 60_000)
.catch(error => { console.info("checkpoint-cost-stream-diagnostic", JSON.stringify({ chunks,
cursor: authority.store.state.ackedSourceSeq,
receivedTextEvents: authority.store.state.committedEvents.filter(event => event.eventType === "item.delta").length })); throw error; });
@@ -1528,13 +1690,14 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
&& query.includes("native_runner_checkpoints")).length;
expect(checkpointWrites).toBeGreaterThan(0); // The collector must see real control writes.
const snapshot = { chunks, checkpointWrites, transactions, changedRows, checkpointBytes,
walBytes: Number(walEnd.bytes), elapsedMs, idleSql: 0,
walBytes: Number(walEnd.bytes), elapsedMs, idleSql: 0, renewalSql: renewalStatements.length,
streamSql: statements.length - beforeStream.sql,
streamTransactions: transactions - beforeStream.transactions,
streamChangedRows: changedRows - beforeStream.changedRows };
// Setup control messages may coalesce differently. The released text-only
// interval must add exactly zero work, independent of that scheduling.
expect(snapshot).toMatchObject({ streamSql: 0, streamTransactions: 0, streamChangedRows: 0 });
expect(leaseLost).toBe(false);
statements.length = 0;
measuring = true;
await new Promise(done => setTimeout(done, 100));
@@ -1547,20 +1710,34 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
expect(statements.some(query => /insert into "native_runner_checkpoints"/i.test(query))).toBe(true);
measuring = false;
samples.push(snapshot);
console.info("native-checkpoint-cost-sample", JSON.stringify(snapshot));
} catch (error) {
costFailure = error;
throw error;
} finally {
measuring = false;
capturedClients.forEach((client, index) => { client.options.debug = priorDebug[index]; });
await transport.transport.close();
try { await transport.transport.close(); }
catch (error) {
if (costFailure) throw new AggregateError([costFailure, error], "Checkpoint cost measurement and cleanup failed");
throw error;
}
finally {
await renewal.stop().catch(error => {
if (costFailure) throw new AggregateError([costFailure, error], "Checkpoint cost measurement and lease renewal failed");
throw error;
});
}
}
}
const report = { schema: "paperclip.checkpoint-cost/v1", samples,
scope: "isolated Postgres; control operations included; WAL is diagnostic; no production capacity claim" };
if (process.env.PAPERCLIP_TEST_CHECKPOINT_COST_REPORT) await writeFile(process.env.PAPERCLIP_TEST_CHECKPOINT_COST_REPORT,
`${JSON.stringify(report, null, 2)}\n`, { flag: "wx", mode: 0o600 });
// Both stream sizes were checked independently; total setup counts remain
// Each stream size was checked independently; total setup counts remain
// diagnostic because control-message batching is scheduling-dependent.
console.info("native-checkpoint-cost", JSON.stringify(report));
}, 120_000);
}, 540_000);
realProcessIt.each([false, true])("adopts the same runner from Postgres after losing the controller filesystem (prepared task: %s)", async preparedTask => {
const fixture = await seedRun(`EMPTY-CONTROLLER-DISK-${preparedTask}`);
@@ -1579,9 +1756,10 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
heartbeatRunId: fixture.runId, issueId: fixture.issueId, status: "active", leasePolicy: "ephemeral",
provider: "plugin", providerLeaseId: "saved-task", metadata: { agentId, runnerTaskLease: true, runnerTaskDispatched: true } });
}
const originalLeaseExpiresAt = new Date(Date.now() + 60_000);
await fixture.db.update(nativeRunFinalizations).set({ controllerBootId: bootId,
controllerGeneration: 1, controllerPid: process.pid, controllerProcessStartedAt: new Date(),
leaseOwner: "original-controller", leaseExpiresAt: new Date(Date.now() + 60_000),
leaseOwner: "original-controller", leaseExpiresAt: originalLeaseExpiresAt,
}).where(eq(nativeRunFinalizations.runId, fixture.runId));
const options = {
...transportOptions(fixture, controllerRoot),
@@ -1647,9 +1825,13 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
await new Promise(resolveOutage => setTimeout(resolveOutage, outageMs));
expect(processAlive(runnerPid)).toBe(true);
expect(processAlive(providerPid)).toBe(true);
await fixture.db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 1) })
.where(eq(nativeRunFinalizations.runId, fixture.runId));
bootId = randomUUID();
if (originalLeaseExpiresAt.getTime() > Date.now()) {
const [waiting] = await claimNativeRestartRecoveries({ db: fixture.db, controller: { ...successor, bootId },
restartKind: "hard", runIds: [fixture.runId] });
expect(waiting).toMatchObject({ kind: "awaiting_evidence", reason: "remote_controller_lease_live", retryAt: originalLeaseExpiresAt });
await new Promise(resolveExpiry => setTimeout(resolveExpiry, Math.max(0, originalLeaseExpiresAt.getTime() - Date.now()) + 20));
}
const [claim] = await claimNativeRestartRecoveries({ db: fixture.db, controller: { ...successor, bootId },
restartKind: "hard", runIds: [fixture.runId] });
expect(claim).toMatchObject({ kind: "reattach_remote_runner", providerAttempt: 0, controllerGeneration: 2 });
@@ -1,6 +1,8 @@
import { onRetainedNativeSessionCleanupRelease } from "../retained-native-session-cleanup-notifications.js";
import { readIdleLocalWork, markIdleStartupComplete, markIdleIngressTracked } from "../idle-local-work.js";
import * as runnerCheckpoint from "./native-runner-checkpoint.js";
import * as stagedAttachmentRecovery from "./native-staged-attachment-recovery.js";
import * as controllerRecovery from "./native-controller-recovery.js";
import { DurablePrpControlPlane } from "../../vendor/paperclip-runner/index.js";
import * as environmentTaskRuntime from "../environment-task-runtime.js";
import * as pluginTaskRuntime from "./plugin-runner-task.js";
@@ -47,6 +49,7 @@ import {
import {
canonicalNativeRuntimeContextDigest,
CURSOR_DISTRIBUTION_PINS,
KIMI_DISTRIBUTION_CLOSURE_SHA256,
QUALIFIED_ACPX_PROFILES,
acpxRuntimeSessionDirectoryName,
resolveAcpxRuntimeRoot,
@@ -96,6 +99,7 @@ import { buildNativeHeartbeatPreparationSpans, createNativeRunTrace } from "./na
import { NativeRunnerOwnershipUnverifiedError } from "./native-runner-ownership.js";
import type { AdapterRuntimeEvent } from "../../adapters/index.js";
import type { AdapterUsageCheckpoint } from "@paperclipai/adapter-utils";
import { acknowledgedNativeStopExecutionHasStopped, isAcknowledgedNativeStop } from "../acknowledged-native-stop.js";
vi.mock("node:fs", async (importOriginal) => {
const actual = await importOriginal<typeof import("node:fs")>();
@@ -391,6 +395,8 @@ import {
REMOTE_RUNNER_CHILD_LAUNCH_SCRIPT,
verifyRemoteRunnerReattachment,
readRemoteProviderPackManifest,
prepareRemoteProviderPackOnTarget,
resolveRemoteProviderPackSource,
readBundledRemoteProviderPackManifest,
providerSessionIdentityFromDurableProviderState,
durableProviderCheckpointFailureReason,
@@ -416,6 +422,10 @@ import {
import { onNativeRunnerEvidence } from "./native-runner-evidence-signal.js";
beforeEach(() => {
// These transport-wiring fixtures use a partial Db double and synthetic IDs.
// The real selector/bootstrap proof is exercised with Postgres by the durable
// discovery and diskless admission suites, not by this backend unit boundary.
vi.spyOn(controllerRecovery, "resolveNativeControllerRecovery").mockResolvedValue({ kind: "no_prior_execution" });
state.drainMaintenance.mockReset().mockResolvedValue(undefined);
usageRecorder.capture.mockReset().mockImplementation(async receipt => receipt);
usageRecorder.create.mockReset().mockResolvedValue({ capture: usageRecorder.capture,
@@ -1946,6 +1956,40 @@ describe("remote provider pack manifest", () => {
expect(readRemoteProviderPackManifest(root).payload.pins.opencode).toBe(
"1.18.34",
);
// Both execution and setup Test consume this same normal installer. Reuse
// the complete controller fixture here instead of inventing a thin pack.
const selectedPack = resolveRemoteProviderPackSource(root);
for (const mode of ["staged", "preinstalled", "uploaded", "failed-final", "missing-transport"] as const) {
let transferred = false;
const syncIn = vi.fn(async (operations: { files: { sourcePath?: string; targetPath: string; kind: string }[] }[]) => {
expect(operations[0]?.files[0]).toMatchObject({ sourcePath: root,
targetPath: "/workspace/.paperclip-runtime/paperclip-runner/provider-pack", kind: "directory" });
transferred = true;
return { operations: [] };
});
const execute = vi.fn(async (command: { command: string; args?: string[]; timeoutMs?: number }) => {
expect(command.timeoutMs).toBeGreaterThan(0);
expect(command.timeoutMs).toBeLessThanOrEqual(30_000);
const verification = command.args?.[0] === "-e";
const packRoot = command.args?.[2];
const valid = mode === "staged" || mode === "preinstalled" && packRoot === "/opt/paperclip-runner/provider-pack"
|| transferred && mode !== "failed-final";
return { pid: null, startedAt: null, exitCode: verification && !valid ? 1 : 0, timedOut: false, signal: null,
stdout: command.args?.[0] === "--version" ? "1.18.34\n"
: command.args?.some(value => value.includes("for candidate in")) ? "/opt/paperclip-runner/provider-pack\n" : "",
stderr: verification && !valid ? "manifest mismatch" : "" };
});
const runner = { execute, ...(mode === "missing-transport" ? {} : { syncIn }) };
const preparation = prepareRemoteProviderPackOnTarget({
target: { kind: "remote", transport: "sandbox", remoteCwd: "/workspace", runner } as never,
runner, configuredProviderPackRoot: selectedPack.configuredRoot, manifest: selectedPack.manifest,
});
if (mode === "failed-final") await expect(preparation).rejects.toThrow("provider pack verification");
else if (mode === "missing-transport") await expect(preparation).rejects.toThrow("transport cannot stage a provider pack");
else await expect(preparation).resolves.toMatchObject({ source: mode, root: "/workspace/.paperclip-runtime/paperclip-runner/provider-pack",
manifest: selectedPack.manifest });
expect(syncIn).toHaveBeenCalledTimes(mode === "uploaded" || mode === "failed-final" ? 1 : 0);
}
const cursorPath = "provider-assets/cursor/linux-x64";
await mkdir(join(root, cursorPath), { recursive: true });
await writeFile(join(root, cursorPath, "runtime"), "pinned Cursor runtime");
@@ -1980,6 +2024,78 @@ describe("remote provider pack manifest", () => {
await writeFile(join(root, cursorPath, "runtime"), "substitute Cursor runtime");
expect(() => readRemoteProviderPackManifest(root)).toThrow("asset tree digest mismatch");
await writeFile(join(root, cursorPath, "runtime"), "pinned Cursor runtime");
const kimiPath = "provider-assets/kimi/linux-x64";
await mkdir(join(root, kimiPath), { recursive: true });
await writeFile(join(root, kimiPath, "runtime"), "pinned Kimi runtime");
const kimi = { version: QUALIFIED_ACPX_PROFILES.kimi.agentServerVersion,
profileDigest: QUALIFIED_ACPX_PROFILES.kimi.commandDigest,
closureDigest: `sha256:${KIMI_DISTRIBUTION_CLOSURE_SHA256["linux-x64"]}`,
qualification: "qualified", path: kimiPath, sha256: sha256DirectoryTree(join(root, kimiPath)) };
Object.assign(payload, { providers: { cursor, kimi } });
await mkdir(releaseMetadata, { recursive: true });
const writeKimiManifest = async () => {
await writeManifest();
await cp(join(root, "provider-pack.json"), manifestPath);
};
await writeKimiManifest();
expect(readRemoteProviderPackManifest(root).payload.providers?.kimi).toEqual(kimi);
expect(readBundledRemoteProviderPackManifest(manifestPath).payload.providers?.kimi?.qualification).toBe("qualified");
for (const field of ["version", "profileDigest", "closureDigest"] as const) {
const previous = kimi[field];
kimi[field] = field === "version" ? "older-kimi" : digest(`old-kimi-${field}`);
await writeKimiManifest();
expect(() => readRemoteProviderPackManifest(root)).toThrow("Kimi profile or closure");
expect(() => readBundledRemoteProviderPackManifest(manifestPath)).toThrow("Kimi profile or closure");
kimi[field] = previous;
}
kimi.closureDigest = `sha256:${KIMI_DISTRIBUTION_CLOSURE_SHA256["darwin-arm64"]}`;
await writeKimiManifest();
expect(() => readBundledRemoteProviderPackManifest(manifestPath)).toThrow("Kimi profile or closure");
kimi.closureDigest = `sha256:${KIMI_DISTRIBUTION_CLOSURE_SHA256["linux-x64"]}`;
for (const target of [{ platform: "linux", architecture: "arm64" }, { platform: "darwin", architecture: "arm64" }]) {
payload.target = target;
kimi.path = `provider-assets/kimi/${target.platform}-${target.architecture}`;
Object.assign(payload, { providers: {} });
await writeKimiManifest();
expect(() => readBundledRemoteProviderPackManifest(manifestPath)).toThrow("bundled image pack must target linux-x64");
}
payload.target = originalTarget;
kimi.path = kimiPath;
Object.assign(payload, { providers: { cursor, kimi } });
kimi.qualification = "pending";
await writeKimiManifest();
expect(() => readBundledRemoteProviderPackManifest(manifestPath)).toThrow("invalid candidate identity");
kimi.qualification = "qualified";
Object.assign(payload, { providers: { cursor, kimi }, candidateProviders: { kimi } });
await writeKimiManifest();
expect(() => readBundledRemoteProviderPackManifest(manifestPath)).toThrow("Kimi profile or closure");
Object.assign(payload, { providers: { cursor, kimi } });
Reflect.deleteProperty(payload, "candidateProviders");
await writeKimiManifest();
await writeFile(join(root, kimiPath, "runtime"), "substitute Kimi runtime");
expect(() => readRemoteProviderPackManifest(root)).toThrow("candidate asset tree digest mismatch");
await writeFile(join(root, kimiPath, "runtime"), "pinned Kimi runtime");
const threeCandidates = { cursor,
pi: { ...kimi, version: "1.0.0", qualification: "qualified", path: "provider-assets/pi/linux-x64" },
copilot: { ...kimi, qualification: "pending", version: QUALIFIED_ACPX_PROFILES.copilot.agentServerVersion,
profileDigest: QUALIFIED_ACPX_PROFILES.copilot.commandDigest,
closureDigest: "sha256:a3d8f4367cfa1694d79e3f8b7930b6fbfe42a229c272b375b11951d631db8d07",
path: "provider-assets/copilot/linux-x64" } };
for (const candidate of [threeCandidates.pi, threeCandidates.copilot]) {
await mkdir(join(root, candidate.path), { recursive: true });
await writeFile(join(root, candidate.path, "runtime"), "candidate runtime");
candidate.sha256 = sha256DirectoryTree(join(root, candidate.path));
}
Object.assign(payload, { candidateProviders: threeCandidates });
await writeKimiManifest();
expect(Object.keys(readRemoteProviderPackManifest(root).payload.candidateProviders!)).toHaveLength(3);
Object.assign(payload, { candidateProviders: { ...threeCandidates, unknown: kimi } });
await writeKimiManifest();
expect(() => readBundledRemoteProviderPackManifest(manifestPath)).toThrow("invalid candidate inventory");
Object.assign(payload, { candidateProviders: { unknown: kimi } });
await writeKimiManifest();
expect(() => readBundledRemoteProviderPackManifest(manifestPath)).toThrow("invalid candidate identity");
await rm(releaseMetadata, { recursive: true, force: true });
const candidatePath = "provider-assets/pi/linux-x64";
await mkdir(join(root, candidatePath), { recursive: true });
await writeFile(join(root, candidatePath, "runtime"), "pinned runtime");
@@ -3791,10 +3907,10 @@ describe("runtime question fallback", () => {
});
describe("native provider bootstrap environment", () => {
it.each(["pi", "cursor", "copilot", "antigravity"] as const)("does not promote ambient %s credentials when bindings are omitted", agent => {
it.each(["pi", "cursor", "copilot", "antigravity", "kimi"] as const)("does not promote ambient %s credentials when bindings are omitted", agent => {
const provider = { kind: "acpx", agent } as NativeExecutionInput["provider"];
const host = { PATH: "/host/bin", HOME: "/host/home", OPENROUTER_API_KEY: "ambient-pi",
CURSOR_API_KEY: "ambient-cursor", CURSOR_AUTH_TOKEN: "ambient-cursor-login", COPILOT_GITHUB_TOKEN: "ambient-copilot",
CURSOR_API_KEY: "ambient-cursor", CURSOR_AUTH_TOKEN: "ambient-cursor-login", COPILOT_GITHUB_TOKEN: "ambient-copilot", KIMI_API_KEY: "ambient-kimi",
GEMINI_API_KEY: "ambient-google", GOOGLE_API_KEY: "ambient-other-google", GEMINI_HOME: "/ambient-google-home",
GOOGLE_APPLICATION_CREDENTIALS: "/ambient-google-credential.json", PAPERCLIP_ACPX_ANTIGRAVITY_AUTH_JSON_SECRET: "ambient-google-subscription",
PAPERCLIP_ACPX_CREDENTIAL_BINDING: "ambient-forged-binding" };
@@ -7748,6 +7864,27 @@ describe("native session cancellation", () => {
expect(state.persistActivity).not.toHaveBeenCalled();
});
it.each([
{ companyId: "another-company" }, { runId: "another-run" }, { issueId: "another-issue" },
{ scope: "issue" }, { effects: ["cancel_issue"] }, { intentAuditId: null },
])("does not fence a failed retry for a foreign or stale Stop receipt %j", async changed => {
const intent = { schema: "paperclip.native-cancellation.v1", intentId: "native-cancellation:earlier-stop",
...execution.binding, scope: "run", reasonCode: "cancellation_run_only", effects: ["release_run_resources"],
dispatchState: "acknowledged", dispatched: true, intentAuditId: "earlier-intent-audit",
acknowledgementAuditId: "earlier-ack-audit", ...changed };
const coordinator = { runId: execution.binding.runId, assessmentId: null, phase: "retryable_failure" };
const persistence = cancellationDb({ status: "failed", coordinator, resultJson: { nativeCancellation: intent } });
const before = structuredClone(persistence.getResultJson());
state.cancel.mockClear();
state.persistActivity.mockClear();
await expect(cancelNativeSession(execution.binding.runId, "retry Stop", { db: persistence.db, scope: "run" })).rejects.toThrow();
expect(persistence.updates).toEqual([]);
expect(coordinator.phase).toBe("retryable_failure");
expect(persistence.getResultJson()).toEqual(before);
expect(state.cancel).not.toHaveBeenCalled();
expect(state.persistActivity).not.toHaveBeenCalled();
});
it("does not mistake a recovered running attempt for the failed retry that reserved Stop", async () => {
const id = "11111111-1111-4111-8111-111111111111";
const persistence = cancellationDb({ status: "running",
@@ -7773,6 +7910,39 @@ describe("native session cancellation", () => {
expect(persistence.getResultJsonUpdateCount()).toBe(writes);
});
it("fences a failed retry whose earlier Stop is already acknowledged without changing its receipts", async () => {
const intent = { schema: "paperclip.native-cancellation.v1", intentId: "native-cancellation:earlier-stop",
...execution.binding, scope: "run", reasonCode: "cancellation_run_only",
effects: ["release_run_resources"], dispatchState: "acknowledged", dispatched: true,
intentAuditId: "earlier-intent-audit", acknowledgementAuditId: "earlier-ack-audit",
acknowledgedAt: "2026-01-01T00:00:00.000Z", decisionId: null, priorCoordinatorDecisionId: null };
const coordinator = { runId: execution.binding.runId, assessmentId: null, phase: "retryable_failure",
nextAttemptAt: new Date(Date.now() + 30_000) };
const persistence = cancellationDb({ status: "failed", coordinator,
resultJson: { nativeCancellation: intent, executionCancellation: { state: "requested", requestedAt: "later-stop" } } });
const before = structuredClone(persistence.getResultJson());
state.cancel.mockClear();
state.persistActivity.mockClear();
for (let attempt = 0; attempt < 2; attempt++) {
await expect(cancelNativeSession(execution.binding.runId, "retry Stop", { db: persistence.db, scope: "run" }))
.resolves.toMatchObject({ dispatched: true, auditId: intent.intentAuditId });
expect(coordinator).toMatchObject({ phase: "terminal_failure", failureCode: "native_retry_cancelled", nextAttemptAt: null });
expect(persistence.getResultJson()).toEqual(before);
}
expect(persistence.updates.filter(update => update.table === nativeRunFinalizations)).toHaveLength(1);
expect(persistence.getResultJsonUpdateCount()).toBe(0);
expect(state.cancel).not.toHaveBeenCalled();
expect(state.persistActivity).not.toHaveBeenCalled();
const stopped = { id: execution.binding.runId, companyId: execution.binding.companyId,
nativeIssueId: execution.binding.issueId, status: "cancelled", processPid: null, processGroupId: null,
resultJson: { ...persistence.getResultJson(), cancelledByActorType: "user", cancelledByUserId: "operator" } };
const lease = { provider: "daytona", providerLeaseId: "unconfirmed-provider-resource", status: "active",
releasedAt: null, cleanupStatus: null, metadata: {} };
const leaseDb = { select: () => ({ from: () => ({ where: async () => [lease] }) }) } as unknown as Db;
expect(isAcknowledgedNativeStop(stopped)).toBe(true);
await expect(acknowledgedNativeStopExecutionHasStopped(leaseDb, stopped)).resolves.toBe(false);
});
it("uses the reserved caller UUID and permits same-intent and default retries", async () => {
const id = "11111111-1111-4111-8111-111111111111";
const persistence = cancellationDb({ resultJson: { startupCancellation: { cancellationRequestId: id } } });
@@ -9196,6 +9366,44 @@ describe("native warm session supervision", () => {
expect(state.execute).toHaveBeenCalledTimes(2);
});
it("claims native execution authority before retiring a superseded warm identity", async () => {
const { base, next, run } = fixture("identity-native-claim-order");
const updates: Array<{ table: unknown; values: Record<string, unknown> }> = [];
let inspected = false;
const close = vi.fn(async () => {
if (inspected) return;
inspected = true;
expect(updates.some(update => update.table === nativeRunFinalizations
&& update.values.leaseOwner && update.values.controllerBootId)).toBe(true);
});
state.execute.mockReset().mockImplementationOnce(async options => {
options.onSession?.({ close }); return result;
}).mockImplementationOnce(async () => result);
await run(base);
// Normal runtime selection already converted preparation to native and
// inserted an unclaimed observed coordinator before calling this facade.
await executePaperclipNativeSession({ db: leaseDb(next, {}, {}, updates),
execution: next, runnerInstanceId: "runner", useRunnerd: true });
expect(close).toHaveBeenCalledOnce();
expect(state.execute).toHaveBeenCalledTimes(2);
});
it.each(["cancelled", "foreign-owner"] as const)("does not retire or stage a warm successor after claim denial: %s", async denial => {
const { base, next, run } = fixture(`identity-native-claim-denied-${denial}`);
const close = vi.fn(async () => undefined);
state.execute.mockReset().mockImplementationOnce(async options => {
options.onSession?.({ close }); return result;
});
await run(base);
state.stageNativeRunnerWakeAttachments.mockClear();
const db = leaseDb(next, denial === "foreign-owner" ? { leaseOwner: "different-controller" } : {},
{}, [], {}, denial === "cancelled" ? "cancelled" : "running");
await expect(executePaperclipNativeSession({ db, execution: next, runnerInstanceId: "runner", useRunnerd: true })).rejects.toThrow();
expect(close).not.toHaveBeenCalled();
expect(state.stageNativeRunnerWakeAttachments).not.toHaveBeenCalled();
expect(state.execute).toHaveBeenCalledTimes(1);
});
it("does not retire an active turn when a new session identity arrives", async () => {
const { base, next, run } = fixture("identity-active");
let finish!: () => void;
@@ -12176,6 +12384,55 @@ describe("runnerd provider runtime wiring", () => {
).toThrow("scope_closed");
});
it.each(["attached", "changed-admission", "changed-template", "changed-template-digest", "changed-preparation"] as const)(
"verifies frozen staged admission after normal wake attachment preparation: %s", async mode => {
const current = scopedNativeExecution(`staged-attachment-${mode}`);
current.task.prompt = "Inspect attachment ref:original";
const root = await initializeScopedRunnerRoot(current, "attachment-runner");
const sourceIdentity = { runnerInstanceId: "attachment-runner", environmentLeaseId: "attachment-lease",
runId: "attachment-source", normalizedSessionId: current.session.normalizedSessionId!, turnId: "source-turn", itemId: "source-item" };
const targetIdentity = { ...sourceIdentity, runId: current.binding.runId, turnId: "target-turn", itemId: "target-item" };
const controller = new DurablePrpControlPlane({ stateDirectory: join(root, "control-plane"), identity: sourceIdentity,
expectedRunnerVersion: "fixture", expectedRunnerDigest: `sha256:${"a".repeat(64)}` });
controller.stageRunAttachment(targetIdentity, { provider: { kind: "codex", instructions: "prepared attachment grant" } });
const path = join(root, "control-plane/control-plane-state.json");
const saved = JSON.parse(await readFile(path, "utf8"));
const canonicalTemplateDigest = saved.stagedRunAttachment.templateDigest;
if (mode === "changed-template" || mode === "changed-template-digest") {
saved.stagedRunAttachment.runAttachTemplate.provider.instructions = "substituted grant";
if (mode === "changed-template-digest") saved.stagedRunAttachment.templateDigest = `sha256:${nativeSha256(saved.stagedRunAttachment.runAttachTemplate)}`;
await writeFile(path, JSON.stringify(saved));
}
state.stageNativeRunnerWakeAttachments.mockImplementationOnce(async () => {
if (mode === "changed-preparation") current.task.prompt = "Inspect attachment ref:mutated-during-preparation";
return { attachments: [{ id: "attachment-original" }], cleanup: async () => undefined } as never;
});
state.renderNativeRunnerStagedAttachmentPrompt.mockReturnValueOnce("Materialized attachment: .paperclip-inbound/current/original");
const admission = structuredClone(current);
if (mode === "changed-admission") admission.task.prompt = "Inspect attachment ref:changed";
const reached = new Error("verified staged admission reached recheck");
const restore = vi.spyOn(runnerCheckpoint, "restoreNativeRunnerCheckpoint").mockResolvedValue(null);
const header = vi.spyOn(runnerCheckpoint, "loadNativeRunnerCheckpointHeader").mockResolvedValue("staged header");
const verify = vi.spyOn(stagedAttachmentRecovery, "resolveNativeStagedAttachmentRecovery")
.mockResolvedValueOnce({ currentExecution: admission, sourceIdentity, targetIdentity,
templateDigest: canonicalTemplateDigest } as never)
.mockRejectedValueOnce(reached);
try {
const resume = executePaperclipNativeSession({ db: leaseDb(current), execution: current, useRunnerd: true,
runnerInstanceId: "attachment-runner", restartRecovery: { kind: "reattach_remote_runner", runId: current.binding.runId,
leaseOwner: "controller", controllerGeneration: 2, providerAttempt: 1, restartKind: "hard", recoveryRequestId: null,
remote: { kind: "sandbox", leaseId: "attachment-lease", providerLeaseId: "sandbox", remoteCwd: "/workspace" } } as never,
runnerExecutionTarget: { kind: "remote", transport: "sandbox", leaseId: "attachment-lease", remoteCwd: "/workspace",
runner: { execute: vi.fn() }, sandboxLeaseAcquisition: { outcome: "resumed", providerLeaseId: "sandbox" } } as never });
if (mode === "attached") { await expect(resume).rejects.toBe(reached); expect(verify).toHaveBeenCalledTimes(2); }
else await expect(resume).rejects.toThrow(mode === "changed-template" ? "staged_attachment_binding_mismatch"
: mode === "changed-preparation" ? "native_attachment_admission_changed" : "staged_attachment_recovery_unproven");
expect(state.stageNativeRunnerWakeAttachments).toHaveBeenCalledWith(expect.objectContaining({ binding: expect.objectContaining({
companyId: current.binding.companyId, issueId: current.binding.issueId, agentId: current.binding.agentId, runId: current.binding.runId,
}) }));
} finally { restore.mockRestore(); header.mockRestore(); verify.mockRestore(); await controller.stop(); }
});
it("passes the run checkpoint active turn into restart recovery", async () => {
state.createBackend.mockClear();
state.createTransport.mockClear();
@@ -12206,6 +12463,27 @@ describe("runnerd provider runtime wiring", () => {
);
});
it.each(["warm", "per_turn"] as const)("defers a %s retained-owner wait before claiming a provider attempt", async mode => {
const fresh = scopedNativeExecution(`retained-owner-${mode}`);
fresh.session.lifecyclePolicy = mode === "warm"
? { mode: "warm", idleTimeoutMs: 300_000 } : { mode: "per_turn", idleTimeoutMs: null };
const deadline = new Date(Date.now() + 60_000);
vi.mocked(controllerRecovery.resolveNativeControllerRecovery).mockResolvedValueOnce({ kind: "waiting_for_owner", deadline });
const defer = vi.spyOn(controllerRecovery, "deferNativeControllerRecovery").mockResolvedValueOnce();
const onExecutionLeaseAcquired = vi.fn();
const updates: Array<{ table: unknown; values: Record<string, unknown> }> = [];
const db = leaseDb(fresh, {}, {}, updates);
await expect(executePaperclipNativeSession({ db, execution: fresh, runnerInstanceId: "waiting-controller",
useRunnerd: true, onExecutionLeaseAcquired,
runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: "/workspace",
environmentId: "retained-environment", leaseId: "retained-lease" },
})).rejects.toMatchObject({ name: "NativeControllerRecoveryWaitingError", retryAt: deadline });
expect(defer).toHaveBeenCalledWith(db, expect.objectContaining({ ...fresh.binding, retryAt: deadline }));
expect(onExecutionLeaseAcquired).not.toHaveBeenCalled();
expect(updates.some(update => update.table === nativeRunFinalizations && update.values.attempt !== undefined)).toBe(false);
expect(state.createBackend).not.toHaveBeenCalled();
});
it("rejects overlapping runs for the same runnerd provider session scope", async () => {
const first = {
...execution,
@@ -12566,7 +12844,7 @@ describe("runnerd provider runtime wiring", () => {
} as NativeExecutionInputV1;
const identity = { runId: current.binding.runId, normalizedSessionId: current.session.normalizedSessionId!,
runnerInstanceId: `admitted-${provider}`, environmentLeaseId: "admitted-lease",
turnId: `turn-${current.binding.runId}`, itemId: `item-${current.binding.runId}` };
turnId: `turn_attachment_${randomUUID()}`, itemId: `item_attachment_${randomUUID()}` };
const root = await initializeScopedRunnerRoot(current, identity.runnerInstanceId);
await mkdir(join(root, "control-plane"), { recursive: true });
await writeFile(join(root, "control-plane", "control-plane-state.json"), JSON.stringify(durableControlPlaneState(identity)));
@@ -12596,9 +12874,56 @@ describe("runnerd provider runtime wiring", () => {
runnerArtifact: artifact, adoptExistingRunner: { pid: 123, startedAt: "2026-09-19T10:00:00.000Z" } });
expect(state.resolveRemoteRunnerBinary).not.toHaveBeenCalled();
expect(execute.mock.calls.some(([request]) => request.args.includes("paperclip-runner-launch") || request.args.includes("--build-metadata"))).toBe(false);
const options = state.createTransport.mock.calls.at(-1)![0] as RunnerTransportOptions & { prepareExternalRunnerState: () => Promise<void> };
await expect(options.prepareExternalRunnerState()).resolves.toBeUndefined();
expect(execute.mock.calls.some(([request]) => request.args.includes("paperclip-runner-launch")
|| request.args.includes("paperclip-runner-claim-unstarted-session"))).toBe(false);
});
it.each(["claude", "codex"].flatMap(provider => (["per_turn", "warm"] as const).flatMap(policy => ["canonical", "local-receipt", "local-receipt-missing-terminal", "local-receipt-pending-command", "quarantined", "empty-runner", "empty-runner-quarantined", "nonempty-runner", "wrong-identity", "active", "foreign-scope", "no-stop-proof", "not-suspended", "pending-outbox", "pending-command", "missing-terminal", "ambiguous", "contradictory"].map(mode => [provider, policy, mode] as const))))(
it.each(["verified", "no-proof", "dead", "stale"] as const)("reuses only verified idle warm process during actual remote preparation: %s", async mode => {
const previous = scopedNativeExecution(`warm-live-preparation-${mode}`);
previous.session.lifecyclePolicy = { mode: "warm", idleTimeoutMs: 300_000 };
const current = { ...previous, binding: { ...previous.binding, runId: `${previous.binding.runId}-next` } };
const identity = { runId: previous.binding.runId, normalizedSessionId: previous.session.normalizedSessionId!,
runnerInstanceId: "retained-warm-runner", environmentLeaseId: "retained-lease", turnId: "prior-turn", itemId: "prior-item" };
const root = await initializeScopedRunnerRoot(previous, identity.runnerInstanceId);
await writeRunnerAuthority(root, identity, "suspended");
const projections = vi.spyOn(controllerRecovery, "resolveNativeControllerRecovery").mockResolvedValue({ kind: "no_prior_execution" });
onTestFinished(() => projections.mockRestore());
const db = leaseDb(current), ordinarySelect = db.select.bind(db);
db.select = ((fields?: Record<string, unknown>) => fields?.status === heartbeatRuns.status && fields?.runnerProfileJson === heartbeatRuns.runnerProfileJson
? priorNativeRunDb(previous).select(fields as never) : ordinarySelect(fields as never)) as Db["select"];
let alive = true;
const execute = vi.fn(async (request: { args: string[] }) => ({ exitCode: request.args[2] === "paperclip-runner-recovery-check" && !alive ? 45 : 0,
timedOut: false, stderr: "", stdout: request.args[0] === "--build-metadata" ? JSON.stringify({
schema: "paperclip-runner/runnerd-build-metadata/v1", binaryName: "paperclip-runnerd",
packageName: "@paperclipai/paperclip-runner", binaryContractVersion: 2,
durableSessionCapabilities: ["unlimited_runtime", "connection_lease_renewal"], prpTransportModes: ["listen_ws"],
}) : request.args[0] === "--version" ? "codex-cli 0.160.0" : request.args[1]?.includes("base64")
? Buffer.from(JSON.stringify({ ...identity, lifecycle: "ready", outbox: [], pendingTerminalDelivery: null })).toString("base64")
: request.args[1]?.includes("cat --") ? `nonce\n123\n2026-09-19T10:00:00.000Z\n${identity.runnerInstanceId}\nlinux:abcd-1234:100\n` : "" }));
const revalidate = vi.fn(async () => undefined);
const syncIn = vi.fn(async () => undefined);
await createRunnerdBackend({ db, execution: current, runnerInstanceId: identity.runnerInstanceId,
runnerIngressAuthorized: true,
...(mode === "no-proof" ? {} : { remoteWarmRecovery: { identity, reuseProcess: true, previousExecution: previous,
checkpoint: {} as never, remote: { providerLeaseId: "retained-sandbox", remoteCwd: "/workspace" }, revalidate } }),
runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: "/workspace", environmentId: "environment",
leaseId: identity.environmentLeaseId, providerKey: "daytona", effectiveCapabilities: { runnerWebSocketIngress: true },
sandboxLeaseAcquisition: { outcome: "resumed", providerLeaseId: "retained-sandbox" }, runner: { execute, syncIn } } as never });
state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!();
const options = state.createTransport.mock.calls.at(-1)![0] as RunnerTransportOptions & { prepareExternalRunnerState: () => Promise<void> };
if (mode === "dead") alive = false;
if (mode === "stale") revalidate.mockRejectedValueOnce(new Error("fixture_warm_owner_changed"));
if (mode === "verified") await expect(options.prepareExternalRunnerState()).resolves.toBeUndefined();
else await expect(options.prepareExternalRunnerState()).rejects.toThrow(mode === "no-proof" ? "resumed_not_suspended"
: mode === "dead" ? "runner_remote_process_identity_unavailable" : "fixture_warm_owner_changed");
expect(syncIn).not.toHaveBeenCalled();
expect(execute.mock.calls.some(([request]) => request.args.includes("paperclip-runner-launch")
|| request.args.includes("paperclip-runner-claim-unstarted-session"))).toBe(false);
});
it.each(["claude", "codex"].flatMap(provider => (["per_turn", "warm"] as const).flatMap(policy => ["canonical", "resumed", "resumed-timeout", "local-receipt", "local-receipt-missing-terminal", "local-receipt-pending-command", "quarantined", "empty-runner", "empty-runner-quarantined", "nonempty-runner", "wrong-identity", "active", "foreign-scope", "no-stop-proof", "not-suspended", "pending-outbox", "pending-command", "missing-terminal", "ambiguous", "contradictory"].filter(mode => policy === "per_turn" || !mode.startsWith("resumed")).map(mode => [provider, policy, mode] as const))))(
"recovers a blocked %s %s plugin checkpoint on a replacement lease: %s", async (provider, policy, mode) => {
const { permissionPolicy: _permissionPolicy, ...claudeProfile } = QUALIFIED_ACPX_PROFILES.claude;
const first = { ...scopedNativeExecution(`plugin-${provider}-${policy}-${mode}`),
@@ -12646,8 +12971,13 @@ describe("runnerd provider runtime wiring", () => {
if (mode === "no-stop-proof") throw new Error("Plugin Runner retained session is not authorized");
return "old-task";
});
let timedOut = false;
const task = vi.fn(async (operation: any) => {
environmentTaskOperationSchema.parse(operation);
if (mode === "resumed-timeout" && operation.kind === "read_state" && !timedOut) {
timedOut = true;
throw new Error("fixture_remote_state_timeout");
}
if (operation.kind === "read_state") return { kind: "state", taskId: "replacement-task", state: {
...durableRunnerState(identity, mode === "not-suspended" ? "ready" : "suspended"),
outbox: mode === "pending-outbox" ? [{}] : [], pendingTerminalDelivery: null,
@@ -12666,7 +12996,7 @@ describe("runnerd provider runtime wiring", () => {
managedAiCredentialHome: home, managedAiCredentialIdentity: "revision", managedAiAccountIdentity: "account",
runnerEnvironment: { ANTHROPIC_API_KEY: "synthetic-fixture" },
runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: ".", leaseId: "replacement-lease", runnerTask: task,
sandboxLeaseAcquisition: { outcome: "replacement", providerLeaseId: "replacement-task" } } as never };
sandboxLeaseAcquisition: { outcome: mode.startsWith("resumed") ? "resumed" : "replacement", providerLeaseId: "replacement-task" } } as never };
if (throughExecutor) {
const validated = validatePrpStructuredRunResult({ schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", summary: "done",
completionClaim: { contractRevision: next.completionContract.contract.revision, objectiveSatisfied: true, criteria: [], remainingWork: [] },
@@ -12676,12 +13006,19 @@ describe("runnerd provider runtime wiring", () => {
normalizedSessionId: next.session.normalizedSessionId, providerSessionId: "original-provider-history", driverKind: next.session.driverKind,
driverVersion: "1", nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null });
}
if (mode === "resumed-timeout") {
await expect(createRunnerdBackend(launch)).rejects.toThrow("fixture_remote_state_timeout");
expect(existsSync(join(scoped, "plugin-runner-state.json"))).toBe(false);
expect(await readFile(join(scoped, "control-plane/control-plane-state.json"), "utf8")).toBe(journal);
}
const backend = throughExecutor ? executePaperclipNativeSession({ ...launch, useRunnerd: true }) : createRunnerdBackend(launch);
if (["canonical", "local-receipt", "quarantined", "empty-runner", "empty-runner-quarantined"].includes(mode)) {
if (["canonical", "resumed", "resumed-timeout", "local-receipt", "quarantined", "empty-runner", "empty-runner-quarantined"].includes(mode)) {
await expect(backend).resolves.toBeDefined();
expect(state.createBackend).toHaveBeenCalledOnce();
expect(owner).toHaveBeenCalledWith(expect.objectContaining({ resumeFrom: "old-task", priorIdentity: identity }));
expect(resume).toHaveBeenCalledWith(db, expect.objectContaining({ leaseId: "replacement-lease", priorIdentity: identity, inspectCheckpoint: true }));
expect(owner).toHaveBeenCalledWith(expect.objectContaining({ priorIdentity: identity,
...(!mode.startsWith("resumed") ? { resumeFrom: "old-task" } : {}) }));
if (!mode.startsWith("resumed")) expect(resume).toHaveBeenCalledWith(db, expect.objectContaining({ leaseId: "replacement-lease", priorIdentity: identity, inspectCheckpoint: true }));
else expect(task).toHaveBeenCalledWith({ kind: "read_state", identity });
expect(JSON.parse(await readFile(join(scoped, "plugin-runner-state.json"), "utf8"))).toMatchObject({ ...identity, lifecycle: "suspended" });
expect(await readFile(join(scoped, "control-plane/control-plane-state.json"), "utf8")).toBe(journal);
} else {
File diff suppressed because it is too large. Load diff
@@ -5,6 +5,7 @@ import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js";
import { findNativeSessionResumeRun } from "./native-session-resume.js";
import { resolveNativeControllerRecovery } from "./native-controller-recovery.js";
const externalUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_TEST_DATABASE_URL");
const supported = Boolean(externalUrl) || (await getEmbeddedPostgresTestSupport()).supported;
@@ -58,7 +59,7 @@ const supported = Boolean(externalUrl) || (await getEmbeddedPostgresTestSupport(
const saved = source();
await db.insert(heartbeatRuns).values([saved, run(1), run(2)]);
const result = await select();
expect(result).toEqual({ ...saved, runnerProfileJson: { nativeExecutionInput, sessionCheckpoint: checkpoint, nativeToolContractFingerprint: "fingerprint" },
expect(result).toEqual({ ...saved, runnerProfileJson: { nativeExecutionInput, sessionCheckpoint: checkpoint, nativeToolContractFingerprint: "fingerprint", nativeControllerRecovery: null },
contextSnapshot: { paperclipWake, paperclipIssue } });
const stored = await db.select({ profile: heartbeatRuns.runnerProfileJson, context: heartbeatRuns.contextSnapshot }).from(heartbeatRuns).where(eq(heartbeatRuns.id, saved.id));
expect(stored[0].profile).toEqual(profile);
@@ -80,6 +81,16 @@ const supported = Boolean(externalUrl) || (await getEmbeddedPostgresTestSupport(
}
});
it("does not call missing or newer incompatible recovery material a fresh session", async () => {
const saved = source();
await db.insert(heartbeatRuns).values([saved, run(30, { id: currentRunId, status: "running" })]);
const { beforeCreatedAt: _before, ...binding } = input;
expect(await resolveNativeControllerRecovery(db, binding)).toEqual({ kind: "missing_evidence", reason: "snapshot_binding" });
// The newest progress barrier remains authoritative even without a snapshot.
await db.insert(heartbeatRuns).values(run(10, { runtimeMode: "legacy" }));
expect(await resolveNativeControllerRecovery(db, binding)).toEqual({ kind: "missing_evidence", reason: "newer_progress_barrier" });
});
it("skips only unused SQL-null, JSON-null, scalar, array and checkpoint-null profiles", async () => {
const saved = source(), newer = run(1, { runnerProfileJson: null });
await db.insert(heartbeatRuns).values([saved, newer]);
@@ -223,6 +223,8 @@ export async function findNativeSessionResumeRun(
normalizedSessionId: string;
currentRunId: string;
beforeCreatedAt: Date;
/** Recovery discovery must retain an ineligible newest progress barrier. */
includeProgressBarrier?: boolean;
},
) {
if (!isNativeSessionId(input.normalizedSessionId)) return null;
@@ -287,10 +289,12 @@ export async function findNativeSessionResumeRun(
nativeToolContractFingerprint: unknown;
nativeExecutionInput: unknown;
sessionCheckpoint: unknown;
nativeControllerRecovery: unknown;
}>`jsonb_build_object(
'nativeToolContractFingerprint', ${heartbeatRuns.runnerProfileJson}->'nativeToolContractFingerprint',
'nativeExecutionInput', ${heartbeatRuns.runnerProfileJson}->'nativeExecutionInput',
'sessionCheckpoint', ${heartbeatRuns.runnerProfileJson}->'sessionCheckpoint'
'sessionCheckpoint', ${heartbeatRuns.runnerProfileJson}->'sessionCheckpoint',
'nativeControllerRecovery', ${heartbeatRuns.runnerProfileJson}->'nativeControllerRecovery'
)`,
contextSnapshot: sql<{
paperclipWake: unknown;
@@ -306,14 +310,16 @@ export async function findNativeSessionResumeRun(
.from(heartbeatRuns)
.innerJoin(barrier, eq(heartbeatRuns.id, barrier.id))
.where(and(
eq(barrier.runtimeMode, "native"),
inArray(barrier.status, terminalStatuses),
eq(barrier.hasCheckpoint, true),
...(input.includeProgressBarrier ? [] : [
eq(barrier.runtimeMode, "native"),
inArray(barrier.status, terminalStatuses),
eq(barrier.hasCheckpoint, true),
sql`date_trunc('milliseconds', ${barrier.createdAt}) <= ${input.beforeCreatedAt.toISOString()}::timestamptz`,
]),
sql`${heartbeatRuns.companyId}::text = ${input.companyId}`,
sql`${heartbeatRuns.agentId}::text = ${input.agentId}`,
sql`${heartbeatRuns.nativeIssueId}::text = ${input.issueId}`,
sql`${heartbeatRuns.nativeSessionId}::text = ${input.normalizedSessionId}`,
sql`date_trunc('milliseconds', ${barrier.createdAt}) <= ${input.beforeCreatedAt.toISOString()}::timestamptz`,
));
return run ?? null;
}
@@ -0,0 +1,187 @@
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { Db } from "@paperclipai/db";
import { afterEach, describe, expect, it, vi } from "vitest";
import { DurablePrpControlPlane, type DurablePrpCheckpoint } from "../../vendor/paperclip-runner/index.js";
import { createLocalEncryptedCipher } from "../../secrets/local-encrypted-provider.js";
import { buildNativeExecutionInput } from "./native-execution-input.js";
import { nativeToolContractFingerprintForTarget } from "./native-session-resume.js";
import { nativeRuntimeContextFixture } from "./runtime-context.test-fixture.js";
import { resolveNativeStagedAttachmentRecovery } from "./native-staged-attachment-recovery.js";
const companyId = "10000000-0000-4000-8000-000000000001";
const issueId = "20000000-0000-4000-8000-000000000002";
const agentId = "30000000-0000-4000-8000-000000000003";
const sourceRunId = "40000000-0000-4000-8000-000000000004";
const currentRunId = "50000000-0000-4000-8000-000000000005";
const normalizedSessionId = "60000000-0000-4000-8000-000000000006";
const runnerInstanceId = "70000000-0000-4000-8000-000000000007";
const sourceIdentity = { runId: sourceRunId, normalizedSessionId, runnerInstanceId,
environmentLeaseId: "environment-1", turnId: "opaque-source-turn", itemId: "opaque-source-item" };
const targetIdentity = { ...sourceIdentity, runId: currentRunId, turnId: "opaque-target-turn", itemId: "opaque-target-item" };
const roots: string[] = [];
afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); });
function execution(runId: string) {
return buildNativeExecutionInput({ companyId, runId, agentId, normalizedSessionId,
issue: { id: issueId, identifier: "TEST-1", title: "Continue", description: null, workMode: "standard" },
taskPrompt: "Continue normally", provider: "codex", workspace: { id: "workspace-1", cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null },
completionContract: { id: "80000000-0000-4000-8000-000000000008", sha256: `sha256:${"a".repeat(64)}`,
schemaVersion: "paperclip.run-result.v1", contract: { revision: "1", objective: "Continue", criteria: [{ id: "done", requirement: "Answer" }] } },
runtimeContext: nativeRuntimeContextFixture(),
});
}
async function fixture() {
const root = await mkdtemp(join(tmpdir(), "native-staged-proof-")); roots.push(root);
const core = new DurablePrpControlPlane({ stateDirectory: root, identity: sourceIdentity,
expectedRunnerVersion: "0.0.0", expectedRunnerDigest: `sha256:${"f".repeat(64)}` });
let saved!: DurablePrpCheckpoint;
core.setCheckpointPersistence(async checkpoint => { saved = checkpoint; });
core.persistRunAttachTemplate({ provider: { provider: "codex", command: "codex" }, authorizedTools: {} });
await core.flushCheckpoint();
const sourceHeader = saved.records.get("header")!;
core.stageRunAttachment(targetIdentity, { provider: { provider: "codex", command: "codex" }, authorizedTools: {} });
await core.flushCheckpoint();
const header = saved.records.get("header")!;
const base = { companyId, agentId, issueId, nativeIssueId: issueId, runnerInstanceId,
nativeSessionId: normalizedSessionId, runtimeMode: "native", status: "running", finishedAt: null as Date | null };
const currentRun = { ...base, id: currentRunId, runnerProfileJson: { nativeExecutionInput: execution(currentRunId) } };
const sourceRun = { ...base, id: sourceRunId, status: "succeeded", finishedAt: new Date(), runnerProfileJson: {
nativeExecutionInput: execution(sourceRunId), nativeToolContractFingerprint: nativeToolContractFingerprintForTarget("remote", "codex"),
sessionCheckpoint: { sessionId: "provider-thread-1", providerSessionId: "provider-thread-1", identity: {
companyId, issueId, agentId, runId: sourceRunId, sessionId: normalizedSessionId },
},
} };
const source = { run: sourceRun, coordinator: { runId: sourceRunId, companyId, issueId,
phase: "committed", resultId: "result-1", leaseOwner: null, leaseExpiresAt: null }, resultId: "result-1", resultStatus: "accepted" };
function database(sourceRows = [source], contents = sourceHeader, envelope = {}) {
const material = createLocalEncryptedCipher("write").encrypt(JSON.stringify({ companyId, issueId, runId: sourceRunId, part: "header", contents, ...envelope }));
const batches: unknown[][] = [sourceRows, [{ material }]];
const select = vi.fn(() => {
const chain = { from: () => chain, innerJoin: () => chain, where: () => chain,
limit: vi.fn(async () => batches.shift() ?? []) };
return chain;
});
return { db: { select } as unknown as Pick<Db, "select">, select };
}
return { header, sourceHeader, source, sourceRun, currentRun, database };
}
describe("private staged attachment restart evidence", () => {
it("proves both frozen run bindings before the target has a session snapshot", async () => {
const f = await fixture(); const { db, select } = f.database();
const matchesExecution = vi.fn(() => true);
const result = await resolveNativeStagedAttachmentRecovery({ db, header: f.header, currentRun: f.currentRun,
expectedTarget: targetIdentity, matchesExecution });
expect(result).toMatchObject({ sourceIdentity, targetIdentity, sourceRun: { id: sourceRunId }, hasTargetCheckpoint: false, targetCheckpoint: null,
protocolIdentity: sourceIdentity, phase: "source_bootstrap", receipt: null });
expect(matchesExecution).toHaveBeenCalledOnce();
expect(select).toHaveBeenCalledTimes(2); // one source point lookup and one encrypted header point lookup
expect(f.currentRun.runnerProfileJson).not.toHaveProperty("sessionCheckpoint");
});
it("retains staged evidence when the target runtime has already published its snapshot", async () => {
const f = await fixture();
const snapshot = structuredClone(f.sourceRun.runnerProfileJson.sessionCheckpoint);
snapshot.identity.runId = currentRunId;
Object.assign(snapshot, { cursor: "target-cursor", activeTurnId: "target-current-turn", pendingRuntimeRequests: [{ requestId: "target-request" }] });
Object.assign(f.currentRun.runnerProfileJson, { sessionCheckpoint: snapshot });
const result = await resolveNativeStagedAttachmentRecovery({ ...f.database(), header: f.header, currentRun: f.currentRun });
expect(result).toMatchObject({ sourceIdentity, targetIdentity, checkpoint: { identity: { runId: currentRunId } },
hasTargetCheckpoint: true, targetCheckpoint: snapshot });
expect(f.currentRun.runnerProfileJson).toHaveProperty("sessionCheckpoint", snapshot);
});
it("returns the codec's queued phase without assuming the Runner has activated the target", async () => {
const f = await fixture(); const header = JSON.parse(f.header);
// Header inspection deliberately does not certify separate command records.
// Full restoration/authentication owns that later gate; no RPC runs here.
header.state.stagedRunAttachment.commandId = "staged-attach-command";
const result = await resolveNativeStagedAttachmentRecovery({ ...f.database(), header: JSON.stringify(header), currentRun: f.currentRun });
expect(result).toMatchObject({ commandId: "staged-attach-command", protocolIdentity: sourceIdentity,
phase: "attachment_queued", receipt: null, targetIdentity });
});
it.each(["runId", "companyId", "issueId", "agentId", "sessionId", "providerSessionId"])(
"rejects a present target checkpoint with changed %s instead of replaying the source", async key => {
const f = await fixture(); const snapshot = structuredClone(f.sourceRun.runnerProfileJson.sessionCheckpoint);
snapshot.identity.runId = currentRunId;
if (key === "providerSessionId") snapshot.providerSessionId = "foreign";
else Object.assign(snapshot.identity, { [key]: "foreign" });
Object.assign(f.currentRun.runnerProfileJson, { sessionCheckpoint: snapshot });
expect(await resolveNativeStagedAttachmentRecovery({ ...f.database(), header: f.header, currentRun: f.currentRun })).toBeNull();
});
it.each(["runId", "runnerInstanceId", "environmentLeaseId", "normalizedSessionId", "turnId", "itemId"] as const)(
"rejects a different expected target %s before querying the source", async key => {
const f = await fixture(); const { db, select } = f.database();
expect(await resolveNativeStagedAttachmentRecovery({ db, header: f.header, currentRun: f.currentRun,
expectedTarget: { ...targetIdentity, [key]: "foreign" } })).toBeNull();
expect(select).not.toHaveBeenCalled();
});
it.each(["agentId", "companyId", "nativeIssueId", "nativeSessionId", "runnerInstanceId"] as const)(
"rejects mixed source DB %s", async key => {
const f = await fixture(); Object.assign(f.source.run, { [key]: "foreign" });
expect(await resolveNativeStagedAttachmentRecovery({ ...f.database(), header: f.header, currentRun: f.currentRun })).toBeNull();
});
it.each(["binding", "schema", "runner", "session"])("rejects current frozen evidence mismatch: %s", async kind => {
const f = await fixture(); const { db, select } = f.database();
if (kind === "binding") f.currentRun.runnerProfileJson.nativeExecutionInput.binding.agentId = sourceRunId;
if (kind === "schema") Object.assign(f.currentRun.runnerProfileJson.nativeExecutionInput, { schema: "unsupported" });
if (kind === "runner") f.currentRun.runnerInstanceId = sourceRunId;
if (kind === "session") f.currentRun.nativeSessionId = sourceRunId;
expect(await resolveNativeStagedAttachmentRecovery({ db, header: f.header, currentRun: f.currentRun })).toBeNull();
expect(select).not.toHaveBeenCalled();
});
it.each(["phase", "leaseOwner", "resultStatus", "status", "finishedAt", "frozenRun", "workspace", "provider", "sourceTuple", "artifactVersion", "artifactDigest"])(
"rejects unsupported or stale source evidence: %s", async kind => {
const f = await fixture(); let sourceHeader = f.sourceHeader;
if (kind === "phase") f.source.coordinator.phase = "observed";
if (kind === "leaseOwner") Object.assign(f.source.coordinator, { leaseOwner: "another-owner", leaseExpiresAt: new Date(0) });
if (kind === "resultStatus") f.source.resultStatus = "rejected";
if (kind === "status") f.source.run.status = "running";
if (kind === "finishedAt") Object.assign(f.source.run, { finishedAt: null });
if (kind === "frozenRun") f.source.run.runnerProfileJson.nativeExecutionInput.binding.runId = currentRunId;
if (kind === "workspace") f.source.run.runnerProfileJson.nativeExecutionInput.binding.executionWorkspaceId = "foreign";
if (kind === "provider") Object.assign(f.source.run.runnerProfileJson.nativeExecutionInput.provider, { model: "another-model" });
if (kind === "sourceTuple") { const h = JSON.parse(sourceHeader); h.state.identity.turnId = "foreign"; sourceHeader = JSON.stringify(h); }
if (kind === "artifactVersion") { const h = JSON.parse(sourceHeader); h.runnerVersion = "other-version"; sourceHeader = JSON.stringify(h); }
if (kind === "artifactDigest") { const h = JSON.parse(sourceHeader); h.runnerDigest = `sha256:${"d".repeat(64)}`; sourceHeader = JSON.stringify(h); }
expect(await resolveNativeStagedAttachmentRecovery({ ...f.database([f.source], sourceHeader), header: f.header, currentRun: f.currentRun })).toBeNull();
});
it("keeps private executor scope/config verification mandatory when supplied", async () => {
const f = await fixture();
expect(await resolveNativeStagedAttachmentRecovery({ ...f.database(), header: f.header, currentRun: f.currentRun,
matchesExecution: () => false })).toBeNull();
});
it.each(["companyId", "issueId", "runId", "part"])("rejects encrypted source header with another %s", async key => {
const f = await fixture();
expect(await resolveNativeStagedAttachmentRecovery({ ...f.database([f.source], f.sourceHeader, { [key]: "foreign" }),
header: f.header, currentRun: f.currentRun })).toBeNull();
});
it("does not accept a source lacking the current tool contract or provider identity", async () => {
const f = await fixture();
f.source.run.runnerProfileJson.nativeToolContractFingerprint = "old-contract";
expect(await resolveNativeStagedAttachmentRecovery({ ...f.database(), header: f.header, currentRun: f.currentRun })).toBeNull();
f.source.run.runnerProfileJson.nativeToolContractFingerprint = nativeToolContractFingerprintForTarget("remote", "codex");
f.source.run.runnerProfileJson.sessionCheckpoint.providerSessionId = "";
expect(await resolveNativeStagedAttachmentRecovery({ ...f.database(), header: f.header, currentRun: f.currentRun })).toBeNull();
});
it("rejects corrupted or ordinary headers instead of promoting them to staged evidence", async () => {
const f = await fixture();
for (const header of [f.sourceHeader, "{}", "not-json"]) {
const { db, select } = f.database();
expect(await resolveNativeStagedAttachmentRecovery({ db, header, currentRun: f.currentRun })).toBeNull();
expect(select).not.toHaveBeenCalled();
}
});
});
@@ -0,0 +1,124 @@
import { isDeepStrictEqual } from "node:util";
import { and, eq } from "drizzle-orm";
import { heartbeatRuns, nativeRunFinalizations, nativeRunResults, type Db } from "@paperclipai/db";
import {
inspectDurablePrpCheckpointHeader,
inspectStagedRunAttachmentHeader,
parseNativeExecutionInput,
type DurableRecoveryIdentity,
type NativeExecutionInput,
type PersistedNativeSession,
type StagedRunAttachment,
} from "../../vendor/paperclip-runner/index.js";
import { loadNativeRunnerCheckpointHeader } from "./native-runner-checkpoint.js";
import { rebindNativeSessionCheckpoint } from "./native-session-resume.js";
type FrozenRun = Pick<typeof heartbeatRuns.$inferSelect,
"id" | "companyId" | "agentId" | "nativeIssueId" | "nativeSessionId" | "runnerInstanceId" |
"runtimeMode" | "status" | "runnerProfileJson" | "finishedAt">;
type StagedHeaderProof = NonNullable<ReturnType<typeof inspectStagedRunAttachmentHeader>>;
export type NativeStagedAttachmentRecovery = Omit<StagedRunAttachment, "runAttachTemplate"> &
Pick<StagedHeaderProof, "protocolIdentity" | "phase" | "receipt"> & {
sourceRun: FrozenRun;
sourceExecution: NativeExecutionInput;
currentExecution: NativeExecutionInput;
/** In-memory input only. It is not a published/resumable target snapshot. */
checkpoint: PersistedNativeSession;
/** A published target snapshot takes precedence over the source rebound. */
hasTargetCheckpoint: boolean;
targetCheckpoint: PersistedNativeSession | null;
runnerVersion: string;
runnerDigest: string;
};
function record(value: unknown): Record<string, unknown> {
return value !== null && typeof value === "object" && !Array.isArray(value)
? value as Record<string, unknown> : {};
}
function boundTargetCheckpoint(run: FrozenRun, source: PersistedNativeSession): PersistedNativeSession | null {
const snapshot = record(run.runnerProfileJson?.sessionCheckpoint), identity = record(snapshot.identity);
if (typeof snapshot.sessionId !== "string" || !snapshot.sessionId || identity.runId !== run.id ||
identity.companyId !== run.companyId || identity.issueId !== run.nativeIssueId || identity.agentId !== run.agentId ||
identity.sessionId !== run.nativeSessionId) return null;
// A target checkpoint may advance cursor/goal/tool progress, but cannot change
// the provider session retained by this attachment. Do not rebound it again.
if ((source.providerSessionId != null && source.providerSessionId !== snapshot.providerSessionId) ||
(source.providerIdentity != null && !isDeepStrictEqual(source.providerIdentity, snapshot.providerIdentity))) return null;
return structuredClone(snapshot) as unknown as PersistedNativeSession;
}
function frozenExecution(run: FrozenRun): NativeExecutionInput | null {
try {
const execution = parseNativeExecutionInput(run.runnerProfileJson?.nativeExecutionInput);
return run.runtimeMode === "native" && execution.binding.runId === run.id &&
execution.binding.companyId === run.companyId && execution.binding.agentId === run.agentId &&
execution.binding.issueId === run.nativeIssueId && execution.session.normalizedSessionId === run.nativeSessionId
? execution : null;
} catch { return null; }
}
function matchesRun(identity: DurableRecoveryIdentity, run: FrozenRun): boolean {
return identity.runId === run.id && identity.runnerInstanceId === run.runnerInstanceId &&
identity.normalizedSessionId === run.nativeSessionId;
}
/** Staging proves which prior attachment may be inspected, not that it is alive
* or that work may be dispatched. Classification reads only two indexed point
* lookups: the named source/result and its encrypted header. The executor must
* re-read this proof with its private scope/config matcher before any RPC, then
* independently prove the physical Runner and authenticate the target tuple. */
export async function resolveNativeStagedAttachmentRecovery(input: {
db: Pick<Db, "select">;
currentRun: FrozenRun;
header: string;
expectedTarget?: DurableRecoveryIdentity;
matchesExecution?: (previous: NativeExecutionInput, current: NativeExecutionInput) => boolean;
}): Promise<NativeStagedAttachmentRecovery | null> {
const proof = inspectStagedRunAttachmentHeader(input.header);
const current = frozenExecution(input.currentRun);
if (!proof || !current || current.provider.kind !== "codex" || !["running", "failed"].includes(input.currentRun.status) ||
!matchesRun(proof.attachment.targetIdentity, input.currentRun) ||
(input.expectedTarget && !isDeepStrictEqual(proof.attachment.targetIdentity, input.expectedTarget))) return null;
const { attachment } = proof;
const [source] = await input.db.select({ run: heartbeatRuns, coordinator: nativeRunFinalizations,
resultId: nativeRunResults.id, resultStatus: nativeRunResults.schemaStatus }).from(heartbeatRuns)
.innerJoin(nativeRunFinalizations, and(eq(nativeRunFinalizations.runId, heartbeatRuns.id),
eq(nativeRunFinalizations.companyId, heartbeatRuns.companyId), eq(nativeRunFinalizations.issueId, heartbeatRuns.nativeIssueId)))
.innerJoin(nativeRunResults, and(eq(nativeRunResults.id, nativeRunFinalizations.resultId),
eq(nativeRunResults.runId, heartbeatRuns.id), eq(nativeRunResults.companyId, heartbeatRuns.companyId),
eq(nativeRunResults.issueId, heartbeatRuns.nativeIssueId)))
.where(and(eq(heartbeatRuns.id, attachment.sourceIdentity.runId), eq(heartbeatRuns.companyId, input.currentRun.companyId),
eq(heartbeatRuns.agentId, input.currentRun.agentId), eq(heartbeatRuns.nativeIssueId, current.binding.issueId)))
.limit(1);
if (!source || source.run.companyId !== current.binding.companyId || source.run.agentId !== current.binding.agentId ||
source.run.nativeIssueId !== current.binding.issueId || !matchesRun(attachment.sourceIdentity, source.run) ||
source.run.status !== "succeeded" || source.run.finishedAt === null || source.coordinator.phase !== "committed" ||
source.coordinator.companyId !== current.binding.companyId || source.coordinator.issueId !== current.binding.issueId ||
source.coordinator.runId !== source.run.id || !source.coordinator.resultId ||
source.coordinator.resultId !== source.resultId || source.resultStatus !== "accepted" ||
source.coordinator.leaseOwner !== null || source.coordinator.leaseExpiresAt !== null) return null;
const previous = frozenExecution(source.run);
if (!previous || (input.matchesExecution && !input.matchesExecution(previous, current))) return null;
// Reuse the existing provider, tool-contract, workspace, runtime-context and
// normalized-session compatibility owner; no second set of digest formulas.
const checkpoint = rebindNativeSessionCheckpoint({ previousRun: source.run, currentExecution: current, executionTargetKind: "remote" });
if (!checkpoint || !(typeof checkpoint.providerSessionId === "string" && checkpoint.providerSessionId.length > 0 ||
checkpoint.providerIdentity && Object.keys(checkpoint.providerIdentity).length > 0)) return null;
const hasTargetCheckpoint = input.currentRun.runnerProfileJson?.sessionCheckpoint != null;
const targetCheckpoint = hasTargetCheckpoint ? boundTargetCheckpoint(input.currentRun, checkpoint) : null;
if (hasTargetCheckpoint && !targetCheckpoint) return null;
const sourceHeader = await loadNativeRunnerCheckpointHeader(input.db, {
companyId: source.run.companyId, issueId: current.binding.issueId, runId: source.run.id,
});
if (!sourceHeader || !isDeepStrictEqual(inspectDurablePrpCheckpointHeader(sourceHeader), attachment.sourceIdentity)) return null;
// The codec above has validated this bounded header's artifact fields. A
// staged target cannot silently select different bytes from its source.
const sourceArtifact = JSON.parse(sourceHeader) as { runnerVersion: string; runnerDigest: string };
if (sourceArtifact.runnerVersion !== proof.runnerVersion || sourceArtifact.runnerDigest !== proof.runnerDigest) return null;
return { ...attachment, sourceRun: source.run, sourceExecution: previous, currentExecution: current,
checkpoint, hasTargetCheckpoint, targetCheckpoint,
protocolIdentity: proof.protocolIdentity, phase: proof.phase, receipt: proof.receipt,
runnerVersion: proof.runnerVersion, runnerDigest: proof.runnerDigest };
}
@@ -0,0 +1,319 @@
import { createRequire } from "node:module";
import { randomUUID } from "node:crypto";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { eq, sql } from "drizzle-orm";
import { drizzle } from "drizzle-orm/postgres-js";
import { agents, agentTaskSessions, companies, completionContracts, createDb, heartbeatRuns, issues,
nativeRunFinalizations, nativeRunResults, type Db } from "@paperclipai/db";
import { CONTROL_PLANE_CONFORMANCE_OPEN, CONTROL_PLANE_CONFORMANCE_RESULT,
CONTROL_PLANE_CONFORMANCE_TERMINAL } from "../../vendor/paperclip-runner/testing.js";
import type { PersistedNativeSession } from "../../vendor/paperclip-runner/index.js";
import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js";
import { startRecoveryPostgres } from "../../__tests__/helpers/recovery-postgres.js";
import { redactRegisteredSecretValues } from "../run-secret-redaction.js";
import { nativeSha256 } from "./canonical.js";
import { PaperclipControlPlanePort, type PaperclipControlPlaneBinding } from "./paperclip-control-plane-port.js";
import { nativeTaskSessionPublicationHandled, publishNativeTaskSessionInTransaction, NATIVE_TASK_SESSION_PUBLICATION_KEY,
type NativeTaskSessionPublicationContext } from "./native-task-session-publication.js";
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_NATIVE_PUBLICATION_TEST_DATABASE_URL");
describe("accepted native task session publication", () => {
let temporary: Awaited<ReturnType<typeof startRecoveryPostgres>> | null = null;
let db: Db;
let measured: Db;
let client: Db["$client"];
const statements: string[] = [];
beforeAll(async () => {
if (!external) temporary = await startRecoveryPostgres("native-publication-");
const url = external ?? temporary!.connectionString;
db = createDb(url, { maxConnections: 2 });
// Test-only wire observer. Production still uses the existing Db/pools.
const postgres = createRequire(import.meta.resolve("@paperclipai/db"))("postgres") as
(url: string, options: object) => Db["$client"];
client = postgres(url, { max: 1, prepare: false, connection: { application_name: "native-publication-observer" },
debug: (_id: number, query: string) => { statements.push(query); } });
measured = drizzle(client) as unknown as Db;
});
afterAll(async () => {
await client?.end({ timeout: 1 });
await db?.$client.end({ timeout: 1 });
await temporary?.cleanup();
});
async function fixture(source: "runner" | "control" = "runner", accepted = true) {
const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID();
const sessionId = randomUUID(), contractId = randomUUID(), runner = randomUUID(), control = randomUUID();
await db.insert(companies).values({ id: companyId, name: "Native pointer", issuePrefix: `P${companyId.slice(0, 7)}` });
await db.insert(agents).values({ id: agentId, companyId, name: "Native pointer", adapterType: "codex_local", status: "running" });
await db.insert(issues).values({ id: issueId, companyId, title: "Native pointer", status: "in_progress", assigneeAgentId: agentId });
await db.insert(completionContracts).values({ id: contractId, companyId, issueId, revision: 1,
schemaVersion: "paperclip.completion-contract.v1", policyVersion: "standalone-v1", risk: "standard",
completionAuthority: "server_arbiter", incompleteCriteriaPolicy: "preserve_non_terminal",
contractJson: { revision: "standalone-v1", objective: "Conformance", criteria: [{ id: "objective", requirement: "Complete" }] },
canonicalSha256: "publication-contract", createdByActorType: "system", createdByActorId: "test" });
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running", runtimeMode: "native",
nativeIssueId: issueId, nativeSessionId: sessionId, runnerInstanceId: runner, completionContractId: contractId,
completionContractSha256: "publication-contract", startedAt: new Date(Date.now() - 2_000), runnerProfileJson: {} });
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId));
const owner = { companyId, issueId, runId, authorityRunId: runId, bootId: "publication-boot", generation: 1, leaseOwner: "publication-owner" };
await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId, phase: "observed",
controllerBootId: owner.bootId, controllerGeneration: owner.generation, leaseOwner: owner.leaseOwner,
leaseExpiresAt: new Date(Date.now() + 60_000) });
const binding: PaperclipControlPlaneBinding = { companyId, agentId, issueId, runId, sessionId,
completionContractId: contractId, completionContractSha256: "publication-contract", sourceInstanceId: runner,
controlPlaneSourceInstanceId: control };
const context: NativeTaskSessionPublicationContext = { taskKey: issueId, adapterType: "codex_local",
sessionParamsJson: { cwd: "/working", paperclipSessionConfigFingerprint: "prepared-once", opaque: "private-prepared-param" } };
const snapshot: PersistedNativeSession = { backendKind: "mock", sessionId, identity: { companyId, agentId, issueId, runId, sessionId },
providerSessionId: "provider-thread", activeTurnId: null, pendingRuntimeRequests: [], goal: null,
semanticResult: CONTROL_PLANE_CONFORMANCE_RESULT, terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL };
const port = new PaperclipControlPlanePort(measured, binding, { taskSessionPublication: context, taskSessionPublicationOwner: owner });
await port.openRun({ ...CONTROL_PLANE_CONFORMANCE_OPEN, identity: snapshot.identity, sourceInstanceId: runner });
if (accepted) await new PaperclipControlPlanePort(db, binding).completeRun({ result: CONTROL_PLANE_CONFORMANCE_RESULT,
terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, turnId: "publication-turn" });
await db.insert(agentTaskSessions).values({ companyId, agentId, adapterType: context.adapterType, taskKey: issueId,
goalSourceId: `${source === "runner" ? runner : control}:${runId}`, goalSourceCursor: 4 });
const read = async () => {
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
const [session] = await db.select().from(agentTaskSessions).where(eq(agentTaskSessions.taskKey, issueId));
return { run, session };
};
return { binding, owner, context, snapshot, port, read };
}
it.each(["runner", "control"] as const)("publishes from the exact accepted %s epoch in one checkpoint transaction", async source => {
const f = await fixture(source); statements.length = 0;
await f.port.checkpointSession(f.snapshot);
const queries = [...statements], { run, session } = await f.read();
expect(session).toMatchObject({ sessionParamsJson: { ...f.context.sessionParamsJson, sessionId: f.binding.sessionId },
sessionDisplayId: "provider-thread", lastRunId: f.binding.runId });
expect(nativeTaskSessionPublicationHandled(run.runnerProfileJson, { ...f.binding, ...f.context })).toBe(true);
expect(queries.filter(query => /^begin\b/i.test(query.trim()))).toHaveLength(1);
expect(queries.filter(query => /^commit\b/i.test(query.trim()))).toHaveLength(1);
expect(queries.filter(query => /native_run_results/i.test(query))).toHaveLength(1);
expect(queries.filter(query => /^select .*from "agent_task_sessions"/i.test(query.trim()))).toHaveLength(1);
expect(queries.filter(query => /^update "agent_task_sessions"/i.test(query.trim()))).toHaveLength(1);
expect(queries).toHaveLength(7);
expect(JSON.stringify(run.runnerProfileJson)).not.toContain("private-prepared-param");
expect(JSON.stringify({ [NATIVE_TASK_SESSION_PUBLICATION_KEY]: run.runnerProfileJson![NATIVE_TASK_SESSION_PUBLICATION_KEY] }).length).toBeLessThan(4_096);
expect(redactRegisteredSecretValues(run.runnerProfileJson, [])).not.toHaveProperty(NATIVE_TASK_SESSION_PUBLICATION_KEY);
expect(nativeTaskSessionPublicationHandled(run.runnerProfileJson, { ...f.binding, ...f.context, companyId: randomUUID() })).toBe(false);
expect(nativeTaskSessionPublicationHandled(run.runnerProfileJson, { ...f.binding, ...f.context, agentId: randomUUID() })).toBe(false);
const malformed = structuredClone(run.runnerProfileJson!);
(malformed[NATIVE_TASK_SESSION_PUBLICATION_KEY] as Record<string, unknown>).resultId = "not-an-id";
expect(nativeTaskSessionPublicationHandled(malformed, { ...f.binding, ...f.context })).toBe(false);
});
it("does not recreate a reset pointer or query task state on repeated terminal maintenance", async () => {
const f = await fixture(); await f.port.checkpointSession(f.snapshot);
await db.delete(agentTaskSessions).where(eq(agentTaskSessions.taskKey, f.binding.issueId)); statements.length = 0;
await f.port.checkpointSession(f.snapshot);
const repeated = [...statements];
expect((await f.read()).session).toBeUndefined();
expect(repeated.filter(query => /agent_task_sessions|native_run_results/i.test(query))).toHaveLength(0);
expect(repeated.filter(query => /^begin\b/i.test(query.trim()))).toHaveLength(1);
expect(repeated.filter(query => /^update /i.test(query.trim()))).toHaveLength(1);
});
it("persists a pre-acceptance terminal observation and publishes only at the existing accepted checkpoint", async () => {
const f = await fixture("runner", false);
// Driver event/startup checkpoints can expose a semantic terminal observation
// before the runtime calls completeRun. That observation is not acceptance.
await f.port.checkpointSession(f.snapshot);
const observed = await f.read();
expect(observed.run.runnerProfileJson?.sessionCheckpoint).toMatchObject({ semanticResult: CONTROL_PLANE_CONFORMANCE_RESULT });
expect(observed.run.runnerProfileJson).not.toHaveProperty(NATIVE_TASK_SESSION_PUBLICATION_KEY);
expect(observed.session?.sessionParamsJson).toBeNull();
expect(nativeTaskSessionPublicationHandled(observed.run.runnerProfileJson, { ...f.binding, ...f.context })).toBe(false);
await f.port.completeRun({ result: CONTROL_PLANE_CONFORMANCE_RESULT,
terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, turnId: "publication-turn" });
expect((await f.read()).session?.sessionParamsJson).toBeNull();
await f.port.checkpointSession(f.snapshot);
const completed = await f.read();
expect(completed.session?.lastRunId).toBe(f.binding.runId);
expect(nativeTaskSessionPublicationHandled(completed.run.runnerProfileJson, { ...f.binding, ...f.context })).toBe(true);
});
it("does not hold the issue parent lock while a result replay waits for the status coordinator", async () => {
const f = await fixture();
let committed = 0;
const port = new PaperclipControlPlanePort(db, f.binding, { nativeRecovery: { owner: f.owner,
compatibility: { configDigest: nativeSha256("config"), processConfigDigest: null } },
onResultCommitted: async () => { committed++; } });
await port.openRun({ ...CONTROL_PLANE_CONFORMANCE_OPEN, identity: f.snapshot.identity, sourceInstanceId: f.binding.sourceInstanceId });
let replay: Promise<unknown> | undefined;
let statusLockError: unknown = null;
try {
await db.transaction(async tx => {
// This is commitNativeStatusDecision's actual lock prefix. Hold its
// coordinator first, then overlap an identical remote result replay.
await tx.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.binding.runId)).for("update");
const [backend] = await tx.execute(sql`select pg_backend_pid() as pid`);
replay = port.completeRun({ result: CONTROL_PLANE_CONFORMANCE_RESULT,
terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, turnId: "publication-turn" }).then(() => null, error => error);
let waiting = false;
for (let attempt = 0; attempt < 100; attempt++) {
const [state] = await measured.execute(sql`select exists (
select 1 from pg_stat_activity where ${backend.pid} = any(pg_blocking_pids(pid))) as waiting`);
if (state.waiting) { waiting = true; break; }
await new Promise(resolve => setTimeout(resolve, 10));
}
expect(waiting).toBe(true);
// A bounded test-local timeout reports the inverse-order cycle without
// relying on which transaction PostgreSQL chooses as deadlock victim.
await tx.execute(sql`set local lock_timeout = '500ms'`);
await tx.select().from(issues).where(eq(issues.id, f.binding.issueId)).for("update");
});
} catch (error) { statusLockError = error; }
const replayError = await replay;
expect(statusLockError).toBeNull();
expect(replayError).toBeNull();
expect(committed).toBe(0);
expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, f.binding.runId))).toHaveLength(1);
await db.update(nativeRunFinalizations).set({ controllerGeneration: 2 }).where(eq(nativeRunFinalizations.runId, f.binding.runId));
await expect(port.completeRun({ result: CONTROL_PLANE_CONFORMANCE_RESULT,
terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, turnId: "publication-turn" })).rejects.toThrow("checkpoint_owner_lost");
});
it.each(["missing", "foreign_epoch", "newer_pointer", "newer_execution", "generation_reset", "stop"])("fences %s and records a handled suppression", async kind => {
const f = await fixture();
if (kind === "missing") await db.delete(agentTaskSessions).where(eq(agentTaskSessions.taskKey, f.binding.issueId));
if (kind === "foreign_epoch") await db.update(agentTaskSessions).set({ goalSourceId: `${randomUUID()}:${f.binding.runId}` }).where(eq(agentTaskSessions.taskKey, f.binding.issueId));
if (kind === "newer_pointer" || kind === "newer_execution") {
const newer = randomUUID();
await db.insert(heartbeatRuns).values({ id: newer, companyId: f.binding.companyId, agentId: f.binding.agentId,
status: "running", issueId: f.binding.issueId, startedAt: new Date() });
if (kind === "newer_pointer") await db.update(agentTaskSessions).set({ lastRunId: newer, sessionParamsJson: { sessionId: "newer" } }).where(eq(agentTaskSessions.taskKey, f.binding.issueId));
else await db.update(issues).set({ executionRunId: newer }).where(eq(issues.id, f.binding.issueId));
}
if (kind === "generation_reset") await db.update(issues).set({ conversationAgentId: f.binding.agentId,
conversationUserId: "publication-test-user", conversationState: "active", conversationSessionGeneration: 2 }).where(eq(issues.id, f.binding.issueId));
if (kind === "stop") await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, f.binding.runId));
await f.port.checkpointSession(f.snapshot);
const { run, session } = await f.read();
expect(nativeTaskSessionPublicationHandled(run.runnerProfileJson, { ...f.binding, ...f.context })).toBe(true);
expect(session?.sessionParamsJson?.sessionId).not.toBe(f.binding.sessionId);
});
it("keeps unsupported task keys and missing remote recovery headers outside handled publication", async () => {
const f = await fixture();
const context = { ...f.context, taskKey: "custom-task" };
const port = new PaperclipControlPlanePort(measured, f.binding, { taskSessionPublication: context, taskSessionPublicationOwner: f.owner });
await port.openRun({ ...CONTROL_PLANE_CONFORMANCE_OPEN, identity: f.snapshot.identity, sourceInstanceId: f.binding.sourceInstanceId });
await port.checkpointSession(f.snapshot);
const { run, session } = await f.read();
expect(session?.sessionParamsJson).toBeNull();
expect(nativeTaskSessionPublicationHandled(run.runnerProfileJson, { ...f.binding, ...context })).toBe(false);
const remote = await fixture();
const remotePort = new PaperclipControlPlanePort(measured, remote.binding, { taskSessionPublication: remote.context,
taskSessionPublicationOwner: remote.owner, nativeRecovery: { owner: remote.owner,
compatibility: { configDigest: nativeSha256("config"), processConfigDigest: null } } });
await remotePort.openRun({ ...CONTROL_PLANE_CONFORMANCE_OPEN, identity: remote.snapshot.identity, sourceInstanceId: remote.binding.sourceInstanceId });
await remotePort.checkpointSession(remote.snapshot);
expect((await remote.read()).session?.sessionParamsJson).toBeNull();
expect(nativeTaskSessionPublicationHandled((await remote.read()).run.runnerProfileJson, { ...remote.binding, ...remote.context })).toBe(false);
});
it("rejects stale owners and a different accepted result without partially publishing", async () => {
const stale = await fixture();
await db.update(nativeRunFinalizations).set({ controllerGeneration: 2 }).where(eq(nativeRunFinalizations.runId, stale.binding.runId));
await expect(stale.port.checkpointSession(stale.snapshot)).rejects.toThrow("checkpoint_owner_lost");
expect((await stale.read()).session?.sessionParamsJson).toBeNull();
const wrong = await fixture();
await expect(wrong.port.checkpointSession({ ...wrong.snapshot, semanticResult: { ...CONTROL_PLANE_CONFORMANCE_RESULT, summary: "different" } })).rejects.toThrow("publication_result_mismatch");
expect((await wrong.read()).session?.sessionParamsJson).toBeNull();
const foreign = await fixture();
const foreignResult = await fixture();
const [otherCoordinator] = await db.select({ resultId: nativeRunFinalizations.resultId }).from(nativeRunFinalizations)
.where(eq(nativeRunFinalizations.runId, foreignResult.binding.runId));
await expect(db.update(nativeRunFinalizations).set({ resultId: otherCoordinator.resultId })
.where(eq(nativeRunFinalizations.runId, foreign.binding.runId)))
.rejects.toMatchObject({ cause: { constraint_name: "native_run_finalizations_result_owner_fk" } });
expect((await foreign.read()).session?.sessionParamsJson).toBeNull();
await db.update(nativeRunFinalizations).set({ resultId: null }).where(eq(nativeRunFinalizations.runId, foreign.binding.runId));
// The CP defers preacceptance snapshots, while the publication helper itself
// still requires accepted-result authority if called directly.
await expect(publishNativeTaskSessionInTransaction({} as never, { binding: foreign.binding,
context: foreign.context, snapshot: foreign.snapshot, run: (await foreign.read()).run,
issue: null, resultId: null })).rejects.toThrow("publication_result_missing");
});
it("adds no publication SQL for nonterminal or goal snapshots", async () => {
const f = await fixture(); statements.length = 0;
await f.port.checkpointSession({ ...f.snapshot, semanticResult: null, terminal: null });
expect(statements.some(query => /agent_task_sessions|native_run_results|publication_issue/i.test(query))).toBe(false);
statements.length = 0;
await f.port.checkpointSession({ ...f.snapshot, goal: {} as NonNullable<PersistedNativeSession["goal"]> });
expect(statements.some(query => /agent_task_sessions|native_run_results|publication_issue/i.test(query))).toBe(false);
expect((await f.read()).session?.sessionParamsJson).toBeNull();
});
it.each(["completed_predecessor", "expired_accepted_lease"])("preserves legitimate %s maintenance", async kind => {
const f = await fixture();
if (kind === "completed_predecessor") {
const prior = randomUUID();
await db.insert(heartbeatRuns).values({ id: prior, companyId: f.binding.companyId, agentId: f.binding.agentId,
issueId: f.binding.issueId, status: "succeeded", finishedAt: new Date(Date.now() - 5_000) });
await db.update(agentTaskSessions).set({ lastRunId: prior, sessionParamsJson: { sessionId: "prior" } })
.where(eq(agentTaskSessions.taskKey, f.binding.issueId));
} else {
await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 1_000) })
.where(eq(nativeRunFinalizations.runId, f.binding.runId));
}
await f.port.checkpointSession(f.snapshot);
expect((await f.read()).session?.lastRunId).toBe(f.binding.runId);
});
it("does not block completeRun's issue KEY SHARE while publishing", async () => {
const f = await fixture(); let release!: () => void; let locked!: () => void;
const ready = new Promise<void>(resolve => { locked = resolve; });
const hold = new Promise<void>(resolve => { release = resolve; });
const blocker = db.transaction(async tx => {
await tx.select().from(issues).where(eq(issues.id, f.binding.issueId)).for("key share"); locked(); await hold;
});
await ready;
try { await f.port.checkpointSession(f.snapshot); }
finally { release(); await blocker; }
expect((await f.read()).session?.lastRunId).toBe(f.binding.runId);
});
it("rechecks the owner clock and issue authority after an issue lock wait", async () => {
const f = await fixture(); let release!: () => void; let locked!: () => void;
await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() + 500) }).where(eq(nativeRunFinalizations.runId, f.binding.runId));
const ready = new Promise<void>(resolve => { locked = resolve; });
const hold = new Promise<void>(resolve => { release = resolve; });
const newer = randomUUID();
await db.insert(heartbeatRuns).values({ id: newer, companyId: f.binding.companyId, agentId: f.binding.agentId,
issueId: f.binding.issueId, status: "running", startedAt: new Date() });
const blocker = db.transaction(async tx => {
await tx.select().from(issues).where(eq(issues.id, f.binding.issueId)).for("no key update");
await tx.update(issues).set({ executionRunId: newer }).where(eq(issues.id, f.binding.issueId));
locked(); await hold;
});
await ready;
const publication = f.port.checkpointSession(f.snapshot);
const rejected = expect(publication).rejects.toThrow("checkpoint_owner_lost");
try {
let waiting = false;
for (let attempt = 0; attempt < 100; attempt++) {
const rows = await db.execute(sql`select 1 from pg_stat_activity where application_name = 'native-publication-observer'
and wait_event_type = 'Lock' and query like '%publication_issue%'`);
if (rows.length) { waiting = true; break; }
await new Promise(resolve => setTimeout(resolve, 10));
}
expect(waiting).toBe(true);
await new Promise(resolve => setTimeout(resolve, 600));
} finally { release(); await blocker; }
await rejected;
expect((await f.read()).session?.sessionParamsJson).toBeNull();
});
it("rolls back the pointer if combined private metadata exceeds its existing bound", async () => {
const f = await fixture();
await db.update(heartbeatRuns).set({ runnerProfileJson: { nativeControllerRecovery: { padding: "x".repeat(4_096) } } }).where(eq(heartbeatRuns.id, f.binding.runId));
await expect(f.port.checkpointSession(f.snapshot)).rejects.toThrow("publication_metadata_too_large");
const { run, session } = await f.read();
expect(session?.sessionParamsJson).toBeNull();
expect(run.runnerProfileJson).not.toHaveProperty(NATIVE_TASK_SESSION_PUBLICATION_KEY);
});
});
@@ -0,0 +1,145 @@
import { and, eq } from "drizzle-orm";
import { agentTaskSessions, heartbeatRuns, nativeRunResults, type Db, type issues } from "@paperclipai/db";
import { validatePrpStructuredRunResult, type PersistedNativeSession } from "../../vendor/paperclip-runner/index.js";
import { nativeSha256 } from "./canonical.js";
import type { PaperclipControlPlaneBinding } from "./paperclip-control-plane-port.js";
import { readRunCancellation } from "../run-cancellation.js";
/** Prepared once by heartbeat; never copied into recovery metadata or receipts. */
export interface NativeTaskSessionPublicationContext {
taskKey: string;
adapterType: string;
sessionParamsJson: Record<string, unknown> | null;
}
export const NATIVE_TASK_SESSION_PUBLICATION_KEY = "nativeTaskSessionPublication";
type Transaction = Parameters<Parameters<Db["transaction"]>[0]>[0];
type Outcome = "published" | "suppressed_reset" | "suppressed_newer" | "suppressed_authority" | "missing_stub" | "unsupported";
interface Receipt {
schema: "paperclip.native-task-session-publication.v1";
companyId: string;
issueId: string;
agentId: string;
runId: string;
taskKey: string;
adapterType: string;
sessionId: string;
resultId: string;
outcome: Outcome;
}
function record(value: unknown): Record<string, unknown> {
return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
}
const uuid = (value: unknown): value is string => typeof value === "string" && /^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/i.test(value);
const outcomes: readonly string[] = ["published", "suppressed_reset", "suppressed_newer", "suppressed_authority", "missing_stub", "unsupported"];
function receiptFromProfile(profile: unknown): Receipt | null {
const root = record(profile), value = record(root[NATIVE_TASK_SESSION_PUBLICATION_KEY]);
if (value.schema !== "paperclip.native-task-session-publication.v1"
|| ![value.companyId, value.issueId, value.agentId, value.runId, value.sessionId, value.resultId].every(uuid)
|| typeof value.taskKey !== "string" || value.taskKey.length > 512
|| typeof value.adapterType !== "string" || !value.adapterType.length || value.adapterType.length > 128
|| !outcomes.includes(String(value.outcome)) || Object.keys(value).length !== 10) return null;
const snapshot = record(root.sessionCheckpoint), identity = record(snapshot.identity);
if (identity.runId !== value.runId || identity.companyId !== value.companyId || identity.issueId !== value.issueId
|| identity.agentId !== value.agentId || identity.sessionId !== value.sessionId) return null;
return value as unknown as Receipt;
}
/** Only the exact server-written outcome may suppress heartbeat's late fallback. */
export function nativeTaskSessionPublicationHandled(profile: unknown, binding: {
companyId: string; agentId: string; runId: string; taskKey: string; adapterType: string;
}): boolean {
const receipt = receiptFromProfile(profile);
return !!receipt && receipt.companyId === binding.companyId && receipt.agentId === binding.agentId
&& receipt.runId === binding.runId && receipt.taskKey === binding.taskKey
&& receipt.adapterType === binding.adapterType && ["published", "suppressed_reset", "suppressed_newer", "suppressed_authority", "missing_stub"].includes(receipt.outcome);
}
export function eligibleNativeTaskSessionPublication(snapshot: PersistedNativeSession): boolean {
return snapshot.goal == null && snapshot.semanticResult != null && snapshot.terminal?.runTerminalState === "succeeded"
&& snapshot.activeTurnId == null && !(snapshot.pendingRuntimeRequests?.length);
}
export type NativeTaskSessionPublicationRun = Pick<typeof heartbeatRuns.$inferSelect,
"id" | "companyId" | "agentId" | "nativeIssueId" | "nativeSessionId" | "status" | "resultJson" | "contextSnapshot" | "startedAt" | "createdAt" | "runnerProfileJson">;
export type NativeTaskSessionPublicationIssue = Pick<typeof issues.$inferSelect,
"id" | "companyId" | "executionRunId" | "conversationAgentId" | "conversationSessionGeneration">;
/** Caller holds coordinator → issue NO KEY UPDATE → run. This helper owns only
* the task-row lock and UPDATE; never starts a transaction or creates a stub. */
export async function publishNativeTaskSessionInTransaction(tx: Transaction, input: {
binding: PaperclipControlPlaneBinding;
context: NativeTaskSessionPublicationContext;
snapshot: PersistedNativeSession;
run: NativeTaskSessionPublicationRun;
issue: NativeTaskSessionPublicationIssue | null;
resultId: string | null;
recoveryReady?: boolean;
}): Promise<Receipt> {
const { binding, context, snapshot, run, issue } = input;
const existingReceipt = receiptFromProfile(run.runnerProfileJson);
if (existingReceipt) {
if (existingReceipt.companyId !== binding.companyId || existingReceipt.issueId !== binding.issueId
|| existingReceipt.agentId !== binding.agentId || existingReceipt.runId !== binding.runId
|| existingReceipt.sessionId !== snapshot.identity.sessionId || existingReceipt.taskKey !== context.taskKey
|| existingReceipt.adapterType !== context.adapterType || existingReceipt.resultId !== input.resultId) {
throw new Error("native_task_session_publication_receipt_mismatch");
}
return existingReceipt;
}
if (record(run.runnerProfileJson)[NATIVE_TASK_SESSION_PUBLICATION_KEY] !== undefined) {
throw new Error("native_task_session_publication_receipt_invalid");
}
if (!input.resultId) throw new Error("native_task_session_publication_result_missing");
const [accepted] = await tx.select({ id: nativeRunResults.id, canonicalSha256: nativeRunResults.canonicalSha256,
turnId: nativeRunResults.turnId }).from(nativeRunResults).where(and(
eq(nativeRunResults.id, input.resultId), eq(nativeRunResults.runId, binding.runId),
eq(nativeRunResults.companyId, binding.companyId), eq(nativeRunResults.issueId, binding.issueId),
eq(nativeRunResults.completionContractId, binding.completionContractId), eq(nativeRunResults.schemaStatus, "accepted"),
)).limit(1);
const validated = validatePrpStructuredRunResult(snapshot.semanticResult);
if (!accepted || !validated.ok || accepted.canonicalSha256 !== nativeSha256({
binding, result: validated.result, terminal: snapshot.terminal, turnId: accepted.turnId ?? null,
})) throw new Error("native_task_session_publication_result_mismatch");
const receipt = (outcome: Outcome): Receipt => ({ schema: "paperclip.native-task-session-publication.v1",
companyId: binding.companyId, issueId: binding.issueId, agentId: binding.agentId, runId: binding.runId,
taskKey: context.taskKey, adapterType: context.adapterType, sessionId: snapshot.identity.sessionId,
resultId: accepted.id, outcome });
if (context.taskKey !== binding.issueId || input.recoveryReady === false) return receipt("unsupported");
const cancellation = record(run.resultJson?.nativeCancellation), stop = readRunCancellation(run.resultJson);
if (!issue || ["cancelled", "interrupted", "failed", "timed_out"].includes(run.status) || issue.companyId !== binding.companyId
|| run.resultJson?.startupCancellation || (stop?.source === "operator" && stop.expected)
|| (["run", "issue"].includes(String(cancellation.scope)) && ["pending", "acknowledged"].includes(String(cancellation.dispatchState)))
|| (issue.conversationAgentId && issue.conversationAgentId !== binding.agentId)) return receipt("suppressed_authority");
if (issue.conversationAgentId
&& run.contextSnapshot?.conversationSessionGeneration !== issue.conversationSessionGeneration) return receipt("suppressed_reset");
if (issue.executionRunId && issue.executionRunId !== binding.runId) return receipt("suppressed_newer");
const [session] = await tx.select({ id: agentTaskSessions.id, goalSourceId: agentTaskSessions.goalSourceId,
lastRunId: agentTaskSessions.lastRunId }).from(agentTaskSessions).where(and(
eq(agentTaskSessions.companyId, binding.companyId), eq(agentTaskSessions.agentId, binding.agentId),
eq(agentTaskSessions.adapterType, context.adapterType), eq(agentTaskSessions.taskKey, context.taskKey),
)).for("update").limit(1);
// Supported native turns synchronously project a stub before this terminal
// snapshot. Absence can be an operator deletion; never recreate it later.
if (!session) return receipt("missing_stub");
const boundSources = [binding.sourceInstanceId, binding.controlPlaneSourceInstanceId]
.filter((value): value is string => typeof value === "string" && value.length > 0)
.map(source => `${source}:${binding.runId}`);
if (!session.goalSourceId || !boundSources.includes(session.goalSourceId)) return receipt("suppressed_newer");
if (session.lastRunId && session.lastRunId !== binding.runId) {
const [prior] = await tx.select({ companyId: heartbeatRuns.companyId, agentId: heartbeatRuns.agentId,
issueId: heartbeatRuns.issueId, nativeIssueId: heartbeatRuns.nativeIssueId, finishedAt: heartbeatRuns.finishedAt,
}).from(heartbeatRuns).where(eq(heartbeatRuns.id, session.lastRunId)).limit(1);
if (!prior || prior.companyId !== binding.companyId || prior.agentId !== binding.agentId
|| (prior.nativeIssueId ?? prior.issueId) !== binding.issueId || !prior.finishedAt || !run.startedAt
|| prior.finishedAt > run.startedAt) return receipt("suppressed_newer");
}
await tx.update(agentTaskSessions).set({
sessionParamsJson: { ...context.sessionParamsJson, sessionId: snapshot.identity.sessionId },
sessionDisplayId: snapshot.providerSessionId ?? snapshot.sessionId, lastRunId: binding.runId,
lastError: null, updatedAt: new Date(),
}).where(eq(agentTaskSessions.id, session.id));
return receipt("published");
}
@@ -12,8 +12,8 @@ import { agents, budgetPolicies, companies, costEvents, createDb, heartbeatRuns,
import { type PrpEvent } from "../../vendor/paperclip-runner/index.js";
import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
import { accountRunCost } from "../run-cost-accounting.js";
import { usageReceiptSpoolPath } from "../usage-receipts.js";
import { createNativeUsageDelivery } from "./native-usage-delivery.js";
import { usageReceiptSpoolPath, type UsageReceiptEnvelope } from "../usage-receipts.js";
import { createNativeUsageContinuationDelivery, createNativeUsageDelivery } from "./native-usage-delivery.js";
const runnerRoot = fileURLToPath(new URL("../../../../packages/paperclip-runner/runner/", import.meta.url));
const fixtureProfile = process.env.CI ? "release" : "debug";
@@ -44,6 +44,89 @@ describe("producer-owned native usage", () => {
payload: type === "usage.reported" ? { runDeltaAvailable: true, runDeltaBasis: "run", runDelta: { inputTokens: 120, outputTokens: 10, providerCostUsd: amount } } : {},
} as PrpEvent);
const finalizedSource = async () => {
const input = await seed();
await db.update(heartbeatRuns).set({ runnerInstanceId: input.runnerInstanceId }).where(eq(heartbeatRuns.id, input.runId));
const delivery = await createNativeUsageDelivery(db, input);
const report = event(input, 1, "usage.reported"), done = event(input, 2, "turn.completed");
await delivery.capture(report);
await delivery.capture(done);
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, input.runId));
expect(await accountRunCost(db, input.runId)).toBe(true);
const { billing: _billing, ...source } = input;
return { input, source, report, done };
};
it("continues exact finalized source receipts without acquiring ownership or resetting accounting", async () => {
const { input, source, report, done } = await finalizedSource();
const before = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId));
await expect(createNativeUsageDelivery(db, input)).rejects.toThrow("native_usage_run_not_available");
const continuation = await createNativeUsageContinuationDelivery(db, source);
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId))).toEqual(before);
await continuation.capture(report);
await continuation.capture(done);
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId))).toEqual(before);
expect(await db.select().from(runUsageReceipts).where(eq(runUsageReceipts.runId, input.runId))).toHaveLength(2);
expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId))).toHaveLength(1);
await expect(continuation.capture({ ...report, sourceInstanceId: randomUUID() })).rejects.toThrow("native_usage_source_mismatch");
await expect(continuation.capture({ ...report, payload: { ...report.payload as object, runDeltaAvailable: false } }))
.rejects.toThrow("native_usage_source_conflict");
});
it("journals an equal finalized source tail using the source billing without repricing", async () => {
const { input, source } = await finalizedSource();
const before = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId));
// Even a structurally supplied extra current-run classification cannot own
// the old receipt: the continuation derives billing from saved source data.
const withCurrentBilling = { ...source, billing: { provider: "anthropic", biller: "current-grant", billingType: "subscription_included" } };
const continuation = await createNativeUsageContinuationDelivery(db, withCurrentBilling);
await continuation.capture(event(input, 3, "usage.reported"));
const receipts = await db.select().from(runUsageReceipts).where(eq(runUsageReceipts.runId, input.runId));
expect(receipts).toHaveLength(3);
expect(receipts.find(row => row.sequence === 3)?.receiptJson.receipt).toMatchObject(billing);
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId))).toEqual(before);
expect((await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId)))[0].costCents).toBe(25);
});
it.each(["tokens", "cost", "incomplete", "unpriceable", "basis"])(
"holds a differing finalized source tail before acknowledgement: %s", async change => {
const { input, source } = await finalizedSource();
const continuation = await createNativeUsageContinuationDelivery(db, source);
const late = event(input, 3, "usage.reported");
if (change === "incomplete") (late.payload as Record<string, unknown>).runDeltaAvailable = false;
if (change === "basis") delete (late.payload as Record<string, unknown>).runDeltaBasis;
if (["tokens", "cost", "unpriceable"].includes(change)) (late.payload as Record<string, unknown>).runDelta = {
inputTokens: change === "tokens" ? 150 : change === "unpriceable" ? "unknown" : 120,
outputTokens: 10, providerCostUsd: change === "cost" ? 0.5 : 0.25,
};
await expect(continuation.capture(late)).rejects.toThrow("native_usage_after_finalization_requires_reconciliation");
await expect(continuation.capture(late)).rejects.toThrow("native_usage_after_finalization_requires_reconciliation");
expect(await db.select().from(runUsageReceipts).where(eq(runUsageReceipts.runId, input.runId))).toHaveLength(2);
expect((await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId)))[0].costCents).toBe(25);
});
it.each(["company", "run", "runner", "source"])("rejects a changed continuation binding without mutation: %s", async change => {
const { input, source } = await finalizedSource();
if (change === "source") await db.update(heartbeatRuns).set({ usageJson: sql`${heartbeatRuns.usageJson} || ${JSON.stringify({ accountingReceiptSourceId: randomUUID() })}::jsonb` })
.where(eq(heartbeatRuns.id, input.runId));
const before = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId));
await expect(createNativeUsageContinuationDelivery(db, { ...source,
...(change === "company" ? { companyId: randomUUID() } : {}),
...(change === "run" ? { runId: randomUUID() } : {}),
...(change === "runner" ? { runnerInstanceId: randomUUID() } : {}),
})).rejects.toThrow("native_usage_continuation_source_mismatch");
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId))).toEqual(before);
});
it("rejects changed immutable source receipt facts during hydration", async () => {
const { input, source } = await finalizedSource();
const [receipt] = await db.select().from(runUsageReceipts).where(and(eq(runUsageReceipts.runId, input.runId), eq(runUsageReceipts.sequence, 1)));
const changed = structuredClone(receipt.receiptJson) as UsageReceiptEnvelope;
changed.producerEvent!.usage!.runDelta = { inputTokens: 999, outputTokens: 10 };
await db.update(runUsageReceipts).set({ receiptJson: changed }).where(eq(runUsageReceipts.id, receipt.id));
await expect(createNativeUsageContinuationDelivery(db, source)).rejects.toThrow("native_usage_source_conflict");
});
it("retains the same Runner report through a failed Postgres receipt and an empty replacement receiver", async () => {
const input = await seed();
expect(existsSync(usageReceiptSpoolPath())).toBe(false);
@@ -16,11 +16,13 @@ const uuid = (identity: string) => {
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-5${hex.slice(13, 16)}-a${hex.slice(17, 20)}-${hex.slice(20, 32)}`;
};
export async function createNativeUsageDelivery(db: Db, input: {
type NativeUsageDeliveryInput = {
companyId: string; runId: string; runnerInstanceId: string;
provider: NativeExecutionInput["provider"];
billing: Pick<AdapterUsageCheckpoint, "provider" | "biller" | "billingType">;
}) {
};
export async function createNativeUsageDelivery(db: Db, input: NativeUsageDeliveryInput) {
const sourceId = uuid(`${input.runId}:${input.runnerInstanceId}`);
const [bound] = await db.update(heartbeatRuns).set({ usageJson: sql`
coalesce(${heartbeatRuns.usageJson}, '{}'::jsonb) || '{"accountingReceiptOwner":"native_producer"}'::jsonb ||
@@ -29,9 +31,39 @@ export async function createNativeUsageDelivery(db: Db, input: {
}).where(and(eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.runId),
isNull(heartbeatRuns.costAccountedAt))).returning({ id: heartbeatRuns.id });
if (!bound) throw new Error("native_usage_run_not_available");
const saved = await db.select({ receipt: runUsageReceipts.receiptJson }).from(runUsageReceipts).where(and(
return createNativeUsageReceiver(db, input, sourceId, await loadNativeUsageReceipts(db, input, sourceId));
}
/** Drain the authenticated previous producer's accounting without acquiring or
* resetting its ownership. Finalized sources retain normal receipt equality
* and reconciliation rules; this grants no business/tool execution authority. */
export async function createNativeUsageContinuationDelivery(db: Db, input: Omit<NativeUsageDeliveryInput, "billing">) {
const sourceId = uuid(`${input.runId}:${input.runnerInstanceId}`);
const [source] = await db.select({ runnerInstanceId: heartbeatRuns.runnerInstanceId, usage: heartbeatRuns.usageJson })
.from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.runId)));
if (source?.runnerInstanceId !== input.runnerInstanceId || source.usage?.accountingReceiptOwner !== "native_producer"
|| source.usage.accountingReceiptSourceId !== sourceId) throw new Error("native_usage_continuation_source_mismatch");
const saved = await loadNativeUsageReceipts(db, input, sourceId);
if (!saved.length) throw new Error("native_usage_continuation_receipts_missing");
const classification = (receipt: AdapterUsageCheckpoint) => ({
provider: receipt.provider, biller: receipt.biller, billingType: receipt.billingType,
});
const receipts = saved.map(row => (row.receipt as UsageReceiptEnvelope).receipt as AdapterUsageCheckpoint);
const billing = classification(receipts[0]);
if (receipts.some(receipt => receiptFingerprint(classification(receipt)) !== receiptFingerprint(billing)
|| receipt.model !== (input.provider.model ?? "unknown"))) throw new Error("native_usage_source_conflict");
return createNativeUsageReceiver(db, { ...input, billing }, sourceId, saved, true);
}
function loadNativeUsageReceipts(db: Db, input: Pick<NativeUsageDeliveryInput, "companyId" | "runId">, sourceId: string) {
return db.select({ receipt: runUsageReceipts.receiptJson, hash: runUsageReceipts.receiptHash,
sequence: runUsageReceipts.sequence }).from(runUsageReceipts).where(and(
eq(runUsageReceipts.companyId, input.companyId), eq(runUsageReceipts.runId, input.runId),
eq(runUsageReceipts.sourceId, sourceId))).orderBy(asc(runUsageReceipts.sequence));
}
function createNativeUsageReceiver(db: Db, input: NativeUsageDeliveryInput, sourceId: string,
saved: Awaited<ReturnType<typeof loadNativeUsageReceipts>>, verifySaved = false) {
const accepted = new Map<string, UsageReceiptEnvelope>();
const turnAccounting = ["opencode", "acpx"].includes(input.provider.kind) ? createNativeTurnAccounting(input.provider) : null;
let observedUsage: Record<string, unknown> | null = null;
@@ -60,6 +92,12 @@ export async function createNativeUsageDelivery(db: Db, input: {
for (const row of saved) {
const envelope = row.receipt as UsageReceiptEnvelope;
if (!envelope.producerEvent) throw new Error("native_usage_receipt_missing_producer");
if (verifySaved && (envelope.companyId !== input.companyId || envelope.runId !== input.runId
|| envelope.sourceId !== sourceId || envelope.sequence !== row.sequence || envelope.adapterType !== "paperclip_runner"
|| envelope.id !== uuid(`${input.runId}:${input.runnerInstanceId}:${envelope.producerEvent.sourceEventId}`)
|| receiptFingerprint(envelope) !== row.hash || accepted.has(envelope.producerEvent.sourceEventId))) {
throw new Error("native_usage_source_conflict");
}
accepted.set(envelope.producerEvent.sourceEventId, envelope);
observe(envelope);
}
@@ -48,6 +48,7 @@ import {
import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
import { startRecoveryPostgres } from "../../__tests__/helpers/recovery-postgres.js";
import { PaperclipControlPlanePort } from "./paperclip-control-plane-port.js";
import { nativeSha256 } from "./canonical.js";
import { NativeRunCoordinatorStore } from "./native-run-coordinator-store.js";
import { finalizeNativeRun, repairCommittedNativeChatResponse } from "./native-run-finalizer.js";
import { nativeRuntimeContextFixture } from "./runtime-context.test-fixture.js";
@@ -313,6 +314,63 @@ describe("PaperclipControlPlanePort conformance", () => {
}
});
it("publishes native recovery metadata in the existing snapshot transaction and fences stale generations", async () => {
const identity = { ...CONTROL_PLANE_CONFORMANCE_OPEN.identity, runId: randomUUID(), sessionId: randomUUID() };
const runnerId = randomUUID();
const owner = { ...identity, authorityRunId: identity.runId, bootId: randomUUID(), generation: 1, leaseOwner: "snapshot-owner" };
await db.insert(heartbeatRuns).values({ id: identity.runId, companyId: identity.companyId, agentId: identity.agentId,
status: "running", runtimeMode: "native", nativeIssueId: identity.issueId, nativeSessionId: identity.sessionId,
runnerInstanceId: runnerId, completionContractId: contractId, completionContractSha256: contractSha,
runnerProfileJson: { unrelated: "preserve" } });
await db.insert(nativeRunFinalizations).values({ runId: identity.runId, companyId: identity.companyId, issueId: identity.issueId,
phase: "observed",
controllerBootId: owner.bootId, controllerGeneration: owner.generation, leaseOwner: owner.leaseOwner,
leaseExpiresAt: new Date(Date.now() + 60_000) });
const port = new PaperclipControlPlanePort(db, { ...identity, completionContractId: contractId,
completionContractSha256: contractSha, sourceInstanceId: runnerId, controlPlaneSourceInstanceId: "snapshot-control" }, {
nativeRecovery: { owner, compatibility: { configDigest: `sha256:${"a".repeat(64)}`, processConfigDigest: null } },
});
await port.openRun({ identity, backendKind: "mock", sourceInstanceId: runnerId });
const snapshot = { backendKind: "mock" as const, sessionId: identity.sessionId, identity };
const statements: string[] = [];
const previousDebug = db.$client.options.debug;
db.$client.options.debug = (_connection, statement) => { statements.push(statement); };
try { await port.checkpointSession(snapshot); }
finally { db.$client.options.debug = previousDebug; }
expect(statements.filter(statement => /^begin\b/i.test(statement.trim()))).toHaveLength(1);
expect(statements.filter(statement => /^update\s+"heartbeat_runs"/i.test(statement.trim()))).toHaveLength(1);
expect(statements.filter(statement => /^(?:with|select)\b/i.test(statement.trim()))).toHaveLength(1);
const [saved] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, identity.runId));
expect(saved.runnerProfileJson).toMatchObject({ unrelated: "preserve", sessionCheckpoint: snapshot,
nativeControllerRecovery: { schema: "paperclip.native-controller-recovery.v1", snapshotSha256: nativeSha256(saved.runnerProfileJson?.sessionCheckpoint),
checkpoint: null, processConfigDigest: null } });
// Missing PRP is an explicit incomplete publication, never fabricated authority.
await db.update(nativeRunFinalizations).set({ controllerGeneration: 2 }).where(eq(nativeRunFinalizations.runId, identity.runId));
await expect(port.checkpointSession({ ...snapshot, providerSessionId: "stale" })).rejects.toThrow("checkpoint_owner_lost");
await expect(port.completeRun({ result: CONTROL_PLANE_CONFORMANCE_RESULT, terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL }))
.rejects.toThrow("checkpoint_owner_lost");
const [unchanged] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, identity.runId));
expect(unchanged.runnerProfileJson).toEqual(saved.runnerProfileJson);
// A valid coordinator lock does not certify a lease that expires while the
// subsequent run-row lock waits. Recheck time at the actual write boundary.
await db.update(nativeRunFinalizations).set({ controllerGeneration: 1, leaseExpiresAt: new Date(Date.now() + 150) })
.where(eq(nativeRunFinalizations.runId, identity.runId));
let locked!: () => void, release!: () => void;
const acquired = new Promise<void>(resolve => { locked = resolve; });
const held = new Promise<void>(resolve => { release = resolve; });
const blocker = db.transaction(async tx => {
await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(eq(heartbeatRuns.id, identity.runId)).for("update");
locked();
await held;
});
await acquired;
const waiting = port.checkpointSession(snapshot);
const rejected = expect(waiting).rejects.toThrow("checkpoint_owner_lost");
try { await new Promise(resolve => setTimeout(resolve, 200)); }
finally { release(); await blocker; }
await rejected;
});
it("permits only read-only admission while an existing startup drains", async () => {
const identity = { ...CONTROL_PLANE_CONFORMANCE_OPEN.identity, runId: randomUUID(), sessionId: randomUUID() };
const runnerId = randomUUID();
@@ -1,7 +1,7 @@
import { notifyNativeResultCommit } from "../native-result-commit-notifications.js";
import { notifyNativeWorkspaceFinalization } from "../native-workspace-finalization-notifications.js";
import { createAgentIdentityRedactor } from "../agent-identity-redaction.js";
import { and, asc, eq, gt, or } from "drizzle-orm";
import { and, asc, eq, gt, or, sql } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import {
heartbeatRunEvents,
@@ -9,6 +9,7 @@ import {
issues,
nativeRunFinalizations,
nativeRunResults,
nativeRunnerCheckpoints,
} from "@paperclipai/db";
import type {
CompleteControlPlaneRunInput,
@@ -30,6 +31,10 @@ import { appendHeartbeatRunEvent } from "../heartbeat-run-events.js";
import { publishLiveEvent } from "../live-events.js";
import { nativeSha256 } from "./canonical.js";
import { redactRunnerApiEvent, redactRunnerApiPayloads } from "./runner-api-log-redaction.js";
import { createNativeControllerRecovery, NATIVE_CONTROLLER_RECOVERY_MAX_BYTES, type NativeControllerCompatibility } from "./native-controller-recovery.js";
import { eligibleNativeTaskSessionPublication, publishNativeTaskSessionInTransaction, NATIVE_TASK_SESSION_PUBLICATION_KEY,
type NativeTaskSessionPublicationContext, type NativeTaskSessionPublicationIssue } from "./native-task-session-publication.js";
import { lockNativeRunnerCheckpointOwner, nativeRunnerCheckpointOwnerLock, validateNativeRunnerCheckpointOwner, inspectNativeRunnerUpgradeHeader, type NativeRunnerCheckpointOwner } from "./native-runner-checkpoint.js";
export interface PaperclipControlPlaneBinding {
companyId: string;
@@ -90,6 +95,9 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
readonly #onEventPersistence?: (event: PrpEvent, startedAtMs: number, endedAtMs: number) => Promise<void>;
readonly #onCommittedEvent?: (event: PrpEvent, cursor: number) => Promise<void>;
readonly #onDuplicateEvent?: (event: PrpEvent, cursor: number) => Promise<void>;
readonly #nativeRecovery?: { compatibility: NativeControllerCompatibility; owner: NativeRunnerCheckpointOwner };
readonly #taskSessionPublication?: NativeTaskSessionPublicationContext;
readonly #taskSessionPublicationOwner?: NativeRunnerCheckpointOwner;
constructor(
db: Db,
@@ -105,6 +113,9 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
onEventPersistence?: (event: PrpEvent, startedAtMs: number, endedAtMs: number) => Promise<void>;
onCommittedEvent?: (event: PrpEvent, cursor: number) => Promise<void>;
onDuplicateEvent?: (event: PrpEvent, cursor: number) => Promise<void>;
nativeRecovery?: { compatibility: NativeControllerCompatibility; owner: NativeRunnerCheckpointOwner };
taskSessionPublication?: NativeTaskSessionPublicationContext;
taskSessionPublicationOwner?: NativeRunnerCheckpointOwner;
} = {},
) {
this.#db = db;
@@ -116,6 +127,9 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
this.#onEventPersistence = options.onEventPersistence;
this.#onResultCommitted = options.onResultCommitted;
this.#onDuplicateEvent = options.onDuplicateEvent;
this.#nativeRecovery = options.nativeRecovery;
this.#taskSessionPublication = options.taskSessionPublication ? structuredClone(options.taskSessionPublication) : undefined;
this.#taskSessionPublicationOwner = options.taskSessionPublicationOwner;
}
#assertActive(options?: { signal: AbortSignal }): void {
@@ -123,7 +137,9 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
options?.signal.throwIfAborted();
}
#matchesPersistedBinding(run: typeof heartbeatRuns.$inferSelect): boolean {
#matchesPersistedBinding(run: Pick<typeof heartbeatRuns.$inferSelect,
"companyId" | "agentId" | "runtimeMode" | "nativeIssueId" | "nativeSessionId" |
"runnerInstanceId" | "completionContractId" | "completionContractSha256">): boolean {
return run.companyId === this.#binding.companyId
&& run.agentId === this.#binding.agentId
&& run.runtimeMode === "native"
@@ -200,19 +216,105 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
|| identity.agentId !== this.#binding.agentId
|| this.#sessionId !== identity.sessionId
) throw new Error("native_session_checkpoint_binding_mismatch");
const publication = this.#taskSessionPublication && eligibleNativeTaskSessionPublication(snapshot)
? this.#taskSessionPublication : null;
if (publication && !this.#taskSessionPublicationOwner) throw new Error("native_task_session_publication_owner_required");
await this.#db.transaction(async (tx) => {
const run = await tx.select().from(heartbeatRuns)
.where(eq(heartbeatRuns.id, this.#binding.runId)).for("update").limit(1)
const recovery = this.#nativeRecovery;
const owner = recovery?.owner ?? (publication ? this.#taskSessionPublicationOwner : undefined);
if (publication && recovery && nativeSha256(recovery.owner) !== nativeSha256(this.#taskSessionPublicationOwner)) {
throw new Error("native_task_session_publication_owner_mismatch");
}
const ownerLock = owner ? nativeRunnerCheckpointOwnerLock(tx, owner) : null;
// NO KEY UPDATE serializes reset/admission fields while remaining compatible
// with completeRun's parent KEY SHARE. Keep coordinator → issue → run order.
const publicationIssue = publication ? tx.$with("publication_issue").as(tx.select({
id: issues.id, companyId: issues.companyId, executionRunId: issues.executionRunId,
conversationAgentId: issues.conversationAgentId, conversationSessionGeneration: issues.conversationSessionGeneration,
}).from(issues).where(and(eq(issues.id, this.#binding.issueId), eq(issues.companyId, this.#binding.companyId),
ownerLock ? sql`exists (select 1 from ${ownerLock})` : undefined)).for("no key update")) : null;
const lockedRun = tx.$with("snapshot_run").as(tx.select({
id: heartbeatRuns.id, status: heartbeatRuns.status,
contextSnapshot: (publication ? sql<Record<string, unknown>>`jsonb_build_object('conversationSessionGeneration', ${heartbeatRuns.contextSnapshot}->'conversationSessionGeneration')` : sql<null>`null`).as("publication_context"),
resultJson: (publication ? sql<Record<string, unknown>>`jsonb_build_object('nativeCancellation', ${heartbeatRuns.resultJson}->'nativeCancellation',
'cancellation', ${heartbeatRuns.resultJson}->'cancellation', 'startupCancellation', ${heartbeatRuns.resultJson}->'startupCancellation')` : sql<null>`null`).as("publication_cancellation"),
startedAt: heartbeatRuns.startedAt, createdAt: heartbeatRuns.createdAt,
companyId: heartbeatRuns.companyId, agentId: heartbeatRuns.agentId,
runtimeMode: heartbeatRuns.runtimeMode, nativeIssueId: heartbeatRuns.nativeIssueId,
nativeSessionId: heartbeatRuns.nativeSessionId, runnerInstanceId: heartbeatRuns.runnerInstanceId,
completionContractId: heartbeatRuns.completionContractId,
completionContractSha256: heartbeatRuns.completionContractSha256,
runnerProfileJson: heartbeatRuns.runnerProfileJson,
}).from(heartbeatRuns)
.where(and(eq(heartbeatRuns.id, this.#binding.runId), ownerLock
? sql`exists (select 1 from ${ownerLock})` : undefined,
publicationIssue ? sql`exists (select 1 from ${publicationIssue})` : undefined)).for("update"));
// Project the locking CTE's tuples, including any update committed while
// the lock waited. A second scan of heartbeat_runs could see an old MVCC tuple.
// The run CTE depends on the coordinator CTE: both locks precede the
// outer clock sample, including when either lock waits.
const selected = await tx.with(...(ownerLock ? [ownerLock] : []), ...(publicationIssue ? [publicationIssue] : []), lockedRun).select({ run: {
id: lockedRun.id, status: lockedRun.status, contextSnapshot: lockedRun.contextSnapshot, resultJson: lockedRun.resultJson,
startedAt: lockedRun.startedAt, createdAt: lockedRun.createdAt,
companyId: lockedRun.companyId, agentId: lockedRun.agentId,
runtimeMode: lockedRun.runtimeMode, nativeIssueId: lockedRun.nativeIssueId,
nativeSessionId: lockedRun.nativeSessionId, runnerInstanceId: lockedRun.runnerInstanceId,
completionContractId: lockedRun.completionContractId,
completionContractSha256: lockedRun.completionContractSha256,
runnerProfileJson: lockedRun.runnerProfileJson,
}, now: sql<Date>`clock_timestamp()`, ...(ownerLock ? {
leaseOwner: sql<string | null>`(select ${ownerLock.leaseOwner} from ${ownerLock})`,
leaseExpiresAt: sql<Date | null>`(select ${ownerLock.leaseExpiresAt} from ${ownerLock})`.mapWith(nativeRunFinalizations.leaseExpiresAt),
resultId: sql<string | null>`(select ${ownerLock.resultId} from ${ownerLock})`,
} : {}), ...(recovery ? {
checkpointHeader: sql<Record<string, unknown> | null>`(select material from ${nativeRunnerCheckpoints}
where ${nativeRunnerCheckpoints.runId} = ${this.#binding.runId}
and ${nativeRunnerCheckpoints.companyId} = ${this.#binding.companyId}
and ${nativeRunnerCheckpoints.issueId} = ${this.#binding.issueId}
and ${nativeRunnerCheckpoints.part} = 'header')`,
} : {}), ...(publicationIssue ? { publicationIssue: sql<NativeTaskSessionPublicationIssue>`(select jsonb_build_object(
'id', ${publicationIssue.id}, 'companyId', ${publicationIssue.companyId}, 'executionRunId', ${publicationIssue.executionRunId},
'conversationAgentId', ${publicationIssue.conversationAgentId}, 'conversationSessionGeneration', ${publicationIssue.conversationSessionGeneration}) from ${publicationIssue})` } : {}),
}).from(lockedRun).limit(1)
.then((rows) => rows[0] ?? null);
const run = selected?.run;
this.#assertActive(options);
if (owner && !selected) throw new Error("native_runner_checkpoint_owner_lost");
if (!run || !this.#matchesPersistedBinding(run)) {
throw new Error("native_session_checkpoint_binding_mismatch");
}
await tx.update(heartbeatRuns).set({
runnerProfileJson: {
// The run row lock may have waited beyond our lease. Generation cannot
// change while we hold the coordinator, but wall-clock expiry still can.
if (owner && selected) await validateNativeRunnerCheckpointOwner(tx, owner,
{ leaseOwner: selected.leaseOwner ?? null, leaseExpiresAt: selected.leaseExpiresAt ?? null,
resultId: selected.resultId ?? null, now: selected.now }, !snapshot.semanticResult);
const header = recovery && selected?.checkpointHeader
? inspectNativeRunnerUpgradeHeader(this.#binding, selected.checkpointHeader) : null;
if (header && (header.normalizedSessionId !== identity.sessionId || header.runnerInstanceId !== this.#binding.sourceInstanceId)) {
throw new Error("native_session_checkpoint_prp_binding_mismatch");
}
const profile: Record<string, unknown> = {
...record(run.runnerProfileJson),
sessionCheckpoint: structuredClone(snapshot) as unknown as Record<string, unknown>,
},
...(recovery ? { nativeControllerRecovery: createNativeControllerRecovery(snapshot, recovery.compatibility,
header ? { runId: this.#binding.runId, runnerInstanceId: header.runnerInstanceId,
normalizedSessionId: header.normalizedSessionId } : null) } : {}),
};
// Driver/live snapshots may already describe a terminal result before
// completeRun accepts it. Persist that observation without a handled
// receipt; the existing post-acceptance checkpoint publishes the pointer.
if (publication && selected?.resultId) {
const receipt = await publishNativeTaskSessionInTransaction(tx, { binding: this.#binding, context: publication,
snapshot, run, issue: selected.publicationIssue ?? null, resultId: selected.resultId ?? null,
recoveryReady: !recovery || header !== null });
profile[NATIVE_TASK_SESSION_PUBLICATION_KEY] = receipt;
if (Buffer.byteLength(JSON.stringify({ nativeControllerRecovery: profile.nativeControllerRecovery,
[NATIVE_TASK_SESSION_PUBLICATION_KEY]: receipt })) > NATIVE_CONTROLLER_RECOVERY_MAX_BYTES) {
throw new Error("native_task_session_publication_metadata_too_large");
}
}
await tx.update(heartbeatRuns).set({
runnerProfileJson: profile,
sessionIdAfter: snapshot.providerSessionId ?? snapshot.sessionId,
nativePhase: snapshot.semanticResult ? "workspace_finalizing" : "observed",
nativePhaseUpdatedAt: new Date(),
@@ -386,9 +488,12 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
});
const insertedResult = await this.#db.transaction(async (tx) => {
// Result insertion checks the task foreign key. Acquire that parent lock
// before the run, matching task mutations that subsequently update a run.
// Otherwise concurrent chat/status writes can deadlock after generation.
// Status commit and snapshot publication lock coordinator → issue → run.
// A replay must follow that order too: holding the issue while waiting
// for the coordinator deadlocks a status commit's issue UPDATE lock.
const recovery = this.#nativeRecovery;
const owner = recovery ? await lockNativeRunnerCheckpointOwner(tx, recovery.owner) : null;
// Result insertion checks the task FK. Keep its parent lock before run.
await tx.select({ id: issues.id }).from(issues)
.where(and(eq(issues.id, this.#binding.issueId), eq(issues.companyId, this.#binding.companyId)))
.for("key share");
@@ -399,6 +504,10 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
if (!run || !this.#matchesPersistedBinding(run)) {
throw new Error("native_result_binding_mismatch");
}
if (recovery && owner) {
const [clock] = await tx.execute<{ now: Date }>(sql`select clock_timestamp() as now`);
await validateNativeRunnerCheckpointOwner(tx, recovery.owner, { ...owner, now: clock.now }, false);
}
const callerConditions = [eq(nativeRunResults.serverFingerprint, serverFingerprint)];
if (value.callerResultId) callerConditions.push(eq(nativeRunResults.callerResultId, value.callerResultId));
@@ -411,6 +520,7 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
if (existing.canonicalSha256 !== canonicalSha256) throw new Error("structured_result_replay_conflict");
return false;
}
if (owner?.resultId) throw new Error("structured_result_replay_conflict");
const resultJson = {
result: validated.result,
terminal: value.terminal,
@@ -120,7 +120,7 @@ describe("PaperclipRunnerToolAuthority", () => {
issueId,
runId,
});
expect(authority.definitions()).toHaveLength(50);
expect(authority.definitions()).toHaveLength(51);
expect(authority.definitions().map(tool => tool.name)).toEqual(expect.arrayContaining([
"get_workspace", "list_workspaces", "select_workspace", "prepare_repository",
]));
@@ -28,6 +28,7 @@ import { createReadStream } from "node:fs";
import { publicChatTaskUrl } from "../chat-task-url.js";
import { agentCommentaryToolInputSchema } from "@paperclipai/shared";
import { submitAgentCommentary } from "../agent-commentary.js";
import { sendAgentChatMessage } from "../agent-chat-messages.js";
import type { createAssignedMcpTools } from "./assigned-mcp-tools.js";
import { captureRunnerToolResult, fitRunnerToolResultPage, READ_TOOL_RESULT, READ_TOOL_RESULT_DEFINITION, readToolResultSchema, RUNNER_TOOL_RESULT_MAX_BYTES } from "./runner-tool-results.js";
import { assertAssignableAgent } from "../agent-assignability.js";
@@ -126,6 +127,7 @@ import {
const EXTERNAL_WORKSPACE_OPERATIONS = new Set(["get_workspace", "list_workspaces", "select_workspace", "prepare_repository", "register_deliverable"]);
const IMPLEMENTED_OPERATIONS = new Set([
"send_chat_message",
"submit_complaint", "submit_suggestion",
"read_agent_instructions", "update_agent_instructions", "get_agent_instruction_history", "restore_agent_instructions",
"search_api", "call_api", "hire_agent",
@@ -434,6 +436,7 @@ export class PaperclipRunnerToolAuthority {
// retry, without ever falling through to an uncertain mutation.
let recovered: unknown;
if (call.tool === "set_task_title") recovered = await this.#setTaskTitle(input, true);
else if (call.tool === "send_chat_message") recovered = await this.#sendChatMessage(input, this.#privacyActor(run), true);
else if (call.tool === "submit_complaint" || call.tool === "submit_suggestion") recovered = await this.#submitCommentary(call.tool, input, true);
else if (call.tool === "create_task") recovered = await this.#createTask(input, null, true);
else if (call.tool === "request_human_input") recovered = await this.#requestHumanInput(input, null, true);
@@ -959,6 +962,7 @@ export class PaperclipRunnerToolAuthority {
return { approval, tasks: tasks.filter(row => !readable || readable.has(row.issue.id)).map(row => row.issue) };
}
case "report_progress": return this.#reportProgress(input);
case "send_chat_message": return this.#sendChatMessage(input, this.#privacyActor(context.run));
case "request_human_input": return this.#requestHumanInput(input,
(await captureRunIdentity(this.db, this.binding, { readScope: "identity" })).context?.id ?? null);
case "create_task": return this.#createTask(input,
@@ -1486,6 +1490,14 @@ export class PaperclipRunnerToolAuthority {
return { interactionId: review.interaction.id, status: expectedStatus };
}
async #sendChatMessage(input: Record<string, unknown>, actor: AuthorizationActor, receiptOnly = false) {
const result = await sendAgentChatMessage(this.db, { ...this.binding, keyScope: actor.keyScope,
onBehalfOfUserId: actor.onBehalfOfUserId, authorizingUserId: actor.authorizingUserId, source: actor.source }, input, {
authorizeNative: tx => this.#lockAuthorizedMutationContext(tx), receiptOnly,
});
return result ?? undefined;
}
async #reportProgress(input: Record<string, unknown>): Promise<unknown> {
const body = typeof input.body === "string" ? input.body.trim() : "";
const idempotencyKey = typeof input.idempotencyKey === "string" ? input.idempotencyKey.trim() : "";
@@ -91,6 +91,23 @@ it.skipIf(!tar || !hasZstd)("reuses only byte-matched image files, keeps modes,
expect(await fs.readlink(path.join(targetPath, "tool"))).toBe("bin/tool");
});
it.skipIf(!tar || !hasZstd)("stages a complete layered pack into a fresh nested runtime directory", async () => {
const { root, pack, archive } = await layeredFixture();
const remote = await runner(root);
const workspace = path.join(root, "workspace");
await fs.mkdir(workspace);
const targetPath = path.join(workspace, ".paperclip-runtime", "paperclip-runner", "provider-pack");
await expect(fs.lstat(path.dirname(targetPath))).rejects.toMatchObject({ code: "ENOENT" });
expect(await stageProviderPackArchive({ archive, runner: remote, targetPath, reuseRoots: [pack] })).toBe(true);
expect(await fs.readFile(path.join(targetPath, "bin/large"))).toEqual(await fs.readFile(path.join(pack, "bin/large")));
expect(await fs.readFile(path.join(targetPath, "provider-pack.json"))).toEqual(await fs.readFile(path.join(pack, "provider-pack.json")));
expect(await fs.readlink(path.join(targetPath, "tool"))).toBe("bin/tool");
for (const directory of [targetPath, path.dirname(targetPath), path.dirname(path.dirname(targetPath))]) {
expect((await fs.stat(directory)).mode & 0o777).toBe(0o700);
}
expect((await fs.stat(path.join(targetPath, "bin/large"))).ino).not.toBe((await fs.stat(path.join(pack, "bin/large"))).ino);
});
it.skipIf(!tar || !hasZstd).each(["missing", "wrong-digest", "symlink", "ancestor-symlink"])("transfers a large layer when the image file is %s", async reason => {
const { root, pack, archive } = await layeredFixture();
const remote = await runner(root);
@@ -135,13 +152,18 @@ it.skipIf(!tar || !hasZstd)("rejects a corrupt missing layer before transfer and
expect(remote.syncIn).not.toHaveBeenCalled();
});
it.skipIf(!tar || !hasZstd)("does not follow a layered target symlink or accept invalid reuse receipts", async () => {
it.skipIf(!tar || !hasZstd).each(["directory", "file", "symlink"])("rejects an existing layered target %s or invalid reuse receipts", async kind => {
const { root, archive } = await layeredFixture();
const remote = await runner(root);
const targetPath = path.join(root, "target"), other = path.join(root, "other");
await fs.mkdir(other); await fs.symlink(other, targetPath);
await fs.mkdir(other);
if (kind === "directory") await fs.mkdir(targetPath);
else if (kind === "file") await fs.writeFile(targetPath, "untouched");
else await fs.symlink(other, targetPath);
await expect(stageProviderPackArchive({ archive, runner: remote, targetPath })).rejects.toThrow("runner_remote_provider_archive_reuse_failed");
expect(await fs.readdir(other)).toEqual([]);
if (kind === "directory") expect(await fs.readdir(targetPath)).toEqual([]);
if (kind === "file") expect(await fs.readFile(targetPath, "utf8")).toBe("untouched");
remote.execute.mockResolvedValue({ exitCode: 0, timedOut: false, stdout: "[999999]", stderr: "", signal: null, pid: null, startedAt: null });
await expect(stageProviderPackArchive({ archive, runner: remote, targetPath: `${targetPath}-2` })).rejects.toThrow("runner_remote_provider_archive_reuse_failed");
expect(remote.syncIn).not.toHaveBeenCalled();
@@ -70,7 +70,8 @@ const reuseScript = `
(async()=>{
const fs=require('node:fs'),p=require('node:path'),c=require('node:crypto');
const spec=JSON.parse(Buffer.from(process.argv[1],'base64').toString('utf8'));
const f=fs.promises;await f.mkdir(spec.target,{mode:448});
const f=fs.promises;await f.mkdir(p.dirname(spec.target),{recursive:true,mode:448});
await f.mkdir(spec.target,{mode:448});
const missing=[];
const regularPath=async(root,relative)=>{
let current=root;if(!(await f.lstat(current)).isDirectory())return false;
@@ -18,7 +18,7 @@ import {
AGENTCORE_QUALIFIED_MODEL,
CLAUDE_MANAGED_QUALIFIED_MODEL,
} from "../provider-profile-qualification.js";
import { resolveAcpxQualification, resolveAntigravityCandidateAdmission, type AcpxQualificationCandidate } from "./acpx-qualification.js";
import { resolveAcpxQualification, resolveAntigravityCandidateAdmission, resolveKimiCandidateAdmission, type AcpxQualificationCandidate } from "./acpx-qualification.js";
import { compatibleCodexModel } from "./codex-model-fallback.js";
export const QUALIFIED_OPENCODE_RUNNER_VERSION = "1.18.34" as const;
@@ -34,6 +34,7 @@ export const DEFAULT_ACPX_RUNNER_MODELS = {
cursor: null,
copilot: null,
pi: null,
kimi: null,
} as const;
export type QualifiedPaperclipRunnerAcpxAgent =
@@ -564,6 +565,15 @@ export function resolvePaperclipRunnerProviderProfile(
throw new PaperclipRunnerProviderProfileError("paperclip_runner_acpx_qualification_invalid", "Antigravity candidate admission requires the authorized company, exact model and runtime identity.");
}
}
if (pendingAcpxProfile.value === "kimi") {
try {
if (resolveKimiCandidateAdmission({ companyId: context.companyId, model }, process.env)) {
return { provider: "acpx", backend: "acpx_runtime", model, acpxAgent: "kimi" };
}
} catch {
throw new PaperclipRunnerProviderProfileError("paperclip_runner_acpx_qualification_invalid", "Kimi candidate admission requires the authorized company, exact model, target and runtime identity.");
}
}
// Qualification authority belongs to the isolated server operator. Never
// read this allowlist from adapter config, credential refs, or run env.
let qualification: AcpxQualificationCandidate | undefined;
@@ -577,10 +587,10 @@ export function resolvePaperclipRunnerProviderProfile(
}
throw new PaperclipRunnerProviderProfileError("paperclip_runner_acpx_agent_unavailable", `${pendingAcpxProfile.label} is awaiting native Runner acceptance. Its profile is not enabled for production runs.`);
}
if ((acpxAgent === "cursor" || acpxAgent === "pi") && !model) {
if ((acpxAgent === "cursor" || acpxAgent === "pi" || acpxAgent === "kimi") && !model) {
throw new PaperclipRunnerProviderProfileError("paperclip_runner_acpx_model_required", "This ACPX agent requires an explicit model ID; there is no default model.");
}
if (acpxAgent !== "claude" && acpxAgent !== "codex" && acpxAgent !== "grok" && acpxAgent !== "cursor" && acpxAgent !== "copilot" && acpxAgent !== "pi") {
if (acpxAgent !== "claude" && acpxAgent !== "codex" && acpxAgent !== "grok" && acpxAgent !== "cursor" && acpxAgent !== "copilot" && acpxAgent !== "pi" && acpxAgent !== "kimi") {
throw new PaperclipRunnerProviderProfileError(
"paperclip_runner_acpx_agent_unavailable",
"Paperclip Runner ACPX requires a qualified agent profile.",
@@ -10,7 +10,7 @@ export function shouldUseBundledAcpxImageAssets(input: {
}): boolean {
return input.requiresRemoteProviderPack && !input.configuredProviderPackRoot &&
input.provider.kind === "acpx" &&
["cursor", "copilot"].includes(input.provider.agent);
["cursor", "copilot", "kimi"].includes(input.provider.agent);
}
type VerificationCommandResult = {
@@ -0,0 +1,43 @@
import { describe, expect, it } from "vitest";
import { isSettledRemoteWarmControlState } from "./remote-warm-session-recovery.js";
const identity = { runId: "run", normalizedSessionId: "session", runnerInstanceId: "runner", environmentLeaseId: "lease", turnId: "turn", itemId: "item" };
const terminal = { eventType: "run.terminal", envelope: { ...identity } };
const goal = { schema: "paperclip.session_goal.snapshot.v1", goal: null, workingNow: false };
const notification = { schema: "paperclip.prp.event.v1", eventType: "session.goal.snapshot", runId: identity.runId,
normalizedSessionId: identity.normalizedSessionId, sourceInstanceId: "runner", sourceKind: "runner", payload: goal };
const snapshot = { eventType: "session.goal.snapshot", envelope: { ...identity, payload: notification } };
const state = (events: unknown[]) => ({ identity, commands: [{ status: "completed" }], committedEvents: events });
describe("settled remote warm control evidence", () => {
it.each(["paperclip.prp.event.v1", "paperclip.prp.event.v2"])("accepts the terminal and an authenticated %s idle goal-read suffix without changing its order", schema => {
const suffix = { ...snapshot, envelope: { ...snapshot.envelope, payload: { ...notification, schema } } };
const control = state([terminal, suffix]);
expect(isSettledRemoteWarmControlState(control, identity)).toBe(true);
expect(control.committedEvents).toEqual([terminal, suffix]);
expect(isSettledRemoteWarmControlState(state([terminal]), identity)).toBe(true);
});
it.each([
["active goal", { ...snapshot, envelope: { ...snapshot.envelope, payload: { ...notification, payload: { ...goal, goal: { status: "active" } } } } }],
["provider work", { ...snapshot, envelope: { ...snapshot.envelope, payload: { ...notification, payload: { ...goal, workingNow: true } } } }],
["unknown event", { ...snapshot, eventType: "unknown" }],
["unknown protocol version", { ...snapshot, envelope: { ...snapshot.envelope, payload: { ...notification, schema: "paperclip.prp.event.v3" } } }],
["new turn", { ...snapshot, eventType: "turn.started" }],
["foreign run", { ...snapshot, envelope: { ...snapshot.envelope, runId: "other" } }],
["foreign session", { ...snapshot, envelope: { ...snapshot.envelope, normalizedSessionId: "other" } }],
["foreign source", { ...snapshot, envelope: { ...snapshot.envelope, payload: { ...notification, sourceInstanceId: "other" } } }],
["foreign lease", { ...snapshot, envelope: { ...snapshot.envelope, environmentLeaseId: "other" } }],
["foreign notification", { ...snapshot, envelope: { ...snapshot.envelope, payload: { ...notification, runId: "other" } } }],
["missing payload", { ...snapshot, envelope: { ...identity } }],
])("holds a %s suffix", (_name, suffix) => {
expect(isSettledRemoteWarmControlState(state([terminal, suffix]), identity)).toBe(false);
});
it("holds missing terminals, pending commands and mixed core identities", () => {
expect(isSettledRemoteWarmControlState(state([snapshot]), identity)).toBe(false);
expect(isSettledRemoteWarmControlState({ ...state([terminal, snapshot]), commands: [{ status: "dispatched" }] }, identity)).toBe(false);
expect(isSettledRemoteWarmControlState({ ...state([terminal, snapshot]), identity: { ...identity, runnerInstanceId: "other" } }, identity)).toBe(false);
expect(isSettledRemoteWarmControlState(state([{ ...terminal, envelope: { ...identity, runId: "other" } }, snapshot]), identity)).toBe(false);
});
});
@@ -3,6 +3,7 @@ import { agents, companies, environmentLeases, heartbeatRuns, issues, nativeRunF
import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target";
import { parseNativeExecutionInput, type NativeExecutionInput, type PersistedNativeSession } from "../../vendor/paperclip-runner/index.js";
import { currentNativeControllerIdentity, evaluateNativeControllerTakeover } from "./native-restart-recovery.js";
import { readNativeControllerRecovery } from "./native-controller-recovery.js";
export type RemoteWarmSessionRecovery = {
identity: Record<string, string>;
@@ -13,6 +14,36 @@ export type RemoteWarmSessionRecovery = {
revalidate: (leaseOwner: string) => Promise<void>;
};
/** A goal read can publish an idle snapshot after the terminal. It carries no
* new work; every other suffix still requires reconciliation before handoff. */
export function isSettledRemoteWarmControlState(control: Record<string, unknown>, identity: Record<string, string>): boolean {
const record = (value: unknown): Record<string, unknown> =>
value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
const commands = control.commands;
const events = control.committedEvents;
if (!Array.isArray(commands) || commands.some(command => record(command).status !== "completed") ||
!Array.isArray(events) || Object.entries(identity).some(([key, value]) => record(control.identity)[key] !== value)) return false;
const terminalIndex = events.findLastIndex(event => record(event).eventType === "run.terminal");
if (terminalIndex < 0) return false;
const matchesSession = (event: unknown) => {
const envelope = record(record(event).envelope);
return envelope.runId === identity.runId && envelope.normalizedSessionId === identity.normalizedSessionId;
};
if (!matchesSession(events[terminalIndex])) return false;
return events.slice(terminalIndex + 1).every(event => {
const envelope = record(record(event).envelope);
const notification = record(envelope.payload);
const payload = record(notification.payload);
return record(event).eventType === "session.goal.snapshot" && matchesSession(event) &&
Object.entries(identity).every(([key, value]) => envelope[key] === value) &&
(notification.schema === "paperclip.prp.event.v1" || notification.schema === "paperclip.prp.event.v2") &&
notification.eventType === "session.goal.snapshot" &&
notification.sourceInstanceId === identity.runnerInstanceId && notification.sourceKind === "runner" &&
notification.runId === identity.runId && notification.normalizedSessionId === identity.normalizedSessionId &&
payload.schema === "paperclip.session_goal.snapshot.v1" && payload.goal === null && payload.workingNow === false;
});
}
export async function inspectRemoteWarmSessionRecovery(input: {
db: Db;
execution: NativeExecutionInput;
@@ -52,15 +83,14 @@ export async function inspectRemoteWarmSessionRecovery(input: {
checkpoint.driverKind !== previous.session.driverKind || !checkpoint.providerSessionId ||
checkpoint.activeTurnId !== null || checkpoint.pendingRuntimeRequests?.length !== 0) return deny();
const control = input.readControlState();
const commands = control.commands;
const events = control.committedEvents;
const terminal = Array.isArray(events) ? events.at(-1) : undefined;
if (!Array.isArray(commands) || commands.some(command => command.status !== "completed") ||
terminal?.eventType !== "run.terminal" || terminal?.envelope?.runId !== identity.runId ||
terminal?.envelope?.normalizedSessionId !== identity.normalizedSessionId ||
Object.entries(identity).some(([key, value]) => (control.identity as Record<string, unknown>)?.[key] !== value)) return deny();
if (!isSettledRemoteWarmControlState(control, identity)) return deny();
const assertPriorStopped = async () => {
const stopped = await evaluateNativeControllerTakeover({ owner: prior.owner, now: new Date(), requireStoppedController: true });
const persistedRecovery = readNativeControllerRecovery(prior.run.runnerProfileJson?.nativeControllerRecovery,
prior.run.runnerProfileJson?.sessionCheckpoint);
// A PID in another webserver container cannot prove life or death. New
// durable controllers are fenced by the database lease and current turn.
const stopped = await evaluateNativeControllerTakeover({ owner: prior.owner, now: new Date(), requireStoppedController: true,
remoteLeaseFenced: persistedRecovery !== null });
if (!stopped.allowed) return deny();
};
await assertPriorStopped();
@@ -41,6 +41,7 @@ function words(text: string): string[] {
}
function dedicatedTools(method: string, path: string): string[] {
if (method === "POST" && path === "/api/agents/me/chat/messages") return ["send_chat_message"];
if (/\/companies\/\{[^}]+\}\/workflows$/.test(path)) return method === "GET" ? ["list_workflows"] : method === "POST" ? ["create_workflow"] : [];
if (/\/workflows\/\{[^}]+\}$/.test(path)) return method === "GET" ? ["read_workflow"] : method === "PATCH" ? ["update_workflow"] : [];
if (/\/issues\/\{[^}]+\}\/title$/.test(path) && method === "PUT") return ["set_task_title"];
@@ -10,6 +10,7 @@ import { getConversationOwnershipBlocker } from "./conversation-continuation.js"
import { isExternalChatPresentationContext } from "./heartbeat-run-summary.js";
import { finalizeNativeRun, recoverCommittedNativeChatResponse, repairCommittedNativeChatResponse } from "./native-runtime/native-run-finalizer.js";
import { nativeResponsePresentationWork } from "./native-runtime/native-response-presentation-work.js";
import { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js";
import { resolveExternalTestDatabaseUrl } from "../__tests__/helpers/external-test-database.js";
@@ -24,10 +25,12 @@ describe.skipIf(!support.supported)("presentation and ownership PostgreSQL read
}, 60_000);
afterAll(async () => { await db?.$client.end(); await database?.cleanup(); });
async function fixture(patch: Partial<typeof heartbeatRuns.$inferInsert> = {}) {
const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID();
await db.insert(companies).values({ id: companyId, name: "Read fixture", issuePrefix: companyId.slice(0, 8) });
await db.insert(agents).values({ id: agentId, companyId, name: "Read agent", adapterType: "codex_local" });
async function fixture(patch: Partial<typeof heartbeatRuns.$inferInsert> = {}, identity?: { companyId: string; agentId: string }) {
const companyId = identity?.companyId ?? randomUUID(), agentId = identity?.agentId ?? randomUUID(), issueId = randomUUID();
if (!identity) {
await db.insert(companies).values({ id: companyId, name: "Read fixture", issuePrefix: companyId.slice(0, 8) });
await db.insert(agents).values({ id: agentId, companyId, name: "Read agent", adapterType: "codex_local" });
}
await db.insert(issues).values({ id: issueId, companyId, title: "Read task", status: "blocked", assigneeAgentId: agentId });
const payload = randomBytes(128 * 1024).toString("base64");
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, nativeIssueId: issueId,
@@ -138,10 +141,10 @@ describe.skipIf(!support.supported)("presentation and ownership PostgreSQL read
expect(await repairCommittedNativeChatResponse(db, { companyId: f.companyId, issueId: other.id, runId: f.run.id })).toBe(false);
});
async function unpublishedAcceptedReply() {
async function unpublishedAcceptedReply(id?: string, identity?: { companyId: string; agentId: string }) {
const payload = randomBytes(128 * 1024).toString("base64");
const f = await fixture({ runtimeMode: "native", status: "running", finishedAt: null,
contextSnapshot: { source: "internal", prompt: payload, paperclipWake: { prompt: payload } } });
const f = await fixture({ id, runtimeMode: "native", status: "running", finishedAt: null,
contextSnapshot: { source: "internal", prompt: payload, paperclipWake: { prompt: payload } } }, identity);
const [contract] = await db.insert(completionContracts).values({ companyId: f.companyId, issueId: f.issueId,
revision: 1, schemaVersion: "paperclip.completion-contract.v1", policyVersion: "test", risk: "standard",
completionAuthority: "server_arbiter", incompleteCriteriaPolicy: "preserve_non_terminal",
@@ -173,6 +176,40 @@ describe.skipIf(!support.supported)("presentation and ownership PostgreSQL read
return { ...f, before, reply };
}
it("saves an independent task reply while an earlier task waits for its database lock", async () => {
const suffix = randomUUID().replaceAll("-", "").slice(0, 12);
const first = await unpublishedAcceptedReply(`00000000-0000-4000-8000-${suffix}`);
const second = await unpublishedAcceptedReply(`00000000-0000-4001-8000-${suffix}`, first);
let release!: () => void;
let locked!: () => void;
const hold = new Promise<void>(resolve => { release = resolve; });
const acquired = new Promise<void>(resolve => { locked = resolve; });
const blocker = db.transaction(async tx => {
await tx.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, first.run.id)).for("update");
locked(); await hold;
});
await Promise.race([acquired, blocker]);
const onError = vi.fn();
const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError });
const worker = nativeResponsePresentationWork(db).register(coordinator, () => true);
try {
await vi.waitFor(async () => {
const rows = await db.select().from(issueComments).where(eq(issueComments.createdByRunId, second.run.id));
expect(rows).toHaveLength(1);
expect(rows[0].body).toBe(second.reply);
}, { timeout: 5000 });
expect(await db.select().from(issueComments).where(eq(issueComments.createdByRunId, first.run.id))).toHaveLength(0);
} finally {
release(); await blocker; await worker.ready; worker.stop(); await coordinator.stop();
}
for (const f of [first, second]) {
const rows = await db.select().from(issueComments).where(eq(issueComments.createdByRunId, f.run.id));
expect(rows).toHaveLength(1);
expect(rows[0].body).toBe(f.reply);
}
expect(onError).not.toHaveBeenCalled();
});
async function repairFixture(patch: Partial<typeof heartbeatRuns.$inferInsert>) {
+1 -1
View File
@@ -52,7 +52,7 @@ export function redactRegisteredSecretValues<T>(input: T, values: string[], cred
if (!record) return input;
return Object.fromEntries(
Object.entries(record)
.filter(([key]) => key !== REGISTRY_KEY)
.filter(([key]) => key !== REGISTRY_KEY && key !== "nativeControllerRecovery" && key !== "nativeTaskSessionPublication")
.map(([key, value]) => [key, key === "credentialAccess"
? (credentialConnectionName ? { connectionName: redactText(credentialConnectionName, values) } : {})
: redactRegisteredSecretValues(value, values, credentialConnectionName)]),
+26 -6
View File
@@ -12,6 +12,7 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
queue: DeliveryQueue;
batchSize?: number;
minimumPassIntervalMs?: number;
concurrentChecks?: { limit: number; key(entry: Entry): string };
reconstructionIntervalMs?: number | ((scope: Scope | undefined) => number);
beforePass?(): Promise<void>;
decodeScope(hint: string): Scope | undefined;
@@ -21,6 +22,9 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
},
canRun: () => boolean,
) {
if (options.concurrentChecks && (!Number.isInteger(options.concurrentChecks.limit) || options.concurrentChecks.limit < 1)) {
throw new RangeError("Concurrent check limit must be a positive integer");
}
const dirty = new Set<string>(["*"]);
const versions = new Map<string, number>();
const scans = new Map<string, { version: number; seen: Set<string> }>();
@@ -112,14 +116,30 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
&& Math.max(entry.dueAt, failures.get(entry.id)?.retryAt ?? 0) <= Date.now())
.sort((a, b) => a.dueAt - b.dueAt || a.id.localeCompare(b.id));
const batch = takeCompanyWorkBatch(due, options.batchSize ?? 25, lastCompany); lastCompany = batch.lastCompany;
const groups = new Map<string, Entry[]>();
for (const entry of batch.entries) {
if (signal.aborted || !canRun()) break;
try {
const current = await options.check(entry, signal);
if (current) entries.set(entry.id, current); else entries.delete(entry.id);
failures.delete(entry.id);
} catch (error) { failed(failures, entry.id); errors.push(error); }
const key = options.concurrentChecks?.key(entry) ?? "";
const group = groups.get(key) ?? [];
group.push(entry); groups.set(key, group);
}
const pending = [...groups.values()];
let nextGroup = 0;
const checkGroups = async () => {
while (!signal.aborted && canRun()) {
const group = pending[nextGroup++];
if (!group) return;
for (const entry of group) {
if (signal.aborted || !canRun()) return;
try {
const current = await options.check(entry, signal);
if (current) entries.set(entry.id, current); else entries.delete(entry.id);
failures.delete(entry.id);
} catch (error) { failed(failures, entry.id); errors.push(error); }
}
}
};
const checks = await Promise.allSettled(Array.from({ length: Math.min(options.concurrentChecks?.limit ?? 1, pending.length) }, checkGroups));
for (const result of checks) if (result.status === "rejected") errors.push(result.reason);
if (errors.length) throw new AggregateError(errors, `${options.queue} could not settle`);
},
// Retain scopes/deadlines for resume without retaining a settled write's
+5 -1
View File
@@ -753,7 +753,11 @@ export function statusCardService(
createdByAgentId: input.actor?.agentId ?? null,
createdByUserId: input.actor?.userId ?? null,
hiddenAt: now,
idempotencyKey: `status-card-update:${card.id}:${fingerprintHash}`,
// Claim ownership is revision-scoped too. A superseded refresh may cancel
// its unclaimed task; it must not share that task with a newer revision
// which has the same watched fingerprint but has not claimed the card yet.
// Concurrent refreshes of this revision still deduplicate normally.
idempotencyKey: `status-card-update:${card.id}:${revision}:${fingerprintHash}`,
onDeduplicated: (reason) => { deduplicated = reason === "idempotency_key"; },
});
const reopened = deduplicated && TERMINAL_ISSUE_STATUSES.has(created.status)
Loaded 100 of 484 files, more files were not shown because too many files have changed in this diff. Show more