Merge master and keep current attachment announcement handling

This commit is contained in:
cryppadotta committed 2026-10-10 01:29:25 +00:00
commit c41b6bcc12
1221 files changed
+190319 -83787

No files matched your search

+6 -3
View File
@@ -44,6 +44,7 @@
"typecheck": "pnpm run prepare:runner-vendor && pnpm --filter @paperclipai/plugin-sdk ensure-build-deps && tsc --noEmit"
},
"dependencies": {
"@agentclientprotocol/codex-acp": "1.6.2",
"ai": "7.0.130",
"@ai-sdk/openai": "4.0.86",
"@openrouter/ai-sdk-provider": "3.1.0",
@@ -55,7 +56,7 @@
"@chat-adapter/telegram": "4.39.0",
"@discordjs/ws": "1.2.3",
"@grpc/grpc-js": "1.14.5",
"@modelcontextprotocol/sdk": "^1.30.0",
"@modelcontextprotocol/sdk": "^1.31.0",
"@opentelemetry/api": "^1.9.0",
"@paperclipai/adapter-claude-local": "workspace:*",
"@paperclipai/adapter-codex-local": "workspace:*",
@@ -96,6 +97,7 @@
"nice-grpc": "2.1.17",
"nice-grpc-common": "2.0.4",
"open": "^11.0.4",
"opencode-ai": "1.18.34",
"pino": "^10.0.0",
"pino-http": "^11.0.0",
"pino-pretty": "^13.1.3",
@@ -103,10 +105,11 @@
"smol-toml": "^1.4.2",
"ssh2": "^1.17.0",
"svix": "1.76.1",
"ws": "^8.21.3",
"ws": "^8.22.0",
"zod": "^4.4.3"
},
"bundleDependencies": [
"@agentclientprotocol/codex-acp",
"@chat-adapter/discord",
"@chat-adapter/github",
"@chat-adapter/slack",
@@ -125,7 +128,7 @@
"@types/node": "^24.0.0",
"@types/sharp": "^0.32.0",
"@types/supertest": "^7.2.1",
"@types/ws": "^8.18.1",
"@types/ws": "^8.18.2",
"cross-env": "^10.1.0",
"supertest": "^7.0.0",
"tsx": "^4.23.15",
+97
View File
@@ -0,0 +1,97 @@
import { createDb } from "@paperclipai/db";
import { createAgentLifecycle } from "../src/services/agent-lifecycle.js";
import {
companies,
goals,
projects,
issues,
} from "@paperclipai/db";
const url = process.env.DATABASE_URL;
if (!url) throw new Error("DATABASE_URL is required");
const db = createDb(url);
console.log("Seeding database...");
const [company] = await db
.insert(companies)
.values({
name: "Paperclip Demo Co",
description: "A demo autonomous company",
status: "active",
budgetMonthlyCents: 50000,
})
.returning();
const ceo = await createAgentLifecycle(db).requestHire(company!.id, {
name: "CEO Agent",
role: "ceo",
title: "Chief Executive Officer",
status: "idle",
adapterType: "process",
adapterConfig: { command: "echo", args: ["hello from ceo"] },
budgetMonthlyCents: 15000,
});
const engineer = await createAgentLifecycle(db).requestHire(company!.id, {
name: "Engineer Agent",
role: "engineer",
title: "Software Engineer",
status: "idle",
reportsTo: ceo!.id,
adapterType: "process",
adapterConfig: { command: "echo", args: ["hello from engineer"] },
budgetMonthlyCents: 10000,
});
const [goal] = await db
.insert(goals)
.values({
companyId: company!.id,
title: "Ship V1",
description: "Deliver first control plane release",
level: "company",
status: "active",
ownerAgentId: ceo!.id,
})
.returning();
const [project] = await db
.insert(projects)
.values({
companyId: company!.id,
goalId: goal!.id,
name: "Control Plane MVP",
description: "Implement core board + agent loop",
status: "in_progress",
leadAgentId: ceo!.id,
})
.returning();
await db.insert(issues).values([
{
companyId: company!.id,
projectId: project!.id,
goalId: goal!.id,
title: "Implement atomic task checkout",
description: "Ensure in_progress claiming is conflict-safe",
status: "todo",
priority: "high",
assigneeAgentId: engineer!.id,
createdByAgentId: ceo!.id,
},
{
companyId: company!.id,
projectId: project!.id,
goalId: goal!.id,
title: "Add budget auto-pause",
description: "Pause agent at hard budget ceiling",
status: "backlog",
priority: "medium",
createdByAgentId: ceo!.id,
},
]);
console.log("Seed complete");
process.exit(0);
+16 -2
View File
@@ -1,4 +1,4 @@
/** Opt-in live qualification: creates an isolated database/workspace and spends two Codex turns.
/** Opt-in live qualification: creates an isolated database/workspace and runs a harness test and two Codex turns.
* PATH=<pnpm-bin>:<codex-bin>:$PATH node --import tsx server/scripts/smoke-native-task-monitor.ts --run
*/
import assert from "node:assert/strict";
@@ -17,6 +17,9 @@ process.env.PAPERCLIP_INSTANCE_ID = "monitor-smoke";
process.env.PAPERCLIP_TELEMETRY_ENABLED = "false";
const { createDb, companies, agents, authUsers, companyMemberships, issues, heartbeatRuns, agentWakeupRequests, nativeRunResults, statusDecisions } = await import("@paperclipai/db");
const { startEmbeddedPostgresTestDatabase } = await import("../src/__tests__/helpers/embedded-postgres.js");
const { createAgentLifecycle, startAgentLifecycle } = await import("../src/services/agent-lifecycle.js");
const { createLifecycleDriver } = await import("../src/services/agent-lifecycle-driver.js");
const { createPluginWorkerManager } = await import("../src/services/plugin-worker-manager.js");
const { heartbeatService } = await import("../src/services/heartbeat.js");
const { setupRunnerPrpWebSocketServer, runnerPrpWebSocketInternals } = await import("../src/realtime/runner-prp-ws.js");
const { closeIdleWarmNativeSessionsForRestart } = await import("../src/services/native-runtime/native-session-executor.js");
@@ -30,6 +33,7 @@ process.env.PAPERCLIP_API_URL = `http://127.0.0.1:${address.port}`;
setupRunnerPrpWebSocketServer(server, { apiUrl: process.env.PAPERCLIP_API_URL });
const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID();
const heartbeat = heartbeatService(db);
const lifecycle = startAgentLifecycle(db, createLifecycleDriver(db, createPluginWorkerManager()), () => true);
const readRuns = () => db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId)).orderBy(heartbeatRuns.createdAt);
console.log(JSON.stringify({ root, companyId, agentId, issueId }));
try {
@@ -38,9 +42,18 @@ try {
await db.insert(companies).values({ id: companyId, name: "Isolated monitor smoke", issuePrefix: "MON", defaultResponsibleUserId: "monitor-smoke", requireBoardApprovalForNewAgents: false });
await db.insert(authUsers).values({ id: "monitor-smoke", name: "Monitor smoke", email: "monitor-smoke@example.test", createdAt: new Date(), updatedAt: new Date() });
await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: "monitor-smoke", status: "active", membershipRole: "owner" });
await db.insert(agents).values({ id: agentId, companyId, name: "Monitor verifier", status: "active", adapterType: "paperclip_runner",
await createAgentLifecycle(db).requestHire(companyId, { id: agentId, name: "Monitor verifier", status: "active", adapterType: "paperclip_runner",
adapterConfig: { provider: "codex", cwd, lifecycleMode: "warm", idleTimeoutMs: 300_000, timeoutSeconds: 180 },
runtimeConfig: { heartbeat: { enabled: false, wakeOnDemand: true } } });
const setupDeadline = Date.now() + 3 * 60_000;
while (true) {
const agent = await createAgentLifecycle(db).get(agentId);
assert.ok(agent, "the hired agent must exist");
assert.equal(agent.lifecycleError, null, "agent setup failed");
if (agent.lifecycleState === "ready") break;
assert.ok(Date.now() < setupDeadline, "agent setup timed out");
await delay(250);
}
await db.insert(issues).values({ id: issueId, companyId, title: "Verify a one-shot native task monitor", identifier: "MON-1", status: "in_progress", assigneeAgentId: agentId,
description: "This is an isolated two-run integration test. First run: use set_task_monitor on this task with a fresh nextCheckAt about 45 seconds in the future (compute the current UTC time if needed), notes saying to verify issue_monitor_due, and idempotencyKey monitor-live-first. Confirm the receipt. Then call paperclip_finish with yielded and continuation.kind monitor, accurately retaining the second run as blocking remaining work. End immediately after acceptance. Do not sleep or poll. Second run: if the wake reason is issue_monitor_due, the requested check succeeded. Report that reason and finish done using the current completion contract. Do not schedule another monitor or request human review. No files or other deliverables are required." });
await heartbeat.wakeup(agentId, { source: "on_demand", triggerDetail: "manual", reason: "monitor_smoke",
@@ -90,6 +103,7 @@ try {
await writeFile(join(root, "evidence.json"), JSON.stringify(evidence, null, 2));
console.log(JSON.stringify({ passed: true, evidence: join(root, "evidence.json"), warmReuse: evidence.warmReuse }));
} finally {
await lifecycle.stop();
await writeFile(join(root, "run-records.json"), JSON.stringify(await readRuns(), null, 2));
await closeIdleWarmNativeSessionsForRestart();
runnerPrpWebSocketInternals.resetForTests();
@@ -2,6 +2,7 @@
* Uses real Codex, disposable database/workspaces, and a private copy of login credentials.
* Writes only content-free evidence to stdout; never runs as part of the unit suite.
*/
import { createAgentLifecycle } from "../src/services/agent-lifecycle.js";
import assert from "node:assert/strict";
import { randomUUID } from "node:crypto";
import { chmod, copyFile, lstat, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
@@ -63,7 +64,7 @@ try {
let result;
if (mode === "legacy") {
await server.db.update(heartbeatRuns).set({ runtimeMode: "legacy", nativeIssueId: null }).where(eq(heartbeatRuns.id, f.runId));
await server.db.update(agents).set({ adapterType: "codex_local" }).where(eq(agents.id, f.agentId));
await createAgentLifecycle(server.db).updateConfiguration(f.agentId, { adapterType: "codex_local" });
result = await executeLegacy({
runId: f.runId,
agent: { id: f.agentId, companyId: f.companyId, name: "Feedback smoke", adapterType: "codex_local", adapterConfig: {} },
@@ -0,0 +1,64 @@
import { randomUUID } from "node:crypto";
import { eq } from "drizzle-orm";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { agents, companies, connectionGrants, connectionGrantMembers, createDb, heartbeatRuns, issues, toolApplications, toolConnections } from "@paperclipai/db";
import { activityService } from "../services/activity.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
const support = await getEmbeddedPostgresTestSupport();
const suite = support.supported ? describe : describe.skip;
suite("task credential-access run summary", () => {
let db: ReturnType<typeof createDb>;
let temporaryDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
beforeAll(async () => {
temporaryDb = await startEmbeddedPostgresTestDatabase("credential-access-summary-");
db = createDb(temporaryDb.connectionString);
}, 60_000);
afterAll(async () => { await temporaryDb?.cleanup(); });
it("projects only bounded credential-access display metadata in task runs", async () => {
const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID();
await db.insert(companies).values({ id: companyId, name: "Credential fixture", issuePrefix: `C${companyId.slice(0, 6)}` });
await db.insert(agents).values({ id: agentId, companyId, name: "Codie", role: "engineer", status: "idle", adapterType: "codex_local" });
await db.insert(issues).values({ id: issueId, companyId, title: "Credential denial", status: "blocked", assigneeAgentId: agentId });
const applicationId = randomUUID(), connectionId = randomUUID(), grantId = randomUUID();
await db.insert(toolApplications).values({ id: applicationId, companyId, name: "AI fixture", type: "rest_api" });
await db.insert(toolConnections).values({ id: connectionId, companyId, applicationId, name: "Dotta’s API Key", uid: "credential-fixture", connectionPurpose: "ai", transport: "runtime_auth" });
await db.insert(connectionGrants).values({ id: grantId, companyId, connectionId, kind: "user", subjectUserId: "dotta" });
const runId = randomUUID();
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, scopeKind: "issue", issueId,
responsibleUserId: "nicky",
status: "failed", errorCode: "configuration_incomplete", error: "This credential is not shared with the responsible user",
resultJson: { configurationIncomplete: { selectionFailure: "ai_connection_credential_not_shared",
credentialAccess: { connectionName: "Dotta’s API Key".repeat(30), grantId, secretValue: "must-not-leak" },
fingerprint: "must-not-leak", unrelated: "must-not-leak" } },
});
const service = activityService(db);
const runs = await service.runsForIssue(companyId, issueId, "nicky");
expect(runs[0]?.error).toBe("This credential is not shared with the responsible user");
expect(runs[0]?.resultJson).toEqual({ configurationIncomplete: { selectionFailure: "ai_connection_credential_not_shared",
credentialAccess: { connectionName: "Dotta’s API Key".repeat(30).slice(0, 240) } } });
expect((await service.runsForIssue(companyId, issueId, "dotta"))[0]?.resultJson).toEqual(runs[0]?.resultJson);
const hidden = { configurationIncomplete: { selectionFailure: "ai_connection_credential_not_shared", credentialAccess: {} } };
expect((await service.runsForIssue(companyId, issueId, "another-reader"))[0]?.resultJson).toEqual(hidden);
expect((await service.runsForIssue(companyId, issueId))[0]?.resultJson).toEqual(hidden);
// Organization credentials follow the same audience rules as Connections.
await db.update(connectionGrants).set({ kind: "organization", subjectUserId: null }).where(eq(connectionGrants.id, grantId));
expect((await service.runsForIssue(companyId, issueId, "another-reader"))[0]?.resultJson).toEqual(runs[0]?.resultJson);
await db.insert(connectionGrantMembers).values({ companyId, grantId, subjectType: "user", subjectId: "dotta" });
expect((await service.runsForIssue(companyId, issueId, "dotta"))[0]?.resultJson).toEqual(runs[0]?.resultJson);
expect((await service.runsForIssue(companyId, issueId, "another-reader"))[0]?.resultJson).toEqual(hidden);
// A resumed native execution keeps its diagnostic name without acquiring
// the fresh-execution selection-failure marker used by recovery.
await db.update(heartbeatRuns).set({ resultJson: { configurationIncomplete: {
credentialAccess: { connectionName: "Dotta’s API Key", grantId },
} } }).where(eq(heartbeatRuns.id, runId));
expect((await service.runsForIssue(companyId, issueId, "nicky"))[0]?.resultJson).toEqual({
configurationIncomplete: { credentialAccess: { connectionName: "Dotta’s API Key" } },
});
await db.update(heartbeatRuns).set({ error: null, resultJson: { configurationIncomplete: { selectionFailure: "ai_connection_unavailable", credentialAccess: { connectionName: "hidden" } } } }).where(eq(heartbeatRuns.id, runId));
expect((await service.runsForIssue(companyId, issueId, "nicky"))[0]?.resultJson).toEqual({});
});
});
+1 -1
View File
@@ -268,7 +268,7 @@ describe("activity routes", () => {
expect(res.status).toBe(200);
expect(mockIssueService.getByIdentifier).toHaveBeenCalledWith("PC1A2-475");
expect(mockIssueService.getById).not.toHaveBeenCalled();
expect(mockActivityService.runsForIssue).toHaveBeenCalledWith("company-1", "issue-uuid-1");
expect(mockActivityService.runsForIssue).toHaveBeenCalledWith("company-1", "issue-uuid-1", "user-1");
expect(res.body).toEqual([{ runId: "run-1", adapterType: "codex_local" }]);
});
@@ -254,6 +254,7 @@ describeEmbeddedPostgres("activity service", () => {
},
resultJson: {
conversationReset: true,
configurationIncomplete: { reason: "workspace_base_ref_unresolved", requestedRef: "main", fetchError: "private diagnostic" },
billing_type: "metered",
total_cost_usd: 0.42,
stopReason: "timeout",
@@ -295,6 +296,7 @@ describeEmbeddedPostgres("activity service", () => {
});
expect(runs[0]?.resultJson).toEqual({
conversationReset: true,
configurationIncomplete: { reason: "workspace_base_ref_unresolved" },
billingType: "metered",
billing_type: "metered",
costUsd: 0.42,
@@ -413,3 +413,9 @@ describe("adapter auth-signal route", () => {
expect(mockResolveEnvironmentExecutionTarget).not.toHaveBeenCalled();
});
});
vi.mock("../services/agent-lifecycle.js", async (importOriginal) => {
const actual = await importOriginal<typeof import("../services/agent-lifecycle.js")>();
return { ...actual, createAgentLifecycle: () => ({
}) };
});
+9 -20
View File
@@ -1,4 +1,4 @@
import { probeAcpxClaudeInstallation, probeAcpxCursorInstallation } from "@paperclipai/paperclip-runner/live";
import { probeAcpxClaudeInstallation, probeAcpxPiInstallation } from "../vendor/paperclip-runner/live/index.js";
import { describe, expect, it, beforeEach, afterEach, vi } from "vitest";
import { buildSandboxNpmInstallCommand } from "@paperclipai/adapter-utils";
import type { ServerAdapterModule } from "../adapters/index.js";
@@ -17,9 +17,10 @@ import {
setOverridePaused,
} from "../adapters/registry.js";
vi.mock("@paperclipai/paperclip-runner/live", () => ({
vi.mock("../vendor/paperclip-runner/live/index.js", () => ({
probeAcpxClaudeInstallation: vi.fn(async () => undefined),
probeAcpxGrokInstallation: vi.fn(async () => undefined),
probeAcpxPiInstallation: vi.fn(async () => undefined),
probeAcpxCursorInstallation: vi.fn(async () => undefined),
}));
@@ -327,23 +328,10 @@ describe("server adapter registry", () => {
});
});
it.each([true, false])("checks ordinary Cursor runtime readiness (%s)", async (ready) => {
const probe = vi.mocked(probeAcpxCursorInstallation);
it.each([true, false])("probes the exact Pi installation without qualification opt-in (ready=%s)", async (ready) => {
const probe = vi.mocked(probeAcpxPiInstallation);
if (ready) probe.mockResolvedValueOnce(undefined);
else probe.mockRejectedValueOnce(new Error("Run paperclipai runtime setup cursor"));
const model = "gpt-5.6-luna[context=272k,reasoning=medium,fast=false]";
const result = await requireServerAdapter("paperclip_runner").testEnvironment({
companyId: "company-1", adapterType: "paperclip_runner",
config: { provider: "acpx", acpxAgent: "cursor", model },
});
expect(probe).toHaveBeenLastCalledWith(model);
expect(result).toMatchObject({
status: ready ? "pass" : "fail",
checks: [expect.objectContaining({ code: ready ? "acpx_runtime_ready" : "acpx_runtime_unavailable" })],
});
});
it("keeps the ACPX Pi profile unavailable", async () => {
else probe.mockRejectedValueOnce(new Error("Pi closure unavailable"));
const result = await requireServerAdapter("paperclip_runner").testEnvironment({
companyId: "company-1",
adapterType: "paperclip_runner",
@@ -355,9 +343,10 @@ describe("server adapter registry", () => {
});
expect(result).toMatchObject({
status: "fail",
checks: [{ code: "paperclip_runner_acpx_agent_unavailable" }],
status: ready ? "pass" : "fail",
checks: [{ code: ready ? "acpx_runtime_ready" : "acpx_runtime_unavailable" }],
});
expect(probe).toHaveBeenLastCalledWith("openrouter/deepseek/deepseek-v4-flash-0731");
});
it("reports qualification-only readiness for an exact host-authorized candidate", async () => {
const key = "PAPERCLIP_RUNNER_ACPX_QUALIFICATION";
@@ -1199,3 +1199,10 @@ describe("agent routes adapter validation", () => {
expect(mockAgentService.update).toHaveBeenCalledOnce();
});
});
vi.mock("../services/agent-lifecycle.js", async (importOriginal) => {
const actual = await importOriginal<typeof import("../services/agent-lifecycle.js")>();
return { ...actual, createAgentLifecycle: () => ({
requestHire: (...args: unknown[]) => mockAgentService.create(...args),
}) };
});
@@ -212,7 +212,10 @@ describe("internal agent commentary through both transports", () => {
await server.db.insert(heartbeatRuns).values({ id: runId, companyId: f.companyId, agentId, status: "running", invocationSource: "on_demand" });
await submitAgentCommentary(server.db, { companyId: f.companyId, agentId, runId }, { ...input, kind: "complaint" });
expect((await rows({ ...f, runId }))[0].issueId).toBeNull();
if (owner === "agent") await agentService(server.db).remove(agentId);
if (owner === "agent") {
await server.db.update(agents).set({ lifecycleState: "terminated", status: "terminated" }).where(eq(agents.id, agentId));
await agentService(server.db).remove(agentId);
}
else await companyService(server.db).remove(f.companyId);
expect(await rows({ ...f, runId })).toHaveLength(0);
});
@@ -48,6 +48,7 @@ let currentAccessCanUser = false;
const mockAgentService = vi.hoisted(() => ({
getById: vi.fn(),
retryLifecycle: vi.fn(),
pause: vi.fn(),
resume: vi.fn(),
clearError: vi.fn(),
@@ -302,6 +303,7 @@ function resetMockDefaults() {
currentKeyAgentId = agentId;
currentAccessCanUser = false;
mockAgentService.getById.mockImplementation(async () => ({ ...baseAgent }));
mockAgentService.retryLifecycle.mockImplementation(async () => ({ ...baseAgent, lifecycleState: "preparing", lifecycleError: null }));
mockAgentService.pause.mockImplementation(async () => ({ ...baseAgent }));
mockAgentService.resume.mockImplementation(async () => ({ ...baseAgent }));
mockAgentService.clearError.mockImplementation(async () => ({ ...baseAgent, status: "idle" }));
@@ -755,3 +757,43 @@ describe("agent cross-tenant route authorization", () => {
expect(mockLogActivity).not.toHaveBeenCalled();
});
});
vi.mock("../services/agent-lifecycle.js", async (importOriginal) => {
const actual = await importOriginal<typeof import("../services/agent-lifecycle.js")>();
return { ...actual, createAgentLifecycle: () => ({
requestHire: (...args: unknown[]) => mockAgentService.create(...args),
retry: (...args: unknown[]) => mockAgentService.retryLifecycle(...args),
pauseAgent: (...args: unknown[]) => mockAgentService.pause(...args),
resumeAgent: (...args: unknown[]) => mockAgentService.resume(...args),
terminateAgent: (...args: unknown[]) => mockAgentService.terminate(...args),
}) };
});
describe("agent lifecycle retry authorization", () => {
beforeEach(resetMockDefaults);
it("retries and audits a board-authorized agent in its company", async () => {
const app = await createApp({ type: "board", userId: "board", source: "local_implicit" });
const res = await requestApp(app, base => request(base).post(`/api/agents/${agentId}/lifecycle/retry`).send({}));
expect(res.status).toBe(200);
expect(res.body).toMatchObject({ id: agentId, lifecycleState: "preparing", lifecycleError: null });
expect(mockAgentService.retryLifecycle).toHaveBeenCalledWith(agentId);
expect(mockLogActivity).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({
companyId, entityId: agentId, action: "agent.lifecycle_retried", actorType: "user", actorId: "board",
}));
});
it.each([
[{ type: "none" }, 403],
[{ type: "agent", agentId, companyId }, 403],
[{ type: "board", userId: "outsider", source: "session", companyIds: ["other-company"] }, 404],
[{ type: "board", userId: "reader", source: "session", companyIds: [companyId] }, 403],
])("refuses retry for %j", async (actor, status) => {
const app = await createApp(actor);
const res = await requestApp(app, base => request(base).post(`/api/agents/${agentId}/lifecycle/retry`).send({}));
expect(res.status).toBe(status);
expect(mockAgentService.retryLifecycle).not.toHaveBeenCalled();
expect(mockLogActivity).not.toHaveBeenCalled();
});
});
@@ -1,3 +1,4 @@
import { createAgentLifecycle } from "../services/agent-lifecycle.js";
import { randomUUID } from "node:crypto";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { eq } from "drizzle-orm";
@@ -50,7 +51,7 @@ describeDatabase("new agent configuration defaults", () => {
issuePrefix: `PD${companyId.slice(0, 6).toUpperCase()}`,
});
const create = (name: string, permissions: Record<string, unknown> = {}, metadata?: Record<string, unknown>) =>
agentService(db).create(companyId, {
createAgentLifecycle(db).requestHire(companyId, {
name,
role: "engineer",
adapterType: "process",
@@ -130,6 +131,7 @@ describeDatabase("new agent configuration defaults", () => {
expect(await db.select().from(principalPermissionGrants)
.where(eq(principalPermissionGrants.principalId, standard.id))).toEqual([]);
await db.update(agents).set({ status: "terminated", lifecycleState: "terminated" }).where(eq(agents.id, peer.id));
await agentService(db).remove(peer.id);
expect(await db.select().from(principalPermissionGrants)
.where(eq(principalPermissionGrants.principalId, peer.id))).toEqual([]);
@@ -190,7 +192,7 @@ describeDatabase("new agent configuration defaults", () => {
name: "Pending permission default",
issuePrefix: `PP${companyId.slice(0, 6).toUpperCase()}`,
});
const pending = await agentService(db).create(companyId, {
const pending = await createAgentLifecycle(db).requestHire(companyId, {
name: "Pending hire",
status: "pending_approval",
role: "engineer",
@@ -201,7 +203,7 @@ describeDatabase("new agent configuration defaults", () => {
expect(await db.select().from(principalPermissionGrants)
.where(eq(principalPermissionGrants.companyId, companyId))).toEqual([]);
const result = await agentService(db).activatePendingApproval(pending.id);
const result = await createAgentLifecycle(db).approveHire(pending.id);
expect(result?.activated).toBe(true);
expect((await db.select().from(principalPermissionGrants)
.where(eq(principalPermissionGrants.companyId, companyId)))
@@ -220,7 +222,7 @@ describeDatabase("new agent configuration defaults", () => {
name: "Invited permission default",
issuePrefix: `PI${companyId.slice(0, 6).toUpperCase()}`,
});
const invited = await agentService(db).create(companyId, {
const invited = await createAgentLifecycle(db).requestHire(companyId, {
name: "Invited agent",
role: "engineer",
adapterType: "process",
@@ -1709,3 +1709,10 @@ describe("adapter device-login routes", () => {
expect(loggedText).not.toContain("provider-lease-");
});
});
vi.mock("../services/agent-lifecycle.js", async (importOriginal) => {
const actual = await importOriginal<typeof import("../services/agent-lifecycle.js")>();
return { ...actual, createAgentLifecycle: () => ({
requestHire: (...args: unknown[]) => mockAgentService.create(...args),
}) };
});
@@ -0,0 +1,221 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import fs from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { createRequire } from "node:module";
import childProcess, { execFileSync } from "node:child_process";
import { prepareAdapterExecutionTargetRuntime } from "@paperclipai/adapter-utils/execution-target";
import { captureDirectorySnapshot } from "@paperclipai/adapter-utils/workspace-restore-merge";
import { prepareSandboxManagedRuntime } from "@paperclipai/adapter-utils/sandbox-managed-runtime";
import { agentDirectoryBaselineDigest, agentDirectoryProbeProgram, observeLocalAgentDirectory, probeAgentDirectory, retireAgentDirectoryTransferScratch, agentDirectoryTransferCleanupProgram } from "../services/agent-directory-probe.js";
describe("stable live agent directory observation", () => {
let root: string;
beforeEach(() => {
root = fs.realpathSync(fs.mkdtempSync(join(tmpdir(), "agent-directory-probe-")));
fs.mkdirSync(join(root, "notes"));
fs.writeFileSync(join(root, "AGENTS.md"), "Instructions\n");
fs.writeFileSync(join(root, "notes", "memory.bin"), Buffer.from([0, 1, 255]));
});
afterEach(() => { vi.restoreAllMocks(); fs.rmSync(root, { recursive: true, force: true }); });
function productionPrograms(): { probe: string; cleanup: string } {
// Product/dev starts the server with tsx. Vitest's own transform does not
// reproduce the lexical helpers injected into function.toString() there.
const loader = createRequire(import.meta.url).resolve("tsx");
const source = new URL("../services/agent-directory-probe.ts", import.meta.url).href;
const program = `import { agentDirectoryProbeProgram, agentDirectoryTransferCleanupProgram } from ${JSON.stringify(source)}; process.stdout.write(JSON.stringify({probe:agentDirectoryProbeProgram,cleanup:agentDirectoryTransferCleanupProgram}));`;
return JSON.parse(execFileSync(process.execPath, ["--import", loader, "--input-type=module", "--eval", program], {
cwd: root, env: { TSX_DISABLE_CACHE: "1", HOME: root, TMPDIR: root }, encoding: "utf8", timeout: 10_000, maxBuffer: 64 * 1024,
}));
}
it("runs a self-contained live probe after the production tsx transform", () => {
const { probe } = productionPrograms();
const observe = () => JSON.parse(execFileSync(process.execPath, ["-e", probe, root], {
env: {}, encoding: "utf8", timeout: 10_000, maxBuffer: 64 * 1024, stdio: "pipe",
}));
const before = probeAgentDirectory(root);
expect(observe()).toEqual(before);
fs.writeFileSync(join(root, "AGENTS.md"), "Changed instructions\n");
expect(observe()).toEqual(probeAgentDirectory(root));
expect(observe().digest).not.toBe(before.digest);
fs.symlinkSync(join(root, "AGENTS.md"), join(root, "unsafe"));
expect(observe).toThrow();
});
it("runs bounded transfer cleanup after the production tsx transform", () => {
const { cleanup } = productionPrograms();
const scratch = join(root, ".paperclip-runtime", "agent-files");
const retire = () => execFileSync(process.execPath, ["-e", cleanup, root], {
env: {}, encoding: "utf8", timeout: 10_000, maxBuffer: 64 * 1024, stdio: "pipe",
});
fs.mkdirSync(scratch, { recursive: true });
fs.writeFileSync(join(scratch, "unexpected"), "retain");
expect(retire).toThrow();
expect(fs.readFileSync(join(scratch, "unexpected"), "utf8")).toBe("retain");
fs.unlinkSync(join(scratch, "unexpected"));
retire();
expect(fs.existsSync(join(root, ".paperclip-runtime"))).toBe(false);
expect(fs.readFileSync(join(root, "AGENTS.md"), "utf8")).toBe("Instructions\n");
expect(fs.readFileSync(join(root, "notes", "memory.bin"))).toEqual(Buffer.from([0, 1, 255]));
});
it("matches the complete materialized baseline and the actual remote Node program", async () => {
const baseline = await captureDirectorySnapshot(root);
const observed = probeAgentDirectory(root);
expect(observed.digest).toBe(agentDirectoryBaselineDigest(baseline));
const remote = JSON.parse(execFileSync(process.execPath, ["-e", agentDirectoryProbeProgram, root], { encoding: "utf8", env: {}, timeout: 10_000 }));
expect(remote).toEqual(observed);
});
it.runIf(process.platform === "darwin")("accepts only the observing Darwin host's fixed system aliases", () => {
const alias = root.replace(/^\/private(?=\/(tmp|var|etc)\/)/, "");
expect(alias).not.toBe(root);
const observed = probeAgentDirectory(root);
expect(probeAgentDirectory(alias)).toEqual(observed);
const remote = JSON.parse(execFileSync(process.execPath, ["-e", agentDirectoryProbeProgram, alias], { encoding: "utf8", env: {}, timeout: 10_000 }));
expect(remote).toEqual(observed);
// A Linux remote cannot inherit the controller's Darwin exception.
expect(() => execFileSync(process.execPath, ["-e", agentDirectoryProbeProgram.replace("platform:process.platform", "platform:'linux'"), alias],
{ encoding: "utf8", env: {}, timeout: 10_000, stdio: "pipe" })).toThrow();
fs.symlinkSync(root, join(root, "redirect"));
expect(() => probeAgentDirectory(`${alias}/redirect`)).toThrow();
});
it("observes the complete agent directory after the actual remote transfer", async () => {
const remote = `${root}-remote`;
fs.mkdirSync(remote);
const target = { kind: "remote" as const, transport: "sandbox" as const, remoteCwd: remote,
runner: { execute: async (input: Parameters<import("@paperclipai/adapter-utils/command-managed-runtime").CommandManagedRuntimeRunner["execute"]>[0]) => ({
stdout: execFileSync(input.command, input.args ?? [], { cwd: input.cwd, input: input.stdin, encoding: "utf8",
env: { PATH: process.env.PATH, ...input.env }, timeout: input.timeoutMs, maxBuffer: 32 * 1024 * 1024 }),
stderr: "", exitCode: 0, signal: null, timedOut: false, pid: null, startedAt: new Date().toISOString(),
}) } };
let runtime: Awaited<ReturnType<typeof prepareAdapterExecutionTargetRuntime>> | undefined;
try {
const baseline = await captureDirectorySnapshot(root);
runtime = await prepareAdapterExecutionTargetRuntime({ target, runId: "probe-transfer", adapterKey: "agent-files",
workspaceLocalDir: root, workspaceRemoteDir: remote, syncWorkspace: true, workspaceBaseline: baseline,
workspaceGitSnapshot: null, workspaceFileMode: "all", workspaceExclude: [".paperclip-runtime", ".paperclip-runtime/**"] });
expect(fs.readdirSync(`${remote}/.paperclip-runtime`)).toEqual(["agent-files"]);
expect(fs.readdirSync(`${remote}/.paperclip-runtime/agent-files`)).toEqual([]);
execFileSync(process.execPath, ["-e", agentDirectoryTransferCleanupProgram, remote], { env: {}, timeout: 10_000 });
expect(() => probeAgentDirectory(remote)).not.toThrow();
expect(probeAgentDirectory(remote).digest).toBe(agentDirectoryBaselineDigest(baseline));
} finally { await runtime?.cleanupWorkspaceSnapshot?.(); fs.rmSync(remote, { recursive: true, force: true }); }
});
it("keeps restart fallback transfer scratch outside the native memory directory", async () => {
const remoteCwd = `${root}-remote`;
const agentHome = join(remoteCwd, ".paperclip-runtime", "agent-files", "agent", "original-run");
const scratch = join(remoteCwd, ".paperclip-runtime", "paperclip-runner", "agent-file-transfers", "agent", "original-run");
fs.mkdirSync(agentHome, { recursive: true });
fs.cpSync(root, agentHome, { recursive: true });
const target = { kind: "remote" as const, transport: "sandbox" as const, remoteCwd,
// A newly bound controller target may have no native sync hooks. Exercise
// the production command/base64 fallback rather than a transport mock.
runner: { execute: async (input: Parameters<import("@paperclipai/adapter-utils/command-managed-runtime").CommandManagedRuntimeRunner["execute"]>[0]) => ({
stdout: execFileSync(input.command, input.args ?? [], { cwd: input.cwd, input: input.stdin, encoding: "utf8",
env: { PATH: process.env.PATH, ...input.env }, timeout: input.timeoutMs, maxBuffer: 32 * 1024 * 1024 }),
stderr: "", exitCode: 0, signal: null, timedOut: false, pid: null, startedAt: new Date().toISOString(),
}) } };
let runtime: Awaited<ReturnType<typeof prepareAdapterExecutionTargetRuntime>> | undefined;
try {
const baseline = await captureDirectorySnapshot(root);
runtime = await prepareAdapterExecutionTargetRuntime({ target, runId: "restart-transfer", adapterKey: "agent-files",
workspaceLocalDir: root, workspaceRemoteDir: agentHome, runtimeRootDir: scratch,
syncWorkspace: true, workspaceInboundMode: "adopt_remote", workspaceBaseline: baseline,
workspaceGitSnapshot: null, workspaceFileMode: "all", workspaceExclude: [".paperclip-runtime", ".paperclip-runtime/**"] });
const memory = "0123456789abcdef0123456789abcdef\n";
fs.writeFileSync(join(agentHome, "notes", "memory.txt"), memory);
await runtime.restoreWorkspace();
expect(fs.readFileSync(join(root, "notes", "memory.txt"), "utf8")).toBe(memory);
// A transfer must not introduce a reserved path that the product's own
// full agent-directory probe rejects, including after controller recovery.
expect(() => probeAgentDirectory(agentHome)).not.toThrow();
expect(fs.existsSync(join(agentHome, ".paperclip-runtime"))).toBe(false);
expect(runtime.runtimeRootDir).toBe(scratch);
expect(fs.readdirSync(scratch)).toEqual([]);
} finally { await runtime?.cleanupWorkspaceSnapshot?.(); fs.rmSync(remoteCwd, { recursive: true, force: true }); }
});
it.each(["outside", "reserved-root", "relative", "traversal", "trailing-slash", "nul"])("rejects %s transfer scratch before filesystem work", async kind => {
const reserved = join(root, ".paperclip-runtime");
const invalid = {
outside: `${root}-foreign/scratch`, "reserved-root": reserved, relative: ".paperclip-runtime/scratch",
traversal: `${reserved}/../../scratch`, "trailing-slash": `${reserved}/scratch/`, nul: `${reserved}/scratch\0`,
}[kind]!;
await expect(prepareSandboxManagedRuntime({
spec: { transport: "sandbox", provider: "fixture", sandboxId: "owned", remoteCwd: root, apiKey: null },
adapterKey: "agent-files", client: {} as never, workspaceLocalDir: root, runtimeRootDir: invalid,
})).rejects.toThrow("Transfer scratch must remain within the lease runtime tree");
expect(fs.existsSync(reserved)).toBe(false);
});
it.each(["file", "extra-directory", "symlink", "race"])("fails closed on %s in transfer-owned scratch without deleting contents", kind => {
const parent = join(root, ".paperclip-runtime"), scratch = join(parent, "agent-files");
fs.mkdirSync(scratch, { recursive: true });
const retained = join(parent, "retained.txt");
if (kind === "file") fs.writeFileSync(join(scratch, "retained.txt"), "keep");
if (kind === "extra-directory") fs.mkdirSync(join(parent, "unknown"));
if (kind === "symlink") { fs.rmdirSync(scratch); fs.symlinkSync(join(root, "notes"), scratch); }
if (kind === "race") {
const rmdir = fs.rmdirSync;
vi.spyOn(fs, "rmdirSync").mockImplementation(((directory: fs.PathLike) => {
if (directory === scratch) fs.writeFileSync(retained, "concurrent bytes");
return rmdir(directory);
}) as typeof rmdir);
}
expect(() => retireAgentDirectoryTransferScratch(root)).toThrow();
expect(fs.existsSync(parent)).toBe(true);
if (kind === "file") expect(fs.readFileSync(join(scratch, "retained.txt"), "utf8")).toBe("keep");
if (kind === "race") expect(fs.readFileSync(retained, "utf8")).toBe("concurrent bytes");
expect(fs.existsSync(join(root, "notes", "memory.bin"))).toBe(true);
});
it("runs the local observation in its bounded credential-free child", async () => {
expect(await observeLocalAgentDirectory(root)).toEqual(probeAgentDirectory(root));
});
it.each(["timeout", "oversized", "malformed"])("rejects %s local-child observations", async kind => {
const stub = vi.spyOn(childProcess, "execFile").mockImplementation(((file: string, args: string[], options: object, callback: (error: Error | null, stdout: string, stderr: string) => void) => {
expect(file).toBe(process.execPath);
expect(args).toEqual(["-e", agentDirectoryProbeProgram, root]);
expect(options).toMatchObject({ cwd: root, env: {}, timeout: 10_000, maxBuffer: 1024, killSignal: "SIGKILL" });
callback(kind === "timeout" ? Object.assign(new Error("timed out"), { killed: true, signal: "SIGKILL" }) : null,
kind === "oversized" ? "x".repeat(1025) : "{}", "");
return {} as childProcess.ChildProcess;
}) as typeof childProcess.execFile);
await expect(observeLocalAgentDirectory(root)).rejects.toThrow();
expect(stub).toHaveBeenCalledOnce();
});
it.each(["edit", "add", "remove", "mode"])("detects a legitimate %s", change => {
const before = probeAgentDirectory(root);
if (change === "edit") fs.writeFileSync(join(root, "AGENTS.md"), "New instructions\n");
if (change === "add") fs.writeFileSync(join(root, "new.txt"), "New");
if (change === "remove") fs.unlinkSync(join(root, "notes", "memory.bin"));
if (change === "mode") fs.chmodSync(join(root, "AGENTS.md"), 0o700);
expect(probeAgentDirectory(root).digest).not.toBe(before.digest);
});
it.each(["symlink", "hardlink", "reserved"])("rejects %s entries", kind => {
if (kind === "symlink") fs.symlinkSync(join(root, "AGENTS.md"), join(root, "unsafe"));
if (kind === "hardlink") fs.linkSync(join(root, "AGENTS.md"), join(root, "unsafe"));
if (kind === "reserved") fs.mkdirSync(join(root, ".paperclip-runtime"));
expect(() => probeAgentDirectory(root)).toThrow();
});
it("detects file mutation during a bounded read", () => {
const read = fs.readSync;
let changed = false;
vi.spyOn(fs, "readSync").mockImplementation(((...args: Parameters<typeof read>) => {
const result = Reflect.apply(read, fs, args);
if (!changed) { changed = true; fs.appendFileSync(join(root, "AGENTS.md"), "Concurrent change"); }
return result;
}) as typeof read);
expect(() => probeAgentDirectory(root)).toThrow(/changed|grew/);
});
it("fails on unreadable observations instead of reporting unchanged", () => {
vi.spyOn(fs, "openSync").mockImplementation(() => { throw Object.assign(new Error("denied"), { code: "EACCES" }); });
expect(() => probeAgentDirectory(root)).toThrow("denied");
});
it("does not let matching replacement bytes preserve the pinned root identity", () => {
const before = probeAgentDirectory(root);
const retired = `${root}-retired`;
fs.renameSync(root, retired);
try { fs.cpSync(retired, root, { recursive: true }); expect(probeAgentDirectory(root).identity).not.toBe(before.identity); }
finally { fs.rmSync(retired, { recursive: true, force: true }); }
});
it("rejects excluded/incomplete baselines", async () => {
const baseline = await captureDirectorySnapshot(root);
expect(() => agentDirectoryBaselineDigest({ ...baseline, exclude: ["notes"] })).toThrow("Incomplete");
});
});
@@ -74,6 +74,183 @@ describe("persistent agent directories", () => {
await fs.writeFile(path.join(root, entryFile), initial);
});
describe("warm directory ownership", () => {
async function nextRun() {
const runId = randomUUID();
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "on_demand", responsibleUserId: userId });
return runId;
}
it("adopts one physical root and fences stale cleanup through final save", async () => {
const first = await run();
expect(await copies.hasChanges({ companyId, runId: first.runId })).toBe(false);
const secondId = await nextRun();
const second = await copies.adopt({ companyId, agentId, runId: secondId, previousRunId: first.runId, cwd: home });
expect(second).toMatchObject({ runId: secondId, localRoot: first.localRoot, executionRoot: first.executionRoot,
responsibleUserId: userId, receipt: { materializationRunId: first.runId } });
expect((await copies.get(companyId, first.runId))?.receipt?.retainedByRunId).toBe(secondId);
await copies.release(companyId, first.runId);
expect((await copies.collectStopped({ companyId, runId: first.runId }))?.processStoppedAt).toBeNull();
await copies.reportUnavailable(companyId, first.runId);
expect(await fs.readFile(path.join(first.localRoot, entryFile), "utf8")).toBe(initial);
await copies.release(companyId, secondId);
expect(await copies.hasChanges({ companyId, runId: secondId })).toBe(false);
const thirdId = await nextRun();
const third = await copies.adopt({ companyId, agentId, runId: thirdId, previousRunId: secondId, cwd: home });
expect(third?.executionRoot).toBe(first.executionRoot);
await fs.writeFile(path.join(first.localRoot, "warm-memory.txt"), "third turn edit");
expect(await copies.hasChanges({ companyId, runId: thirdId })).toBe(true);
expect(await copies.collectStopped({ companyId, runId: thirdId })).toMatchObject({ state: "saved", receipt: { cleanupPending: false } });
expect(await fs.readFile(path.join(root, "warm-memory.txt"), "utf8")).toBe("third turn edit");
await expect(fs.stat(first.localRoot)).rejects.toMatchObject({ code: "ENOENT" });
await copies.release(companyId, first.runId);
});
it.each(["company", "agent", "workspace", "environment"])("rejects foreign %s adoption", async mismatch => {
const first = await run();
expect(await copies.hasChanges({ companyId, runId: first.runId })).toBe(false);
const secondId = await nextRun();
expect(await copies.adopt({ companyId: mismatch === "company" ? randomUUID() : companyId,
agentId: mismatch === "agent" ? randomUUID() : agentId, runId: secondId, previousRunId: first.runId,
cwd: mismatch === "workspace" ? `${home}-other` : home,
...(mismatch === "environment" ? { target: { kind: "remote" as const, transport: "sandbox" as const, environmentId: randomUUID(), remoteCwd: "/workspace" } } : {}),
})).toBeNull();
expect((await copies.get(companyId, first.runId))?.receipt?.retainedByRunId).toBeUndefined();
await copies.collectStopped({ companyId, runId: first.runId });
});
it("allows only one stale concurrent adoption with identical updatedAt timestamps", async () => {
const fixed = new Date("2030-01-01T00:00:00.000Z");
vi.useFakeTimers({ toFake: ["Date"] });
vi.setSystemTime(fixed);
try {
const first = await run();
expect(await copies.hasChanges({ companyId, runId: first.runId })).toBe(false);
const next = await Promise.all([nextRun(), nextRun()]);
let reads = 0;
let releaseReads!: () => void;
const bothRead = new Promise<void>(resolve => { releaseReads = resolve; });
// Both claimants see the same unclaimed row before either can enter
// the agent-row transaction. Date precision cannot distinguish them.
const staleReader = async () => {
const captured = await copies.get(companyId, first.runId);
reads++;
if (reads >= 3) { if (reads === 4) releaseReads(); await bothRead; }
return captured;
};
const service = agentDirectoryWorkingCopyService(db, staleReader, async (row, values) => {
const [updated] = await db.update(agentInstructionWorkingCopies).set({ ...values, updatedAt: fixed })
.where(eq(agentInstructionWorkingCopies.runId, row.runId)).returning();
return updated;
});
const settled = await Promise.allSettled(next.map(runId => service.adopt({ companyId, agentId, runId, previousRunId: first.runId, cwd: home })));
const winners = settled.filter(result => result.status === "fulfilled" && result.value);
expect(winners).toHaveLength(1);
expect(settled.filter(result => result.status === "rejected")).toHaveLength(1);
const prior = await copies.get(companyId, first.runId);
expect(prior?.updatedAt).toEqual(fixed);
const winnerId = prior?.receipt?.retainedByRunId as string;
expect(next).toContain(winnerId);
expect(await copies.get(companyId, next.find(id => id !== winnerId)!)).toBeNull();
await copies.collectStopped({ companyId, runId: winnerId });
} finally { vi.useRealTimers(); }
});
it("fences a stale collector when adoption commits immediately before its stop CAS", async () => {
const first = await run();
expect(await copies.hasChanges({ companyId, runId: first.runId })).toBe(false);
const secondId = await nextRun();
const update = db.update.bind(db);
let injected = false;
const spy = vi.spyOn(db, "update").mockImplementation(table => {
const builder = update(table);
if (table !== agentInstructionWorkingCopies) return builder;
const set = builder.set.bind(builder);
builder.set = (values => {
const query = set(values);
if (!("processStoppedAt" in values) || !values.processStoppedAt || injected) return query;
const returning = query.returning.bind(query);
query.returning = (async () => {
injected = true;
expect(await copies.adopt({ companyId, agentId, runId: secondId, previousRunId: first.runId, cwd: home })).not.toBeNull();
return returning();
}) as typeof query.returning;
return query;
}) as typeof builder.set;
return builder;
});
try {
expect(await copies.collectStopped({ companyId, runId: first.runId })).toMatchObject({
receipt: { retainedByRunId: secondId }, processStoppedAt: null, attempts: 0,
});
expect(injected).toBe(true);
expect(await fs.readFile(path.join(first.localRoot, entryFile), "utf8")).toBe(initial);
expect((await copies.get(companyId, secondId))?.processStoppedAt).toBeNull();
} finally { spy.mockRestore(); }
await copies.collectStopped({ companyId, runId: secondId });
});
it("requires current-run authorization before transferring collection authority", async () => {
const first = await run();
expect(await copies.hasChanges({ companyId, runId: first.runId })).toBe(false);
const secondId = await nextRun();
await db.update(heartbeatRuns).set({ responsibleUserId: null }).where(eq(heartbeatRuns.id, secondId));
await expect(copies.adopt({ companyId, agentId, runId: secondId, previousRunId: first.runId, cwd: home })).rejects.toMatchObject({ status: 403 });
expect((await copies.get(companyId, first.runId))?.receipt?.retainedByRunId).toBeUndefined();
await copies.collectStopped({ companyId, runId: first.runId });
});
it.each(["owner-close", "restart-proof"])("durably preserves an adopted directory after retirement failure and final cleanup: %s", async recovery => {
const first = await run();
expect(await copies.hasChanges({ companyId, runId: first.runId })).toBe(false);
const secondId = await nextRun();
await copies.adopt({ companyId, agentId, runId: secondId, previousRunId: first.runId, cwd: home });
await db.update(heartbeatRuns).set({ status: "failed", runtimeMode: "native" }).where(eq(heartbeatRuns.id, secondId));
expect(await copies.reportRetirementUnconfirmed(companyId, secondId)).toMatchObject({
state: "pending_collection", errorCode: "INSTRUCTION_STOP_UNCONFIRMED", processStoppedAt: null,
});
await fs.writeFile(path.join(first.localRoot, "unsaved-memory.txt"), "Keep failed retirement edits");
// Mirror heartbeat's final cleanup after preparation throws. Neither the
// current pending owner nor the prior alias may become terminal loss.
await copies.reportUnavailable(companyId, secondId);
await copies.reportUnavailable(companyId, first.runId);
expect(await copies.get(companyId, secondId)).toMatchObject({ state: "pending_collection", errorCode: "INSTRUCTION_STOP_UNCONFIRMED", processStoppedAt: null });
await copies.reportRetirementUnconfirmed(companyId, first.runId);
await copies.release(companyId, first.runId);
await copies.release(companyId, secondId);
await copies.recoverStopped(); // A terminal run with no stop receipt grants no collection authority.
expect((await copies.get(companyId, secondId))?.processStoppedAt).toBeNull();
expect(await fs.readFile(path.join(first.localRoot, entryFile), "utf8")).toBe(initial);
if (recovery === "owner-close") await copies.collectStopped({ companyId, runId: secondId }); // Explicit test owner retirement.
else {
const { appendHeartbeatRunEvent } = await import("../services/heartbeat-run-events.js");
const stop = (runId: string) => appendHeartbeatRunEvent(db, { companyId, runId, agentId,
eventType: "native.local_process_stopped", stream: "system", level: "info", message: "Test-owned process retirement proof", payload: {} });
await stop(first.runId);
copies = agentInstructionWorkingCopyService(db); // No in-memory owner or transport survives.
await copies.recoverStopped();
expect((await copies.get(companyId, secondId))?.processStoppedAt).toBeNull();
await stop(secondId);
// Make the deferred maintenance retry due after the new owner stop proof.
await db.update(agentInstructionWorkingCopies).set({ nextAttemptAt: null }).where(eq(agentInstructionWorkingCopies.runId, secondId));
await copies.recoverStopped();
}
expect(await copies.get(companyId, secondId)).toMatchObject({ state: "saved", receipt: { cleanupPending: false } });
expect(await fs.readFile(path.join(root, "unsaved-memory.txt"), "utf8")).toBe("Keep failed retirement edits");
await expect(fs.stat(first.localRoot)).rejects.toMatchObject({ code: "ENOENT" });
});
it("declines reuse after canonical instruction changes", async () => {
const first = await run();
expect(await copies.hasChanges({ companyId, runId: first.runId })).toBe(false);
await fs.writeFile(path.join(root, entryFile), "Changed canonical instructions");
const secondId = await nextRun();
expect(await copies.adopt({ companyId, agentId, runId: secondId, previousRunId: first.runId, cwd: home })).toBeNull();
await copies.collectStopped({ companyId, runId: first.runId });
});
it("requires stop for matching bytes at a replaced materialized root", async () => {
const first = await run();
const replaced = `${first.localRoot}-old`;
await fs.rename(first.localRoot, replaced);
await fs.cp(replaced, first.localRoot, { recursive: true });
expect(await copies.hasChanges({ companyId, runId: first.runId })).toBe(true);
await copies.collectStopped({ companyId, runId: first.runId });
});
});
it.each([".paperclip-runtime/state", "notes/.paperclip-runtime/state", "promptTemplate.legacy.md"])("rejects reserved board path %s before mutation", async (reserved) => {
await expect(agentFileStore(db).write({ ...target(), path: reserved, bytes: Buffer.from("reserved"), baseHash: null }, board())).rejects.toMatchObject({ status: 422 });
await expect(fs.stat(path.join(root, reserved))).rejects.toMatchObject({ code: "ENOENT" });
@@ -381,7 +558,8 @@ describe("persistent agent directories", () => {
}
expect(execute).toHaveBeenCalledTimes(2);
expect(execute).toHaveBeenLastCalledWith(expect.objectContaining({ lease: expect.objectContaining({ id: leaseId, providerLeaseId: "original-sandbox" }),
command: "rm", args: ["-rf", "--", executionRoot], bypassSession: true }));
command: "rm", args: ["-rf", "--", executionRoot,
path.posix.join(remoteCwd, ".paperclip-runtime", "paperclip-runner", "agent-file-transfers", agentId, copy.runId)], bypassSession: true }));
await copies.recoverCaptured();
expect(execute).toHaveBeenCalledTimes(2);
await copies.release(companyId, copy.runId);
@@ -389,6 +567,60 @@ describe("persistent agent directories", () => {
expect(execute).toHaveBeenCalledTimes(2);
});
it.each(["missing", "stopped", "destroyed"])("recovers a retained remote unchanged turn only with exact stop proof: %s", async stopped => {
const copy = await run();
const environmentId = randomUUID(), leaseId = randomUUID(), remoteCwd = "/fixture/task";
const lease = { id: leaseId, companyId, environmentId, heartbeatRunId: copy.runId, provider: "daytona", providerLeaseId: "retained-sandbox" };
await db.insert(environments).values({ id: environmentId, name: environmentId, driver: "sandbox" });
await db.insert(environmentLeases).values({ ...lease, status: "released", releasedAt: new Date(), cleanupStatus: "success",
metadata: stopped !== "missing" ? { remoteExecutionTermination: remoteTerminationReceipt(lease, { providerLeaseId: lease.providerLeaseId, state: stopped }) } : {} });
await db.update(heartbeatRuns).set({ status: "succeeded", runtimeMode: "native" }).where(eq(heartbeatRuns.id, copy.runId));
await db.update(agentInstructionWorkingCopies).set({ state: "unchanged_turn", location: `remote:${environmentId}`,
executionRoot: path.posix.join(remoteCwd, ".paperclip-runtime", "agent-files", agentId, copy.runId),
receipt: { ...copy.receipt, cleanup: { leaseId, remoteCwd } } }).where(eq(agentInstructionWorkingCopies.runId, copy.runId));
const execute = vi.fn();
copies = agentInstructionWorkingCopyService(db, { environmentRuntime: { execute } as unknown as EnvironmentRuntimeService });
expect((await copies.reportUnavailable(companyId, copy.runId))?.state).toBe("unchanged_turn");
await copies.recoverStopped();
const recovered = (await copies.get(companyId, copy.runId))!;
if (stopped === "destroyed") {
expect(recovered).toMatchObject({ state: "unavailable", errorCode: "INSTRUCTION_COLLECTION_UNAVAILABLE", receipt: { cleanupPending: false } });
expect(recovered.processStoppedAt).toBeInstanceOf(Date);
await expect(fs.stat(copy.localRoot)).rejects.toMatchObject({ code: "ENOENT" });
await copies.recoverStopped();
expect((await copies.get(companyId, copy.runId))?.updatedAt).toEqual(recovered.updatedAt);
} else {
expect(recovered).toMatchObject({ state: stopped === "stopped" ? "pending_collection" : "unchanged_turn", processStoppedAt: null });
if (stopped === "stopped") {
expect(recovered.errorCode).toBe("INSTRUCTION_STOPPED_REMOTE_COLLECTION_PENDING");
expect(recovered.receipt?.baseline).toBeDefined();
await copies.release(companyId, copy.runId);
await copies.recoverCaptured();
}
expect(await fs.readFile(path.join(copy.localRoot, entryFile), "utf8")).toBe(initial);
}
expect(execute).not.toHaveBeenCalled(); // Never restart a remote provider to recover bytes.
});
it.each([0, 2])("preserves a legacy no-ID remote copy with %s matching leases", async count => {
const copy = await run();
const environmentId = randomUUID(), remoteCwd = "/fixture/task";
await db.insert(environments).values({ id: environmentId, name: `Legacy remote ${environmentId}`, driver: "sandbox" });
for (let index = 0; index < count; index++) await db.insert(environmentLeases).values({
companyId, environmentId, heartbeatRunId: copy.runId, provider: "daytona", providerLeaseId: `allocation-${index}` });
await db.update(agentInstructionWorkingCopies).set({ location: `remote:${environmentId}`,
executionRoot: path.posix.join(remoteCwd, ".paperclip-runtime", "agent-files", agentId, copy.runId),
receipt: { ...copy.receipt, cleanup: { remoteCwd } } }).where(eq(agentInstructionWorkingCopies.runId, copy.runId));
const execute = vi.fn();
copies = agentInstructionWorkingCopyService(db, { environmentRuntime: { execute } as unknown as EnvironmentRuntimeService });
const pending = await copies.collectStopped({ companyId, runId: copy.runId });
expect(pending).toMatchObject({ state: "pending_collection", attempts: 0, processStoppedAt: null, errorCode: "INSTRUCTION_REMOTE_LEASE_UNVERIFIED" });
expect(pending!.receipt?.baseline).toBeDefined();
await copies.release(companyId, copy.runId);
expect(await fs.readFile(path.join(copy.localRoot, entryFile), "utf8")).toBe(initial);
expect(execute).not.toHaveBeenCalled();
});
it("finishes remote cleanup from a destruction receipt after the environment is deleted", async () => {
const copy = await run();
const environmentId = randomUUID(), leaseId = randomUUID(), remoteCwd = "/fixture/task";
@@ -614,15 +846,22 @@ describe("persistent agent directories", () => {
}
},
};
const executionTarget = { kind: "remote" as const, transport: "sandbox" as const, environmentId: randomUUID(), remoteCwd, runner };
const executionTarget = { kind: "remote" as const, transport: "sandbox" as const, environmentId: randomUUID(), leaseId: "", remoteCwd, runner };
await db.insert(environments).values({ id: executionTarget.environmentId, name: "Fixture sandbox", driver: "sandbox" });
const prepare = async () => {
const runId = randomUUID();
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "on_demand", responsibleUserId: userId });
executionTarget.leaseId = randomUUID();
await db.insert(environmentLeases).values({ id: executionTarget.leaseId, companyId, environmentId: executionTarget.environmentId,
heartbeatRunId: runId, provider: "daytona", providerLeaseId: "fixture-allocation" });
return (await copies.prepare({ ...target(), runId, cwd: home, target: executionTarget }))!;
};
await fs.mkdir(path.join(root, "build"));
await fs.writeFile(path.join(root, "build", "personal.txt"), "cache-like names are still agent files");
const first = await prepare();
const firstScratch = path.join(remoteCwd, ".paperclip-runtime", "paperclip-runner", "agent-file-transfers", agentId, first.runId);
expect(await fs.readdir(firstScratch)).toEqual([]);
await expect(fs.stat(path.join(first.executionRoot, ".paperclip-runtime"))).rejects.toMatchObject({ code: "ENOENT" });
expect(await fs.readFile(path.join(first.executionRoot, "build", "personal.txt"), "utf8")).toBe("cache-like names are still agent files");
await fs.mkdir(path.join(first.executionRoot, "notes"));
await fs.writeFile(path.join(first.executionRoot, "notes", "bytes.bin"), Buffer.from([0, 128, 255]));
@@ -632,6 +871,7 @@ describe("persistent agent directories", () => {
expect((await execFile("git", ["-C", remoteCwd, "status", "--porcelain", "--untracked-files=all"])).stdout).toBe("?? task-only.txt\n");
expect((await copies.collectStopped({ companyId, runId: first.runId, target: executionTarget }))?.state).toBe("saved");
await copies.release(companyId, first.runId);
await expect(fs.stat(firstScratch)).rejects.toMatchObject({ code: "ENOENT" });
expect((await copies.get(companyId, first.runId))?.receipt?.baseline).toBeUndefined();
await expect(fs.stat(first.localRoot)).rejects.toMatchObject({ code: "ENOENT" });
await fs.rm(remoteCwd, { recursive: true });
@@ -666,10 +906,14 @@ describe("persistent agent directories", () => {
}
},
};
const executionTarget = { kind: "remote" as const, transport: "sandbox" as const, environmentId: randomUUID(), remoteCwd, runner };
const executionTarget = { kind: "remote" as const, transport: "sandbox" as const, environmentId: randomUUID(), leaseId: "", remoteCwd, runner };
await db.insert(environments).values({ id: executionTarget.environmentId, name: "Warm fixture sandbox", driver: "sandbox" });
const prepare = async (reuseRunId?: string) => {
const runId = randomUUID();
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "on_demand", responsibleUserId: userId });
executionTarget.leaseId = randomUUID();
await db.insert(environmentLeases).values({ id: executionTarget.leaseId, companyId, environmentId: executionTarget.environmentId,
heartbeatRunId: runId, provider: "daytona", providerLeaseId: "warm-fixture-allocation" });
return (await copies.prepare({ ...target(), runId, cwd: home, target: executionTarget, warm: true, reuseRunId }))!;
};
let copy = await prepare();
@@ -687,7 +931,7 @@ describe("persistent agent directories", () => {
}
await copies.collectStopped({ companyId, runId: original.runId, target: executionTarget });
expect(await fs.stat(copy.executionRoot)).toBeTruthy();
await copies.collectStopped({ companyId, runId: copy.runId, target: executionTarget });
expect((await copies.collectStopped({ companyId, runId: copy.runId, target: executionTarget }))?.receipt).toMatchObject({ cleanupPending: false });
await expect(fs.stat(copy.executionRoot)).rejects.toMatchObject({ code: "ENOENT" });
});
@@ -706,6 +950,9 @@ describe("persistent agent directories", () => {
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "on_demand", responsibleUserId: userId });
const target = { kind: "remote" as const, transport: "ssh" as const, environmentId: randomUUID(), remoteCwd,
spec: { host: "unused.invalid", port: 22, username: "test", remoteCwd } };
await db.insert(environments).values({ id: target.environmentId, name: "Fixture SSH", driver: "ssh" });
// Legacy no-ID receipt: only the unique run/environment lease authorizes retrieval.
await db.insert(environmentLeases).values({ companyId, environmentId: target.environmentId, heartbeatRunId: runId, provider: "ssh", providerLeaseId: "fixture-host" });
const copy = (await copies.prepare({ companyId, agentId, runId, cwd: home, target }))!;
expect(stage).toHaveBeenCalledWith(expect.objectContaining({ remoteDir: copy.executionRoot }));
expect(await fs.readFile(path.join(copy.executionRoot, entryFile), "utf8")).toBe(initial);
@@ -948,8 +1195,8 @@ describe("persistent agent directories", () => {
const execute = vi.fn(), restoreWorkspace = vi.fn(), cleanupWorkspaceSnapshot = vi.fn().mockResolvedValue(undefined);
const executionTarget = { kind: "remote" as const, transport: "sandbox" as const, environmentId, leaseId, remoteCwd, runner: { execute } };
const shell = vi.spyOn(executionTargetTools, "runAdapterExecutionTargetShellCommand").mockResolvedValue({ exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "" });
const stage = vi.spyOn(executionTargetTools, "prepareAdapterExecutionTargetRuntime").mockImplementation(async () => ({ target: executionTarget,
workspaceRemoteDir: path.posix.join(remoteCwd, ".paperclip-runtime", "agent-files", agentId, runId), runtimeRootDir: null,
const stage = vi.spyOn(executionTargetTools, "prepareAdapterExecutionTargetRuntime").mockImplementation(async options => ({ target: executionTarget,
workspaceRemoteDir: path.posix.join(remoteCwd, ".paperclip-runtime", "agent-files", agentId, runId), runtimeRootDir: options.runtimeRootDir ?? null,
assetDirs: {}, additionalSourceDirs: {}, additionalSourceFailures: [], workspaceSyncSnapshot: null, restoreWorkspace, cleanupWorkspaceSnapshot }));
try {
const copy = (await copies.prepare({ ...target(), runId, cwd: home, target: executionTarget }))!;
@@ -0,0 +1,27 @@
import { describe, expect, it } from "vitest";
import type { AdapterEnvironmentTestResult } from "@paperclipai/adapter-utils";
import { ADAPTER_AUTH_MISSING_CHECK_CODE } from "@paperclipai/shared";
import { assertHarnessTestPassed } from "../services/agent-harness-verification.js";
describe("saved harness verification", () => {
const result = (status: AdapterEnvironmentTestResult["status"], codes: string[]): AdapterEnvironmentTestResult => ({
adapterType: "test", status, testedAt: new Date().toISOString(),
checks: codes.map(code => ({ code, level: "info", message: code })),
});
it.each([
["fail", []], ["warn", ["claude_hello_probe_auth_required"]],
["warn", ["gemini_hello_probe_timed_out"]], ["pass", ["claude_hello_probe_skipped_custom_command"]],
["warn", ["claude_acp_anthropic_api_key_detected", "claude_acp_login_probe_unavailable"]],
["warn", ["cli_version_probe_mismatch"]], ["pass", [ADAPTER_AUTH_MISSING_CHECK_CODE]],
["warn", ["claude_hello_probe_passed", "claude_hello_probe_auth_required"]],
] as const)("rejects %s with %j", (status, codes) => {
expect(() => assertHarnessTestPassed(result(status, [...codes]))).toThrow("The harness test failed");
});
it.each([
["pass", ["process_command_available"]], ["warn", ["claude_hello_probe_passed", "optional_environment_warning"]],
["warn", ["claude_acp_anthropic_api_key_detected", "claude_hello_probe_passed"]],
["pass", ["codex_hello_probe_succeeded"]], ["warn", ["ai_connection_api_key_reverified"]],
] as const)("accepts %s with %j", (status, codes) => {
expect(() => assertHarnessTestPassed(result(status, [...codes]))).not.toThrow();
});
});
@@ -8,7 +8,7 @@ import express from "express";
import request from "supertest";
import { and, eq } from "drizzle-orm";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { activityLog, agents, companies, companyMemberships, createDb, heartbeatRuns, issues, plugins, principalPermissionGrants, toolConnectionInstalls } from "@paperclipai/db";
import { activityLog, agents, companies, companyMemberships, createDb, closeRegisteredClients, heartbeatRuns, issues, plugins, principalPermissionGrants, toolConnectionInstalls } from "@paperclipai/db";
import { type AiConnectionBinding, type AiConnectionPoolMember, type PaperclipPluginManifestV1 } from "@paperclipai/shared";
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { agentRoutes } from "../routes/agents.js";
@@ -21,8 +21,17 @@ import { secretService } from "../services/secrets.js";
import { aiConnectionRouterService } from "../services/ai-connection-router.js";
import { instanceSettingsService } from "../services/instance-settings.js";
import { toolAccessService } from "../services/tool-access.js";
import { waitForPendingRunFailureReports } from "../services/run-failure-report.js";
import { configureAgentLifecycle } from "../services/agent-lifecycle.js";
const captureRunFailure = vi.hoisted(() => vi.fn());
vi.mock("../sentry.js", async (importOriginal) => ({
...await importOriginal<typeof import("../sentry.js")>(),
captureRunFailure,
}));
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
const externalTestDatabaseUrl = process.env.PAPERCLIP_TEST_DATABASE_URL?.trim();
let db: ReturnType<typeof createDb>;
let home: string;
@@ -31,11 +40,17 @@ beforeAll(async () => {
vi.stubEnv("PAPERCLIP_HOME", home);
vi.stubEnv("PAPERCLIP_INSTANCE_ID", "hire-ai");
vi.stubEnv("PAPERCLIP_IN_WORKTREE", "false");
database = await startEmbeddedPostgresTestDatabase("paperclip-hire-ai-db-");
db = createDb(database.connectionString);
if (externalTestDatabaseUrl) db = createDb(externalTestDatabaseUrl);
else {
database = await startEmbeddedPostgresTestDatabase("paperclip-hire-ai-db-");
db = createDb(database.connectionString);
}
}, 90_000);
afterAll(async () => {
await waitForPendingRunFailureReports();
if (externalTestDatabaseUrl) await closeRegisteredClients(externalTestDatabaseUrl);
await db?.$client.end();
await database?.cleanup();
vi.unstubAllEnvs();
if (home) await rm(home, { recursive: true, force: true });
@@ -360,7 +375,7 @@ describe("agent-created hires use managed AI connections", () => {
const agent = hired(await request(f.app).post(`/api/companies/${f.companyId}/agent-hires`).send({ name: "Other provider", role: "engineer", adapterType }));
expect(agent.runtimeConfig.aiConnection).toMatchObject({ provider: otherProvider, mode: "responsible_user" });
await expect(prepareManagedAiRuntime(db, { companyId: f.companyId, agentId: agent.id, responsibleUserId: f.userId, adapterType, binding: agent.runtimeConfig.aiConnection, config: agent.adapterConfig })).rejects.toMatchObject({ details: { code: "ai_connection_default_missing" } });
expect(agent.status).toBe("idle");
expect(agent).toMatchObject({ status: "paused", lifecycleState: "preparing" });
});
for (const endpoint of ["agent-hires", "agents"]) {
@@ -472,11 +487,17 @@ describe("agent-created hires use managed AI connections", () => {
});
describe("hired agents sharing a subscription", () => {
async function finishSetup(agentId: string) {
// These tests cover task credentials after setup; lifecycle tests cover verification.
const worker = configureAgentLifecycle(db, { requiredPluginIds: async () => [], runPlugin: async () => "complete", runHost: async () => "complete" });
try { await worker.process(agentId); } finally { await worker.stop(); }
}
it("keeps a credential-lock timeout on automatic retry without blocking the task or starting a provider", async () => {
const f = await fixture("openai", "subscription");
const agent = hired(await request(f.app).post(`/api/companies/${f.companyId}/agent-hires`).send({ name: "Waiting teammate", role: "engineer", adapterType: f.adapterType, reportsTo: f.agentId, adapterConfig: { engine: "cli" }, runtimeConfig: { heartbeat: { enabled: false } } }));
// Only an operator may set host execution paths after the agent is hired.
await db.update(agents).set({ adapterConfig: { ...agent.adapterConfig, cwd: home } }).where(eq(agents.id, agent.id));
await finishSetup(agent.id);
const [issue] = await db.insert(issues).values({ companyId: f.companyId, title: "Wait for credential rotation", status: "todo", assigneeAgentId: agent.id, responsibleUserId: f.userId, createdByUserId: f.userId }).returning();
const execute = vi.fn(async () => ({ exitCode: 0, signal: null, timedOut: false, resultJson: {} }));
registerServerAdapter({ ...getServerAdapter(f.adapterType), execute });
@@ -488,10 +509,21 @@ describe("hired agents sharing a subscription", () => {
expect(run).not.toBeNull();
await expect.poll(async () => (await heartbeat.getRun(run!.id))?.status, { timeout: 20_000 }).toBe("cancelled");
await heartbeat.drainActiveRunExecutions();
expect(await heartbeat.getRun(run!.id)).toMatchObject({ errorCode: "ai_connection_busy", resultJson: { executionRecovery: { providerWorkStarted: false } } });
const cancelled = await heartbeat.getRun(run!.id);
expect(cancelled).toMatchObject({ errorCode: "ai_connection_busy", resultJson: {
executionRecovery: { kind: "ai_connection_wait", providerWorkStarted: false },
cancellation: {
source: "control_plane", expected: true, initiator: { type: "system" },
reason: "Waiting for shared AI credentials",
recordedAt: cancelled!.finishedAt!.toISOString(),
},
} });
await waitForPendingRunFailureReports();
expect(captureRunFailure.mock.calls.filter(([report]) => report.runId === run!.id)).toHaveLength(0);
const retries = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, run!.id));
expect(retries).toHaveLength(1);
expect(retries[0]).toMatchObject({ status: "scheduled_retry", scheduledRetryReason: "ai_connection_busy" });
expect(retries[0].resultJson?.cancellation).toBeUndefined();
const [savedIssue] = await db.select().from(issues).where(eq(issues.id, issue.id));
expect(savedIssue.status).not.toBe("blocked");
expect(savedIssue.executionRunId).toBe(retries[0].id);
@@ -509,6 +541,7 @@ describe("hired agents sharing a subscription", () => {
const agent = hired(await request(f.app).post(`/api/companies/${f.companyId}/agent-hires`).send({ name: "Concurrent teammate", role: "engineer", adapterType: f.adapterType, reportsTo: f.agentId, adapterConfig: { engine: "cli" }, runtimeConfig: { heartbeat: { enabled: false } } }));
// Host working directories are configured by an operator, not an agent key.
await db.update(agents).set({ adapterConfig: { ...agent.adapterConfig, cwd: home } }).where(eq(agents.id, agent.id));
await finishSetup(agent.id);
const [issue] = await db.insert(issues).values({ companyId: f.companyId, title: "Subscription child task", status: "todo", assigneeAgentId: agent.id, responsibleUserId: f.userId, createdByUserId: f.userId }).returning();
const parentRuntime = await prepareManagedAiRuntime(db, { companyId: f.companyId, agentId: f.agentId, responsibleUserId: f.userId, adapterType: f.adapterType, binding: f.binding, config: { cwd: home } });
const execute = vi.fn(async () => {
+3 -2
View File
@@ -1,3 +1,4 @@
import { createAgentLifecycle } from "../services/agent-lifecycle.js";
import { randomBytes, randomUUID, verify, generateKeyPairSync } from "node:crypto";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
@@ -74,7 +75,7 @@ describe("agent cryptographic identity", () => {
});
it("creates identities in the agent creation transaction and rolls both writes back on failure", async () => {
const created = await agentService(db).create(companyId, { name: "New identity", role: "engineer", status: "pending_approval" });
const created = await createAgentLifecycle(db).requestHire(companyId, { name: "New identity", role: "engineer", status: "pending_approval" });
expect(await agentIdentityService(db).getPublicIdentity(companyId, created.id)).toMatchObject({ algorithm: "Ed25519" });
const id = randomUUID();
await expect(db.transaction(async tx => {
@@ -85,7 +86,7 @@ describe("agent cryptographic identity", () => {
expect(await db.select().from(agents).where(eq(agents.id, id))).toHaveLength(0);
expect(await agentIdentityService(db).getPublicIdentity(companyId, id)).toBeNull();
const fail = vi.spyOn(localEncryptedProvider, "createSecret").mockRejectedValueOnce(new Error("encryption unavailable"));
await expect(agentService(db).create(companyId, { name: "Must roll back", role: "engineer", status: "pending_approval" })).rejects.toThrow("encryption unavailable");
await expect(createAgentLifecycle(db).requestHire(companyId, { name: "Must roll back", role: "engineer", status: "pending_approval" })).rejects.toThrow("encryption unavailable");
fail.mockRestore();
expect(await db.select().from(agents).where(eq(agents.name, "Must roll back"))).toHaveLength(0);
});
@@ -300,6 +300,10 @@ describe("registered run instruction copies", () => {
expect((await revisions.readCurrent(target(), board()))?.content).toBe(initial);
expect((await copies.list(companyId, agentId, board()))[0]).toMatchObject({ state: "pending_collection", content: null });
await appendHeartbeatRunEvent(db, { ...target(), runId: copy.runId, eventType: "native.local_process_stopped", stream: "system" });
// Recovery scheduled a retry while stop authority was missing. Make that
// retry due before asking the restarted controller to collect the copy.
await db.update(agentInstructionWorkingCopies).set({ nextAttemptAt: new Date(0) })
.where(eq(agentInstructionWorkingCopies.runId, copy.runId));
await copies.recoverStopped();
expect((await revisions.readCurrent(target(), board()))?.content).toBe("edit before controller restart");
});
@@ -844,3 +844,10 @@ describe("agent instructions bundle routes", () => {
expect(res.body.adapterConfig.instructionsFilePath).toBeUndefined();
});
});
vi.mock("../services/agent-lifecycle.js", async (importOriginal) => {
const actual = await importOriginal<typeof import("../services/agent-lifecycle.js")>();
return { ...actual, createAgentLifecycle: () => ({
requestHire: (...args: unknown[]) => mockAgentService.create(...args),
}) };
});
@@ -1940,3 +1940,9 @@ describe("agent live run routes", () => {
expect(mockWorkspaceDiffReprojection.persist).not.toHaveBeenCalled();
});
});
vi.mock("../services/agent-lifecycle.js", async (importOriginal) => {
const actual = await importOriginal<typeof import("../services/agent-lifecycle.js")>();
return { ...actual, createAgentLifecycle: () => ({
}) };
});
@@ -73,8 +73,8 @@ const mockApprovalService = vi.hoisted(() => ({
create: vi.fn(),
getById: vi.fn(),
findOpenHireApprovalForAgent: vi.fn(),
approve: vi.fn(),
reject: vi.fn(),
approveHire: vi.fn(),
rejectHire: vi.fn(),
}));
const mockBudgetService = vi.hoisted(() => ({
@@ -317,8 +317,8 @@ describe("agent permission routes", () => {
mockApprovalService.create.mockReset();
mockApprovalService.getById.mockReset();
mockApprovalService.findOpenHireApprovalForAgent.mockReset();
mockApprovalService.approve.mockReset();
mockApprovalService.reject.mockReset();
mockApprovalService.approveHire.mockReset();
mockApprovalService.rejectHire.mockReset();
mockBudgetService.upsertPolicy.mockReset();
mockHeartbeatService.listTaskSessions.mockReset();
mockHeartbeatService.resetRuntimeSession.mockReset();
@@ -679,6 +679,16 @@ describe("agent permission routes", () => {
]);
});
it.each([{ spentMonthlyCents: 0 }, { name: "Renamed", spentMonthlyCents: 0 }])(
"rejects accounting fields before applying an agent update: %j", async payload => {
const app = await createApp({ type: "board", userId: "board-user", source: "session", isInstanceAdmin: true, companyIds: [companyId] });
const res = await requestApp(app, baseUrl => request(baseUrl).patch(`/api/agents/${agentId}`).send(payload));
expect(res.status).toBe(400);
expect(res.body.details).toEqual(expect.arrayContaining([expect.objectContaining({ path: ["spentMonthlyCents"] })]));
expect(mockAgentService.update).not.toHaveBeenCalled();
},
);
it("blocks agent updates for authenticated company members without agent admin permission", async () => {
mockAccessService.canUser.mockResolvedValue(false);
@@ -1251,7 +1261,7 @@ describe("agent permission routes", () => {
expect.objectContaining({
status: "idle",
}),
{ createdByUserId: "agent-admin-user", claudeLogin: { storedSessionId: null, ownerUserId: "agent-admin-user", applyExistingWithoutClaim: false } },
{ createdByUserId: "agent-admin-user", responsibleUserId: "agent-admin-user", claudeLogin: { storedSessionId: null, ownerUserId: "agent-admin-user", applyExistingWithoutClaim: false } },
);
expect(mockAccessService.setPrincipalPermission).toHaveBeenCalledWith(
companyId,
@@ -1381,7 +1391,7 @@ describe("agent permission routes", () => {
},
},
}),
{ createdByUserId: "board-user", claudeLogin: { storedSessionId: null, ownerUserId: "board-user", applyExistingWithoutClaim: false } },
{ createdByUserId: "board-user", responsibleUserId: "board-user", claudeLogin: { storedSessionId: null, ownerUserId: "board-user", applyExistingWithoutClaim: false } },
);
});
@@ -1417,7 +1427,7 @@ describe("agent permission routes", () => {
model: DEFAULT_OPENCODE_LOCAL_MODEL,
}),
}),
{ createdByUserId: "board-user", claudeLogin: { storedSessionId: null, ownerUserId: "board-user", applyExistingWithoutClaim: false } },
{ createdByUserId: "board-user", responsibleUserId: "board-user", claudeLogin: { storedSessionId: null, ownerUserId: "board-user", applyExistingWithoutClaim: false } },
);
});
@@ -1455,7 +1465,7 @@ describe("agent permission routes", () => {
model: "anthropic/claude-sonnet-4-5",
}),
}),
{ createdByUserId: "board-user", claudeLogin: { storedSessionId: null, ownerUserId: "board-user", applyExistingWithoutClaim: false } },
{ createdByUserId: "board-user", responsibleUserId: "board-user", claudeLogin: { storedSessionId: null, ownerUserId: "board-user", applyExistingWithoutClaim: false } },
);
});
@@ -1495,7 +1505,7 @@ describe("agent permission routes", () => {
},
}),
{
createdByUserId: "board-user",
createdByUserId: "board-user", responsibleUserId: "board-user",
claudeLogin: {
storedSessionId: null,
ownerUserId: "board-user",
@@ -1516,9 +1526,10 @@ describe("agent permission routes", () => {
status: "idle",
};
mockAgentService.getById.mockResolvedValue(pendingAgent);
mockAgentService.activatePendingApproval.mockResolvedValue({
mockApprovalService.approveHire.mockResolvedValue({
agent: approvedAgent,
activated: true,
applied: true,
approval: null,
});
const app = await createApp({
@@ -1534,8 +1545,7 @@ describe("agent permission routes", () => {
.send({}));
expect(res.status).toBe(200);
expect(mockAgentService.activatePendingApproval).toHaveBeenCalledWith(agentId);
expect(mockApprovalService.approve).not.toHaveBeenCalled();
expect(mockApprovalService.approveHire).toHaveBeenCalledWith(agentId, "board-user");
expect(mockLogActivity).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({
companyId,
actorType: "user",
@@ -1556,19 +1566,9 @@ describe("agent permission routes", () => {
...baseAgent,
status: "idle",
};
// First getById (getAccessibleAgent) sees the pending agent; the second
// (after the approval resolves) sees the activated agent.
mockAgentService.getById
.mockResolvedValueOnce(pendingAgent)
.mockResolvedValue(approvedAgent);
mockApprovalService.findOpenHireApprovalForAgent.mockResolvedValue({
id: "approval-1",
companyId,
type: "hire_agent",
status: "pending",
payload: { agentId },
});
mockApprovalService.approve.mockResolvedValue({
mockAgentService.getById.mockResolvedValue(pendingAgent);
mockApprovalService.approveHire.mockResolvedValue({
agent: approvedAgent,
approval: { id: "approval-1", status: "approved" },
applied: true,
});
@@ -1587,7 +1587,7 @@ describe("agent permission routes", () => {
expect(res.status).toBe(200);
// The shared approval flow handles activation; we must not double-activate.
expect(mockApprovalService.approve).toHaveBeenCalledWith("approval-1", "board-user");
expect(mockApprovalService.approveHire).toHaveBeenCalledWith(agentId, "board-user");
expect(mockAgentService.activatePendingApproval).not.toHaveBeenCalled();
expect(mockLogActivity).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({
action: "agent.approved",
@@ -1604,19 +1604,9 @@ describe("agent permission routes", () => {
...baseAgent,
status: "terminated",
};
// getAccessibleAgent sees the pending agent; after the rejection resolves
// (which terminates internally) the route re-reads the terminated agent.
mockAgentService.getById
.mockResolvedValueOnce(pendingAgent)
.mockResolvedValue(terminatedAgent);
mockApprovalService.findOpenHireApprovalForAgent.mockResolvedValue({
id: "approval-1",
companyId,
type: "hire_agent",
status: "pending",
payload: { agentId },
});
mockApprovalService.reject.mockResolvedValue({
mockAgentService.getById.mockResolvedValue(pendingAgent);
mockApprovalService.rejectHire.mockResolvedValue({
agent: terminatedAgent,
approval: { id: "approval-1", status: "rejected" },
applied: true,
});
@@ -1639,8 +1629,8 @@ describe("agent permission routes", () => {
.send({}));
expect(res.status).toBe(200);
expect(mockApprovalService.reject).toHaveBeenCalledWith("approval-1", "board-user");
// reject() terminates the agent internally; the route must not terminate again.
expect(mockApprovalService.rejectHire).toHaveBeenCalledWith(agentId, "board-user");
// The hire decision rejects the pending agent; the route must not terminate again.
expect(mockAgentService.terminate).not.toHaveBeenCalled();
});
@@ -1670,7 +1660,7 @@ describe("agent permission routes", () => {
expect(res.status).toBe(200);
expect(mockAgentService.terminate).toHaveBeenCalledWith(agentId);
expect(mockApprovalService.findOpenHireApprovalForAgent).not.toHaveBeenCalled();
expect(mockApprovalService.reject).not.toHaveBeenCalled();
expect(mockApprovalService.rejectHire).not.toHaveBeenCalled();
});
it("rejects direct approval for agents that are not pending approval", async () => {
@@ -1687,7 +1677,7 @@ describe("agent permission routes", () => {
.send({}));
expect(res.status).toBe(409);
expect(mockAgentService.activatePendingApproval).not.toHaveBeenCalled();
expect(mockApprovalService.approveHire).not.toHaveBeenCalled();
expect(mockLogActivity).not.toHaveBeenCalledWith(expect.anything(), expect.objectContaining({
action: "agent.approved",
}));
@@ -1726,7 +1716,7 @@ describe("agent permission routes", () => {
expect.objectContaining({
defaultEnvironmentId: environmentId,
}),
{ createdByUserId: "board-user", claudeLogin: { storedSessionId: null, ownerUserId: "board-user", applyExistingWithoutClaim: false } },
{ createdByUserId: "board-user", responsibleUserId: "board-user", claudeLogin: { storedSessionId: null, ownerUserId: "board-user", applyExistingWithoutClaim: false } },
);
});
@@ -1812,7 +1802,7 @@ describe("agent permission routes", () => {
adapterType: adapterCase.adapterType,
defaultEnvironmentId: environmentId,
}),
{ createdByUserId: "board-user", claudeLogin: { storedSessionId: null, ownerUserId: "board-user", applyExistingWithoutClaim: false } },
{ createdByUserId: "board-user", responsibleUserId: "board-user", claudeLogin: { storedSessionId: null, ownerUserId: "board-user", applyExistingWithoutClaim: false } },
);
});
}
@@ -2291,3 +2281,12 @@ describe("agent permission routes", () => {
expect(mockHeartbeatService.cancelRun).not.toHaveBeenCalled();
});
});
vi.mock("../services/agent-lifecycle.js", async (importOriginal) => {
const actual = await importOriginal<typeof import("../services/agent-lifecycle.js")>();
return { ...actual, createAgentLifecycle: () => ({
requestHire: (...args: unknown[]) => mockAgentService.create(...args),
approveHire: (...args: unknown[]) => mockAgentService.activatePendingApproval(...args),
terminateAgent: (...args: unknown[]) => mockAgentService.terminate(...args),
}) };
});
@@ -8,7 +8,7 @@ import {
defaultAgentPermissions,
normalizeAgentPermissions,
permissionsImplyLowTrust,
} from "../services/agent-permissions.js";
} from "../lib/agent-permissions.js";
describe("agent permissions service", () => {
it("grants agent-creation authority to new agents by default", () => {
@@ -1,3 +1,4 @@
import { createAgentLifecycle } from "../services/agent-lifecycle.js";
import { randomUUID } from "node:crypto";
import { readFile } from "node:fs/promises";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
@@ -20,15 +21,17 @@ describe("persisted agent personas", () => {
afterAll(async () => { await database?.cleanup(); });
it("randomizes once, accepts a saved draft and preserves identity across edits and revision rollback", async () => {
const service = agentService(db);
const random = await service.create(companyId, { name: "Random", role: "engineer", adapterType: "process" });
const serviceLifecycle = createAgentLifecycle(db);
const random = await serviceLifecycle.requestHire(companyId, { name: "Random", role: "engineer", adapterType: "process" });
expect(agentAppearanceSchema.safeParse(random.appearance).success).toBe(true);
const appearance = appearanceForPalette("arctic-blue");
const original = await service.create(companyId, { name: "Draft", role: "engineer", adapterType: "process", appearance });
const original = await serviceLifecycle.requestHire(companyId, { name: "Draft", role: "engineer", adapterType: "process", appearance });
expect(original.appearance).toEqual(appearance);
const updated = await service.update(original.id, { name: "Renamed" }, { recordRevision: { source: "test" } });
expect(updated?.appearance).toEqual(appearance);
const [revision] = await service.listConfigRevisions(original.id);
await service.update(original.id, { name: "Changed again", status: "paused" });
await service.update(original.id, { name: "Changed again" });
await serviceLifecycle.pauseAgent(original.id);
const restored = await service.rollbackConfigRevision(original.id, revision.id, {});
expect(restored?.appearance).toEqual(appearance);
expect((await agentService(db).getById(original.id))?.avatarUrl).toContain("/arctic-blue/rest.png?size=512");
@@ -0,0 +1,156 @@
import { randomUUID } from "node:crypto";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import express, { type Request, type ErrorRequestHandler } from "express";
import request from "supertest";
import sharp from "sharp";
import { eq } from "drizzle-orm";
import { afterAll, beforeAll, expect, it, vi } from "vitest";
import { createDb, agents, companies, assets, activityLog, authUsers, companyMemberships, mcpOauthClients, mcpOauthGrants, dotAgentBindings, principalPermissionGrants } from "@paperclipai/db";
import { appearanceForPalette, MAX_AGENT_AVATAR_BYTES } from "@paperclipai/shared";
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { createLocalDiskStorageProvider } from "../storage/local-disk-provider.js";
import { createStorageService } from "../storage/service.js";
import type { StorageService } from "../storage/types.js";
import { normalizeAgentAvatar, setAgentProfileAvatar } from "../services/agent-profile-avatar.js";
import { agentProfileAvatarRoutes } from "../routes/agent-profile-avatar.js";
import { assetRoutes } from "../routes/assets.js";
import { agentService } from "../services/agents.js";
import { instanceSettingsService } from "../services/instance-settings.js";
import { createDotRunnerMcpTools } from "../services/dot-runner-broker.js";
import type { McpPrincipal } from "../services/public-mcp/oauth.js";
import { isSecretSensitiveHttpRequest } from "../middleware/http-log-policy.js";
let storage: StorageService;
vi.mock("../storage/index.js", () => ({ getStorageService: () => storage }));
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let db: ReturnType<typeof createDb>;
let root: string;
let imageBase64: string;
beforeAll(async () => {
temporary = await startEmbeddedPostgresTestDatabase("paperclip-profile-avatar-");
db = createDb(temporary.connectionString);
root = await mkdtemp(join(tmpdir(), "profile-avatar-"));
storage = createStorageService(createLocalDiskStorageProvider(root));
imageBase64 = (await sharp({ create: { width: 800, height: 600, channels: 3, background: "orange" } }).png().toBuffer()).toString("base64");
await instanceSettingsService(db).updateExperimental({ enablePublicMcp: true, enableOpenAiDot: true });
}, 120000);
afterAll(async () => { await temporary?.cleanup(); if (root) await rm(root, { recursive: true, force: true }); });
async function fixture() {
const [company] = await db.insert(companies).values({ name: "Avatar test", issuePrefix: randomUUID().slice(0, 8) }).returning();
const [agent] = await db.insert(agents).values({ companyId: company!.id, name: "Dot", status: "active", adapterType: "paperclip_runner", appearance: appearanceForPalette("deep-tide") }).returning();
const actor = { actorType: "agent" as const, actorId: agent!.id, agentId: agent!.id };
return { company: company!, agent: agent!, actor };
}
function appFor(actor: Request["actor"]) {
const app = express();
app.use(express.json({ limit: "2mb" }));
app.use((req, _res, next) => { req.actor = actor; next(); });
app.use("/api", agentProfileAvatarRoutes(db, storage), assetRoutes(db, storage));
app.use(((error, _req, res, _next) => res.status(error.status ?? 400).json({ error: error.message })) as ErrorRequestHandler);
return app;
}
it("uploads, normalizes, serves privately, retries without duplication, and resets the avatar", async () => {
const f = await fixture();
const app = appFor({ type: "agent", agentId: f.agent.id, companyId: f.company.id });
const path = `/api/companies/${f.company.id}/agents/${f.agent.id}/avatar`;
const upload = await request(app).put(path).send({ imageBase64 }).expect(200);
expect(upload.body.appearance.paletteId).toBe(f.agent.appearance?.paletteId);
const retry = await request(app).put(path).send({ imageBase64 }).expect(200);
expect(retry.body).toEqual(upload.body);
expect(await db.select().from(assets).where(eq(assets.companyId, f.company.id))).toHaveLength(1);
const audit = await db.select().from(activityLog).where(eq(activityLog.companyId, f.company.id));
expect(audit.filter(row => row.action === "agent.avatar_updated")).toHaveLength(1);
expect(JSON.stringify(audit)).not.toContain(imageBase64);
const content = await request(app).get(upload.body.avatarUrl).expect(200);
expect(content.headers["cache-control"]).toContain("private");
expect(await sharp(content.body).metadata()).toMatchObject({ format: "png", width: 512, height: 384 });
expect((await agentService(db).getById(f.agent.id))?.avatarUrl).toBe(upload.body.avatarUrl);
const reset = await request(app).put(path).send({ imageBase64: null }).expect(200);
expect(reset.body.appearance).toEqual(f.agent.appearance);
expect(reset.body.avatarUrl).toContain("/api/agent-avatars/");
});
it("rejects other agents, companies, viewers, and foreign asset references", async () => {
const f = await fixture(); const other = await fixture();
const path = `/api/companies/${f.company.id}/agents/${f.agent.id}/avatar`;
for (const actor of [
{ type: "agent", agentId: other.agent.id, companyId: f.company.id },
{ type: "agent", agentId: other.agent.id, companyId: other.company.id },
{ type: "board", source: "session", companyIds: [f.company.id], memberships: [{ companyId: f.company.id, status: "active", membershipRole: "viewer" }] },
] satisfies Request["actor"][]) await request(appFor(actor)).put(path).send({ imageBase64 }).expect(403);
await request(appFor({ type: "none" })).put(path).send({ imageBase64 }).expect(401);
const foreign = await setAgentProfileAvatar(db, storage, other.company.id, other.agent.id, { imageBase64 }, other.actor);
await expect(agentService(db).update(f.agent.id, { appearance: foreign.appearance })).rejects.toThrow("avatar upload");
await request(appFor({ type: "agent", agentId: f.agent.id, companyId: f.company.id })).get(foreign.avatarUrl).expect(404);
});
it("requires an agent configuration grant for board avatar changes", async () => {
const f = await fixture();
const userId = randomUUID();
await db.insert(authUsers).values({ id: userId, name: "Member", email: `${userId}@example.test`, createdAt: new Date(), updatedAt: new Date() });
await db.insert(companyMemberships).values({ companyId: f.company.id, principalType: "user", principalId: userId, membershipRole: "member", status: "active" });
const app = appFor({ type: "board", source: "session", userId, companyIds: [f.company.id], memberships: [{ companyId: f.company.id, status: "active", membershipRole: "member" }] });
const path = `/api/companies/${f.company.id}/agents/${f.agent.id}/avatar`;
await request(app).put(path).send({ imageBase64 }).expect(403);
await request(app).put(path).send({ imageBase64: null }).expect(403);
expect(await db.select().from(assets).where(eq(assets.companyId, f.company.id))).toHaveLength(0);
await db.insert(principalPermissionGrants).values({ companyId: f.company.id, principalType: "user", principalId: userId, permissionKey: "agents:suggest-changes" });
await request(app).put(path).send({ imageBase64 }).expect(403);
await db.insert(principalPermissionGrants).values({ companyId: f.company.id, principalType: "user", principalId: userId, permissionKey: "agents:configure" });
await request(app).put(path).send({ imageBase64 }).expect(200);
await request(app).put(path).send({ imageBase64: null }).expect(200);
});
it.each([
{ kind: "task_bridge" as const, parentIssueId: randomUUID() },
{ kind: "skill_test" as const, issueId: randomUUID() },
])("rejects profile changes by restricted %s credentials", async keyScope => {
const f = await fixture();
const app = appFor({ type: "agent", agentId: f.agent.id, companyId: f.company.id, keyScope });
const path = `/api/companies/${f.company.id}/agents/${f.agent.id}/avatar`;
await request(app).put(path).send({ imageBase64 }).expect(403);
await request(app).put(path).send({ imageBase64: null }).expect(403);
expect(await db.select().from(assets).where(eq(assets.companyId, f.company.id))).toHaveLength(0);
expect(await db.select().from(activityLog).where(eq(activityLog.companyId, f.company.id))).toHaveLength(0);
expect((await agentService(db).getById(f.agent.id))?.appearance).toEqual(f.agent.appearance);
});
it("rejects URLs, SVG, corrupt and oversized images, unsupported fields and inactive agents", async () => {
for (const input of ["https://example.test/avatar.png", Buffer.from("<svg xmlns=\"http://www.w3.org/2000/svg\"/>").toString("base64"), Buffer.from("broken").toString("base64"), Buffer.alloc(MAX_AGENT_AVATAR_BYTES + 1).toString("base64")]) {
await expect(normalizeAgentAvatar(input)).rejects.toThrow();
}
const tooManyPixels = await sharp({ create: { width: 5000, height: 5000, channels: 3, background: "white" } }).png().toBuffer();
await expect(normalizeAgentAvatar(tooManyPixels.toString("base64"))).rejects.toThrow("16 megapixels");
const f = await fixture();
const app = appFor({ type: "agent", agentId: f.agent.id, companyId: f.company.id });
await request(app).put(`/api/companies/${f.company.id}/agents/${f.agent.id}/avatar`).send({ imageBase64, agentId: randomUUID() }).expect(400);
await db.update(agents).set({ status: "pending_approval" }).where(eq(agents.id, f.agent.id));
await expect(setAgentProfileAvatar(db, storage, f.company.id, f.agent.id, { imageBase64 }, f.actor)).rejects.toThrow("current state");
});
it("lets a paired Dot upload without an assignment, with feature and current grant authorization", async () => {
const f = await fixture(); const userId = randomUUID(); const clientId = randomUUID();
await db.insert(authUsers).values({ id: userId, name: "Owner", email: `${userId}@example.test`, createdAt: new Date(), updatedAt: new Date() });
await db.insert(companyMemberships).values({ companyId: f.company.id, principalType: "user", principalId: userId, membershipRole: "owner", status: "active" });
await db.insert(mcpOauthClients).values({ id: clientId, name: "Dot", redirectUris: [] });
const [grant] = await db.insert(mcpOauthGrants).values({ companyId: f.company.id, userId, clientId, resource: "https://paperclip.example/mcp/runner", purpose: "agent", agentId: f.agent.id, scopes: ["paperclip:agent"] }).returning();
await db.insert(dotAgentBindings).values({ companyId: f.company.id, agentId: f.agent.id, operatorId: userId, grantId: grant!.id, status: "connected" });
const principal: McpPrincipal = { grant: grant!, company: f.company, actor: { type: "agent", companyId: f.company.id, agentId: f.agent.id } };
const tools = createDotRunnerMcpTools(db);
expect((await tools.listTools(principal)).find(tool => tool.name === "paperclip_dot_set_avatar")?.annotations).toMatchObject({ readOnlyHint: false });
expect(await tools.callTool(principal, "paperclip_dot_set_avatar", { imageBase64 })).toMatchObject({ outcome: "completed", result: { agentId: f.agent.id } });
await expect(tools.callTool(principal, "paperclip_dot_set_avatar", { imageBase64, agentId: randomUUID() })).rejects.toThrow();
await instanceSettingsService(db).updateExperimental({ enableOpenAiDot: false });
try { await expect(tools.callTool(principal, "paperclip_dot_set_avatar", { imageBase64 })).rejects.toThrow("disabled"); }
finally { await instanceSettingsService(db).updateExperimental({ enableOpenAiDot: true }); }
await db.update(mcpOauthGrants).set({ revokedAt: new Date() }).where(eq(mcpOauthGrants.id, grant!.id));
await expect(tools.callTool(principal, "paperclip_dot_set_avatar", { imageBase64 })).rejects.toThrow("authority");
});
it("redacts HTTP diagnostics containing image bytes", () => {
expect(isSecretSensitiveHttpRequest("PUT", "/api/companies/company/agents/agent/avatar")).toBe(true);
expect(isSecretSensitiveHttpRequest("POST", "/mcp/runner")).toBe(true);
});
@@ -1005,7 +1005,7 @@ describe("agent skill routes", () => {
}),
}),
}),
{ createdByUserId: "local-board", claudeLogin: { storedSessionId: null, ownerUserId: "local-board", applyExistingWithoutClaim: false } },
{ createdByUserId: "local-board", responsibleUserId: "local-board", claudeLogin: { storedSessionId: null, ownerUserId: "local-board", applyExistingWithoutClaim: false } },
);
expect(mockTrackAgentCreated).toHaveBeenCalledWith(
expect.anything(),
@@ -1055,7 +1055,7 @@ describe("agent skill routes", () => {
expect.objectContaining({
role: "security",
}),
{ createdByUserId: "local-board", claudeLogin: { storedSessionId: null, ownerUserId: "local-board", applyExistingWithoutClaim: false } },
{ createdByUserId: "local-board", responsibleUserId: "local-board", claudeLogin: { storedSessionId: null, ownerUserId: "local-board", applyExistingWithoutClaim: false } },
);
expect(mockTrackAgentCreated).toHaveBeenCalledWith(
expect.anything(),
@@ -1491,7 +1491,7 @@ describe("agent skill routes", () => {
}),
}),
{
createdByUserId: "local-board",
createdByUserId: "local-board", responsibleUserId: "local-board",
claudeLogin: {
storedSessionId: null,
ownerUserId: "local-board",
@@ -1575,3 +1575,10 @@ describe("agent skill routes", () => {
expect(mockAgentInstructionsService.materializeManagedBundle).not.toHaveBeenCalled();
});
});
vi.mock("../services/agent-lifecycle.js", async (importOriginal) => {
const actual = await importOriginal<typeof import("../services/agent-lifecycle.js")>();
return { ...actual, createAgentLifecycle: () => ({
requestHire: (...args: unknown[]) => mockAgentService.create(...args),
}) };
});
@@ -93,6 +93,10 @@ vi.mock("../services/instance-settings.js", () => ({
instanceSettingsService: () => mockInstanceSettingsService,
}));
vi.mock("../services/dot-runner-broker.js", () => ({
dotRunnerBroker: () => ({ enabled: async () => true, bindingForAgent: async () => null }),
}));
const testEnvironmentSpy = vi.fn();
const externalAdapter: ServerAdapterModule = {
@@ -116,10 +120,7 @@ vi.mock("../services/ai-connection-runtime.js", async (importOriginal) => ({
},
}));
const mockValidateAiApiKey = vi.hoisted(() => vi.fn(async () => undefined));
vi.mock("../routes/ai-connections.js", async (importOriginal) => ({
...(await importOriginal<typeof import("../routes/ai-connections.js")>()),
validateAiApiKey: mockValidateAiApiKey,
}));
vi.mock("../services/ai-api-key-test.js", () => ({ validateAiApiKey: mockValidateAiApiKey }));
const mockMarkAuthenticationFailed = vi.hoisted(() => vi.fn(async () => undefined));
vi.mock("../services/ai-connections.js", async (importOriginal) => ({
...(await importOriginal<typeof import("../services/ai-connections.js")>()),
@@ -252,6 +253,35 @@ describe("agent test-environment route", () => {
await unregisterTestAdapter("external_test");
});
it("accepts a managed Dot environment without allocating an idle sandbox", async () => {
vi.stubEnv("PAPERCLIP_PUBLIC_URL", "https://paperclip.example");
mockInstanceSettingsService.get.mockResolvedValue({ defaultEnvironmentId: "11111111-1111-4111-8111-111111111111" });
const app = await createApp();
try {
const res = await request(app).post("/api/companies/company-1/adapters/paperclip_runner/test-environment")
.send({ adapterConfig: { provider: "openai_dot", allowUnmeteredProvider: true } });
expect(res.status, JSON.stringify(res.body)).toBe(200);
expect(res.body.checks).toContainEqual(expect.objectContaining({ code: "dot_controller", level: "info" }));
expect(mockEnvironmentRuntime.acquireRunLease).not.toHaveBeenCalled();
expect(testEnvironmentSpy).not.toHaveBeenCalled();
} finally { vi.unstubAllEnvs(); }
});
it("fails the Cloud Dot check without a managed sandbox instead of probing the host", async () => {
vi.stubEnv("PAPERCLIP_PUBLIC_URL", "https://paperclip.example");
vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "test-cloud");
const app = await createApp();
try {
const res = await request(app).post("/api/companies/company-1/adapters/paperclip_runner/test-environment")
.send({ adapterConfig: { provider: "openai_dot", allowUnmeteredProvider: true } });
expect(res.status, JSON.stringify(res.body)).toBe(200);
expect(res.body.status).toBe("fail");
expect(res.body.checks).toContainEqual(expect.objectContaining({ code: "dot_controller", level: "error" }));
expect(mockEnvironmentRuntime.acquireRunLease).not.toHaveBeenCalled();
expect(testEnvironmentSpy).not.toHaveBeenCalled();
} finally { vi.unstubAllEnvs(); }
});
it("tests the instance default sandbox when the agent inherits its environment", async () => {
const environmentId = "11111111-1111-4111-8111-111111111111";
mockInstanceSettingsService.get.mockResolvedValue({ defaultEnvironmentId: environmentId });
@@ -1030,3 +1060,9 @@ describe("agent test-environment route", () => {
});
});
});
vi.mock("../services/agent-lifecycle.js", async (importOriginal) => {
const actual = await importOriginal<typeof import("../services/agent-lifecycle.js")>();
return { ...actual, createAgentLifecycle: () => ({
}) };
});
@@ -416,3 +416,9 @@ describeEmbeddedPostgres("agents adapter-config user-secret resolution routes",
function beforeEachActor(actor: TestActor) {
currentActor = actor;
}
vi.mock("../services/agent-lifecycle.js", async (importOriginal) => {
const actual = await importOriginal<typeof import("../services/agent-lifecycle.js")>();
return { ...actual, createAgentLifecycle: () => ({
}) };
});
@@ -1,3 +1,4 @@
import { createAgentLifecycle } from "../services/agent-lifecycle.js";
import { randomUUID } from "node:crypto";
import { mkdirSync, rmSync } from "node:fs";
import os from "node:os";
@@ -362,7 +363,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
const scope = await seedScope();
const sessionId = await seedStoredClaim(scope, Date.now() + 60_000);
const created = await agentService(db).create(scope.companyId, createInput(scope), {
const created = await createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), {
claudeLogin: { storedSessionId: sessionId, ownerUserId: scope.ownerUserId },
});
@@ -376,7 +377,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
// A replay of the same claim inserts no second agent.
await expect(
agentService(db).create(scope.companyId, createInput(scope), {
createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), {
claudeLogin: { storedSessionId: sessionId, ownerUserId: scope.ownerUserId },
}),
).rejects.toMatchObject({ message: CLAUDE_OAUTH_CLAIM_REJECTED });
@@ -386,7 +387,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
it("rejects a missing claim and inserts no binding", async () => {
const scope = await seedScope();
await expect(
agentService(db).create(scope.companyId, createInput(scope), {
createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), {
claudeLogin: { storedSessionId: undefined, ownerUserId: scope.ownerUserId },
}),
).rejects.toMatchObject({ message: CLAUDE_OAUTH_CLAIM_REJECTED });
@@ -397,7 +398,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
const scope = await seedScope();
const sessionId = await seedStoredClaim(scope, Date.now() + 60_000);
await expect(
agentService(db).create(scope.companyId, createInput(scope), {
createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), {
claudeLogin: { storedSessionId: sessionId, ownerUserId: "intruder" },
}),
).rejects.toMatchObject({ message: CLAUDE_OAUTH_CLAIM_REJECTED });
@@ -409,7 +410,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
const scope = await seedScope();
const sessionId = await seedStoredClaim(scope, Date.now() - 1_000);
await expect(
agentService(db).create(scope.companyId, createInput(scope), {
createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), {
claudeLogin: { storedSessionId: sessionId, ownerUserId: scope.ownerUserId },
}),
).rejects.toMatchObject({ message: CLAUDE_OAUTH_CLAIM_REJECTED });
@@ -420,7 +421,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
const scope = await seedScope();
const sessionId = await seedStoredClaim(scope, Date.now() + 60_000, "awaiting_code");
await expect(
agentService(db).create(scope.companyId, createInput(scope), {
createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), {
claudeLogin: { storedSessionId: sessionId, ownerUserId: scope.ownerUserId },
}),
).rejects.toMatchObject({ message: CLAUDE_OAUTH_CLAIM_REJECTED });
@@ -432,8 +433,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
// Missing claim.
const missingScope = await seedScope();
messages.push(
await agentService(db)
.create(missingScope.companyId, createInput(missingScope), {
await createAgentLifecycle(db).requestHire(missingScope.companyId, createInput(missingScope), {
claudeLogin: { storedSessionId: undefined, ownerUserId: missingScope.ownerUserId },
})
.then(() => "no-error")
@@ -443,8 +443,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
const foreignScope = await seedScope();
const foreignSession = await seedStoredClaim(foreignScope, Date.now() + 60_000);
messages.push(
await agentService(db)
.create(foreignScope.companyId, createInput(foreignScope), {
await createAgentLifecycle(db).requestHire(foreignScope.companyId, createInput(foreignScope), {
claudeLogin: { storedSessionId: foreignSession, ownerUserId: "intruder" },
})
.then(() => "no-error")
@@ -454,8 +453,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
const expiredScope = await seedScope();
const expiredSession = await seedStoredClaim(expiredScope, Date.now() - 1_000);
messages.push(
await agentService(db)
.create(expiredScope.companyId, createInput(expiredScope), {
await createAgentLifecycle(db).requestHire(expiredScope.companyId, createInput(expiredScope), {
claudeLogin: { storedSessionId: expiredSession, ownerUserId: expiredScope.ownerUserId },
})
.then(() => "no-error")
@@ -465,8 +463,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
const nonStoredScope = await seedScope();
const nonStoredSession = await seedStoredClaim(nonStoredScope, Date.now() + 60_000, "submitting");
messages.push(
await agentService(db)
.create(nonStoredScope.companyId, createInput(nonStoredScope), {
await createAgentLifecycle(db).requestHire(nonStoredScope.companyId, createInput(nonStoredScope), {
claudeLogin: { storedSessionId: nonStoredSession, ownerUserId: nonStoredScope.ownerUserId },
})
.then(() => "no-error")
@@ -475,12 +472,11 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
// Already consumed.
const consumedScope = await seedScope();
const consumedSession = await seedStoredClaim(consumedScope, Date.now() + 60_000);
await agentService(db).create(consumedScope.companyId, createInput(consumedScope), {
await createAgentLifecycle(db).requestHire(consumedScope.companyId, createInput(consumedScope), {
claudeLogin: { storedSessionId: consumedSession, ownerUserId: consumedScope.ownerUserId },
});
messages.push(
await agentService(db)
.create(consumedScope.companyId, createInput(consumedScope), {
await createAgentLifecycle(db).requestHire(consumedScope.companyId, createInput(consumedScope), {
claudeLogin: { storedSessionId: consumedSession, ownerUserId: consumedScope.ownerUserId },
})
.then(() => "no-error")
@@ -495,10 +491,10 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
const sessionId = await seedStoredClaim(scope, Date.now() + 60_000);
const results = await Promise.allSettled([
agentService(db).create(scope.companyId, createInput(scope), {
createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), {
claudeLogin: { storedSessionId: sessionId, ownerUserId: scope.ownerUserId },
}),
agentService(db).create(scope.companyId, createInput(scope), {
createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), {
claudeLogin: { storedSessionId: sessionId, ownerUserId: scope.ownerUserId },
}),
]);
@@ -536,8 +532,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
});
await locked;
const createPromise = agentService(db)
.create(scope.companyId, createInput(scope), {
const createPromise = createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), {
claudeLogin: { storedSessionId: sessionId, ownerUserId: scope.ownerUserId },
})
.then(() => "created")
@@ -555,7 +550,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
it("removes the fixed binding on a normal update", async () => {
const scope = await seedScope();
const sessionId = await seedStoredClaim(scope, Date.now() + 60_000);
const created = await agentService(db).create(scope.companyId, createInput(scope), {
const created = await createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), {
claudeLogin: { storedSessionId: sessionId, ownerUserId: scope.ownerUserId },
});
@@ -568,7 +563,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
it("re-points the binding to a plain value on a normal update", async () => {
const scope = await seedScope();
const sessionId = await seedStoredClaim(scope, Date.now() + 60_000);
const created = await agentService(db).create(scope.companyId, createInput(scope), {
const created = await createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), {
claudeLogin: { storedSessionId: sessionId, ownerUserId: scope.ownerUserId },
});
@@ -583,7 +578,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
it("removes the binding by a move to another adapter type on a normal update", async () => {
const scope = await seedScope();
const sessionId = await seedStoredClaim(scope, Date.now() + 60_000);
const created = await agentService(db).create(scope.companyId, createInput(scope), {
const created = await createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), {
claudeLogin: { storedSessionId: sessionId, ownerUserId: scope.ownerUserId },
});
@@ -603,7 +598,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
it("keeps an existing binding when the update changes another field", async () => {
const scope = await seedScope();
const sessionId = await seedStoredClaim(scope, Date.now() + 60_000);
const created = await agentService(db).create(scope.companyId, createInput(scope), {
const created = await createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), {
claudeLogin: { storedSessionId: sessionId, ownerUserId: scope.ownerUserId },
});
@@ -616,7 +611,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
it("rejects a newly introduced binding on update, because it carries no claim", async () => {
const scope = await seedScope();
const created = await agentService(db).create(scope.companyId, {
const created = await createAgentLifecycle(db).requestHire(scope.companyId, {
name: "Api Key Agent",
role: "engineer",
status: "idle",
@@ -645,7 +640,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
const scope = await seedScope();
await seedStoredOwnerValue(scope);
const created = await agentService(db).create(scope.companyId, createInput(scope), {
const created = await createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), {
claudeLogin: { ownerUserId: scope.ownerUserId, applyExistingWithoutClaim: true },
});
@@ -666,9 +661,9 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
const scope = await seedScope();
await seedStoredOwnerValue(scope);
const created = await agentService(
const created = await createAgentLifecycle(
db,
).create(scope.companyId, { ...createInput(scope), status: "pending_approval" }, {
).requestHire(scope.companyId, { ...createInput(scope), status: "pending_approval" }, {
claudeLogin: { ownerUserId: scope.ownerUserId, applyExistingWithoutClaim: true },
});
@@ -680,7 +675,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
it("binds the fixed reference on an update from a stored owner value with no claim", async () => {
const scope = await seedScope();
await seedStoredOwnerValue(scope);
const created = await agentService(db).create(scope.companyId, {
const created = await createAgentLifecycle(db).requestHire(scope.companyId, {
name: "Claude Agent",
role: "engineer",
status: "idle",
@@ -706,7 +701,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
const scope = await seedScope();
await expect(
agentService(db).create(scope.companyId, createInput(scope), {
createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), {
claudeLogin: { ownerUserId: scope.ownerUserId, applyExistingWithoutClaim: true },
}),
).rejects.toMatchObject({ message: CLAUDE_OAUTH_CLAIM_REJECTED });
@@ -723,7 +718,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
// The route sets no owner for an agent actor. The gate rejects the no-claim
// bind, so an agent actor never binds the fixed reference.
await expect(
agentService(db).create(scope.companyId, createInput(scope), {
createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), {
claudeLogin: { ownerUserId: null, applyExistingWithoutClaim: true },
}),
).rejects.toMatchObject({ message: CLAUDE_OAUTH_CLAIM_REJECTED });
@@ -739,7 +734,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
foreignScope.ownerUserId = ownerScope.ownerUserId;
await expect(
agentService(db).create(foreignScope.companyId, createInput(foreignScope), {
createAgentLifecycle(db).requestHire(foreignScope.companyId, createInput(foreignScope), {
claudeLogin: { ownerUserId: foreignScope.ownerUserId, applyExistingWithoutClaim: true },
}),
).rejects.toMatchObject({ message: CLAUDE_OAUTH_CLAIM_REJECTED });
@@ -753,7 +748,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
// The apply-existing flag is set, but the config carries no fixed binding. The
// gate mints the fixed reference only when the client presents the exact fixed
// binding, so the flag alone binds nothing and creates no fixed definition.
const created = await agentService(db).create(
const created = await createAgentLifecycle(db).requestHire(
scope.companyId,
{
name: "Plain Agent",
@@ -781,7 +776,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
// A client adds a secret id and a foreign owner to the binding. The persist
// normalization keeps only the fixed reference fields, so the extra selectors
// never reach the stored binding. The runtime derives the owner from context.
const created = await agentService(db).create(
const created = await createAgentLifecycle(db).requestHire(
scope.companyId,
createInput(scope, {
CLAUDE_CODE_OAUTH_TOKEN: {
@@ -804,7 +799,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
it("resolves the bound reference to the stored value without leaking it in a log", async () => {
const scope = await seedScope();
await seedStoredOwnerValue(scope, "sk-secret-resolve");
const created = await agentService(db).create(scope.companyId, createInput(scope), {
const created = await createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), {
claudeLogin: { ownerUserId: scope.ownerUserId, applyExistingWithoutClaim: true },
});
@@ -858,7 +853,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
const scope = await seedScope();
const parent = await seedParentAgent(scope);
const created = await agentService(db).create(scope.companyId, createInput(scope), {
const created = await createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), {
claudeLogin: { inheritedFromAgentId: parent.id },
});
@@ -871,7 +866,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
const scope = await seedScope();
const parent = await seedParentAgent(scope, { version: 5 });
const created = await agentService(db).create(
const created = await createAgentLifecycle(db).requestHire(
scope.companyId,
createInput(scope, { CLAUDE_CODE_OAUTH_TOKEN: { ...FIXED_BINDING, version: 5 } }),
{ claudeLogin: { inheritedFromAgentId: parent.id } },
@@ -890,7 +885,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
const parent = await seedParentAgent(scope, { version: 5 });
await expect(
agentService(db).create(
createAgentLifecycle(db).requestHire(
scope.companyId,
createInput(scope, { CLAUDE_CODE_OAUTH_TOKEN: { ...FIXED_BINDING, version: 2 } }),
{ claudeLogin: { inheritedFromAgentId: parent.id } },
@@ -906,7 +901,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
const parent = await seedParentAgent(scope, { holdsFixedBinding: false });
await expect(
agentService(db).create(scope.companyId, createInput(scope), {
createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), {
claudeLogin: { inheritedFromAgentId: parent.id },
}),
).rejects.toMatchObject({ message: CLAUDE_OAUTH_CLAIM_REJECTED });
@@ -921,7 +916,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
const parent = await seedParentAgent(foreignScope);
await expect(
agentService(db).create(scope.companyId, createInput(scope), {
createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), {
claudeLogin: { inheritedFromAgentId: parent.id },
}),
).rejects.toMatchObject({ message: CLAUDE_OAUTH_CLAIM_REJECTED });
@@ -934,7 +929,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
const scope = await seedScope();
await expect(
agentService(db).create(scope.companyId, createInput(scope), {
createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), {
claudeLogin: { inheritedFromAgentId: randomUUID() },
}),
).rejects.toMatchObject({ message: CLAUDE_OAUTH_CLAIM_REJECTED });
@@ -950,7 +945,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
const parent = await seedParentAgent(scope, { adapterType: "codex_local" });
await expect(
agentService(db).create(scope.companyId, createInput(scope), {
createAgentLifecycle(db).requestHire(scope.companyId, createInput(scope), {
claudeLogin: { inheritedFromAgentId: parent.id },
}),
).rejects.toMatchObject({ message: CLAUDE_OAUTH_CLAIM_REJECTED });
@@ -988,8 +983,7 @@ describeEmbeddedPostgres("agent service Claude OAuth binding claim", () => {
// The child's copied reference still names version 5, the version the
// route read before the rotation started. The create call must wait for
// the parent row lock, so it can only proceed once the rotation commits.
const createPromise = agentService(db)
.create(
const createPromise = createAgentLifecycle(db).requestHire(
scope.companyId,
createInput(scope, { CLAUDE_CODE_OAUTH_TOKEN: { ...FIXED_BINDING, version: 5 } }),
{ claudeLogin: { inheritedFromAgentId: parent.id } },
@@ -1,3 +1,4 @@
import { createAgentLifecycle } from "../services/agent-lifecycle.js";
import { randomUUID } from "node:crypto";
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
import { eq } from "drizzle-orm";
@@ -66,8 +67,9 @@ describeEmbeddedPostgres("pending approval agent config integrity", () => {
it("freezes generic pending hire config and reapplies the approval snapshot on activation", async () => {
const companyId = await seedCompany();
const agentSvc = agentService(db);
const agentSvcLifecycle = createAgentLifecycle(db);
const approvalSvc = approvalService(db);
const pending = await agentSvc.create(companyId, {
const pending = await agentSvcLifecycle.requestHire(companyId, {
name: "Pending Coder",
role: "engineer",
title: "Software Engineer",
@@ -145,7 +147,8 @@ describeEmbeddedPostgres("pending approval agent config integrity", () => {
await approvalSvc.approve(approval.id, "board-user", "Approved generic hire");
await expect(agentSvc.getById(pending.id)).resolves.toMatchObject({
status: "idle",
status: "paused",
lifecycleState: "preparing",
appearance: pending.appearance,
name: "Pending Coder",
role: "engineer",
@@ -1,3 +1,4 @@
import { createAgentLifecycle } from "../services/agent-lifecycle.js";
import { randomUUID } from "node:crypto";
import { mkdirSync, rmSync } from "node:fs";
import os from "node:os";
@@ -86,7 +87,7 @@ describeEmbeddedPostgres("agent service secret binding sync", () => {
value: "sk-ant-123",
});
const created = await agentService(db).create(companyId, {
const created = await createAgentLifecycle(db).requestHire(companyId, {
name: "Claude Novita",
role: "engineer",
status: "pending_approval",
@@ -137,7 +138,7 @@ describeEmbeddedPostgres("agent service secret binding sync", () => {
value: "slack-test-token",
});
const created = await agentService(db).create(companyId, {
const created = await createAgentLifecycle(db).requestHire(companyId, {
name: "Slack Briefing",
role: "briefing",
adapterType: "codex_local",
@@ -185,7 +186,7 @@ describeEmbeddedPostgres("agent service secret binding sync", () => {
});
await expect(
agentService(db).create(companyId, {
createAgentLifecycle(db).requestHire(companyId, {
name: "Unlisted Static Lease",
role: "engineer",
adapterType: "codex_local",
@@ -220,7 +221,7 @@ describeEmbeddedPostgres("agent service secret binding sync", () => {
const companyId = await seedCompany();
const literalApiKey = `hermes-key-${randomUUID()}`;
const created = await agentService(db).create(companyId, {
const created = await createAgentLifecycle(db).requestHire(companyId, {
name: "Hermes Gateway",
role: "engineer",
status: "idle",
@@ -289,7 +290,7 @@ describeEmbeddedPostgres("agent service secret binding sync", () => {
value: "next-value",
});
const created = await agentService(db).create(companyId, {
const created = await createAgentLifecycle(db).requestHire(companyId, {
name: "Binding Swapper",
role: "engineer",
adapterType: "codex_local",
@@ -359,13 +360,14 @@ describeEmbeddedPostgres("agent service secret binding sync", () => {
.where(eq(companySecretBindings.targetId, agentId));
expect(beforeBindings).toHaveLength(0);
const approved = await agentService(db).activatePendingApproval(agentId);
const approved = await createAgentLifecycle(db).approveHire(agentId);
expect(approved).toMatchObject({
activated: true,
agent: {
id: agentId,
status: "idle",
status: "paused",
lifecycleState: "preparing",
},
});
@@ -390,7 +392,7 @@ describeEmbeddedPostgres("agent service secret binding sync", () => {
const missingSecretId = randomUUID();
await expect(
agentService(db).create(companyId, {
createAgentLifecycle(db).requestHire(companyId, {
name: "Broken Create",
role: "engineer",
adapterType: "claude_local",
@@ -420,7 +422,7 @@ describeEmbeddedPostgres("agent service secret binding sync", () => {
provider: "local_encrypted",
value: "valid-value",
});
const created = await agentService(db).create(companyId, {
const created = await createAgentLifecycle(db).requestHire(companyId, {
name: "Transactional Update",
role: "engineer",
adapterType: "codex_local",
@@ -483,7 +485,7 @@ describeEmbeddedPostgres("agent service secret binding sync", () => {
permissions: {},
});
await expect(agentService(db).activatePendingApproval(agentId)).rejects.toBeTruthy();
await expect(createAgentLifecycle(db).approveHire(agentId)).rejects.toBeTruthy();
const reloaded = await agentService(db).getById(agentId);
expect(reloaded?.status).toBe("pending_approval");
@@ -4,7 +4,7 @@ import { mkdtemp, rm } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { and, eq } from "drizzle-orm";
import { createDb, companies, agents, companyMemberships, connectionGrants, plugins, aiConnectionRouterCursors, aiConnectionTaskPins, toolConnections, companySecrets, heartbeatRuns, issues } from "@paperclipai/db";
import { createDb, closeRegisteredClients, companies, agents, companyMemberships, connectionGrants, plugins, aiConnectionRouterCursors, aiConnectionTaskPins, toolConnections, companySecrets, heartbeatRuns, issues } from "@paperclipai/db";
import { startEmbeddedPostgresTestDatabase } from "@paperclipai/db/test-embedded-postgres";
import type { AiConnectionPoolMember, AiConnectionRouterRequest, AiConnectionRouterResult, PaperclipPluginManifestV1 } from "@paperclipai/shared";
import { aiConnectionRouterService, AiConnectionPoolExhausted, poolMemberRuntimeConfig, applyAiConnectionRouterTaskSettings } from "../services/ai-connection-router.js";
@@ -24,8 +24,16 @@ import { heartbeatService, buildEffectiveRunSessionConfigMetadata, resolveTaskSe
import { connectionIntentService } from "../services/connection-intents.js";
import { legacyExecutionNeedsReconciliation } from "../services/legacy-execution-recovery.js";
import { getServerAdapter, registerServerAdapter, unregisterServerAdapter } from "../adapters/index.js";
import { waitForPendingRunFailureReports } from "../services/run-failure-report.js";
const captureRunFailure = vi.hoisted(() => vi.fn());
vi.mock("../sentry.js", async (importOriginal) => ({
...await importOriginal<typeof import("../sentry.js")>(),
captureRunFailure,
}));
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
const externalTestDatabaseUrl = process.env.PAPERCLIP_TEST_DATABASE_URL?.trim();
let db: ReturnType<typeof createDb>;
let home: string;
const companyId = randomUUID(), otherCompanyId = randomUUID(), agentId = randomUUID(), secondAgentId = randomUUID(), pluginId = randomUUID();
@@ -46,7 +54,8 @@ const makePool = async (config = {}) => service().save(pluginKey, { companyId, c
beforeAll(async () => {
home = await mkdtemp(path.join(os.tmpdir(), "paperclip-router-tests-"));
vi.stubEnv("PAPERCLIP_HOME", home); vi.stubEnv("PAPERCLIP_INSTANCE_ID", "router-fixture");
database = await startEmbeddedPostgresTestDatabase("paperclip-router-db-"); db = createDb(database.connectionString);
if (externalTestDatabaseUrl) db = createDb(externalTestDatabaseUrl);
else { database = await startEmbeddedPostgresTestDatabase("paperclip-router-db-"); db = createDb(database.connectionString); }
await db.insert(companies).values([{ id: companyId, name: "Router fixture", issuePrefix: "RTF" }, { id: otherCompanyId, name: "Other", issuePrefix: "RTO" }]);
await db.insert(agents).values([agentId, secondAgentId].map(id => ({ id, companyId, name: "Router agent", adapterType: "paperclip_runner" })));
await db.insert(companyMemberships).values(["alice", "bob"].map(principalId => ({ companyId, principalId, principalType: "user", status: "active", membershipRole: "member" })));
@@ -59,7 +68,12 @@ beforeAll(async () => {
{ id: randomUUID(), binding: { mode: "delegated", provider: "anthropic", method: "subscription", ...claude }, profile: { provider: "acpx", acpxAgent: "claude", model: "claude-sonnet-5" } },
];
}, 90000);
afterAll(async () => { await database?.cleanup(); vi.unstubAllEnvs(); if (home) await rm(home, { recursive: true, force: true }); });
afterAll(async () => {
await waitForPendingRunFailureReports();
if (externalTestDatabaseUrl) await closeRegisteredClients(externalTestDatabaseUrl);
await db?.$client.end();
await database?.cleanup(); vi.unstubAllEnvs(); if (home) await rm(home, { recursive: true, force: true });
});
describe("durable, authorized connection routing", () => {
it("defaults off and does not expose selectable pools before opt-in", async () => {
@@ -321,16 +335,48 @@ describe("durable, authorized connection routing", () => {
expect(run).not.toBeNull();
await expect.poll(async () => (await heartbeat.getRun(run!.id))?.status, { timeout: 20_000 }).toBe("cancelled");
const cancelled = await heartbeat.getRun(run!.id); expect(cancelled?.errorCode).toBe("ai_connection_pool_exhausted");
expect(cancelled?.resultJson).toMatchObject({
executionRecovery: { kind: "ai_connection_wait", providerWorkStarted: false },
cancellation: {
source: "control_plane", expected: true, initiator: { type: "system" },
reason: "Waiting for an AI connection pool account",
recordedAt: cancelled!.finishedAt!.toISOString(),
},
});
expect(legacyExecutionNeedsReconciliation(cancelled!)).toBe(false);
expect(legacyExecutionNeedsReconciliation({ ...cancelled!, resultJson: { executionRecovery: { kind: "ai_connection_wait", providerWorkStarted: true } } })).toBe(true);
await expect.poll(async () => (await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.agentId, agent!.id), eq(heartbeatRuns.scheduledRetryReason, "ai_connection_pool_wait")))).length, { timeout: 5000 }).toBe(1);
const [retry] = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.agentId, agent!.id), eq(heartbeatRuns.scheduledRetryReason, "ai_connection_pool_wait")));
await waitForPendingRunFailureReports();
expect(captureRunFailure.mock.calls.filter(([report]) => report.runId === run!.id)).toHaveLength(0);
expect(retry?.resultJson?.cancellation).toBeUndefined();
expect(retry?.contextSnapshot?.aiRouterTaskKey).toBe(issue!.id); expect(retry?.contextSnapshot?.executionRetryAccounting).toMatchObject({ failureRetries: 0 }); expect(execute).not.toHaveBeenCalled();
const [pin] = await db.select().from(aiConnectionTaskPins).where(and(eq(aiConnectionTaskPins.poolId, pool.id), eq(aiConnectionTaskPins.agentId, agent!.id))); expect(pin?.selection).toEqual(pinned);
const [cursor] = await db.select().from(aiConnectionRouterCursors).where(eq(aiConnectionRouterCursors.poolId, pool.id)); expect(cursor?.version).toBe(1);
await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date() }).where(eq(heartbeatRuns.agentId, agent!.id));
} finally { fetch.mockRestore(); proposal.mockReset(); proposal.mockImplementation(defaultProposal); unregisterServerAdapter("claude_local"); await heartbeat.drainActiveRunExecutions(); }
}, 30_000);
it("keeps unexpected pool policy failures reportable before provider work", async () => {
await instanceSettingsService(db).updateExperimental({ enableAiConnectionRouters: true, enableWorktreeRunExecution: true });
const pool = await makePool({ members: [members[1]!] });
const [agent] = await db.insert(agents).values({ companyId, name: "Unexpected router failure", adapterType: "claude_local", adapterConfig: { cwd: home, engine: "cli" }, runtimeConfig: { aiConnection: { mode: "router", connectionId: pool.id }, heartbeat: { enabled: false } } }).returning();
const [issue] = await db.insert(issues).values({ companyId, title: "Router failure fixture", status: "todo", assigneeAgentId: agent!.id, responsibleUserId: "alice", createdByUserId: "alice" }).returning();
const execute = vi.fn(); registerServerAdapter({ ...getServerAdapter("claude_local"), execute });
proposal.mockRejectedValueOnce(new Error("Synthetic unexpected router policy failure"));
const heartbeat = heartbeatService(db, { pluginWorkerManager: worker, runtimeEnv: {} });
try {
const run = await heartbeat.wakeup(agent!.id, { source: "assignment", reason: "issue_assigned", payload: { issueId: issue!.id }, contextSnapshot: { issueId: issue!.id, wakeReason: "issue_assigned", responsibleUserId: "alice" }, triggerDetail: "system" });
expect(run).not.toBeNull();
await expect.poll(async () => (await heartbeat.getRun(run!.id))?.status, { timeout: 20_000 }).toBe("failed");
const failed = await heartbeat.getRun(run!.id);
expect(failed?.errorCode).toBe("configuration_incomplete");
expect(failed?.resultJson).toMatchObject({ configurationIncomplete: { reason: "ai_connection_unavailable" } });
expect(failed?.resultJson?.cancellation).toBeUndefined();
expect(execute).not.toHaveBeenCalled();
await expect.poll(() => captureRunFailure.mock.calls.filter(([report]) => report.runId === run!.id)).toHaveLength(1);
expect(captureRunFailure).toHaveBeenCalledWith(expect.objectContaining({ runId: run!.id, runStatus: "failed", errorCode: "configuration_incomplete" }));
} finally { proposal.mockReset(); proposal.mockImplementation(defaultProposal); unregisterServerAdapter("claude_local"); await heartbeat.drainActiveRunExecutions(); }
}, 30_000);
it("repairs the failed pool account without replacing the pool or adopting the configured provider", async () => {
const pool = await makePool({ members: [members[1]!] });
const [agent] = await db.insert(agents).values({ companyId, name: "Pool repair", adapterType: "paperclip_runner", adapterConfig: { provider: "codex", model: "gpt-5.6-sol" }, runtimeConfig: { aiConnection: { mode: "router", connectionId: pool.id } } }).returning();
+66 -4
View File
@@ -2,6 +2,9 @@ import { connectionIntentService } from "../services/connection-intents.js";
import { connectionIntentDeliveryService } from "../services/connection-intent-delivery.js";
import { issueRecoveryActionService } from "../services/issue-recovery-actions.js";
import { localAiLoginService } from "../services/local-ai-login.js";
// Background plan observations have their own provider-fixture coverage. Runtime
// auth tests must not send the selected fixture credentials to a real provider.
vi.mock("../services/subscription-refresh.js", () => ({ refreshSubscriptionConnection: vi.fn().mockResolvedValue(undefined) }));
import * as localCredentials from "../services/local-ai-credentials.js";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { createHash, randomUUID } from "node:crypto";
@@ -16,6 +19,7 @@ import { createDb, companies, agents, agentTaskSessions, heartbeatRuns, companyM
import { startEmbeddedPostgresTestDatabase } from "@paperclipai/db/test-embedded-postgres";
import { fetchCompanyQuotaWindows } from "../services/quota-windows.js";
import { quotaCredentialRecoveryPath, quotaCredentialRecovery, quotaCredentialHash } from "../services/quota-credential-recovery.js";
import { readAiConnectionConfigurationFailure } from "../services/ai-connection-configuration-failure.js";
import { aiConnectionService } from "../services/ai-connections.js";
import { syncConnectionCredentialBindings } from "../services/connection-credential-bindings.js";
import * as codexAdapter from "@paperclipai/adapter-codex-local/server";
@@ -60,6 +64,25 @@ beforeAll(async () => {
afterAll(async () => { await database?.cleanup(); vi.unstubAllEnvs(); if (home) await rm(home, { recursive: true, force: true }); });
describe("managed AI connections", () => {
it("marks explicit AI selection rejections without changing their HTTP contract", async () => {
const owner = "selection-blocker-owner";
await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" });
const expectSelectionFailure = async (promise: Promise<unknown>, reason: string) => {
const error = await promise.catch((error: unknown) => error);
expect(error).toMatchObject({ status: 422, details: { code: reason } });
expect(readAiConnectionConfigurationFailure(error)).toBe(reason);
};
await expectSelectionFailure(service.select({ ...input, userId: null }), "ai_connection_responsible_user_missing");
await expectSelectionFailure(service.select({ ...input, userId: owner, adapterType: "codex_local" }), "ai_connection_incompatible");
await expectSelectionFailure(service.select({ ...input, userId: owner }), "ai_connection_default_missing");
await expectSelectionFailure(service.select({ ...input, userId: owner, binding: {
...binding, mode: "delegated", connectionId: randomUUID(), grantId: randomUUID(),
} }), "ai_connection_missing");
const account = await create(owner, "Selection blocker account");
await db.update(connectionGrants).set({ status: "revoked" }).where(eq(connectionGrants.id, account.grantId));
await expectSelectionFailure(service.select({ ...input, userId: owner }), "ai_connection_unavailable");
});
it("persists first-time recovery directories before consuming a single-use token", async () => {
const owner = "quota-durable-path";
await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" });
@@ -1210,7 +1233,10 @@ console.log(JSON.stringify({ type: "result", subtype: "success", result: "hello"
it("uses human access for every selection; an agent delegation cannot override Just me", async () => {
const personal = await service.select({ ...input, userId: "alice" });
const delegated = { ...binding, mode: "delegated" as const, connectionId: personal.connection.id, grantId: personal.grant.id };
await expect(service.select({ ...input, userId: "bob", binding: delegated })).rejects.toThrow("not shared");
const rejection = await service.select({ ...input, userId: "bob", binding: delegated }).catch(error => error);
expect(rejection).toMatchObject({ status: 403, message: "This credential is not shared with the responsible user" });
expect(rejection.details).toBeUndefined();
expect(readAiConnectionConfigurationFailure(rejection)).toBe("ai_connection_credential_not_shared");
// Existing delegation records no longer confer an independent AI permission.
await db.insert(connectionGrantDelegations).values({ companyId, grantId: personal.grant.id, agentId, createdByUserId: "alice" });
await expect(service.select({ ...input, userId: "bob", binding: delegated })).rejects.toThrow("not shared");
@@ -1223,7 +1249,9 @@ console.log(JSON.stringify({ type: "result", subtype: "success", result: "hello"
const sharedBinding = { ...binding, mode: "shared" as const, ...shared };
const tools = toolAccessService(db);
await tools.replaceConnectionGrantMembers(shared.connectionId, shared.grantId, ["alice"], { userId: "alice" });
await expect(service.select({ ...input, userId: "bob", binding: sharedBinding })).rejects.toThrow("not shared");
const rejection = await service.select({ ...input, userId: "bob", binding: sharedBinding }).catch(error => error);
expect(rejection).toMatchObject({ status: 403, message: "This credential is not shared with the responsible user" });
expect(readAiConnectionConfigurationFailure(rejection)).toBe("ai_connection_credential_not_shared");
expect((await service.list(companyId, "bob", agentId)).some(account => account.id === shared.connectionId)).toBe(false);
await tools.replaceConnectionGrantMembers(shared.connectionId, shared.grantId, ["bob"], { userId: "alice" });
expect((await service.select({ ...input, userId: "bob", binding: sharedBinding })).grant.id).toBe(shared.grantId);
@@ -1233,12 +1261,43 @@ console.log(JSON.stringify({ type: "result", subtype: "success", result: "hello"
for (const userId of ["alice", "bob"]) {
expect((await service.select({ ...input, userId, binding: sharedBinding })).grant.id).toBe(shared.grantId);
}
await expect(service.select({ ...input, userId: null, binding: sharedBinding })).rejects.toThrow("not shared");
const missingIdentity = await service.select({ ...input, userId: null, binding: sharedBinding }).catch(error => error);
expect(missingIdentity).toMatchObject({ status: 403, message: "This credential is not shared with the responsible user" });
expect(readAiConnectionConfigurationFailure(missingIdentity)).toBeNull();
const inactiveMember = await service.select({ ...input, userId: "inactive-user", binding: sharedBinding }).catch(error => error);
expect(inactiveMember).toMatchObject({ status: 403, message: "The responsible user is not an active company member" });
expect(readAiConnectionConfigurationFailure(inactiveMember)).toBeNull();
// Human permission still cannot bypass the separate agent-access setting.
await db.delete(toolConnectionInstalls).where(eq(toolConnectionInstalls.connectionId, shared.connectionId));
await expect(service.select({ ...input, userId: "bob", binding: sharedBinding })).rejects.toThrow("not permitted for this agent");
const missingAgentPermission = await service.select({ ...input, userId: "bob", binding: sharedBinding }).catch(error => error);
expect(missingAgentPermission).toMatchObject({ status: 403 });
expect(missingAgentPermission.message).toContain("not permitted for this agent");
expect(readAiConnectionConfigurationFailure(missingAgentPermission)).toBeNull();
expect(sharedBinding).toEqual({ ...binding, mode: "shared", ...shared });
});
it.each(["inactive_owner", "wrong_binding_kind", "wrong_company_membership"] as const)(
"does not classify other credential authorization denials as sharing configuration: %s", async (cause) => {
const owner = `sharing-${cause}-owner`;
await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" });
const account = await create(owner, "Sharing permission fixture");
const selected = { ...binding, mode: cause === "wrong_binding_kind" ? "shared" as const : "delegated" as const, ...account };
let userId = "bob";
if (cause === "inactive_owner") {
await db.update(companyMemberships).set({ status: "inactive" }).where(and(eq(companyMemberships.companyId, companyId), eq(companyMemberships.principalId, owner)));
}
if (cause === "wrong_company_membership") {
userId = "other-company-sharing-user";
await db.insert(companyMemberships).values({ companyId: otherCompanyId, principalId: userId, principalType: "user", status: "active", membershipRole: "member" });
}
const error = await service.select({ ...input, userId, binding: selected }).catch(error => error);
expect(error).toMatchObject({ status: 403, message: cause === "inactive_owner"
? "The account owner is no longer an active company member"
: cause === "wrong_binding_kind" ? "Select a company-shared account"
: "The responsible user is not an active company member" });
expect(readAiConnectionConfigurationFailure(error)).toBeNull();
},
);
it("isolates concurrent homes and overrides ambient credentials without changing the model", async () => {
vi.stubEnv("ANTHROPIC_API_KEY", "ambient-never-use");
const config = { model: "unchanged-model", env: { ANTHROPIC_API_KEY: "project-never-use" } };
@@ -1351,6 +1410,9 @@ console.log(JSON.stringify({ type: "result", subtype: "success", result: "hello"
try {
expect(child.identity).toBe(parent.identity);
expect(child.attribution.grantId).toBe(account.grantId);
expect(parent.attribution.subscriptionId).toEqual(expect.any(String));
expect(child.attribution.subscriptionId).toBe(parent.attribution.subscriptionId);
expect(child.config.managedAiConnection).toMatchObject({ subscriptionId: parent.attribution.subscriptionId });
} finally {
await Promise.all([parent.cleanup(), child.cleanup()]);
}
@@ -2,21 +2,23 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
import { approvalService } from "../services/approvals.ts";
import { companies } from "@paperclipai/db";
vi.mock("../services/budgets.js", () => ({
vi.mock("../services/budgets.js", async importOriginal => ({
...await importOriginal<typeof import("../services/budgets.js")>(),
budgetService: () => ({ deliverPendingEnforcement: vi.fn(async () => {}) }),
budgetServiceInTransaction: () => ({ upsertPolicy: vi.fn(async () => {}) }),
}));
const mockAgentService = vi.hoisted(() => ({
activatePendingApproval: vi.fn(),
getById: vi.fn(async () => ({ id: "agent-1" })),
create: vi.fn(),
terminate: vi.fn(),
}));
const mockNotifyHireApproved = vi.hoisted(() => vi.fn());
vi.mock("../services/agents.js", () => ({
agentService: vi.fn(() => mockAgentService),
vi.mock("../modules/agent-lifecycle/adapters/records.js", () => ({
agentRecords: vi.fn(() => ({ ...mockAgentService, rejectPendingHire: mockAgentService.terminate })),
}));
vi.mock("../services/hire-hook.js", () => ({
@@ -114,7 +116,7 @@ describe("approvalService resolution idempotency", () => {
const result = await svc.approve("approval-1", "board", "ship it");
expect(result.applied).toBe(true);
expect(mockAgentService.activatePendingApproval).toHaveBeenCalledWith("agent-1", approved.payload);
expect(mockAgentService.activatePendingApproval).toHaveBeenCalledWith("agent-1", approved.payload, approved.requestedByUserId);
expect(mockNotifyHireApproved).toHaveBeenCalledTimes(1);
});
@@ -161,7 +163,7 @@ describe("approvalService resolution idempotency", () => {
expect.objectContaining({
adapterConfig: approved.payload.adapterConfig,
}),
{ createdByUserId: expectedCreator },
{ createdByUserId: expectedCreator, responsibleUserId: requestedByUserId },
);
});
});
@@ -1,3 +1,4 @@
import { createAgentLifecycle } from "../services/agent-lifecycle.js";
import { randomUUID } from "node:crypto";
import express from "express";
import request from "supertest";
@@ -795,8 +796,8 @@ describeEmbeddedPostgres("attention service", () => {
},
]);
await agentService(db).pause(pausedReviewerId);
await agentService(db).terminate(terminatedReviewerId);
await createAgentLifecycle(db).pauseAgent(pausedReviewerId);
await createAgentLifecycle(db).terminateAgent(terminatedReviewerId);
const feed = await attentionService(db).list(companyId, { userId: "board-user" });
const interactionTitles = feed.items
@@ -886,7 +887,7 @@ describeEmbeddedPostgres("attention service", () => {
effectiveResolverPolicy: "human_only",
payload: { version: 1, questions: [] },
});
await agentService(db).pause(reviewerId);
await createAgentLifecycle(db).pauseAgent(reviewerId);
const feed = await attentionService(db).list(companyId, { userId: "board-user" });
const otherUserFeed = await attentionService(db).list(companyId, { userId: "other-user" });
+67 -16
View File
@@ -1,3 +1,4 @@
import { createAgentLifecycle, configureAgentLifecycle } from "../services/agent-lifecycle.js";
import fs from "node:fs/promises";
import { fileHash } from "../services/agent-file-store.js";
import { stockHash } from "../services/managed-resource-drift.js";
@@ -9,6 +10,9 @@ import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest
import { and, eq, sql } from "drizzle-orm";
import {
activityLog,
heartbeatRuns,
agentWakeupRequests,
routineRuns,
agentConfigRevisions,
agents,
approvals,
@@ -35,14 +39,14 @@ import { agentInstructionsService } from "../services/agent-instructions.ts";
import { agentService } from "../services/agents.ts";
import { approvalService } from "../services/approvals.ts";
import {
builtInAgentService,
builtInAgentService as createBuiltInAgentService,
deriveBuiltInAgentStatus,
listBuiltInAgentDefinitions,
readBuiltInTextWithFallback,
reconcileBuiltInAgentsOnStartup,
validateBuiltInAgentDefinitions,
} from "../services/built-in-agents.ts";
import { readBuiltInAgentMarker, withBuiltInAgentMarker } from "../services/built-in-agent-metadata.ts";
import { readBuiltInAgentMarker, withBuiltInAgentMarker } from "../lib/built-in-agent-metadata.ts";
import { issueThreadInteractionService } from "../services/issue-thread-interactions.ts";
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
@@ -119,6 +123,22 @@ describe("built-in agent asset loading", () => {
describeEmbeddedPostgres("built-in agents", () => {
let db!: ReturnType<typeof createDb>;
let lifecycleWorker: ReturnType<typeof configureAgentLifecycle>;
function builtInAgentService(database: typeof db) {
const service = createBuiltInAgentService(database);
return { ...service,
async ensure(...args: Parameters<typeof service.ensure>) {
const result = await service.ensure(...args);
if (result.agentId) await lifecycleWorker.process(result.agentId);
return service.get(args[0], args[1]);
},
async get(...args: Parameters<typeof service.get>) {
const result = await service.get(...args);
if (result.agentId) await lifecycleWorker.process(result.agentId);
return service.get(...args);
},
};
}
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
const instructionOperator = { type: "board", userId: "local-board", source: "local_implicit" } as const;
async function writeInstructionEntry(agent: { id: string; companyId: string }, entryFile: string, content: string) {
@@ -134,9 +154,11 @@ describeEmbeddedPostgres("built-in agents", () => {
beforeAll(async () => {
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-built-in-agents-");
db = createDb(tempDb.connectionString);
lifecycleWorker = configureAgentLifecycle(db, { requiredPluginIds: async () => [], runPlugin: async () => "complete", runHost: async () => "complete" });
}, 20_000);
afterEach(async () => {
await lifecycleWorker.sweep();
await db.delete(routineTriggers);
await db.delete(routines);
await db.delete(issueThreadInteractions);
@@ -158,6 +180,7 @@ describeEmbeddedPostgres("built-in agents", () => {
});
afterAll(async () => {
await lifecycleWorker.stop();
await tempDb?.cleanup();
});
@@ -498,7 +521,7 @@ describeEmbeddedPostgres("built-in agents", () => {
});
expect(ready.status).toBe("ready");
await agentService(db).pause(ready.agentId!, "manual");
await createAgentLifecycle(db).pauseAgent(ready.agentId!, "manual");
await expect(builtIns.get(companyId, "learning")).resolves.toMatchObject({
status: "paused",
agentId: ready.agentId,
@@ -540,7 +563,7 @@ describeEmbeddedPostgres("built-in agents", () => {
warning: null,
});
await agentService(db).pause(ready.agentId!, "maintenance");
await createAgentLifecycle(db).pauseAgent(ready.agentId!, "maintenance");
await expect(builtIns.requireBuiltInAgent(companyId, "briefs")).resolves.toMatchObject({
agent: { id: ready.agentId },
warning: {
@@ -585,7 +608,7 @@ describeEmbeddedPostgres("built-in agents", () => {
it("reconciles an enabled Reflection Coach bundle with skill sync and a disabled routine", async () => {
const companyId = await seedCompany({ requireApproval: false });
const root = await agentService(db).create(companyId, {
const root = await createAgentLifecycle(db).requestHire(companyId, {
name: "CEO",
role: "ceo",
status: "idle",
@@ -677,7 +700,7 @@ describeEmbeddedPostgres("built-in agents", () => {
it("recreates missing managed resource bindings idempotently during concurrent reconcile", async () => {
const companyId = await seedCompany({ requireApproval: false });
await agentService(db).create(companyId, {
await createAgentLifecycle(db).requestHire(companyId, {
name: "CEO",
role: "ceo",
status: "idle",
@@ -715,7 +738,7 @@ describeEmbeddedPostgres("built-in agents", () => {
it("preserves new-agent approval gates during on-demand Reflection Coach provisioning", async () => {
const companyId = await seedCompany({ requireApproval: true });
const root = await agentService(db).create(companyId, {
const root = await createAgentLifecycle(db).requestHire(companyId, {
name: "CEO",
role: "ceo",
status: "idle",
@@ -959,7 +982,7 @@ describeEmbeddedPostgres("built-in agents", () => {
it("prevents direct marker add, remove, or mutation", async () => {
const companyId = await seedCompany();
const builtIn = await builtInAgentService(db).ensure(companyId, "briefs");
const normal = await agentService(db).create(companyId, {
const normal = await createAgentLifecycle(db).requestHire(companyId, {
name: "Normal",
role: "engineer",
status: "idle",
@@ -969,7 +992,7 @@ describeEmbeddedPostgres("built-in agents", () => {
permissions: {},
});
await expect(agentService(db).create(companyId, {
await expect(createAgentLifecycle(db).requestHire(companyId, {
name: "Spoof",
role: "engineer",
status: "idle",
@@ -1222,7 +1245,7 @@ describeEmbeddedPostgres("built-in agents", () => {
it("automatically materializes the Reflection Coach bundle without enabling background work", async () => {
const companyId = await seedCompany();
const root = await agentService(db).create(companyId, {
const root = await createAgentLifecycle(db).requestHire(companyId, {
name: "CEO",
role: "ceo",
status: "idle",
@@ -1297,7 +1320,7 @@ describeEmbeddedPostgres("built-in agents", () => {
it("materializes the Summarizer bundle paused on Claude Haiku with a disabled routine", async () => {
const companyId = await seedCompany();
const root = await agentService(db).create(companyId, {
const root = await createAgentLifecycle(db).requestHire(companyId, {
name: "CEO",
role: "ceo",
status: "idle",
@@ -1409,9 +1432,35 @@ describeEmbeddedPostgres("built-in agents", () => {
expect(resetFile.content).not.toContain("Operator edit.");
});
it("keeps a built-in routine task while the agent prepares", async () => {
const companyId = await seedCompany({ requireApproval: false });
const builtIns = createBuiltInAgentService(db);
const created = await builtIns.ensure(companyId, "reflection-coach");
await lifecycleWorker.stop();
lifecycleWorker = configureAgentLifecycle(db, { requiredPluginIds: async () => [], runPlugin: async () => "complete", runHost: async () => "pending" });
try {
const result = await builtIns.runRoutine(companyId, "reflection-coach", "recent-agent-reflection", { userId: "responsible-user" });
expect(result.status).toBe("issue_created");
const tasks = await db.select().from(issues).where(eq(issues.companyId, companyId));
expect(tasks).toHaveLength(1);
expect(tasks[0]!.assigneeAgentId).toBe(created.agentId);
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId)))
.toEqual([expect.objectContaining({ status: "queued", agentId: created.agentId })]);
// A second request during setup must not try to resume an already preparing agent.
await expect(builtIns.runRoutine(companyId, "reflection-coach", "recent-agent-reflection", { userId: "responsible-user" }))
.resolves.toHaveProperty("id");
} finally {
await lifecycleWorker.stop();
await db.delete(routineRuns).where(eq(routineRuns.companyId, companyId));
await db.delete(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId));
await db.delete(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId));
lifecycleWorker = configureAgentLifecycle(db, { requiredPluginIds: async () => [], runPlugin: async () => "complete", runHost: async () => "complete" });
}
});
it("controls the Reflection Coach routine schedule without enabling it by default", async () => {
const companyId = await seedCompany();
await agentService(db).create(companyId, {
await createAgentLifecycle(db).requestHire(companyId, {
name: "CEO",
role: "ceo",
status: "idle",
@@ -1434,7 +1483,8 @@ describeEmbeddedPostgres("built-in agents", () => {
"recent-agent-reflection",
{ userId: "board-user" },
);
expect(enabled.status).toBe("needs_setup");
await lifecycleWorker.process(enabled.agentId!);
expect((await builtIns.get(companyId, "reflection-coach")).status).toBe("needs_setup");
expect(enabled.resources.find((resource) => resource.resourceKind === "routine")).toMatchObject({
stockStatus: "stock_current",
scheduleEnabled: true,
@@ -1462,7 +1512,7 @@ describeEmbeddedPostgres("built-in agents", () => {
it("surfaces pending Reflection Coach proposal interactions on the routine resource", async () => {
const companyId = await seedCompany();
await agentService(db).create(companyId, {
await createAgentLifecycle(db).requestHire(companyId, {
name: "CEO",
role: "ceo",
status: "idle",
@@ -1515,7 +1565,8 @@ describeEmbeddedPostgres("built-in agents", () => {
it("gates Reflection Coach proposal mutations until an accepted follow-up apply step", async () => {
const companyId = await seedCompany();
const agentsSvc = agentService(db);
await agentsSvc.create(companyId, {
const agentsSvcLifecycle = createAgentLifecycle(db);
await agentsSvcLifecycle.requestHire(companyId, {
name: "CEO",
role: "ceo",
status: "idle",
@@ -1524,7 +1575,7 @@ describeEmbeddedPostgres("built-in agents", () => {
runtimeConfig: {},
permissions: {},
});
const target = await agentsSvc.create(companyId, {
const target = await agentsSvcLifecycle.requestHire(companyId, {
name: "Target Coder",
role: "engineer",
status: "idle",
File diff suppressed because it is too large. Load diff
@@ -1,3 +1,5 @@
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
import { subscribeDeliveryWork, DELIVERY_QUEUES } from "../services/delivery-work-notifications.js";
import { randomUUID } from "node:crypto";
import { and, asc, eq } from "drizzle-orm";
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
@@ -7,7 +9,7 @@ import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } fro
import { settleInterruptedNativeBootstrap, terminalizeLegacyExecution } from "../services/legacy-execution-recovery.js";
import { getExecutionBlocker } from "../services/execution-blocker.js";
import { settleUnrecoverableExecutions } from "../services/execution-recovery-resolution.js";
import { issueService } from "../services/issues.js";
import { executeIssuePostCommitActions, type IssuePostCommitAction, issueService } from "../services/issues.js";
import { buildLowTrustSourceTrust } from "../services/source-trust.js";
import { documentService } from "../services/documents.js";
import { instanceSettingsService } from "../services/instance-settings.js";
@@ -51,6 +53,57 @@ const support = await getEmbeddedPostgresTestSupport();
};
return { companyId, agentId, sourceId, runId, task, create, finish, rows, due, wakeup, service, run };
}
it("dispatches a committed completion without activity publication and recovers after restart", async () => {
const f = await seed();
const run = vi.fn(() => f.service.sweepPending());
const makeWorker = () => {
const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => { throw error; } });
const worker = coordinator.register(DELIVERY_QUEUES.chatCompletion, { retryMs: 60_000, run, hasPending: f.service.hasPending });
return { ...worker, stop: coordinator.stop };
};
const worker = makeWorker();
try {
await worker.ready;
expect(await f.service.hasPending()).toBe(false);
expect(run).toHaveBeenCalledTimes(1);
await f.finish(); // Service mutation alone: no route activity callback.
await vi.waitFor(() => expect(f.wakeup).toHaveBeenCalledTimes(1));
expect(await f.service.hasPending()).toBe(true); // queued receipt still owed
} finally { await worker.stop(); }
// Admit the first reporting turn; later completions need a new turn.
await f.run();
// A second task commits while no consumer is registered.
const next = await f.create();
await f.finish(next.id);
const restarted = makeWorker();
try {
await restarted.ready;
expect(f.wakeup).toHaveBeenCalledTimes(2);
} finally { await restarted.stop(); }
});
it("automatically notifies after a caller-owned transaction commits", async () => {
const f = await seed();
const notified = vi.fn();
const unsubscribe = subscribeDeliveryWork(db, DELIVERY_QUEUES.chatCompletion, notified);
try {
const actions: IssuePostCommitAction[] = [];
await db.transaction(async tx => {
await issueService(db).update(f.task.id, { status: "done" }, tx, [], actions);
expect(notified).not.toHaveBeenCalled();
});
expect(notified).toHaveBeenCalledTimes(1);
expect(actions).toEqual([]);
await executeIssuePostCommitActions(db, actions);
expect(notified).toHaveBeenCalledTimes(1);
const rolledBack: IssuePostCommitAction[] = [];
await expect(db.transaction(async tx => {
await issueService(db).update(f.task.id, { status: "todo" }, tx, [], rolledBack);
throw new Error("rollback");
})).rejects.toThrow("rollback");
expect(notified).toHaveBeenCalledTimes(1);
expect((await f.rows())[0]?.status).toBe("pending");
} finally { unsubscribe(); }
});
it("records authenticated origins and atomically creates one event per Done transition", async () => {
const f = await seed();
expect(await db.select().from(handoffs).where(eq(handoffs.taskId, f.task.id))).toMatchObject([{ conversationId: f.sourceId, sessionGeneration: 0 }]);
@@ -1,3 +1,4 @@
import { createAgentLifecycle, configureAgentLifecycle } from "../services/agent-lifecycle.js";
import { randomUUID } from "node:crypto";
import { eq } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
@@ -41,6 +42,7 @@ describeEmbeddedPostgres("cleanup removal services", () => {
beforeAll(async () => {
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-cleanup-removal-");
db = createDb(tempDb.connectionString);
configureAgentLifecycle(db, { requiredPluginIds: async () => [], runPlugin: async () => "complete", runHost: async () => "complete" });
}, 20_000);
afterEach(async () => {
@@ -146,6 +148,7 @@ describeEmbeddedPostgres("cleanup removal services", () => {
createdByRunId: runId,
});
await db.update(agents).set({ status: "terminated", lifecycleState: "terminated" }).where(eq(agents.id, agentId));
const removed = await agentService(db).remove(agentId);
expect(removed?.id).toBe(agentId);
+4 -1
View File
@@ -91,7 +91,7 @@ async function createApp(actor: any, db: any = {} as any) {
}
describe("cli auth routes", () => {
beforeEach(() => {
beforeEach(async () => {
vi.resetModules();
vi.doUnmock("../services/index.js");
vi.doUnmock("../routes/authz.js");
@@ -99,6 +99,9 @@ describe("cli auth routes", () => {
vi.doUnmock("../middleware/index.js");
registerModuleMocks();
vi.resetAllMocks();
// Cold route transforms are fixture setup, not HTTP request latency.
// Use the hook budget while preserving the request tests' normal timeout.
await Promise.all([import("../routes/access.js"), import("../middleware/index.js")]);
});
it("creates a CLI auth challenge with approval metadata", async () => {
@@ -3,6 +3,7 @@ import express from "express";
import request from "supertest";
import { afterEach, describe, expect, it, vi } from "vitest";
import type { Db } from "@paperclipai/db";
import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js";
import { cloudWarmStandbyMiddleware, cloudWarmStandbyServerOptions } from "../middleware/cloud-warm-standby.js";
import { healthRoutes } from "../routes/health.js";
import { emailChannelService } from "../services/email-channels.js";
@@ -124,7 +125,7 @@ describe("unclaimed Cloud background work", () => {
vi.useFakeTimers();
let enabled = false;
const select = vi.fn(() => { throw new Error("SQL probe"); });
const db = { select } as unknown as Db;
const db = installDatabaseWorkSignals({ select, execute: vi.fn(), transaction: vi.fn() }) as unknown as Db;
const email = emailChannelService(db, { heartbeat: { wakeup: vi.fn() }, isBackgroundWorkEnabled: () => enabled });
const scheduler = createPluginJobScheduler({
db,
+24 -10
View File
@@ -33,9 +33,10 @@ vi.mock("../services/cloud-lifecycle-sync.js", () => ({
notifyCloudOfPrimaryCompanyLifecycleChange: notifyCloudSpy,
}));
import { configureAgentLifecycle } from "../services/agent-lifecycle.js";
import { companyService } from "../services/companies.js";
import { deriveIssuePrefixBase } from "../services/issue-prefix.js";
import { readBuiltInAgentMarker } from "../services/built-in-agent-metadata.js";
import { readBuiltInAgentMarker } from "../lib/built-in-agent-metadata.js";
import { builtInAgentService, reconcileBuiltInAgentsOnStartup } from "../services/built-in-agents.js";
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
@@ -49,6 +50,16 @@ if (!embeddedPostgresSupport.supported) {
describeEmbeddedPostgres("companyService", () => {
let db!: ReturnType<typeof createDb>;
let lifecycleWorker: ReturnType<typeof configureAgentLifecycle>;
beforeEach(() => { lifecycleWorker = configureAgentLifecycle(db, { requiredPluginIds: async () => [], runPlugin: async () => "complete", runHost: async () => "complete" }); });
async function insertAgents(input: typeof agents.$inferInsert | Array<typeof agents.$inferInsert>) {
const rows = Array.isArray(input) ? input : [input];
await db.insert(agents).values(rows.map(row => ({ ...row,
lifecycleState: ["paused", "pending_approval", "terminated"].includes(row.status ?? "") ? row.status! : "ready",
lifecycleHolds: row.status === "paused" ? [row.pauseReason ?? "manual"] : [],
})));
}
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
beforeAll(async () => {
@@ -57,6 +68,7 @@ describeEmbeddedPostgres("companyService", () => {
}, 20_000);
afterEach(async () => {
await lifecycleWorker.stop();
await db.delete(routineTriggers);
await db.delete(routines);
await db.delete(builtInManagedResources);
@@ -165,7 +177,7 @@ describeEmbeddedPostgres("companyService", () => {
requireBoardApprovalForNewAgents: false,
});
await db.insert(agents).values([
await insertAgents([
{
id: runningAgentId,
companyId,
@@ -335,7 +347,7 @@ describeEmbeddedPostgres("companyService", () => {
requireBoardApprovalForNewAgents: false,
});
await db.insert(agents).values([
await insertAgents([
{
id: archivedPausedAgentId,
companyId,
@@ -382,6 +394,7 @@ describeEmbeddedPostgres("companyService", () => {
);
expect(reactivated?.status).toBe("active");
await lifecycleWorker.process(archivedPausedAgentId);
const reactivateActivity = await db
.select({
@@ -441,7 +454,7 @@ describeEmbeddedPostgres("companyService", () => {
requireBoardApprovalForNewAgents: false,
});
await db.insert(agents).values([
await insertAgents([
{
id: runningAgentId,
companyId,
@@ -551,7 +564,7 @@ describeEmbeddedPostgres("companyService", () => {
requireBoardApprovalForNewAgents: false,
});
await db.insert(agents).values([
await insertAgents([
{
id: archivedPausedAgentId,
companyId,
@@ -587,6 +600,7 @@ describeEmbeddedPostgres("companyService", () => {
);
expect(reactivated?.status).toBe("active");
await lifecycleWorker.process(archivedPausedAgentId);
const rows = await db
.select({ id: agents.id, status: agents.status, pauseReason: agents.pauseReason })
@@ -618,7 +632,7 @@ describeEmbeddedPostgres("companyService", () => {
requireBoardApprovalForNewAgents: false,
});
await db.insert(agents).values({
await insertAgents({
id: terminatedAgentId,
companyId,
name: "Terminated Agent",
@@ -661,7 +675,7 @@ describeEmbeddedPostgres("companyService", () => {
requireBoardApprovalForNewAgents: false,
});
await db.insert(agents).values({
await insertAgents({
id: manualPausedAgentId,
companyId,
name: "Manual Paused Agent",
@@ -713,7 +727,7 @@ describeEmbeddedPostgres("companyService", () => {
requireBoardApprovalForNewAgents: false,
});
await db.insert(agents).values({
await insertAgents({
id: agentId,
companyId,
name: "Idle Agent",
@@ -788,7 +802,7 @@ describeEmbeddedPostgres("companyService", () => {
requireBoardApprovalForNewAgents: false,
});
await db.insert(agents).values({
await insertAgents({
id: agentId,
companyId,
name: "Idle Agent",
@@ -863,7 +877,7 @@ describeEmbeddedPostgres("companyService", () => {
requireBoardApprovalForNewAgents: false,
});
await db.insert(agents).values({
await insertAgents({
id: idleAgentId,
companyId,
name: "Idle Agent",
@@ -122,6 +122,13 @@ vi.mock("../services/agents.js", () => ({
agentService: () => agentSvc,
}));
vi.mock("../services/agent-lifecycle.js", () => ({
createAgentLifecycle: () => ({
requestHire: (...args: unknown[]) => agentSvc.create(...args),
pauseAgent: vi.fn(),
}),
}));
vi.mock("../services/access.js", () => ({
accessService: () => accessSvc,
}));
@@ -1727,20 +1734,30 @@ describe("company portability", () => {
expect(exported.warnings.filter((warning) => warning.includes("could not be exported portably"))).toHaveLength(1);
});
it("round-trips the saved visual persona through the portable bundle", async () => {
it.each([undefined, "22222222-2222-4222-8222-222222222222"])("round-trips the preset without private avatar references (%s)", async customAvatarAssetId => {
const appearance = { schemaVersion: 1, characterVersion: "cap-v1", paletteId: "arctic-blue" };
const source = await agentSvc.list();
agentSvc.list.mockResolvedValue(source.map((agent: Record<string, unknown>) => ({ ...agent, appearance })));
agentSvc.list.mockResolvedValue(source.map((agent: Record<string, unknown>) => ({ ...agent, appearance: { ...appearance, ...(customAvatarAssetId ? { customAvatarAssetId } : {}) } })));
const portability = companyPortabilityService({} as any);
const include = { company: true, agents: true, projects: false, issues: false, skills: false };
const exported = await portability.exportBundle("company-1", { include });
expect(asTextFile(exported.files[".paperclip.yaml"])).toContain("arctic-blue");
const extensionText = asTextFile(exported.files[".paperclip.yaml"]);
expect(extensionText).not.toContain("customAvatarAssetId");
if (customAvatarAssetId) {
expect(exported.warnings.some(warning => warning.includes("uploaded avatar was omitted"))).toBe(true);
// Even a hand-edited package must not carry another agent's private asset into import.
exported.files[".paperclip.yaml"] = extensionText.replace(/^(\s*)paletteId:.*$/gm,
match => `${match}\n${match.match(/^\s*/)?.[0]}customAvatarAssetId: ${customAvatarAssetId}`);
expect(asTextFile(exported.files[".paperclip.yaml"])).toContain(customAvatarAssetId);
}
agentSvc.list.mockResolvedValue([]);
agentSvc.create.mockImplementation(async (_companyId: string, input: Record<string, unknown>) => ({ ...input, id: `imported-${input.name}` }));
await portability.importBundle({ source: { type: "inline", files: exported.files, rootPath: exported.rootPath }, include,
target: { mode: "new_company", newCompanyName: "Imported personas" }, collisionStrategy: "rename", agents: "all" }, "user-1");
expect(agentSvc.create).toHaveBeenCalled();
for (const [, input] of agentSvc.create.mock.calls) expect(input).toMatchObject({ appearance });
for (const [, input] of agentSvc.create.mock.calls) expect(input.appearance).toEqual(appearance);
});
it("reads env inputs back from .paperclip.yaml during preview import", async () => {
@@ -1,3 +1,5 @@
import { subscribeDeliveryWork } from "../services/delivery-work-notifications.js";
import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js";
import { randomUUID } from "node:crypto";
import { and, eq, sql } from "drizzle-orm";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
@@ -38,6 +40,7 @@ import { materializeNativeInteractionResponses } from "../services/native-runtim
import { toolAccessService } from "../services/tool-access.js";
import { toolAccessPolicyService } from "../services/tool-access-policy.js";
import { connectionIntentService } from "../services/connection-intents.js";
import { instanceSettingsService } from "../services/instance-settings.js";
import {
getEmbeddedPostgresTestSupport,
startEmbeddedPostgresTestDatabase,
@@ -198,6 +201,29 @@ describeEmbeddedPostgres("connectionIntentService", () => {
expect(serialized).not.toContain(claims.sub);
});
it("discovers GitHub review bots independently from chat connectors", async () => {
const settings = instanceSettingsService(db);
const previous = await settings.getExperimental();
const service = connectionIntentService(db);
try {
await settings.updateExperimental({ enableChatConnectors: false, enableGitHubReviewBots: true });
expect((await service.search(claims, "github")).results).toEqual(expect.arrayContaining([
expect.objectContaining({ service: "github-code-review-bot" }),
expect.objectContaining({ service: "github" }),
]));
expect((await service.search(claims, "slack")).results.some(item =>
item.methods.some(method => method.purpose === "channel"))).toBe(false);
await settings.updateExperimental({ enableChatConnectors: true, enableGitHubReviewBots: false });
expect((await service.search(claims, "github")).results.some(item =>
item.service === "github-code-review-bot")).toBe(false);
} finally {
await settings.updateExperimental({
enableChatConnectors: previous.enableChatConnectors,
enableGitHubReviewBots: previous.enableGitHubReviewBots,
});
}
});
it("creates one addressed request, resolves after install, and then reports ready for the responsible user", async () => {
const service = connectionIntentService(db);
const first = await service.request(claims, "notion");
@@ -773,7 +799,11 @@ describeEmbeddedPostgres("connectionIntentService", () => {
it("recovers a resolution after a failed dispatch and acknowledges an already queued wake exactly once", async () => {
const service = connectionIntentService(db);
const pending = await service.request(claims, "airtable");
const notified = vi.fn();
const unsubscribe = subscribeDeliveryWork(db, DELIVERY_QUEUES.connection, notified);
await service.decline(pending.interactionId!, claims.responsible_user_id);
unsubscribe();
expect(notified).toHaveBeenCalledTimes(1);
const wakeup = vi.fn().mockRejectedValueOnce(new Error("simulated crash before durable enqueue"));
const deliveries = connectionIntentDeliveryService(db, { wakeup } as never);
await expect(deliveries.deliver(pending.interactionId!)).rejects.toThrow("simulated crash");
@@ -537,6 +537,7 @@ describeEmbeddedPostgres("cost and finance aggregate overflow handling", () => {
cachedInputTokens: 10,
outputTokens: 200_000_000,
costCents: 2_000_000_000,
costStatus: "estimated",
occurredAt: new Date("2026-04-11T00:00:00.000Z"),
},
]);
@@ -555,6 +556,11 @@ describeEmbeddedPostgres("cost and finance aggregate overflow handling", () => {
expect(byAgentRow?.inputTokens).toBe(3_999_999_990);
expect(byAgentRow!.inputTokens + byAgentRow!.cachedInputTokens).toBe(4_000_000_000);
expect(byProjectRow?.costCents).toBe(4_000_000_000);
expect(byProjectRow).toMatchObject({ eventCount: 2, estimatedEventCount: 1 });
const [firstDayProject] = await costs.byProject(companyId, {
...range, to: new Date("2026-04-10T23:59:59.999Z"),
});
expect(firstDayProject).toMatchObject({ eventCount: 1, estimatedEventCount: 0 });
expect(byAgentModelRow?.costCents).toBe(4_000_000_000);
});
+5 -1
View File
@@ -189,12 +189,16 @@ describe("company decision service", () => {
const context: DecisionContext = { companyId: f.companyId, feature: settingsDecisionTest,
actor: { type: "agent", agentId: f.agentId, runId, companyId: f.companyId, source: "agent_jwt" } };
const remove = () => agentService(db).remove(f.agentId);
let inFlightDeletionError: unknown;
f.provider.mockImplementationOnce(async () => {
await expect(remove()).rejects.toMatchObject({ status: 409, details: { code: "agent_decision_accounting_pending" } });
// Exercise the accounting guard after lifecycle termination.
await db.update(agents).set({ status: "terminated", lifecycleState: "terminated" }).where(eq(agents.id, f.agentId));
inFlightDeletionError = await remove().catch(error => error);
expect(await f.service.decide(f.context, DECISION_TEST_REQUEST)).toMatchObject({ status: "unavailable", reason: "budget_blocked" });
return { errorCode: "timeout", receipt: decisionReceipt("openai", null) };
});
expect((await f.service.decide(context, DECISION_TEST_REQUEST)).status).toBe("failed");
expect(inFlightDeletionError).toMatchObject({ status: 409, details: { code: "agent_decision_accounting_pending" } });
await expect(remove()).rejects.toMatchObject({ status: 409, details: { code: "agent_decision_accounting_pending" } });
expect((await db.select().from(budgetReservations).where(eq(budgetReservations.companyId, f.companyId)))[0]).toMatchObject({ state: "held", amountCents: "1.0000000" });
});
@@ -0,0 +1,207 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js";
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
import { notifyDeliveryWork, DELIVERY_QUEUES } from "../services/delivery-work-notifications.js";
import { idleWorkSnapshot } from "../services/task-admission.js";
const coordinators: ReturnType<typeof createDeliveryWorkCoordinator>[] = [];
afterEach(async () => { for (const c of coordinators.splice(0)) await c.stop(); vi.useRealTimers(); });
function setup() {
vi.useFakeTimers();
let outcome = "committed";
let rejectCommit = false;
const execute = vi.fn(async () => [{ xid: "42", status: outcome }]);
const owner = installDatabaseWorkSignals({ execute, async transaction(callback: (tx: any) => Promise<any>) {
const result = await callback({ execute, transaction: owner.transaction });
if (rejectCommit) throw new Error("lost commit reply");
return result;
} });
const canRun = vi.fn(() => true), canReconcile = vi.fn(() => false), onError = vi.fn();
const coordinator = createDeliveryWorkCoordinator({ owner, canRun, canReconcile, onError });
coordinators.push(coordinator);
const enqueue = (queue = DELIVERY_QUEUES.feedback as typeof DELIVERY_QUEUES[keyof typeof DELIVERY_QUEUES]) =>
owner.transaction(tx => notifyDeliveryWork(tx, queue));
return { owner, canRun, canReconcile, onError, coordinator, enqueue, execute,
failCommit: () => { rejectCommit = true; outcome = "in progress"; }, settle: (status = "committed") => { outcome = status; } };
}
function task() { return { retryMs: 5000, run: vi.fn(async () => {}), hasPending: vi.fn(async () => false) }; }
describe("delivery work coordinator", () => {
it("leaves empty queues with no deadlines, timers, or periodic queries", async () => {
const s = setup();
const tasks = Object.values(DELIVERY_QUEUES).map(queue => {
const t = task(); return { queue, t, worker: s.coordinator.register(queue, t) };
});
await Promise.all(tasks.map(t => t.worker.ready));
expect(vi.getTimerCount()).toBe(0);
expect(s.coordinator.nextWakeAt()).toBeNull();
await vi.advanceTimersByTimeAsync(3_600_000);
for (const { queue, t } of tasks) {
expect(t.run).toHaveBeenCalledTimes(1);
await s.enqueue(queue); await s.enqueue(queue);
}
await vi.advanceTimersByTimeAsync(1);
for (const { t } of tasks) expect(t.run).toHaveBeenCalledTimes(2);
expect(vi.getTimerCount()).toBe(0);
expect(idleWorkSnapshot().active).toBe(0);
});
it("schedules maintenance without holding idle intent, and lets new work preempt it", async () => {
const s = setup(), t = task();
let next: number | null = Date.now() + 60_000;
await s.coordinator.register(DELIVERY_QUEUES.email, { ...t, nextRunAt: () => next }).ready;
await vi.advanceTimersByTimeAsync(59_000);
expect(t.run).toHaveBeenCalledTimes(1);
await s.enqueue(DELIVERY_QUEUES.email);
await vi.advanceTimersByTimeAsync(1);
expect(t.run).toHaveBeenCalledTimes(2);
expect(idleWorkSnapshot().active).toBe(0);
next = null;
await vi.advanceTimersByTimeAsync(1000);
expect(t.run).toHaveBeenCalledTimes(3);
expect(s.coordinator.nextWakeAt()).toBeNull();
});
it("does not postpone an already committed wake when another producer starts writing", async () => {
const s = setup(), t = task();
await s.coordinator.register(DELIVERY_QUEUES.feedback, t).ready;
await s.enqueue();
let release!: () => void;
const gate = new Promise<void>(resolve => { release = resolve; });
const writing = s.owner.transaction(async tx => {
await notifyDeliveryWork(tx, DELIVERY_QUEUES.feedback);
await gate;
});
await vi.advanceTimersByTimeAsync(1);
expect(t.run).toHaveBeenCalledTimes(2);
release(); await writing;
await vi.advanceTimersByTimeAsync(1);
expect(t.run).toHaveBeenCalledTimes(3);
expect(idleWorkSnapshot().active).toBe(0);
});
it("fences unresolved writes until the server settles, then scans and releases the idle hold", async () => {
const s = setup(), t = task();
await s.coordinator.register(DELIVERY_QUEUES.feedback, t).ready;
s.failCommit();
await expect(s.enqueue()).rejects.toThrow("lost commit reply");
expect(idleWorkSnapshot().active).toBe(1);
await vi.advanceTimersByTimeAsync(1);
expect(t.run).toHaveBeenCalledTimes(2); // Empty scan cannot clear unresolved write.
expect(idleWorkSnapshot().active).toBe(1);
expect(s.coordinator.nextWakeAt()).not.toBeNull();
s.settle();
await vi.advanceTimersByTimeAsync(5001);
expect(t.run.mock.calls.length).toBeGreaterThanOrEqual(3);
expect(idleWorkSnapshot().active).toBe(0);
expect(vi.getTimerCount()).toBe(0);
});
it("retains unresolved intent across coordinator replacement", async () => {
const s = setup(), t = task();
await s.coordinator.register(DELIVERY_QUEUES.feedback, t).ready;
s.failCommit(); await expect(s.enqueue()).rejects.toThrow();
await s.coordinator.stop();
expect(idleWorkSnapshot().active).toBe(0);
const replacement = createDeliveryWorkCoordinator({ owner: s.owner, canRun: () => true, onError: vi.fn() });
coordinators.push(replacement);
await replacement.register(DELIVERY_QUEUES.feedback, t).ready;
expect(idleWorkSnapshot().active).toBe(1);
s.settle(); await vi.advanceTimersByTimeAsync(5001);
expect(idleWorkSnapshot().active).toBe(0);
expect(replacement.nextWakeAt()).toBeNull();
});
it("retries pending deliveries and transient failures, then goes quiet", async () => {
const s = setup(), t = task();
t.run.mockRejectedValueOnce(new Error("database disconnected"));
t.hasPending.mockResolvedValueOnce(true);
await s.coordinator.register(DELIVERY_QUEUES.feedback, t).ready;
expect(s.onError).toHaveBeenCalledTimes(1);
await vi.advanceTimersByTimeAsync(10_000);
expect(t.run).toHaveBeenCalledTimes(3);
expect(s.coordinator.nextWakeAt()).toBeNull();
expect(idleWorkSnapshot().active).toBe(0);
});
it("does not overlap a sweep or lose a commit during its final pending check", async () => {
const s = setup(), t = task();
let release!: (pending: boolean) => void;
t.hasPending.mockImplementationOnce(() => new Promise(resolve => { release = resolve; }));
s.coordinator.register(DELIVERY_QUEUES.connection, t);
await vi.advanceTimersByTimeAsync(1);
await s.enqueue(DELIVERY_QUEUES.connection);
await vi.advanceTimersByTimeAsync(1000);
expect(t.run).toHaveBeenCalledTimes(1);
release(false);
await vi.advanceTimersByTimeAsync(1);
expect(t.run).toHaveBeenCalledTimes(2);
});
it("reconciles a rollback during idle drain and releases the empty-queue hold without dispatching", async () => {
const s = setup(), t = task();
await s.coordinator.register(DELIVERY_QUEUES.feedback, t).ready;
s.canRun.mockReturnValue(false);
s.canReconcile.mockReturnValue(true);
s.failCommit(); await expect(s.enqueue()).rejects.toThrow("lost commit reply");
await vi.advanceTimersByTimeAsync(1);
expect(idleWorkSnapshot().active).toBe(1);
expect(t.run).toHaveBeenCalledTimes(1);
s.settle("aborted");
await vi.advanceTimersByTimeAsync(5001);
expect(t.run).toHaveBeenCalledTimes(1);
expect(t.hasPending).toHaveBeenCalled();
expect(idleWorkSnapshot().active).toBe(0);
expect(s.coordinator.nextWakeAt()).toBeNull();
});
it("keeps committed pending work recoverable during drain without starting deliveries", async () => {
const s = setup(), t = task();
await s.coordinator.register(DELIVERY_QUEUES.feedback, t).ready;
s.canRun.mockReturnValue(false);
s.canReconcile.mockReturnValue(true);
t.hasPending.mockResolvedValue(true);
await s.enqueue();
await vi.advanceTimersByTimeAsync(1);
expect(t.run).toHaveBeenCalledTimes(1);
expect(s.coordinator.nextWakeAt()).not.toBeNull();
s.canRun.mockReturnValue(true);
t.hasPending.mockResolvedValue(false);
await vi.advanceTimersByTimeAsync(5001);
expect(t.run).toHaveBeenCalledTimes(2);
expect(idleWorkSnapshot().active).toBe(0);
});
it("does not query during standby and resumes when admission opens", async () => {
const s = setup(), t = task(); s.canRun.mockReturnValue(false);
await s.coordinator.register(DELIVERY_QUEUES.toolAction, t).ready;
await s.enqueue(DELIVERY_QUEUES.toolAction);
await vi.advanceTimersByTimeAsync(120_000);
expect(t.run).not.toHaveBeenCalled();
expect(t.hasPending).not.toHaveBeenCalled();
s.canRun.mockReturnValue(true);
await vi.advanceTimersByTimeAsync(5001);
expect(t.run).toHaveBeenCalledTimes(1);
expect(idleWorkSnapshot().active).toBe(0);
});
it("awaits in-flight work on shutdown and releases the subscription for restart", async () => {
const s = setup(), t = task();
let release!: () => void;
t.run.mockImplementationOnce(() => new Promise(resolve => { release = resolve; }));
s.coordinator.register(DELIVERY_QUEUES.chatCompletion, t);
await vi.advanceTimersByTimeAsync(1);
const stopped = vi.fn();
const stopping = s.coordinator.stop().then(stopped);
await s.enqueue(DELIVERY_QUEUES.chatCompletion);
expect(stopped).not.toHaveBeenCalled();
release(); await stopping;
expect(vi.getTimerCount()).toBe(0);
expect(idleWorkSnapshot().active).toBe(0);
});
it("cancels an abortable delivery before awaiting shutdown", async () => {
const s = setup();
const hasPending = vi.fn(async () => false);
const run = vi.fn((signal: AbortSignal) => new Promise<void>((_resolve, reject) => {
signal.addEventListener("abort", () => reject(signal.reason), { once: true });
}));
s.coordinator.register(DELIVERY_QUEUES.feedback, { retryMs: 5000, run, hasPending });
await vi.advanceTimersByTimeAsync(1);
await s.coordinator.stop();
expect(run.mock.calls[0]?.[0].aborted).toBe(true);
expect(hasPending).not.toHaveBeenCalled();
expect(s.onError).not.toHaveBeenCalled();
expect(idleWorkSnapshot().active).toBe(0);
expect(vi.getTimerCount()).toBe(0);
});
});
@@ -0,0 +1,158 @@
import { logActivity } from "../services/activity-log.js";
vi.mock("../services/activity-log.js", async importOriginal => {
const actual = await importOriginal<typeof import("../services/activity-log.js")>();
return { ...actual, logActivity: vi.fn(actual.logActivity) };
});
import { randomBytes, randomUUID } from "node:crypto";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import express from "express";
import request from "supertest";
import { and, eq } from "drizzle-orm";
import { afterAll, beforeAll, expect, it, vi } from "vitest";
import { createDb, agents, approvals, principalPermissionGrants, authUsers, companies, companyMemberships, dotAgentBindings, activityLog } from "@paperclipai/db";
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { dotInvitationService } from "../services/dot-invitations.js";
import { dotRunnerBroker } from "../services/dot-runner-broker.js";
import { dotRunnerRoutes } from "../routes/dot-runner.js";
import { instanceSettingsService } from "../services/instance-settings.js";
import { HttpError } from "../errors.js";
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let db: ReturnType<typeof createDb>;
let root: string;
beforeAll(async () => {
temporary = await startEmbeddedPostgresTestDatabase("dot-invitations-");
db = createDb(temporary.connectionString);
root = await mkdtemp(join(tmpdir(), "dot-invite-home-"));
vi.stubEnv("PAPERCLIP_HOME", root);
vi.stubEnv("PAPERCLIP_IN_WORKTREE", "false");
await instanceSettingsService(db).updateExperimental({ enableOpenAiDot: true, enablePublicMcp: true });
}, 120000);
afterAll(async () => { await temporary?.cleanup(); if (root) await rm(root, { recursive: true, force: true }); vi.unstubAllEnvs(); });
async function fixture(requireApproval = false) {
const userId = randomUUID();
await db.insert(authUsers).values({ id: userId, name: "Operator", email: `${userId}@example.test`, createdAt: new Date(), updatedAt: new Date() });
const [company] = await db.insert(companies).values({ name: "Dot invitation", issuePrefix: "DI" + randomBytes(3).toString("hex"), requireBoardApprovalForNewAgents: requireApproval }).returning();
await db.insert(companyMemberships).values({ companyId: company!.id, principalType: "user", principalId: userId, membershipRole: "owner", status: "active" });
await db.insert(principalPermissionGrants).values({ companyId: company!.id, principalType: "user", principalId: userId, permissionKey: "agents:create" });
return { userId, companyId: company!.id };
}
it("atomically resumes one pending invite across retries and never persists a pairing secret", async () => {
const f = await fixture();
const service = dotInvitationService(db);
const [a, b] = await Promise.all([service.create(f.companyId, f.userId), service.create(f.companyId, f.userId)]);
expect(a.agent.id).toBe(b.agent.id);
expect(a.agent.status).toBe("paused");
expect((await db.select().from(agents).where(eq(agents.id, a.agent.id)))[0]!.lifecycleState).toBe("preparing");
expect(await service.resume(f.companyId, f.userId)).toMatchObject({ agent: { id: a.agent.id } });
expect(await service.resume(f.companyId, randomUUID())).toBeNull();
expect(await db.select().from(agents).where(eq(agents.companyId, f.companyId))).toHaveLength(1);
const code = await dotRunnerBroker(db).createPairing({ ...f, operatorId: f.userId, agentId: a.agent.id });
const logs = await db.select().from(activityLog).where(eq(activityLog.companyId, f.companyId));
expect(JSON.stringify(logs)).not.toContain(code.pairingCode);
expect(JSON.stringify(await service.resume(f.companyId, f.userId))).not.toContain(code.pairingCode);
});
it("reuses a committed hire when invitation completion must retry", async () => {
const f = await fixture(true);
const actual = await vi.importActual<typeof import("../services/activity-log.js")>("../services/activity-log.js");
vi.mocked(logActivity).mockImplementation(async (...args) => {
if (args[1].action === "agent.hire_created") {
vi.mocked(logActivity).mockImplementation(actual.logActivity);
throw new Error("Invitation audit unavailable");
}
return actual.logActivity(...args);
});
const service = dotInvitationService(db);
await expect(service.create(f.companyId, f.userId)).rejects.toThrow("Invitation audit unavailable");
const before = await db.select().from(agents).where(eq(agents.companyId, f.companyId));
expect(before).toHaveLength(1);
expect(await service.resume(f.companyId, f.userId)).toBeNull();
const retried = await service.create(f.companyId, f.userId);
expect(retried.agent.id).toBe(before[0]!.id);
expect(await db.select().from(approvals).where(eq(approvals.companyId, f.companyId))).toHaveLength(1);
expect(await db.select().from(agents).where(eq(agents.companyId, f.companyId))).toHaveLength(1);
});
it("honors hiring approval before issuing a pairing capability", async () => {
const f = await fixture(true);
const invite = await dotInvitationService(db).create(f.companyId, f.userId);
expect(invite.agent.status).toBe("pending_approval");
expect(invite.approvalId).toBeTruthy();
const [approval] = await db.select().from(approvals).where(eq(approvals.id, invite.approvalId!));
expect(approval).toMatchObject({ status: "pending", type: "hire_agent", payload: { agentId: invite.agent.id } });
await expect(dotRunnerBroker(db).createPairing({ ...f, operatorId: f.userId, agentId: invite.agent.id })).rejects.toThrow("approved");
expect((await dotInvitationService(db).create(f.companyId, f.userId)).approvalId).toBe(invite.approvalId);
});
it("replaces only the named pending capability and refuses to revoke a connected binding", async () => {
const f = await fixture(); const broker = dotRunnerBroker(db);
const invite = await dotInvitationService(db).create(f.companyId, f.userId);
const input = { companyId: f.companyId, agentId: invite.agent.id, operatorId: f.userId };
const first = await broker.createPairing(input);
const second = await broker.createPairing({ ...input, replaceBindingId: first.bindingId });
expect(second.bindingId).not.toBe(first.bindingId);
const [old] = await db.select().from(dotAgentBindings).where(eq(dotAgentBindings.id, first.bindingId));
expect(old).toMatchObject({ status: "revoked", pairingCodeHash: null });
await expect(broker.createPairing({ ...input, replaceBindingId: first.bindingId })).rejects.toThrow("changed");
await db.update(dotAgentBindings).set({ status: "connected" }).where(eq(dotAgentBindings.id, second.bindingId));
await expect(broker.createPairing({ ...input, replaceBindingId: second.bindingId })).rejects.toThrow("changed");
});
it("keeps invite routes company-scoped and denies viewers and agent actors", async () => {
const f = await fixture();
const app = express(); app.use(express.json());
app.use((req, _res, next) => {
const mode = req.headers["x-test-actor"];
req.actor = mode === "agent" ? { type: "agent", agentId: randomUUID(), companyId: f.companyId }
: { type: "board", source: "session", userId: f.userId, companyIds: [f.companyId],
memberships: [{ companyId: f.companyId, membershipRole: mode === "viewer" ? "viewer" : "owner", status: "active" }] };
next();
});
app.use(dotRunnerRoutes(db, "https://paperclip.example/mcp/runner"));
app.use((error: Error, _req: express.Request, res: express.Response, _next: express.NextFunction) => {
res.status(error instanceof HttpError ? error.status : 500).json({ error: error.message });
});
const path = `/companies/${f.companyId}/dot-invitations`;
for (const actor of ["viewer", "agent"]) expect((await request(app).post(path).set("x-test-actor", actor).send({})).status).toBe(403);
expect((await request(app).post(`/companies/${randomUUID()}/dot-invitations`).send({})).status).toBe(403);
const result = await request(app).post(path).send({});
expect(result.status, JSON.stringify(result.body)).toBe(200);
expect(result.body.agent.id).toBeTruthy();
expect((await request(app).get(path)).body.agent.id).toBe(result.body.agent.id);
expect(await db.select().from(agents).where(and(eq(agents.companyId, f.companyId), eq(agents.lifecycleState, "preparing")))).toHaveLength(1);
});
it("does not trust editable agent metadata as an invitation ownership receipt", async () => {
const f = await fixture();
const [spoof] = await db.insert(agents).values({ companyId: f.companyId, name: "Unrelated agent", adapterType: "paperclip_runner",
adapterConfig: { provider: "openai_dot" }, metadata: { dotInvitation: { operatorId: f.userId } } }).returning();
const service = dotInvitationService(db);
expect(await service.resume(f.companyId, f.userId)).toBeNull();
expect((await service.create(f.companyId, f.userId)).agent.id).not.toBe(spoof!.id);
});
it("reports connection setup authority separately from compatibility status", async () => {
const f = await fixture();
const invite = await dotInvitationService(db).create(f.companyId, f.userId);
const app = express();
app.use((req, _res, next) => {
req.actor = { type: "board", source: "session", userId: f.userId, companyIds: [f.companyId],
memberships: [{ companyId: f.companyId, membershipRole: "owner", status: "active" }] };
next();
});
app.use(dotRunnerRoutes(db, "https://paperclip.example/mcp/runner"));
const path = `/companies/${f.companyId}/agents/${invite.agent.id}/dot-binding`;
const cases = [
{ lifecycleState: "preparing", status: "paused", allowed: true },
{ lifecycleState: "verifying", status: "paused", allowed: true },
{ lifecycleState: "ready", status: "idle", allowed: true },
{ lifecycleState: "paused", status: "paused", allowed: false },
{ lifecycleState: "pending_approval", status: "pending_approval", allowed: false },
{ lifecycleState: "terminated", status: "terminated", allowed: false },
] as const;
for (const state of cases) {
await db.update(agents).set({ lifecycleState: state.lifecycleState, status: state.status }).where(eq(agents.id, invite.agent.id));
const result = await request(app).get(path);
expect(result.status).toBe(200);
expect(result.body).toMatchObject({ agentStatus: state.status, agentLifecycleState: state.lifecycleState,
canConfigureConnection: state.allowed });
}
});
@@ -1,11 +1,12 @@
import { readFile } from "node:fs/promises";
import { createHash, randomBytes, randomUUID } from "node:crypto";
import express from "express";
import request from "supertest";
import { and, eq } from "drizzle-orm";
import { and, eq, sql } from "drizzle-orm";
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { createDb, authUsers, companies, companyMemberships, agents, dotAgentBindings, mcpOauthGrants, mcpOauthRequests } from "@paperclipai/db";
import { createDb, authUsers, companies, companyMemberships, agents, dotAgentBindings, dotMailboxItems, mcpEventSubscriptions, mcpOauthGrants, mcpOauthRequests, mcpOauthTokens } from "@paperclipai/db";
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { createPublicMcpOAuth, publicMcpConfig, DEVICE_GRANT } from "../services/public-mcp/oauth.js";
import { createPublicMcpOAuth, publicMcpConfig, hashMcpSecret, DEVICE_GRANT } from "../services/public-mcp/oauth.js";
import { instanceSettingsService } from "../services/instance-settings.js";
import { dotRunnerBroker } from "../services/dot-runner-broker.js";
import { publicMcpManagementRoutes, publicMcpIngressRoutes } from "../routes/public-mcp.js";
@@ -21,12 +22,12 @@ describe("Dot onboarding with an operator-issued pairing capability", () => {
}, 120000);
beforeEach(async () => { await instanceSettingsService(db).updateExperimental({ enablePublicMcp: true, enableOpenAiDot: true, enableNativeRunner: false }); });
afterAll(async () => { await temp?.cleanup(); });
async function fixture(authorizationOrigin?: string) {
async function fixture(authorizationOrigin?: string, lifecycleState = "ready") {
const userId = randomUUID();
await db.insert(authUsers).values({ id: userId, name: "Operator", email: userId + "@example.test", createdAt: new Date(), updatedAt: new Date() });
const [company] = await db.insert(companies).values({ name: "Dot onboarding", issuePrefix: "DO" + randomBytes(3).toString("hex") }).returning();
await db.insert(companyMemberships).values({ companyId: company!.id, principalType: "user", principalId: userId, membershipRole: "owner", status: "active" });
const [agent] = await db.insert(agents).values({ companyId: company!.id, name: "Dot", adapterType: "paperclip_runner", status: "active" }).returning();
const [agent] = await db.insert(agents).values({ companyId: company!.id, name: "Dot", adapterType: "paperclip_runner", adapterConfig: { provider: "openai_dot" }, lifecycleState, status: lifecycleState === "ready" ? "active" : "paused" }).returning();
const oauth = createPublicMcpOAuth(db, { ...config, ...(authorizationOrigin ? { authorizationOrigin } : {}) });
const client = await oauth.register({ client_name: "Dot", redirect_uris: [callback], grant_types: ["authorization_code", "refresh_token", DEVICE_GRANT] }, randomUUID());
const verifier = randomBytes(32).toString("base64url");
@@ -37,6 +38,115 @@ describe("Dot onboarding with an operator-issued pairing capability", () => {
const id = await begin();
return { userId, company: company!, agent: agent!, oauth, client, verifier, input, pairing, id, begin };
}
async function connect(f: Awaited<ReturnType<typeof fixture>>) {
const consent = await f.oauth.consentDotPairing(f.id, f.pairing.pairingCode);
return f.oauth.token({ grant_type: "authorization_code", client_id: f.client.client_id,
redirect_uri: callback, resource: config.resource, code: new URL(consent.redirectUrl).searchParams.get("code"), code_verifier: f.verifier });
}
it("invalidates setup when a broker pairing is applied or revoked", async () => {
const f = await fixture(undefined, "preparing");
const consent = await f.oauth.consent(f.id, { type: "board", source: "session", userId: f.userId },
{ decision: "approve", companyId: f.company.id, allowWrites: false });
const tokens = await f.oauth.token({ grant_type: "authorization_code", client_id: f.client.client_id,
redirect_uri: callback, resource: config.resource, code: new URL(consent.redirectUrl).searchParams.get("code"), code_verifier: f.verifier });
const principal = await f.oauth.authenticate(tokens.access_token);
const broker = dotRunnerBroker(db);
await broker.pair(principal, f.pairing.pairingCode);
const [paired] = await db.select().from(agents).where(eq(agents.id, f.agent.id));
expect(paired.adapterConfig.dotBindingId).toBe(f.pairing.bindingId);
expect(paired.lifecycleVersion).toBe(f.agent.lifecycleVersion + 1);
await expect(broker.pair(principal, f.pairing.pairingCode)).rejects.toThrow("Pairing code expired or was consumed");
await broker.revoke(f.company.id, f.agent.id, f.userId);
const [revoked] = await db.select().from(agents).where(eq(agents.id, f.agent.id));
expect(revoked.adapterConfig).not.toHaveProperty("dotBindingId");
expect(revoked.lifecycleVersion).toBe(paired.lifecycleVersion + 1);
await expect(f.oauth.authenticate(tokens.access_token)).rejects.toThrow();
});
it.each(["preparing", "verifying"])("allows Dot setup during %s without allowing task access", async (state) => {
const f = await fixture(undefined, state);
const tokens = await connect(f);
const principal = await f.oauth.authenticate(tokens.access_token);
const broker = dotRunnerBroker(db);
await db.insert(mcpEventSubscriptions).values({ id: randomUUID(), companyId: f.company.id,
grantId: principal.grant.id, name: "paperclip.dot.mailbox_updated", bindingId: f.pairing.bindingId,
arguments: {}, deliveryMaterial: {}, verifiedAt: new Date(), expiresAt: new Date(Date.now() + 60_000) });
await db.insert(dotMailboxItems).values({ companyId: f.company.id, bindingId: f.pairing.bindingId,
bindingGeneration: 1, kind: "assignment", sourceEventId: randomUUID(), references: { task: "hidden" } });
await broker.challenge(f.company.id, f.agent.id);
const inbox = await broker.mailbox(principal);
expect(inbox.items.map(item => item.kind)).toEqual(["readiness_challenge"]);
expect(inbox.nextCursor).toBe(0);
await expect(broker.confirmChallenge(principal, String(inbox.items[0]!.references.challenge))).resolves.toEqual({ status: "ready" });
expect(await broker.capabilities(principal)).toMatchObject({ ready: false });
await expect(broker.tasks(principal)).rejects.toThrow("authority");
await expect(broker.snapshot(f.company.id, f.agent.id, f.pairing.bindingId)).rejects.toThrow("authority");
await db.update(agents).set({ lifecycleState: "paused" }).where(eq(agents.id, f.agent.id));
expect((await broker.mailbox(principal)).items).toEqual([]);
await expect(broker.confirmChallenge(principal, "already-used")).rejects.toThrow("authority");
});
it("refreshes a Dot connection after years of inactivity, still rotates and revokes on replay", async () => {
const f = await fixture();
const tokens = await connect(f);
const [stored] = await db.select().from(mcpOauthTokens).where(eq(mcpOauthTokens.tokenHash, hashMcpSecret(tokens.refresh_token!)));
expect(stored!.expiresAt).toBeNull();
const refresh = { grant_type: "refresh_token", client_id: f.client.client_id, resource: config.resource, refresh_token: tokens.refresh_token };
vi.useFakeTimers({ toFake: ["Date"] });
try {
vi.setSystemTime(new Date(Date.now() + 2 * 365 * 24 * 60 * 60_000));
await expect(f.oauth.authenticate(tokens.access_token)).rejects.toThrow();
const next = await f.oauth.token(refresh);
expect(next.refresh_token).not.toBe(tokens.refresh_token);
expect(next.expires_in).toBe(900);
expect((await f.oauth.authenticate(next.access_token)).actor).toMatchObject({ type: "agent", agentId: f.agent.id });
const [rotated] = await db.select().from(mcpOauthTokens).where(eq(mcpOauthTokens.tokenHash, hashMcpSecret(next.refresh_token!)));
expect(rotated!.expiresAt).toBeNull();
await expect(f.oauth.token(refresh)).rejects.toThrow();
await expect(f.oauth.authenticate(next.access_token)).rejects.toThrow();
expect(await dotRunnerBroker(db).bindingForAgent(f.company.id, f.agent.id)).toBeNull();
} finally { vi.useRealTimers(); }
});
it("upgrades valid Dot refresh tokens without reviving expired, used or revoked credentials", async () => {
const f = await fixture();
const tokens = await connect(f);
const grant = (await f.oauth.authenticate(tokens.access_token)).grant;
const validExpiry = new Date(Date.now() + 30 * 24 * 60 * 60_000);
await db.update(mcpOauthTokens).set({ expiresAt: validExpiry }).where(eq(mcpOauthTokens.tokenHash, hashMcpSecret(tokens.refresh_token!)));
const stale = await fixture();
const staleTokens = await connect(stale);
const expired = new Date(0);
await db.update(mcpOauthTokens).set({ expiresAt: expired }).where(eq(mcpOauthTokens.tokenHash, hashMcpSecret(staleTokens.refresh_token!)));
const personalId = randomUUID();
const revokedId = randomUUID();
const usedHash = randomUUID();
await db.insert(mcpOauthGrants).values([
{ ...grant, id: personalId, purpose: "personal", agentId: null, resource: config.origin + "/mcp/paperclip" },
{ ...grant, id: revokedId, revokedAt: new Date() },
]);
await db.insert(mcpOauthTokens).values([
{ grantId: personalId, tokenHash: randomUUID(), kind: "refresh", expiresAt: validExpiry },
{ grantId: revokedId, tokenHash: randomUUID(), kind: "refresh", expiresAt: validExpiry },
{ grantId: grant.id, tokenHash: usedHash, kind: "refresh", expiresAt: validExpiry, usedAt: new Date() },
]);
const migration = await readFile(new URL("../../../packages/db/src/migrations/0319_heavy_captain_midlands.sql", import.meta.url), "utf8");
for (let attempt = 0; attempt < 2; attempt++) {
for (const statement of migration.split("--> statement-breakpoint")) await db.execute(sql.raw(statement));
}
const [personal] = await db.select().from(mcpOauthTokens).where(eq(mcpOauthTokens.grantId, personalId));
const [revoked] = await db.select().from(mcpOauthTokens).where(eq(mcpOauthTokens.grantId, revokedId));
const [used] = await db.select().from(mcpOauthTokens).where(eq(mcpOauthTokens.tokenHash, usedHash));
const [staleToken] = await db.select().from(mcpOauthTokens).where(eq(mcpOauthTokens.tokenHash, hashMcpSecret(staleTokens.refresh_token!)));
expect(personal!.expiresAt).toEqual(validExpiry);
expect(revoked!.expiresAt).toEqual(validExpiry);
expect(used!.expiresAt).toEqual(validExpiry);
expect(staleToken!.expiresAt).toEqual(expired);
await expect(stale.oauth.token({ grant_type: "refresh_token", client_id: stale.client.client_id, resource: config.resource, refresh_token: staleTokens.refresh_token })).rejects.toThrow();
const next = await f.oauth.token({ grant_type: "refresh_token", client_id: f.client.client_id, resource: config.resource, refresh_token: tokens.refresh_token });
expect((await f.oauth.authenticate(next.access_token)).actor).toMatchObject({ type: "agent", agentId: f.agent.id });
await f.oauth.revokeToken(next.access_token, f.client.client_id);
await expect(f.oauth.token({ grant_type: "refresh_token", client_id: f.client.client_id, resource: config.resource, refresh_token: next.refresh_token })).rejects.toThrow();
});
it("pins browser ingress without changing the issuer, resource or token endpoint", async () => {
const browserOrigin = "https://paperclip-browser.example:10000";
const f = await fixture(browserOrigin);
@@ -51,6 +161,7 @@ describe("Dot onboarding with an operator-issued pairing capability", () => {
const authorization = await request(app).get("/mcp/runner/oauth/authorize").query(f.input);
expect(authorization.status).toBe(303);
expect(new URL(authorization.headers.location).origin).toBe(browserOrigin);
expect(new URL(authorization.headers.location).pathname).toMatch(/^\/dot-connect\/pcmcp_request_[A-Za-z0-9_-]{43}$/);
const path = `/api/mcp/requests/${f.id}/dot-pairing`;
expect((await request(app).post(path).set("Origin", "https://other.example").send({ pairingCode: f.pairing.pairingCode })).status).toBe(403);
const approval = await request(app).post(path).set("Origin", browserOrigin).send({ pairingCode: f.pairing.pairingCode });
@@ -65,16 +176,18 @@ describe("Dot onboarding with an operator-issued pairing capability", () => {
for (const invalid of ["http://browser.example", "https://secret@browser.example", "https://browser.example/path", "https://browser.example?next=evil", "https://browser.example#fragment"])
expect(() => publicMcpConfig({ PAPERCLIP_PUBLIC_URL: config.origin, PAPERCLIP_MCP_AUTHORIZATION_ORIGIN: invalid })).toThrow();
});
it("connects without a board session, binds the exact agent, and consumes the capability once", async () => {
it.each(["mcp", "dot-mcp"])("connects via %s without a board session, binds the exact agent, and consumes the capability once", async prefix => {
const f = await fixture();
const personal = createPublicMcpOAuth(db, { ...config, resource: config.origin + "/mcp/paperclip" });
const app = express(); app.use(express.json());
app.use((req, _res, next) => { req.actor = { type: "none" }; next(); });
app.use("/api", publicMcpManagementRoutes(personal, f.oauth));
const path = `/api/mcp/requests/${f.id}/dot-pairing`;
const path = `/api/${prefix}/requests/${f.id}/dot-pairing`;
expect((await request(app).get(`/api/${prefix}/requests/${f.id}`)).body.agentConnection).toBe(true);
expect((await request(app).post(path).send({ pairingCode: f.pairing.pairingCode })).status).toBe(403);
const preview = await request(app).post(path + "/preview").set("Origin", config.origin).send({ pairingCode: f.pairing.pairingCode });
expect(preview.status).toBe(200);
expect(preview.body.accessDuration).toBe("Ongoing until revoked. This connection does not expire from inactivity.");
expect(preview.body).toMatchObject({ company: { id: f.company.id, name: f.company.name }, agent: { id: f.agent.id, name: f.agent.name } });
expect(JSON.stringify(preview.body)).not.toContain(f.pairing.pairingCode);
expect(await db.select().from(mcpOauthGrants).where(eq(mcpOauthGrants.userId, f.userId))).toHaveLength(0);
@@ -97,6 +210,20 @@ describe("Dot onboarding with an operator-issued pairing capability", () => {
await expect(f.oauth.consentDotPairing(f.id, f.pairing.pairingCode)).rejects.toThrow();
await expect(f.oauth.consentDotPairing(await f.begin(), f.pairing.pairingCode)).rejects.toThrow();
});
it("does not expose personal assistant requests through the public Dot pairing lane", async () => {
const f = await fixture();
const personal = createPublicMcpOAuth(db, { ...config, resource: config.origin + "/mcp/paperclip" });
const client = await personal.register({ client_name: "Personal", redirect_uris: [callback], grant_types: ["authorization_code"] }, randomUUID());
const id = (await personal.authorize({ ...f.input, client_id: client.client_id, resource: config.origin + "/mcp/paperclip", scope: "paperclip:read" })).split("/").at(-1)!;
const app = express(); app.use(express.json());
app.use((req, _res, next) => { req.actor = { type: "none" }; next(); });
app.use("/api", publicMcpManagementRoutes(personal, f.oauth));
expect((await request(app).get(`/api/dot-mcp/requests/${id}`)).status).toBe(404);
for (const suffix of ["", "/preview"]) {
expect((await request(app).post(`/api/dot-mcp/requests/${id}/dot-pairing${suffix}`).set("Origin", config.origin).send({ pairingCode: f.pairing.pairingCode })).status).toBe(404);
}
expect((await request(app).get(`/api/mcp/requests/${id}`)).status).toBe(200);
});
it.each(["expired-code", "expired-request", "revoked", "paused", "viewer", "wrong-company", "dot-disabled", "mcp-disabled"])("rejects %s without creating a grant", async failure => {
const f = await fixture();
let id = f.id;
+358 -115
View File
@@ -10,7 +10,7 @@ import express, { type Request } from "express";
import request from "supertest";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { agents, authUsers, companies, companyMemberships, createDb, dotAgentBindings, dotMailboxItems, dotRunnerAssignments, dotRunnerOperations,
heartbeatRuns, agentWakeupRequests, issueComments, issues, nativeRunResults, nativeRunFinalizations, completionContracts, mcpEventDeliveries, workspaceOperations, workAssessments, statusDecisions, issueThreadInteractions } from "@paperclipai/db";
heartbeatRuns, agentWakeupRequests, issueComments, issues, nativeRunResults, nativeRunFinalizations, completionContracts, mcpEventDeliveries, mcpEventSubscriptions, workspaceOperations, workAssessments, statusDecisions, issueThreadInteractions } from "@paperclipai/db";
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { createPublicMcpOAuth, DEVICE_GRANT } from "../services/public-mcp/oauth.js";
import { createPublicMcpExecutor } from "../services/public-mcp/capabilities.js";
@@ -30,6 +30,9 @@ import { setupRunnerPrpWebSocketServer } from "../realtime/runner-prp-ws.js";
import { finalizeNativeRun } from "../services/native-runtime/native-run-finalizer.js";
import { resolveHeartbeatNativeRuntimeMode, resolveNativeRuntimeMode } from "../services/native-runtime/runtime-mode.js";
import { heartbeatService } from "../services/heartbeat.js";
import { consumeDotHistoryReceipt, publishActiveDotComment } from "../services/dot-assignment-follow-up.js";
import { createPostgresWakeQueueAdapter } from "../modules/wake-queue/adapters/postgres.js";
import { createReleaseIssueExecution } from "../modules/wake-queue/application/use-cases.js";
describe("durable Dot Runner integration", () => {
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
@@ -37,7 +40,7 @@ describe("durable Dot Runner integration", () => {
let root: string;
beforeAll(async () => {
const runnerRoot = fileURLToPath(new URL("../../../packages/paperclip-runner/", import.meta.url));
execFileSync("cargo", ["build", "--release", "--locked", "--manifest-path", join(runnerRoot, "runner/Cargo.toml"), "-p", "paperclip-runner-core", "--bin", "paperclip-runnerd"], { cwd: runnerRoot, stdio: "pipe", timeout: 300000 });
execFileSync("cargo", ["build", "--release", "--locked", "--manifest-path", join(runnerRoot, "runner/Cargo.toml"), "-p", "paperclip-runner-core", "--bin", "paperclip-runnerd"], { cwd: runnerRoot, stdio: "inherit", timeout: 300000 });
temporary = await startEmbeddedPostgresTestDatabase("paperclip-dot-runner-");
db = createDb(temporary.connectionString);
root = await mkdtemp(join(tmpdir(), "paperclip-dot-state-"));
@@ -47,7 +50,17 @@ describe("durable Dot Runner integration", () => {
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", randomBytes(32).toString("base64"));
await instanceSettingsService(db).updateExperimental({ enablePublicMcp: true, enableOpenAiDot: true, enableNativeRunner: false });
}, 360000);
afterAll(async () => { await temporary?.cleanup(); await rm(root, { recursive: true, force: true }); vi.unstubAllEnvs(); });
afterAll(async () => {
try {
await temporary?.cleanup();
} finally {
try {
if (root) await rm(root, { recursive: true, force: true });
} finally {
vi.unstubAllEnvs();
}
}
});
it("requires each persisted prerequisite for pairing and new work without relying on the retired environment flag", async () => {
const settings = instanceSettingsService(db);
@@ -84,7 +97,7 @@ describe("durable Dot Runner integration", () => {
})).toMatchObject({ kind: "native", profile: { backend: "openai_dot_mcp" } });
});
async function fixture() {
async function fixture(options: { cloudOrigin?: string } = {}) {
const userId = randomUUID();
await db.insert(authUsers).values({ id: userId, name: "Dot operator", email: userId + "@example.test", createdAt: new Date(), updatedAt: new Date() });
const [company] = await db.insert(companies).values({ name: "Dot test", issuePrefix: "DT" + randomBytes(3).toString("hex") }).returning();
@@ -122,10 +135,10 @@ describe("durable Dot Runner integration", () => {
if (body.type === "verification") { verifications++; return Response.json({ challenge: body.challenge }); }
received.push(body); return new Response(null, { status: 204 });
};
const events = createPublicMcpEvents(db, oauth, async () => { throw new Error("personal API dispatch forbidden"); }, { enableDotRunner: true, fetch: fetcher });
const events = createPublicMcpEvents(db, oauth, async () => { throw new Error("personal API dispatch forbidden"); }, { enableDotRunner: true, fetch: fetcher, ...options });
const subscription = { name: "paperclip.dot.mailbox_updated", arguments: { companyId: company!.id, bindingId: pairing.bindingId }, delivery: { mode: "webhook", url: "https://example.com/dot-hook", secret } };
await events.subscribe(principal, subscription);
await broker.challenge(company!.id, agent!.id); await events.tick();
await events.tick();
if (!received.length) throw new Error("Dot readiness delivery missing: " + JSON.stringify(await db.select({ outcome: mcpEventDeliveries.outcome, event: mcpEventDeliveries.event }).from(mcpEventDeliveries)));
expect(received.at(-1)?.data.kind).toBe("readiness_challenge");
expect(received.at(-1)?.data).not.toHaveProperty("challenge");
@@ -136,6 +149,14 @@ describe("durable Dot Runner integration", () => {
return { company: company!, agent: agent!, userId, broker, principal, oauth, events, subscription, received, snapshot, tokens, client, verifications: () => verifications };
}
it("keeps scoped Dot events on tenant OAuth when the Cloud personal broker is configured", async () => {
const f = await fixture({ cloudOrigin: "https://cloud-broker.example" });
expect(f.verifications()).toBe(1);
expect(f.received.at(-1)?.data.kind).toBe("readiness_challenge");
await f.broker.revoke(f.company.id, f.agent.id, f.userId);
await expect(f.events.subscribe(f.principal, f.subscription)).rejects.toThrow();
}, 30000);
async function offeredWork(f: Awaited<ReturnType<typeof fixture>>) {
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id,
status: "queued", invocationSource: "assignment", triggerDetail: "system" }).returning();
@@ -168,6 +189,25 @@ describe("durable Dot Runner integration", () => {
}
});
it.each(["revoked binding", "removed subscription"])("handles an automatic event-test race with a %s", async change => {
const f = await fixture();
await db.update(dotAgentBindings).set({ readyAt: null, status: "connected" }).where(eq(dotAgentBindings.id, f.snapshot.bindingId));
const readBinding = f.broker.bindingForAgent.bind(f.broker);
const read = vi.spyOn(f.broker, "bindingForAgent").mockImplementationOnce(async (companyId, agentId) => {
const state = await readBinding(companyId, agentId);
if (change === "revoked binding") await db.update(dotAgentBindings).set({ revokedAt: new Date() }).where(eq(dotAgentBindings.id, f.snapshot.bindingId));
else await db.update(mcpEventSubscriptions).set({ stoppedAt: new Date() }).where(eq(mcpEventSubscriptions.bindingId, f.snapshot.bindingId));
return state;
});
try {
await expect(f.broker.challenge(f.company.id, f.agent.id, { automatic: true, bindingId: f.snapshot.bindingId })).resolves.toEqual({ status: "unavailable" });
await expect(f.broker.challenge(f.company.id, f.agent.id)).rejects.toThrow("Connect and subscribe");
} finally {
read.mockRestore();
await f.events.unsubscribe(f.principal, f.subscription); await f.events.stop();
}
});
it("admits one durable idle intake without requiring a preassigned task", async () => {
const f = await fixture();
await db.update(companies).set({ defaultResponsibleUserId: f.userId }).where(eq(companies.id, f.company.id));
@@ -253,6 +293,139 @@ describe("durable Dot Runner integration", () => {
} finally { await f.events.unsubscribe(f.principal, f.subscription); await f.events.stop(); }
});
it("coalesces a normal comment wake into the accepted Dot mailbox without a second run", async () => {
const f = await fixture(), { run, assignment } = await offeredWork(f);
const [task] = await db.insert(issues).values({ companyId: f.company.id, title: "Active Dot comment",
status: "in_progress", assigneeAgentId: f.agent.id, executionRunId: run.id, checkoutRunId: run.id }).returning();
await db.update(heartbeatRuns).set({ nativeIssueId: task!.id, issueId: task!.id, status: "running",
runtimeMode: "native", nativeSessionId: assignment.normalizedSessionId }).where(eq(heartbeatRuns.id, run.id));
await db.update(dotRunnerAssignments).set({ status: "accepted" }).where(eq(dotRunnerAssignments.id, assignment.id));
const [comment] = await db.insert(issueComments).values({ companyId: f.company.id, issueId: task!.id,
authorUserId: f.userId, body: "Continue the same assignment" }).returning();
try {
const admitted = await heartbeatService(db).wakeup(f.agent.id, { source: "assignment", triggerDetail: "system",
reason: "issue_commented", payload: { issueId: task!.id, commentId: comment!.id },
requestedByActorType: "user", requestedByActorId: f.userId,
contextSnapshot: { issueId: task!.id, wakeCommentId: comment!.id, wakeReason: "issue_commented" } });
expect(admitted?.id).toBe(run.id);
await f.events.tick(); await f.events.tick();
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.agentId, f.agent.id))).toHaveLength(1);
const wakes = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.agentId, f.agent.id));
expect(wakes).toHaveLength(1);
expect(wakes[0]).toMatchObject({ status: "deferred_issue_execution", runId: null, finishedAt: null });
const mailbox = await db.select().from(dotMailboxItems).where(and(eq(dotMailboxItems.assignmentId, assignment.id), eq(dotMailboxItems.kind, "follow_up")));
expect(mailbox).toHaveLength(1);
expect(mailbox[0]!.references).toEqual({ assignmentId: assignment.id, commentId: comment!.id });
expect(f.received.filter(event => event.data?.kind === "follow_up")).toHaveLength(1);
await f.broker.mailbox(f.principal);
expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakes[0]!.id)))[0]?.status).toBe("deferred_issue_execution");
const [receipt] = await db.insert(dotRunnerOperations).values({ companyId: f.company.id,
assignmentId: assignment.id, requestId: randomUUID(), digest: "history", status: "completed",
command: { action: "tool", input: { name: "get_task_history", arguments: {} } },
outcome: { status: "completed", isError: false, result: { comments: [{ id: comment!.id }] } } }).returning();
await consumeDotHistoryReceipt(db, receipt!);
await consumeDotHistoryReceipt(db, receipt!);
expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakes[0]!.id)))[0]).toMatchObject({ status: "coalesced", runId: run.id });
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.agentId, f.agent.id))).toHaveLength(1);
} finally {
await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, run.id));
await db.update(dotRunnerAssignments).set({ status: "fenced" }).where(eq(dotRunnerAssignments.id, assignment.id));
await f.events.unsubscribe(f.principal, f.subscription); await f.events.stop();
}
}, 30000);
it.each(["finish before delivery", "read before wake admission", "partial history", "failed history"])(
"preserves comment delivery when %s", async scenario => {
const f = await fixture(), { run, assignment } = await offeredWork(f);
await db.update(companies).set({ defaultResponsibleUserId: f.userId }).where(eq(companies.id, f.company.id));
const [task] = await db.insert(issues).values({ companyId: f.company.id, title: "Dot comment race",
status: "in_progress", assigneeAgentId: f.agent.id, responsibleUserId: f.userId,
executionRunId: run.id, checkoutRunId: run.id }).returning();
await db.update(heartbeatRuns).set({ nativeIssueId: task!.id, issueId: task!.id, status: "running",
contextSnapshot: { issueId: task!.id }, runtimeMode: "native", nativeSessionId: assignment.normalizedSessionId }).where(eq(heartbeatRuns.id, run.id));
await db.update(dotRunnerAssignments).set({ status: "accepted" }).where(eq(dotRunnerAssignments.id, assignment.id));
const comments = await db.insert(issueComments).values([1, 2].map(index => ({ companyId: f.company.id,
issueId: task!.id, authorUserId: f.userId, body: "Input " + index }))).returning();
const [history] = await db.insert(dotRunnerOperations).values({ companyId: f.company.id,
assignmentId: assignment.id, requestId: randomUUID(), digest: "history", status: "completed",
command: { action: "tool", input: { name: "get_task_history", arguments: {} } },
outcome: { status: "completed", isError: scenario === "failed history",
result: { comments: [{ id: comments[0]!.id }] } } }).returning();
try {
if (scenario === "read before wake admission") await consumeDotHistoryReceipt(db, history!);
for (const comment of comments) await heartbeatService(db).wakeup(f.agent.id, {
source: "assignment", triggerDetail: "system", reason: "issue_commented",
payload: { issueId: task!.id, commentId: comment.id }, requestedByActorType: "user", requestedByActorId: f.userId,
contextSnapshot: { issueId: task!.id, wakeCommentId: comment.id, wakeReason: "issue_commented" },
});
if (scenario === "finish before delivery") {
const requestId = randomUUID();
await db.insert(dotRunnerOperations).values({ companyId: f.company.id, assignmentId: assignment.id,
requestId, digest: "finish", command: { action: "finish" }, status: "admitted" });
const port = f.broker.port({ binding: { companyId: f.company.id, agentId: f.agent.id, runId: run.id }, provider: { binding: f.snapshot } });
await port.settle({ sourceEventId: randomUUID(), payload: { requestId,
binding: { ...f.snapshot, runId: run.id, normalizedSessionId: assignment.normalizedSessionId,
turnId: assignment.turnId, assignmentRevision: assignment.revision }, outcome: { status: "completed" } } } as Parameters<typeof port.settle>[0]);
await f.events.tick();
expect(f.received.filter(event => event.data?.kind === "follow_up")).toHaveLength(0);
}
await consumeDotHistoryReceipt(db, history!);
let wakes = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.agentId, f.agent.id));
expect(wakes).toHaveLength(scenario === "read before wake admission" ? 2 : 1);
const pending = wakes.find(wake => wake.status === "deferred_issue_execution")!;
expect(pending.payload?._paperclipWakeContext).toMatchObject({ wakeCommentIds:
["finish before delivery", "failed history"].includes(scenario)
? comments.map(comment => comment.id) : [comments[1]!.id] });
if (scenario === "read before wake admission") expect(wakes.find(wake => wake.payload?.commentId === comments[0]!.id)?.status).toBe("coalesced");
if (scenario === "finish before delivery") {
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run.id));
await db.update(issues).set({ status: "done" }).where(eq(issues.id, task!.id));
const release = createReleaseIssueExecution({ issueLock: createPostgresWakeQueueAdapter(db, {
resolveResponsibleUserId: async () => f.userId,
getRoutineEnv: async () => ({ routineId: null, env: null, responsibleUserId: null }),
resolveSessionBeforeForWakeup: async () => null,
}), recovery: {
escalateStrandedAssignedIssue: async () => { throw new Error("unexpected escalation"); },
escalateStrandedRecoveryIssueInPlace: async () => { throw new Error("unexpected escalation"); },
} });
await release({ companyId: f.company.id, runId: run.id, now: new Date(), suppressImmediateRecovery: true });
await release({ companyId: f.company.id, runId: run.id, now: new Date(), suppressImmediateRecovery: true });
const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.agentId, f.agent.id));
expect(runs).toHaveLength(2);
expect(runs.find(row => row.id !== run.id)).toMatchObject({ status: "queued",
contextSnapshot: { wakeCommentIds: comments.map(comment => comment.id) } });
wakes = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.agentId, f.agent.id));
expect(wakes).toHaveLength(1);
expect(wakes[0]).toMatchObject({ status: "queued", runId: runs.find(row => row.id !== run.id)!.id });
}
} finally {
await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.agentId, f.agent.id));
await db.update(dotRunnerAssignments).set({ status: "fenced" }).where(eq(dotRunnerAssignments.id, assignment.id));
await f.events.unsubscribe(f.principal, f.subscription); await f.events.stop();
}
}, 30000);
it.each(["foreign comment", "self comment", "revoked binding", "fenced assignment", "expired assignment", "prior generation"])(
"refuses active comment delivery for %s", async kind => {
const f = await fixture(), { run, assignment } = await offeredWork(f);
const [task] = await db.insert(issues).values({ companyId: f.company.id, title: "Bounded follow-up", assigneeAgentId: f.agent.id }).returning();
await db.update(heartbeatRuns).set({ nativeIssueId: task!.id, status: "running", runtimeMode: "native",
nativeSessionId: assignment.normalizedSessionId }).where(eq(heartbeatRuns.id, run.id));
await db.update(dotRunnerAssignments).set({ status: "accepted" }).where(eq(dotRunnerAssignments.id, assignment.id));
const [comment] = await db.insert(issueComments).values({ companyId: f.company.id, issueId: task!.id,
...(kind === "self comment" ? { authorAgentId: f.agent.id } : { authorUserId: f.userId }), body: "Bounded input" }).returning();
if (kind === "revoked binding") await db.update(dotAgentBindings).set({ revokedAt: new Date() }).where(eq(dotAgentBindings.id, f.snapshot.bindingId));
if (kind === "fenced assignment") await db.update(dotRunnerAssignments).set({ status: "fenced" }).where(eq(dotRunnerAssignments.id, assignment.id));
if (kind === "expired assignment") await db.update(dotRunnerAssignments).set({ expiresAt: new Date(0) }).where(eq(dotRunnerAssignments.id, assignment.id));
if (kind === "prior generation") await db.update(dotAgentBindings).set({ generation: f.snapshot.bindingGeneration + 1 }).where(eq(dotAgentBindings.id, f.snapshot.bindingId));
try {
expect(await db.transaction(tx => publishActiveDotComment(tx as unknown as typeof db, {
companyId: f.company.id, agentId: f.agent.id, bindingId: f.snapshot.bindingId, runId: run.id,
issueId: task!.id, commentId: kind === "foreign comment" ? randomUUID() : comment!.id }))).toBe(false);
expect(await db.select().from(dotMailboxItems).where(and(eq(dotMailboxItems.assignmentId, assignment.id), eq(dotMailboxItems.kind, "follow_up")))).toHaveLength(0);
} finally { await f.events.stop(); }
}, 30000);
it("serializes tool-result writes and cursor reads with the binding mailbox lock", async () => {
const f = await fixture();
const { run, assignment } = await offeredWork(f);
@@ -288,6 +461,23 @@ describe("durable Dot Runner integration", () => {
await f.events.unsubscribe(f.principal, f.subscription); await f.events.stop();
}, 30000);
it.each(["succeeded", "failed", "cancelled", "timed_out"] as const)("fences a stranded %s assignment before new admission without claiming external stop", async status => {
const f = await fixture();
const { run, assignment } = await offeredWork(f);
try {
await expect(f.broker.snapshot(f.company.id, f.agent.id, f.snapshot.bindingId)).rejects.toThrow("active assignment");
await db.update(heartbeatRuns).set({ status }).where(eq(heartbeatRuns.id, run.id));
await f.broker.snapshot(f.company.id, f.agent.id, f.snapshot.bindingId);
await f.broker.snapshot(f.company.id, f.agent.id, f.snapshot.bindingId);
expect((await db.select().from(dotRunnerAssignments).where(eq(dotRunnerAssignments.id, assignment.id)))[0]?.status).toBe("fenced");
const inbox = await f.broker.mailbox(f.principal);
const fences = inbox.items.filter(item => item.assignmentId === assignment.id && item.kind === "authority_revoked");
expect(fences).toHaveLength(1);
expect(fences[0]?.references).toMatchObject({ assignmentId: assignment.id, runId: run.id, externalStopConfirmed: false });
await expect(createDotRunnerMcpTools(db).callTool(f.principal, "paperclip_dot_accept", { assignmentId: assignment.id, requestId: randomUUID() })).rejects.toThrow("revoked");
} finally { await f.events.unsubscribe(f.principal, f.subscription); await f.events.stop(); }
}, 30000);
it("allows paused Dot fence delivery and acknowledgements without task authority", async () => {
const f = await fixture();
const { run, assignment } = await offeredWork(f);
@@ -474,117 +664,170 @@ describe("durable Dot Runner integration", () => {
await f.events.unsubscribe(f.principal, f.subscription);
}, 30000);
it("signed readiness, normal native authority, one document write and finalization through real Rust", async () => {
const f = await fixture();
const [issue] = await db.insert(issues).values({ companyId: f.company.id, title: "Save the arithmetic report", description: "Save 17 + 25 = 42 as a task document.",
status: "in_progress", workMode: "standard", assigneeAgentId: f.agent.id }).returning();
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id, status: "running", invocationSource: "assignment", triggerDetail: "system", contextSnapshot: { issueId: issue!.id } }).returning();
await db.update(issues).set({ executionRunId: run!.id, checkoutRunId: run!.id }).where(eq(issues.id, issue!.id));
const prepared = await prepareNativeHeartbeatRun({ db, run: run!, issue: issue!, environmentLeaseId: randomUUID() });
await db.insert(nativeRunFinalizations).values({ runId: run!.id, companyId: f.company.id, issueId: issue!.id, phase: "observed" });
const [boundRun] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run!.id));
const [contract] = await db.select().from(completionContracts).where(eq(completionContracts.id, boundRun!.completionContractId!));
const runtimeContext = nativeRuntimeContextFixture();
await writeFile(join(root, "AGENTS.md"), "Use Paperclip tools to save the report.");
runtimeContext.instructions.bundle.rootPath = root;
runtimeContext.aggregateDigest = canonicalNativeRuntimeContextDigest(runtimeContext);
const execution = buildNativeExecutionInput({ companyId: f.company.id, agentId: f.agent.id, runId: run!.id, issue: issue!,
taskPrompt: issue!.description!, normalizedSessionId: prepared.normalizedSessionId, provider: "openai_dot", dotBinding: f.snapshot,
workspace: { id: randomUUID(), cwd: root, repoUrl: null, repoRef: null, branchName: null }, runtimeContext,
completionContract: { id: contract!.id, sha256: contract!.canonicalSha256,
schemaVersion: contract!.schemaVersion, contract: contract!.contractJson as unknown as StrictCompletionContractInput } });
await db.update(heartbeatRuns).set({ runnerProfileJson: { nativeExecutionInput: execution } }).where(eq(heartbeatRuns.id, run!.id));
const server = createServer(); await new Promise<void>(done => server.listen(0, "127.0.0.1", done));
const address = server.address(); if (!address || typeof address === "string") throw new Error("Missing test server");
setupRunnerPrpWebSocketServer(server, { apiUrl: `http://127.0.0.1:${address.port}` });
const observed: Array<Record<string, any>> = [];
const appCall = vi.fn().mockResolvedValue({ status: "completed", result: { content: [{ type: "text", text: "synthetic gateway result" }] } });
const gatewayTools = [{ name: "fixture:read_status", displayName: "Read fixture status", description: "Read a synthetic app", parametersSchema: { type: "object", properties: {}, additionalProperties: false }, risk: "read" }];
registerAssignedMcpGateway(db, { listToolsForNamedGateway: vi.fn().mockResolvedValue(gatewayTools), executeTool: appCall } as unknown as ToolGatewayService);
await db.update(agents).set({ adapterConfig: { ...f.agent.adapterConfig, dotWorkspaceAccess: true, dotAttachmentAccess: true } }).where(eq(agents.id, f.agent.id));
const resultPromise = executePaperclipNativeSession({ db, execution, dotWorkspaceRoot: root, runnerInstanceId: prepared.runnerInstanceId,
runnerEnvironment: { PAPERCLIP_NATIVE_MCP_NAME: "paperclip-assigned", PAPERCLIP_NATIVE_MCP_URL: "http://127.0.0.1:3217/mcp/gateways/dot-fixture", PAPERCLIP_NATIVE_MCP_TOKEN: "fixture-gateway-secret" },
useRunnerd: true, turnTimeoutMs: 45000, onEvent: async event => { observed.push(event); }, onLog: async () => {} });
// Attach a handler immediately: a bootstrap error must never become an unhandled rejection.
let startupError: unknown;
let startupOutcome: unknown;
void resultPromise.then(result => { startupOutcome = result; }, () => {});
void resultPromise.catch(error => { startupError = error; });
it.each([
{ name: "local Rust", managed: false },
...(process.env.PAPERCLIP_DOT_DAYTONA_LIVE === "1" ? [{ name: "private Daytona Rust", managed: true }] : []),
])("signed readiness, native tools and finalization through $name", async ({ managed }) => {
const remote = managed ? await (await import("./helpers/dot-daytona.js")).startDotDaytona(root) : null;
try {
await vi.waitFor(async () => { if (startupError) throw startupError; expect(await db.select().from(dotRunnerAssignments).where(eq(dotRunnerAssignments.runId, run!.id))).toHaveLength(1); }, { timeout: 30000, interval: 50 });
const [assignment] = await db.select().from(dotRunnerAssignments).where(eq(dotRunnerAssignments.runId, run!.id));
await f.events.tick();
expect(JSON.stringify(f.received)).not.toContain("17 + 25");
const work = await f.broker.read(f.principal, assignment!.id);
expect(work.accounting).toEqual({ usage: null, cost: null });
expect(work.status).toBe("offered");
expect(observed.filter(event => event.eventType === "turn.started")).toHaveLength(0);
expect(JSON.stringify(work)).not.toContain(root);
expect((work.tools as Array<{ operationId: string }>).some(tool => tool.operationId === "register_deliverable")).toBe(true);
expect(await f.broker.operation(f.principal, assignment!.id, randomUUID(), "tool", { name: "write_document", arguments: {} })).toMatchObject({ status: "rejected" });
await f.broker.operation(f.principal, assignment!.id, randomUUID(), "accept", {});
expect(JSON.stringify(work)).not.toContain("fixture-gateway-secret");
const app = (work.tools as Array<{ operationId: string }>).find(tool => tool.operationId.startsWith("app_"))!;
expect(app).toBeTruthy();
const missingReadId = randomUUID();
await f.broker.operation(f.principal, assignment!.id, missingReadId, "tool", { name: "workspace_read", arguments: { path: "missing-fixture.txt", offset: 0 } });
await vi.waitFor(async () => expect(await f.broker.operationStatus(f.principal, assignment!.id, missingReadId)).toMatchObject({ status: "completed", isError: true, result: { outcome: "failed", code: "runner_bridge_file_not_found" } }), { timeout: 10000 });
// A definite read failure must not poison later calls or finalization.
const attachmentListId = randomUUID();
expect((work.tools as Array<{ operationId: string }>).some(tool => tool.operationId === "read_task_attachment")).toBe(true);
await f.broker.operation(f.principal, assignment!.id, attachmentListId, "tool", { name: "list_task_attachments", arguments: {} });
await vi.waitFor(async () => expect(await f.broker.operationStatus(f.principal, assignment!.id, attachmentListId)).toMatchObject({ status: "completed", isError: false, result: { attachments: [] } }), { timeout: 10000 });
const appRequestId = randomUUID();
await f.broker.operation(f.principal, assignment!.id, appRequestId, "tool", { name: app.operationId, arguments: {} });
await vi.waitFor(async () => expect(await f.broker.operationStatus(f.principal, assignment!.id, appRequestId)).toMatchObject({ status: "completed", isError: false }), { timeout: 10000 });
expect(appCall).toHaveBeenCalledWith(expect.objectContaining({ gatewayPublicId: "dot-fixture", sessionToken: "fixture-gateway-secret", tool: "fixture:read_status" }));
await db.update(agents).set({ budgetMonthlyCents: 100, spentMonthlyCents: 100 }).where(eq(agents.id, f.agent.id));
await expect(f.broker.read(f.principal, assignment!.id)).rejects.toThrow("execution authority");
await db.update(agents).set({ budgetMonthlyCents: 0, spentMonthlyCents: 0, status: "paused" }).where(eq(agents.id, f.agent.id));
await expect(f.broker.read(f.principal, assignment!.id)).rejects.toThrow("authority");
await db.update(agents).set({ status: "active" }).where(eq(agents.id, f.agent.id));
await db.update(issues).set({ executionRunId: null, checkoutRunId: null }).where(eq(issues.id, issue!.id));
await expect(f.broker.read(f.principal, assignment!.id)).rejects.toThrow("execution authority");
const f = await fixture();
const [issue] = await db.insert(issues).values({ companyId: f.company.id, title: "Save the arithmetic report", description: "Save 17 + 25 = 42 as a task document.",
status: "in_progress", workMode: "standard", assigneeAgentId: f.agent.id }).returning();
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id, status: "running", invocationSource: "assignment", triggerDetail: "system", contextSnapshot: { issueId: issue!.id } }).returning();
await db.update(issues).set({ executionRunId: run!.id, checkoutRunId: run!.id }).where(eq(issues.id, issue!.id));
const writeId = randomUUID();
const args = { name: "write_document", arguments: { key: "report", title: "Arithmetic", body: "17 + 25 = 42.", baseRevisionId: null, idempotencyKey: writeId } };
await f.broker.operation(f.principal, assignment!.id, writeId, "tool", args);
await vi.waitFor(async () => expect(await f.broker.operationStatus(f.principal, assignment!.id, writeId)).toMatchObject({ status: "completed", isError: false }), { timeout: 10000 });
await f.broker.operation(f.principal, assignment!.id, writeId, "tool", args);
await expect(f.broker.operation(f.principal, assignment!.id, writeId, "tool", { ...args, arguments: { ...args.arguments, body: "changed" } })).rejects.toThrow("different arguments");
const document = await documentService(db).getIssueDocumentByKey(issue!.id, "report");
expect(document?.body).toBe("17 + 25 = 42.");
const result = { reportedWorkDisposition: "completed", summary: "Saved the arithmetic report.",
completionClaim: { contractRevision: execution.completionContract.contract.revision, objectiveSatisfied: true,
criteria: execution.completionContract.contract.criteria.map(c => ({ criterionId: c.id, status: "passed", evidenceRefs: [] })), remainingWork: [] },
evidence: [], verification: [{ commandOrCheck: "17 + 25", status: "pass" }] };
const completionId = randomUUID();
await f.broker.operation(f.principal, assignment!.id, completionId, "tool", { name: "paperclip_finish", arguments: result });
await vi.waitFor(async () => expect(await f.broker.operationStatus(f.principal, assignment!.id, completionId)).toMatchObject({ status: "completed", isError: false }), { timeout: 10000 });
expect(await f.broker.operationStatus(f.principal, assignment!.id, completionId)).toMatchObject({ result: { completionReport: {
schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", artifacts: [], attentionRequests: [],
verification: [{ commandOrCheck: "17 + 25", status: "passed" }],
} } });
await f.broker.operation(f.principal, assignment!.id, randomUUID(), "finish", { result });
await resultPromise;
// The heartbeat records its controller workspace settlement before the
// ordinary status finalizer commits the task disposition.
await db.insert(workspaceOperations).values({ companyId: f.company.id, heartbeatRunId: run!.id, issueId: issue!.id,
phase: "workspace_finalize", status: "succeeded", exitCode: 0, cwd: root, finishedAt: new Date() });
await finalizeNativeRun({ db, runId: run!.id, workspaceFinalizeStatus: "succeeded" });
await finalizeNativeRun({ db, runId: run!.id, workspaceFinalizeStatus: "succeeded" });
expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, run!.id))).toHaveLength(1);
expect((await db.select().from(issues).where(eq(issues.id, issue!.id)))[0]?.status).toBe("done");
expect(await db.select().from(dotRunnerOperations).where(eq(dotRunnerOperations.requestId, writeId))).toHaveLength(1);
expect(startupOutcome).toMatchObject({ exitCode: 0, model: null, nativeFinalization: { providerSessionId: null, driverKind: "openai_dot_mcp" } });
expect(observed.filter(event => event.eventType === "turn.started")).toHaveLength(1);
} catch (error) {
if (startupError) throw startupError;
await Promise.race([resultPromise.catch(() => {}), new Promise(resolve => setTimeout(resolve, 1000))]);
if (startupError) throw startupError;
throw error;
} finally { await f.events.unsubscribe(f.principal, f.subscription); await f.events.stop(); server.closeAllConnections(); await new Promise<void>(done => server.close(() => done())); }
}, 90000);
const prepared = await prepareNativeHeartbeatRun({ db, run: run!, issue: issue!, environmentLeaseId: randomUUID() });
await db.insert(nativeRunFinalizations).values({ runId: run!.id, companyId: f.company.id, issueId: issue!.id, phase: "observed" });
const [boundRun] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run!.id));
const [contract] = await db.select().from(completionContracts).where(eq(completionContracts.id, boundRun!.completionContractId!));
const runtimeContext = nativeRuntimeContextFixture();
await writeFile(join(root, "AGENTS.md"), "Use Paperclip tools to save the report.");
runtimeContext.instructions.bundle.rootPath = root;
runtimeContext.aggregateDigest = canonicalNativeRuntimeContextDigest(runtimeContext);
const execution = buildNativeExecutionInput({ companyId: f.company.id, agentId: f.agent.id, runId: run!.id, issue: issue!,
taskPrompt: issue!.description!, normalizedSessionId: prepared.normalizedSessionId, provider: "openai_dot", dotBinding: f.snapshot,
workspace: { id: randomUUID(), cwd: root, repoUrl: null, repoRef: null, branchName: null }, runtimeContext,
completionContract: { id: contract!.id, sha256: contract!.canonicalSha256,
schemaVersion: contract!.schemaVersion, contract: contract!.contractJson as unknown as StrictCompletionContractInput } });
await db.update(heartbeatRuns).set({ runnerProfileJson: { nativeExecutionInput: execution } }).where(eq(heartbeatRuns.id, run!.id));
const server = createServer(); await new Promise<void>(done => server.listen(0, "127.0.0.1", done));
const address = server.address(); if (!address || typeof address === "string") throw new Error("Missing test server");
setupRunnerPrpWebSocketServer(server, { apiUrl: `http://127.0.0.1:${address.port}` });
const observed: Array<Record<string, any>> = [];
const appCall = vi.fn().mockResolvedValue({ status: "completed", result: { content: [{ type: "text", text: "synthetic gateway result" }] } });
const gatewayTools = [{ name: "fixture:read_status", displayName: "Read fixture status", description: "Read a synthetic app", parametersSchema: { type: "object", properties: {}, additionalProperties: false }, risk: "read" }];
registerAssignedMcpGateway(db, { listToolsForNamedGateway: vi.fn().mockResolvedValue(gatewayTools), executeTool: appCall } as unknown as ToolGatewayService);
await db.update(agents).set({ adapterConfig: { ...f.agent.adapterConfig, dotWorkspaceAccess: true, dotAttachmentAccess: true } }).where(eq(agents.id, f.agent.id));
vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "fixture-cloud");
try {
for (const useRunnerd of [false, true]) {
await expect(executePaperclipNativeSession({ db, execution, runnerInstanceId: prepared.runnerInstanceId,
useRunnerd, turnTimeoutMs: 1000, onEvent: async () => {} }))
.rejects.toThrow("dot_cloud_requires_managed_runner");
}
} finally { vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", ""); }
const resultPromise = executePaperclipNativeSession({ db, execution,
...(remote ? { runnerExecutionTarget: remote.target, runnerRemoteBinaryPath: remote.runnerBinary, runnerIngressAuthorized: true }
: { dotWorkspaceRoot: root }),
runnerInstanceId: prepared.runnerInstanceId,
runnerEnvironment: { PAPERCLIP_NATIVE_MCP_NAME: "paperclip-assigned", PAPERCLIP_NATIVE_MCP_URL: "http://127.0.0.1:3217/mcp/gateways/dot-fixture", PAPERCLIP_NATIVE_MCP_TOKEN: "fixture-gateway-secret" },
useRunnerd: true, turnTimeoutMs: 45000, onEvent: async event => { observed.push(event); }, onLog: async () => {} });
// Attach a handler immediately: a bootstrap error must never become an unhandled rejection.
let startupError: unknown;
let startupOutcome: unknown;
void resultPromise.then(result => { startupOutcome = result; }, () => {});
void resultPromise.catch(error => { startupError = error; });
try {
await vi.waitFor(async () => { if (startupError) throw startupError; expect(await db.select().from(dotRunnerAssignments).where(eq(dotRunnerAssignments.runId, run!.id))).toHaveLength(1); }, { timeout: 30000, interval: 50 });
const [assignment] = await db.select().from(dotRunnerAssignments).where(eq(dotRunnerAssignments.runId, run!.id));
await f.events.tick();
expect(JSON.stringify(f.received)).not.toContain("17 + 25");
const work = await f.broker.read(f.principal, assignment!.id);
expect(work.accounting).toEqual({ usage: null, cost: null });
expect(work.status).toBe("offered");
expect(observed.filter(event => event.eventType === "turn.started")).toHaveLength(0);
expect(JSON.stringify(work)).not.toContain(root);
expect((work.tools as Array<{ operationId: string }>).some(tool => tool.operationId === "register_deliverable")).toBe(true);
expect(await f.broker.operation(f.principal, assignment!.id, randomUUID(), "tool", { name: "write_document", arguments: {} })).toMatchObject({ status: "rejected" });
await f.broker.operation(f.principal, assignment!.id, randomUUID(), "accept", {});
expect(JSON.stringify(work)).not.toContain("fixture-gateway-secret");
const app = (work.tools as Array<{ operationId: string }>).find(tool => tool.operationId.startsWith("app_"))!;
expect(app).toBeTruthy();
if (remote) {
expect((work.tools as Array<{ operationId: string }>).some(tool => tool.operationId.startsWith("workspace_"))).toBe(false);
} else {
const missingReadId = randomUUID();
await f.broker.operation(f.principal, assignment!.id, missingReadId, "tool", { name: "workspace_read", arguments: { path: "missing-fixture.txt", offset: 0 } });
await vi.waitFor(async () => expect(await f.broker.operationStatus(f.principal, assignment!.id, missingReadId)).toMatchObject({ status: "completed", isError: true, result: { outcome: "failed", code: "runner_bridge_file_not_found" } }), { timeout: 10000 });
}
// A definite read failure must not poison later calls or finalization.
const attachmentListId = randomUUID();
expect((work.tools as Array<{ operationId: string }>).some(tool => tool.operationId === "read_task_attachment")).toBe(true);
await f.broker.operation(f.principal, assignment!.id, attachmentListId, "tool", { name: "list_task_attachments", arguments: {} });
await vi.waitFor(async () => expect(await f.broker.operationStatus(f.principal, assignment!.id, attachmentListId)).toMatchObject({ status: "completed", isError: false, result: { attachments: [] } }), { timeout: 10000 });
const appRequestId = randomUUID();
await f.broker.operation(f.principal, assignment!.id, appRequestId, "tool", { name: app.operationId, arguments: {} });
await vi.waitFor(async () => expect(await f.broker.operationStatus(f.principal, assignment!.id, appRequestId)).toMatchObject({ status: "completed", isError: false }), { timeout: 10000 });
expect(appCall).toHaveBeenCalledWith(expect.objectContaining({ gatewayPublicId: "dot-fixture", sessionToken: "fixture-gateway-secret", tool: "fixture:read_status" }));
await db.update(agents).set({ budgetMonthlyCents: 100, spentMonthlyCents: 100 }).where(eq(agents.id, f.agent.id));
await expect(f.broker.read(f.principal, assignment!.id)).rejects.toThrow("execution authority");
await db.update(agents).set({ budgetMonthlyCents: 0, spentMonthlyCents: 0, status: "paused" }).where(eq(agents.id, f.agent.id));
await expect(f.broker.read(f.principal, assignment!.id)).rejects.toThrow("authority");
await db.update(agents).set({ status: "active" }).where(eq(agents.id, f.agent.id));
await db.update(issues).set({ executionRunId: null, checkoutRunId: null }).where(eq(issues.id, issue!.id));
await expect(f.broker.read(f.principal, assignment!.id)).rejects.toThrow("execution authority");
await db.update(issues).set({ executionRunId: run!.id, checkoutRunId: run!.id }).where(eq(issues.id, issue!.id));
const writeId = randomUUID();
const args = { name: "write_document", arguments: { key: "report", title: "Arithmetic", body: "17 + 25 = 42.", baseRevisionId: null, idempotencyKey: writeId } };
await f.broker.operation(f.principal, assignment!.id, writeId, "tool", args);
await vi.waitFor(async () => expect(await f.broker.operationStatus(f.principal, assignment!.id, writeId)).toMatchObject({ status: "completed", isError: false }), { timeout: 10000 });
await f.broker.operation(f.principal, assignment!.id, writeId, "tool", args);
await expect(f.broker.operation(f.principal, assignment!.id, writeId, "tool", { ...args, arguments: { ...args.arguments, body: "changed" } })).rejects.toThrow("different arguments");
const document = await documentService(db).getIssueDocumentByKey(issue!.id, "report");
expect(document?.body).toBe("17 + 25 = 42.");
const result = { reportedWorkDisposition: "completed", summary: "Saved the arithmetic report.",
completionClaim: { contractRevision: execution.completionContract.contract.revision, objectiveSatisfied: true,
criteria: execution.completionContract.contract.criteria.map(c => ({ criterionId: c.id, status: "passed", evidenceRefs: [] })), remainingWork: [] },
evidence: [], verification: [{ commandOrCheck: "17 + 25", status: "pass" }] };
const completionId = randomUUID();
await f.broker.operation(f.principal, assignment!.id, completionId, "tool", { name: "paperclip_finish", arguments: result });
await vi.waitFor(async () => expect(await f.broker.operationStatus(f.principal, assignment!.id, completionId)).toMatchObject({ status: "completed", isError: false }), { timeout: 10000 });
expect(await f.broker.operationStatus(f.principal, assignment!.id, completionId)).toMatchObject({ result: { completionReport: {
schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", artifacts: [], attentionRequests: [],
verification: [{ commandOrCheck: "17 + 25", status: "passed" }],
} } });
await f.broker.operation(f.principal, assignment!.id, randomUUID(), "finish", { result });
await resultPromise;
// The heartbeat records its controller workspace settlement before the
// ordinary status finalizer commits the task disposition.
await db.insert(workspaceOperations).values({ companyId: f.company.id, heartbeatRunId: run!.id, issueId: issue!.id,
phase: "workspace_finalize", status: "succeeded", exitCode: 0, cwd: root, finishedAt: new Date() });
await finalizeNativeRun({ db, runId: run!.id, workspaceFinalizeStatus: "succeeded" });
await finalizeNativeRun({ db, runId: run!.id, workspaceFinalizeStatus: "succeeded" });
expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, run!.id))).toHaveLength(1);
expect((await db.select().from(issues).where(eq(issues.id, issue!.id)))[0]?.status).toBe("done");
expect(await db.select().from(dotRunnerOperations).where(eq(dotRunnerOperations.requestId, writeId))).toHaveLength(1);
expect(startupOutcome).toMatchObject({ exitCode: 0, model: null, nativeFinalization: { providerSessionId: null, driverKind: "openai_dot_mcp" } });
expect(observed.filter(event => event.eventType === "turn.started")).toHaveLength(1);
} catch (error) {
if (startupError) throw startupError;
await Promise.race([resultPromise.catch(() => {}), new Promise(resolve => setTimeout(resolve, 1000))]);
if (startupError) throw startupError;
throw error;
} finally { await f.events.unsubscribe(f.principal, f.subscription); await f.events.stop(); server.closeAllConnections(); await new Promise<void>(done => server.close(() => done())); }
} finally { await remote?.close(); }
}, 360000);
it.each(["failed", "cancelled", "timed_out"])("reports %s admission without creating another wake or waiting for a nonexistent assignment", async status => {
const f = await fixture();
const { run } = await offeredWork(f);
await db.update(heartbeatRuns).set({ status }).where(eq(heartbeatRuns.id, run.id));
const requestId = randomUUID(), issueId = randomUUID();
await db.insert(agentWakeupRequests).values({ companyId: f.company.id, agentId: f.agent.id,
source: "assignment", status: "admitted", runId: run.id, payload: { issueId },
idempotencyKey: `dot-work:${f.snapshot.bindingId}:${f.snapshot.bindingGeneration}:${requestId}` });
const result = await f.broker.requestWork(f.principal, issueId, requestId);
expect(result).toMatchObject({ status: "admission_failed", runId: run.id, admissionStatus: status });
expect(result.message).toContain("Stop waiting");
expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.agentId, f.agent.id))).toHaveLength(1);
});
it("uses the app-configured heartbeat for Dot-originated managed admission and cancellation", async () => {
const f = await fixture();
const wakeup = vi.fn().mockResolvedValue(null);
const cancelRun = vi.fn().mockResolvedValue(null);
createDotRunnerMcpTools(db, { wakeup, cancelRun });
try {
const requestId = randomUUID();
const intake = await f.broker.requestTurn(f.principal, "Managed intake context test", requestId);
expect(wakeup).toHaveBeenCalledExactlyOnceWith(f.agent.id, expect.objectContaining({
source: "assignment", requestedByActorType: "agent", requestedByActorId: f.agent.id,
durableDotRequest: expect.objectContaining({ issueId: intake.issueId, requestId }),
}));
const assignment = await offeredWork(f);
await f.broker.revoke(f.company.id, f.agent.id, f.userId);
expect(cancelRun).toHaveBeenCalledExactlyOnceWith(assignment.run.id, "Dot connection revoked");
} finally { dotRunnerBroker(db, { heartbeat: heartbeatService(db) }); }
});
it("revocation, membership loss and a foreign company cannot recover authority", async () => {
const f = await fixture(); const other = await fixture();
@@ -17,6 +17,7 @@ import {
agentWakeupRequests,
chatConversations,
chatActions,
chatDeliveries,
chatEndpoints,
chatMessageLinks,
chatPublications,
@@ -190,6 +191,84 @@ describe("durable inbound chat scheduler receipts", () => {
};
}
it.each(["unmentioned", "mentioned", "coalesced", "recovery", "deferred"])("checks Slack mentions after a real heartbeat queue wait (%s)", async (scenario) => {
const mentioned = scenario === "mentioned" || scenario === "coalesced";
const allowed = scenario === "mentioned" || scenario === "recovery";
const f = await fixture();
await db.update(agents).set({ adapterType: "durable_chat_retry_test" }).where(eq(agents.id, f.agentId));
const applicationId = randomUUID(), connectionId = randomUUID(), endpointId = randomUUID();
await db.insert(toolApplications).values({ id: applicationId, companyId: f.companyId, name: "Mention gate", type: "chat" });
await db.insert(toolConnections).values({ id: connectionId, companyId: f.companyId, applicationId,
name: "Mention gate", uid: randomUUID(), connectionPurpose: "channel", transport: "chat_sdk", status: "active" });
await db.insert(chatEndpoints).values({ id: endpointId, companyId: f.companyId, connectionId,
provider: "slack", publicId: randomUUID(), assignedAgentId: f.agentId, status: "active" });
const request = f.request();
await db.insert(issueComments).values({ id: request.commentId, companyId: f.companyId, issueId: f.issueId,
authorUserId: "board-user", body: mentioned ? "@bot work" : "Work" });
const [delivery] = await db.insert(chatDeliveries).values({ companyId: f.companyId, endpointId,
providerEventId: randomUUID(), deduplicationKey: randomUUID(), eventKind: "direct_message", state: "processed",
normalizedEvent: { trigger: "direct_message", message: { mentionedBot: mentioned } } }).returning();
await db.insert(chatActions).values({ id: request.id, companyId: f.companyId, endpointId,
deliveryId: delivery.id, kind: "inbound_wakeup", providerActionId: randomUUID(), status: "processed",
payload: { issueId: f.issueId, commentId: request.commentId } });
await f.wake(request);
const [receipt] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.id));
expect(receipt.status).toBe("queued");
const [queuedRun] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, receipt.runId!));
expect(queuedRun.startedAt).toBeNull();
const originalStartedAt = new Date(Date.now() - 60_000);
if (scenario === "recovery") {
// Same-run retries retain their first claim time, even when queued again.
await db.update(heartbeatRuns).set({ startedAt: originalStartedAt })
.where(eq(heartbeatRuns.id, receipt.runId!));
}
let deferredRequestId: string | undefined;
if (scenario === "coalesced" || scenario === "deferred") {
const second = f.request();
await db.insert(issueComments).values({ id: second.commentId, companyId: f.companyId, issueId: f.issueId,
authorUserId: "board-user", body: scenario === "deferred" ? "@bot follow-up" : "Unmentioned follow-up" });
const [secondDelivery] = await db.insert(chatDeliveries).values({ companyId: f.companyId, endpointId,
providerEventId: randomUUID(), deduplicationKey: randomUUID(), eventKind: "direct_message", state: "processed",
normalizedEvent: { trigger: "direct_message", message: { mentionedBot: scenario === "deferred" } } }).returning();
await db.insert(chatActions).values({ id: second.id, companyId: f.companyId, endpointId,
deliveryId: secondDelivery.id, kind: "inbound_wakeup", providerActionId: randomUUID(), status: "processed",
payload: { issueId: f.issueId, commentId: second.commentId } });
await db.update(issues).set({ executionRunId: receipt.runId, executionAgentNameKey: "maya", executionLockedAt: new Date() })
.where(eq(issues.id, f.issueId));
if (scenario === "deferred") {
// Seed the persisted queue state left by a deferred follow-up before
// its predecessor is returned to the unstarted queue.
await db.update(heartbeatRuns).set({ status: "running" }).where(eq(heartbeatRuns.id, receipt.runId!));
runningProcesses.set(receipt.runId!, { child: {} as never, graceSec: 0, processGroupId: null });
liveRunIds.add(receipt.runId!);
}
await f.wake(second);
const [coalesced] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, second.id));
if (scenario === "deferred") {
expect(coalesced).toMatchObject({ status: "deferred_issue_execution", runId: null });
deferredRequestId = second.id;
runningProcesses.delete(receipt.runId!);
liveRunIds.delete(receipt.runId!);
await db.update(heartbeatRuns).set({ status: "queued" }).where(eq(heartbeatRuns.id, receipt.runId!));
} else {
expect(coalesced).toMatchObject({ status: "coalesced", runId: receipt.runId });
}
}
await db.update(chatEndpoints).set({ requireAtMention: true }).where(eq(chatEndpoints.id, endpointId));
await f.heartbeat.cancelRun(f.activeRunId, "Fixture slot released", { suppressImmediateRecovery: true });
await f.heartbeat.drainActiveRunExecutions();
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, receipt.runId!));
expect(run.status).toBe(allowed ? "succeeded" : "cancelled");
expect(execute).toHaveBeenCalledTimes(allowed || scenario === "deferred" ? 1 : 0);
if (deferredRequestId) {
const [promoted] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, deferredRequestId));
const [followup] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, promoted.runId!));
expect(followup.status).toBe("succeeded");
}
if (!allowed) expect(run.errorCode).toBe("chat_mention_required");
if (scenario === "recovery") expect(run.startedAt).toEqual(originalStartedAt);
});
async function executionHold(f: { companyId: string; issueId: string; agentId: string }) {
const [action] = await db.insert(issueRecoveryActions).values({
companyId: f.companyId, sourceIssueId: f.issueId, kind: "active_run_watchdog",
@@ -199,7 +199,9 @@ describe("AgentMail durable email pipeline", () => {
expect(await db.select().from(authUsers).where(eq(authUsers.id, "email-board"))).toHaveLength(1);
});
async function fixture(mode: "websocket" | "webhook" = "webhook", storage?: StorageService) {
async function fixture(mode: "websocket" | "webhook" = "webhook", storage?: StorageService,
beforeSetup?: (input: { fetcher: typeof fetch; wakeup: ReturnType<typeof vi.fn> }) => Promise<void>,
) {
const companyId = randomUUID(),
agentId = randomUUID(),
endpointId = randomUUID();
@@ -312,6 +314,7 @@ describe("AgentMail durable email pipeline", () => {
storage,
});
services.push(service);
await beforeSetup?.({ fetcher, wakeup });
await service.setup(
companyId,
{
@@ -374,6 +377,73 @@ describe("AgentMail durable email pipeline", () => {
setWebhookError: (status: number) => { webhookError = status; },
};
}
it("activates an empty running worker from another service and drains committed inbound and outbound work", async () => {
// Connector setup and tool calls construct short-lived service objects.
// Their committed writes must wake the app-owned worker sharing this DB.
let running!: EmailChannelService;
const f = await fixture("webhook", undefined, async ({ fetcher, wakeup }) => {
running = emailChannelService(db, { heartbeat: { wakeup }, fetch: fetcher });
services.push(running);
await running.start();
const select = vi.spyOn(db, "select");
await new Promise(resolve => setTimeout(resolve, 1200));
expect(select).not.toHaveBeenCalled();
select.mockRestore();
});
await vi.waitFor(async () => {
const [config] = await db.select().from(emailEndpoints).where(eq(emailEndpoints.endpointId, f.endpointId));
expect(config.lastSyncAt).not.toBeNull();
}, { timeout: 5000 });
await running.tick();
await f.service.control(f.endpointId, "pause", { userId: "email-board" });
await f.service.control(f.endpointId, "resume", { userId: "email-board" });
const message = f.message();
f.messages.set(message.message_id, message);
await f.service.admit(await f.service.getEndpoint(f.endpointId), {
event_type: "message.received", message,
});
await vi.waitFor(() => expect(f.wakeup).toHaveBeenCalledTimes(1), { timeout: 5000 });
const [parent] = await db.select().from(issues).where(eq(issues.companyId, f.companyId));
const input = emailSendSchema.parse({ endpointId: f.endpointId, parentIssueId: parent.id,
to: ["recipient@example.test"], subject: "Wake on send", text: "Hello", idempotencyKey: randomUUID() });
await f.service.queueSend(f.companyId, input, { userId: "email-board" });
await vi.waitFor(async () => expect((await f.service.publication(input.idempotencyKey, f.companyId)).outcome).toBe("sent"), { timeout: 5000 });
expect(f.sends).toHaveLength(1);
await running.tick();
await f.service.control(f.endpointId, "pause", { userId: "email-board" });
await new Promise(resolve => setTimeout(resolve, 100));
const select = vi.spyOn(db, "select");
await new Promise(resolve => setTimeout(resolve, 1200));
expect(select).not.toHaveBeenCalled();
});
it("restores a retry deadline without polling replies blocked behind that send", async () => {
const f = await fixture();
const message = f.message("received", "outbound-thread");
await f.receive(message);
const [conversation] = await db.select().from(chatConversations).where(eq(chatConversations.endpointId, f.endpointId));
const input = emailSendSchema.parse({ endpointId: f.endpointId, conversationId: conversation.id,
replyToMessageId: message.message_id, text: "First reply", idempotencyKey: randomUUID() });
await f.service.queueSend(f.companyId, input, { userId: "email-board" });
const second = { ...input, text: "Second reply", idempotencyKey: randomUUID() };
await f.service.queueSend(f.companyId, second, { userId: "email-board" });
// Simulate a previous process persisting a provider retry deadline. The
// second reply has no deadline but cannot overtake this first reply.
await db.update(chatPublications).set({ state: "retry", nextAttemptAt: new Date(Date.now() + 5000) })
.where(eq(chatPublications.id, input.idempotencyKey));
await f.service.start();
expect(f.sends).toHaveLength(0);
const select = vi.spyOn(db, "select");
await new Promise(resolve => setTimeout(resolve, 1100));
expect(select).not.toHaveBeenCalled();
expect(f.sends).toHaveLength(0);
await vi.waitFor(() => expect(f.sends).toHaveLength(2), { timeout: 8000 });
await f.service.tick();
expect(f.sends.map(send => send.key)).toEqual([input.idempotencyKey, second.idempotencyKey]);
for (const id of [input.idempotencyKey, second.idempotencyKey])
expect((await f.service.publication(id, f.companyId)).outcome).toBe("sent");
});
it("admits signed webhooks through the durable queue and rejects a valid signature for another inbox", async () => {
const f = await fixture();
const endpoint = await f.service.getEndpoint(f.endpointId);
@@ -0,0 +1,81 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { Db } from "@paperclipai/db";
import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js";
import { emailChannelService } from "../services/email-channels.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "../services/delivery-work-notifications.js";
import { idleWorkSnapshot } from "../services/task-admission.js";
const services: ReturnType<typeof emailChannelService>[] = [];
afterEach(async () => {
for (const service of services.splice(0)) await service.shutdown();
vi.useRealTimers();
});
function fixture() {
vi.useFakeTimers();
const select = vi.fn(() => ({ from: () => ({ where: async () => [] }) }));
const execute = vi.fn(async () => [{ xid: "42", status: "aborted" }]);
const db = installDatabaseWorkSignals({ select, execute,
async transaction(callback: (tx: any) => Promise<any>) {
return callback({ execute, transaction: db.transaction });
},
});
let enabled = true;
const service = emailChannelService(db as unknown as Db, {
heartbeat: { wakeup: vi.fn() },
isBackgroundWorkEnabled: () => enabled,
isReconciliationEnabled: () => true,
});
services.push(service);
return { db, select, service, enable: (value: boolean) => { enabled = value; } };
}
describe("email idle work", () => {
it("scans at startup, then leaves an unconfigured stack with zero timers and queries", async () => {
const f = fixture();
await f.service.start();
expect(f.select).toHaveBeenCalledTimes(1);
await vi.advanceTimersByTimeAsync(3_600_000);
expect(f.select).toHaveBeenCalledTimes(1);
expect(vi.getTimerCount()).toBe(0);
});
it("wakes on an outer transaction commit from a separate producer, never an inner savepoint", async () => {
const f = fixture();
await f.service.start();
await f.db.transaction(async tx => {
await tx.transaction(async (inner: object) => notifyDeliveryWork(inner, DELIVERY_QUEUES.email));
expect(f.select).toHaveBeenCalledTimes(1);
});
await vi.advanceTimersByTimeAsync(1);
expect(f.select).toHaveBeenCalledTimes(2);
expect(vi.getTimerCount()).toBe(0);
expect(idleWorkSnapshot().active).toBe(0);
});
it("retains startup and committed work across idle drain without SQL or dispatch", async () => {
const f = fixture();
f.enable(false);
await f.service.start();
await f.db.transaction(tx => notifyDeliveryWork(tx, DELIVERY_QUEUES.email));
await vi.advanceTimersByTimeAsync(60_000);
expect(f.select).not.toHaveBeenCalled();
expect(idleWorkSnapshot().active).toBe(0);
f.enable(true);
await vi.advanceTimersByTimeAsync(1001);
expect(f.select).toHaveBeenCalledTimes(1);
expect(vi.getTimerCount()).toBe(0);
});
it("retries a failed startup scan and unsubscribes on shutdown", async () => {
const f = fixture();
const warning = vi.spyOn(process, "emitWarning").mockImplementation(() => {});
f.select.mockImplementationOnce(() => { throw new Error("database unavailable"); });
await f.service.start();
await vi.advanceTimersByTimeAsync(1001);
expect(f.select).toHaveBeenCalledTimes(2);
expect(vi.getTimerCount()).toBe(0);
await f.service.shutdown();
await f.db.transaction(tx => notifyDeliveryWork(tx, DELIVERY_QUEUES.email));
await vi.advanceTimersByTimeAsync(60_000);
expect(f.select).toHaveBeenCalledTimes(2);
expect(vi.getTimerCount()).toBe(0);
warning.mockRestore();
});
});
@@ -0,0 +1,56 @@
import { createRequire } from "node:module";
import { fileURLToPath } from "node:url";
import { describe, expect, it } from "vitest";
import { JsonRpcCallError, readEnvironmentAcquisitionDiagnostic, readEnvironmentCreationCleanupError } from "@paperclipai/plugin-sdk";
import type { heartbeatRuns } from "@paperclipai/db";
import { createPluginWorkerHandle } from "../services/plugin-worker-manager.js";
import { captureEnvironmentAcquisitionDiagnostic } from "../services/environment-acquisition-diagnostics.js";
import { collectRunFailureDiagnostics } from "../services/run-failure-diagnostics.js";
describe("acquisition diagnostics through the built worker and real host client", () => {
it("keeps concurrent RPC errors isolated through wrapping and failure projection", async () => {
const handle = createPluginWorkerHandle("test.acquisition", {
entrypointPath: fileURLToPath(new URL("./fixtures/plugin-worker-acquisition-diagnostic.mjs", import.meta.url)),
manifest: { id: "test.acquisition", apiVersion: 1, version: "1.0.0", displayName: "Acquisition fixture",
description: "Acquisition fixture", author: "Paperclip", categories: ["automation"], capabilities: [],
entrypoints: { worker: "fixture.mjs" } },
config: {}, instanceInfo: { instanceId: "instance-fixture", hostVersion: "1.0.0" }, apiVersion: 1,
hostHandlers: {}, rpcTimeoutMs: 5_000,
execArgv: ["--import", createRequire(import.meta.url).resolve("tsx")],
});
try {
await handle.start();
const first = { driverKey: "daytona", companyId: "company-fixture", environmentId: "environment-fixture",
runId: "run-first", config: { phase: "create" } };
const second = { ...first, runId: "run-second", config: { phase: "shell" } };
const [a, b] = await Promise.all([first, second].map(params => handle.call("environmentAcquireLease", params).catch(error => error)));
expect(a).toBeInstanceOf(JsonRpcCallError);
expect(b).toBeInstanceOf(JsonRpcCallError);
expect(a).not.toBe(b);
for (const [error, params] of [[a, first], [b, second]] as const) {
expect(error.code).toBe(-32002);
expect(readEnvironmentCreationCleanupError(error)?.runId).toBe(params.runId);
expect(error.message).toBe("Daytona lease acquisition timed out; allocation cleanup is pending");
expect(JSON.stringify(error)).not.toContain("private-provider-");
const diagnostic = readEnvironmentAcquisitionDiagnostic(error);
expect(diagnostic).toEqual({ phase: params.config.phase, elapsedMs: 2_000, budgetMs: 2_000 });
const properties = Object.getOwnPropertyDescriptors(error);
Object.freeze(error);
captureEnvironmentAcquisitionDiagnostic(error, params, diagnostic);
const run = { id: params.runId, companyId: params.companyId, errorCode: "setup_failed", resultJson: null } as typeof heartbeatRuns.$inferSelect;
const diagnostics = collectRunFailureDiagnostics(run, { error: new Error("Cleanup confirmed", { cause: error }), phase: "setup" });
expect(diagnostics.execution).toMatchObject({ environmentAcquisitionPhase: params.config.phase,
environmentAcquisitionElapsedMs: 2_000, environmentAcquisitionBudgetMs: 2_000 });
expect(JSON.stringify(diagnostics)).not.toContain("private-provider-");
// Object.freeze changes descriptor flags but no error values or identity.
for (const [key, descriptor] of Object.entries(properties)) expect(Object.getOwnPropertyDescriptor(error, key)?.value).toBe(descriptor.value);
expect(collectRunFailureDiagnostics({ ...run, id: "another-run" }, { error, phase: "setup" }).execution)
.not.toHaveProperty("environmentAcquisitionPhase");
}
const destroyed = await handle.call("environmentDestroyLease", { ...first, providerLeaseId: "fixture" }).catch(error => error);
expect(destroyed).toBeInstanceOf(JsonRpcCallError);
expect(readEnvironmentCreationCleanupError(destroyed)).not.toBeNull();
expect(readEnvironmentAcquisitionDiagnostic(destroyed)).toBeNull();
} finally { await handle.stop(); }
});
});
@@ -12,6 +12,7 @@ const mockUpdateExecutionWorkspace = vi.hoisted(() => vi.fn());
const mockLogActivity = vi.hoisted(() => vi.fn());
const mockLoggerInfo = vi.hoisted(() => vi.fn());
const mockGetEnvironment = vi.hoisted(() => vi.fn());
const mockGetLease = vi.hoisted(() => vi.fn());
vi.mock("../services/environment-execution-target.js", () => ({
resolveEnvironmentExecutionTarget: mockResolveEnvironmentExecutionTarget,
@@ -30,6 +31,7 @@ vi.mock("../services/environments.js", () => ({
environmentService: vi.fn(() => ({
ensureLocalEnvironment: vi.fn(),
getById: mockGetEnvironment,
getLeaseById: mockGetLease,
acquireLease: vi.fn(),
releaseLease: vi.fn(),
updateLeaseMetadata: mockUpdateLeaseMetadata,
@@ -815,3 +817,56 @@ describe("admitted native lifecycle recovery", () => {
expect(environment.config.reuseLease).toBe(false);
});
});
describe("live remote runner lease reattachment", () => {
const input = {
companyId: "company-1", selectedEnvironmentId: "env-1", localEnvironmentId: "local",
adapterType: "paperclip_runner", admittedLifecycleMode: "per_turn" as const,
issueId: "task-1", heartbeatRunId: "run-1", agentId: "agent-1",
persistedExecutionWorkspace: { id: "ew-1", mode: "shared_workspace" as const },
executionWorkspaceSettings: null,
reattachRemoteLease: { leaseId: "lease-1", providerLeaseId: "original-sandbox", remoteCwd: "/remote/workspace" },
};
const originalLease = () => makeLease({
provider: "daytona", providerLeaseId: "original-sandbox", issueId: "task-1", executionWorkspaceId: "ew-1",
metadata: { agentId: "agent-1", remoteCwd: "/remote/workspace",
sandboxLeaseAcquisition: { outcome: "created", providerLeaseId: "original-sandbox" } },
});
beforeEach(() => {
vi.clearAllMocks();
mockGetEnvironment.mockResolvedValue({ ...makeEnvironment("sandbox"), config: { provider: "daytona", reuseLease: false } });
mockGetLease.mockResolvedValue(originalLease());
});
it("reattaches the original active ephemeral lease without acquiring or resuming a sandbox", async () => {
const savedLease = originalLease();
mockGetLease.mockResolvedValue(savedLease);
const runtime = makeMockRuntime();
const result = await environmentRunOrchestrator({} as never, { environmentRuntime: runtime }).acquireForRun(input);
expect(mockGetLease).toHaveBeenCalledWith("lease-1");
expect(runtime.acquireRunLease).not.toHaveBeenCalled();
expect(result.lease).toEqual(savedLease);
expect(result.leaseContext.executionWorkspaceId).toBe("ew-1");
expect(result.environment.config.reuseLease).toBe(false);
});
it.each([
{ companyId: "foreign" }, { environmentId: "foreign" }, { heartbeatRunId: "another-run" },
{ issueId: "another-task" }, { executionWorkspaceId: "another-workspace" },
{ providerLeaseId: "replacement" }, { provider: "another-provider" },
{ status: "released" }, { expiresAt: new Date(0) }, { releasedAt: new Date() }, { cleanupStatus: "pending" },
{ metadata: { agentId: "another-agent", remoteCwd: "/remote/workspace" } },
{ metadata: { agentId: "agent-1", remoteCwd: "/other/workspace" } },
] as Partial<EnvironmentLease>[])("fails closed before provider acquisition for a mismatched lease %j", async (override) => {
mockGetLease.mockResolvedValue({ ...originalLease(), ...override });
const runtime = makeMockRuntime();
await expect(environmentRunOrchestrator({} as never, { environmentRuntime: runtime }).acquireForRun(input))
.rejects.toThrow("native_remote_recovery_lease_mismatch");
expect(runtime.acquireRunLease).not.toHaveBeenCalled();
});
it("never replaces a missing original lease", async () => {
mockGetLease.mockResolvedValue(null);
const runtime = makeMockRuntime();
await expect(environmentRunOrchestrator({} as never, { environmentRuntime: runtime }).acquireForRun(input))
.rejects.toThrow("native_remote_recovery_lease_mismatch");
expect(runtime.acquireRunLease).not.toHaveBeenCalled();
});
});
+197 -11
View File
@@ -1,3 +1,5 @@
import { JsonRpcCallError } from "@paperclipai/plugin-sdk";
import { collectRunFailureDiagnostics } from "../services/run-failure-diagnostics.js";
import { idleWorkSnapshot } from "../services/task-admission.js";
import { createHash, randomUUID } from "node:crypto";
import { mkdir, mkdtemp, readdir, rm, writeFile } from "node:fs/promises";
@@ -12,6 +14,7 @@ import {
stopSshEnvLabFixture,
} from "@paperclipai/adapter-utils/ssh";
import {
activityLog,
costEvents,
agents,
builtInManagedResources,
@@ -219,6 +222,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
await stopSshEnvLabFixture(path.join(root, "state.json")).catch(() => undefined);
await rm(root, { recursive: true, force: true }).catch(() => undefined);
}
await db.delete(activityLog);
await db.delete(environmentLeases);
await db.delete(issues);
await db.delete(costEvents);
@@ -767,7 +771,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
expect(call.mock.calls.filter((entry) => entry[1] === "environmentResumeLease")).toHaveLength(shouldResume ? 1 : 0);
});
it.each(["pending", "inline", "foreign scope", "foreign environment", "foreign run", "malformed", "deleted environment"])("retains uncertain plugin creation for durable cleanup: %s", async (mode) => {
it.each(["pending", "inline", "foreign scope", "foreign environment", "foreign run", "malformed", "deleted environment", "forged error", "invalid diagnostic", "old worker"])("retains uncertain plugin creation for durable cleanup: %s", async (mode) => {
const seeded = await seedReusablePluginSandboxLease();
await environmentService(db).releaseLease(seeded.reusableLease.id, "expired");
const attemptId = randomUUID();
@@ -778,9 +782,14 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
runId: mode === "foreign run" ? randomUUID() : seeded.runId,
accountFingerprint: mode === "malformed" ? "invalid" : "a".repeat(64), labels: { "paperclip-provider": "fake-plugin" },
};
const failure = Object.assign(new Error("Provider creation cleanup required"), {
data: { schema: "paperclip/environment-creation-cleanup/v1", cleanup },
});
const acquisitionDiagnostic = { phase: "shell", elapsedMs: 300_000, budgetMs: 300_000 };
const data = { schema: "paperclip/environment-creation-cleanup/v1", cleanup,
...(mode === "old worker" ? {} : { acquisitionDiagnostic: mode === "invalid diagnostic"
? { ...acquisitionDiagnostic, phase: "private-command" } : acquisitionDiagnostic }),
};
const failure = mode === "forged error"
? Object.assign(new Error("Provider creation cleanup required"), { data })
: new JsonRpcCallError({ code: -32002, message: "Provider creation cleanup required", data });
let available = mode === "inline";
const call = vi.fn(async (_id: string, method: string, params: Record<string, unknown>) => {
if (method === "environmentAcquireLease") throw failure;
@@ -798,6 +807,14 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
const candidate = environmentRuntimeService(db, { pluginWorkerManager: worker });
const acquisition = candidate.acquireRunLease({ companyId: seeded.companyId, environment: seeded.environment,
issueId: null, heartbeatRunId: seeded.runId, persistedExecutionWorkspace: null });
const caught = await acquisition.catch(error => error);
const execution = collectRunFailureDiagnostics({ id: seeded.runId, companyId: seeded.companyId,
errorCode: "setup_failed", resultJson: null,
} as typeof heartbeatRuns.$inferSelect, { error: caught, phase: "setup" }).execution;
if (["pending", "inline", "deleted environment"].includes(mode)) {
expect(execution).toMatchObject({ environmentAcquisitionPhase: "shell",
environmentAcquisitionElapsedMs: 300_000, environmentAcquisitionBudgetMs: 300_000 });
} else expect(execution).not.toHaveProperty("environmentAcquisitionPhase");
if (mode === "inline") {
await expect(acquisition).rejects.toMatchObject({
message: "Sandbox creation failed; allocated sandbox cleanup was confirmed.", cause: failure,
@@ -5780,6 +5797,21 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
});
});
it("destroys a failed retained lease when the agent stops", async () => {
const { lease, workerManager, runtimeWithPlugin, runId } = await seedStaleLifecycleReusableLease("retain_on_failure");
await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, runId));
await runtimeWithPlugin.releaseRunLeases(runId, "failed");
await expect(environmentService(db).getLeaseById(lease.id)).resolves.toMatchObject({ status: "retained", cleanupStatus: "failed" });
vi.mocked(workerManager.getWorker).mockReturnValue({ supportedMethods: ["environmentDestroyLease"] } as never);
vi.mocked(workerManager.call).mockResolvedValue({ providerLeaseId: lease.providerLeaseId, state: "destroyed" });
const { heartbeatService } = await import("../services/heartbeat.js");
const stopped = await heartbeatService(db, { pluginWorkerManager: workerManager })
.stopInvocationsForAgents([lease.metadata!.agentId as string], "Agent lifecycle stop requested");
expect(stopped).toBe(true);
expect(workerManager.call).toHaveBeenCalledWith(expect.any(String), "environmentDestroyLease", expect.objectContaining({ providerLeaseId: lease.providerLeaseId }), expect.any(Number));
await expect(environmentService(db).getLeaseById(lease.id)).resolves.toMatchObject({ status: "expired", cleanupStatus: "success" });
});
it("fails closed on expiry destruction when the worker no longer advertises the destroy lifecycle method", async () => {
const { pluginId, lease, workerManager, runtimeWithPlugin } = await seedStaleLifecycleReusableLease();
@@ -6679,13 +6711,24 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
});
});
it("destroys scoped reusable plugin-backed sandbox leases", async () => {
it.each([
{ status: "active", scope: "workspace" },
{ status: "failed", scope: "workspace" },
{ status: "failed", scope: "issue" },
] as const)("destroys $status reusable plugin-backed sandbox leases scoped to an $scope", async ({ status, scope }) => {
const { pluginId, companyId, runId, executionWorkspaceId, reusableLease } =
await seedReusablePluginSandboxLease();
await db
.update(heartbeatRuns)
.set({ status: "succeeded" })
.set({ status: status === "failed" ? "failed" : "succeeded" })
.where(eq(heartbeatRuns.id, runId));
if (status === "failed") await environmentService(db).releaseLease(reusableLease.id, "failed");
const issueId = randomUUID();
if (scope === "issue") {
await db.insert(issues).values({ id: issueId, companyId, title: "Failed reusable lease cleanup", status: "done", priority: "medium" });
await db.update(environmentLeases).set({ issueId }).where(eq(environmentLeases.id, reusableLease.id));
}
const selection = scope === "issue" ? { issueId } : { executionWorkspaceId };
const workerManager = {
isRunning: vi.fn((id: string) => id === pluginId),
@@ -6699,9 +6742,12 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
} as unknown as PluginWorkerManager;
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
// A known issue/workspace id does not authorize another company's teardown.
expect(await runtimeWithPlugin.destroyReusableSandboxLeases({ companyId: randomUUID(), ...selection })).toEqual([]);
expect(workerManager.call).not.toHaveBeenCalled();
const destroyed = await runtimeWithPlugin.destroyReusableSandboxLeases({
companyId,
executionWorkspaceId,
...selection,
failureReason: "execution_workspace_closed",
});
@@ -6724,9 +6770,10 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
});
});
it("does not destroy a scoped reusable lease while its run is active", async () => {
it.each(["active", "failed"] as const)("does not destroy a %s scoped reusable lease while its run is active", async (status) => {
const { pluginId, companyId, executionWorkspaceId, reusableLease } =
await seedReusablePluginSandboxLease();
if (status === "failed") await environmentService(db).releaseLease(reusableLease.id, "failed");
const workerManager = {
isRunning: vi.fn((id: string) => id === pluginId),
call: vi.fn(async () => undefined),
@@ -6752,9 +6799,106 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
expect(workerManager.call).not.toHaveBeenCalled();
await expect(
environmentService(db).getLeaseById(reusableLease.id),
).resolves.toMatchObject({ status: "active" });
).resolves.toMatchObject({ status });
});
it.each(["issue", "workspace", "environment"] as const)(
"durably claims failed reusable cleanup before %s teardown and recovers a provider failure",
async (scope) => {
const seeded = await seedReusablePluginSandboxLease();
await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, seeded.runId));
await environmentService(db).releaseLease(seeded.reusableLease.id, "failed");
const issueId = randomUUID();
if (scope === "issue") {
await db.insert(issues).values({ id: issueId, companyId: seeded.companyId, title: "Cleanup crash window", status: "done" });
await db.update(environmentLeases).set({ issueId }).where(eq(environmentLeases.id, seeded.reusableLease.id));
}
let enterProvider!: () => void;
let finishProvider!: () => void;
const entered = new Promise<void>((resolve) => { enterProvider = resolve; });
const finish = new Promise<void>((resolve) => { finishProvider = resolve; });
let unavailable = true;
const call = vi.fn(async (_id: string, method: string) => {
if (method !== "environmentDestroyLease") throw new Error(`Unexpected method ${method}`);
if (unavailable) {
enterProvider();
await finish;
throw new Error("Provider response lost");
}
return { providerLeaseId: seeded.reusableLease.providerLeaseId, state: "destroyed" };
});
const makeRuntime = () => environmentRuntimeService(db, { pluginWorkerManager: {
isRunning: () => true, call,
getWorker: () => ({ supportedMethods: ["environmentDestroyLease"] }),
} as unknown as PluginWorkerManager });
const runtime = makeRuntime();
const close = (service: ReturnType<typeof makeRuntime>) => scope === "environment"
? service.destroyReusableSandboxLeasesForEnvironment({ environmentId: seeded.environment.id })
: service.destroyReusableSandboxLeases({ companyId: seeded.companyId,
...(scope === "issue" ? { issueId } : { executionWorkspaceId: seeded.executionWorkspaceId }) });
const closing = close(runtime);
try {
await entered;
// This independently persisted row is what survives controller loss
// during the provider call. No provider receipt has arrived yet.
expect(await environmentService(db).getLeaseById(seeded.reusableLease.id)).toMatchObject({
status: "pending_cleanup", cleanupStatus: "failed", companyId: seeded.companyId,
providerLeaseId: seeded.reusableLease.providerLeaseId,
metadata: { pendingCleanupAttemptId: expect.any(String), pendingCleanupInFlight: true,
pendingCleanupLeaseExpiresAtMs: expect.any(Number) },
});
await close(makeRuntime());
expect(await heartbeatService(db, { environmentRuntime: makeRuntime() }).sweepPendingCleanupLeases({ backoffMs: 0 }))
.toEqual({ swept: 0, destroyed: 0, capped: 0 });
expect(call).toHaveBeenCalledOnce();
} finally {
finishProvider();
await closing;
}
expect(await environmentService(db).getLeaseById(seeded.reusableLease.id)).toMatchObject({
status: "pending_cleanup", metadata: { pendingCleanupInFlight: false },
});
unavailable = false;
expect(await heartbeatService(db, { environmentRuntime: makeRuntime() }).sweepPendingCleanupLeases({ backoffMs: 0 }))
.toEqual({ swept: 1, destroyed: 1, capped: 0 });
expect(call).toHaveBeenCalledTimes(2);
expect(await environmentService(db).getLeaseById(seeded.reusableLease.id)).toMatchObject({
status: "expired", cleanupStatus: "success", metadata: { remoteExecutionTermination: { state: "destroyed" } },
});
},
);
it.each(["queued", "scheduled_retry", "running"] as const)(
"fences scoped cleanup when its holding run becomes %s after selection", async (status) => {
const seeded = await seedReusablePluginSandboxLease();
await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, seeded.runId));
await environmentService(db).releaseLease(seeded.reusableLease.id, "failed");
const originalService = environmentsModule.environmentService;
const serviceSpy = vi.spyOn(environmentsModule, "environmentService").mockImplementation((client) => {
const service = originalService(client);
return { ...service, getById: async (id: string) => {
const environment = await service.getById(id);
if (id === seeded.environment.id) {
await db.update(heartbeatRuns).set({ status }).where(eq(heartbeatRuns.id, seeded.runId));
}
return environment;
} };
});
const call = vi.fn(async () => undefined);
const runtime = environmentRuntimeService(db, { pluginWorkerManager: {
isRunning: () => true, call, getWorker: () => ({ supportedMethods: ["environmentDestroyLease"] }),
} as unknown as PluginWorkerManager });
try {
expect(await runtime.destroyReusableSandboxLeases({ companyId: seeded.companyId,
executionWorkspaceId: seeded.executionWorkspaceId })).toEqual([]);
expect(call).not.toHaveBeenCalled();
expect(await originalService(db).getLeaseById(seeded.reusableLease.id)).toMatchObject({ status: "failed" });
} finally {
serviceSpy.mockRestore();
}
},
);
it("destroys reusable plugin-backed sandbox leases scoped to an environment", async () => {
const { pluginId, runId, reusableLease } = await seedReusablePluginSandboxLease();
// The holding run is finished, so the reservation is stale and destroyable.
@@ -6796,9 +6940,51 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
});
});
it("keeps a reusable lease held by an in-flight run out of the environment-scoped destroy", async () => {
it("destroys a failed reusable sandbox after successful release before deleting its environment", async () => {
const { pluginId, companyId, runId, environment, reusableLease } = await seedReusablePluginSandboxLease();
const workerManager = {
isRunning: vi.fn((id: string) => id === pluginId),
call: vi.fn(async (_pluginId: string, method: string) => {
if (method === "environmentReleaseLease") {
return { providerLeaseId: reusableLease.providerLeaseId, state: "stopped" };
}
if (method === "environmentDestroyLease") {
// The provider call must already have a durable cleanup reference.
expect(await environmentService(db).getLeaseById(reusableLease.id)).toMatchObject({
status: "pending_cleanup", environmentId: environment.id,
});
return { providerLeaseId: reusableLease.providerLeaseId, state: "destroyed" };
}
throw new Error(`Unexpected plugin method: ${method}`);
}),
getWorker: vi.fn(() => ({ supportedMethods: ["environmentReleaseLease", "environmentDestroyLease"] })),
} as unknown as PluginWorkerManager;
const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, runId));
const released = await runtimeWithPlugin.releaseRunLeases(runId, "failed", undefined, undefined, true);
expect(released[0]?.lease).toMatchObject({
id: reusableLease.id, status: "failed", cleanupStatus: "success",
metadata: { remoteExecutionTermination: { state: "stopped" } },
});
const result = await runtimeWithPlugin.destroyReusableSandboxLeasesForEnvironment({
environmentId: environment.id, failureReason: "environment_deleted",
});
expect(result).toEqual({ destroyed: 1, failed: 0, skippedLiveRun: 0 });
expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentDestroyLease",
expect.objectContaining({ companyId, environmentId: environment.id, providerLeaseId: reusableLease.providerLeaseId }),
31234);
await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({
status: "expired", cleanupStatus: "success",
metadata: { remoteExecutionTermination: { state: "destroyed" } },
});
expect((await environmentService(db).removeIfDeletable(environment.id))?.id).toBe(environment.id);
});
it.each(["active", "failed"] as const)("keeps a %s reusable lease held by an in-flight run out of the environment-scoped destroy", async (status) => {
const { pluginId, reusableLease } = await seedReusablePluginSandboxLease();
// seedEnvironment leaves the holding run in `running` status.
if (status === "failed") await environmentService(db).releaseLease(reusableLease.id, "failed");
const workerManager = {
isRunning: vi.fn((id: string) => id === pluginId),
@@ -6816,7 +7002,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
expect(workerManager.call).not.toHaveBeenCalled();
// The lease keeps its reusable status, so the delete guard still blocks.
await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({
status: "active",
status,
leasePolicy: "reuse_by_environment",
});
});
@@ -636,6 +636,18 @@ describeEmbeddedPostgres("environmentService leases", () => {
expect(releasedImpact?.reusableSandboxLeaseCount).toBe(1);
expect(await svc.removeIfDeletable(environmentId)).toBeNull();
// A failed run can release its reusable sandbox successfully without
// destroying it. The provider handle still needs environment-scoped cleanup.
await svc.releaseLease(lease.id, "failed", {
failureReason: "adapter_or_run_failure",
cleanupStatus: "success",
});
const failedImpact = await svc.getDeleteBlastRadius(environmentId);
expect.soft(failedImpact?.canDelete).toBe(false);
expect.soft(failedImpact?.deleteBlockedReasons).toContain("reusable_sandbox_lease");
expect.soft(failedImpact?.reusableSandboxLeaseCount).toBe(1);
expect(await svc.removeIfDeletable(environmentId)).toBeNull();
const rows = await db.select().from(environments).where(eq(environments.id, environmentId));
expect(rows).toHaveLength(1);
const storedLease = await svc.getLeaseById(lease.id);
@@ -3,6 +3,7 @@ import { PassThrough } from "node:stream";
import { describe, expect, it } from "vitest";
import {
createRequest, definePlugin, isJsonRpcResponse, JsonRpcCallError, parseMessage,
readEnvironmentSyncErrorDiagnostic, withEnvironmentSyncTransferStep, recordEnvironmentSyncError,
PLUGIN_RPC_ERROR_CODES, serializeMessage, startWorkerRpcHost, type JsonRpcResponse,
} from "@paperclipai/plugin-sdk";
import {
@@ -53,8 +54,9 @@ describe("environment sync-out restore diagnostics", () => {
await expect(withWorkspaceRestoreDiagnostics("workspace", () => withWorkspaceRestoreStep("workspace_transfer", async () => {
throw preserveEnvironmentSyncOutErrorDiagnostic(error);
}), async (line) => { logs.push(line); })).rejects.toBe(error);
expect(getWorkspaceRestoreDiagnostic(error)).toEqual({ phase: "workspace", step: "workspace_transfer", errorCode: "ETIMEDOUT", httpStatus: 504 });
expect(logs).toEqual(['[paperclip] Workspace restore diagnostic: {"phase":"workspace","step":"workspace_transfer","errorCode":"ETIMEDOUT","httpStatus":504}\n']);
expect(getWorkspaceRestoreDiagnostic(error)).toEqual({ phase: "workspace", step: "workspace_transfer", errorCode: "ETIMEDOUT", httpStatus: 504, rpcCode: PLUGIN_RPC_ERROR_CODES.WORKER_ERROR });
expect(logs).toHaveLength(1);
expect(JSON.parse(logs[0].slice(logs[0].indexOf("{")))).toEqual(getWorkspaceRestoreDiagnostic(error));
expect({ message: error.message, code: error.code, data: error.data }).toEqual(before);
expect(error).not.toHaveProperty("cause");
});
@@ -76,3 +78,69 @@ describe("environment sync-out restore diagnostics", () => {
expect(preserveEnvironmentSyncOutErrorDiagnostic(null)).toBeNull();
});
});
it("isolates overlapping and sequential worker calls that throw the same object", async () => {
const original = Object.freeze(new Error("Transfer failed"));
const before = Object.getOwnPropertyDescriptors(original);
const stdin = new PassThrough();
const stdout = new PassThrough();
const lines = createInterface({ input: stdout });
let release!: () => void;
const gate = new Promise<void>(resolve => { release = resolve; });
let started!: () => void;
const ready = new Promise<void>(resolve => { started = resolve; });
const worker = startWorkerRpcHost({ stdin, stdout, plugin: definePlugin({
async setup() {},
async onEnvironmentSyncOut(params) {
if (params.operations?.[0]?.operationId === "first") {
return withEnvironmentSyncTransferStep("archive_create", async () => {
recordEnvironmentSyncError(original, { exitCode: 2, transferFailureKind: "command_failed" });
started(); await gate; throw original;
});
}
if (params.operations?.[0]?.operationId === "second") {
return withEnvironmentSyncTransferStep("file_download", async () => { throw original; });
}
throw original;
},
async onEnvironmentSyncIn() { throw original; },
}) });
const call = (id: string, method = "environmentSyncOut") => new Promise<JsonRpcCallError>((resolve) => {
const listener = (wire: string) => {
const message = parseMessage(wire);
if (isJsonRpcResponse(message) && message.id === id && message.error) {
lines.off("line", listener); resolve(new JsonRpcCallError(message.error));
}
};
lines.on("line", listener);
stdin.write(serializeMessage(createRequest(method, { operations: [{ operationId: id, files: [] }] }, id)));
});
try {
const first = call("first");
await ready;
const second = await call("second");
expect(readEnvironmentSyncErrorDiagnostic(second)).toEqual({ errorCode: "unknown", transferStep: "file_download" });
release();
expect(readEnvironmentSyncErrorDiagnostic(await first)).toEqual({
errorCode: "unknown", transferStep: "archive_create", transferFailureKind: "command_failed", exitCode: 2,
});
expect((await call("third")).data).toBeUndefined();
expect((await call("unrelated", "environmentSyncIn")).data).toBeUndefined();
expect(Object.getOwnPropertyDescriptors(original)).toEqual(before);
} finally { release(); worker.stop(); lines.close(); stdin.destroy(); stdout.destroy(); }
});
it.each([
[new JsonRpcCallError({ code: PLUGIN_RPC_ERROR_CODES.TIMEOUT, message: "private-rpc-message" }), PLUGIN_RPC_ERROR_CODES.TIMEOUT],
[new JsonRpcCallError({ code: 123, message: "private-rpc-message" }), undefined],
[Object.assign(new Error("private-rpc-message"), { code: PLUGIN_RPC_ERROR_CODES.TIMEOUT }), undefined],
])("records known typed RPC codes without interpreting arbitrary errors", async (error, rpcCode) => {
const logs: string[] = [];
await expect(withWorkspaceRestoreDiagnostics("workspace", () => withWorkspaceRestoreStep("workspace_transfer", async () => {
throw preserveEnvironmentSyncOutErrorDiagnostic(error);
}), async line => { logs.push(line); })).rejects.toBe(error);
expect(getWorkspaceRestoreDiagnostic(error)?.rpcCode).toBe(rpcCode);
expect(getWorkspaceRestoreDiagnostic(error)?.transferStep).toBeUndefined();
expect(JSON.stringify(logs)).not.toContain("private-");
});
@@ -2,6 +2,7 @@ import type { NextFunction, Request, Response } from "express";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { HttpError } from "../errors.js";
import { errorHandler } from "../middleware/error-handler.js";
import { mcpDiscoveryHttpFailure, retainMcpConnectionFailure } from "../services/mcp-connection-failure.js";
const recordResponsibleUserDenialOnActiveRunMock = vi.hoisted(() => vi.fn());
const captureExceptionMock = vi.hoisted(() => vi.fn());
@@ -121,6 +122,35 @@ describe("errorHandler", () => {
expect(res.__errorContext?.error?.message).toBe("db exploded");
});
it("keeps HTTP failures and existing telemetry for proven external MCP outages while skipping Sentry", () => {
const req = makeReq();
const res = makeRes() as any;
const original = mcpDiscoveryHttpFailure(new Response(null, { status: 503 }), "Remote app returned HTTP 503");
const err = retainMcpConnectionFailure(original, new HttpError(502, original.message));
errorHandler(err, req, res, vi.fn());
expect(res.status).toHaveBeenCalledWith(502);
expect(res.json).toHaveBeenCalledWith({ error: original.message });
expect(res.err).toBe(err);
expect(res.__errorContext.error.message).toBe(original.message);
expect(telemetryMocks.trackErrorHandlerCrash).toHaveBeenCalledOnce();
expect(captureExceptionMock).not.toHaveBeenCalled();
});
it("reports copied connection markers and unrelated database failures", () => {
for (const err of [
Object.assign(new HttpError(502, "Remote app returned HTTP 503", { status: 503 }), {
connectionFailure: { schemaVersion: 1, provider: "mcp_http", operation: "discover_tools", reason: "remote_unavailable" },
}),
new HttpError(500, "database unavailable"),
]) {
errorHandler(err, makeReq(), makeRes(), vi.fn());
}
expect(captureExceptionMock).toHaveBeenCalledTimes(2);
expect(telemetryMocks.trackErrorHandlerCrash).toHaveBeenCalledTimes(2);
});
it("sanitizes chat setup errors before logs and crash reporting", () => {
const req = {
...makeReq(),
+34 -1
View File
@@ -1,3 +1,5 @@
import { subscribeDeliveryWork } from "../services/delivery-work-notifications.js";
import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js";
import { randomUUID } from "node:crypto";
import fs from "node:fs";
import os from "node:os";
@@ -800,6 +802,8 @@ describeEmbeddedPostgres("feedbackService.saveIssueVote", () => {
sessionId,
});
const notified = vi.fn();
const unsubscribe = subscribeDeliveryWork(db, DELIVERY_QUEUES.feedback, notified);
await flushingSvc.saveIssueVote({
issueId,
targetType: "issue_comment",
@@ -808,7 +812,12 @@ describeEmbeddedPostgres("feedbackService.saveIssueVote", () => {
authorUserId: "user-1",
allowSharing: true,
});
await flushingSvc.flushPendingFeedbackTraces();
unsubscribe();
expect(notified).toHaveBeenCalledTimes(1);
expect(await flushingSvc.hasPendingFeedbackTraces()).toBe(true);
// The route and worker may request a flush together after the commit.
await Promise.all([flushingSvc.flushPendingFeedbackTraces(), flushingSvc.flushPendingFeedbackTraces()]);
expect(await flushingSvc.hasPendingFeedbackTraces()).toBe(false);
expect(uploadTraceBundle).toHaveBeenCalledTimes(1);
const bundle = uploadTraceBundle.mock.calls[0]?.[0] as Record<string, unknown> | undefined;
@@ -1152,6 +1161,30 @@ describeEmbeddedPostgres("feedbackService.saveIssueVote", () => {
expect(uploadTraceBundle).toHaveBeenCalledTimes(1);
});
it("leaves aborted and unstarted exports available for recovery after shutdown", async () => {
const first = await seedIssueWithAgentComment();
const second = await seedIssueWithAgentComment();
const shutdown = new AbortController();
const uploadTraceBundle = vi.fn().mockResolvedValue({ objectKey: "saved" })
.mockImplementationOnce(async (_bundle, signal: AbortSignal) => {
shutdown.abort(new Error("shutdown"));
signal.throwIfAborted();
});
const flushingSvc = feedbackService(db, { shareClient: { uploadTraceBundle } });
for (const item of [first, second]) {
await flushingSvc.saveIssueVote({ issueId: item.issueId, targetType: "issue_comment",
targetId: item.commentId, vote: "up", authorUserId: "user-1", allowSharing: true });
}
expect(await flushingSvc.flushPendingFeedbackTraces({ signal: shutdown.signal }))
.toMatchObject({ attempted: 1, sent: 0, failed: 1 });
const rows = await db.select().from(feedbackExports);
expect(rows.filter(row => row.status === "failed")).toHaveLength(1);
expect(rows.filter(row => row.status === "pending")).toHaveLength(1);
expect(await flushingSvc.hasPendingFeedbackTraces()).toBe(true);
expect(await flushingSvc.flushPendingFeedbackTraces()).toMatchObject({ attempted: 2, sent: 2, failed: 0 });
expect(await flushingSvc.hasPendingFeedbackTraces()).toBe(false);
});
it("marks pending shared traces as failed when no feedback export backend is configured", async () => {
const { companyId, issueId, commentId } = await seedIssueWithAgentComment();
@@ -1,6 +1,6 @@
import { gunzipSync } from "node:zlib";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { createFeedbackTraceShareClientFromConfig } from "../services/feedback-share-client.js";
import { FEEDBACK_UPLOAD_TIMEOUT_MS, createFeedbackTraceShareClientFromConfig } from "../services/feedback-share-client.js";
describe("feedback trace share client", () => {
beforeEach(() => {
@@ -12,6 +12,7 @@ describe("feedback trace share client", () => {
afterEach(() => {
vi.restoreAllMocks();
vi.unstubAllGlobals();
});
it("defaults to telemetry.paperclip.ing when no backend url is configured", async () => {
@@ -98,4 +99,43 @@ describe("feedback trace share client", () => {
expect(parsed.objectKey.endsWith("/export-1.json")).toBe(true);
expect(parsed.bundle.envelope).toEqual({ hello: "world" });
});
it.each(["headers", "body"])("cancels a stalled upload at its deadline while waiting for %s", async phase => {
const deadline = new AbortController();
const timeout = vi.spyOn(AbortSignal, "timeout").mockReturnValue(deadline.signal);
let started!: () => void;
const waiting = new Promise<void>(resolve => { started = resolve; });
vi.mocked(fetch).mockImplementation(async (_url, options) => {
const block = () => new Promise<never>((_resolve, reject) => {
options!.signal!.addEventListener("abort", () => reject(options!.signal!.reason), { once: true });
started();
});
if (phase === "headers") return block();
return { ok: true, json: block } as unknown as Response;
});
const client = createFeedbackTraceShareClientFromConfig({ feedbackExportBackendUrl: undefined, feedbackExportBackendToken: undefined });
const bundle = { companyId: "company", traceId: "trace" } as Parameters<typeof client.uploadTraceBundle>[0];
const uploading = client.uploadTraceBundle(bundle);
const rejected = expect(uploading).rejects.toThrow("upload deadline");
await waiting;
expect(timeout).toHaveBeenCalledWith(FEEDBACK_UPLOAD_TIMEOUT_MS);
deadline.abort(new Error("upload deadline"));
await rejected;
});
it("passes shutdown cancellation through to a pending upload", async () => {
const shutdown = new AbortController();
let started!: () => void;
const waiting = new Promise<void>(resolve => { started = resolve; });
vi.mocked(fetch).mockImplementation((_url, options) => new Promise((_resolve, reject) => {
options!.signal!.addEventListener("abort", () => reject(options!.signal!.reason), { once: true });
started();
}));
const client = createFeedbackTraceShareClientFromConfig({ feedbackExportBackendUrl: undefined, feedbackExportBackendToken: undefined });
const bundle = { companyId: "company", traceId: "trace" } as Parameters<typeof client.uploadTraceBundle>[0];
const rejected = expect(client.uploadTraceBundle(bundle, shutdown.signal)).rejects.toThrow("shutdown");
await waiting;
shutdown.abort(new Error("shutdown"));
await rejected;
});
});
@@ -134,50 +134,65 @@ describe("workspace Git scan route load regression", () => {
...unavailableMethods(),
};
const app = createLoadApp(db, companyId, service);
const pendingResponses = Array.from({ length: 500 }, (_, index) => request(app)
// Model one busy API server, not 500 independent ephemeral listeners whose
// accept/close races can reset clients before they reach the scheduler.
const server = app.listen(0, "127.0.0.1");
await new Promise<void>(resolve => server.once("listening", resolve));
const pendingResponses = Array.from({ length: 500 }, (_, index) => request(server)
.get(`/api/issues/issue-${index}/file-resources/list`)
.set("x-test-actor", `actor-${index % 73}`)
.query({ mode: "changed" })
.then((response) => response));
// Observe every client rejection immediately, including on assertion failure.
const settledResponses = Promise.allSettled(pendingResponses);
try {
await vi.waitFor(
() => expect(scheduler.snapshot().totals.singleFlightJoins).toBe(498),
{ timeout: 15_000, interval: 20 },
);
const loadedSnapshot = scheduler.snapshot();
expect(loadedSnapshot).toMatchObject({ activeCount: 2, queuedCount: 0, inFlightCount: 2 });
await vi.waitFor(
() => expect(scheduler.snapshot().totals.singleFlightJoins).toBe(498),
{ timeout: 15_000, interval: 20 },
);
const loadedSnapshot = scheduler.snapshot();
expect(loadedSnapshot).toMatchObject({ activeCount: 2, queuedCount: 0, inFlightCount: 2 });
const healthLatencies: number[] = [];
for (let index = 0; index < 25; index += 1) {
const startedAt = performance.now();
const response = await request(server).get("/api/health");
healthLatencies.push(performance.now() - startedAt);
expect(response.status).toBe(200);
}
healthLatencies.sort((left, right) => left - right);
const healthP99Ms = healthLatencies[Math.ceil(healthLatencies.length * 0.99) - 1]!;
expect(healthP99Ms).toBeLessThan(250);
const healthLatencies: number[] = [];
for (let index = 0; index < 25; index += 1) {
const startedAt = performance.now();
const response = await request(app).get("/api/health");
healthLatencies.push(performance.now() - startedAt);
expect(response.status).toBe(200);
releaseScans();
const responses = (await settledResponses).map(result => {
if (result.status === "rejected") throw result.reason;
return result.value;
});
const outcomeCounts = responses.reduce<Record<number, number>>((counts, response) => {
counts[response.status] = (counts[response.status] ?? 0) + 1;
return counts;
}, {});
expect(outcomeCounts).toEqual({ 200: 500 });
expect({ runnerCalls, peakActive }).toEqual({ runnerCalls: 2, peakActive: 2 });
expect(scheduler.snapshot()).toMatchObject({ activeCount: 0, queuedCount: 0, inFlightCount: 0 });
console.info("workspace Git scan regression metrics", {
requests: responses.length,
repositories: roots.length,
underlyingScans: runnerCalls,
peakActive,
peakQueued: loadedSnapshot.queuedCount,
outcomes: outcomeCounts,
healthP99Ms: Math.round(healthP99Ms * 100) / 100,
unreapedChildCount: 0,
});
} finally {
releaseScans();
server.closeAllConnections();
await settledResponses;
await new Promise<void>((resolve, reject) => server.close(error => error ? reject(error) : resolve()));
}
healthLatencies.sort((left, right) => left - right);
const healthP99Ms = healthLatencies[Math.ceil(healthLatencies.length * 0.99) - 1]!;
expect(healthP99Ms).toBeLessThan(250);
releaseScans();
const responses = await Promise.all(pendingResponses);
const outcomeCounts = responses.reduce<Record<number, number>>((counts, response) => {
counts[response.status] = (counts[response.status] ?? 0) + 1;
return counts;
}, {});
expect(outcomeCounts).toEqual({ 200: 500 });
expect({ runnerCalls, peakActive }).toEqual({ runnerCalls: 2, peakActive: 2 });
expect(scheduler.snapshot()).toMatchObject({ activeCount: 0, queuedCount: 0, inFlightCount: 0 });
console.info("workspace Git scan regression metrics", {
requests: responses.length,
repositories: roots.length,
underlyingScans: runnerCalls,
peakActive,
peakQueued: loadedSnapshot.queuedCount,
outcomes: outcomeCounts,
healthP99Ms: Math.round(healthP99Ms * 100) / 100,
unreapedChildCount: 0,
});
}, 60_000);
it.each([
@@ -0,0 +1,19 @@
import { definePlugin, PluginEnvironmentCreationCleanupError, runWorker } from "@paperclipai/plugin-sdk";
function failure(params) {
const cleanup = {
providerLeaseId: "paperclip-create-fixture", attemptId: "fixture", companyId: params.companyId,
environmentId: params.environmentId, runId: params.runId, accountFingerprint: "a".repeat(64),
labels: { "paperclip-provider": "daytona" },
};
const diagnostic = { phase: params.config.phase ?? "create", elapsedMs: 2_000, budgetMs: 2_000,
output: "private-provider-output", command: "private-provider-command", url: "private-provider-url" };
return new PluginEnvironmentCreationCleanupError([new Error("private-provider-cause")],
"Daytona lease acquisition timed out; allocation cleanup is pending", cleanup, diagnostic);
}
runWorker(definePlugin({
async setup() {},
async onEnvironmentAcquireLease(params) { throw failure(params); },
async onEnvironmentDestroyLease(params) { throw failure(params); },
}), import.meta.url);
@@ -0,0 +1,27 @@
// Use the real SDK so host tests cross a process and the NDJSON boundary.
void Promise.all([
import("../../../../packages/plugins/sdk/src/define-plugin.ts"),
import("../../../../packages/plugins/sdk/src/worker-rpc-host.ts"),
]).then(([{ definePlugin }, { startWorkerRpcHost }]) => {
let context;
startWorkerRpcHost({ plugin: definePlugin({
async setup(ctx) {
context = ctx;
ctx.events.on("fixture.write", async () => {
await ctx.state.set({ scopeKind: "instance", stateKey: "event" }, { value: 1 });
});
},
async onIdleDrain() { return "none"; },
async onHealth() {
if (process.env.IDLE_TEST_WRITE === "1") {
await context.state.set({ scopeKind: "instance", stateKey: "fixture" }, { value: 1 });
}
return { status: "ok" };
},
async onEnvironmentProbe(params) {
if (params.config.crash) process.exit(1);
await new Promise((resolve) => setTimeout(resolve, params.config.delayMs));
return { ok: true, summary: "fixture complete" };
},
}) });
});
@@ -24,12 +24,12 @@ describe("GitHub grant metadata", () => {
}
if (url.pathname === "/user/installations/101/repositories") {
return json({ total_count: 2, repositories: secondPage
? [{ id: 2, full_name: "paperclipai/b", private: true, description: "must-not-persist", clone_url: "must-not-persist" }]
: [{ id: 1, full_name: "paperclipai/a", private: false }],
? [{ id: 2, full_name: "paperclipai/b", owner: { type: "Organization" }, private: true, description: "must-not-persist", clone_url: "must-not-persist" }]
: [{ id: 1, full_name: "paperclipai/a", owner: { type: "Organization" }, private: false }],
}, !secondPage);
}
if (url.pathname === "/user/installations/102/repositories") {
return json({ total_count: 1, repositories: [{ id: 3, full_name: "octocat/c" }] });
return json({ total_count: 1, repositories: [{ id: 3, full_name: "octocat/c", owner: { type: "User" } }] });
}
throw new Error(`Unexpected GitHub path: ${url.pathname}`);
});
@@ -44,9 +44,9 @@ describe("GitHub grant metadata", () => {
installationIds: ["101", "102"],
installationOwnerLogins: ["paperclipai", "octocat"],
repositories: [
{ id: "3", fullName: "octocat/c", installationId: "102" },
{ id: "1", fullName: "paperclipai/a", installationId: "101", private: false },
{ id: "2", fullName: "paperclipai/b", installationId: "101", private: true },
{ id: "3", fullName: "octocat/c", installationId: "102", ownerType: "personal" },
{ id: "1", fullName: "paperclipai/a", installationId: "101", ownerType: "organization", private: false },
{ id: "2", fullName: "paperclipai/b", installationId: "101", ownerType: "organization", private: true },
],
installationUrl: "https://github.com/apps/paperclip-development/installations/new",
managementUrl: "https://github.com/settings/installations/101",
@@ -301,6 +301,11 @@ describe("buildPaperclipTaskMarkdown", () => {
externalChatProvider: "slack", nativeRunner, includeDescription,
});
expect(markdown).toContain("Interactive questions in Slack:");
expect(markdown).toContain("When you have results, update the user in the originating Slack");
expect(markdown).toContain("When the user needs to review a plan, approve an action, or review a result");
expect(markdown).toContain("create the required human-input or approval interaction before you yield");
expect(markdown).toContain("Saving a plan or setting a task to in_review alone does not ask the user to review it");
expect(markdown).toContain("do not continue gated work until the required approval is recorded");
expect(markdown).toContain("ask_user_questions");
expect(markdown).toContain("single_select");
expect(markdown).toContain("resumes this task after the answer");
@@ -8,6 +8,7 @@ import { ensureManagedProjectWorkspace, prepareProjectRepositoryWorkspaces } fro
import { buildGitAuthInvocation, GIT_CREDENTIAL_TOKEN_ENV_KEY } from "../services/git-credentials.ts";
import { sanitizeRuntimeServiceBaseEnv } from "../services/workspace-runtime.ts";
import { resolveManagedProjectWorkspaceDir } from "../home-paths.ts";
import { readGitConnectionFailure } from "../services/git-connection-failure.js";
const execFile = promisify(execFileCallback);
@@ -38,6 +39,79 @@ async function createLocalSourceRepo() {
}
describe("ensureManagedProjectWorkspace clone credentials", () => {
it.each([
["authentication_failed", "fatal: Authentication failed for 'https://example.test/team/repo.git/'"],
["dns_failure", "ssh: Could not resolve hostname repository-alias.example.test: Name or service not known\nfatal: Could not read from remote repository."],
[null, "fatal: Authentication failed for 'https://example.test/team/repo.git/'\nfatal: bad config line 1 in file /tmp/config"],
[null, "fatal: corrupt loose object"],
] as const)("preserves only proven outbound clone classification through the actionable wrapper: %s", async (reason, stderr) => {
const bin = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-fake-clone-"));
await fs.writeFile(path.join(bin, "git"), `#!/bin/sh
printf '%s\\n' "$FIXTURE_CLONE_STDERR" >&2
exit 128
`, { mode: 0o755 });
try {
const failure = await ensureManagedProjectWorkspace({
companyId: "remote-clone-fixture", projectId: path.basename(bin),
repoUrl: "https://fixture-user:fixture-password@example.test/team/repo.git",
resolveGitAuth: async () => ({
configArgs: [], env: { PATH: bin, FIXTURE_CLONE_STDERR: stderr },
source: "company_secret", secretName: "GITHUB_TOKEN",
}),
}).catch((error: unknown) => error);
expect(failure).toBeInstanceOf(Error);
expect((failure as Error).message).toContain("Failed to prepare managed checkout");
expect((failure as Error).message).not.toContain("fixture-password");
expect(readGitConnectionFailure(failure)).toEqual(reason ? {
schemaVersion: 1, provider: "git", operation: "clone", reason,
} : null);
const homeEntries = await fs.readdir(path.dirname(resolveManagedProjectWorkspaceDir({
companyId: "remote-clone-fixture", projectId: path.basename(bin), repoName: "repo",
})));
expect(homeEntries.some((name) => name.includes(".clone-"))).toBe(false);
} finally {
await fs.rm(bin, { recursive: true, force: true });
}
});
it("does not classify a credential-provider exception that resembles a Git failure", async () => {
const failure = Object.assign(new Error("fatal: Authentication failed for 'https://example.test/repo.git'"), {
code: 128, stderr: "fatal: Authentication failed for 'https://example.test/repo.git'",
});
const caught = await ensureManagedProjectWorkspace({
companyId: "credential-provider-fixture", projectId: "project", repoUrl: "https://example.test/repo.git",
resolveGitAuth: async () => { throw failure; },
}).catch((error: unknown) => error);
expect(caught).toBe(failure);
expect(readGitConnectionFailure(caught)).toBeNull();
});
it("does not transfer clone classification to a later local checkout failure", async () => {
const bin = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-fake-checkout-"));
const previousPath = process.env.PATH;
await fs.writeFile(path.join(bin, "git"), `#!/bin/sh
if [ "$1" = clone ]; then
for argument in "$@"; do destination="$argument"; done
/bin/mkdir -p "$destination/.git"
exit 0
fi
printf '%s\\n' "fatal: Authentication failed for 'https://example.test/repo.git'" >&2
exit 128
`, { mode: 0o755 });
try {
process.env.PATH = bin;
const failure = await ensureManagedProjectWorkspace({
companyId: "checkout-failure-fixture", projectId: "project", repoUrl: "https://example.test/repo.git", repoRef: "missing-branch",
}).catch((error: unknown) => error);
expect(failure).toBeInstanceOf(Error);
expect((failure as Error).message).toContain("Authentication failed");
expect(readGitConnectionFailure(failure)).toBeNull();
} finally {
if (previousPath === undefined) delete process.env.PATH; else process.env.PATH = previousPath;
await fs.rm(bin, { recursive: true, force: true });
}
});
it("materializes every repository-only project row inside the task workspace and reuses local edits", async () => {
const first = await createLocalSourceRepo();
const second = await createLocalSourceRepo();
@@ -41,6 +41,13 @@ describe("native heartbeat cancellation authority", () => {
);
});
it("passes the caller identity into the audited native boundary", async () => {
const cancel = vi.fn(async () => ({ decision: {}, auditId: "audit" }));
const db = {} as Db, cancellationRequestId = "11111111-1111-4111-8111-111111111111";
await cancelHeartbeatNativeRun({ db, runId: "run", reason: "Stop", runtimeMode: "native", cancellationRequestId, cancel });
expect(cancel).toHaveBeenCalledWith("run", "Stop", { db, scope: "run", cancellationRequestId });
});
it("fails closed when a native cancellation lacks its decision audit", async () => {
const cancel = vi.fn(async () => ({
decision: null,
@@ -1,3 +1,13 @@
import * as conversationContinuation from "../services/conversation-continuation.js";
import { environmentRuntimeService } from "../services/environment-runtime.js";
import * as workspaceRuntime from "../services/workspace-runtime.js";
import { ensureNativeCompletionContract } from "../services/native-runtime/completion-contracts.js";
import { readNativePlanWait, hasCommittedNativePlanWait } from "../services/native-runtime/native-plan-wait.js";
import { nativeSha256 } from "../services/native-runtime/canonical.js";
import { buildQuestionResponseDeliveryEnvelope } from "../services/question-response-delivery.js";
import * as aiConnectionRuntime from "../services/ai-connection-runtime.js";
import { forbidden, unprocessable } from "../errors.js";
import { aiConnectionCredentialNotSharedFailure } from "../services/ai-connection-configuration-failure.js";
import * as executionContinuation from "../services/execution-continuation.js";
import * as environmentOrchestrator from "../services/environment-run-orchestrator.js";
import { remoteTerminationReceipt } from "../services/remote-execution-termination.js";
@@ -12,7 +22,7 @@ import { recordLegacyWorkspaceRestoreFailure, terminalizeLegacyExecution } from
import { issueService } from "../services/issues.js";
import { getExecutionBlocker } from "../services/execution-blocker.js";
import { adapterExecutionControls, createAdapterExecutionControl } from "../services/adapter-execution-control.js";
import { spawn, type ChildProcess } from "node:child_process";
import { execFileSync, spawn, type ChildProcess } from "node:child_process";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
@@ -21,6 +31,7 @@ import {
afterAll,
afterEach,
beforeAll,
beforeEach,
describe,
expect,
it,
@@ -44,6 +55,7 @@ import {
chatExternalPrincipals,
chatMessageLinks,
chatPublications,
companyMemberships,
companySecretBindings,
companySecrets,
companySkills,
@@ -63,6 +75,7 @@ import {
heartbeatRunEvents,
heartbeatRuns,
issueComments,
issueQuestionResponseDeliveries,
issueApprovals,
issueDocuments,
issuePlanDecompositions,
@@ -1522,6 +1535,133 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
return { companyId, agentId, runId, wakeupRequestId, issueId };
}
it.each(["owned", "forged"] as const)("keeps unresolved-ref recovery and reporting while persisting only owned diagnostics: %s", async provenance => {
const { companyId, agentId, issueId, runId, wakeupRequestId } = await seedQueuedIssueRunFixture();
const repoRoot = await fs.mkdtemp(path.join(os.tmpdir(), "base-ref-heartbeat-"));
execFileSync("git", ["init", "-b", "master"], { cwd: repoRoot, stdio: "ignore" });
execFileSync("git", ["-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid", "commit", "--allow-empty", "-m", "Fixture"], { cwd: repoRoot, stdio: "ignore" });
const original = await workspaceRuntime.realizeExecutionWorkspace({
base: { baseCwd: repoRoot, source: "project_primary", projectId: null, workspaceId: null, repoUrl: null, repoRef: "absent-fixture" },
config: { workspaceStrategy: { type: "git_worktree" } },
issue: { id: issueId, identifier: "FIXTURE-1", title: "Fixture" }, agent: { id: agentId, companyId, name: "Fixture" },
}).catch(error => error);
expect(original).toBeInstanceOf(workspaceRuntime.UnresolvedWorkspaceBaseRefError);
const diagnostic = workspaceRuntime.readUnresolvedWorkspaceBaseRefDiagnostic(original);
expect(diagnostic).toMatchObject({ remoteLookup: "failed", fetch: "not_attempted" });
const error = provenance === "owned" ? original : Object.assign(new workspaceRuntime.UnresolvedWorkspaceBaseRefError({
requestedRef: "absent-fixture", recoveryIdentityRef: "origin/absent-fixture", attemptedRefs: ["origin/absent-fixture"],
}), { baseRefDiagnostic: diagnostic });
const realize = vi.spyOn(workspaceRuntime, "realizeExecutionWorkspace").mockRejectedValueOnce(error);
try {
const heartbeat = heartbeatService(db);
await heartbeat.resumeQueuedRuns();
await waitForRunToSettle(heartbeat, runId, 5_000);
await heartbeat.waitForRunExecutionDrain(runId);
await waitForPendingRunFailureReports();
expect(realize).toHaveBeenCalled();
expect(mockAdapterExecute).not.toHaveBeenCalled();
const failed = await heartbeat.getRun(runId);
expect(failed).toMatchObject({ status: "failed", errorCode: "configuration_incomplete", resultJson: {
configurationIncomplete: { reason: "workspace_base_ref_unresolved" },
executionRecovery: { kind: "bootstrap", providerWorkStarted: false },
} });
if (provenance === "owned") expect(failed?.resultJson?.configurationIncomplete).toHaveProperty("baseRefDiagnostic", diagnostic);
else expect(failed?.resultJson?.configurationIncomplete).not.toHaveProperty("baseRefDiagnostic");
expect(mockCaptureRunFailure).toHaveBeenCalledWith(expect.objectContaining({ runId, errorCode: "configuration_incomplete" }));
const captured = mockCaptureRunFailure.mock.calls.find(([entry]) => entry.runId === runId)?.[0];
if (provenance === "owned") expect(captured?.diagnostics.execution).toMatchObject({ workspaceBaseRefRemoteLookup: "failed", workspaceBaseRefFetch: "not_attempted" });
else expect(captured?.diagnostics.execution).not.toHaveProperty("workspaceBaseRefFetch");
const [issue] = await db.select().from(issues).where(eq(issues.id, issueId));
expect(issue).toMatchObject({ status: "blocked", executionRunId: null });
const [action] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId));
expect(action).toMatchObject({ status: "active", kind: "configuration_validation", ownerType: "board" });
const [wakeup] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeupRequestId));
expect(wakeup.status).toBe("failed");
} finally { realize.mockRestore(); await fs.rm(repoRoot, { recursive: true, force: true }); }
});
it("keeps a repair notice on the latest run when an unresolved branch fails twice", async () => {
const { agentId, issueId, runId } = await seedQueuedIssueRunFixture();
const error = new workspaceRuntime.UnresolvedWorkspaceBaseRefError({
requestedRef: "main", recoveryIdentityRef: "origin/main", attemptedRefs: ["origin/main"], defaultBranch: "master",
});
const realize = vi.spyOn(workspaceRuntime, "realizeExecutionWorkspace").mockRejectedValue(error);
try {
const heartbeat = heartbeatService(db);
await heartbeat.resumeQueuedRuns();
await waitForRunToSettle(heartbeat, runId, 5_000);
await heartbeat.waitForRunExecutionDrain(runId);
const [firstAction] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId));
const next = await heartbeat.wakeup(agentId, { source: "on_demand", triggerDetail: "manual",
reason: "retry_failed_run", failedRunId: runId, payload: { issueId }, contextSnapshot: { issueId },
requestedByActorType: "user", requestedByActorId: "responsible-user" });
expect(next).toHaveProperty("id");
await waitForRunToSettle(heartbeat, next!.id, 5_000);
await heartbeat.waitForRunExecutionDrain(next!.id);
const actions = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId));
expect(actions).toHaveLength(1);
expect(actions[0]).toMatchObject({ id: firstAction.id, status: "active", evidence: { latestRunId: next!.id } });
const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId));
expect(comments.filter(c => c.authorType === "system" && c.metadata?.sourceRunId === next!.id)).toHaveLength(1);
expect(comments.filter(c => c.authorType === "system" && c.metadata?.sourceRunId === runId)).toHaveLength(1);
expect(mockAdapterExecute).not.toHaveBeenCalled();
} finally { realize.mockRestore(); }
});
it.each(["missing_default", "credential_not_shared", "database_error", "unmarked_http_error", "unmarked_forbidden", "provider_error"] as const)(
"preserves AI configuration recovery and reports only unexpected causes: %s", async (cause) => {
const { companyId, agentId, issueId, runId, wakeupRequestId } = await seedQueuedIssueRunFixture();
await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: "responsible-user", status: "active", membershipRole: "member" });
await db.update(agents).set({ runtimeConfig: {
heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 },
aiConnection: { provider: "openai", method: "api_key", mode: "responsible_user" },
} }).where(eq(agents.id, agentId));
const prepare = vi.spyOn(aiConnectionRuntime, "prepareManagedAiRuntime");
if (cause !== "missing_default") prepare.mockRejectedValueOnce(
cause === "credential_not_shared"
? aiConnectionCredentialNotSharedFailure()
: cause === "unmarked_forbidden"
? forbidden("This credential is not shared with the responsible user", { code: "ai_connection_credential_not_shared" })
: cause === "unmarked_http_error"
? unprocessable("Connect an account and choose your personal default", { code: "ai_connection_default_missing" })
: new Error(cause),
);
try {
const heartbeat = heartbeatService(db);
await heartbeat.resumeQueuedRuns();
await waitForRunToSettle(heartbeat, runId, 5_000);
await heartbeat.waitForRunExecutionDrain(runId);
await waitForPendingRunFailureReports();
expect(prepare).toHaveBeenCalled();
const failed = await heartbeat.getRun(runId);
expect(failed).toMatchObject({ status: "failed", errorCode: "configuration_incomplete",
resultJson: { configurationIncomplete: { reason: "ai_connection_unavailable" }, executionRecovery: { kind: "bootstrap", providerWorkStarted: false } },
});
if (cause === "missing_default" || cause === "credential_not_shared") {
expect(failed?.error).toBe(cause === "missing_default"
? "Connect an account and choose your personal default"
: "This credential is not shared with the responsible user");
expect(failed?.resultJson?.configurationIncomplete).toMatchObject({ selectionFailure: cause === "missing_default"
? "ai_connection_default_missing" : "ai_connection_credential_not_shared" });
expect(mockCaptureRunFailure).not.toHaveBeenCalled();
} else {
expect(failed?.resultJson?.configurationIncomplete).not.toHaveProperty("selectionFailure");
expect(mockCaptureRunFailure).toHaveBeenCalledWith(expect.objectContaining({ runId, errorCode: "configuration_incomplete" }));
}
expect(mockAdapterExecute).not.toHaveBeenCalled();
const [issue] = await db.select().from(issues).where(eq(issues.id, issueId));
expect(issue).toMatchObject({ status: "blocked", executionRunId: null });
const [action] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId));
expect(action).toMatchObject({ status: "active", kind: "configuration_validation", cause: "configuration_incomplete", ownerType: "board" });
const [wakeup] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeupRequestId));
expect(wakeup.status).toBe("failed");
} finally {
prepare.mockRestore();
await db.delete(companyMemberships).where(eq(companyMemberships.companyId, companyId));
}
},
);
it.each(["timeout", "upstream", "cleanup_pending", "cleanup_pending_edited", "cleanup_pending_exhausted", "edited", "exhausted", "new_message", "no_claim", "reassigned", "superseded", "stopped"])(
"settles explicit retry admission through real executor cleanup: %s", async scenario => {
const { companyId, agentId, runId, issueId } = await seedQueuedIssueRunFixture();
@@ -2032,6 +2172,259 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
expect(result.continuationRequeued).toBe(0);
});
async function seedPendingNativeFinalization(accept = true) {
const fixture = await seedRunFixture({ runtimeMode: "native", adapterType: "paperclip_runner", processPid: 2_000_000_000 });
const { companyId, agentId, issueId, runId } = fixture;
const contractId = randomUUID(), runnerInstanceId = randomUUID();
const contractSha = `finalization-contract-${runId}`;
await db.insert(completionContracts).values({
id: contractId, companyId, issueId, revision: 1,
schemaVersion: "paperclip.completion-contract.v1", policyVersion: "phase6-v1",
risk: "low", completionAuthority: "agent_claim_policy", incompleteCriteriaPolicy: "preserve_non_terminal",
contractJson: { revision: CONTROL_PLANE_CONFORMANCE_RESULT.completionClaim.contractRevision,
objective: "Keep completed work", criteria: [{ id: "objective", requirement: "Keep completed work" }] },
canonicalSha256: contractSha, createdByActorType: "system", createdByActorId: "test",
});
await db.update(heartbeatRuns).set({ nativeIssueId: issueId, nativeSessionId: runId,
runnerInstanceId, completionContractId: contractId, completionContractSha256: contractSha,
}).where(eq(heartbeatRuns.id, runId));
const makePort = (targetDb = db) => new PaperclipControlPlanePort(targetDb, { companyId, issueId, runId, agentId, sessionId: runId,
completionContractId: contractId, completionContractSha256: contractSha,
sourceInstanceId: runnerInstanceId, controlPlaneSourceInstanceId: `finalization-control-${runId}`,
});
const port = makePort();
await port.openRun({ ...CONTROL_PLANE_CONFORMANCE_OPEN,
identity: { companyId, issueId, runId, agentId, sessionId: runId }, sourceInstanceId: runnerInstanceId });
const complete = (targetDb = db) => makePort(targetDb).completeRun({ result: CONTROL_PLANE_CONFORMANCE_RESULT,
terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, callerResultId: `finalization-result-${runId}` });
if (accept) await complete();
const [local] = await db.select().from(environments).where(eq(environments.driver, "local"));
const lease = local
? { environmentId: local.id, leaseId: (await db.insert(environmentLeases).values({
companyId, issueId, heartbeatRunId: runId, environmentId: local.id,
status: "active", leasePolicy: "ephemeral", provider: "local", metadata: { driver: "local" },
updatedAt: new Date(0),
}).returning())[0].id }
: await seedEnvironmentLeaseFixture(fixture);
return { ...fixture, ...lease, complete };
}
it.each(["foreign", "malformed", "expired_controller"] as const)(
"preserves native finalization source and board hold after restart with %s owner",
async (kind) => {
const f = await seedPendingNativeFinalization();
const owner = kind === "malformed" ? { unknown: true } : {
token: randomUUID(), controllerBootId: randomUUID(),
hostname: kind === "foreign" ? "previous-controller.invalid" : os.hostname(),
pid: 2_000_000_000, processStartedAt: new Date(0).toISOString(),
};
await db.update(heartbeatRuns).set({ runnerProfileJson: { nativeWorkspaceFinalizationOwner: owner } })
.where(eq(heartbeatRuns.id, f.runId));
await db.update(nativeRunFinalizations).set({ leaseOwner: null, leaseExpiresAt: new Date(0),
controllerPid: 2_000_000_000, controllerProcessStartedAt: new Date(0),
}).where(eq(nativeRunFinalizations.runId, f.runId));
const accepted = await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, f.runId));
const runtime = environmentRuntimeService(db);
const release = vi.spyOn(runtime, "releaseRunLeases");
const destroy = vi.spyOn(runtime, "destroyRunLease");
const heartbeat = heartbeatService(db, { environmentRuntime: runtime });
await heartbeat.recoverNativeRunsAfterRestart();
const [hold] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId));
expect(hold).toMatchObject({ status: "active", cause: "native_workspace_finalization_owner_unverified", ownerType: "board", wakePolicy: null });
for (let tick = 0; tick < 2; tick += 1) {
expect(await heartbeat.reapOrphanedRuns()).toEqual({ reaped: 0, runIds: [] });
// The held run itself is an execution path; it does not settle copyback.
await heartbeat.reconcileStrandedAssignedIssues();
const actions = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId));
expect(actions).toHaveLength(1);
expect(actions[0]).toMatchObject({ id: hold.id });
expect(["active", "escalated"]).toContain(actions[0].status);
}
expect((await heartbeat.getRun(f.runId))).toMatchObject({ status: "running", nativePhase: "workspace_finalizing", errorCode: null,
runnerProfileJson: { nativeWorkspaceFinalizationOwner: owner } });
expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.leaseId)))[0])
.toMatchObject({ status: "active", releasedAt: null });
expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, f.runId))).toEqual(accepted);
expect(await db.select().from(workspaceOperations).where(eq(workspaceOperations.heartbeatRunId, f.runId))).toEqual([]);
expect(release).not.toHaveBeenCalled();
expect(destroy).not.toHaveBeenCalled();
expect(mockAdapterExecute).not.toHaveBeenCalled();
expect(mockExecutePaperclipNativeSession).not.toHaveBeenCalled();
await waitForPendingRunFailureReports();
expect(mockCaptureRunFailure).not.toHaveBeenCalled();
await heartbeat.drainActiveRunExecutions();
},
);
it("rejects a stale orphan candidate when native finalization commits before owner acquisition", async () => {
const f = await seedPendingNativeFinalization(false);
// Avoid the unrelated observed-owner gate while the old enumeration still
// lacks a result. Publish through the real result transaction just before
// generic orphan recovery reaches its conditional status write.
await db.update(nativeRunFinalizations).set({ phase: "workspace_finalizing" }).where(eq(nativeRunFinalizations.runId, f.runId));
const usedConversation = vi.spyOn(conversationContinuation, "runUsedConversationAdapter").mockImplementationOnce(async () => {
await f.complete();
return false;
});
const runtime = environmentRuntimeService(db);
const release = vi.spyOn(runtime, "releaseRunLeases");
try {
expect(await heartbeatService(db, { environmentRuntime: runtime }).reapOrphanedRuns()).toEqual({ reaped: 0, runIds: [] });
expect(usedConversation).toHaveBeenCalledOnce();
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0])
.toMatchObject({ status: "running", nativePhase: "workspace_finalizing", errorCode: null });
expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.leaseId)))[0].status).toBe("active");
expect(release).not.toHaveBeenCalled();
} finally { usedConversation.mockRestore(); }
});
it("fences a native finalization publisher that commits while the orphan UPDATE waits on its row lock", async () => {
const f = await seedPendingNativeFinalization(false);
await db.update(nativeRunFinalizations).set({ phase: "workspace_finalizing" }).where(eq(nativeRunFinalizations.runId, f.runId));
let release!: () => void;
const gate = new Promise<void>(resolve => { release = resolve; });
let entered!: () => void;
const writerEntered = new Promise<void>(resolve => { entered = resolve; });
let publisherPid = 0;
let publisher: Promise<void> | undefined;
const usedConversation = vi.spyOn(conversationContinuation, "runUsedConversationAdapter").mockImplementationOnce(async () => {
publisher = db.transaction(async tx => {
const rows = await tx.execute(sql`select pg_backend_pid() as pid`);
publisherPid = Number(rows[0].pid);
// This is the real publication protocol, not a synthetic version bump:
// completeRun locks heartbeat, accepts the result, and writes its phase.
await f.complete(tx as unknown as typeof db);
entered();
await gate;
});
await writerEntered;
return false;
});
const runtime = environmentRuntimeService(db);
const releaseLease = vi.spyOn(runtime, "releaseRunLeases");
const heartbeat = heartbeatService(db, { environmentRuntime: runtime });
const reaping = heartbeat.reapOrphanedRuns();
try {
await writerEntered;
await vi.waitFor(async () => {
const waiting = await db.execute(sql`select pid from pg_stat_activity
where ${publisherPid} = any(pg_blocking_pids(pid)) and wait_event_type = 'Lock'`);
expect(waiting.length).toBeGreaterThan(0);
}, { timeout: 5_000 });
release();
await publisher;
expect(await reaping).toEqual({ reaped: 0, runIds: [] });
expect((await heartbeat.getRun(f.runId))).toMatchObject({ status: "running", errorCode: null, nativePhase: "workspace_finalizing" });
expect((await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, f.runId)))).toHaveLength(1);
expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.leaseId)))[0].status).toBe("active");
expect(releaseLease).not.toHaveBeenCalled();
expect(mockAdapterExecute).not.toHaveBeenCalled();
expect(mockExecutePaperclipNativeSession).not.toHaveBeenCalled();
await waitForPendingRunFailureReports();
expect(mockCaptureRunFailure).not.toHaveBeenCalled();
} finally {
release();
await publisher?.catch(() => undefined);
await reaping.catch(() => undefined);
usedConversation.mockRestore();
}
});
it.each(["phase_only", "rejected_result", "terminal_failure", "unknown_phase"] as const)(
"does not invent native finalization authority from %s",
async (kind) => {
const f = await seedPendingNativeFinalization(kind !== "phase_only");
await db.update(heartbeatRuns).set({ nativePhase: "workspace_finalizing",
runnerProfileJson: { nativeWorkspaceFinalizationOwner: { unknown: true } },
}).where(eq(heartbeatRuns.id, f.runId));
await db.update(nativeRunFinalizations).set({ phase: kind === "terminal_failure" ? "terminal_failure"
: kind === "unknown_phase" ? "unknown_phase" : "workspace_finalizing",
// Prevent the finalizer from changing this negative control first.
leaseOwner: "unverified-old-owner", leaseExpiresAt: new Date(Date.now() + 60_000),
}).where(eq(nativeRunFinalizations.runId, f.runId));
if (kind === "rejected_result") await db.update(nativeRunResults).set({ schemaStatus: "rejected" }).where(eq(nativeRunResults.runId, f.runId));
const result = await heartbeatService(db).reapOrphanedRuns();
expect(result.runIds).toContain(f.runId);
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0])
.toMatchObject({ status: "failed", errorCode: "process_lost" });
},
);
it("resolves the exact native finalization hold after verified workspace settlement without replay", async () => {
const f = await seedPendingNativeFinalization();
await db.update(heartbeatRuns).set({ runnerProfileJson: { nativeWorkspaceFinalizationOwner: {
token: randomUUID(), hostname: "previous-controller.invalid", pid: 2_000_000_000, processStartedAt: new Date(0).toISOString(),
} } }).where(eq(heartbeatRuns.id, f.runId));
const heartbeat = heartbeatService(db);
await heartbeat.recoverNativeRunsAfterRestart();
const [hold] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId));
const accepted = await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, f.runId));
// Model exact operator stop verification and a completed copyback receipt;
// no test or product path infers these facts from a dead provider PID.
await db.update(heartbeatRuns).set({ runnerProfileJson: {} }).where(eq(heartbeatRuns.id, f.runId));
await db.insert(workspaceOperations).values({ companyId: f.companyId, issueId: f.issueId,
heartbeatRunId: f.runId, phase: "workspace_finalize", status: "succeeded" });
await heartbeat.reapOrphanedRuns();
expect((await heartbeat.getRun(f.runId))).toMatchObject({ status: "succeeded", nativePhase: "committed" });
expect((await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, hold.id)))[0])
.toMatchObject({ status: "resolved", outcome: "restored" });
expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, f.runId))).toEqual(accepted);
expect(mockExecutePaperclipNativeSession).not.toHaveBeenCalled();
expect(mockAdapterExecute).not.toHaveBeenCalled();
await heartbeat.drainActiveRunExecutions();
});
it("rejects foreign result, company, and issue bindings for native finalization", async () => {
const source = await seedPendingNativeFinalization();
const other = await seedPendingNativeFinalization();
const [otherResult] = await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, other.runId));
for (const patch of [{ resultId: otherResult.id }, { companyId: other.companyId }, { issueId: other.issueId }]) {
await expect(db.update(nativeRunFinalizations).set(patch).where(eq(nativeRunFinalizations.runId, source.runId)))
.rejects.toMatchObject({ cause: { code: "23503" } });
}
// The composite foreign keys are part of the guard's durable authority;
// invalid foreign bindings never become a candidate through a valid write.
expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, source.runId)))[0])
.toMatchObject({ companyId: source.companyId, issueId: source.issueId, phase: "workspace_finalizing" });
});
it("does not extend native finalization authority to a legacy runtime", async () => {
const f = await seedPendingNativeFinalization();
await db.update(heartbeatRuns).set({ runtimeMode: "legacy" }).where(eq(heartbeatRuns.id, f.runId));
expect((await heartbeatService(db).reapOrphanedRuns()).runIds).toContain(f.runId);
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0])
.toMatchObject({ status: "failed", errorCode: "process_lost" });
});
it("does not clear another run's native finalization hold on the same task", async () => {
const f = await seedPendingNativeFinalization();
const [otherHold] = await db.insert(issueRecoveryActions).values({ companyId: f.companyId,
sourceIssueId: f.issueId, kind: "active_run_watchdog", status: "active", ownerType: "board",
cause: "native_workspace_finalization_owner_unverified", fingerprint: randomUUID(),
evidence: { runId: randomUUID() }, nextAction: "Verify the other physical owner has stopped.",
}).returning();
await db.insert(workspaceOperations).values({ companyId: f.companyId, issueId: f.issueId,
heartbeatRunId: f.runId, phase: "workspace_finalize", status: "succeeded" });
await expect(finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true }))
.resolves.toMatchObject({ phase: "committed" });
expect((await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, otherHold.id)))[0])
.toMatchObject({ status: "active", resolvedAt: null });
});
it("keeps existing terminal-task disposition for a native finalization hold", async () => {
const f = await seedPendingNativeFinalization();
await db.update(heartbeatRuns).set({ runnerProfileJson: { nativeWorkspaceFinalizationOwner: { unknown: true } } })
.where(eq(heartbeatRuns.id, f.runId));
const heartbeat = heartbeatService(db);
await heartbeat.recoverNativeRunsAfterRestart();
const [hold] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId));
await db.update(issues).set({ status: "done" }).where(eq(issues.id, f.issueId));
await heartbeat.reconcileStrandedAssignedIssues();
expect((await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, hold.id)))[0])
.toMatchObject({ status: "resolved", resolutionNote: "source_terminal" });
await heartbeat.drainActiveRunExecutions();
});
it.each(["settled", "rejected", "late_callback", "provider_transport_failed"] as const)(
"joins startup and reap retained cleanup, keeps recovery live, and drains its %s operation",
async (outcome) => {
@@ -5591,6 +5984,179 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
}
});
it.each([
{ name: "HTTPS authentication", failures: ["auth"], report: false },
{ name: "SSH alias DNS", failures: ["dns"], report: false },
{ name: "mixed recognized and unknown candidates", failures: ["auth", "unknown"], report: true },
{ name: "mixed local and authentication diagnostics", failures: ["mixed"], report: true },
{ name: "unknown clone failure", failures: ["unknown"], report: true },
])("preserves failed workspace recovery while classifying managed Git connections: $name", async ({ failures, report }) => {
const { companyId, agentId, runId, issueId } = await seedQueuedIssueRunFixture();
const home = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-git-connection-lifecycle-"));
const bin = path.join(home, "bin");
const callsPath = path.join(home, "git-calls.jsonl");
const projectId = randomUUID();
const workspaceIds = failures.map(() => randomUUID());
const remotes = failures.map((failure, index) => failure === "dns"
? `git@clone-alias.example.test:team/repository-${index}.git`
: `https://clone.example.test/team/repository-${index}.git`);
const diagnostics = failures.map((failure) => failure === "dns"
? "ssh: Could not resolve hostname clone-alias.example.test: Name or service not known\nfatal: Could not read from remote repository."
: failure === "unknown" ? "fatal: unexpected internal clone failure"
: "fatal: Authentication failed for 'https://clone.example.test/'" +
(failure === "mixed" ? "\nfatal: No space left on device" : ""));
await fs.mkdir(bin);
// A real child process supplies execFile's numeric exit/stderr evidence.
// Every command is intercepted, so no fixture can reach a remote Git host.
await fs.writeFile(path.join(bin, "git"), `#!${process.execPath}\n` +
`const fs = require('node:fs');\n` +
`const args = process.argv.slice(2);\n` +
`fs.appendFileSync(${JSON.stringify(callsPath)}, JSON.stringify(args) + '\\n');\n` +
`const remotes = ${JSON.stringify(remotes)};\n` +
`const diagnostics = ${JSON.stringify(diagnostics)};\n` +
`const index = remotes.indexOf(args.at(-2));\n` +
`if (!args.includes('clone') || index < 0) { process.stderr.write('Unexpected fixture Git command\\n'); process.exit(2); }\n` +
`process.stderr.write(diagnostics[index] + '\\n'); process.exitCode = 128;\n`, { mode: 0o755 });
const heartbeat = heartbeatService(db);
const previousSettings = await instanceSettingsService(db).getExperimental();
const previousPath = process.env.PATH;
const previousHome = process.env.PAPERCLIP_HOME;
try {
process.env.PATH = `${bin}${path.delimiter}${previousPath ?? ""}`;
process.env.PAPERCLIP_HOME = home;
await instanceSettingsService(db).updateExperimental({ enableIsolatedWorkspaces: true });
await db.insert(projects).values({ id: projectId, companyId, name: "Git connection fixture", status: "in_progress" });
await db.insert(projectWorkspaces).values(workspaceIds.map((id, index) => ({
id, companyId, projectId, name: `Repository ${index}`, sourceType: "git_repo",
repoUrl: remotes[index], cwd: null, isPrimary: index === 0,
})));
await db.update(issues).set({
projectId, projectWorkspaceId: workspaceIds[0], executionWorkspaceSettings: { mode: "isolated_workspace" },
}).where(eq(issues.id, issueId));
await db.update(agents).set({ adapterConfig: { workspaceStrategy: { type: "git_worktree" } } }).where(eq(agents.id, agentId));
await heartbeat.resumeQueuedRuns();
await waitForRunToSettle(heartbeat, runId, 5_000);
await heartbeat.waitForRunExecutionDrain(runId);
await waitForPendingRunFailureReports();
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
expect(run).toMatchObject({
status: "failed", errorCode: "workspace_validation_failed", processStartedAt: null,
resultJson: {
workspaceValidation: { reason: "git_worktree_base_materialization_failed" },
executionRecovery: { kind: "bootstrap", providerWorkStarted: false },
},
});
expect(run.error).toContain("checkout could not be prepared");
const validation = run.resultJson!.workspaceValidation as Record<string, unknown>;
const materializations = validation.materializationFailures as Array<Record<string, unknown>>;
expect(materializations).toHaveLength(failures.length);
expect(materializations.map((failure) => failure.connectionFailure)).toEqual(failures.map((failure) =>
failure === "auth" || failure === "dns" ? {
schemaVersion: 1, provider: "git", operation: "clone",
reason: failure === "auth" ? "authentication_failed" : "dns_failure",
} : undefined));
expect(run.resultJson!.connectionFailure).toEqual(report ? undefined : materializations[0].connectionFailure);
expect(mockAdapterExecute).not.toHaveBeenCalled();
const [issue] = await db.select().from(issues).where(eq(issues.id, issueId));
expect(issue).toMatchObject({ status: "blocked", executionRunId: null, projectId, projectWorkspaceId: workspaceIds[0] });
const [action] = await db.select().from(issueRecoveryActions).where(and(
eq(issueRecoveryActions.companyId, companyId), eq(issueRecoveryActions.sourceIssueId, issueId),
));
expect(action).toMatchObject({ status: "active", kind: "workspace_validation", ownerType: "board" });
expect(action.nextAction).toContain("workspace");
const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId));
expect(comments.some((comment) => comment.body.includes("workspace failed validation"))).toBe(true);
expect(mockCaptureRunFailure.mock.calls.filter(([event]) => event.runId === runId)).toHaveLength(report ? 1 : 0);
const calls = (await fs.readFile(callsPath, "utf8")).trim().split("\n").map((line) => JSON.parse(line) as string[]);
expect(calls).toHaveLength(failures.length);
expect(calls.every((args) => args.includes("clone"))).toBe(true);
} finally {
await heartbeat.waitForRunExecutionDrain(runId);
if (previousPath === undefined) delete process.env.PATH; else process.env.PATH = previousPath;
if (previousHome === undefined) delete process.env.PAPERCLIP_HOME; else process.env.PAPERCLIP_HOME = previousHome;
await instanceSettingsService(db).updateExperimental({ enableIsolatedWorkspaces: previousSettings.enableIsolatedWorkspaces });
await fs.rm(home, { recursive: true, force: true });
}
});
it("keeps a direct additional-repository connection failure visible without reporting it to Sentry", async () => {
const { companyId, agentId, runId, issueId } = await seedQueuedIssueRunFixture();
const home = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-additional-git-connection-"));
const cwd = path.join(home, "primary");
const bin = path.join(home, "bin");
const cloneCalls = path.join(home, "clone-calls.jsonl");
const projectId = randomUUID();
const workspaceId = randomUUID();
const repoUrl = "git@clone-alias.example.test:team/additional.git";
const previousPath = process.env.PATH;
const previousHome = process.env.PAPERCLIP_HOME;
const heartbeat = heartbeatService(db);
await fs.mkdir(cwd);
await fs.mkdir(bin);
execFileSync("git", ["init", "-b", "main"], { cwd, stdio: "ignore" });
await fs.writeFile(path.join(cwd, "README.md"), "Keep this local repository intact.\n");
execFileSync("git", ["add", "README.md"], { cwd, stdio: "ignore" });
execFileSync("git", ["-c", "user.name=Test", "-c", "user.email=test@example.test", "commit", "-m", "Fixture"], { cwd, stdio: "ignore" });
await fs.writeFile(path.join(bin, "git"), `#!${process.execPath}\n` +
`const fs = require('node:fs'); const { spawnSync } = require('node:child_process');\n` +
`const args = process.argv.slice(2);\n` +
`if (args.includes('clone')) {\n` +
` fs.appendFileSync(${JSON.stringify(cloneCalls)}, JSON.stringify(args) + '\\n');\n` +
` if (args.at(-2) !== ${JSON.stringify(repoUrl)}) process.exit(2);\n` +
` process.stderr.write('ssh: Could not resolve hostname clone-alias.example.test: Name or service not known\\nfatal: Could not read from remote repository.\\n'); process.exit(128);\n` +
`}\n` +
`if (args.some(arg => ['fetch', 'pull', 'push', 'ls-remote', 'submodule'].includes(arg))) process.exit(2);\n` +
`const result = spawnSync('git', args, { stdio: 'inherit', env: { ...process.env, PATH: ${JSON.stringify(previousPath)} } });\n` +
`process.exit(result.status ?? 2);\n`, { mode: 0o755 });
try {
process.env.PATH = `${bin}${path.delimiter}${previousPath ?? ""}`;
process.env.PAPERCLIP_HOME = home;
await db.insert(projects).values({ id: projectId, companyId, name: "Additional repository fixture", status: "in_progress" });
await db.insert(projectWorkspaces).values([
{ id: workspaceId, companyId, projectId, name: "Primary", sourceType: "local_path", cwd, isPrimary: true },
{ id: randomUUID(), companyId, projectId, name: "Additional", sourceType: "git_repo", repoUrl, cwd: null, isPrimary: false },
]);
await db.update(issues).set({ projectId, projectWorkspaceId: workspaceId,
executionWorkspaceSettings: { mode: "shared_workspace" },
}).where(eq(issues.id, issueId));
await db.update(agents).set({ adapterConfig: { cwd } }).where(eq(agents.id, agentId));
await heartbeat.resumeQueuedRuns();
await waitForRunToSettle(heartbeat, runId, 5_000);
await heartbeat.waitForRunExecutionDrain(runId);
await waitForPendingRunFailureReports();
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
expect(run).toMatchObject({ status: "failed", errorCode: "setup_failed", executionStage: "preparing", processStartedAt: null,
resultJson: {
connectionFailure: { schemaVersion: 1, provider: "git", operation: "clone", reason: "dns_failure" },
executionRecovery: { kind: "bootstrap", providerWorkStarted: false },
},
});
expect(run.resultJson?.workspaceValidation).toBeUndefined();
expect(run.error).toContain("Could not resolve hostname");
expect(mockAdapterExecute).not.toHaveBeenCalled();
const [issue] = await db.select().from(issues).where(eq(issues.id, issueId));
expect(issue).toMatchObject({ status: "blocked", executionRunId: null, projectId, projectWorkspaceId: workspaceId });
const [action] = await db.select().from(issueRecoveryActions).where(and(
eq(issueRecoveryActions.companyId, companyId), eq(issueRecoveryActions.sourceIssueId, issueId),
));
expect(action).toMatchObject({ status: "active", ownerType: "board" });
expect(mockCaptureRunFailure.mock.calls.filter(([event]) => event.runId === runId)).toHaveLength(0);
const calls = (await fs.readFile(cloneCalls, "utf8")).trim().split("\n").map((line) => JSON.parse(line) as string[]);
expect(calls).toHaveLength(1);
expect(calls[0].at(-2)).toBe(repoUrl);
expect(await fs.readFile(path.join(cwd, "README.md"), "utf8")).toBe("Keep this local repository intact.\n");
} finally {
await heartbeat.waitForRunExecutionDrain(runId);
if (previousPath === undefined) delete process.env.PATH; else process.env.PATH = previousPath;
if (previousHome === undefined) delete process.env.PAPERCLIP_HOME; else process.env.PAPERCLIP_HOME = previousHome;
await fs.rm(home, { recursive: true, force: true });
}
});
it.each(["throw", "result"])("redacts opaque environment-bound credentials from Sentry diagnostics: %s", async (mode) => {
const { companyId, agentId, runId } = await seedQueuedIssueRunFixture();
const svc = secretService(db);
@@ -7751,7 +8317,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
});
expect(next?.contextSnapshot?.wakeCommentIds).toEqual([pending!.id, go!.id]);
expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, deferred!.id)))[0]).toMatchObject({ status: "coalesced", runId: next!.id });
await vi.waitFor(async () => expect((await heartbeat.getRun(next!.id))?.status).not.toBe("running"));
await heartbeat.drainActiveRunExecutions();
expect((await heartbeat.getRun(next!.id))?.status).not.toBe("running");
});
it.each(["dedicated deferred donor", "non-coalescing recipient", "persistent agent conversation"] as const)(
@@ -8816,7 +9383,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
expect(mockTerminateLocalService).toHaveBeenCalledWith(
expect.objectContaining({ pid: 81_501, processGroupId: 81_502 }),
{ forceAfterMs: 3000 },
{ forceAfterMs: 3000, signal: "SIGINT" },
);
expect(runningProcesses.has(runId)).toBe(false);
});
@@ -16227,4 +16794,237 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
.where(eq(heartbeatRuns.id, runId));
expect(runs).toHaveLength(1);
});
describe("accepted Cursor planning boundaries", () => {
const nativeImplementation = mockExecutePaperclipNativeSession.getMockImplementation();
beforeEach(() => {
mockExecutePaperclipNativeSession.mockImplementation(async () => { throw new Error("Accepted-plan tests must never execute a provider"); });
});
afterEach(() => { mockExecutePaperclipNativeSession.mockImplementation(nativeImplementation!); });
async function seedAcceptedCursorPlanWait(semanticFinish = false, sourceSequenceOffset = 0) {
const f = await seedStrandedIssueFixture({ status: "in_progress", runStatus: "succeeded", livenessState: "advanced" });
const instance = randomUUID(), interactionId = randomUUID();
const contractInput = { db, companyId: f.companyId,
issue: { id: f.issueId, title: "Review the plan before further work", description: "Explicit completion required" }, actorId: "test" };
const { row: persistedContract, contract } = await ensureNativeCompletionContract(contractInput);
const reused = await ensureNativeCompletionContract(contractInput);
expect(reused.row.id).toBe(persistedContract.id);
expect(reused.contract).toEqual(contract);
const contractId = persistedContract.id, contractSha = persistedContract.canonicalSha256;
// Production binds policy/schema as well as the body; a body-only hash is not a valid contract receipt.
expect(contractSha).not.toBe(nativeSha256(contract));
const model = "gpt-5.6-luna[context=272k,reasoning=medium,fast=false]";
await db.update(agents).set({ adapterType: "paperclip_runner", adapterConfig: { provider: "acpx", acpxAgent: "cursor", model, acpxSessionMode: "plan", acpxPermissionMode: "approve-all" } }).where(eq(agents.id, f.agentId));
await db.update(agentWakeupRequests).set({ status: "completed" }).where(eq(agentWakeupRequests.id, f.wakeupRequestId));
await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: f.issueId, nativeSessionId: f.runId,
completionContractId: contractId, completionContractSha256: contractSha, runnerInstanceId: instance,
runnerProfileJson: { nativeExecutionInput: { binding: { companyId: f.companyId, issueId: f.issueId, runId: f.runId, agentId: f.agentId }, provider: { kind: "acpx", agent: "cursor", model, mode: "plan", permissionMode: "approve-all",
profile: paperclipRunner.resolveQualifiedAcpxProfile("cursor", model) }, session: { normalizedSessionId: f.runId },
completionContract: { id: contractId, sha256: contractSha, contract } } } }).where(eq(heartbeatRuns.id, f.runId));
const planId = `plan-${"a".repeat(64)}`;
const questionSet = { schema: "paperclip.question_set.v1", title: "Native plan", description: "Exact accepted revision", questions: [{ id: planId, prompt: "Proceed?", required: true,
answerMode: "single_select", options: [{ id: "accept", label: "Accept" }, { id: "reject", label: "Reject" }, { id: "cancel", label: "Cancel" }] }] };
const [interaction] = await db.insert(issueThreadInteractions).values({ id: interactionId, companyId: f.companyId, issueId: f.issueId, sourceRunId: f.runId,
createdByAgentId: f.agentId, kind: "ask_user_questions", status: "answered", continuationPolicy: "none",
idempotencyKey: `paperclip-runner-question:${f.runId}:request`, resolvedByUserId: "responsible-user", resolvedAt: new Date(Date.now() - 1000),
payload: { version: 1, questions: [{ id: planId, prompt: "Proceed?", selectionMode: "single", required: true, allowOther: false, options: [{ id: "accept", label: "Accept" }, { id: "reject", label: "Reject" }, { id: "cancel", label: "Cancel" }] }], runtimeRequestId: "request", questionSet: questionSet as never }, result: { version: 1, answers: [{ questionId: planId, optionIds: ["accept"] }] } }).returning();
const answer = buildQuestionResponseDeliveryEnvelope(interaction as never);
const [delivery] = await db.insert(issueQuestionResponseDeliveries).values({ companyId: f.companyId, issueId: f.issueId, interactionId,
sourceRunId: f.runId, targetRunId: f.runId, correlationId: randomUUID(), status: "delivered", deliveryMode: "steered", acknowledgedAt: new Date(), payloadSha256: nativeSha256(answer) }).returning();
const port = new PaperclipControlPlanePort(db, { companyId: f.companyId, issueId: f.issueId, runId: f.runId, agentId: f.agentId,
sessionId: f.runId, completionContractId: contractId, completionContractSha256: contractSha, sourceInstanceId: instance, controlPlaneSourceInstanceId: `control-${f.runId}` });
const events = [
{ eventType: "runtime_request.created", payload: { request: { schema: "paperclip.runtime_request.v2", requestKind: "runtime", requestId: "request", type: "input", status: "pending", turnId: "turn", itemId: "native-plan-tool", prompt: "Review plan",
origin: { adapter: "acpx-runtime-sidecar", provider: "cursor", method: "cursor/create_plan" }, input: questionSet } } },
{ eventType: "tool.execution.started", payload: { schema: "paperclip.tool.execution.v1", executionId: "native-plan-tool", transport: "builtin", operation: "execute", status: "running", readOnly: false, namespace: null, name: "Create Plan", target: null, inputUpdated: true, output: null, outputBytes: 0, outputTruncated: false, outputDigest: null, progress: null, exitCode: null, durationMs: null } },
{ eventType: "runtime_request.resolved", payload: { requestId: "request", turnId: "turn", action: "submit", response: answer.response } },
{ eventType: "tool.execution.completed", payload: { schema: "paperclip.tool.execution.v1", executionId: "native-plan-tool", transport: "builtin", operation: "execute", status: "completed", readOnly: false, namespace: null, name: "Create Plan", target: null, inputUpdated: false, output: null, outputBytes: 0, outputTruncated: false, outputDigest: null, progress: null, exitCode: null, durationMs: null } },
{ eventType: "turn.completed", payload: { status: "completed", error: null } },
];
for (const [index, event] of events.entries()) await port.appendEvent({ schema: "paperclip.prp.event.v1", sourceEventId: `${instance}:${sourceSequenceOffset + index + 1}`, sourceSeq: sourceSequenceOffset + index + 1,
sourceInstanceId: instance, sourceKind: "runner", runId: f.runId, normalizedSessionId: f.runId, turnId: "turn", schemaVersion: 1, priority: 0,
emittedAt: new Date().toISOString(), ...event } as paperclipRunner.PrpEvent);
const proof = await readNativePlanWait(db, f);
expect(proof).not.toBeNull();
const result = semanticFinish ? { ...proof!.result, reportedWorkDisposition: "done" as const, summary: "Explicit semantic finish wins", continuation: undefined,
completionClaim: { contractRevision: "1", objectiveSatisfied: true, criteria: [{ criterionId: "objective", status: "satisfied" as const, evidenceRefs: [] }], remainingWork: [] } } : proof!.result;
await port.completeRun({ result, turnId: "turn", terminal: { schema: "paperclip.prp.terminal.v1", runTerminalState: "succeeded", turnTerminalState: "completed", reportedWorkDisposition: result.reportedWorkDisposition } });
return { ...f, interactionId, deliveryId: delivery!.id, proof: proof! };
}
it("keeps an accepted Cursor plan passive across restart, future configuration changes, and paused recovery", async () => {
const f = await seedAcceptedCursorPlanWait();
await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
expect(await hasCommittedNativePlanWait(db, f)).toBe(true);
expect(await db.select().from(statusDecisions).where(eq(statusDecisions.runId, f.runId))).toEqual([expect.objectContaining({ reasonCode: "native_plan_accepted_waiting_for_continuation", toStatus: "in_progress" })]);
expect(await db.select().from(statusDecisionEffects).where(eq(statusDecisionEffects.issueId, f.issueId))).toEqual([expect.objectContaining({ effectKind: "issue_status_projection", targetType: "issue", deliveryState: "delivered" })]);
expect(await db.select().from(issueComments).where(eq(issueComments.createdByRunId, f.runId))).toEqual([expect.objectContaining({ body: expect.stringContaining("next message") })]);
const current = paperclipRunner.resolveQualifiedAcpxProfile("cursor", "gpt-5.6-luna[context=272k,reasoning=medium,fast=false]");
const resolver = vi.spyOn(paperclipRunner, "resolveQualifiedAcpxProfile").mockReturnValue({ ...current,
agentProfileVersion: current.agentProfileVersion + 1, commandDigest: `sha256:${"b".repeat(64)}` });
try {
expect(await readNativePlanWait(db, f)).toBeNull(); // Old profile cannot create a new wait.
for (const status of ["idle", "paused"] as const) {
await db.update(agents).set({ status, adapterConfig: { provider: "acpx", acpxAgent: "cursor",
model: "future-model", acpxSessionMode: "agent", acpxPermissionMode: "approve-reads" } }).where(eq(agents.id, f.agentId));
expect(await hasCommittedNativePlanWait(db, f)).toBe(true);
const recovered = await heartbeatService(db).reconcileStrandedAssignedIssues();
expect(recovered.continuationRequeued).toBe(0); expect(recovered.escalated).toBe(0);
}
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId));
const changed = structuredClone(run!.runnerProfileJson!);
(changed.nativeExecutionInput as { provider: { profile: { commandDigest: string } } }).provider.profile.commandDigest = "tampered-original-profile";
await db.update(heartbeatRuns).set({ runnerProfileJson: changed }).where(eq(heartbeatRuns.id, f.runId));
expect(await hasCommittedNativePlanWait(db, f)).toBe(false);
await db.update(heartbeatRuns).set({ runnerProfileJson: run!.runnerProfileJson }).where(eq(heartbeatRuns.id, f.runId));
expect(await hasCommittedNativePlanWait(db, f)).toBe(true);
const [tool] = await db.select().from(heartbeatRunEvents).where(and(eq(heartbeatRunEvents.runId, f.runId), eq(heartbeatRunEvents.eventType, "tool.execution.completed")));
const changedTool = structuredClone(tool!.payload!) as { prpEvent: { payload: { name: string } } };
changedTool.prpEvent.payload.name = "mutated committed tool evidence";
await db.update(heartbeatRunEvents).set({ payload: changedTool, sourcePayloadSha256: nativeSha256(changedTool.prpEvent) }).where(eq(heartbeatRunEvents.id, tool!.id));
expect(await hasCommittedNativePlanWait(db, f)).toBe(false);
await db.update(heartbeatRunEvents).set({ payload: tool!.payload, sourcePayloadSha256: tool!.sourcePayloadSha256 }).where(eq(heartbeatRunEvents.id, tool!.id));
expect(await hasCommittedNativePlanWait(db, f)).toBe(true);
} finally { resolver.mockRestore(); }
expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId))).toHaveLength(1);
expect(mockAdapterExecute).not.toHaveBeenCalled(); expect(mockExecutePaperclipNativeSession).not.toHaveBeenCalled();
});
it("retains an exact historical Cursor6 committed wait across more than 1000 ignored progress rows without allowing new unbound admission", async () => {
const progressCount = 1002;
const f = await seedAcceptedCursorPlanWait(false, progressCount);
await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId));
const profile = structuredClone(run!.runnerProfileJson!) as any;
Object.assign(profile.nativeExecutionInput.provider.profile, { agentProfileVersion: 6, commandDigest: "sha256:377dcea64a727ce799cc112458d4b40ba4bc6574cd6c6f7233b6efd5917a6c4b" });
await db.update(heartbeatRuns).set({ runnerProfileJson: profile }).where(eq(heartbeatRuns.id, f.runId));
await db.delete(heartbeatRunEvents).where(and(eq(heartbeatRunEvents.runId, f.runId), inArray(heartbeatRunEvents.eventType, ["tool.execution.started", "tool.execution.completed"])));
// Reconstruct the persisted Cursor6 receipt format using its unchanged
// source facts. This historical format had no tool lifecycle binding.
const rows = await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, f.runId));
const event = (kind: string) => (rows.find(r => r.eventType === kind)!.payload as any).prpEvent;
const [contract] = await db.select().from(completionContracts).where(eq(completionContracts.id, run!.completionContractId!));
const [interaction] = await db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.id, f.interactionId));
const [delivery] = await db.select().from(issueQuestionResponseDeliveries).where(eq(issueQuestionResponseDeliveries.id, f.deliveryId));
profile.nativeExecutionInput.provider.cursorMode = profile.nativeExecutionInput.provider.mode;
delete profile.nativeExecutionInput.provider.mode;
await db.update(heartbeatRuns).set({ runnerProfileJson: profile }).where(eq(heartbeatRuns.id, f.runId));
const legacy = { ...f.proof.source, schema: "paperclip.native_cursor_plan_wait.v1" }; delete legacy.toolExecutionId; delete legacy.toolLifecycleSha256;
legacy.authoritySha256 = nativeSha256({ admission: profile.nativeExecutionInput, contract, created: event("runtime_request.created"), resolved: event("runtime_request.resolved"), terminal: event("turn.completed"), interaction, delivery, resolvedAt: interaction!.resolvedAt!.toISOString(), acknowledgedAt: delivery!.acknowledgedAt!.toISOString() });
const [decision] = await db.select().from(statusDecisions).where(eq(statusDecisions.runId, f.runId));
const [savedResult] = await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, f.runId));
const resultJson = structuredClone(savedResult!.resultJson!) as any;
resultJson.result.continuation.idempotencyKey = resultJson.result.continuation.idempotencyKey.replace("native-plan-wait:", "cursor-plan-wait:");
const canonicalSha256 = nativeSha256(resultJson);
await db.update(nativeRunResults).set({ resultJson, canonicalSha256 }).where(eq(nativeRunResults.id, savedResult!.id));
const decisionJson = { ...decision!.decisionJson } as any;
delete decisionJson.planWait; delete decisionJson.planWaitResult;
decisionJson.cursorPlanWait = legacy;
decisionJson.cursorPlanWaitResult = { resultId: savedResult!.id, resultSha256: canonicalSha256 };
await db.update(statusDecisions).set({ decisionJson }).where(eq(statusDecisions.id, decision!.id));
expect(await readNativePlanWait(db, f)).toBeNull();
expect(await hasCommittedNativePlanWait(db, f)).toBe(true);
// Fill the reserved source prefix with genuine durable progress rows. Shift
// only DB ordering keys; the original PRP facts and committed receipt stay
// byte-identical. Cursor6 never queried these rows, even above its budget.
await db.update(heartbeatRunEvents).set({ seq: sql`${heartbeatRunEvents.seq} + ${progressCount}` }).where(eq(heartbeatRunEvents.runId, f.runId));
const progress = Array.from({ length: progressCount }, (_, index) => {
const seq = index + 1;
const prpEvent = { ...event("runtime_request.created"), sourceEventId: `${run!.runnerInstanceId}:${seq}`, sourceSeq: seq, eventType: "tool.execution.progressed",
payload: { schema: "paperclip.tool.execution.v1", executionId: "earlier-tool", transport: "builtin", operation: "read", status: "running" } };
return { companyId: f.companyId, runId: f.runId, agentId: f.agentId, seq, eventType: prpEvent.eventType, payload: { prpEvent }, sourceInstanceId: run!.runnerInstanceId,
sourceEventId: prpEvent.sourceEventId, sourceSeq: seq, sourcePayloadSha256: nativeSha256(prpEvent), protocolSchemaVersion: 1 };
});
await db.insert(heartbeatRunEvents).values(progress);
expect(await hasCommittedNativePlanWait(db, f)).toBe(true);
const [unchangedDecision] = await db.select().from(statusDecisions).where(eq(statusDecisions.id, decision!.id));
expect(unchangedDecision!.decisionJson!.cursorPlanWait).toEqual(legacy);
// Completed rows belonged to the old query. A later completion must still
// invalidate the normal-terminal boundary rather than being filtered away.
const completion = { ...event("turn.completed"), sourceEventId: `${run!.runnerInstanceId}:99999`, sourceSeq: 99999, eventType: "tool.execution.completed", payload: { ...progress[0]!.payload.prpEvent.payload, status: "completed" } };
const [extra] = await db.insert(heartbeatRunEvents).values({ ...progress[0]!, seq: 99999, eventType: completion.eventType, payload: { prpEvent: completion }, sourceEventId: completion.sourceEventId, sourceSeq: 99999, sourcePayloadSha256: nativeSha256(completion) }).returning();
expect(await hasCommittedNativePlanWait(db, f)).toBe(false);
await db.delete(heartbeatRunEvents).where(eq(heartbeatRunEvents.id, extra!.id));
expect(await hasCommittedNativePlanWait(db, f)).toBe(true);
profile.nativeExecutionInput.provider.profile.commandDigest = "tampered-history";
await db.update(heartbeatRuns).set({ runnerProfileJson: profile }).where(eq(heartbeatRuns.id, f.runId));
expect(await hasCommittedNativePlanWait(db, f)).toBe(false);
});
it("admits a later ordinary user message after an accepted Cursor plan without changing Plan mode or replaying its run", async () => {
const f = await seedAcceptedCursorPlanWait();
await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
// Occupy the single dispatch slot: this checks actual user-wake admission
// without executing any provider or fabricating a second semantic result.
const occupied = randomUUID();
await db.update(agents).set({ runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 } } }).where(eq(agents.id, f.agentId));
await db.insert(heartbeatRuns).values({ id: occupied, companyId: f.companyId, agentId: f.agentId, status: "running", startedAt: new Date(), contextSnapshot: {} });
const [comment] = await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId, authorType: "user", authorUserId: "responsible-user", body: "Continue reviewing the accepted plan in Plan mode." }).returning();
try {
const heartbeat = heartbeatService(db);
const next = await heartbeat.wakeup(f.agentId, { source: "automation", triggerDetail: "system", reason: "issue_commented", requestedByActorType: "user", requestedByActorId: "responsible-user",
payload: { issueId: f.issueId, commentId: comment!.id }, contextSnapshot: { issueId: f.issueId, taskId: f.issueId, wakeCommentId: comment!.id, wakeCommentIds: [comment!.id] } });
expect(next).not.toBeNull(); expect(next!.id).not.toBe(f.runId);
const [admitted] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, next!.id));
expect(admitted).toMatchObject({ status: "queued", retryOfRunId: null });
expect(admitted!.contextSnapshot?.wakeCommentIds).toContain(comment!.id);
const [agent] = await db.select().from(agents).where(eq(agents.id, f.agentId));
expect(agent!.adapterConfig.acpxSessionMode).toBe("plan");
expect(await hasCommittedNativePlanWait(db, f)).toBe(false);
expect(mockExecutePaperclipNativeSession).not.toHaveBeenCalled();
} finally {
await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date() }).where(and(eq(heartbeatRuns.companyId, f.companyId), inArray(heartbeatRuns.status, ["running", "queued"])));
await db.update(agentWakeupRequests).set({ status: "cancelled", finishedAt: new Date() }).where(and(eq(agentWakeupRequests.companyId, f.companyId), inArray(agentWakeupRequests.status, ["queued", "claimed"])));
}
});
it.each(["delivery", "assignment", "admission", "user_request", "result_or_decision"] as const)("revokes an accepted Cursor plan passive wait after %s changes", async change => {
const f = await seedAcceptedCursorPlanWait();
await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
if (change === "delivery") await db.update(issueQuestionResponseDeliveries).set({ acknowledgedAt: null }).where(eq(issueQuestionResponseDeliveries.id, f.deliveryId));
if (change === "assignment") await db.update(issues).set({ assigneeAgentId: null }).where(eq(issues.id, f.issueId));
if (change === "admission") {
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId));
const profile = structuredClone(run!.runnerProfileJson!);
(profile.nativeExecutionInput as { provider: { cursorMode: string } }).provider.cursorMode = "agent";
await db.update(heartbeatRuns).set({ runnerProfileJson: profile }).where(eq(heartbeatRuns.id, f.runId));
}
if (change === "user_request") await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId, authorType: "user", authorUserId: "responsible-user", body: "Continue reviewing this plan in Plan mode." });
if (change === "result_or_decision") {
const [accepted] = await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, f.runId));
await db.update(nativeRunResults).set({ canonicalSha256: "changed" }).where(eq(nativeRunResults.id, accepted!.id));
expect(await hasCommittedNativePlanWait(db, f)).toBe(false);
await db.update(nativeRunResults).set({ canonicalSha256: accepted!.canonicalSha256 }).where(eq(nativeRunResults.id, accepted!.id));
expect(await hasCommittedNativePlanWait(db, f)).toBe(true);
await db.update(issues).set({ lastStatusDecisionId: null }).where(eq(issues.id, f.issueId));
}
expect(await hasCommittedNativePlanWait(db, f)).toBe(false);
});
it("rechecks accepted Cursor plan delivery under the status transaction before presentation or effects", async () => {
const f = await seedAcceptedCursorPlanWait();
await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", failpoint: "status_projection" });
const [coordinator] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId));
const [issue] = await db.select().from(issues).where(eq(issues.id, f.issueId));
await db.update(issueQuestionResponseDeliveries).set({ payloadSha256: "changed" }).where(eq(issueQuestionResponseDeliveries.id, f.deliveryId));
await expect(commitNativeStatusDecision({ db, companyId: f.companyId, issueId: f.issueId, runId: f.runId, assessmentId: coordinator!.assessmentId!,
priorStatus: issue!.status, priorStatusVersion: issue!.statusVersion, priorDecisionId: issue!.lastStatusDecisionId,
decision: { policyVersion: "phase6-v7", statusAction: "in_progress", toStatus: "in_progress", reasonCode: "native_plan_accepted_waiting_for_continuation", unblockDescriptor: null, effects: [] },
requirePlanWaitSource: f.proof.source })).rejects.toBeInstanceOf(NativeStatusRaceError);
expect(await db.select().from(statusDecisions).where(eq(statusDecisions.runId, f.runId))).toHaveLength(0);
expect(await db.select().from(issueComments).where(eq(issueComments.createdByRunId, f.runId))).toHaveLength(0);
});
it("preserves explicit semantic finish priority over accepted Cursor plan evidence", async () => {
const f = await seedAcceptedCursorPlanWait(true);
await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
expect(await hasCommittedNativePlanWait(db, f)).toBe(false);
const decisions = await db.select().from(statusDecisions).where(eq(statusDecisions.runId, f.runId));
expect(decisions).toHaveLength(1); expect(decisions[0]!.reasonCode).not.toBe("native_plan_accepted_waiting_for_continuation");
});
});
});
@@ -10,11 +10,14 @@ import {
authUsers,
companies,
companyMemberships,
connectionGrants,
createDb,
heartbeatRunEvents,
heartbeatRuns,
issueAccessGrants,
issues,
toolApplications,
toolConnections,
workspaceOperations,
} from "@paperclipai/db";
import {
@@ -316,6 +319,35 @@ describeEmbeddedPostgres("heartbeat run privacy routes", { concurrent: false },
}
});
it("filters credential diagnostics in full-run and history responses by viewer", async () => {
const fixture = await seedFixture();
const readerId = `reader-${randomUUID()}`;
await db.insert(authUsers).values({ id: readerId, name: "Task reader", email: `${readerId}@example.test`, createdAt: new Date(), updatedAt: new Date() });
await db.insert(companyMemberships).values({ companyId: fixture.companyId, principalType: "user", principalId: readerId, status: "active", membershipRole: "operator" });
await db.update(issues).set({ visibility: "open", privacyRootIssueId: null }).where(eq(issues.id, fixture.issueId));
const applicationId = randomUUID(), connectionId = randomUUID(), grantId = randomUUID();
await db.insert(toolApplications).values({ id: applicationId, companyId: fixture.companyId, name: "AI fixture", type: "rest_api" });
await db.insert(toolConnections).values({ id: connectionId, companyId: fixture.companyId, applicationId, name: "Owner's API key", uid: "credential-fixture", connectionPurpose: "ai", transport: "runtime_auth" });
await db.insert(connectionGrants).values({ id: grantId, companyId: fixture.companyId, connectionId, kind: "user", subjectUserId: fixture.ownerUserId });
await db.update(heartbeatRuns).set({ status: "failed", responsibleUserId: fixture.otherUserId,
errorCode: "configuration_incomplete", error: "This credential is not shared with the responsible user",
resultJson: { configurationIncomplete: { selectionFailure: "ai_connection_credential_not_shared",
credentialAccess: { connectionName: "Owner's API key", grantId } } },
}).where(eq(heartbeatRuns.id, fixture.privateRunId));
for (const userId of [fixture.ownerUserId, fixture.otherUserId, readerId]) {
const app = createBoardApp(fixture.companyId, userId);
const detail = await request(app).get(`/api/heartbeat-runs/${fixture.privateRunId}`);
const history = await request(app).get(`/api/companies/${fixture.companyId}/heartbeat-runs`);
expect(detail.status).toBe(200);
expect(history.status).toBe(200);
const expected = userId === readerId ? {} : { connectionName: "Owner's API key" };
expect(detail.body.resultJson.configurationIncomplete.credentialAccess).toEqual(expected);
expect(JSON.stringify(detail.body)).not.toContain(grantId);
expect(JSON.stringify(history.body)).not.toContain(grantId);
if (userId === readerId) expect(JSON.stringify(history.body)).not.toContain("Owner's API key");
}
});
it("returns no linked issue metadata to a non-member", async () => {
const fixture = await seedFixture();
const response = await request(createApp(fixture.companyId, fixture.otherAgentId))
@@ -216,6 +216,26 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
return { companyId, agentId };
}
it.each(["preparing", "verifying", "resuming"] as const)("keeps assigned work queued through %s, including recovery with timers disabled", async lifecycleState => {
const { companyId, agentId } = await seedCompanyAndAgent({ heartbeatConfig: { enabled: false } });
await db.update(agents).set({ status: "paused", lifecycleState }).where(eq(agents.id, agentId));
const [issue] = await db.insert(issues).values({ companyId, title: "Work accepted during setup", status: "todo", assigneeAgentId: agentId }).returning();
const run = await heartbeat.wakeup(agentId, { source: "assignment", triggerDetail: "system", reason: "issue_assigned",
requestedByActorType: "system", payload: { issueId: issue.id } });
expect(run?.status).toBe("queued");
await heartbeat.resumeQueuedRuns();
expect((await heartbeat.getRun(run!.id))?.status).toBe("queued");
expect(countExecuteCallsForRun(run!.id)).toBe(0);
expect((await heartbeat.reconcileStrandedAssignedIssues()).escalated).toBe(0);
expect((await db.select().from(issues).where(eq(issues.id, issue.id)))[0]).toMatchObject({ status: "todo", assigneeAgentId: agentId });
await db.update(agents).set({ status: "idle", lifecycleState: "ready" }).where(eq(agents.id, agentId));
const recovered = heartbeatService(db, { runtimeEnv: { ...process.env, PAPERCLIP_IN_WORKTREE: "false" } });
await recovered.resumeQueuedRuns();
expect(await waitForCondition(async () => countExecuteCallsForRun(run!.id) === 1)).toBe(true);
await recovered.waitForRunExecutionDrain(run!.id);
expect(countExecuteCallsForRun(run!.id)).toBe(1);
});
async function seedQueuedRun(input: {
companyId: string;
agentId: string;
@@ -14,6 +14,7 @@ import {
companies,
companySkills,
createDb,
closeRegisteredClients,
environments,
environmentLeases,
executionWorkspaces,
@@ -46,6 +47,13 @@ import {
heartbeatService,
} from "../services/heartbeat.ts";
import { instanceSettingsService } from "../services/instance-settings.ts";
import { reportRunFailure, waitForPendingRunFailureReports } from "../services/run-failure-report.js";
const captureRunFailure = vi.hoisted(() => vi.fn());
vi.mock("../sentry.js", async (importOriginal) => ({
...await importOriginal<typeof import("../sentry.js")>(),
captureRunFailure,
}));
// Exercise the real SSH lease and heartbeat paths without connecting to a host.
vi.mock("@paperclipai/adapter-utils/ssh", async (importOriginal) => ({
@@ -55,7 +63,8 @@ vi.mock("@paperclipai/adapter-utils/ssh", async (importOriginal) => ({
}),
}));
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
const externalTestDatabaseUrl = process.env.PAPERCLIP_TEST_DATABASE_URL?.trim();
const embeddedPostgresSupport = externalTestDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
const WORKSPACE_BUSY_TEST_ADAPTER = "workspace_busy_test";
@@ -91,10 +100,14 @@ describeEmbeddedPostgres("shared-workspace run serialization", () => {
let workspaceCwd!: string;
const executedRunIds: string[] = [];
const executedInputs = new Map<string, AdapterExecutionContext>();
const failExecutionRunIds = new Set<string>();
beforeAll(async () => {
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-heartbeat-workspace-busy-");
db = createDb(tempDb.connectionString);
if (externalTestDatabaseUrl) db = createDb(externalTestDatabaseUrl);
else {
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-heartbeat-workspace-busy-");
db = createDb(tempDb.connectionString);
}
heartbeat = heartbeatService(db);
workspaceCwd = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-workspace-busy-"));
registerServerAdapter({
@@ -102,6 +115,15 @@ describeEmbeddedPostgres("shared-workspace run serialization", () => {
execute: async (input) => {
executedRunIds.push(input.runId);
executedInputs.set(input.runId, input);
if (failExecutionRunIds.has(input.runId)) {
return {
exitCode: 1,
signal: null,
timedOut: false,
errorMessage: "Synthetic unexpected adapter failure after workspace wait",
resultJson: {},
};
}
return {
exitCode: 0,
signal: null,
@@ -127,15 +149,20 @@ describeEmbeddedPostgres("shared-workspace run serialization", () => {
.set({ status: "cancelled", finishedAt: new Date() })
.where(eq(heartbeatRuns.status, "running"));
await drainHeartbeatRunsToQuiescence(db, heartbeat);
await waitForPendingRunFailureReports();
await cleanupFixture();
executedRunIds.length = 0;
executedInputs.clear();
failExecutionRunIds.clear();
captureRunFailure.mockClear();
await instanceSettingsService(db).updateGeneral({ executionMode: "any" });
});
afterAll(async () => {
unregisterServerAdapter(WORKSPACE_BUSY_TEST_ADAPTER);
if (workspaceCwd) await fs.rm(workspaceCwd, { recursive: true, force: true });
if (externalTestDatabaseUrl) await closeRegisteredClients(externalTestDatabaseUrl);
await db.$client.end();
await tempDb?.cleanup();
});
@@ -547,6 +574,16 @@ describeEmbeddedPostgres("shared-workspace run serialization", () => {
holderIssueId: fixture.holderIssueId,
deferralAttempt: 0,
});
expect(finishedRun?.resultJson).toMatchObject({
executionRecovery: { kind: "workspace_wait", providerWorkStarted: false },
cancellation: {
source: "control_plane",
expected: true,
initiator: { type: "system" },
reason: "Waiting for the shared project workspace",
recordedAt: finishedRun!.finishedAt!.toISOString(),
},
});
// The deferred run's adapter never executed — the whole point of the gate.
expect(executedRunIds).not.toContain(run!.id);
@@ -557,6 +594,8 @@ describeEmbeddedPostgres("shared-workspace run serialization", () => {
scheduledRetryAttempt: 1,
scheduledRetryReason: WORKSPACE_BUSY_RETRY_REASON,
});
await waitForPendingRunFailureReports();
expect(captureRunFailure.mock.calls.filter(([report]) => report.runId === run!.id)).toHaveLength(0);
expect((retryRun?.contextSnapshot as Record<string, unknown> | null)?.wakeReason).toBe(
WORKSPACE_BUSY_RETRY_WAKE_REASON,
);
@@ -603,7 +642,7 @@ describeEmbeddedPostgres("shared-workspace run serialization", () => {
}
});
it("executes the scheduled retry once the holder has finished", async () => {
it.each([false, true])("executes the scheduled retry without inheriting expected cancellation (adapter fails: %s)", async (adapterFails) => {
const fixture = await seedWorkspaceFixture();
const run = await heartbeat.invoke(
@@ -618,6 +657,8 @@ describeEmbeddedPostgres("shared-workspace run serialization", () => {
const retryRun = await waitForRetryRun(run!.id);
expect(retryRun?.status).toBe("scheduled_retry");
expect(retryRun?.resultJson?.cancellation).toBeUndefined();
if (adapterFails) failExecutionRunIds.add(retryRun!.id);
// Holder finishes; the due retry promotes, queues, and executes.
await db
@@ -631,8 +672,50 @@ describeEmbeddedPostgres("shared-workspace run serialization", () => {
await heartbeat.resumeQueuedRuns();
const finishedRetry = await waitForRunToLeaveActiveStates(retryRun!.id);
expect(finishedRetry?.status).toBe("succeeded");
expect(finishedRetry?.status).toBe(adapterFails ? "failed" : "succeeded");
expect(executedRunIds).toContain(retryRun!.id);
expect(finishedRetry?.resultJson?.cancellation).toBeUndefined();
if (adapterFails) {
await expect.poll(() => captureRunFailure.mock.calls.filter(([report]) => report.runId === retryRun!.id)).toHaveLength(1);
expect(captureRunFailure).toHaveBeenCalledWith(expect.objectContaining({
runId: retryRun!.id,
runStatus: "failed",
exitCode: 1,
}));
}
await waitForPendingRunFailureReports();
expect(captureRunFailure.mock.calls.filter(([report]) => report.runId === run!.id)).toHaveLength(0);
});
it("does not hide ambiguous, malformed, provider, or transport cancellations behind workspace_busy", async () => {
const fixture = await seedWorkspaceFixture();
const run = await heartbeat.invoke(fixture.agentId, "assignment", {
issueId: fixture.issueId, wakeReason: "issue_assigned",
}, "system");
expect(run).not.toBeNull();
await waitForRunToLeaveActiveStates(run!.id);
await waitForRetryRun(run!.id);
const [deferred] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run!.id));
expect(deferred?.status).toBe("cancelled");
const receipt = deferred!.resultJson!.cancellation as Record<string, unknown>;
for (const cancellation of [
undefined,
{ ...receipt, expected: "true" },
{ ...receipt, expected: false, source: "provider", initiator: { type: "provider" } },
{ ...receipt, expected: false, source: "transport" },
]) {
captureRunFailure.mockClear();
const unexpected = {
...deferred!, id: randomUUID(),
resultJson: { ...deferred!.resultJson, cancellation },
};
await reportRunFailure(db, unexpected);
expect(captureRunFailure).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({
runId: unexpected.id,
runStatus: "cancelled",
errorCode: WORKSPACE_BUSY_ERROR_CODE,
}));
}
});
it("preserves a legacy accepted mention run through workspace retry", async () => {
@@ -462,6 +462,29 @@ describe("assertGitSensitiveAdapterWorkspaceValid", () => {
});
describe("assertGitWorktreeBaseWorkspaceReady", () => {
it.each(["all_external", "mixed", "malformed"])("keeps materialization aggregation fail-closed: %s", async (kind) => {
const first = { schemaVersion: 1, provider: "git", operation: "clone", reason: "authentication_failed" } as const;
const second = kind === "all_external" ? { ...first, reason: "dns_failure" as const } :
kind === "malformed" ? { ...first, reason: "unknown" } : undefined;
const failures = [first, second].map((connectionFailure, index) => ({
projectWorkspaceId: `workspace-${index}`, repoUrl: "https://example.test/team/repo.git",
error: "Managed checkout failed", ...(connectionFailure ? { connectionFailure } : {}),
}));
const error = await assertGitWorktreeBaseWorkspaceReady({
requestedExecutionWorkspaceMode: "isolated_workspace",
config: { workspaceStrategy: { type: "git_worktree" } },
issue: { id: "issue-1", identifier: "TEST-1", projectId: "project-1", projectWorkspaceId: "workspace-0" },
base: { baseCwd: "/tmp/unused-fallback", source: "project_primary", projectId: "project-1", workspaceId: "workspace-0", repoUrl: "https://example.test/team/repo.git", repoRef: null },
anchor: { baseCwdFallback: true, materializationFailures: failures as never },
}).catch((error: unknown) => error);
expect(error).toMatchObject({
code: "workspace_validation_failed",
resultJson: { workspaceValidation: { reason: "git_worktree_base_materialization_failed", materializationFailures: failures } },
});
expect((error as { resultJson: Record<string, unknown> }).resultJson.connectionFailure)
.toEqual(kind === "all_external" ? first : undefined);
});
it("rejects projectless isolated git worktrees that resolved to agent_home", async () => {
const fallbackCwd = resolveDefaultAgentWorkspaceDir("agent-1");
@@ -0,0 +1,97 @@
import { createRequire } from "node:module";
import { createHash, randomUUID } from "node:crypto";
import { readFile, writeFile } from "node:fs/promises";
import { dirname, join } from "node:path";
import type { AdapterSandboxExecutionTarget } from "@paperclipai/adapter-utils";
import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime";
import { shellQuote } from "@paperclipai/adapter-utils/ssh";
/** Opt-in, bounded live fixture. Every invocation owns and deletes its sandbox. */
export async function startDotDaytona(localRoot: string) {
const image = process.env.PAPERCLIP_DOT_DAYTONA_IMAGE;
if (!image || !/@sha256:[a-f0-9]{64}$/.test(image)) throw new Error("Dot live tests require an immutable Daytona image digest");
const require = createRequire(new URL("../../../../packages/plugins/sandbox-providers/daytona/package.json", import.meta.url));
const { Daytona } = require("@daytonaio/sdk");
const sandbox = await new Daytona({ apiKey: process.env.DAYTONA_API_KEY }).create({
image, public: false, autoStopInterval: 10, ttlMinutes: 20,
labels: { "paperclip-test": "dot-managed-runner" },
}, { timeout: 180 });
const workspace = "/home/daytona/dot-managed-test";
const executeSource = `
const {spawn} = require('node:child_process');
const input = JSON.parse(process.argv[1]);
const child = spawn(input.command, input.args || [], {
cwd: input.cwd, env: {...process.env, ...input.env}, timeout: input.timeoutMs || 30000
});
const stdout = [], stderr = [];
child.stdout.on('data', chunk => stdout.push(chunk));
child.stderr.on('data', chunk => stderr.push(chunk));
child.on('error', error => stderr.push(Buffer.from(error.message)));
child.on('close', (exitCode, signal) => process.stdout.write(JSON.stringify({
exitCode, signal, timedOut: signal === 'SIGTERM', pid: null,
startedAt: new Date().toISOString(),
stdout: Buffer.concat(stdout).toString(), stderr: Buffer.concat(stderr).toString()
})));
child.stdin.on('error', error => { if (error.code !== 'EPIPE') stderr.push(Buffer.from(error.message)); });
child.stdin.end(input.stdin || '');
`;
const runner: CommandManagedRuntimeRunner = {
execute: async input => {
const command = `node --input-type=commonjs -e ${shellQuote(executeSource)} ${shellQuote(JSON.stringify(input))}`;
const result = await sandbox.process.executeCommand(command, undefined, undefined, Math.ceil((input.timeoutMs ?? 30000) / 1000) + 10);
if (result.exitCode !== 0) throw new Error(`Daytona command transport failed (exit ${result.exitCode})`);
return JSON.parse(result.result);
},
syncIn: async operations => {
const results = [];
for (const operation of operations) {
let bytesTransferred = 0;
for (const file of operation.files) {
if (file.kind !== "file") throw new Error("Dot test stages only the verified runner artifact");
await sandbox.fs.createFolder(dirname(file.targetPath), "700");
const bytes = await readFile(file.sourcePath);
await sandbox.fs.uploadFile(bytes, file.targetPath);
await sandbox.fs.setFilePermissions(file.targetPath, { mode: (file.mode ?? 0o600).toString(8) });
bytesTransferred += bytes.length;
}
results.push({ operationId: operation.operationId, filesTransferred: operation.files.length, bytesTransferred });
}
return { operations: results };
},
};
try {
const setup = await runner.execute({ command: "mkdir", args: ["-p", workspace], timeoutMs: 10000 });
if (setup.exitCode !== 0) throw new Error("Daytona test workspace creation failed");
const binary = process.env.PAPERCLIP_DOT_DAYTONA_RUNNER_BINARY
? await readFile(process.env.PAPERCLIP_DOT_DAYTONA_RUNNER_BINARY)
: await sandbox.fs.downloadFile("/usr/local/bin/paperclip-runnerd");
const runnerBinary = join(localRoot, "daytona-paperclip-runnerd");
await writeFile(runnerBinary, binary, { mode: 0o700 });
const qualificationBinary = join(workspace, "qualification-runnerd");
await sandbox.fs.uploadFile(binary, qualificationBinary);
await sandbox.fs.setFilePermissions(qualificationBinary, { mode: "700" });
const metadata = await runner.execute({ command: qualificationBinary, args: ["--build-metadata"], timeoutMs: 10000 });
if (metadata.exitCode !== 0) throw new Error("Daytona qualification runner build metadata missing");
const evidence = { image, sandboxId: sandbox.id, binarySha256: createHash("sha256").update(binary).digest("hex"), metadata: JSON.parse(metadata.stdout) };
const evidencePath = process.env.PAPERCLIP_DOT_DAYTONA_EVIDENCE;
if (evidencePath) await writeFile(evidencePath, JSON.stringify(evidence, null, 2));
const target: AdapterSandboxExecutionTarget = {
kind: "remote", transport: "sandbox", providerKey: "daytona", environmentId: randomUUID(),
leaseId: randomUUID(), remoteCwd: workspace, runner,
effectiveCapabilities: { runnerWebSocketIngress: true } as AdapterSandboxExecutionTarget["effectiveCapabilities"],
getRunnerIngressEndpoint: async ({ port, path }) => {
const preview = await sandbox.getPreviewLink(port);
const url = new URL(preview.url);
url.protocol = "wss:";
url.pathname = `${url.pathname.replace(/\/$/, "")}${path}`;
return { kind: "authenticated_websocket", websocketUrl: url.toString(),
secretHeaders: [{ name: "X-Daytona-Preview-Token", value: preview.token }],
generation: "dot-test", refresh: () => target.getRunnerIngressEndpoint!({ leaseId: target.leaseId!, port, path }), close: async () => {} };
},
};
return { target, runnerBinary, evidence, close: async () => {
await sandbox.delete();
if (evidencePath) await writeFile(evidencePath, JSON.stringify({ ...evidence, cleanup: "deleted" }, null, 2));
} };
} catch (error) { await sandbox.delete(); throw error; }
}
@@ -49,6 +49,36 @@ describe("idle admission", () => {
expect(idleWorkSnapshot().active).toBe(0);
});
it("tracks real Vite middleware without confusing Connect mount paths with Express routes", async () => {
const { createServer } = await import("vite");
const vite = await createServer({
configFile: false,
appType: "custom",
server: { middlewareMode: true, hmr: false, watch: null },
optimizeDeps: { noDiscovery: true, include: [] },
});
const server = app(), done = deferred();
vite.middlewares.use("/vite-work", async (_req, res) => {
res.statusCode = 202;
res.end();
await done.promise;
});
server.use(vite.middlewares);
try {
expect(() => trackIdleRequestHandlers(server)).not.toThrow();
await request(server).get("/vite-work").expect(202);
expect(idleWorkSnapshot().active).toBe(1);
startTaskDrain({ purpose: "idle", ttlMs: 60_000 });
await request(server).get("/vite-work").expect(503);
done.resolve();
await new Promise(resolve => setImmediate(resolve));
expect(idleWorkSnapshot().active).toBe(0);
} finally {
done.resolve();
await vite.close();
}
});
it("does not mistake a client disconnect for completed accepted work", async () => {
const server = app();
const entered = deferred(), done = deferred(), closed = deferred();
@@ -0,0 +1,92 @@
import { mkdtemp, readFile, rm, symlink, utimes, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { gzipSync } from "node:zlib";
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { sql } from "drizzle-orm";
import { createDb, ensurePostgresDatabase, getEmbeddedPostgresTestSupport, runDatabaseBackup, runDatabaseRestore, startEmbeddedPostgresTestDatabase } from "@paperclipai/db";
import { clearIdleBackupWakeRequired, idleBackupWakeMarker, markIdleBackupWakeRequired, verifyIdleDatabaseBackup } from "../services/idle-database-backup.js";
let directory: string;
beforeEach(async () => { directory = await mkdtemp(path.join(os.tmpdir(), "idle-backup-")); });
afterEach(async () => { await rm(directory, { recursive: true, force: true }); });
describe("idle database backup persistence", () => {
it("keeps a restart marker until a fresh valid backup is complete", async () => {
const archive = path.join(directory, "backup.sql.gz");
await markIdleBackupWakeRequired(directory);
await writeFile(archive, gzipSync("CREATE TABLE fixture (id integer);\n"));
await verifyIdleDatabaseBackup(archive);
// Verification on the sleep path must not remove the wake obligation.
expect(await readFile(idleBackupWakeMarker(directory), "utf8")).toContain("restarts");
await clearIdleBackupWakeRequired(directory);
await expect(readFile(idleBackupWakeMarker(directory))).rejects.toMatchObject({ code: "ENOENT" });
expect(await readFile(archive)).toEqual(gzipSync("CREATE TABLE fixture (id integer);\n"));
});
it.each([
["empty file", Buffer.alloc(0)], ["empty SQL", gzipSync("")],
["truncated gzip", gzipSync("SELECT 1;").subarray(0, -4)],
["plain text", Buffer.from("SELECT 1;")],
])("rejects %s and retains the restart obligation", async (_label, contents) => {
await markIdleBackupWakeRequired(directory);
const archive = path.join(directory, "invalid.sql.gz");
await writeFile(archive, contents);
await expect(verifyIdleDatabaseBackup(archive)).rejects.toThrow();
expect(await readFile(idleBackupWakeMarker(directory), "utf8")).toContain("restarts");
});
it("rejects a missing archive and symlinked archive or marker", async () => {
const archive = path.join(directory, "backup.sql.gz");
await expect(verifyIdleDatabaseBackup(archive)).rejects.toThrow();
const target = path.join(directory, "target");
await writeFile(target, gzipSync("SELECT 1;"));
await symlink(target, archive);
await expect(verifyIdleDatabaseBackup(archive)).rejects.toThrow();
await symlink(target, idleBackupWakeMarker(directory));
await expect(markIdleBackupWakeRequired(directory)).rejects.toThrow();
expect(await readFile(target)).toEqual(gzipSync("SELECT 1;"));
});
});
const support = await getEmbeddedPostgresTestSupport();
describe.skipIf(!support.supported)("idle backup restore", () => {
it("restores the last pre-sleep state into a separate database", async () => {
const source = await startEmbeddedPostgresTestDatabase("idle-backup-restore-");
const sourceDb = createDb(source.connectionString);
const targetUrl = new URL(source.connectionString);
targetUrl.pathname = "/idle_restore";
try {
await sourceDb.execute(sql`CREATE TABLE backup_fixture (id integer PRIMARY KEY, value text NOT NULL)`);
await sourceDb.execute(sql`INSERT INTO backup_fixture VALUES (1, 'last committed state')`);
await markIdleBackupWakeRequired(directory);
const result = await runDatabaseBackup({
connectionString: source.connectionString, backupDir: directory,
retention: { dailyDays: 14, weeklyWeeks: 4, monthlyMonths: 6 },
verifyBeforePrune: verifyIdleDatabaseBackup,
});
await verifyIdleDatabaseBackup(result.backupFile);
await ensurePostgresDatabase(source.connectionString, "idle_restore");
await runDatabaseRestore({ connectionString: targetUrl.toString(), backupFile: result.backupFile });
const target = createDb(targetUrl.toString());
expect(await target.execute(sql`SELECT id, value FROM backup_fixture`)).toEqual([{ id: 1, value: "last committed state" }]);
const checkpointBytes = await readFile(result.backupFile);
expect(await readFile(idleBackupWakeMarker(directory), "utf8")).toContain("restarts");
// A failed verification must not delete existing recovery points, even
// when their age would make them eligible for the configured pruning.
const history = path.join(directory, "paperclip-history.sql.gz");
const historyBytes = gzipSync("SELECT 'historical recovery point';");
await writeFile(history, historyBytes);
await utimes(history, new Date(0), new Date(0));
await expect(runDatabaseBackup({
connectionString: source.connectionString, backupDir: directory,
backupEngine: "javascript", retention: { dailyDays: 7, weeklyWeeks: 4, monthlyMonths: 1 },
verifyBeforePrune: async () => { throw new Error("verification failed"); },
})).rejects.toThrow("verification failed");
expect(await readFile(history)).toEqual(historyBytes);
expect(await readFile(result.backupFile)).toEqual(checkpointBytes);
} finally {
await source.cleanup();
}
}, 60_000);
});
@@ -40,10 +40,12 @@ describe("idle cleanup inspection", () => {
expect(await readIdleLocalWork()).toBe("none");
markIdleStartupComplete({ scheduledBackups: true });
expect(await readIdleLocalWork()).toBe("present");
expect(await readIdleLocalWork({ backupCheckpoint: true })).toBe("none");
markIdleStartupComplete({ scheduledBackups: false });
await fs.mkdir(custom);
await fs.writeFile(path.join(custom, "receipt.tmp"), "incomplete");
expect(await readIdleLocalWork()).toBe("present");
expect(await readIdleLocalWork({ backupCheckpoint: true })).toBe("present");
await fs.rm(custom, { recursive: true });
expect(await readIdleLocalWork()).toBe("none");
});
+215 -5
View File
@@ -1,4 +1,4 @@
import { beginIdleTrackedWork } from "../services/task-admission.js";
import { beginIdleTrackedWork, idleWorkSnapshot } from "../services/task-admission.js";
import { randomUUID } from "node:crypto";
import { eq, sql } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
@@ -22,6 +22,73 @@ const ownerId = "d0b833f4-4098-42de-8420-1907f3aa4895";
const owned = () => ({ ...held(), ownerId });
const emptyLocal = async () => "none" as const;
describe("idle backup checkpoint", () => {
const database = (blocked = false) => {
const transaction = vi.fn(async (run: (tx: unknown) => Promise<unknown>) => run({ execute: async () => [{ blocked }] }));
return { db: { transaction } as unknown as Db, transaction };
};
const backup = (work: () => Promise<boolean> = async () => true) => vi.fn(async () => {
const finish = beginIdleTrackedWork();
try { return await work(); } finally { finish(); }
});
it("backs up after durable inspection and rechecks local work before authorizing sleep", async () => {
const { db, transaction } = database();
const local = vi.fn(emptyLocal);
const checkpoint = backup(async () => {
expect(transaction).toHaveBeenCalledOnce();
expect(idleWorkSnapshot().active).toBe(1);
return true;
});
expect(await readIdleSleepSafety(db, owned, () => now, ownerId, local, undefined, checkpoint)).toEqual(none);
expect(checkpoint).toHaveBeenCalledOnce();
expect(local).toHaveBeenCalledTimes(2);
});
it.each(["present", "unknown"] as const)("avoids DB probes and backups for known local %s work", async state => {
const { db, transaction } = database();
const checkpoint = backup();
const plugins = vi.fn();
expect(await readIdleSleepSafety(db, owned, () => now, ownerId, async () => state, plugins, checkpoint))
.toEqual({ version: 1, backgroundWork: state });
expect(transaction).not.toHaveBeenCalled();
expect(plugins).not.toHaveBeenCalled();
expect(checkpoint).not.toHaveBeenCalled();
});
it("does not start a backup for durable work or an unowned hold", async () => {
const checkpoint = backup();
expect(await readIdleSleepSafety(database(true).db, owned, () => now, ownerId, emptyLocal, undefined, checkpoint)).toEqual(present);
expect(await readIdleSleepSafety(database().db, owned, () => now, "stale", emptyLocal, undefined, checkpoint)).toEqual(unknown);
expect(checkpoint).not.toHaveBeenCalled();
});
it.each(["failed", "threw", "untracked", "concurrent", "expired", "replaced"])("keeps the instance awake when the backup is %s", async kind => {
let clock = now;
let status = owned();
const work = async () => {
if (kind === "threw") throw new Error("private archive path");
if (kind === "concurrent") { const finish = beginIdleTrackedWork(); finish(); }
if (kind === "expired") clock += 60_000;
if (kind === "replaced") status = { ...status, ownerId: "replacement" };
return kind !== "failed";
};
const checkpoint = kind === "untracked" ? work : backup(work);
expect(await readIdleSleepSafety(database().db, () => status, () => clock, ownerId, emptyLocal, undefined, checkpoint)).toEqual(unknown);
expect(idleWorkSnapshot().active).toBe(0);
});
it("retains actual backup work after the owning hold expires", async () => {
let finishBackup!: () => void;
const pending = new Promise<void>(resolve => { finishBackup = resolve; });
const checkpoint = backup(async () => { await pending; return true; });
let clock = now;
const scan = readIdleSleepSafety(database().db, owned, () => clock, ownerId, emptyLocal, undefined, checkpoint);
await vi.waitFor(() => expect(checkpoint).toHaveBeenCalledOnce());
clock += 60_000;
expect(idleWorkSnapshot().active).toBe(1);
expect(await readIdleSleepSafety(database().db, owned, () => clock, ownerId, emptyLocal)).toEqual(unknown);
finishBackup();
expect(await scan).toEqual(unknown);
expect(idleWorkSnapshot().active).toBe(0);
});
});
describe("idle sleep safety failure boundaries", () => {
it.each([
@@ -36,7 +103,7 @@ describe("idle sleep safety failure boundaries", () => {
it("fails closed when the database is unavailable", async () => {
const db = { transaction: vi.fn().mockRejectedValue(new Error("private connection detail")) } as unknown as Db;
expect(await readIdleSleepSafety(db, held, () => now)).toEqual(unknown);
expect(await readIdleSleepSafety(db, held, () => now, undefined, emptyLocal)).toEqual(unknown);
});
it.each([
@@ -46,14 +113,14 @@ describe("idle sleep safety failure boundaries", () => {
const getStatus = vi.fn().mockReturnValueOnce(held()).mockReturnValue({ ...held(), ...change });
const execute = vi.fn().mockResolvedValue([{ blocked: true }]);
const transaction = vi.fn(async (fn: (tx: unknown) => Promise<unknown>) => fn({ execute }));
expect(await readIdleSleepSafety({ transaction } as unknown as Db, getStatus, () => now)).toEqual(unknown);
expect(await readIdleSleepSafety({ transaction } as unknown as Db, getStatus, () => now, undefined, emptyLocal)).toEqual(unknown);
expect(transaction).toHaveBeenCalledWith(expect.any(Function), { isolationLevel: "repeatable read", accessMode: "read only" });
});
it("reports persisted work when the admission hold remains unchanged", async () => {
const execute = vi.fn().mockResolvedValue([{ blocked: true }]);
const transaction = vi.fn(async (fn: (tx: unknown) => Promise<unknown>) => fn({ execute }));
expect(await readIdleSleepSafety({ transaction } as unknown as Db, held, () => now)).toEqual(present);
expect(await readIdleSleepSafety({ transaction } as unknown as Db, held, () => now, undefined, emptyLocal)).toEqual(present);
});
});
@@ -111,7 +178,7 @@ if (!support.supported) console.warn(`Skipping idle sleep Postgres tests: ${supp
const companyId = randomUUID();
const agentId = randomUUID();
const runId = randomUUID();
await db.insert(companies).values({ id: companyId, name: "Idle test", issuePrefix: "IDLE" });
await db.insert(companies).values({ id: companyId, name: "Idle test", issuePrefix: `T${companyId.slice(0, 6).toUpperCase()}` });
await db.insert(agents).values({ id: agentId, companyId, name: "On-demand agent", role: "engineer", status: "idle", adapterType: "process" });
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "on_demand", status: "succeeded" });
return { companyId, agentId, runId };
@@ -179,6 +246,44 @@ if (!support.supported) console.warn(`Skipping idle sleep Postgres tests: ${supp
expect(await read()).toEqual(present);
});
it("allows consumed coalesced wakes only after their linked run finishes", async () => {
const { companyId, agentId, runId } = await seed();
await db.update(heartbeatRuns).set({ status: "running" }).where(eq(heartbeatRuns.id, runId));
await db.insert(agentWakeupRequests).values({
companyId, agentId, source: "assignment", status: "coalesced", runId, finishedAt: new Date(now),
});
expect(await read()).toEqual(present);
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date(now) })
.where(eq(heartbeatRuns.id, runId));
expect(await read()).toEqual(none);
});
it.each(["unfinished", "missing_run", "wrong_company", "wrong_agent", "unfinished_run", "retry", "accounting"])(
"keeps a coalesced wake awake with %s evidence", async kind => {
const { companyId, agentId, runId } = await seed();
await db.update(heartbeatRuns).set({ finishedAt: new Date(now) }).where(eq(heartbeatRuns.id, runId));
let linkedRunId = runId;
if (kind === "missing_run") linkedRunId = randomUUID();
if (kind === "wrong_company") {
linkedRunId = (await seed()).runId;
await db.update(heartbeatRuns).set({ finishedAt: new Date(now) }).where(eq(heartbeatRuns.id, linkedRunId));
}
if (kind === "wrong_agent") {
const otherAgentId = randomUUID();
await db.insert(agents).values({ id: otherAgentId, companyId, name: "Other agent", role: "engineer", status: "idle", adapterType: "process" });
await db.update(heartbeatRuns).set({ agentId: otherAgentId }).where(eq(heartbeatRuns.id, runId));
}
if (kind === "unfinished_run") await db.update(heartbeatRuns).set({ finishedAt: null }).where(eq(heartbeatRuns.id, runId));
if (kind === "retry") await db.update(heartbeatRuns).set({ scheduledRetryAt: new Date(now + 1000) }).where(eq(heartbeatRuns.id, runId));
if (kind === "accounting") await db.update(heartbeatRuns).set({ costAccountingPending: true }).where(eq(heartbeatRuns.id, runId));
await db.insert(agentWakeupRequests).values({
companyId, agentId, source: "assignment", status: "coalesced", runId: linkedRunId,
finishedAt: kind === "unfinished" ? null : new Date(now),
});
expect(await read()).toEqual(present);
},
);
it("blocks an active routine without waiting for its next due time", async () => {
const { companyId } = await seed();
await db.insert(routines).values({ companyId, title: "Tomorrow", status: "active" });
@@ -201,6 +306,95 @@ if (!support.supported) console.warn(`Skipping idle sleep Postgres tests: ${supp
expect(await read()).toEqual(present);
});
async function seedFinishedLogin() {
const { companyId } = await seed();
const [environment] = await db.insert(environments).values({ name: "Finished login fixture" }).returning();
const [lease] = await db.insert(environmentLeases).values({
companyId, environmentId: environment!.id, providerLeaseId: "fixture-login-resource",
status: "released", cleanupStatus: "success", releasedAt: new Date(now),
}).returning();
const [session] = await db.insert(adapterAuthSessions).values({
companyId, environmentId: environment!.id, adapterType: "codex_local",
startedByUserId: "fixture-user", publicSessionId: "finished-fixture-session",
status: "authenticated", finishedAt: new Date(now), providerLeaseId: lease!.providerLeaseId,
}).returning();
return { companyId, environment: environment!, lease: lease!, session: session! };
}
it.each(["authenticated", "completed", "failed", "timed_out", "cancelled"] as const)(
"permits finished %s login history after confirmed provider cleanup", async status => {
const { session } = await seedFinishedLogin();
await db.update(adapterAuthSessions).set({ status }).where(eq(adapterAuthSessions.id, session.id));
expect(await read()).toEqual(none);
},
);
it.each(["canonical", "uppercase"])("accepts a %s cleanup reference to the internal lease id", async format => {
const { session, lease } = await seedFinishedLogin();
await db.update(adapterAuthSessions).set({ providerLeaseId: format === "uppercase" ? lease.id.toUpperCase() : lease.id })
.where(eq(adapterAuthSessions.id, session.id));
expect(await read()).toEqual(none);
});
it.each(["fixture-provider-id", "00000000-0000-0000-0000-00000000000z", randomUUID()])(
"accepts cleaned external provider reference %s without an unsafe UUID cast", async providerLeaseId => {
const { session, lease } = await seedFinishedLogin();
await db.update(environmentLeases).set({ providerLeaseId }).where(eq(environmentLeases.id, lease.id));
await db.update(adapterAuthSessions).set({ providerLeaseId }).where(eq(adapterAuthSessions.id, session.id));
expect(await read()).toEqual(none);
},
);
it("keeps a UUID-shaped external reference awake when an internal lease conflicts", async () => {
const { companyId, environment, session, lease } = await seedFinishedLogin();
const conflictingId = randomUUID();
await db.update(environmentLeases).set({ providerLeaseId: conflictingId }).where(eq(environmentLeases.id, lease.id));
await db.update(adapterAuthSessions).set({ providerLeaseId: conflictingId }).where(eq(adapterAuthSessions.id, session.id));
await db.insert(environmentLeases).values({
id: conflictingId, companyId, environmentId: environment.id,
status: "released", cleanupStatus: "failed", releasedAt: new Date(now),
});
expect(await read()).toEqual(present);
await db.update(environmentLeases).set({ cleanupStatus: "success" }).where(eq(environmentLeases.id, conflictingId));
expect(await read()).toEqual(none);
});
it("permits terminal login history without a provider resource", async () => {
const { session } = await seedFinishedLogin();
await db.update(adapterAuthSessions).set({ providerLeaseId: null }).where(eq(adapterAuthSessions.id, session.id));
expect(await read()).toEqual(none);
});
it.each(["cleanup_pending", "starting", "promoting", "awaiting_code", "submitting", "stored"] as const)(
"retains %s login work even with an old successful cleanup receipt", async status => {
const { session } = await seedFinishedLogin();
await db.update(adapterAuthSessions).set({ status }).where(eq(adapterAuthSessions.id, session.id));
expect(await read()).toEqual(present);
},
);
it.each(["unfinished", "promotion_claim", "missing_lease", "wrong_company", "wrong_environment", "cleanup_failed", "cleanup_unknown", "unreleased", "conflicting_lease"])(
"retains terminal login history with %s cleanup evidence", async kind => {
const { companyId, session, lease, environment } = await seedFinishedLogin();
if (kind === "unfinished") await db.update(adapterAuthSessions).set({ finishedAt: null }).where(eq(adapterAuthSessions.id, session.id));
if (kind === "promotion_claim") await db.update(adapterAuthSessions).set({ promotionExpiresAt: new Date(now + 1000) }).where(eq(adapterAuthSessions.id, session.id));
if (kind === "missing_lease") await db.delete(environmentLeases).where(eq(environmentLeases.id, lease.id));
if (kind === "wrong_company") await db.update(environmentLeases).set({ companyId: (await seed()).companyId }).where(eq(environmentLeases.id, lease.id));
if (kind === "wrong_environment") {
const [other] = await db.insert(environments).values({ name: "Other environment", driver: "sandbox" }).returning();
await db.update(environmentLeases).set({ environmentId: other!.id }).where(eq(environmentLeases.id, lease.id));
}
if (kind === "cleanup_failed") await db.update(environmentLeases).set({ cleanupStatus: "failed" }).where(eq(environmentLeases.id, lease.id));
if (kind === "cleanup_unknown") await db.update(environmentLeases).set({ cleanupStatus: null }).where(eq(environmentLeases.id, lease.id));
if (kind === "unreleased") await db.update(environmentLeases).set({ releasedAt: null }).where(eq(environmentLeases.id, lease.id));
if (kind === "conflicting_lease") await db.insert(environmentLeases).values({
companyId, environmentId: environment.id, providerLeaseId: lease.providerLeaseId,
status: "released", cleanupStatus: "failed", releasedAt: new Date(now),
});
expect(await read()).toEqual(present);
},
);
it("reports a pending secret proposal with future expiry", async () => {
const { companyId, agentId, runId } = await seed();
await db.insert(companySecretProposals).values({
@@ -245,6 +439,22 @@ if (!support.supported) console.warn(`Skipping idle sleep Postgres tests: ${supp
expect(await read()).toEqual(none);
});
it("accepts only exact worker drain receipts and still checks durable work", async () => {
const [{ id }] = await db.execute<{ id: string }>(sql`INSERT INTO plugins (plugin_key, package_name, version, manifest_json)
VALUES ('demo.idle', 'demo-idle', '1.0.0', '{}'::jsonb) RETURNING id`);
const inspect = vi.fn(async () => ({ backgroundWork: "none" as const, pluginIds: [id!] }));
const scan = () => readIdleSleepSafety(db, owned, () => now, ownerId, emptyLocal, inspect);
expect(await scan()).toEqual(none);
expect(inspect).toHaveBeenCalledWith({ ownerId, expiresAt: owned().expiresAt!.getTime() });
await db.execute(sql`INSERT INTO plugins (plugin_key, package_name, version, manifest_json)
VALUES ('demo.unknown', 'demo-unknown', '1.0.0', '{}'::jsonb)`);
expect(await scan()).toEqual(present);
await db.execute(sql`UPDATE plugins SET status = 'disabled' WHERE plugin_key = 'demo.unknown'`);
const { companyId } = await seed();
await db.insert(issues).values({ companyId, title: "Still pending", status: "todo" });
expect(await scan()).toEqual(present);
});
it("fails closed when the installed schema is older than the report", async () => {
await db.execute(sql`ALTER TABLE heartbeat_runs RENAME COLUMN cost_accounting_pending TO hidden_pending`);
try { expect(await read()).toEqual(unknown); }
@@ -20,6 +20,7 @@ import {
import { LOW_TRUST_REVIEW_PRESET } from "@paperclipai/shared";
import { errorHandler } from "../middleware/index.js";
import { issueRoutes } from "../routes/issues.js";
import { issueService } from "../services/issues.js";
import { subscribeCompanyLiveEvents } from "../services/live-events.js";
import {
getEmbeddedPostgresTestSupport,
@@ -194,6 +195,168 @@ describeEmbeddedPostgres("inbox archive routes", () => {
.expect(({ body }) => expect(body).toEqual({ ok: true, userId: seeded.responsibleUserId }));
});
it.each([false, true])("allows manual archive during human completion (existing archive: %s)", async (alreadyArchived) => {
const seeded = await seed();
const svc = issueService(db);
if (alreadyArchived) {
await svc.archiveInbox(seeded.companyId, seeded.issueId, seeded.responsibleUserId);
}
const app = appFor({
type: "board",
source: "session",
userId: seeded.responsibleUserId,
companyIds: [seeded.companyId],
memberships: [{ companyId: seeded.companyId, membershipRole: "operator", status: "active" }],
isInstanceAdmin: false,
});
let manualArchive: Promise<{ status: number }> | undefined;
let completed: Awaited<ReturnType<typeof svc.update>>;
let manualStatus: number | undefined;
try {
completed = await db.transaction(async (tx) => {
await tx.execute(sql`set local statement_timeout = '5s'`);
// Human completion locks the issue before it writes the archive.
await tx.select({ id: issues.id }).from(issues).where(eq(issues.id, seeded.issueId)).for("update");
const [{ pid }] = await tx.execute<{ pid: number }>(sql`select pg_backend_pid() as pid`);
manualArchive = request(app).post(`/api/issues/${seeded.issueId}/inbox-archive`).send({}).then((response) => response);
await expect.poll(async () => {
const blocked = await db.execute(sql`
select 1 from pg_stat_activity
where datname = current_database() and ${pid} = any(pg_blocking_pids(pid))
`);
return blocked.length;
}, { timeout: 3_000 }).toBeGreaterThan(0);
// The manual request is now waiting on our parent row. Previously it
// already owned the archive insert, so this completion formed a cycle.
return svc.update(seeded.issueId, {
status: "done",
actorUserId: seeded.responsibleUserId,
companyGuard: seeded.companyId,
}, tx, []);
});
} finally {
// Do not leave an in-flight route writing into the next test's fixture.
if (manualArchive) manualStatus = (await manualArchive).status;
}
expect(manualStatus).toBe(200);
expect(completed!.status).toBe("done");
const archives = await db.select().from(issueInboxArchives);
expect(archives).toHaveLength(1);
expect(archives[0]).toMatchObject({
companyId: seeded.companyId,
issueId: seeded.issueId,
userId: seeded.responsibleUserId,
archivedByActorType: "user",
archivedByAgentId: null,
archivedByRunId: null,
});
const audits = await db.select().from(activityLog).where(eq(activityLog.action, "issue.inbox_archived"));
expect(audits).toHaveLength(2);
expect(audits.some((audit) => audit.details?.source === "issue_status_done")).toBe(true);
expect(archives[0].archivedAt.getTime()).toBeGreaterThanOrEqual(completed!.completedAt!.getTime());
// The status route records this activity after the completion transaction.
await db.insert(activityLog).values({
companyId: seeded.companyId,
actorType: "user",
actorId: seeded.responsibleUserId,
action: "issue.updated",
entityType: "issue",
entityId: seeded.issueId,
details: { status: "done", _previous: { status: "todo" } },
});
await request(app).get(`/api/companies/${seeded.companyId}/issues`)
.query({ touchedByUserId: seeded.responsibleUserId, inboxArchivedByUserId: seeded.responsibleUserId, status: "done" })
.expect(200)
.expect(({ body }) => expect(body.map((issue: { id: string }) => issue.id)).not.toContain(seeded.issueId));
});
it("keeps caller rollback atomic without blocking a different user's archive", async () => {
const seeded = await seed();
const svc = issueService(db);
const rollback = new Error("Roll back the first user's archive");
await expect(db.transaction(async (tx) => {
await svc.archiveInbox(seeded.companyId, seeded.issueId, seeded.responsibleUserId, new Date(), undefined, tx);
// An exclusive parent lock would unnecessarily serialize these users.
await db.transaction(async (otherTx) => {
await otherTx.execute(sql`set local statement_timeout = '3s'`);
await svc.archiveInbox(seeded.companyId, seeded.issueId, seeded.targetUserId, new Date(), undefined, otherTx);
});
throw rollback;
})).rejects.toBe(rollback);
const archives = await db.select().from(issueInboxArchives);
expect(archives).toHaveLength(1);
expect(archives[0].userId).toBe(seeded.targetUserId);
});
it.each([-1, 0, 1])("keeps the newest archive time and matching attribution (offset: %s)", async (offset) => {
const seeded = await seed();
const svc = issueService(db);
const firstTime = new Date("2000-01-02T00:00:00.000Z");
const nextTime = new Date(firstTime.getTime() + offset * 1_000);
const first = await svc.archiveInbox(seeded.companyId, seeded.issueId, seeded.responsibleUserId, firstTime, {
archivedByActorType: "agent",
archivedByAgentId: seeded.agentId,
archivedByRunId: seeded.runId,
});
const next = await svc.archiveInbox(seeded.companyId, seeded.issueId, seeded.responsibleUserId, nextTime);
expect(next.id).toBe(first.id);
expect(next).toMatchObject(offset < 0 ? first : {
archivedAt: nextTime,
archivedByActorType: "user",
archivedByAgentId: null,
archivedByRunId: null,
});
expect(await db.select().from(issueInboxArchives)).toEqual([next]);
await svc.unarchiveInbox(seeded.companyId, seeded.issueId, seeded.responsibleUserId);
const afterUndo = await svc.archiveInbox(seeded.companyId, seeded.issueId, seeded.responsibleUserId, nextTime);
expect(afterUndo).toMatchObject({ archivedAt: nextTime, archivedByActorType: "user" });
expect(afterUndo.id).not.toBe(first.id);
});
it("holds the issue company stable until its archive transaction ends", async () => {
const seeded = await seed();
const otherCompany = await seed();
const svc = issueService(db);
let move: Promise<unknown> | undefined;
try {
await db.transaction(async (tx) => {
await svc.archiveInbox(seeded.companyId, seeded.issueId, seeded.responsibleUserId, new Date(), undefined, tx);
const [{ pid }] = await tx.execute<{ pid: number }>(sql`select pg_backend_pid() as pid`);
move = db.update(issues).set({ companyId: otherCompany.companyId })
.where(eq(issues.id, seeded.issueId)).then((rows) => rows);
await expect.poll(async () => {
const blocked = await db.execute(sql`
select 1 from pg_stat_activity
where datname = current_database() and ${pid} = any(pg_blocking_pids(pid))
`);
return blocked.length;
}, { timeout: 3_000 }).toBeGreaterThan(0);
const [stillOwned] = await tx.select().from(issues).where(eq(issues.id, seeded.issueId));
expect(stillOwned.companyId).toBe(seeded.companyId);
});
} finally {
await move;
}
const [moved] = await db.select().from(issues).where(eq(issues.id, seeded.issueId));
expect(moved.companyId).toBe(otherCompany.companyId);
});
it("rejects missing and cross-company issues before archiving", async () => {
const seeded = await seed();
const svc = issueService(db);
const otherCompany = await seed();
for (const [companyId, issueId] of [
[otherCompany.companyId, seeded.issueId],
[seeded.companyId, randomUUID()],
]) {
await expect(svc.archiveInbox(companyId, issueId, seeded.responsibleUserId))
.rejects.toMatchObject({ status: 404, message: "Issue not found" });
}
expect(await db.select().from(issueInboxArchives)).toHaveLength(0);
});
it("silently archives an issue for the board user who moves it to done", async () => {
const seeded = await seed();
const app = appFor({
@@ -2,6 +2,7 @@ import express from "express";
import request from "supertest";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { hoistModuleGraph } from "./helpers/hoist-module-graph.js";
import type { PluginWorkerManager } from "../services/plugin-worker-manager.js";
const mockInstanceSettingsService = vi.hoisted(() => ({
get: vi.fn(),
@@ -76,7 +77,7 @@ describe("instance settings routes", () => {
return { errorHandler, instanceSettingsRoutes };
});
function createApp(actor: any) {
function createApp(actor: any, pluginWorkers?: PluginWorkerManager, prepareBackup?: () => Promise<boolean>) {
const { errorHandler, instanceSettingsRoutes } = routeModules.value;
const app = express();
app.use(express.json());
@@ -84,7 +85,7 @@ describe("instance settings routes", () => {
req.actor = actor;
next();
});
app.use("/api", instanceSettingsRoutes(mockDb as any));
app.use("/api", instanceSettingsRoutes(mockDb as any, pluginWorkers, prepareBackup));
app.use(errorHandler);
return app;
}
@@ -972,7 +973,7 @@ describe("instance settings routes", () => {
await request(createApp(adminActor)).post("/api/instance/task-drain").send({ purpose: "idle", ttlMs: 60_000 }).expect(200);
expect(mockHeartbeatService.computeTaskDrain).toHaveBeenCalledWith({ purpose: "idle", ttlMs: 60_000 });
await request(createApp(adminActor)).get(`/api/instance/task-drain?idleSleepSafety=1&ownerId=${ownerId}`).expect(200);
expect(mockReadIdleSleepSafety).toHaveBeenLastCalledWith(mockDb, expect.any(Function), Date.now, ownerId);
expect(mockReadIdleSleepSafety).toHaveBeenLastCalledWith(mockDb, expect.any(Function), Date.now, ownerId, expect.any(Function), undefined, undefined);
});
it.each([{}, { ttlMs: null }, { ttlMs: 1 }, { ttlMs: 300_001 }])("rejects an unbounded or invalid idle TTL: %j", async body => {
@@ -999,10 +1000,70 @@ describe("instance settings routes", () => {
const res = await request(createApp(adminActor)).get("/api/instance/task-drain?idleSleepSafety=1");
expect(res.status).toBe(200);
expect(res.body).toEqual({ ...idleStatus, idleSleepSafety: report });
expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now, undefined);
expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now, undefined, expect.any(Function), undefined, undefined);
expect(mockReadIdleSleepSafety.mock.calls[0][1]()).toEqual(idleStatus);
});
it("uses the live worker manager for inspection and releases only the owned hold", async () => {
const inspect = vi.fn(async () => ({ backgroundWork: "none", pluginIds: [] }));
const release = vi.fn();
const workers = {
inspectIdleSleep: inspect,
releaseIdleSleep: release,
} as unknown as PluginWorkerManager;
const ownerId = "fixture-idle-owner";
const lease = { ownerId, expiresAt: Date.now() + 60_000 };
mockHeartbeatService.getTaskDrainStatus.mockReturnValue({ ...idleStatus, draining: true, ownerId });
mockReadIdleSleepSafety.mockImplementation(async (_db, _status, _now, owner, _local, inspectPlugins) => {
expect(owner).toBe(ownerId);
return { version: 1, backgroundWork: (await inspectPlugins(lease)).backgroundWork };
});
const app = createApp(adminActor, workers);
await request(app).get(`/api/instance/task-drain?idleSleepSafety=1&ownerId=${ownerId}`).expect(200);
expect(inspect).toHaveBeenCalledWith(lease);
expect(inspect.mock.contexts[0]).toBe(workers);
await request(app).delete("/api/instance/task-drain?ownerId=stale-owner").expect(409);
expect(release).not.toHaveBeenCalled();
await request(app).delete(`/api/instance/task-drain?ownerId=${ownerId}`).expect(200);
expect(release).toHaveBeenCalledOnce();
});
it("passes the operator backup hook only through the protected safety report", async () => {
const backup = vi.fn(async () => true);
mockHeartbeatService.getTaskDrainStatus.mockReturnValue(idleStatus);
const app = createApp(adminActor, undefined, backup);
await request(app).get("/api/instance/task-drain").expect(200);
expect(mockReadIdleSleepSafety).not.toHaveBeenCalled();
await request(app).get("/api/instance/task-drain?idleSleepSafety=1&ownerId=fixture-owner").expect(200);
expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now,
"fixture-owner", expect.any(Function), undefined, backup);
mockReadIdleSleepSafety.mockClear();
await request(createApp(nonAdminActor, undefined, backup)).get("/api/instance/task-drain?idleSleepSafety=1").expect(403);
expect(mockReadIdleSleepSafety).not.toHaveBeenCalled();
expect(backup).not.toHaveBeenCalled();
});
it("reserves managed checkpoint creation for verified Cloud control actors", async () => {
const backup = vi.fn(async () => true);
process.env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN = "fixture-managed-signal";
mockHeartbeatService.getTaskDrainStatus.mockReturnValue(idleStatus);
try {
for (const source of ["session", "cloud_tenant", "local_implicit"]) {
const tenantAdmin = createApp({ ...adminActor, source }, undefined, backup);
await request(tenantAdmin).get("/api/instance/task-drain?idleSleepSafety=1&ownerId=fixture-owner")
.set("x-paperclip-cloud-control", "unverified-header").expect(403);
}
expect(mockReadIdleSleepSafety).not.toHaveBeenCalled();
expect(backup).not.toHaveBeenCalled();
const control = createApp({ ...adminActor, source: "cloud_control" }, undefined, backup);
await request(control).get("/api/instance/task-drain?idleSleepSafety=1&ownerId=fixture-owner").expect(200);
expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now,
"fixture-owner", expect.any(Function), undefined, backup);
} finally {
delete process.env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN;
}
});
it.each([
["company member", nonAdminActor],
["agent", { type: "agent", agentId: "agent-1", companyId: "company-1" }],
@@ -24,6 +24,11 @@ describe("instance settings service", () => {
const disabled = applyExperimentalSettingsPatch(enabled, { enableChatConnectors: false });
expect(disabled).toMatchObject({ enableApps: true, enableChatConnectors: false });
});
it("keeps GitHub review bots independent across legacy normalization and persistence", () => {
expect(normalizeExperimentalSettings({ enableChatConnectors: true })).toMatchObject({ enableChatConnectors: true, enableGitHubReviewBots: false });
const saved = applyExperimentalSettingsPatch({}, { enableGitHubReviewBots: true });
expect(normalizeExperimentalSettings(JSON.parse(JSON.stringify(saved)))).toMatchObject({ enableChatConnectors: false, enableGitHubReviewBots: true });
});
it("ignores retired experimental flags without resetting current settings", () => {
expect(normalizeExperimentalSettings({
enableEnvironments: true,
@@ -57,6 +62,7 @@ describe("instance settings service", () => {
enableOpenAiDot: false,
enableCombinedInboxTasks: false,
enableChatConnectors: false,
enableGitHubReviewBots: false,
enableMemoryConnectors: false,
enableConferenceRoomChat: false,
enableClassicTaskInterface: false,
@@ -7,6 +7,10 @@ const canUserMock = vi.fn();
const insertMock = vi.fn();
function registerModuleMocks() {
vi.doMock("../services/agent-lifecycle.js", () => ({
createAgentLifecycle: () => ({}),
AgentLifecycleConflict: class extends Error {},
}));
vi.doMock("../services/index.js", () => ({
accessService: () => ({
isInstanceAdmin: vi.fn(),
@@ -8,7 +8,7 @@ import {
normalizeHumanRole,
resolveHumanInviteRole,
} from "../services/company-member-roles.js";
import { NEW_STANDARD_AGENT_DEFAULT_GRANT_KEYS, newStandardAgentGrantScope } from "../services/agent-permissions.js";
import { NEW_STANDARD_AGENT_DEFAULT_GRANT_KEYS, newStandardAgentGrantScope } from "../lib/agent-permissions.js";
const invitedAgentId = "agent-1";
const defaultAgentGrants = NEW_STANDARD_AGENT_DEFAULT_GRANT_KEYS.map((permissionKey) => ({
@@ -40,6 +40,9 @@ describe("buildInviteOnboardingTextDocument", () => {
});
expect(text).toContain("Paperclip Agent Onboarding");
expect(text).toContain("use your existing Slack connection");
expect(text).toContain("Setup or test restrictions apply only to their stated scope");
expect(text).toContain("Never use another Paperclip connection or identity to bypass a denial");
expect(text).toContain("/api/invites/token-123/accept");
expect(text).toContain("/api/join-requests/{requestId}/claim-api-key");
expect(text).toContain("/api/invites/token-123/onboarding.txt");
@@ -1,6 +1,6 @@
import express from "express";
import request from "supertest";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
function createSelectChain(rows: unknown[]) {
const query = {
@@ -44,6 +44,13 @@ function createInvite(overrides: Record<string, unknown> = {}) {
};
}
function loadAppModules() {
return Promise.all([
import("../routes/access.js"),
import("../middleware/index.js"),
]);
}
async function createApp(
db: Record<string, unknown>,
network: {
@@ -51,10 +58,7 @@ async function createApp(
requestHead: ReturnType<typeof vi.fn>;
},
) {
const [access, middleware] = await Promise.all([
import("../routes/access.js"),
import("../middleware/index.js"),
]);
const [access, middleware] = await loadAppModules();
const app = express();
app.use((req, _res, next) => {
(req as any).actor = { type: "anon" };
@@ -74,7 +78,12 @@ async function createApp(
return app;
}
describe("GET /invites/:token/test-resolution", () => {
describe("GET /invites/:token/test-resolution", { sequential: true }, () => {
beforeAll(async () => {
// Route transformation is fixture setup, not part of the network assertions.
await loadAppModules();
});
beforeEach(() => {
vi.clearAllMocks();
});
@@ -2439,6 +2439,71 @@ describe("agent issue mutation checkout ownership", () => {
);
});
describe("repairing an unavailable starting branch", () => {
const repair = { actionId: recoveryActionId, outcome: "restored", sourceIssueStatus: "todo",
workspaceBaseRef: { requestedRef: "main", branch: "master" } };
function seedBranchRepair() {
mockAccessService.decide.mockResolvedValue({ allowed: true, reason: "allow_explicit_grant", explanation: "Allowed" });
const issue = makeIssue({ status: "blocked", assigneeAgentId: ownerAgentId,
executionWorkspaceSettings: { mode: "isolated_workspace", workspaceStrategy: { type: "git_worktree", baseRef: "main", branchTemplate: "task/{issueId}" } } });
mockIssueService.getById.mockResolvedValue(issue);
mockIssueService.update.mockImplementation(async (_id, patch) => ({ ...issue, ...patch }));
mockIssueRecoveryActionService.getActiveForIssue.mockResolvedValue({ id: recoveryActionId,
status: "active", cause: "configuration_incomplete", ownerType: "board", returnOwnerAgentId: ownerAgentId,
evidence: { latestRunId: ownerRunId } });
return createRunContextDb({}, [{ id: ownerRunId, companyId, agentId: ownerAgentId,
contextSnapshot: { issueId }, errorCode: "configuration_incomplete",
resultJson: { configurationIncomplete: { reason: "workspace_base_ref_unresolved", requestedRef: "main" } } }]);
}
it("saves the task-only branch in the recovery transaction and requests a retry", async () => {
const db = seedBranchRepair();
const res = await request(await createApp(boardActor(), db)).post(`/api/issues/${issueId}/recovery-actions/resolve`).send(repair);
expect(res.status, JSON.stringify(res.body)).toBe(200);
expect(mockIssueService.update).toHaveBeenCalledWith(issueId, expect.objectContaining({ status: "todo",
executionWorkspaceSettings: { mode: "isolated_workspace", workspaceStrategy: { type: "git_worktree", baseRef: "master", branchTemplate: "task/{issueId}" } },
}), expect.anything(), expect.anything());
expect(mockHeartbeatService.wakeup).toHaveBeenCalledTimes(1);
});
it("creates a valid task override when the failing branch was inherited", async () => {
const db = seedBranchRepair();
mockIssueService.getById.mockResolvedValue(makeIssue({ status: "blocked", assigneeAgentId: ownerAgentId, executionWorkspaceSettings: null }));
const res = await request(await createApp(boardActor(), db)).post(`/api/issues/${issueId}/recovery-actions/resolve`).send(repair);
expect(res.status, JSON.stringify(res.body)).toBe(200);
expect(mockIssueService.update).toHaveBeenCalledWith(issueId, expect.objectContaining({
executionWorkspaceSettings: { workspaceStrategy: { type: "git_worktree", baseRef: "master" } },
}), expect.anything(), expect.anything());
});
it("preserves inherited agent setup when repairing only the base branch", async () => {
const db = seedBranchRepair();
mockIssueService.getById.mockResolvedValue(makeIssue({ status: "blocked", assigneeAgentId: ownerAgentId, executionWorkspaceSettings: null }));
const strategy = { type: "git_worktree", baseRef: "main", provisionCommand: "npm run setup", branchTemplate: "task/{{issue.identifier}}", worktreeParentDir: ".worktrees" };
mockAgentService.getById.mockResolvedValue({ ...makeAgent(ownerAgentId), adapterConfig: { workspaceStrategy: strategy } });
const res = await request(await createApp(boardActor(), db)).post(`/api/issues/${issueId}/recovery-actions/resolve`).send(repair);
expect(res.status, JSON.stringify(res.body)).toBe(200);
expect(mockIssueService.update).toHaveBeenCalledWith(issueId, expect.objectContaining({
executionWorkspaceSettings: { workspaceStrategy: { ...strategy, baseRef: "master" } },
}), expect.anything(), expect.anything());
expect(strategy.baseRef).toBe("main");
});
it.each(["stale", "project", "strategy", "agent", "permission", "paused", "run", "approval", "budget", "blocker"])("rejects %s repairs without saving or waking", async gate => {
const db = seedBranchRepair();
if (gate === "project") mockIssueService.getById.mockResolvedValue(makeIssue({ status: "blocked", assigneeAgentId: ownerAgentId, projectId: "new-project" }));
if (gate === "strategy") mockIssueService.getById.mockResolvedValue(makeIssue({ status: "blocked", assigneeAgentId: ownerAgentId, executionWorkspaceSettings: { workspaceStrategy: { type: "adapter_managed" } } }));
if (gate === "permission") mockAccessService.decide.mockImplementation(async ({ action }) => ({ allowed: action !== "runtime:manage", reason: "deny_missing_grant", explanation: "No runtime permission" }));
if (gate === "stale") mockIssueService.getById.mockResolvedValue(makeIssue({ status: "blocked", assigneeAgentId: ownerAgentId,
executionWorkspaceSettings: { workspaceStrategy: { baseRef: "release" } } }));
if (gate === "paused") mockAgentService.getById.mockResolvedValue({ ...makeAgent(ownerAgentId), status: "paused" });
if (gate === "run") mockIssueService.getById.mockResolvedValue(makeIssue({ status: "blocked", assigneeAgentId: ownerAgentId, executionRunId: ownerRunId }));
if (gate === "approval") mockIssueApprovalService.listApprovalsForIssue.mockResolvedValue([{ status: "pending" }]);
if (gate === "budget") mockBudgetService.getInvocationBlock.mockResolvedValue({ scope: "agent", reason: "hard_limit_reached" });
if (gate === "blocker") mockIssueService.getDependencyReadiness.mockResolvedValue({ unresolvedBlockerCount: 1 });
const res = await request(await createApp(gate === "agent" ? ownerActor() : boardActor(), db)).post(`/api/issues/${issueId}/recovery-actions/resolve`).send(repair);
expect([403, 409], JSON.stringify(res.body)).toContain(res.status);
expect(mockIssueService.update).not.toHaveBeenCalled();
expect(mockHeartbeatService.wakeup).not.toHaveBeenCalled();
});
});
describe("retrying an escalated disposition repair", () => {
function seedRetry() {
mockIssueService.getById.mockResolvedValue(makeIssue({ status: "blocked", assigneeAgentId: ownerAgentId }));
@@ -0,0 +1,45 @@
import { randomUUID } from "node:crypto";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { companies, createDb, issues } from "@paperclipai/db";
import { buildExecutionPolicy } from "../../../ui/src/lib/issue-execution-policy.ts";
import { normalizeIssueExecutionPolicy } from "../services/issue-execution-policy.js";
import { issueService } from "../services/issues.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
const support = await getEmbeddedPostgresTestSupport();
(support.supported ? describe : describe.skip)("edited execution policy persistence", () => {
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let db: ReturnType<typeof createDb>;
beforeAll(async () => {
temporary = await startEmbeddedPostgresTestDatabase("paperclip-policy-persistence-");
db = createDb(temporary.connectionString);
}, 30_000);
afterAll(async () => { await temporary?.cleanup(); });
it.each(["reviewer", "monitor"] as const)("retains the independent review limit after removing the last %s", async (removed) => {
const companyId = randomUUID();
const issueId = randomUUID();
const original = normalizeIssueExecutionPolicy({
maxReviewRounds: 4,
...(removed === "monitor"
? { monitor: { nextCheckAt: "2099-01-01T00:00:00.000Z" } }
: { stages: [{ type: "review", participants: [{ type: "user", userId: "board-reviewer" }] }] }),
});
await db.insert(companies).values({ id: companyId, name: "Policy fixture", issuePrefix: `P${companyId.slice(0, 6)}` });
await db.insert(issues).values({ id: issueId, companyId, title: "Review limit fixture", executionPolicy: original });
const edited = buildExecutionPolicy({ existingPolicy: original, reviewerValues: [], approverValues: [] })!;
// The monitor editor removes this key after building the unchanged stages.
const { monitor: _removedMonitor, ...withoutMonitor } = edited;
const normalized = normalizeIssueExecutionPolicy(removed === "monitor" ? withoutMonitor : edited);
const service = issueService(db);
await service.update(issueId, { executionPolicy: normalized, companyGuard: companyId });
const saved = await service.getById(issueId);
expect(saved?.executionPolicy).toEqual({ mode: "normal", commentRequired: true, stages: [], maxReviewRounds: 4 });
const restored = normalizeIssueExecutionPolicy(buildExecutionPolicy({
existingPolicy: saved?.executionPolicy,
reviewerValues: ["user:board-reviewer"], approverValues: [],
}));
expect(restored).toMatchObject({ maxReviewRounds: 4, stages: [{ type: "review" }] });
});
});
@@ -173,11 +173,15 @@ type TestActor =
runId: string | null;
};
async function createApp(actor?: TestActor) {
const [{ errorHandler }, { issueRoutes }] = await Promise.all([
function loadAppModules() {
return Promise.all([
import("../middleware/index.js"),
import("../routes/issues.js"),
]);
}
async function createApp(actor?: TestActor) {
const [{ errorHandler }, { issueRoutes }] = await loadAppModules();
const app = express();
app.use(express.json());
app.use((req, _res, next) => {
@@ -196,7 +200,7 @@ async function createApp(actor?: TestActor) {
}
describe("issue execution policy routes", () => {
beforeEach(() => {
beforeEach(async () => {
vi.resetModules();
vi.doUnmock("../services/index.js");
vi.doUnmock("../routes/issues.js");
@@ -263,6 +267,37 @@ describe("issue execution policy routes", () => {
};
});
mockAccessService.hasPermission.mockResolvedValue(false);
// Finish cold imports before a test configures its request-specific mocks.
// A timed-out import must not resume a request against the next test's mocks.
await loadAppModules();
});
it.each([
{ status: "done" },
{ status: "cancelled" },
{ assigneeAgentId: "33333333-3333-4333-8333-333333333333" },
{ assigneeUserId: "operator" },
{ interrupt: true },
{ comment: "Continue now" },
])("rejects mixed snapshot-guarded updates before execution side effects: %j", async (controlFields) => {
const res = await request(await createApp({ type: "board", source: "local_implicit", userId: "local-board" }))
.patch("/api/issues/aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa")
.send({ expectedExecutionPolicy: null, executionPolicy: null, ...controlFields });
expect(res.status).toBe(422);
expect(res.body.error).toContain("policy-only");
expect(mockIssueService.getById).not.toHaveBeenCalled();
expect(mockIssueService.update).not.toHaveBeenCalled();
expect(mockRunnerGoalService.act).not.toHaveBeenCalled();
expect(mockHeartbeatService.cancelRun).not.toHaveBeenCalled();
});
it("rejects a snapshot without a policy update", async () => {
const res = await request(await createApp({ type: "board", source: "local_implicit", userId: "local-board" }))
.patch("/api/issues/aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa")
.send({ expectedExecutionPolicy: null });
expect(res.status).toBe(422);
expect(mockIssueService.getById).not.toHaveBeenCalled();
expect(mockIssueService.update).not.toHaveBeenCalled();
});
it("reauthorizes a terminal verdict against the review policy held under the update lock", async () => {
@@ -1,6 +1,7 @@
import { describe, expect, it } from "vitest";
import { applyIssueExecutionPolicyTransition, normalizeIssueExecutionPolicy, parseIssueExecutionState } from "../services/issue-execution-policy.ts";
import type { IssueExecutionPolicy, IssueExecutionState } from "@paperclipai/shared";
import { buildExecutionPolicy } from "../../../ui/src/lib/issue-execution-policy.ts";
const coderAgentId = "11111111-1111-4111-8111-111111111111";
const qaAgentId = "22222222-2222-4222-8222-222222222222";
@@ -43,6 +44,23 @@ describe("normalizeIssueExecutionPolicy", () => {
expect(normalizeIssueExecutionPolicy({ stages: [] })).toBeNull();
});
it("retains an independent review limit through removing and restoring the last reviewer", () => {
const original = normalizeIssueExecutionPolicy({
maxReviewRounds: 4,
stages: [{ type: "review", participants: [{ type: "agent", agentId: qaAgentId }] }],
});
const edited = buildExecutionPolicy({ existingPolicy: original, reviewerValues: [], approverValues: [] });
const saved = normalizeIssueExecutionPolicy(edited);
expect(saved).toEqual({ mode: "normal", commentRequired: true, stages: [], maxReviewRounds: 4 });
expect(applyIssueExecutionPolicyTransition({
issue: { status: "in_progress", assigneeAgentId: coderAgentId },
policy: saved, requestedStatus: "done", requestedAssigneePatch: {}, actor: { agentId: coderAgentId },
})).toEqual({ patch: {} });
expect(normalizeIssueExecutionPolicy(buildExecutionPolicy({
existingPolicy: saved, reviewerValues: [`agent:${qaAgentId}`], approverValues: [],
}))).toMatchObject({ maxReviewRounds: 4, stages: [{ type: "review" }] });
});
it("throws when all participants are invalid (missing agentId)", () => {
expect(() =>
normalizeIssueExecutionPolicy({
@@ -8,6 +8,7 @@ const mockFeedbackService = vi.hoisted(() => ({
listIssueVotesForUser: vi.fn(),
listFeedbackTraces: vi.fn(),
saveIssueVote: vi.fn(),
flushPendingFeedbackTraces: vi.fn(() => new Promise(() => {})),
}));
const mockIssueService = vi.hoisted(() => ({
@@ -18,9 +19,6 @@ const mockIssueService = vi.hoisted(() => ({
findMentionedAgents: vi.fn(),
}));
const mockFeedbackExportService = vi.hoisted(() => ({
flushPendingFeedbackTraces: vi.fn(async () => ({ attempted: 1, sent: 1, failed: 0 })),
}));
const mockAccessService = vi.hoisted(() => ({
canUser: vi.fn(),
hasPermission: vi.fn(),
@@ -138,7 +136,7 @@ async function createApp(actor: Record<string, unknown>) {
(req as any).actor = actor;
next();
});
app.use("/api", issueRoutes({} as any, {} as any, { feedbackExportService: mockFeedbackExportService }));
app.use("/api", issueRoutes({} as any, {} as any));
const routeErrors: string[] = [];
app.locals.routeErrors = routeErrors;
app.use((error: unknown, _req: express.Request, _res: express.Response, next: express.NextFunction) => {
@@ -152,11 +150,6 @@ async function createApp(actor: Record<string, unknown>) {
describe("issue feedback trace routes", () => {
beforeEach(() => {
vi.clearAllMocks();
mockFeedbackExportService.flushPendingFeedbackTraces.mockResolvedValue({
attempted: 1,
sent: 1,
failed: 0,
});
mockHeartbeatService.wakeup.mockResolvedValue(undefined);
mockHeartbeatService.reportRunActivity.mockResolvedValue(undefined);
mockHeartbeatService.getRun.mockResolvedValue(null);
@@ -174,7 +167,7 @@ describe("issue feedback trace routes", () => {
mockLogActivity.mockResolvedValue(undefined);
});
it("flushes a newly shared feedback trace immediately after saving the vote", async () => {
it("returns the saved vote without waiting for feedback uploads", async () => {
const targetId = "11111111-1111-4111-8111-111111111111";
mockIssueService.getById.mockResolvedValue({
id: "issue-1",
@@ -211,11 +204,9 @@ describe("issue feedback trace routes", () => {
});
expect([200, 201]).toContain(res.status);
expect(mockFeedbackExportService.flushPendingFeedbackTraces).toHaveBeenCalledWith({
companyId: "company-1",
traceId: "trace-1",
limit: 1,
});
expect(mockFeedbackService.saveIssueVote).toHaveBeenCalledTimes(1);
expect(mockFeedbackService.flushPendingFeedbackTraces).not.toHaveBeenCalled();
});
it("rejects non-board callers before fetching a feedback trace", async () => {
@@ -660,17 +660,19 @@ describeEmbeddedPostgres("issue recovery actions", () => {
// The operator retries the same remote branch under two spellings. Both map
// to the canonical `origin/fix/foo` identity, so recovery must not reset the
// attempt count or post a second notice.
// attempt count. Each failed run still needs its own actionable notice.
const firstRun = makeRun("fix/foo", "origin/fix/foo");
const secondRun = makeRun("origin/fix/foo", "origin/fix/foo");
await recovery.escalateStrandedAssignedIssue({
issue: sourceIssue,
previousStatus: "in_progress",
latestRun: makeRun("fix/foo", "origin/fix/foo"),
latestRun: firstRun,
recoveryCause: "configuration_incomplete",
});
await recovery.escalateStrandedAssignedIssue({
issue: sourceIssue,
previousStatus: "in_progress",
latestRun: makeRun("origin/fix/foo", "origin/fix/foo"),
latestRun: secondRun,
recoveryCause: "configuration_incomplete",
});
@@ -689,7 +691,12 @@ describeEmbeddedPostgres("issue recovery actions", () => {
`source_scoped_recovery:${sourceIssue.companyId}:${sourceIssue.id}:configuration_incomplete:workspace_base_ref:origin/fix/foo`,
);
// The operator gets one notice, bound to the one action.
// Each run gets one notice bound to the same action; reconciling the
// latest run again must not duplicate its notice.
await recovery.escalateStrandedAssignedIssue({
issue: sourceIssue, previousStatus: "in_progress", latestRun: secondRun,
recoveryCause: "configuration_incomplete",
});
const notices = await db
.select({ metadata: issueComments.metadata })
.from(issueComments)
@@ -703,7 +710,8 @@ describeEmbeddedPostgres("issue recovery actions", () => {
notices.filter((row) =>
noticeMetadataReferencesRecoveryAction(row.metadata, actions[0]!.id),
),
).toHaveLength(1);
).toHaveLength(2);
expect(notices.map((row) => row.metadata?.sourceRunId).sort()).toEqual([firstRun.id, secondRun.id].sort());
});
it("gives a distinct recovery identity and a new operator notice when the unresolved base ref changes", async () => {
@@ -1,3 +1,5 @@
import { subscribeDeliveryWork } from "../services/delivery-work-notifications.js";
import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js";
import { randomUUID } from "node:crypto";
import { eq, sql } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
@@ -231,11 +233,15 @@ describeEmbeddedPostgres("issueThreadInteractionService", () => {
expect(created.payload.questions.map((question) => question.id)).toEqual(["repo", "scope", "hosting"]);
expect(await interactionsSvc.create(issue, input, { userId: "local-board" })).toEqual(created);
await expect(interactionsSvc.answerQuestions(issue, created.id, { answers: [{ questionId: "repo", optionIds: [], otherText: "https://example.com/repo" }] }, { userId: "local-board" })).rejects.toThrow("requires an answer");
const notified = vi.fn();
const unsubscribe = subscribeDeliveryWork(db, DELIVERY_QUEUES.question, notified);
const answered = await interactionsSvc.answerQuestions(issue, created.id, { answers: [
{ questionId: "repo", optionIds: [], otherText: "https://example.com/repo" },
{ questionId: "scope", optionIds: ["all"] },
{ questionId: "hosting", optionIds: ["existing", "new"] },
] }, { userId: "local-board" });
unsubscribe();
expect(notified).toHaveBeenCalledTimes(1);
expect(answered.status).toBe("answered");
});
@@ -1025,6 +1031,7 @@ describeEmbeddedPostgres("issueThreadInteractionService", () => {
{ agentId: creatorAgentId },
);
await db.update(agents).set({ lifecycleState: "terminated", status: "terminated" }).where(eq(agents.id, addresseeAgentId));
await agentService(db).remove(addresseeAgentId);
const cancelled = await interactionsSvc.getById(created.id);
@@ -540,6 +540,34 @@ describeEmbeddedPostgres("issueService.list participantAgentId", () => {
await ensureIssueRelationsTable(db);
}, 20_000);
it("rejects stale monitor cancellation under the task lock and preserves saved policy settings", async () => {
const companyId = randomUUID();
const issueId = randomUUID();
await db.insert(companies).values({ id: companyId, name: "Monitor cancellation", issuePrefix: "MCL", requireBoardApprovalForNewAgents: false });
const oldPolicy = {
mode: "normal" as const, commentRequired: true, stages: [],
monitor: { nextCheckAt: "2026-10-10T12:00:00.000Z", notes: "Check status", scheduledBy: "board" as const },
};
const savedPolicy = {
...oldPolicy,
stages: [{ type: "review" as const, approvalsNeeded: 1 as const, participants: [{ type: "user" as const, userId: "reviewer" }] }],
maxReviewRounds: 4,
authorizationPolicy: { assignmentPolicy: { mode: "protected" as const } },
};
await db.insert(issues).values({ id: issueId, companyId, title: "Monitored task", executionPolicy: oldPolicy });
await svc.update(issueId, { executionPolicy: savedPolicy });
const { monitor: _oldMonitor, ...staleCancellation } = oldPolicy;
await expect(svc.update(issueId, { executionPolicy: staleCancellation, expectedExecutionPolicy: oldPolicy }))
.rejects.toMatchObject({ status: 409 });
expect((await svc.getById(issueId))?.executionPolicy).toEqual(savedPolicy);
const { monitor: _savedMonitor, ...cancellation } = savedPolicy;
await svc.update(issueId, { executionPolicy: cancellation, expectedExecutionPolicy: savedPolicy });
expect((await svc.getById(issueId))?.executionPolicy).toEqual(cancellation);
await expect(svc.update(issueId, { executionPolicy: null, expectedExecutionPolicy: null }))
.rejects.toMatchObject({ status: 409 });
expect((await svc.getById(issueId))?.executionPolicy).toEqual(cancellation);
});
afterEach(async () => {
await db.delete(issueComments);
await db.delete(issueThreadInteractions);
@@ -0,0 +1,98 @@
import { createServer } from "node:http";
import { once } from "node:events";
import { getTableName, type SQL } from "drizzle-orm";
import { PgDialect } from "drizzle-orm/pg-core";
import { activityToastIdentity } from "../realtime/activity-toast-identity.js";
import { WebSocket } from "ws";
import { describe, expect, it, vi } from "vitest";
import { setupLiveEventsWebSocketServer } from "../realtime/live-events-ws.js";
import { publishLiveEvent } from "../services/live-events.js";
vi.mock("../services/authorization.js", () => ({
authorizationService: () => ({ decide: async ({ resource }: { resource: { issueId: string } }) => ({ allowed: resource.issueId !== "hidden" }) }),
}));
vi.mock("../services/heartbeat-run-privacy.js", () => ({ canActorReadHeartbeatRun: async () => false }));
function database(onPredicate: (kind: "where" | "join", condition: SQL) => void = () => {}) {
return { select: (fields: Record<string, unknown>) => {
let table = "";
const query = {
from: (value: Parameters<typeof getTableName>[0]) => { table = getTableName(value); return query; },
innerJoin: (_table: unknown, condition: SQL) => { onPredicate("join", condition); return query; },
where: (condition: SQL) => { onPredicate("where", condition); return query; },
limit: async () => {
if (table === "issues") return [{ id: "task" }];
if (table === "company_memberships") return fields.name ? [{ name: "Alex Example", image: "/alex.png" }] : [{ id: "membership" }];
if (table === "agents") return [{ name: "Coder", appearance: { schemaVersion: 1, characterVersion: "cap-v1", paletteId: "bubblegum-sky" } }];
return [];
},
};
return query;
} };
}
describe("authenticated activity notifications", () => {
it("keeps actor identity after authorization and excludes hidden tasks and private details", async () => {
const server = createServer();
setupLiveEventsWebSocketServer(server, database() as never, { deploymentMode: "authenticated",
resolveCloudActor: async () => ({ userId: "viewer", companyIds: ["company"] }) });
server.listen(0, "127.0.0.1");
await once(server, "listening");
const socket = new WebSocket(`ws://127.0.0.1:${(server.address() as { port: number }).port}/api/companies/company/events/ws`);
try {
await once(socket, "open");
const messages: Array<{ payload: Record<string, unknown> }> = [];
socket.on("message", (data) => messages.push(JSON.parse(data.toString())));
const activity = (entityId: string, actorType = "user") => publishLiveEvent({ companyId: "company", type: "activity.logged", payload: {
action: "issue.created", entityType: "issue", entityId, actorType, actorId: actorType === "user" ? "viewer" : "agent",
details: { bodySnippet: "private comment", secret: "not for the socket" },
} });
activity("hidden");
const first = once(socket, "message");
activity("visible");
await first;
expect(messages).toHaveLength(1);
expect(messages[0].payload).toEqual({ action: "issue.created", entityType: "issue", entityId: "visible", actorType: "user", actorId: "viewer", actorName: "Alex Example", actorImage: "/alex.png" });
const second = once(socket, "message");
activity("visible", "agent");
await second;
expect(messages[1].payload).toMatchObject({ actorName: "Coder", actorType: "agent", actorId: "agent", actorAppearance: { paletteId: "bubblegum-sky" } });
expect(messages[1].payload).not.toHaveProperty("details");
} finally {
socket.terminate();
await new Promise<void>((resolve) => server.close(() => resolve()));
}
});
});
describe("activity profile query boundaries", () => {
it("requires the exact company, human actor, and active membership before reading a person profile", async () => {
const predicates: { kind: string; sql: string; params: unknown[] }[] = [];
const db = database((kind, condition) => {
const { sql, params } = new PgDialect().sqlToQuery(condition);
predicates.push({ kind, sql, params });
});
await activityToastIdentity(db as never, "company-A", {
action: "issue.created", actorType: "user", actorId: "person-A",
});
expect(predicates).toEqual([
{ kind: "join", sql: '"user"."id" = "company_memberships"."principal_id"', params: [] },
{ kind: "where", sql: '("company_memberships"."company_id" = $1 and "company_memberships"."principal_type" = $2 and "company_memberships"."principal_id" = $3 and "company_memberships"."status" = $4)',
params: ["company-A", "user", "person-A", "active"] },
]);
});
it("requires both the exact company and agent ID before reading an agent profile", async () => {
const predicates: { kind: string; sql: string; params: unknown[] }[] = [];
const db = database((kind, condition) => {
const { sql, params } = new PgDialect().sqlToQuery(condition);
predicates.push({ kind, sql, params });
});
await activityToastIdentity(db as never, "company-A", {
action: "issue.created", actorType: "agent", actorId: "agent-A",
});
expect(predicates).toEqual([
{ kind: "where", sql: '("agents"."company_id" = $1 and "agents"."id" = $2)',
params: ["company-A", "agent-A"] },
]);
});
});
@@ -208,7 +208,9 @@ describeEmbeddedPostgres("POST /api/companies/:companyId/onboarding-seed", () =>
expect(companyGoals[0]?.title).toBe("Make robotics boring");
expect(companyGoals[0]?.description).toBe("Boring enough that hospitals buy it.");
// No agent and no task were sent, so none were invented.
expect(await ctx.db.select().from(agents).where(eq(agents.companyId, companyId))).toHaveLength(0);
const committedAgents = await ctx.db.select().from(agents).where(eq(agents.companyId, companyId));
expect(committedAgents).toHaveLength(0);
expect(await ctx.db.select().from(projects).where(eq(projects.companyId, companyId))).toHaveLength(0);
expect(await ctx.db.select().from(issues).where(eq(issues.companyId, companyId))).toHaveLength(0);
expect(response.body.agentId).toBeNull();
expect(response.body.issueId).toBeNull();
@@ -245,12 +247,7 @@ describeEmbeddedPostgres("POST /api/companies/:companyId/onboarding-seed", () =>
expect(entries).toHaveLength(1);
});
it("rolls the whole seed back when the audit entry cannot be written", async () => {
// The audit entry shares the seed's transaction, so a failure to write it
// must leave nothing behind. The alternative — commit the seed and lose the
// entry — is unrecoverable: Cloud stops retrying on a 2xx, and a later
// replay reports `changed: false` and never logs, so the entry would be
// permanently absent.
it("reuses the committed hire after the content transaction fails", async () => {
const { companyId, app } = await seedCompany();
// Injected at the module boundary, not on `ctx.db`: the audit write goes
@@ -261,18 +258,20 @@ describeEmbeddedPostgres("POST /api/companies/:companyId/onboarding-seed", () =>
const failed = await post(app, companyId, SEED);
expect(failed.status).toBeGreaterThanOrEqual(500);
// Nothing committed: no seed record, so Cloud has no acknowledged revision
// and keeps retrying, and no orphaned agent from the rolled-back attempt.
// The failed content transaction must not acknowledge the revision.
expect(
await ctx.db
.select()
.from(companyOnboardingSeeds)
.where(eq(companyOnboardingSeeds.companyId, companyId)),
).toHaveLength(0);
expect(await ctx.db.select().from(agents).where(eq(agents.companyId, companyId))).toHaveLength(0);
const committedAgents = await ctx.db.select().from(agents).where(eq(agents.companyId, companyId));
expect(committedAgents).toHaveLength(1);
expect(await ctx.db.select().from(projects).where(eq(projects.companyId, companyId))).toHaveLength(0);
// And the retry recovers completely — seed applied, entry present.
await post(app, companyId, SEED).expect(200);
expect((await ctx.db.select().from(agents).where(eq(agents.companyId, companyId))).map(agent => agent.id))
.toEqual(committedAgents.map(agent => agent.id));
expect(
await ctx.db
.select()
@@ -299,7 +298,9 @@ describeEmbeddedPostgres("POST /api/companies/:companyId/onboarding-seed", () =>
const response = await post(strangerApp, companyId, SEED);
expect(response.status).toBe(403);
expect(await ctx.db.select().from(agents).where(eq(agents.companyId, companyId))).toHaveLength(0);
const committedAgents = await ctx.db.select().from(agents).where(eq(agents.companyId, companyId));
expect(committedAgents).toHaveLength(0);
expect(await ctx.db.select().from(projects).where(eq(projects.companyId, companyId))).toHaveLength(0);
expect(await ctx.db.select().from(companyOnboardingSeeds)).toHaveLength(0);
});
+95 -4
View File
@@ -21,6 +21,7 @@ const apiPrefixes: Record<string, string> = {
"agents.ts": "/api",
"agent-commentary.ts": "/api",
"agent-avatars.ts": "/api",
"agent-profile-avatar.ts": "/api",
"announcements.ts": "/api",
"ai-connections.ts": "/api",
"decision-models.ts": "/api",
@@ -84,6 +85,7 @@ const apiPrefixes: Record<string, string> = {
const ROUTE_LITERAL_PATTERN =
/router\.(get|post|put|patch|delete)\(\s*["'`]([^"'`]+)["'`]/g;
const ROUTE_ARRAY_PATTERN = /router\.(get|post|put|patch|delete)\(\s*\[([^\]]+)\]/g;
const ROUTER_METHOD_PATTERN = /router\.(get|post|put|patch|delete)\(/;
const HTTP_METHODS = new Set([
"get",
@@ -105,6 +107,8 @@ const explicitOpenApiOperationCoverageExclusions = new Set([
"POST /api/customer-success/v1/read",
// OAuth discovery and protocol endpoints have their own metadata contract;
// browser connection-management operations remain documented in the board API.
// Markdown rendering of the separately documented assistant setup page.
"GET /api/mcp/setup.md",
"GET /.well-known/oauth-authorization-server",
"POST /mcp/oauth/register",
"POST /mcp/oauth/device_authorization",
@@ -214,6 +218,14 @@ function loadActualRoutes() {
}
}
for (const match of source.matchAll(ROUTE_ARRAY_PATTERN)) {
for (const literal of match[2].matchAll(/["'`]([^"'`]+)["'`]/g)) {
const operation = `${match[1].toUpperCase()} ${normalizeExpressPath(resolveMountedPath(file, prefix, literal[1]))}`;
if (explicitOpenApiOperationCoverageExclusions.has(operation)) excludedRoutes.add(operation);
else routes.add(operation);
}
}
if (file === "public-mcp.ts") {
// The shared gateway mounts these protocol paths for each OAuth resource.
if (source.includes("router.get(metadataPath,")) {
@@ -229,10 +241,13 @@ function loadActualRoutes() {
}
}
if (file === "dot-runner.ts") {
const basePath = /const path = "([^"]+)"/.exec(source)?.[1];
if (!basePath) throw new Error("Dot binding route prefix is missing");
for (const match of source.matchAll(/router\.(get|post|delete)\(path(?: \+ "([^"]+)")?/g)) {
routes.add(`${match[1].toUpperCase()} ${normalizeExpressPath(prefix + basePath + (match[2] ?? ""))}`);
for (const name of ["path", "invitePath"]) {
const basePath = new RegExp(`const ${name} = "([^"]+)"`).exec(source)?.[1];
if (!basePath) throw new Error(`Dot ${name} route prefix is missing`);
const methods = new RegExp(`router\\.(get|post|delete)\\(${name}(?: \\+ "([^"]+)")?`, "g");
for (const match of source.matchAll(methods)) {
routes.add(`${match[1].toUpperCase()} ${normalizeExpressPath(prefix + basePath + (match[2] ?? ""))}`);
}
}
}
@@ -275,6 +290,28 @@ function loadSpecRoutes() {
}
describe("openapi routes", () => {
it("documents only writable agent update fields and lifecycle status requests", () => {
const document = buildOpenApiSpec() as any;
const properties = document.paths["/api/agents/{id}"].patch.requestBody.content["application/json"].schema.properties;
expect(properties).not.toHaveProperty("spentMonthlyCents");
expect(properties.status.enum).toEqual(["paused", "idle", "terminated"]);
expect(properties).toHaveProperty("name");
expect(properties).toHaveProperty("budgetMonthlyCents");
});
it("documents public Dot pairing capabilities while ordinary consent keeps board authentication", () => {
const spec = buildOpenApiSpec() as any;
for (const [method, path] of [
["get", "/api/dot-mcp/requests/{id}"],
["post", "/api/dot-mcp/requests/{id}/dot-pairing"],
["post", "/api/dot-mcp/requests/{id}/dot-pairing/preview"],
]) {
expect(spec.paths[path][method].security).toEqual([]);
expect(spec.paths[path][method].responses["404"]).toBeDefined();
}
expect(spec.paths["/api/mcp/requests/{id}/consent"].post.security).not.toEqual([]);
});
it("documents manager-only task privacy hints without protected scope identity", () => {
const spec = buildOpenApiSpec() as any;
const operation = spec.paths["/api/issues/{id}/privacy-constraints"].get;
@@ -858,6 +895,19 @@ describe("openapi routes", () => {
expect(path.put.responses["422"]).toBeDefined();
});
it("documents operator Dot invitations and current pairing and event-test fields", () => {
const { spec } = loadSpecRoutes();
const invitations = spec.paths["/api/companies/{companyId}/dot-invitations"];
for (const operation of [invitations.get, invitations.post]) {
expect(operation["x-paperclip-authorization"]).toEqual({ actor: "board" });
expect(operation.responses["200"]).toBeDefined();
}
expect(invitations.post.requestBody.content["application/json"].schema.additionalProperties).toBe(false);
const base = "/api/companies/{companyId}/agents/{agentId}/dot-binding";
expect(spec.paths[base].post.requestBody.content["application/json"].schema.properties.replaceBindingId).toMatchObject({ type: "string", format: "uuid" });
expect(spec.paths[base + "/event-test"].post.requestBody.content["application/json"].schema.properties.bindingId).toMatchObject({ type: "string", format: "uuid" });
});
it("documents board-only repository discovery and selection", () => {
const { spec } = loadSpecRoutes();
const discovery = spec.paths["/api/companies/{companyId}/project-repositories"].get;
@@ -870,6 +920,47 @@ describe("openapi routes", () => {
expect(replacement.responses["422"]).toBeDefined();
});
it("documents the manager-only GitHub wizard and compatible owner-aware registration", () => {
const { spec } = loadSpecRoutes();
for (const [suffix, method] of [["draft", "put"], ["setup", "post"], ["identity/start", "post"], ["identity/confirm", "post"]]) {
const operation = spec.paths[`/api/chat-endpoints/{endpointId}/github/${suffix}`][method];
expect(operation["x-paperclip-authorization"]).toEqual({ actor: "board" });
expect(operation.security).toEqual([{ BoardSessionAuth: [] }, { BoardApiKeyAuth: [] }]);
}
const registration = spec.paths["/api/chat-endpoints/{endpointId}/github/registration"].post;
const input = registration.requestBody.content["application/json"].schema;
expect(input.properties.ownerType.enum).toEqual(["personal", "organization"]);
expect(input.required).not.toContain("ownerType");
const response = registration.responses["200"].content["application/json"].schema;
expect(response.oneOf ?? response.anyOf).toHaveLength(2);
});
it("documents GitHub recovery, draft receipts, paged repositories and old review lookups", () => {
const { spec } = loadSpecRoutes();
const root = "/api/chat-endpoints/{endpointId}/github/";
const setup = spec.paths[root + "setup"].post.responses["200"].content["application/json"].schema;
expect(setup.properties.restartableRegistrationId.format).toBe("uuid");
const restart = spec.paths[root + "registration/restart"].post;
expect(restart["x-paperclip-authorization"]).toEqual({ actor: "board" });
const restartInput = restart.requestBody.content["application/json"].schema;
expect(restartInput.required).toEqual(expect.arrayContaining(["registrationId", "appNotCreated"]));
expect(restartInput.properties.appNotCreated.enum ?? [restartInput.properties.appNotCreated.const]).toEqual([true]);
const receipt = spec.paths[root + "draft"].put.responses["200"].content["application/json"].schema;
expect(receipt.required).toEqual(["saved"]);
expect(receipt.properties).not.toHaveProperty("state");
const app = spec.paths[root + "app"].post.requestBody.content["application/json"].schema;
expect(app.properties.clientId).toBeDefined();
expect(app.properties.clientSecret).toBeDefined();
expect(app.required).not.toContain("clientId");
const page = spec.paths[root + "repositories"].get;
expect(page.parameters.map((param: { name: string }) => param.name)).toEqual(expect.arrayContaining(["endpointId", "offset", "limit", "search"]));
expect(page.responses["200"].content["application/json"].schema.properties).toHaveProperty("totalCount");
expect(spec.paths[root + "repositories/access"].put["x-paperclip-authorization"]).toEqual({ actor: "board" });
const review = spec.paths[root + "reviews/{reviewId}"].get;
expect(review.parameters.map((param: { name: string }) => param.name)).toEqual(["endpointId", "reviewId"]);
expect(review.responses["404"]).toBeDefined();
});
it("documents auth and reviewed response-code invariants", () => {
const { spec } = loadSpecRoutes();
@@ -0,0 +1,98 @@
import { fileURLToPath } from "node:url";
import { createRequire } from "node:module";
import { setTimeout as delay } from "node:timers/promises";
import { afterEach, describe, expect, it } from "vitest";
import { createPluginWorkerHandle } from "../services/plugin-worker-manager.js";
import { startTaskDrain, stopTaskDrain } from "../services/task-admission.js";
const manifest = { id: "test.idle", apiVersion: 1 as const, version: "1.0.0", displayName: "Idle fixture",
description: "Idle fixture", author: "Paperclip", categories: ["automation" as const], capabilities: [], entrypoints: { worker: "worker.cjs" } };
function worker(write = false, hostHandler = async () => undefined) {
return createPluginWorkerHandle("test.idle", {
entrypointPath: fileURLToPath(new URL("./fixtures/plugin-worker-idle.cjs", import.meta.url)),
execArgv: ["--import", createRequire(import.meta.url).resolve("tsx")],
manifest, config: {}, instanceInfo: { instanceId: "fixture", hostVersion: "1.0.0" }, apiVersion: 1,
env: { IDLE_TEST_WRITE: write ? "1" : "0" },
hostHandlers: { "state.set": hostHandler, "events.subscribe": async () => ({}) },
});
}
function hold() {
const lease = startTaskDrain({ purpose: "idle", ttlMs: 30_000 });
return { ownerId: lease.ownerId!, expiresAt: lease.expiresAt!.getTime() };
}
describe("plugin manager idle receipts", () => {
afterEach(() => stopTaskDrain());
it("retains timed-out worker operations and admits work after the owned hold releases", async () => {
const handle = worker();
await handle.start();
try {
await expect(handle.call("environmentProbe", { driverKey: "fixture", companyId: "fixture", environmentId: "fixture", config: { delayMs: 200 } }, 10)).rejects.toThrow("timed out");
const lease = hold();
expect(await handle.prepareIdleSleep!(lease)).toBe("present");
await expect.poll(() => handle.prepareIdleSleep!(lease)).toBe("none");
await expect(handle.call("health", {})).rejects.toThrow("held");
expect(await handle.prepareIdleSleep!({ ...lease, ownerId: "stale" })).toBe("unknown");
stopTaskDrain(); handle.releaseIdleSleep!();
expect(await handle.call("health", {})).toMatchObject({ status: "ok" });
expect(await handle.prepareIdleSleep!(hold())).toBe("none");
stopTaskDrain();
// No intervening request or explicit worker release: the next controller
// attempt must discard the worker manager's previous ownership record.
expect(await handle.prepareIdleSleep!(hold())).toBe("none");
} finally { stopTaskDrain(); await handle.stop(); }
});
it("keeps a host write counted after the caller stops waiting", async () => {
let finish!: () => void;
const handle = worker(true, () => new Promise<void>((resolve) => { finish = resolve; }));
await handle.start();
try {
await expect(handle.call("health", {}, 30)).rejects.toThrow("timed out");
const lease = hold();
expect(await handle.prepareIdleSleep!(lease)).toBe("present");
finish();
await expect.poll(() => handle.prepareIdleSleep!(lease)).toBe("none");
} finally { stopTaskDrain(); await handle.stop(); }
});
it("lets a queued notification finish its write while idle preparation is in flight", async () => {
let finish!: () => void;
let writes = 0;
const handle = worker(false, () => new Promise<void>((resolve) => {
writes++;
finish = resolve;
}));
await handle.start();
try {
// Finish setup and its subscription receipt before queuing the event.
await handle.call("health", {});
handle.notify("onEvent", { event: { eventType: "fixture.write" } });
// No await: the host has set its hold before it can read the event's
// worker-to-host write. The worker must still be allowed to complete it.
const lease = hold();
const preparation = handle.prepareIdleSleep!(lease);
expect(await preparation).not.toBe("none");
await expect.poll(() => writes).toBe(1);
expect(await handle.prepareIdleSleep!(lease)).toBe("present");
finish();
await expect.poll(() => handle.prepareIdleSleep!(lease)).toBe("none");
} finally { finish?.(); stopTaskDrain(); await handle.stop(); }
});
it("resumes crash recovery after an idle hold without discarding the restart", async () => {
const handle = worker();
await handle.start();
handle.on("crash", () => { hold(); });
try {
await expect(handle.call("environmentProbe", { driverKey: "fixture", companyId: "fixture", environmentId: "fixture", config: { crash: true } })).rejects.toThrow();
await delay(1_600);
expect(handle.status).toBe("backoff");
expect(handle.diagnostics().nextRestartAt).not.toBeNull();
stopTaskDrain();
await expect.poll(() => handle.status, { timeout: 5_000 }).toBe("running");
// Recovery restores service, but lost completion receipts remain unsafe.
expect(await handle.prepareIdleSleep!(hold())).toBe("unknown");
} finally { stopTaskDrain(); await handle.stop(); }
});
});
@@ -0,0 +1,109 @@
import { PassThrough } from "node:stream";
import { describe, expect, it } from "vitest";
import { definePlugin, type PluginDefinition } from "../../../packages/plugins/sdk/src/define-plugin.js";
import { startWorkerRpcHost } from "../../../packages/plugins/sdk/src/worker-rpc-host.js";
function fixture(definition: PluginDefinition, rpcTimeoutMs = 30_000) {
const stdin = new PassThrough();
const stdout = new PassThrough();
const worker = startWorkerRpcHost({ plugin: definePlugin(definition), stdin, stdout, rpcTimeoutMs });
const messages: any[] = [];
stdout.on("data", (chunk) => { for (const line of String(chunk).trim().split("\n")) messages.push(JSON.parse(line)); });
let id = 0;
async function call(method: string, params: unknown = {}) {
const callId = ++id;
stdin.write(JSON.stringify({ jsonrpc: "2.0", id: callId, method, params }) + "\n");
await expect.poll(() => messages.find((message) => message.id === callId && !message.method)).toBeDefined();
return messages.find((message) => message.id === callId && !message.method);
}
const initialize = () => call("initialize", { manifest: { id: "test.idle", apiVersion: 1 }, config: {}, apiVersion: 1 });
return { call, initialize, messages, stdin, close: () => worker.stop() };
}
describe("plugin idle RPC", () => {
it.each(["preparing", "verifying", "pausing", "resuming", "terminating", "cleaning_up"] as const)("advertises and dispatches the %s lifecycle phase", async phase => {
const request = { companyId: "company", agentId: "agent", operationId: "operation", version: 7, phase };
const f = fixture({ async setup() {}, async onAgentLifecycle(input) {
expect(input).toEqual(request);
return { operationId: input.operationId, version: input.version, status: "pending" };
} });
try {
expect((await f.initialize()).result.supportedMethods).toContain("agentLifecycle");
expect((await f.call("agentLifecycle", request)).result).toEqual({ operationId: "operation", version: 7, status: "pending" });
} finally { f.close(); }
});
it("does not acknowledge lifecycle work without a handler", async () => {
const f = fixture({ async setup() {} });
try {
expect((await f.initialize()).result.supportedMethods).not.toContain("agentLifecycle");
expect((await f.call("agentLifecycle", { companyId: "company", agentId: "agent", operationId: "operation", version: 1, phase: "preparing" })).error).toBeDefined();
} finally { f.close(); }
});
const hold = () => ({ ownerId: "idle-owner", expiresAt: Date.now() + 30_000 });
it("seals an idle worker, fences stale releases, and restores normal calls", async () => {
const f = fixture({ async setup() {}, async onIdleDrain() { return "none"; } });
try {
expect((await f.initialize()).result.supportedMethods).toContain("prepareIdleSleep");
const lease = hold();
expect((await f.call("prepareIdleSleep", lease)).result).toEqual({ ...lease, backgroundWork: "none" });
expect((await f.call("health")).error).toBeDefined();
await f.call("releaseIdleSleep", { ownerId: "stale" });
expect((await f.call("health")).error).toBeDefined();
await f.call("releaseIdleSleep", lease);
expect((await f.call("health")).result.status).toBe("ok");
} finally { f.close(); }
});
it("does not mistake a caller timeout for a completed worker handler", async () => {
let finish!: () => void;
const f = fixture({ async setup() {}, async onIdleDrain() { return "none"; },
async onHealth() { await new Promise<void>((resolve) => { finish = resolve; }); return { status: "ok" }; } });
try {
await f.initialize();
const accepted = f.call("health");
await expect.poll(() => finish).toBeDefined();
expect((await f.call("prepareIdleSleep", hold())).result.backgroundWork).toBe("present");
finish(); await accepted;
expect((await f.call("prepareIdleSleep", hold())).result.backgroundWork).toBe("none");
} finally { f.close(); }
});
it("retains timed-out worker-to-host writes until the host's late receipt", async () => {
let write!: () => Promise<unknown>;
const f = fixture({ async setup(ctx) { write = () => ctx.state.set({ scopeKind: "instance", stateKey: "fixture" }, { value: 1 }); },
async onIdleDrain() { return "none"; } }, 10);
try {
await f.initialize();
await expect(write()).rejects.toThrow("timed out");
const request = f.messages.find((message) => message.method === "state.set");
expect((await f.call("prepareIdleSleep", hold())).result.backgroundWork).toBe("present");
f.stdin.write(JSON.stringify({ jsonrpc: "2.0", id: request.id, result: null }) + "\n");
expect((await f.call("prepareIdleSleep", hold())).result.backgroundWork).toBe("none");
} finally { f.close(); }
});
it.each([false, true])("logs session callback failures (async: %s)", async (asynchronous) => {
let subscribe!: () => Promise<unknown>;
const f = fixture({ async setup(ctx) {
subscribe = () => ctx.agents.sessions.sendMessage("fixture-session", "fixture-company", {
prompt: "fixture",
onEvent: asynchronous
? async () => { throw new Error("fixture callback failure"); }
: () => { throw new Error("fixture callback failure"); },
});
} });
try {
await f.initialize();
const subscription = subscribe();
const request = f.messages.find((message) => message.method === "agents.sessions.sendMessage");
f.stdin.write(JSON.stringify({ jsonrpc: "2.0", id: request.id, result: {} }) + "\n");
await subscription;
f.stdin.write(JSON.stringify({ jsonrpc: "2.0", method: "agents.sessions.event", params: { sessionId: "fixture-session" } }) + "\n");
await expect.poll(() => f.messages.find((message) => message.method === "log" &&
message.params.level === "error" && message.params.message.includes("fixture callback failure"))).toBeDefined();
} finally { f.close(); }
});
});
@@ -1,3 +1,4 @@
import { createAgentLifecycle, configureAgentLifecycle } from "../services/agent-lifecycle.js";
import { randomUUID } from "node:crypto";
import { promises as fs } from "node:fs";
import os from "node:os";
@@ -256,11 +257,15 @@ describeEmbeddedPostgres("plugin-managed agents", () => {
it("keeps an explicit resume durable across managed-agent reconcile", async () => {
const { companyId, services } = await seedCompanyAndPlugin({ manifest: pausedManifest() });
const created = await services.agents.managedReconcile({ companyId, agentKey: "wiki-maintainer" });
await agentService(db).resume(created.agentId!);
await createAgentLifecycle(db).resumeAgent(created.agentId!);
const reconciled = await services.agents.managedReconcile({ companyId, agentKey: "wiki-maintainer" });
expect(reconciled.agent).toMatchObject({
expect(reconciled.agent).toMatchObject({ lifecycleState: "resuming", pauseReason: null });
const worker = configureAgentLifecycle(db, { requiredPluginIds: async () => [], runPlugin: async () => "complete", runHost: async () => "complete" });
await worker.process(created.agentId!);
await worker.stop();
expect((await services.agents.managedReconcile({ companyId, agentKey: "wiki-maintainer" })).agent).toMatchObject({
status: "idle",
pauseReason: null,
pausedAt: null,
Loaded 100 of 1221 files, more files were not shown because too many files have changed in this diff. Show more