diff --git a/doc/connections/AI-CONNECTIONS.md b/doc/connections/AI-CONNECTIONS.md index 4a0cb77adf3..bb90256852a 100644 --- a/doc/connections/AI-CONNECTIONS.md +++ b/doc/connections/AI-CONNECTIONS.md @@ -251,3 +251,35 @@ revoking credentials or submitting work. Delete the disposable instance and revo its provider key after the test; failed tests may leave a paused task for inspection. Authenticated public deployments must configure a trusted runtime host (`PAPERCLIP_TRUSTED_MCP_RUNTIME_HOST` or `PAPERCLIP_TOOL_RUNTIME_TRUSTED_HOST`) before offering server-host subscription login, matching the local stdio runtime boundary. Health reports this capability so setup can offer a supported environment or API key instead of an unusable terminal command. Private authenticated self-hosted instances support isolated local login without that extra setting. Isolated Claude credential files must be private, owned by the server user, bounded, and free of symlinks. + +### Hiring and delegated work + +When a managed agent creates or hires another agent without an explicit AI binding +or adapter auth setting, the server inherits its compatible managed connection choice. +Explicit credentials, blank overrides, credential directories, and provider routing +settings for the child provider take precedence. Unrelated provider keys do not +suppress the default. Unmanaged parents keep their existing authentication path. The new agent resolves +the responsible user's account at execution time; it never copies the parent's +credentials or identity. Same-provider hires preserve subscription/API-key choice. +A different provider selects the responsible user's default for that provider. +Native Codex and ACPX/Claude provider selections follow the same compatibility rules. + +Hiring may succeed before that personal account exists or while it needs repair, +including hires awaiting board approval. The first assigned task then shows an AI +connection card. First-time setup presents the provider's subscription/API controls +inside the task. Connecting installs access for that agent and resumes the pending +work automatically. Explicit incompatible bindings and shared-account permission +denials still fail; hiring never expands a restricted shared account's audience. + +Concurrent runs using the same subscription wait through scheduled retries while +the credential lease is held. They do not request new credentials or consume the +provider-failure retry allowance. Each retry revalidates the account and existing +run-dispatch rules still suppress cancelled, reassigned, or otherwise ineligible work. +Task retries must retain execution-lock ownership at scheduling, promotion, and dispatch. + +`server/src/__tests__/agent-hire-ai-connections.test.ts` covers both creation routes, +both providers and methods, approval gates, native provider mapping, shared access +boundaries, and subscription contention. The opt-in +[`tests/hiring-ai-connections/README.md`](../../tests/hiring-ai-connections/README.md) +describes real browser hiring, subtask, connection, and automatic-resume checks on +local and Daytona environments, plus the production component Storybook checks. diff --git a/doc/run-log-events.md b/doc/run-log-events.md index c65aab2775f..b2b999a6f24 100644 --- a/doc/run-log-events.md +++ b/doc/run-log-events.md @@ -168,3 +168,13 @@ and records cumulative usage counters. It does not include provider credentials or message content. The event establishes the accounting baseline; it is not a new billable usage receipt or a user-facing provider warning. Other provider identity checks remain in force. + +## AI subscription contention + +A run that waits for a managed subscription lease writes an informational +`lifecycle` event to the local run log. Its payload contains only +`retryScheduled`, a boolean that reports whether the scheduler created a retry. +The message distinguishes an automatic retry from work that is no longer eligible. +This pre-provider wait records `ai_connection_busy` on the cancelled run and does +not consume the provider-failure retry allowance. The event contains no credentials +and creates no Telemetry or OpenTelemetry export. diff --git a/doc/runner-api-tools.md b/doc/runner-api-tools.md index 121b03fbbe9..55b19339275 100644 --- a/doc/runner-api-tools.md +++ b/doc/runner-api-tools.md @@ -53,6 +53,10 @@ can be supplied. Routes still validate payloads and enforce permissions. Requests have a 30-second HTTP timeout, 16 KiB URL limit and 10 MiB payload/response transfer limit. Responses above 24 KiB and binary responses become company-owned assets with retrievable references; text previews are limited to 2,000 bytes. +Tool responses identify the HTTP route with `apiOperationId`. The native protocol +reserves `operationId` and `callId` for semantic tool-call identity; API metadata +must not masquerade as that envelope. Saved mutation receipts are normalized at +the tool boundary as well, without repeating their HTTP request. All redirects are refused. Oversized or interrupted mutation responses have an unknown outcome, requiring inspection before another mutation. Mutation responses with HTTP 5xx, HTTP 408, redirects, or malformed JSON also diff --git a/packages/plugins/sandbox-providers/daytona/src/file-sync.test.ts b/packages/plugins/sandbox-providers/daytona/src/file-sync.test.ts index 7c883c6c303..b193b587c23 100644 --- a/packages/plugins/sandbox-providers/daytona/src/file-sync.test.ts +++ b/packages/plugins/sandbox-providers/daytona/src/file-sync.test.ts @@ -265,6 +265,7 @@ function createRecordingSandbox(input: { */ function createRealExecSandbox(input?: { uploadOverride?: (uploads: Array<{ source: string; destination: string }>) => Promise; + commandEnv?: NodeJS.ProcessEnv; }) { const commands: RecordedCommand[] = []; return { @@ -273,7 +274,7 @@ function createRealExecSandbox(input?: { process: { executeCommand: async (command: string) => { commands.push({ command }); - const result = spawnSync("/bin/sh", ["-c", command], { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 }); + const result = spawnSync("/bin/sh", ["-c", command], { encoding: "utf8", maxBuffer: 64 * 1024 * 1024, env: input?.commandEnv }); return { exitCode: result.status ?? 1, result: (result.stdout ?? "") + (result.stderr ?? "") }; }, }, @@ -290,6 +291,53 @@ function createRealExecSandbox(input?: { }; } +// Daytona uses GNU tar. macOS contributors can install gnu-tar; the usual +// Linux CI path runs this directly without extra dependencies. +const gnuTar = ["gtar", "tar"].map((candidate) => { + const resolved = spawnSync("/bin/sh", ["-c", 'command -v "$1"', "sh", candidate], { encoding: "utf8" }).stdout.trim(); + return resolved && spawnSync(resolved, ["--version"], { encoding: "utf8" }).stdout?.includes("GNU tar") ? resolved : null; +}).find(Boolean); + +it.skipIf(!gnuTar)("extracts interleaved read-only skill directories with GNU tar and preserves their modes", async () => { + const root = await fs.mkdtemp("/tmp/paperclip-daytona-readonly-"); + const source = path.join(root, "source"); + const remoteDir = path.join(root, "remote"); + const target = path.join(remoteDir, "skill"); + const bin = path.join(root, "bin"); + await fs.mkdir(path.join(source, "references", "agents"), { recursive: true }); + await fs.mkdir(remoteDir); + await fs.mkdir(bin); + await fs.symlink(gnuTar!, path.join(bin, "tar")); + await fs.writeFile(path.join(source, "references", "overview.md"), "overview", { mode: 0o444 }); + await fs.writeFile(path.join(source, "references", "agents", "qa.md"), "QA instructions", { mode: 0o444 }); + for (const dir of ["references/agents", "references"]) await fs.chmod(path.join(source, dir), 0o555); + try { + const { sandbox } = createRealExecSandbox({ + commandEnv: { ...process.env, PATH: `${bin}:${process.env.PATH}` }, + uploadOverride: async (uploads) => { + for (const upload of uploads) { + // BSD tar can list a child directory before its parent's files, then + // visit that child later. GNU tar must not finalize its 0555 mode early. + const archive = spawnSync("tar", ["-c", "--no-xattrs", "--no-recursion", "-f", upload.destination, "-C", source, + "references", "references/agents", "references/overview.md", "references/agents/qa.md"], { encoding: "utf8", env: { ...process.env, COPYFILE_DISABLE: "1" } }); + expect(archive.status, archive.stderr).toBe(0); + } + return true; + }, + }); + await performSyncIn({ sandbox: sandbox as never, remoteDir, timeoutSeconds: 30, + operations: [{ operationId: "readonly-skill", files: [{ sourcePath: source, targetPath: target, kind: "directory", mode: 0o555 }] }] }); + expect(await fs.readFile(path.join(target, "references", "agents", "qa.md"), "utf8")).toBe("QA instructions"); + expect((await fs.stat(path.join(target, "references", "agents"))).mode & 0o777).toBe(0o555); + expect((await fs.stat(path.join(target, "references", "agents", "qa.md"))).mode & 0o777).toBe(0o444); + } finally { + for (const base of [source, target]) { + for (const dir of ["references/agents", "references"]) await fs.chmod(path.join(base, dir), 0o700).catch(() => undefined); + } + await fs.rm(root, { recursive: true, force: true }); + } +}, 30_000); + describe("daytona file-sync inbound zstd transport compression", () => { const cleanupDirs: string[] = []; diff --git a/packages/plugins/sandbox-providers/daytona/src/file-sync.ts b/packages/plugins/sandbox-providers/daytona/src/file-sync.ts index e9296deed5b..984ee6d682f 100644 --- a/packages/plugins/sandbox-providers/daytona/src/file-sync.ts +++ b/packages/plugins/sandbox-providers/daytona/src/file-sync.ts @@ -902,7 +902,10 @@ async function syncInDirectoryMapping(input: { // Extract the uploaded tarball onto the already-created target directory, // then remove the scratch tarball. const extractScript = [ - `tar -xf ${shellQuote(remoteTar)} -C ${shellQuote(mapping.targetPath)} || { echo "extract failed"; exit 43; };`, + // BSD archives may revisit a directory after its parent's files. Keep + // GNU tar from restoring a read-only skill directory's mode before all + // of its children are extracted; final permissions remain unchanged. + `tar -xf ${shellQuote(remoteTar)} --delay-directory-restore -C ${shellQuote(mapping.targetPath)} || { echo "extract failed"; exit 43; };`, `rm -f ${shellQuote(remoteTar)};`, ].join("\n"); // `extractTarball` span: one round trip — re-check the path, `tar -xf`, and diff --git a/server/src/__tests__/agent-hire-ai-connections.test.ts b/server/src/__tests__/agent-hire-ai-connections.test.ts new file mode 100644 index 00000000000..d2586f74d6f --- /dev/null +++ b/server/src/__tests__/agent-hire-ai-connections.test.ts @@ -0,0 +1,242 @@ +import { randomUUID } from "node:crypto"; +import { mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import express from "express"; +import request from "supertest"; +import { eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { agents, companies, companyMemberships, createDb, heartbeatRuns, issues, issueThreadInteractions, principalPermissionGrants, toolConnectionInstalls } from "@paperclipai/db"; +import { type AiConnectionBinding } from "@paperclipai/shared"; +import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { agentRoutes } from "../routes/agents.js"; +import { errorHandler } from "../middleware/index.js"; +import { aiConnectionService } from "../services/ai-connections.js"; +import { heartbeatService } from "../services/heartbeat.js"; +import { getServerAdapter, registerServerAdapter, unregisterServerAdapter } from "../adapters/index.js"; +import { prepareManagedAiRuntime } from "../services/ai-connection-runtime.js"; +import { secretService } from "../services/secrets.js"; + +let database: Awaited>; +let db: ReturnType; +let home: string; + +beforeAll(async () => { + home = await mkdtemp(path.join(os.tmpdir(), "paperclip-hire-ai-")); + vi.stubEnv("PAPERCLIP_HOME", home); + vi.stubEnv("PAPERCLIP_INSTANCE_ID", "hire-ai"); + database = await startEmbeddedPostgresTestDatabase("paperclip-hire-ai-db-"); + db = createDb(database.connectionString); +}, 90_000); + +afterAll(async () => { + await database?.cleanup(); + vi.unstubAllEnvs(); + if (home) await rm(home, { recursive: true, force: true }); +}); + +async function fixture(provider: "anthropic" | "openai", method: "api_key" | "subscription" = "api_key") { + const companyId = randomUUID(); + const agentId = randomUUID(); + const userId = `owner-${companyId}`; + const binding = { provider, method, mode: "responsible_user" } as const; + const adapterType = provider === "anthropic" ? "claude_local" : "codex_local"; + await db.insert(companies).values({ id: companyId, name: "Hiring connection test", issuePrefix: `H${companyId.slice(0, 7)}`, defaultResponsibleUserId: userId }); + await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: userId, membershipRole: "owner", status: "active" }); + await db.insert(principalPermissionGrants).values({ companyId, principalType: "user", principalId: userId, permissionKey: "agents:create" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Manager", role: "ceo", adapterType, runtimeConfig: { aiConnection: binding } }); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "running", responsibleUserId: userId }).returning(); + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + req.actor = { type: "agent", agentId, companyId, runId: run!.id, source: "agent_jwt", onBehalfOfUserId: userId, onBehalfOfMemberships: [{ companyId, membershipRole: "owner", status: "active" }] }; + next(); + }); + app.use("/api", agentRoutes(db)); + app.use(errorHandler); + const credential = method === "api_key" ? "fixture-api-key" : provider === "anthropic" ? "fixture-subscription-token" : JSON.stringify({ tokens: { access_token: "fixture-access", refresh_token: "fixture-refresh", id_token: "fixture-id", account_id: "fixture-account" } }); + const account = await aiConnectionService(db).save(companyId, userId, { + provider, method, name: "Manager's connection", ownership: "personal", agentIds: [agentId], allAgents: false, + ...(method === "api_key" ? { apiKey: credential } : { loginSessionId: "fixture" }), + }, credential); + return { app, companyId, agentId, userId, binding, adapterType, account, runId: run!.id }; +} + +function hired(response: request.Response) { + expect(response.status, JSON.stringify(response.body)).toBe(201); + return response.body.agent ?? response.body; +} + +describe("agent-created hires use managed AI connections", () => { + for (const endpoint of ["agent-hires", "agents"]) { + it.each([ + ["anthropic", "api_key"], ["anthropic", "subscription"], + ["openai", "api_key"], ["openai", "subscription"], + ] as const)(`${endpoint}: %s hires its own provider using the same %s binding`, async (provider, method) => { + const f = await fixture(provider, method); + const agent = hired(await request(f.app).post(`/api/companies/${f.companyId}/${endpoint}`).send({ name: "Teammate", role: "engineer", adapterType: f.adapterType, reportsTo: f.agentId })); + expect(agent.runtimeConfig.aiConnection).toEqual(f.binding); + const runtime = await prepareManagedAiRuntime(db, { companyId: f.companyId, agentId: agent.id, responsibleUserId: f.userId, adapterType: agent.adapterType, binding: agent.runtimeConfig.aiConnection, config: agent.adapterConfig }); + try { + expect(runtime.attribution).toMatchObject({ connectionId: f.account.connectionId, grantId: f.account.grantId, method, responsibleUserId: f.userId }); + } finally { await runtime.cleanup(); } + }); + } + + for (const endpoint of ["agent-hires", "agents"]) { + it.each([ + ["ANTHROPIC_API_KEY", "child-key"], + ["ANTHROPIC_API_KEY", ""], + ["CLAUDE_CONFIG_DIR", "/tmp/child-claude-home"], + ["ANTHROPIC_BASE_URL", "https://example.invalid"], + ])(`${endpoint}: preserves an explicit child auth setting %s=%s`, async (key, value) => { + const f = await fixture("anthropic"); + const agent = hired(await request(f.app).post(`/api/companies/${f.companyId}/${endpoint}`).send({ + name: "Explicit auth", role: "engineer", adapterType: f.adapterType, + adapterConfig: { env: { [key]: value } }, + })); + expect(agent.runtimeConfig.aiConnection).toBeUndefined(); + const [saved] = await db.select().from(agents).where(eq(agents.id, agent.id)); + expect((saved.adapterConfig.env as Record)[key]).toEqual({ type: "plain", value }); + }); + } + + it.each([true, false])("managed bindings do not inherit legacy credentials (managed parent: %s)", async (managedParent) => { + const f = await fixture("openai"); + const secret = await secretService(db).create(f.companyId, { + name: "Legacy parent key", provider: "local_encrypted", value: "fixture-legacy-key", + }); + await db.update(agents).set({ + runtimeConfig: managedParent ? { aiConnection: f.binding } : {}, + adapterConfig: { env: { OPENAI_API_KEY: { type: "secret_ref", secretId: secret.id } } }, + }).where(eq(agents.id, f.agentId)); + const agent = hired(await request(f.app).post(`/api/companies/${f.companyId}/agent-hires`).send({ + name: "Managed child", role: "engineer", adapterType: f.adapterType, + ...(managedParent ? {} : { runtimeConfig: { aiConnection: f.binding } }), + })); + expect(agent.runtimeConfig.aiConnection).toEqual(f.binding); + expect(agent.adapterConfig.env?.OPENAI_API_KEY).toBeUndefined(); + }); + + it("keeps unmanaged parent hires on their existing authentication path", async () => { + const f = await fixture("openai"); + await db.update(agents).set({ runtimeConfig: {} }).where(eq(agents.id, f.agentId)); + const agent = hired(await request(f.app).post(`/api/companies/${f.companyId}/agent-hires`).send({ + name: "Legacy authentication", role: "engineer", adapterType: f.adapterType, + })); + expect(agent.runtimeConfig.aiConnection).toBeUndefined(); + }); + + it.each(["anthropic", "openai"] as const)("%s can hire the other provider before that user connects it", async (provider) => { + const f = await fixture(provider); + const otherProvider = provider === "anthropic" ? "openai" : "anthropic"; + const adapterType = otherProvider === "anthropic" ? "claude_local" : "codex_local"; + const agent = hired(await request(f.app).post(`/api/companies/${f.companyId}/agent-hires`).send({ name: "Other provider", role: "engineer", adapterType })); + expect(agent.runtimeConfig.aiConnection).toMatchObject({ provider: otherProvider, mode: "responsible_user" }); + await expect(prepareManagedAiRuntime(db, { companyId: f.companyId, agentId: agent.id, responsibleUserId: f.userId, adapterType, binding: agent.runtimeConfig.aiConnection, config: agent.adapterConfig })).rejects.toMatchObject({ details: { code: "ai_connection_default_missing" } }); + expect(agent.status).toBe("idle"); + }); + + for (const endpoint of ["agent-hires", "agents"]) { + it.each([ + ["anthropic", "codex_local", {}, "ANTHROPIC_API_KEY"], + ["openai", "claude_local", {}, "OPENAI_API_KEY"], + ["anthropic", "paperclip_runner", { provider: "codex" }, "ANTHROPIC_API_KEY"], + ["openai", "paperclip_runner", { provider: "acpx", acpxAgent: "claude" }, "OPENAI_API_KEY"], + ] as const)(`${endpoint}: ignores the %s auth key for a different provider in %s`, async (provider, adapterType, config, key) => { + const f = await fixture(provider); + const agent = hired(await request(f.app).post(`/api/companies/${f.companyId}/${endpoint}`).send({ + name: "Cross-provider config", role: "engineer", adapterType, + adapterConfig: { ...config, env: { [key]: "leftover-parent-setting" } }, + })); + expect(agent.runtimeConfig.aiConnection).toMatchObject({ + provider: provider === "anthropic" ? "openai" : "anthropic", mode: "responsible_user", + }); + }); + } + + it("accepts an explicit personal default before authentication, including approval-gated hires", async () => { + const f = await fixture("anthropic"); + await db.update(companies).set({ requireBoardApprovalForNewAgents: true }).where(eq(companies.id, f.companyId)); + const binding: AiConnectionBinding = { provider: "openai", method: "subscription", mode: "responsible_user" }; + const response = await request(f.app).post(`/api/companies/${f.companyId}/agent-hires`).send({ name: "Future Codex", role: "engineer", adapterType: "codex_local", runtimeConfig: { aiConnection: binding } }); + const agent = hired(response); + expect(agent.runtimeConfig.aiConnection).toEqual(binding); + expect(agent.status).toBe("pending_approval"); + expect(response.body.approval.payload.runtimeConfig.aiConnection).toEqual(binding); + expect(await db.select().from(toolConnectionInstalls).where(eq(toolConnectionInstalls.targetId, agent.id))).toEqual([]); + }); + + it.each(["anthropic", "openai"] as const)("inherits %s when the hire uses the native runner", async (provider) => { + const f = await fixture(provider, "subscription"); + const agent = hired(await request(f.app).post(`/api/companies/${f.companyId}/agent-hires`).send({ name: "Native teammate", role: "engineer", adapterType: "paperclip_runner", adapterConfig: provider === "anthropic" ? { provider: "acpx", acpxAgent: "claude" } : { provider: "codex" } })); + expect(agent.runtimeConfig.aiConnection).toEqual(f.binding); + const runtime = await prepareManagedAiRuntime(db, { companyId: f.companyId, agentId: agent.id, responsibleUserId: f.userId, adapterType: agent.adapterType, binding: agent.runtimeConfig.aiConnection, config: agent.adapterConfig }); + try { expect(runtime.attribution.connectionId).toBe(f.account.connectionId); } finally { await runtime.cleanup(); } + }); + + it.each([true, false])("preserves shared connection access boundaries (company access: %s)", async (allAgents) => { + const f = await fixture("anthropic"); + const account = await aiConnectionService(db).save(f.companyId, f.userId, { provider: "anthropic", method: "api_key", name: "Shared Claude", ownership: "shared", apiKey: "fixture", agentIds: [f.agentId], allAgents }, "fixture"); + const binding = { provider: "anthropic", method: "api_key", mode: "shared", ...account }; + await db.update(agents).set({ runtimeConfig: { aiConnection: binding } }).where(eq(agents.id, f.agentId)); + const response = await request(f.app).post(`/api/companies/${f.companyId}/agent-hires`).send({ name: "Shared teammate", role: "engineer", adapterType: f.adapterType }); + if (!allAgents) { + expect(response.status).toBe(403); + expect(await db.select().from(agents).where(eq(agents.companyId, f.companyId))).toHaveLength(1); + } else { + const agent = hired(response); + expect(agent.runtimeConfig.aiConnection).toEqual(binding); + const runtime = await prepareManagedAiRuntime(db, { companyId: f.companyId, agentId: agent.id, responsibleUserId: f.userId, adapterType: agent.adapterType, binding: agent.runtimeConfig.aiConnection, config: agent.adapterConfig }); + try { expect(runtime.attribution.connectionId).toBe(account.connectionId); } finally { await runtime.cleanup(); } + } + }); + + it("still rejects an explicitly incompatible provider without creating a hire", async () => { + const f = await fixture("anthropic"); + const response = await request(f.app).post(`/api/companies/${f.companyId}/agent-hires`).send({ name: "Wrong provider", role: "engineer", adapterType: "codex_local", runtimeConfig: { aiConnection: f.binding } }); + expect(response.status).toBe(422); + expect(response.body.details.code).toBe("ai_connection_incompatible"); + expect(await db.select().from(agents).where(eq(agents.companyId, f.companyId))).toHaveLength(1); + }); +}); + +describe("hired agents sharing a subscription", () => { + it.each(["anthropic", "openai"] as const)("waits for the %s parent and resumes without asking for a new connection", async (provider) => { + const f = await fixture(provider, "subscription"); + const agent = hired(await request(f.app).post(`/api/companies/${f.companyId}/agent-hires`).send({ name: "Waiting teammate", role: "engineer", adapterType: f.adapterType, reportsTo: f.agentId, adapterConfig: { cwd: home, engine: "cli" }, runtimeConfig: { heartbeat: { enabled: false } } })); + const [issue] = await db.insert(issues).values({ companyId: f.companyId, title: "Subscription child task", status: "todo", assigneeAgentId: agent.id, responsibleUserId: f.userId, createdByUserId: f.userId }).returning(); + const parentRuntime = await prepareManagedAiRuntime(db, { companyId: f.companyId, agentId: f.agentId, responsibleUserId: f.userId, adapterType: f.adapterType, binding: f.binding, config: { cwd: home } }); + const execute = vi.fn(async () => { + await db.update(issues).set({ status: "done", completedAt: new Date() }).where(eq(issues.id, issue.id)); + return { exitCode: 0, signal: null, timedOut: false, resultJson: {} }; + }); + registerServerAdapter({ ...getServerAdapter(f.adapterType), execute }); + const heartbeat = heartbeatService(db); + let parentReleased = false; + try { + const run = await heartbeat.invoke(agent.id, "assignment", { issueId: issue.id, wakeReason: "issue_assigned", responsibleUserId: f.userId }, "system"); + expect(run).not.toBeNull(); + await expect.poll(async () => (await heartbeat.getRun(run!.id))?.status, { timeout: 20_000 }).not.toBe("running"); + expect(await heartbeat.getRun(run!.id)).toMatchObject({ status: "cancelled", errorCode: "ai_connection_busy" }); + let retry: typeof heartbeatRuns.$inferSelect | undefined; + await expect.poll(async () => { [retry] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, run!.id)); return retry?.status; }, { timeout: 10_000 }).toBe("scheduled_retry"); + expect(execute).not.toHaveBeenCalled(); + expect(await db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.issueId, issue.id))).toHaveLength(0); + expect((await db.select().from(issues).where(eq(issues.id, issue.id)))[0].executionRunId).toBe(retry!.id); + parentReleased = true; + await parentRuntime.cleanup(); + const promoted = await heartbeat.promoteDueScheduledRetries(new Date(retry!.scheduledRetryAt!.getTime() + 1000)); + expect(promoted.runIds).toContain(retry!.id); + await heartbeat.resumeQueuedRuns(); + await expect.poll(async () => (await heartbeat.getRun(retry!.id))?.status, { timeout: 20_000 }).toBe("succeeded"); + expect(execute).toHaveBeenCalledTimes(1); + expect((await heartbeat.getRun(retry!.id))?.contextSnapshot?.aiConnection).toMatchObject({ connectionId: f.account.connectionId, responsibleUserId: f.userId, method: "subscription" }); + } finally { + if (!parentReleased) await parentRuntime.cleanup(); + await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.runId)); + await heartbeat.drainActiveRunExecutions(); + unregisterServerAdapter(f.adapterType); + } + }); +}); diff --git a/server/src/__tests__/heartbeat-retry-scheduling.test.ts b/server/src/__tests__/heartbeat-retry-scheduling.test.ts index 490e01e1508..6484e678b0b 100644 --- a/server/src/__tests__/heartbeat-retry-scheduling.test.ts +++ b/server/src/__tests__/heartbeat-retry-scheduling.test.ts @@ -1374,6 +1374,55 @@ describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => { expect(issue?.executionRunId).toBe(retryRuns[0]?.id); }); + it.each(["schedule", "transaction", "promote", "dispatch"] as const)( + "suppresses a busy-subscription retry whose task lock was cleared before %s", + async (phase) => { + const { companyId, issueId, runId, now } = await seedMaxTurnFixture(); + await db.update(heartbeatRuns).set({ + status: "cancelled", errorCode: "ai_connection_busy", + resultJson: { executionRecovery: { kind: "ai_connection_wait", providerWorkStarted: false } }, + }).where(eq(heartbeatRuns.id, runId)); + const clearLock = () => db.update(issues).set({ executionRunId: null }).where(eq(issues.id, issueId)); + if (phase === "schedule") await clearLock(); + const transaction = db.transaction.bind(db); + let raceApplied = false; + const transactionSpy = phase === "transaction" + ? vi.spyOn(db, "transaction").mockImplementationOnce(async (callback, config) => { + await clearLock(); + raceApplied = true; + return transaction(callback, config); + }) + : null; + const scheduled = await heartbeat.scheduleBoundedRetry(runId, { + now, retryReason: "ai_connection_busy", wakeReason: "ai_connection_busy_retry", + maxAttempts: 1, delayMs: 1_000, + }).finally(() => transactionSpy?.mockRestore()); + if (phase === "transaction") expect(raceApplied).toBe(true); + if (phase === "schedule" || phase === "transaction") { + expect(scheduled).toMatchObject({ outcome: "not_scheduled", errorCode: "issue_execution_lock_changed" }); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, runId))).toHaveLength(0); + return; + } + expect(scheduled.outcome).toBe("scheduled"); + if (scheduled.outcome !== "scheduled") return; + if (phase === "promote") await clearLock(); + const promotion = await heartbeat.promoteDueScheduledRetries(scheduled.dueAt); + if (phase === "promote") { + expect(promotion).toEqual({ promoted: 0, runIds: [] }); + } else { + expect(promotion.runIds).toContain(scheduled.run.id); + await clearLock(); + const adapter = createPostgresRunDispatchAdapter(db); + expect(await adapter.cancelStaleQueuedRun({ + companyId, runId: scheduled.run.id, expectedStatus: "queued", now: scheduled.dueAt, + })).toMatchObject({ outcome: "cancelled", errorCode: "issue_execution_lock_changed" }); + } + expect(await heartbeat.getRun(scheduled.run.id)).toMatchObject({ + status: "cancelled", errorCode: "issue_execution_lock_changed", + }); + }, + ); + it("does not promote a duplicate max-turn continuation that does not own the issue lock", async () => { const { companyId, agentId, issueId, runId, now } = await seedMaxTurnFixture(); diff --git a/server/src/modules/run-dispatch/adapters/postgres.ts b/server/src/modules/run-dispatch/adapters/postgres.ts index 036b247754c..b89f2f07316 100644 --- a/server/src/modules/run-dispatch/adapters/postgres.ts +++ b/server/src/modules/run-dispatch/adapters/postgres.ts @@ -113,6 +113,7 @@ function readNonEmptyString(value: unknown): string | null { function classifyRetryReasonKind(retryReason: string | null): RetryReasonKind { if (retryReason === MAX_TURN_CONTINUATION_RETRY_REASON) return "max_turn_continuation"; if (retryReason === ISSUE_DISPOSITION_REPAIR_RETRY_REASON) return "disposition_repair"; + if (retryReason === "ai_connection_busy") return "ai_connection_wait"; if (retryReason === "native_safe_replacement") return "native_safe_replacement"; return "other"; } @@ -274,7 +275,7 @@ export function createPostgresRunDispatchAdapter( runAgentId: input.agentId, issueId, retryReasonKind, - enforceIssueExecutionLock: retryReasonKind === "max_turn_continuation", + enforceIssueExecutionLock: retryReasonKind === "max_turn_continuation" || retryReasonKind === "ai_connection_wait", isNonAssigneeWorkspaceBusyRetry: isNonAssigneeWorkspaceBusyRetry(retryReason, input.contextSnapshot), budgetBlock: null, agentInvokable: true, diff --git a/server/src/modules/run-dispatch/domain/policy.ts b/server/src/modules/run-dispatch/domain/policy.ts index f8351553db4..3049b2f141d 100644 --- a/server/src/modules/run-dispatch/domain/policy.ts +++ b/server/src/modules/run-dispatch/domain/policy.ts @@ -5,11 +5,12 @@ // then packs the result into a facts object. This file only branches on // that facts object; it never queries a database or reads the clock. -/** The retry reason a run carries, reduced to the three kinds a gate cares about. */ +/** The retry reason a run carries, reduced to the kinds a gate cares about. */ export type RetryReasonKind = | "max_turn_continuation" | "disposition_repair" | "native_safe_replacement" + | "ai_connection_wait" | "other"; export type BudgetBlockFacts = { @@ -403,7 +404,7 @@ export function decideScheduledRetryGate( } const lockOutcome = decideExecutionLock({ - requiresExecutionLock: requiresInProgress && facts.enforceIssueExecutionLock, + requiresExecutionLock: (requiresInProgress || facts.retryReasonKind === "ai_connection_wait") && facts.enforceIssueExecutionLock, runId: facts.runId, issueExecutionRunId: facts.issueExecutionRunId, }); @@ -411,7 +412,7 @@ export function decideScheduledRetryGate( return { allowed: false, reason: - "Scheduled max-turn continuation suppressed because the issue execution lock belongs to a different run", + "Scheduled retry suppressed because the issue execution lock belongs to a different run", errorCode: "issue_execution_lock_changed", issueId: facts.issueId, details: { @@ -629,7 +630,7 @@ export function decideQueuedRunStaleness( } const lockOutcome = decideExecutionLock({ - requiresExecutionLock: requiresInProgress, + requiresExecutionLock: requiresInProgress || facts.retryReasonKind === "ai_connection_wait", runId: facts.runId, issueExecutionRunId: facts.issueExecutionRunId, }); @@ -638,7 +639,7 @@ export function decideQueuedRunStaleness( stale: true, errorCode: "issue_execution_lock_changed", reason: - "Cancelled because max-turn continuation no longer owns the issue execution lock before the queued run could start", + "Cancelled because the retry no longer owns the issue execution lock before the queued run could start", details: { issueId: facts.issueId, expectedExecutionRunId: facts.runId, diff --git a/server/src/routes/agents.ts b/server/src/routes/agents.ts index e826efc7053..8baa38c3d22 100644 --- a/server/src/routes/agents.ts +++ b/server/src/routes/agents.ts @@ -3,6 +3,7 @@ import { prepareManagedAiRuntime, assertManagedAiProjectAuth, stripAiAuthBinding import { ADAPTER_AUTH_MISSING_CHECK_CODE, AI_CONNECTION_CAPABILITIES, aiConnectionBindingSchema, type AiConnectionBinding } from "@paperclipai/shared"; import { toolConnections } from "@paperclipai/db"; import { aiConnectionService } from "../services/ai-connections.js"; +import { defaultAiConnectionForHire } from "../services/agent-ai-connection-default.js"; import { assertAiConnectionCreateAccess, canInstallSharedAiConnectionForNewAgent, responsibleUserForAiRequest, validateAiApiKey } from "./ai-connections.js"; import { isAiConnectionCompatible } from "@paperclipai/shared"; import { applyConnectorSkills, resolveConnectorAssignments, annotateConnectorSkills, isConnectorSkill } from "../services/connector-runtime.js"; @@ -2412,6 +2413,22 @@ export function agentRoutes( return normalizedRuntimeConfig; } + async function normalizeCreatedAgentRuntimeConfig( + req: Request, + companyId: string, + adapterType: string, + adapterConfig: Record, + runtimeConfig: unknown, + ) { + const normalized = normalizeNewAgentRuntimeConfig(runtimeConfig); + if (req.actor.type !== "agent" || normalized.aiConnection) return normalized; + const manager = req.actor.agentId ? await svc.getById(req.actor.agentId) : null; + if (!manager || manager.companyId !== companyId) throw forbidden("Hiring agent is unavailable"); + const binding = defaultAiConnectionForHire(adapterType, adapterConfig, manager.runtimeConfig?.aiConnection); + if (binding) normalized.aiConnection = binding; + return normalized; + } + async function normalizeMediatedAdapterConfigForPersistence(input: { companyId: string; adapterType: string | null | undefined; @@ -2528,14 +2545,18 @@ export function agentRoutes( companyId: string, adapterType: string | null | undefined, adapterConfig: Record, + runtimeConfig: unknown, ): Promise<{ adapterConfig: Record; inheritedFixedClaudeOAuthBinding: boolean }> { const noInheritance = { adapterConfig, inheritedFixedClaudeOAuthBinding: false }; + if (asRecord(runtimeConfig)?.aiConnection) return noInheritance; if (req.actor.type !== "agent" || !req.actor.agentId) return noInheritance; const credentialKeys = adapterType ? INHERITABLE_AGENT_CREDENTIAL_ENV_KEYS[adapterType] : undefined; if (!credentialKeys) return noInheritance; const parent = await svc.getById(req.actor.agentId); if (!parent || parent.companyId !== companyId || parent.adapterType !== adapterType) return noInheritance; + // Managed parents must not pass stale legacy credential references to hires. + if (aiConnectionBindingSchema.safeParse(parent.runtimeConfig.aiConnection).success) return noInheritance; const parentEnv = asRecord(asRecord(parent.adapterConfig)?.env); if (!parentEnv) return noInheritance; @@ -3297,7 +3318,17 @@ export function agentRoutes( async function validateManagedAgentBinding(req: Request, companyId: string, agentId: string, adapterType: string, config: Record, binding: AiConnectionBinding, environmentId: string | null | undefined, test: boolean, newAgent = false) { const userId = responsibleUserForAiRequest(req); const allowUninstalledShared = newAgent && await canInstallSharedAiConnectionForNewAgent(db, req, companyId, binding); - const selection = await aiConnectionService(db).select({ companyId, agentId, userId, adapterType, model: config.model, runnerProvider: config.provider, acpxAgent: config.acpxAgent, binding, allowUninstalledPersonal: newAgent, allowUninstalledShared, allowLegacyValidation: test }); + const selection = await aiConnectionService(db).select({ companyId, agentId, userId, adapterType, model: config.model, runnerProvider: config.provider, acpxAgent: config.acpxAgent, binding, allowUninstalledPersonal: newAgent, allowUninstalledShared, allowLegacyValidation: test }).catch((error: unknown) => { + // Hiring is allowed before the responsible user has connected this + // provider. Execution still resolves credentials and creates the normal + // task connection request; compatibility and access denials stay errors. + if (newAgent && !test && binding.mode === "responsible_user" && error instanceof HttpError + && ["ai_connection_default_missing", "ai_connection_missing", "ai_connection_unavailable", "ai_connection_responsible_user_missing"].includes(String(asRecord(error.details)?.code))) { + return null; + } + throw error; + }); + if (!selection) return undefined; if (test) { if (environmentId) await assertAdapterTestEnvironmentForCompany(companyId, environmentId); const target = await resolveAdapterTestExecutionContext({ companyId, adapterType, environmentId: environmentId ?? null }); @@ -4399,6 +4430,7 @@ export function agentRoutes( hireInput.adapterType, rawHireAdapterConfig, ), + hireInput.runtimeConfig, ); const requestedAdapterConfig = applyCodexLocalKeyIsolation( companyId, @@ -4431,7 +4463,7 @@ export function agentRoutes( adapterType: hireInput.adapterType, adapterConfig: desiredSkillAssignment.adapterConfig, }); - const normalizedRuntimeConfig = normalizeNewAgentRuntimeConfig(hireInput.runtimeConfig); + const normalizedRuntimeConfig = await normalizeCreatedAgentRuntimeConfig(req, companyId, hireInput.adapterType, normalizedAdapterConfig, hireInput.runtimeConfig); const normalizedHireInput = { ...hireInput, adapterConfig: normalizedAdapterConfig, @@ -4726,7 +4758,7 @@ export function agentRoutes( adapterType: createInput.adapterType, adapterConfig: desiredSkillAssignment.adapterConfig, }); - const normalizedRuntimeConfig = normalizeNewAgentRuntimeConfig(createInput.runtimeConfig); + const normalizedRuntimeConfig = await normalizeCreatedAgentRuntimeConfig(req, companyId, createInput.adapterType, normalizedAdapterConfig, createInput.runtimeConfig); await assertAgentEnvironmentSelection(companyId, createInput.adapterType, createInput.defaultEnvironmentId); await assertAgentDefaultEnvironmentSelection(companyId, createInput.defaultEnvironmentId, { allowedDrivers: allowedEnvironmentDriversForAgent(createInput.adapterType), diff --git a/server/src/services/agent-ai-connection-default.ts b/server/src/services/agent-ai-connection-default.ts new file mode 100644 index 00000000000..fc0224f1246 --- /dev/null +++ b/server/src/services/agent-ai-connection-default.ts @@ -0,0 +1,42 @@ +import { + AI_PROVIDERS, + aiConnectionBindingSchema, + isAiConnectionCompatible, + type AiConnectionBinding, + type AiProvider, +} from "@paperclipai/shared"; + +// Only keys read by the child's provider express a child auth override. +// A config copied from another provider can retain unrelated keys. +const PROVIDER_AUTH_ENV_KEYS: Record = { + anthropic: ["ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", "CLAUDE_CODE_OAUTH_TOKEN", "CLAUDE_CONFIG_DIR", "ANTHROPIC_BASE_URL", "CLAUDE_CODE_USE_BEDROCK", "CLAUDE_CODE_USE_VERTEX", "CLAUDE_CODE_USE_FOUNDRY"], + openai: ["OPENAI_API_KEY", "CODEX_API_KEY", "CODEX_HOME", "OPENAI_BASE_URL"], + openrouter: ["OPENROUTER_API_KEY", "OPENCODE_AUTH_JSON", "OPENCODE_CONFIG_CONTENT", "OPENCODE_CONFIG", "OPENCODE_CONFIG_DIR", "PAPERCLIP_OPENCODE_PROVIDERS"], + xai: ["XAI_API_KEY", "GROK_API_KEY", "GROK_HOME", "XAI_BASE_URL"], +}; + +/** A hire inherits a connection choice, never its manager's credentials or identity. */ +export function defaultAiConnectionForHire( + adapterType: string, + config: Record, + managerBinding: unknown, +): AiConnectionBinding | undefined { + const compatible = (binding: AiConnectionBinding) => + isAiConnectionCompatible(binding, adapterType, config.model, config.provider, config.acpxAgent); + const inherited = aiConnectionBindingSchema.safeParse(managerBinding); + // Unmanaged parents keep their existing login and credential-reference paths. + if (!inherited.success) return undefined; + const env = config.env && typeof config.env === "object" ? config.env as Record : {}; + const withChildAuthPrecedence = (binding: AiConnectionBinding) => + PROVIDER_AUTH_ENV_KEYS[binding.provider].some((key) => env[key] !== undefined) ? undefined : binding; + if (inherited.data.mode !== "delegated" && compatible(inherited.data)) { + return withChildAuthPrecedence(inherited.data); + } + // The selected provider's personal default supplies the actual sign-in method + // at run time. A provider without an account can be connected on the first task. + for (const provider of AI_PROVIDERS) { + const binding = { provider, method: "api_key", mode: "responsible_user" } as const; + if (compatible(binding)) return withChildAuthPrecedence(binding); + } + return undefined; +} diff --git a/server/src/services/execution-recovery-attempt.test.ts b/server/src/services/execution-recovery-attempt.test.ts index 757d01ec17f..f1757f5e914 100644 --- a/server/src/services/execution-recovery-attempt.test.ts +++ b/server/src/services/execution-recovery-attempt.test.ts @@ -2,6 +2,12 @@ import { describe, expect, it } from "vitest"; import { executionFailureRetryCount } from "./execution-recovery-attempt.js"; describe("failure attempts across resource waits", () => { + it("preserves prior failures through repeated AI subscription waits", () => { + expect(executionFailureRetryCount({ scheduledRetryReason: "ai_connection_busy", scheduledRetryAttempt: 12, + contextSnapshot: { failureRetriesBeforeAiConnectionWait: 1 } })).toBe(1); + expect(executionFailureRetryCount({ scheduledRetryReason: "transient_failure", scheduledRetryAttempt: 2, + contextSnapshot: { failureRetriesBeforeAiConnectionWait: 0 } })).toBe(2); + }); it("preserves prior failures through repeated workspace waits", () => { expect(executionFailureRetryCount({ scheduledRetryReason: "workspace_busy", scheduledRetryAttempt: 12, contextSnapshot: { failureRetriesBeforeWorkspaceWait: 1 } })).toBe(1); diff --git a/server/src/services/execution-recovery-attempt.ts b/server/src/services/execution-recovery-attempt.ts index a87c9ce192f..11449966a8e 100644 --- a/server/src/services/execution-recovery-attempt.ts +++ b/server/src/services/execution-recovery-attempt.ts @@ -5,6 +5,10 @@ export function executionFailureRetryCount(run: { contextSnapshot?: Record | null; }): number { if (run.scheduledRetryReason === "max_turns_continuation") return 0; + if (run.scheduledRetryReason === "ai_connection_busy") { + const count = run.contextSnapshot?.failureRetriesBeforeAiConnectionWait; + if (typeof count === "number" && Number.isInteger(count) && count >= 0) return count; + } if (run.scheduledRetryReason === "workspace_busy") { // Only a server-created workspace retry can consume this field. Its // scheduler overwrites caller context with the predecessor's durable count. diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 02847cf88a3..220621e69a7 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -842,6 +842,10 @@ const MAX_TURN_CONTINUATION_LIVE_RUN_STATUSES = [ export { WORKSPACE_BUSY_RETRY_REASON }; export const WORKSPACE_BUSY_RETRY_WAKE_REASON = "workspace_busy_retry"; export const WORKSPACE_BUSY_ERROR_CODE = "workspace_busy"; +const AI_CONNECTION_BUSY_RETRY_REASON = "ai_connection_busy"; +function isAiConnectionBusy(error: unknown): error is HttpError { + return error instanceof HttpError && parseObject(error.details).code === AI_CONNECTION_BUSY_RETRY_REASON; +} export const WORKSPACE_BUSY_RETRY_BASE_DELAY_MS = 60 * 1000; export const WORKSPACE_BUSY_RETRY_JITTER_MS = 60 * 1000; // A running run stops counting as a shared-workspace holder once it has been @@ -15018,6 +15022,7 @@ export function heartbeatService( ); const nextAttempt = (retryReason === WORKSPACE_BUSY_RETRY_REASON || + retryReason === AI_CONNECTION_BUSY_RETRY_REASON || retryReason === MAX_TURN_CONTINUATION_RETRY_REASON ? (run.scheduledRetryAttempt ?? 0) : executionFailureRetryCount(run)) + 1; @@ -15144,6 +15149,7 @@ export function heartbeatService( const requiresIssueGate = hasConversationContinuationPolicy(run.resultJson) || + retryReason === AI_CONNECTION_BUSY_RETRY_REASON || retryReason === MAX_TURN_CONTINUATION_RETRY_REASON || retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON; if (requiresIssueGate) { @@ -15212,6 +15218,9 @@ export function heartbeatService( executionFailureRetryCount(run), } : {}), + ...(retryReason === AI_CONNECTION_BUSY_RETRY_REASON + ? { failureRetriesBeforeAiConnectionWait: executionFailureRetryCount(run) } + : {}), ...(shouldQuarantineWorkspaceForRetry ? { workspaceValidationRecovery: { @@ -15534,6 +15543,20 @@ export function heartbeatService( } } + if (retryReason === AI_CONNECTION_BUSY_RETRY_REASON && issueId) { + // The issue row is locked above. Recheck after the preflight gate so + // cancellation or recovery cannot leave a successor without its lock. + const [lockedIssue] = await tx.select({ executionRunId: issues.executionRunId }) + .from(issues).where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId))); + if (lockedIssue?.executionRunId !== run.id) { + return { + outcome: "not_scheduled", issueId, errorCode: "issue_execution_lock_changed", + reason: "Subscription retry suppressed because the task execution lock changed", + details: { issueId, expectedExecutionRunId: run.id, currentExecutionRunId: lockedIssue?.executionRunId ?? null }, + }; + } + } + const wakeupRequest = await tx .insert(agentWakeupRequests) .values({ @@ -15916,6 +15939,41 @@ export function heartbeatService( .then((rows) => rows[0] ?? null); } + // Subscription homes are serialized so refresh tokens cannot be overwritten + // by another run. Contention is a resource wait, not broken authentication. + // The database lease is released on completion/disconnect; retrying remains + // safe across processes and each attempt revalidates the selected account. + async function finalizeAiConnectionBusyDeferral(run: typeof heartbeatRuns.$inferSelect, error: HttpError) { + const now = new Date(); + const cancelled = await setRunStatusIfRunning(run.id, "cancelled", { + error: error.message, errorCode: AI_CONNECTION_BUSY_RETRY_REASON, finishedAt: now, + resultJson: { executionRecovery: { kind: "ai_connection_wait", providerWorkStarted: false } }, + }); + if (!cancelled.updated) return; + await setWakeupStatus(run.wakeupRequestId, "cancelled", { finishedAt: now, error: error.message }).catch(() => undefined); + const cancelledRun = cancelled.run ?? await getRun(run.id); + const agent = await getAgent(run.agentId); + let scheduled = false; + try { + if (cancelledRun && agent) { + const retry = await scheduleBoundedRetryForRun(cancelledRun, agent, { + now, retryReason: AI_CONNECTION_BUSY_RETRY_REASON, wakeReason: "ai_connection_busy_retry", + maxAttempts: (cancelledRun.scheduledRetryAttempt ?? 0) + 1, + delayMs: computeWorkspaceBusyRetryDelayMs(), + }); + scheduled = retry.outcome === "scheduled"; + await appendRunEvent(cancelledRun, { + eventType: "lifecycle", stream: "system", level: "info", + message: scheduled ? "Waiting for the shared AI subscription. This task will retry automatically." : "The AI subscription is busy; this task can no longer retry automatically.", + payload: { retryScheduled: scheduled }, + }); + } + } finally { + if (cancelledRun && !scheduled) await releaseIssueExecutionAndPromote(cancelledRun); + await finalizeAgentStatus(run.agentId, "cancelled", null, { wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run) }); + } + } + // Terminal handling for a WorkspaceBusyDeferral thrown by the pre-dispatch // gate: cancel the run (contention is not a failure), schedule a // workspace_busy retry, and leave the agent idle. The issue execution lock @@ -20711,6 +20769,7 @@ export function heartbeatService( try { managedAiRuntime = await prepareManagedAiRuntime(db, { companyId: agent.companyId, agentId: agent.id, responsibleUserId, adapterType: agent.adapterType, binding: aiBinding, config: resolvedConfig }); } catch (error) { + if (isAiConnectionBusy(error)) throw error; if (responsibleUserId && issueId && aiBinding.mode === "responsible_user") { await connectionIntentService(db).request({ sub: agent.id, company_id: agent.companyId, run_id: run.id, responsible_user_id: responsibleUserId }, aiBinding.provider, { purpose: "ai" }).catch(() => { logger.warn({ runId: run.id, agentId: agent.id }, "Could not attach AI connection request; runtime configuration action remains available"); @@ -25018,6 +25077,10 @@ export function heartbeatService( ? outerErr.reason : "adopted_runner_authentication_timeout", }).catch(() => undefined); + } else if (isAiConnectionBusy(outerErr)) { + await finalizeAiConnectionBusyDeferral(run, outerErr).catch((error) => { + logger.error({ err: error, runId }, "failed to schedule a retry for the busy AI subscription"); + }); } else if (isWorkspaceBusyDeferral(outerErr)) { // Expected contention on a shared project workspace, not a // failure: park the run as a bounded scheduled retry and leave the diff --git a/server/src/services/legacy-execution-recovery.test.ts b/server/src/services/legacy-execution-recovery.test.ts index 3567091ffb5..62558c60535 100644 --- a/server/src/services/legacy-execution-recovery.test.ts +++ b/server/src/services/legacy-execution-recovery.test.ts @@ -38,3 +38,11 @@ it("retains the hold until the provider actually acknowledges cancellation", () })).toBe(false); } }); + +it("retries a busy AI subscription only when no provider work started", () => { + const waiting = { runtimeMode: "legacy", status: "cancelled", errorCode: "ai_connection_busy", scheduledRetryAttempt: 10, + resultJson: { executionRecovery: { kind: "ai_connection_wait", providerWorkStarted: false } } }; + expect(legacyExecutionNeedsReconciliation(waiting)).toBe(false); + expect(legacyExecutionNeedsReconciliation({ ...waiting, resultJson: {} })).toBe(true); + expect(legacyExecutionNeedsReconciliation({ ...waiting, resultJson: { executionRecovery: { kind: "ai_connection_wait", providerWorkStarted: true } } })).toBe(true); + }); diff --git a/server/src/services/legacy-execution-recovery.ts b/server/src/services/legacy-execution-recovery.ts index f414ee333e2..fa967dc9378 100644 --- a/server/src/services/legacy-execution-recovery.ts +++ b/server/src/services/legacy-execution-recovery.ts @@ -31,8 +31,10 @@ export function legacyExecutionNeedsReconciliation( && evidence.providerStopped === true && evidence.sessionPreserved === true && evidence.actionOutcomes === "settled" && (run.resultJson?.executionCancellation as Record | undefined)?.state === "acknowledged") return false; - // Waiting for a live workspace holder precedes provider execution. It is a - // resource wait, not a failed provider attempt or permission to replay work. + // Waiting for a subscription or workspace precedes provider execution. It is + // a resource wait, not a failed provider attempt or permission to replay work. + if (run.status === "cancelled" && run.errorCode === "ai_connection_busy" && + evidence?.kind === "ai_connection_wait" && evidence.providerWorkStarted === false) return false; if (run.status === "cancelled" && run.errorCode === "workspace_busy" && evidence?.kind === "workspace_wait" && evidence.providerWorkStarted === false) return false; if (executionFailureRetryCount(run) >= 2) return true; diff --git a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts index f0262f3a434..cce1e6b1fa9 100644 --- a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts +++ b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts @@ -320,7 +320,13 @@ export class PaperclipRunnerToolAuthority { conversation: Boolean(context.issue.conversationAgentId) }); } case "search_api": return searchRunnerApi(call.arguments); - case "call_api": return this.#callApi(call.callId, call.arguments); + case "call_api": { + // PRP reserves operationId/callId for semantic result identity. The + // HTTP operation is metadata, including in previously saved receipts; + // exposing it as operationId makes the runner reject a valid response. + const { operationId, ...response } = record(await this.#callApi(call.callId, call.arguments)); + return { ...response, apiOperationId: operationId }; + } case "get_task_context": return { company: { id: this.binding.companyId }, actor: redactedActor(context.actor), diff --git a/server/src/services/native-runtime/runner-api.integration.test.ts b/server/src/services/native-runtime/runner-api.integration.test.ts index 9dcecc474ec..98e42f01af6 100644 --- a/server/src/services/native-runtime/runner-api.integration.test.ts +++ b/server/src/services/native-runtime/runner-api.integration.test.ts @@ -34,7 +34,7 @@ describe("runner API against real HTTP routes", () => { import { createInterface } from 'node:readline'; const send = value => process.stdout.write(JSON.stringify(value)+'\\n'); let step = 0; -const calls = [{tool:'search_api',arguments:{query:'list projects'}},{tool:'call_api',arguments:{operationId:'GET /api/companies/{companyId}/projects'}}]; +const calls = [{tool:'search_api',arguments:{query:'list projects'}},{tool:'call_api',arguments:{operationId:'GET /api/companies/{companyId}/projects'}},{tool:'call_api',arguments:{operationId:'GET /api/projects/{id}',pathParams:{id:'${fixture.foreignProjectId}'}}}]; const next = () => { const c=calls[step++]; if(c) send({id:'call-'+step,method:'item/tool/call',params:{threadId:'api-thread',turnId:'api-turn',itemId:'api-item-'+step,callId:'api-call-'+step,...c}}); else send({method:'turn/completed',params:{turn:{id:'api-turn',status:'completed'}}}); }; for await (const line of createInterface({input:process.stdin})) { const m=JSON.parse(line); @@ -57,15 +57,24 @@ else if(m.id!==undefined) send({id:m.id,result:{}}); const params = request.params as any; const result = await fixture.authority.execute({ tool: params.tool, arguments: params.arguments, callId: params.callId }); results.push(result); - return { success: true, contentItems: [{ type: "inputText", text: JSON.stringify({ ok: true, result }) }] }; + // Match production's dynamicToolResponse: an extra test-only envelope + // hides collisions between API response fields and PRP tool identity. + return { success: true, contentItems: [{ type: "inputText", text: JSON.stringify(result) }] }; }); try { await bundle.transport.request("initialize", {}); await bundle.transport.request("thread/start", { cwd: fixture.workspace, dynamicTools: await fixture.authority.definitions() }); await bundle.transport.request("turn/start", { input: [{ type: "text", text: "Find the project" }] }); - for await (const notification of bundle.transport.notifications()) if (notification.method === "turn/completed") break; - expect(results).toHaveLength(2); - expect(results[1]).toMatchObject({ status: 200, data: [{ id: fixture.projectId, name: "Aurora" }] }); + let timeout: ReturnType | undefined; + try { + await Promise.race([ + (async () => { for await (const notification of bundle.transport.notifications()) if (notification.method === "turn/completed") return; })(), + new Promise((_, reject) => { timeout = setTimeout(() => reject(new Error("Native provider did not continue after the API tool result")), 10_000); }), + ]); + } finally { clearTimeout(timeout); } + expect(results).toHaveLength(3); + expect(results[1]).toMatchObject({ status: 200, apiOperationId: "GET /api/companies/{companyId}/projects", data: [{ id: fixture.projectId, name: "Aurora" }] }); + expect(results[2]).toMatchObject({ ok: false, status: 404 }); expect(bundle.evidence().diagnostics).toContain("runnerd authenticated to the durable PRP control plane"); } finally { await bundle.transport.close(); } }, 30_000); diff --git a/tests/hiring-ai-connections/.gitignore b/tests/hiring-ai-connections/.gitignore new file mode 100644 index 00000000000..12b46b848a1 --- /dev/null +++ b/tests/hiring-ai-connections/.gitignore @@ -0,0 +1 @@ +/test-results/ diff --git a/tests/hiring-ai-connections/README.md b/tests/hiring-ai-connections/README.md new file mode 100644 index 00000000000..748f672a3e5 --- /dev/null +++ b/tests/hiring-ai-connections/README.md @@ -0,0 +1,132 @@ +# AI connections through hiring + +Paid, opt-in acceptance against a separate loopback `local_trusted` instance. Each +scenario creates its own company and accounts. It never edits an existing company. +Do not start the server with provider API keys: ambient credentials could mask +missing managed-connection bindings. + +Start from a fresh worktree with dependencies installed: + +```sh +pnpm exec tsx cli/src/index.ts test-drive --data-dir /tmp/paperclip-hiring-qa --no-browser +``` + +Use the URL printed at startup. Test-drive may choose another available port when +restarted. In a separate shell, export `ANTHROPIC_API_KEY` and `OPENAI_API_KEY`, then: + +```sh +HIRING_AI_LIVE=1 HIRING_AI_TEST_URL=http://127.0.0.1:3100 \ + pnpm exec playwright test --config tests/hiring-ai-connections/playwright.config.ts --grep 'local:' +``` + +For both Claude and Codex, the browser creates a task instructing the real parent to +hire one agent of each provider and create a self-assigned subtask. The server's +activity log must attribute both hires to the parent agent. The test verifies the +subtask and a task assigned to the same-provider hire finish under the original +managed account. It then assigns work to the other provider, checks the pending AI +connection card, connects with an API key in the card, and verifies the task resumes +and finishes without sending another message or clicking retry. Completion comments +must be authored by the assigned agent; account and responsible-user attribution +are checked on the successful runs. + +Provider calls use real credentials and incur usage. The fixture uses Claude Sonnet +4.6 and Codex GPT-5.4 through their CLI adapters. Provider subscription sign-in is +not automated by this live suite. Integration tests cover subscription inheritance, +credential serialization, and automatic retry after the parent's lease is released. + +For representative coverage of the newer native runner, put +`PAPERCLIP_RUNNER_API_TOOLS_ENABLED=true` in the disposable data directory's +`instances/default/.env`, then restart test-drive. Test-drive clears inherited +`PAPERCLIP_*` variables before loading that file. The existing opt-in managed API +tools are how native agents hire. Then use: + +```sh +HIRING_AI_LIVE=1 HIRING_AI_RUNNER=native HIRING_AI_TEST_URL=http://127.0.0.1:3100 \ + pnpm exec playwright test --config tests/hiring-ai-connections/playwright.config.ts --grep 'local: native: Codex' +``` + +This uses native Codex and native ACPX/Claude. It verifies native run persistence, +managed API hiring, native `create_task` delegation, and `paperclip_finish` +completion. Set `HIRING_AI_ENVIRONMENT=daytona` to exercise the same native path +remotely. The default remains legacy CLI adapters; it also asserts that successful +runs stayed on the legacy path. A single native parent scenario exercises both +native worker providers without repeating the full matrix. + +## Daytona + +Build the plugin and provide `DAYTONA_API_KEY` plus an immutable, qualified runner +image digest. See `tests/runner-e2e/README.md` for the image publication workflow. + +```sh +pnpm -C packages/plugins/sandbox-providers/daytona build +HIRING_AI_LIVE=1 HIRING_AI_ENVIRONMENT=daytona \ + HIRING_AI_TEST_URL=http://127.0.0.1:3100 \ + HIRING_AI_DAYTONA_IMAGE=ghcr.io/paperclipai/paperclip-daytona-runner@sha256:YOUR_DIGEST \ + pnpm exec playwright test --config tests/hiring-ai-connections/playwright.config.ts --grep 'daytona:' +``` + +The same tests install the plugin only in the disposable instance, create a Daytona +environment, and assign it to the parent and both hires. Successful runs must report +that environment. The finalizer cancels remaining active runs in the test company +and deletes the environment and its leases. Task history and encrypted connections +remain in the disposable data directory for inspection. + +Native Daytona also requires the controller's remote runner binary and provider +pack. On a Mac, use the Linux artifacts from the same qualified image, not the local +runner binary. The pack must match this checkout's provider pins and profile +digests; a newer image is not automatically compatible. Extract into an empty +destination directory without starting the container: + +```sh +mkdir -p /tmp/hiring-native-artifacts +hiring_image_container=$(docker create "$HIRING_AI_DAYTONA_IMAGE" /bin/true) +docker cp "$hiring_image_container:/opt/paperclip-runner/provider-pack" /tmp/hiring-native-artifacts/provider-pack +docker cp "$hiring_image_container:/usr/local/bin/paperclip-runnerd" /tmp/hiring-native-artifacts/paperclip-runnerd +docker rm "$hiring_image_container" +``` + +Add these settings to the disposable instance's `instances/default/.env`, alongside +the native API-tools flag, and restart test-drive: + +```dotenv +PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH=/tmp/hiring-native-artifacts/provider-pack +PAPERCLIP_RUNNER_REMOTE_BINARY_PATH=/tmp/hiring-native-artifacts/paperclip-runnerd +``` + +Run the Daytona command with `HIRING_AI_RUNNER=native` and +`--grep 'daytona: native: Claude'` for one representative native remote scenario. +If no matching image is available, use the `runnerd-build` and +`provider-pack-build` Docker targets from this checkout, following the runner E2E +workflow's lockfile resolution. Configure both resulting Linux artifacts together; +native startup can stage them over a different image's preinstalled artifacts. +Mixing a newer runner binary with an older provider pack can fail the sidecar +handshake even when the individual provider version pins match. + +## Regression and Storybook checks + +```sh +pnpm exec vitest run server/src/__tests__/agent-hire-ai-connections.test.ts \ + server/src/services/legacy-execution-recovery.test.ts \ + server/src/services/execution-recovery-attempt.test.ts \ + server/src/services/native-runtime/runner-api.integration.test.ts \ + ui/src/features/connections/ConnectionIntentInteractionBody.test.tsx +pnpm --filter @paperclipai/ui exec storybook dev -p 6010 -c storybook/.storybook --ci --no-open +HIRING_AI_STORYBOOK_URL=http://127.0.0.1:6010 \ + pnpm exec playwright test --config tests/hiring-ai-connections/playwright.config.ts --grep storybook +``` + +The Storybook checks use the production card and credential components with fixture +API responses. They cover both providers, API/subscription choice, a narrow viewport, +completion, cancellation, and invalid credentials. They do not contact providers. + +The Daytona plugin's `file-sync.test.ts` also exercises real GNU tar extraction of +interleaved read-only skill directories. It runs on Linux and on macOS with +`gnu-tar` installed. This guards the macOS-to-Daytona staging failure where directory +permissions were restored before all skill files arrived, while asserting that the +finished directory and files retain their read-only modes. + +Screenshots and redacted account/run attribution are written under `test-results/`. +Traces, videos, and automatic failure screenshots are disabled because credential +fields are involved. Each Playwright invocation clears its output directory; use +`--output` to preserve separate local, Daytona, and Storybook evidence. Keep credentials +out of issue descriptions, console output, command-line arguments, and attachments. diff --git a/tests/hiring-ai-connections/hiring.live.spec.ts b/tests/hiring-ai-connections/hiring.live.spec.ts new file mode 100644 index 00000000000..9d1a2810373 --- /dev/null +++ b/tests/hiring-ai-connections/hiring.live.spec.ts @@ -0,0 +1,175 @@ +import { randomUUID } from "node:crypto"; +import { writeFile } from "node:fs/promises"; +import path from "node:path"; +import { test, expect, type Page, type APIRequestContext } from "@playwright/test"; + +// Paid, opt-in acceptance against a dedicated loopback instance. No provider +// mocking, direct database writes, credential traces, or existing company edits. +test.skip(process.env.HIRING_AI_LIVE !== "1", "Set HIRING_AI_LIVE=1 with both provider keys for paid acceptance"); +const environment = process.env.HIRING_AI_ENVIRONMENT ?? "local"; +const runner = process.env.HIRING_AI_RUNNER ?? "legacy"; +const native = runner === "native"; +const providers = { + anthropic: { label: "Claude", adapterType: "claude_local", model: "claude-sonnet-4-6", key: "ANTHROPIC_API_KEY", radio: "Claude API" }, + openai: { label: "Codex", adapterType: "codex_local", model: "gpt-5.4", key: "OPENAI_API_KEY", radio: "OpenAI API" }, +} as const; +type Provider = keyof typeof providers; + +function client(request: APIRequestContext) { + return async (url: string, method = "GET", data?: unknown): Promise => { + const res = await request.fetch(`/api${url}`, { method, data }); + // Only report route/status: credential-bearing request bodies must never + // enter a reporter's assertion or attachment output. + if (!res.ok()) { + const body = await res.json().catch(() => ({})); + throw new Error(`${method} ${url}: ${res.status()} ${body.code ?? "Request failed"}`); + } + return res.json(); + }; +} + +async function createTask(page: Page, prefix: string, agentName: string, title: string, prompt: string) { + await page.goto(`/${prefix}/issues`); + await page.getByRole("navigation").getByRole("button", { name: "New Task", exact: true }).click(); + await page.getByPlaceholder("Task title").fill(title); + await page.getByRole("dialog").getByRole("textbox", { name: "editable markdown", exact: true }).fill(prompt); + await page.getByRole("button", { name: "Assignee", exact: true }).click(); + await page.getByPlaceholder("Search assignees...").fill(agentName); + await page.getByRole("button", { name: agentName, exact: true }).click(); + await page.getByRole("button", { name: "No project", exact: true }).click(); + await page.getByRole("button", { name: "Create Task", exact: true }).click(); + await expect(page.getByPlaceholder("Task title")).toHaveCount(0); +} + +const coordination = native + ? `Use the native Paperclip tools. First get_task_context to verify the current task and your identity. For hiring use call_api with operationId POST /api/companies/{companyId}/agent-hires and the requested JSON body. Use create_task for subtasks. Do not use shell commands or raw HTTP. Never read or print credentials. For completion call paperclip_finish exactly once with reportedWorkDisposition done, summary describing the completed work, completionClaim {contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]}, evidence [], and verification []. Wait for that tool to succeed before your final response.` + : `Use the injected PAPERCLIP_API_URL and PAPERCLIP_API_KEY for every API request. Include Authorization: Bearer $PAPERCLIP_API_KEY and X-Paperclip-Run-Id: $PAPERCLIP_RUN_ID. Never print credentials or the environment. Verify GET /api/agents/me identifies you before writes; stop if it does not. You must act as the agent, not the local board. Use standard Paperclip APIs or the managed API tool, and do not install a CLI. Do not read or modify unrelated files. For completion use PATCH /api/issues/$PAPERCLIP_TASK_ID with status done and a comment.`; + +for (const source of ["anthropic", "openai"] as const) { + test(`${environment}: ${runner}: ${providers[source].label} hires both providers, delegates, and repairs missing auth in the task`, async ({ page, request }, testInfo) => { + const api = client(request); + const base = new URL(testInfo.project.use.baseURL!); + expect(base.protocol).toBe("http:"); + expect(["127.0.0.1", "localhost", "[::1]"]).toContain(base.hostname); + expect((await api("/health")).deploymentMode).toBe("local_trusted"); + expect(["legacy", "native"]).toContain(runner); + if (native) await api("/instance/settings/experimental", "PATCH", { enableNativeRunner: true }); + for (const p of Object.values(providers)) if (!process.env[p.key]) throw new Error(`Missing ${p.key}`); + const target: Provider = source === "anthropic" ? "openai" : "anthropic"; + const nonce = randomUUID().slice(0, 8); + const company = await api("/companies", "POST", { name: `Hiring QA ${environment} ${providers[source].label} ${nonce}`, description: "Disposable live AI hiring acceptance" }); + const account = await api(`/companies/${company.id}/ai-connections`, "POST", { provider: source, method: "api_key", name: `${providers[source].label} QA account`, ownership: "personal", apiKey: process.env[providers[source].key], agentIds: [], allAgents: false }); + let environmentId: string | undefined; + if (environment === "daytona") { + if (!process.env.DAYTONA_API_KEY || !process.env.HIRING_AI_DAYTONA_IMAGE) throw new Error("Daytona requires DAYTONA_API_KEY and HIRING_AI_DAYTONA_IMAGE"); + const installed = (await api("/plugins")).find((plugin: any) => plugin.packageName === "@paperclipai/plugin-daytona"); + if (!installed) await api("/plugins/install", "POST", { packageName: path.resolve(import.meta.dirname, "../../packages/plugins/sandbox-providers/daytona"), isLocalPath: true }); + else expect(installed.status).toBe("ready"); + const secret = await api(`/companies/${company.id}/secrets`, "POST", { name: `Daytona QA ${nonce}`, key: "DAYTONA_API_KEY", value: process.env.DAYTONA_API_KEY }); + const env = await api(`/companies/${company.id}/environments`, "POST", { name: `Hiring QA Daytona ${nonce}`, driver: "sandbox", config: { provider: "daytona", apiKey: { type: "secret_ref", secretId: secret.id, version: "latest" }, image: process.env.HIRING_AI_DAYTONA_IMAGE, cpu: 4, memory: 4, disk: 10, reuseLease: false, runnerLifecycleMode: "per_turn", autoStopInterval: 5, autoArchiveInterval: 15, autoDeleteInterval: 60, timeoutMs: 300_000 }, envVars: {} }); + environmentId = env.id; + } + const binding = { provider: source, method: "api_key", mode: "responsible_user" }; + const adapterType = (provider: Provider) => native ? "paperclip_runner" : providers[provider].adapterType; + const config = (provider: Provider) => native + ? { model: providers[provider].model, lifecycleMode: "per_turn", ...(provider === "anthropic" ? { provider: "acpx", acpxAgent: "claude", acpxPermissionMode: "approve-all" } : { provider: "codex", codexPermissionMode: "never" }) } + : { model: providers[provider].model, engine: "cli", ...(provider === "anthropic" ? { dangerouslySkipPermissions: true } : { dangerouslyBypassApprovalsAndSandbox: true }) }; + const manager = await api(`/companies/${company.id}/agents`, "POST", { name: `${providers[source].label} Manager ${nonce}`, role: "ceo", adapterType: adapterType(source), adapterConfig: config(source), defaultEnvironmentId: environmentId, runtimeConfig: { aiConnection: binding, heartbeat: { enabled: false } }, instructionsBundle: { entryFile: "AGENTS.md", files: { "AGENTS.md": `# Acceptance manager\n${coordination}\nPerform only the requested bounded hiring workflow. This test has no code deliverable.` } } }); + const sameName = `${providers[source].label} Teammate ${nonce}`; + const crossName = `${providers[target].label} Teammate ${nonce}`; + const selfTitle = `Self subtask ${nonce}`; + const proof = (kind: string) => `HIRING-${kind}-${nonce}: 1147`; + const simplePrompt = (kind: string) => `${coordination}\nCalculate 31*37 and use exactly ${proof(kind)} as your ${native ? "paperclip_finish summary and final response" : "completion comment"}. Mark this task done. No subtasks or files.`; + const hire = (provider: Provider, name: string) => ({ name, role: "engineer", reportsTo: manager.id, adapterType: adapterType(provider), adapterConfig: config(provider), ...(environmentId ? { defaultEnvironmentId: environmentId } : {}), instructionsBundle: { entryFile: "AGENTS.md", files: { "AGENTS.md": `# Acceptance worker\n${coordination}` } } }); + const title = `Hire both providers ${nonce}`; + const child = native + ? `Use create_task with idempotencyKey self-${nonce}, title ${JSON.stringify(selfTitle)}, assigneeActorId ${manager.id}, and description ${JSON.stringify(simplePrompt("SELF"))}.` + : `Create a subtask with title ${JSON.stringify(selfTitle)}, parentId equal to PAPERCLIP_TASK_ID, assigneeAgentId ${manager.id}, status todo, and description ${JSON.stringify(simplePrompt("SELF"))}.`; + const prompt = `${coordination}\nPerform exactly these three operations, then mark this manager task done with links. Do not wait for the child and do not configure AI connections.\n1. POST /api/companies/${company.id}/agent-hires with ${JSON.stringify(hire(source, sameName))}.\n2. POST the same endpoint with ${JSON.stringify(hire(target, crossName))}. It is expected that ${providers[target].label} has no connection yet; hiring must still succeed. Do not provide runtimeConfig.aiConnection or credentials for either hire; the server must select defaults.\n3. ${child} The board will assign tasks to the two hires separately.`; + const evidence: Record = { companyId: company.id, prefix: company.issuePrefix, managerId: manager.id, account, source, target, runner, environment, environmentId }; + console.log(`Fixture ${JSON.stringify(evidence)}`); + try { + await createTask(page, company.issuePrefix, manager.name, title, prompt); + const findIssue = async (name: string) => (await api(`/companies/${company.id}/issues`)).find((issue: any) => issue.title === name); + let hiringIssue: any; + await expect.poll(async () => { hiringIssue = await findIssue(title); return hiringIssue?.status; }, { timeout: 300_000, intervals: [1500, 3000] }).toBe("done"); + evidence.hiringIssueId = hiringIssue.id; + const agents = await api(`/companies/${company.id}/agents`); + const same = agents.find((agent: any) => agent.name === sameName); + const cross = agents.find((agent: any) => agent.name === crossName); + expect(same?.runtimeConfig.aiConnection).toEqual(binding); + expect(cross?.runtimeConfig.aiConnection).toMatchObject({ provider: target, mode: "responsible_user" }); + evidence.sameAgentId = same.id; evidence.crossAgentId = cross.id; + const activity = await api(`/companies/${company.id}/activity`); + for (const agent of [same, cross]) expect(activity.some((entry: any) => entry.action === "agent.hire_created" && entry.entityId === agent.id && entry.actorType === "agent" && entry.agentId === manager.id)).toBe(true); + const completed = async (issue: any, kind: string, agentId: string, connectionId?: string) => { + await expect.poll(async () => (await api(`/issues/${issue.id}`)).status, { timeout: 300_000, intervals: [2000, 3000] }).toBe("done"); + const comments = await api(`/issues/${issue.id}/comments`); + expect(comments.some((comment: any) => comment.authorAgentId === agentId && comment.body.includes(proof(kind)))).toBe(true); + let successful: any; + await expect.poll(async () => { + const runs = await api(`/issues/${issue.id}/runs`); + successful = runs.find((run: any) => run.status === "succeeded" && run.agentId === agentId && run.contextIssueId === issue.id); + return Boolean(successful); + }, { timeout: 60_000, intervals: [1000] }).toBe(true); + const run = await api(`/heartbeat-runs/${successful.runId}`); + expect(run.runtimeMode).toBe(runner); + if (native) expect(run.runnerInstanceId).toBeTruthy(); + expect(run.contextSnapshot.aiConnection).toMatchObject({ responsibleUserId: "local-board", ...(connectionId ? { connectionId } : {}) }); + if (environmentId) { + expect(successful.environment).toMatchObject({ id: environmentId, driver: "sandbox" }); + expect(successful.environmentLease).toMatchObject({ provider: "daytona", leasePolicy: "ephemeral" }); + expect(successful.environmentLease.providerLeaseId).toBeTruthy(); + } + evidence[kind] = { issueId: issue.id, runId: run.id, runtimeMode: run.runtimeMode, runnerInstanceId: run.runnerInstanceId, attribution: run.contextSnapshot.aiConnection, environment: successful.environment, environmentLease: successful.environmentLease }; + await page.goto(`/${company.issuePrefix}/issues/${issue.identifier}`); + await expect(page.getByText(proof(kind), { exact: true }).first()).toBeVisible(); + await page.screenshot({ path: testInfo.outputPath(`${kind.toLowerCase()}-completed.png`), fullPage: true }); + }; + const self = await findIssue(selfTitle); + expect(self).toMatchObject({ parentId: hiringIssue.id, createdByAgentId: manager.id, responsibleUserId: "local-board" }); + await completed(self, "SELF", manager.id, account.connectionId); + const sameTitle = `Work for same provider ${nonce}`; + await createTask(page, company.issuePrefix, same.name, sameTitle, simplePrompt("SAME")); + await completed(await findIssue(sameTitle), "SAME", same.id, account.connectionId); + const crossTitle = `Work pending new connection ${nonce}`; + await createTask(page, company.issuePrefix, cross.name, crossTitle, simplePrompt("CROSS")); + const crossIssue = await findIssue(crossTitle); + evidence.crossIssueId = crossIssue.id; + await page.goto(`/${company.issuePrefix}/issues/${crossIssue.identifier}`); + await expect(page.getByRole("button", { name: "Fix connection", exact: true })).toBeVisible({ timeout: 90_000 }); + const before = await api(`/issues/${crossIssue.id}/interactions`); + expect(before.filter((interaction: any) => interaction.kind === "connection_intent" && interaction.payload.purpose === "ai" && interaction.status === "pending")).toHaveLength(1); + await page.screenshot({ path: testInfo.outputPath("missing-connection-card.png"), fullPage: true }); + await page.getByRole("button", { name: "Fix connection", exact: true }).click(); + const inline = page.getByTestId("ai-connection-inline-repair"); + await expect(inline).toBeVisible(); + await expect(page.getByRole("dialog")).toHaveCount(0); + await inline.getByRole("button", { name: "Use API key instead", exact: true }).click(); + await inline.getByRole("radio", { name: providers[target].radio, exact: true }).click(); + await inline.screenshot({ path: testInfo.outputPath("inline-before-credential.png") }); + try { + await inline.getByPlaceholder("Enter API key here").fill(process.env[providers[target].key]!); + await inline.getByRole("button", { name: "Connect", exact: true }).click(); + await expect(inline).toHaveCount(0, { timeout: 60_000 }); + } catch { + const field = inline.getByPlaceholder("Enter API key here"); + if (await field.count()) await field.fill("").catch(() => {}); + throw new Error("Inline credential setup did not complete (credential details suppressed)"); + } + // No follow-up message or explicit retry: connecting must resume this task. + const accounts = (await api(`/companies/${company.id}/ai-connections`)).connections; + const targetAccount = accounts.find((item: any) => item.provider === target && item.isDefault); + expect(targetAccount).toMatchObject({ status: "connected", method: "api_key" }); + await completed(crossIssue, "CROSS", cross.id, targetAccount.id); + expect((await api(`/issues/${crossIssue.id}/interactions`)).filter((interaction: any) => interaction.kind === "connection_intent" && interaction.status === "accepted")).toHaveLength(1); + } finally { + await writeFile(testInfo.outputPath("evidence.json"), JSON.stringify(evidence, null, 2)); + if (environmentId) { + const runs = await api(`/companies/${company.id}/heartbeat-runs?limit=100`); + for (const run of runs) if (["queued", "running"].includes(run.status)) await api(`/heartbeat-runs/${run.id}/cancel`, "POST"); + await expect(async () => { await api(`/environments/${environmentId}?destroyReusableSandboxLeases=true`, "DELETE"); }).toPass({ timeout: 120_000, intervals: [3000] }); + } + } + }); +} diff --git a/tests/hiring-ai-connections/hiring.storybook.spec.ts b/tests/hiring-ai-connections/hiring.storybook.spec.ts new file mode 100644 index 00000000000..24e8fcf214c --- /dev/null +++ b/tests/hiring-ai-connections/hiring.storybook.spec.ts @@ -0,0 +1,15 @@ +import { test, expect } from "@playwright/test"; + +test.skip(!process.env.HIRING_AI_STORYBOOK_URL, "Set HIRING_AI_STORYBOOK_URL to the isolated Storybook server"); +for (const name of ["new-claude-connection", "new-codex-connection", "new-codex-connection-narrow", "new-claude-api-connection", "new-codex-connection-complete", "ai-repair-cancel", "ai-repair-invalid-key"]) { + test(`storybook: ${name}`, async ({ page }, testInfo) => { + if (name.endsWith("narrow")) await page.setViewportSize({ width: 375, height: 812 }); + const id = `connections-in-task-connections--${name}`; + await page.goto(`${process.env.HIRING_AI_STORYBOOK_URL}/iframe.html?id=${id}&viewMode=story`); + await expect.poll(() => page.evaluate(() => document.body.dataset.inFeedStoryReady), { timeout: 60_000 }).toBe(id); + expect(await page.evaluate(() => document.body.dataset.inFeedStoryError)).toBeUndefined(); + await expect(page.getByRole("dialog")).toHaveCount(0); + expect(await page.evaluate(() => document.documentElement.scrollWidth <= window.innerWidth)).toBe(true); + await page.screenshot({ path: testInfo.outputPath(`${name}.png`), fullPage: true }); + }); +} diff --git a/tests/hiring-ai-connections/playwright.config.ts b/tests/hiring-ai-connections/playwright.config.ts new file mode 100644 index 00000000000..1ea76013e6c --- /dev/null +++ b/tests/hiring-ai-connections/playwright.config.ts @@ -0,0 +1,19 @@ +import { defineConfig } from "@playwright/test"; + +export default defineConfig({ + testDir: ".", + testMatch: "*.spec.ts", + outputDir: "test-results", + workers: 1, + timeout: 900_000, + expect: { timeout: 30_000 }, + reporter: "list", + use: { actionTimeout: 30_000, navigationTimeout: 30_000, + baseURL: process.env.HIRING_AI_TEST_URL ?? "http://127.0.0.1:3101", + browserName: "chromium", + viewport: { width: 1440, height: 1000 }, + trace: "off", + video: "off", + screenshot: "off", + }, +}); diff --git a/ui/src/features/connections/ConnectionIntentInteractionBody.test.tsx b/ui/src/features/connections/ConnectionIntentInteractionBody.test.tsx index e64baea43c3..b97d789f0fb 100644 --- a/ui/src/features/connections/ConnectionIntentInteractionBody.test.tsx +++ b/ui/src/features/connections/ConnectionIntentInteractionBody.test.tsx @@ -34,7 +34,7 @@ vi.mock("@/api/connection-intents", () => ({ vi.mock("@/components/ai-connections/AiConnectionCredentialStep", () => ({ AiConnectionCredentialStep: (props: { connectionId?: string; name: string; fixedMethod?: boolean; onComplete: (result: {connectionId: string; grantId: string; method: "api_key"}) => void; onCancel: () => void }) => { credentialRender(props); return
{props.name}{String(props.fixedMethod)} - +
; }, })); @@ -416,6 +416,21 @@ describe("ConnectionIntentInteractionBody dialog behavior", () => { describe("AI repair inside the card", () => { const interaction: ConnectionIntentInteraction = { ...pendingConnectionIntentInteraction, payload: { ...pendingConnectionIntentInteraction.payload, purpose: "ai" } }; const connection = { id: "selected-account", name: "My Codex account", provider: "openai", method: "api_key", ownership: "personal", ownerName: "Dotta", status: "revoked" }; + it.each(["anthropic", "openai"])("connects a missing %s default directly in the task", async (provider) => { + setupOptionsMock.mockResolvedValue({ interaction, existingConnections: [], aiConnection: { provider, method: "api_key", mode: "responsible_user" } }); + completeMock.mockResolvedValue({ ...interaction, status: "accepted" }); + renderBody(interaction); await flush(); + await act(() => button("Fix connection")!.click()); + expect(document.querySelector('[role="dialog"]')).toBeNull(); + expect(document.querySelector('[data-testid="shared-connection-setup"]')).toBeNull(); + expect(document.querySelector('[data-testid="shared-ai-credentials"]')).not.toBeNull(); + expect(credentialRender.mock.lastCall![0]).toMatchObject({ provider, ownership: "personal", agentIds: [interaction.payload.requestingAgentId], allAgents: false }); + expect(credentialRender.mock.lastCall![0].connectionId).toBeUndefined(); + expect(credentialRender.mock.lastCall![0].fixedMethod).toBeUndefined(); + expect(document.body.textContent).toContain("This task can’t run until"); + await act(() => button("Reconnect selected account")!.click()); + expect(completeMock).toHaveBeenCalledWith(interaction.id, "new-ai-account"); + }); it("reuses authentication inline, preserves the selected account, cancels with focus, and completes", async () => { setupOptionsMock.mockResolvedValue({ interaction, existingConnections: [], aiRepair: { connection, canReconnect: true } }); completeMock.mockResolvedValue({ ...interaction, status: "accepted" }); diff --git a/ui/src/features/connections/ConnectionIntentInteractionBody.tsx b/ui/src/features/connections/ConnectionIntentInteractionBody.tsx index 7fd9e5254a7..7e344c73f79 100644 --- a/ui/src/features/connections/ConnectionIntentInteractionBody.tsx +++ b/ui/src/features/connections/ConnectionIntentInteractionBody.tsx @@ -11,6 +11,7 @@ import { import type { ConnectionIntentInteraction } from "@paperclipai/shared"; import { connectionIntentsApi } from "@/api/connection-intents"; import { AiConnectionCredentialStep } from "@/components/ai-connections/AiConnectionCredentialStep"; +import { AI_PROVIDERS } from "@/components/ai-connections/model"; import { AppLogo } from "@/pages/apps/AppLogo"; import { Button } from "@/components/ui/button"; import { @@ -181,7 +182,7 @@ export function ConnectionIntentInteractionBody({ ? { icon: CheckCircle2, title: `${interaction.payload.serviceName} connected`, - body: isAi ? "The connection was restored for this request." : `${interaction.payload.requestingAgentName} can use this connection on the continuation run.`, + body: isAi ? "This agent can now use the connection." : `${interaction.payload.requestingAgentName} can use this connection on the continuation run.`, } : interaction.status === "rejected" ? { @@ -309,7 +310,16 @@ export function ConnectionIntentInteractionBody({

: setupQuery.data?.aiConnection && setupQuery.data.aiConnection.mode !== "responsible_user" ?

The selected account is no longer available to you. Ask its owner to restore access, or choose an available AI connection in the agent’s settings.

- : setupContent; + : setupQuery.data?.aiConnection ? { void finishNewConnection(result); }} + onCancel={() => { closeSetup(); returnFocusToCard(); }} + /> : setupContent; return (

{interaction.payload.purpose === "ai" - ? "Restore the agent’s selected AI account, then continue this task." + ? "This task can’t run until the agent has a valid AI connection. Connect here and the task will resume automatically." : "Connect your identity or reuse an eligible connection. Access is added only for this agent."}

diff --git a/ui/storybook/stories/in-feed-connections.stories.tsx b/ui/storybook/stories/in-feed-connections.stories.tsx index 5cb3aa4977d..0f0df58c764 100644 --- a/ui/storybook/stories/in-feed-connections.stories.tsx +++ b/ui/storybook/stories/in-feed-connections.stories.tsx @@ -31,7 +31,7 @@ const connection = { createdByAgentId: null, createdByUserId: "user-board", createdAt: new Date("2026-09-07"), updatedAt: new Date("2026-09-07"), } satisfies ToolConnection; -type Scenario = { ai?: boolean; ownerOnly?: boolean; checking?: boolean; count?: number; loading?: boolean; loadError?: boolean; completeError?: boolean; submitting?: boolean; denied?: boolean }; +type Scenario = { ai?: boolean; missingAiAccount?: "anthropic" | "openai"; ownerOnly?: boolean; checking?: boolean; count?: number; loading?: boolean; loadError?: boolean; completeError?: boolean; submitting?: boolean; denied?: boolean }; const meta: Meta = { title: "Connections/In-task connections", parameters: { layout: "padded" }, @@ -45,9 +45,22 @@ const meta: Meta = { channel.on("unhandledErrorsWhilePlaying", reportPlayError); const original = window.fetch; const scenario = (parameters.connectionScenario ?? {}) as Scenario; - let current = structuredClone(scenario.ai ? aiPending : pending); + let current = structuredClone(scenario.missingAiAccount ? missingAiInteraction(scenario.missingAiAccount) : scenario.ai ? aiPending : pending); window.fetch = async (input, init) => { const url = new URL(typeof input === "string" ? input : input instanceof URL ? input.href : input.url, window.location.origin); + if (scenario.missingAiAccount) { + if (url.pathname.endsWith("/environments")) return Response.json([{ id: "local-env", companyId: pending.companyId, name: "Local", driver: "local", status: "active", config: {}, isDefault: true }]); + if (url.pathname.endsWith("/environments/capabilities")) return Response.json({ sandboxProviders: {} }); + if (url.pathname === "/api/instance/settings") return Response.json({ defaultEnvironmentId: "local-env" }); + if (url.pathname === "/api/instance/settings/general") return Response.json({ executionMode: "any" }); + if (url.pathname === "/api/instance/settings/experimental") return Response.json({}); + if (url.pathname === "/api/health") return Response.json({ deploymentMode: "local_trusted", localAiLoginSupported: true }); + if (url.pathname.endsWith("/ai-connections/local/attempts")) return Response.json({ sessionId: "storybook-login", command: "codex login", expiresAt: "2099-01-01T00:00:00Z" }); + if (url.pathname.endsWith("/ai-connections/local/check")) return Response.json({ status: "sign_in_required" }); + if (url.pathname.endsWith("/ai-connections") && (!init?.method || init.method === "GET")) return Response.json({ currentUserId: pending.addresseeUserId, connections: [] }); + if (url.pathname.includes("/secrets")) return Response.json([]); + if (url.pathname.includes("claude-oauth")) return Response.json(null); + } if (url.pathname.endsWith("/tools/gallery")) return Response.json({ apps: CONNECTABLE_APP_DEFINITIONS.filter((app) => ["notion", "github", "posthog", "zapier"].includes(app.slug)), capabilities: { canCreateOrganizationGrant: true, canSetCompanyInstall: true }, @@ -62,7 +75,7 @@ const meta: Meta = { if (scenario.loadError) return Response.json({ error: "Connection options are temporarily unavailable. Try again." }, { status: 503 }); return Response.json({ version: 1, interaction: current, requestedAgentId: pending.payload.requestingAgentId, service: { service: "notion", name: "Notion", state: "available", methods: [] }, - ...(scenario.ai ? { aiConnection: { provider: "openrouter", method: "api_key", mode: "responsible_user" }, aiRepair: { connection: aiAccount, canReconnect: !scenario.ownerOnly } } : {}), + ...(scenario.ai ? { aiConnection: { provider: scenario.missingAiAccount ?? "openrouter", method: "api_key", mode: "responsible_user" }, ...(scenario.missingAiAccount ? {} : { aiRepair: { connection: aiAccount, canReconnect: !scenario.ownerOnly } }) } : {}), existingConnections: Array.from({ length: scenario.count ?? 0 }, (_, i) => ({ ...connection, id: `${connection.id.slice(0, -1)}${i}`, name: i ? "Team Notion workspace" : connection.name })), }); } @@ -288,3 +301,36 @@ export const AiRepairOwnerRequired: Story = { ...aiRepair, parameters: { connect await expect(await canvas.findByText(/Alex must reconnect/)).toBeVisible(); await expect(canvas.queryByPlaceholderText("Enter API key here")).not.toBeInTheDocument(); }}; + +function missingAiInteraction(provider: "anthropic" | "openai"): ConnectionIntentInteraction { + return { ...aiPending, payload: { ...aiPending.payload, serviceSlug: provider, serviceName: provider === "anthropic" ? "Claude" : "OpenAI", serviceLogoUrl: `/brands/${provider === "anthropic" ? "claude" : "codex"}-color.svg`, serviceDarkLogoUrl: null } }; +} +const missingAi = (provider: "anthropic" | "openai"): Story => ({ + ...card(missingAiInteraction(provider), { ai: true, missingAiAccount: provider }), + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await userEvent.click(await canvas.findByRole("button", { name: "Fix connection" })); + await expect(await canvas.findByLabelText("Connection name")).toBeEnabled(); + await expect(within(document.body).queryByRole("dialog")).not.toBeInTheDocument(); + await expect(await canvas.findByRole("radio", { name: `${provider === "anthropic" ? "Claude" : "OpenAI"} Subscription` })).toBeVisible(); + }, +}); +export const NewClaudeConnection = missingAi("anthropic"); +export const NewCodexConnection = missingAi("openai"); +export const NewCodexConnectionNarrow: Story = { ...NewCodexConnection, globals: { viewport: { value: "mobile1", isRotated: false } } }; +export const NewClaudeApiConnection: Story = { ...NewClaudeConnection, play: async context => { + await NewClaudeConnection.play!(context); + const canvas = within(context.canvasElement); + await userEvent.click(canvas.getByRole("button", { name: "Use API key instead" })); + await userEvent.click(await canvas.findByRole("radio", { name: "Claude API" })); + await waitFor(() => expect(canvas.getByPlaceholderText("Enter API key here")).toBeVisible()); +}}; +export const NewCodexConnectionComplete: Story = { ...NewCodexConnection, play: async context => { + await NewCodexConnection.play!(context); + const canvas = within(context.canvasElement); + await userEvent.click(canvas.getByRole("button", { name: "Use API key instead" })); + await userEvent.click(await canvas.findByRole("radio", { name: "OpenAI API" })); + await userEvent.type(await canvas.findByPlaceholderText("Enter API key here"), "storybook-placeholder"); + await userEvent.click(canvas.getByRole("button", { name: "Connect" })); + await expect(await canvas.findByText("OpenAI connected")).toBeVisible(); +}};