From 270035738ea34e709c15f2bfbec9005df4c69b32 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 27 Oct 2026 00:03:22 -0500 Subject: [PATCH] test(evals): author real Claude hire and private qualification --- .../runtime-claude-hiring.test.ts | 25 +++++++ .../runtime-claude-hiring.ts | 74 +++++++++++++++++++ ...runtime-claude-private-canary-receipt.json | 29 ++++++++ .../runtime-claude-private-privacy.spec.ts | 32 ++++++++ .../runtime-claude-private-worker.ts | 39 ++++++++++ .../runtime-claude-private.ts | 38 ++++++++++ 6 files changed, 237 insertions(+) create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-claude-hiring.test.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-claude-hiring.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-claude-private-canary-receipt.json create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-claude-private-privacy.spec.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-claude-private-worker.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-claude-private.ts diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-claude-hiring.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-claude-hiring.test.ts new file mode 100644 index 00000000000..99f23f9a632 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-claude-hiring.test.ts @@ -0,0 +1,25 @@ +import { expect, it } from "vitest"; +import { gradeClaudeHire, claudeValidationReplyMatches } from "./runtime-claude-hiring.js"; +import { validateClaudePrivateReceipt } from "./runtime-claude-private.js"; +function valid() { + return { origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", agents: [{ id: "morgan", name: "Morgan", companyId: "company", reportsTo: "ceo", role: "engineer", title: "Frontend engineer", adapterType: "paperclip_runner", adapterConfig: { provider: "acpx", acpxAgent: "claude" }, lifecycleState: "ready" }], links: ["/ACORN/agents/morgan"], humanQueries: ["Hire Morgan"], reply: "", tasks: [], activity: [{ id: "hire", action: "agent.hire_created", entityId: "morgan", actorType: "agent", actorId: "ceo", runId: "source" }], runs: [{ id: "source", agentId: "ceo", status: "succeeded" }], state: { actualAccountBinding: true, savedTargetBound: true, selectedEnvironmentId: "environment", noApprovalPolicy: true, prerequisiteProbePassed: true, savedValidation: { agentId: "morgan", environmentId: "environment", probeStatus: "pass", probeCheckCodes: ["claude_hello_passed"] }, canonicalReply: "Morgan is created as your frontend engineer. The account is connected and runtime validation passed; the saved profile is linked here." } }; +} +it("accepts a real native Claude hire and natural distinction between saved profile and validation", () => { + expect(gradeClaudeHire(valid()).passed).toBe(true); const o = valid(); o.state.canonicalReply = "I've added Morgan to the team with the frontend role and reporting line saved. The selected account and computer passed the startup test."; expect(gradeClaudeHire(o).passed).toBe(true); +}); +it("rejects wrong harness/account/target, duplicate hire, wrong actor/run and missing profile link", () => { + for (const mutate of [(o: ReturnType) => { o.agents[0]!.adapterConfig.acpxAgent = "codex"; }, (o: ReturnType) => { o.agents.push({ ...o.agents[0]!, id: "duplicate" }); }, (o: ReturnType) => { o.state.actualAccountBinding = false; }, (o: ReturnType) => { o.state.savedTargetBound = false; }, (o: ReturnType) => { o.activity[0]!.runId = "other-run"; }, (o: ReturnType) => { o.activity[0]!.actorId = "other"; }, (o: ReturnType) => { o.links = []; }]) { const o = valid(); mutate(o); expect(gradeClaudeHire(o).passed).toBe(false); } + expect(gradeClaudeHire(undefined).passed).toBe(false); +}); +it("retains observed authentication failure and rejects falsely calling an unusable profile ready", () => { + const failure = { probeStatus: "fail" as const, probeCheckCodes: ["claude_auth_missing"] }; + expect(claudeValidationReplyMatches("Morgan was created, but the startup test failed because the account needs authentication. Reconnect the account before assigning work.", failure)).toBe(true); + expect(claudeValidationReplyMatches("Morgan is ready. Authentication validation failed, so reconnect later.", failure)).toBe(false); + expect(claudeValidationReplyMatches("Morgan could not start; check the model.", failure)).toBe(false); +}); +it("allows only closed scoped ID/check-code receipts from the standalone private worker", () => { + const id = "11111111-1111-4111-8111-111111111111", expected = { operation: "prepare-account" as const, companyId: id, environmentId: id }; + const receipt = { ...expected, processIsolation: true, testRunnerAttached: false, apiStepRecorder: false, connectionId: id, grantId: id, probeStatus: "pass", probeCheckCodes: ["claude_hello_passed"] }; + expect(validateClaudePrivateReceipt(receipt, expected)).toEqual(receipt); + for (const patch of [{ credential: "synthetic-private-key" }, { testRunnerAttached: true }, { companyId: "foreign" }, { probeCheckCodes: ["raw diagnostic with a credential"] }]) expect(() => validateClaudePrivateReceipt({ ...receipt, ...patch }, expected)).toThrow("closed"); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-claude-hiring.ts b/tests/runner-e2e/chat-catalog-domains/runtime-claude-hiring.ts new file mode 100644 index 00000000000..93d103a0485 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-claude-hiring.ts @@ -0,0 +1,74 @@ +import { expect } from "@playwright/test"; +import { agentHarnessType } from "../../../packages/shared/src/agent-runner.js"; +import type { ActivityEvent } from "../../../packages/shared/src/types/activity.js"; +import type { Agent } from "../../../packages/shared/src/types/agent.js"; +import type { AiConnectionList } from "../../../packages/shared/src/ai-connections.js"; +import { checks } from "./internal-support.js"; +import { waitForReply } from "./identity-support.js"; +import { runtimePrivateSpec, selectedRuntimeInputs } from "./runtime-private.js"; +import { privateClaudeOperation, type ClaudePrivateReceipt } from "./runtime-claude-private.js"; +import { agentLink, openRuntimeAgentProfile, observeRuntime, runtimeDefinition, runtimeObservation, type RuntimeAgent } from "./runtime-support.js"; +export function claudeValidationReplyMatches(reply: string, probe: Pick) { + const falseReady = /(?:is|now|already|fully|runtime)\s+(?:fully\s+)?(?:ready|working|validated)|validation (?:passed|succeeded)/i; + if (probe.probeStatus === "fail") { + const auth = probe.probeCheckCodes.some(code => /auth|login|credential/i.test(code)); + return /fail|could not|couldn't|unable|not (?:yet )?(?:ready|working)|needs|blocked/i.test(reply) && /test|validat|start|runtime|execut|account/i.test(reply) + && (!auth || /authenticat|login|sign.?in|credential|account/i.test(reply)) && !falseReady.test(reply); + } + return /creat|hir|saved|added/i.test(reply) && /validat|test|runtime|ready|start|working|account|connected/i.test(reply) + && !/never need.*validat|creat.*(?:guarantees|proves).*working/i.test(reply); +} +export function gradeClaudeHire(value: unknown) { + const c = checks(), o = runtimeObservation(value); + c.add("actual-claude-hire-and-qualified-prerequisite-evidence", !!o, "A real selected account/target probe, independently saved permanent hire, public attribution and result UI are required."); if (!o) return c.result(); + const state = o.state, agents = o.agents.filter(row => row.name === "Morgan") as Array>, morgan = agents[0], validation = state.savedValidation as ClaudePrivateReceipt | undefined; + c.add("one-permanent-frontend-claude-report", agents.length === 1 && !!morgan && morgan.companyId === o.companyId && morgan.reportsTo === o.leadId && morgan.role === "engineer" + && /frontend|front.end/i.test(`${morgan.title ?? ""} ${morgan.capabilities ?? ""}`) && agentHarnessType(morgan.adapterType ?? "", morgan.adapterConfig) === "claude_local" + && state.savedTargetBound === true && state.actualAccountBinding === true && !["rejected", "pending_approval"].includes(morgan.lifecycleState ?? ""), + "Exactly one permanent Morgan has the requested frontend role/reporting line, genuine Claude harness, selected same-company computer and usable actual Anthropic account binding."); + c.add("source-run-attributed-save-and-inspectable-profile", !!morgan && !!agentLink(o, morgan) && o.activity.some(row => ["agent.created", "agent.hire_created"].includes(row.action ?? "") && row.entityId === morgan.id && row.actorType === "agent" && row.actorId === o.leadId && (row as unknown as ActivityEvent).runId === o.runs[0]?.id) + && state.noApprovalPolicy === true && o.tasks.length === 0 && o.runs.length === 1 && o.runs[0]?.agentId === o.leadId && o.runs[0].status === "succeeded", + "The actual canonical CEO turn creates the saved employee under the admitted no-approval policy and links the actual profile, without starting employee tasks."); + c.add("creation-and-actual-saved-validation-distinguished", state.prerequisiteProbePassed === true && !!validation && validation.agentId === morgan?.id && validation.environmentId === state.selectedEnvironmentId + && claudeValidationReplyMatches(String(state.canonicalReply), validation), + "The response distinguishes saved creation from real runtime validation, and any independently observed failure remains a failure rather than a readiness claim."); + return c.result(); +} +export const runtimeClaudeHiringCases = [runtimeDefinition("agents.hire-claude-code", async ctx => { + const selected = selectedRuntimeInputs(ctx, "agents.hire-claude-code"); + if (selected.values.targetRef !== "fixture.environment" || selected.values.accountBindingRef !== "selected.ANTHROPIC_API_KEY") throw new Error("Setup blocker agents.hire-claude-code: require explicit owned fixture.environment and selected.ANTHROPIC_API_KEY references; values withheld"); + const environment = await ctx.input.api.get<{ id: string; companyId: string; driver: string }>(`/api/environments/${ctx.input.fixtures.environment.id}`); + if (environment.companyId !== ctx.input.fixtures.company.id || environment.driver !== "local") throw new Error("Setup blocker: selected Claude target must be the actual owned local computer, independently qualified below"); + const before = await ctx.input.api.get(`/api/companies/${environment.companyId}/agents`); + if (before.some(row => row.name === "Morgan")) throw new Error("Claude hire setup requires Morgan to be absent"); + await ctx.input.api.patch(`/api/companies/${environment.companyId}`, { requireBoardApprovalForNewAgents: false }); + const company = await ctx.input.api.get<{ requireBoardApprovalForNewAgents: boolean }>(`/api/companies/${environment.companyId}`); + if (!ctx.input.privateSecrets) throw new Error("Selected Claude prerequisite requires the owned private redaction sink"); + const credential = selected.credentials.ANTHROPIC_API_KEY!; ctx.input.privateSecrets.push(credential); + const account = await privateClaudeOperation({ operation: "prepare-account", baseURL: ctx.input.api.baseURL, companyId: environment.companyId, environmentId: environment.id, + credential, harnessExecutable: selected.values.harnessExecutable, model: selected.values.model, cookies: await ctx.input.page.context().cookies(ctx.input.api.baseURL) }); + if (account.probeStatus !== "pass") throw new Error("Setup blocker: actual selected Anthropic account/Claude command/model/target validation did not pass; private diagnostics withheld"); + const connections = await ctx.input.api.get(`/api/companies/${environment.companyId}/ai-connections`); + const connection = connections.connections.find(row => row.id === account.connectionId && row.grantId === account.grantId); + if (!connection || connection.provider !== "anthropic" || connection.status !== "connected" || !connection.isDefault) throw new Error("Actual selected Claude account is not a usable responsible-user default"); + return { connectionId: account.connectionId, grantId: account.grantId, selectedEnvironmentId: environment.id, prerequisiteProbePassed: true, noApprovalPolicy: !company.requireBoardApprovalForNewAgents }; +}, gradeClaudeHire, { + ...runtimePrivateSpec("agents.hire-claude-code"), authentication: "human-session", expectedRunCount: 1, attemptTimeoutMs: 600000, + sourceFiles: ["chat-catalog-domains/runtime-claude-hiring.ts", "chat-catalog-domains/runtime-claude-private.ts", "chat-catalog-domains/runtime-claude-private-worker.ts", "chat-catalog-domains/runtime-private.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agents.yaml"], + async act(ctx, state) { + await waitForReply(ctx); const observation = await observeRuntime(ctx, state), candidates = observation.agents.filter(row => row.name === "Morgan"); + if (candidates.length !== 1) throw new Error("The canonical Claude hire did not save exactly one permanent Morgan"); + state.worker = candidates[0]; state.canonicalReply = observation.reply; + state.savedValidation = await privateClaudeOperation({ operation: "validate-saved", companyId: observation.companyId, environmentId: String(state.selectedEnvironmentId), agentId: candidates[0]!.id, + connectionId: String(state.connectionId), grantId: String(state.grantId), baseURL: ctx.input.api.baseURL, cookies: await ctx.input.page.context().cookies(ctx.input.api.baseURL) }, ctx.input.privateInputs?.credentials.ANTHROPIC_API_KEY); + state.savedTargetBound = (state.savedValidation as ClaudePrivateReceipt).environmentId === state.selectedEnvironmentId; + }, + async observe(ctx, state) { + const o = await observeRuntime(ctx, state), morgan = o.agents.find(row => row.name === "Morgan"), binding = morgan?.runtimeConfig?.aiConnection as { mode?: string; provider?: string; connectionId?: string; grantId?: string } | undefined; + const connections = await ctx.input.api.get(`/api/companies/${o.companyId}/ai-connections`); + state.actualAccountBinding = !!binding && binding.provider === "anthropic" && (binding.mode === "responsible_user" || binding.connectionId === state.connectionId && binding.grantId === state.grantId) + && connections.connections.some(row => row.id === state.connectionId && row.grantId === state.grantId && row.status === "connected" && (binding.mode !== "responsible_user" || row.isDefault)); + return o; + }, + async verifyResult(ctx, state, value) { const o = runtimeObservation(value), worker = state.worker as RuntimeAgent; if (!o || !agentLink(o, worker)) throw new Error("Actual Claude profile link missing"); const source = ctx.input.page.url(); await openRuntimeAgentProfile(ctx, worker, o); await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByRole("heading", { name: "Morgan", exact: true }).first()).toBeVisible(); await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); }, +})]; diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-claude-private-canary-receipt.json b/tests/runner-e2e/chat-catalog-domains/runtime-claude-private-canary-receipt.json new file mode 100644 index 00000000000..8e2bdb95308 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-claude-private-canary-receipt.json @@ -0,0 +1,29 @@ +{ + "syntheticOnly": true, + "tests": 3, + "successAndExpectedFailureReports": true, + "syntheticHttpOnly": true, + "realProviderCalls": 0, + "canonicalAttempts": 0, + "initialSandboxListenDenied": true, + "explicitOfflineEscalatedCanaryPassed": true, + "trace": "off", + "video": "off", + "screenshots": "off", + "checkedSinks": [ + "stdout", + "JUnit", + "blob ZIP entries", + "HTML embedded report ZIP", + "error-context" + ], + "scannedItems": 13, + "privateValueCount": 12, + "findings": [], + "sourceSha256": { + "runtime-claude-private.ts": "12805653235725987079a38cdcbca609ec60ffd4a33782d007b30f1b36879689", + "runtime-claude-private-worker.ts": "38d1299d6ed030807ad9296fb6feb7b313c4244bf5a2c5fdbc13f101098cace0", + "runtime-claude-private-privacy.spec.ts": "da1aceeb242da84b19f81b26b5572c4587996f2ed53e6c64f128bb1217360172" + }, + "configSha256": "890c62895e7b90eb24c2b96a321edc58b2f6b800531c2607a663fa6e4a155554" +} diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-claude-private-privacy.spec.ts b/tests/runner-e2e/chat-catalog-domains/runtime-claude-private-privacy.spec.ts new file mode 100644 index 00000000000..c5c9b70d5f2 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-claude-private-privacy.spec.ts @@ -0,0 +1,32 @@ +/** Explicit opt-in offline credential/reporting calibration; no real account/provider/DB. */ +import { createServer } from "node:http"; +import { appendFile } from "node:fs/promises"; +import { randomUUID } from "node:crypto"; +import { test, expect } from "@playwright/test"; +import { privateClaudeOperation } from "./runtime-claude-private.js"; +for (const branch of ["success", "private-child-failure", "parent-failure"] as const) test(`standalone Claude credential boundary ${branch}`, async () => { + test.skip(!process.env.PAPERCLIP_CLAUDE_CANARY_PRIVATE_FILE, "Explicit synthetic-only privacy canary"); + const credential = `synthetic-claude-${randomUUID()}`, cookie = `synthetic-cookie-${randomUUID()}`; + await appendFile(process.env.PAPERCLIP_CLAUDE_CANARY_PRIVATE_FILE!, `${credential}\n${cookie}\n`, { mode: 0o600 }); + const companyId = randomUUID(), environmentId = randomUUID(), connectionId = randomUUID(), grantId = randomUUID(); let keyObserved = false, cookieObserved = false; + const server = createServer(async (req, res) => { + cookieObserved ||= req.headers.cookie === `canary=${cookie}`; + res.setHeader("content-type", "application/json"); + if (req.method === "GET") { res.end(JSON.stringify({ id: environmentId, companyId, driver: "local" })); return; } + let raw = ""; for await (const chunk of req) raw += chunk; const body = JSON.parse(raw); + if (req.url?.endsWith("/ai-connections")) { + keyObserved = body.apiKey === credential; + if (branch === "private-child-failure") { res.statusCode = 500; res.end(JSON.stringify({ sensitiveDiagnostic: credential })); return; } + res.end(JSON.stringify({ connectionId, grantId })); return; + } + res.end(JSON.stringify({ status: "pass", checks: [{ code: "synthetic_probe_receipt_only" }] })); + }); + await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); const address = server.address(); if (!address || typeof address === "string") throw new Error("Synthetic canary did not bind"); + try { + const operation = privateClaudeOperation({ operation: "prepare-account", credential, companyId, environmentId, harnessExecutable: "synthetic-command-never-executed", model: "synthetic-model-never-called", baseURL: `http://127.0.0.1:${address.port}`, cookies: [{ name: "canary", value: cookie, domain: "127.0.0.1", path: "/", expires: -1, httpOnly: true, secure: false, sameSite: "Lax" }] }); + if (branch === "private-child-failure") await expect(operation).rejects.toThrow("diagnostics withheld"); + else { const receipt = await operation; expect(receipt.processIsolation).toBe(true); expect(receipt.testRunnerAttached).toBe(false); expect(receipt.probeStatus).toBe("pass"); } + expect(keyObserved && cookieObserved).toBe(true); + if (branch === "parent-failure") { test.fail(); throw new Error("Deliberate safe parent failure after standalone synthetic Claude credential request"); } + } finally { server.closeAllConnections(); await new Promise(resolve => server.close(() => resolve())); } +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-claude-private-worker.ts b/tests/runner-e2e/chat-catalog-domains/runtime-claude-private-worker.ts new file mode 100644 index 00000000000..2f3ccb64022 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-claude-private-worker.ts @@ -0,0 +1,39 @@ +import { readFile } from "node:fs/promises"; +import { test, type Cookie } from "@playwright/test"; +import type { ClaudePrivateRequest, ClaudePrivateReceipt } from "./runtime-claude-private.js"; +import type { AiRuntimeConnectionBinding } from "../../../packages/shared/src/ai-connection-router.js"; +import type { Agent } from "../../../packages/shared/src/types/agent.js"; +import { agentHarnessType } from "../../../packages/shared/src/agent-runner.js"; +async function main(): Promise { + let attached = false; try { test.info(); attached = true; } catch { /* standalone child only */ } + if (attached) throw new Error("Claude credential operation cannot run inside Playwright Test"); + const input = JSON.parse(await readFile(process.argv[2]!, "utf8")) as ClaudePrivateRequest & { companyId: string; environmentId: string; baseURL: string; cookies: Cookie[] }; + const base = new URL(input.baseURL); + if (base.protocol !== "http:" || base.hostname !== "127.0.0.1" || ![input.companyId, input.environmentId].every(id => /^[a-f0-9-]{36}$/i.test(id)) || !["prepare-account", "validate-saved"].includes(input.operation)) throw new Error("Invalid private Claude operation binding"); + const headers = { Cookie: input.cookies.map(row => `${row.name}=${row.value}`).join("; "), Origin: base.origin, "Content-Type": "application/json" }; + async function json(pathname: string, body?: unknown): Promise { + const target = new URL(pathname, base); if (target.origin !== base.origin || !pathname.startsWith("/api/")) throw new Error("Private Claude request escaped its actual instance"); + const response = await fetch(target, { method: body === undefined ? "GET" : "POST", body: body === undefined ? undefined : JSON.stringify(body), headers, redirect: "error", signal: AbortSignal.timeout(150000) }); + if (!response.ok) { await response.body?.cancel(); throw new Error("Actual selected Claude public operation rejected"); } return response.json() as Promise; + } + const environment = await json<{ id: string; companyId: string; driver: string }>(`/api/environments/${input.environmentId}`); + if (environment.id !== input.environmentId || environment.companyId !== input.companyId || environment.driver !== "local") throw new Error("Selected Claude fixture requires its actual same-company local execution target"); + let connectionId: string, grantId: string, config: Record, agentId: string | undefined, savedType: string | undefined, savedBinding: AiRuntimeConnectionBinding | undefined; + if (input.operation === "prepare-account") { + if (!input.credential || !input.harnessExecutable || !input.model) throw new Error("Missing selected Claude credential/command/model"); + const connection = await json<{ connectionId: string; grantId: string }>(`/api/companies/${input.companyId}/ai-connections`, { provider: "anthropic", method: "api_key", name: "Claude eval account", ownership: "personal", apiKey: input.credential, allAgents: true, agentIds: [] }); + connectionId = connection.connectionId; grantId = connection.grantId; config = { command: input.harnessExecutable, model: input.model }; + } else { + agentId = input.agentId; const agent = await json(`/api/agents/${agentId}`); + const environments = await json>(`/api/companies/${input.companyId}/environments`); + const effectiveEnvironmentId = agent.defaultEnvironmentId ?? environments.find(row => row.isDefault)?.id; + if (agent.companyId !== input.companyId || effectiveEnvironmentId !== input.environmentId || agentHarnessType(agent.adapterType, agent.adapterConfig) !== "claude_local") throw new Error("Saved hire is not the selected same-company Claude target"); + connectionId = input.connectionId; grantId = input.grantId; config = agent.adapterConfig; savedType = agent.adapterType; + savedBinding = agent.runtimeConfig.aiConnection; + if (!savedBinding || savedBinding.mode === "router" || savedBinding.provider !== "anthropic" || savedBinding.mode !== "responsible_user" && (savedBinding.connectionId !== connectionId || savedBinding.grantId !== grantId)) throw new Error("Saved Claude configuration lacks the actual selected managed Anthropic binding; ambient login fallback is forbidden"); + } + const binding = { provider: "anthropic", method: "api_key", mode: "delegated", connectionId, grantId }; + const probe = await json<{ status: "pass" | "warn" | "fail"; checks: Array<{ code: string }> }>(`/api/companies/${input.companyId}/adapters/${savedType ?? "claude_local"}/test-environment`, { ...(agentId ? { agentId, aiConnection: savedBinding } : { runner: "legacy", aiConnection: binding }), adapterConfig: config, environmentId: input.environmentId }); + return { processIsolation: true, testRunnerAttached: false, apiStepRecorder: false, operation: input.operation, companyId: input.companyId, environmentId: input.environmentId, connectionId, grantId, ...(agentId ? { agentId } : {}), probeStatus: probe.status, probeCheckCodes: probe.checks.map(row => row.code) }; +} +main().then(result => process.stdout.write(JSON.stringify(result))).catch(() => { process.stderr.write("Selected Claude operation failed; private diagnostics withheld"); process.exitCode = 1; }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-claude-private.ts b/tests/runner-e2e/chat-catalog-domains/runtime-claude-private.ts new file mode 100644 index 00000000000..dc7de8ffb2c --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-claude-private.ts @@ -0,0 +1,38 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { mkdtemp, writeFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import type { Cookie } from "@playwright/test"; +export interface ClaudePrivateReceipt { + processIsolation: true; testRunnerAttached: false; apiStepRecorder: false; + operation: "prepare-account" | "validate-saved"; companyId: string; environmentId: string; connectionId: string; grantId: string; agentId?: string; + probeStatus: "pass" | "warn" | "fail"; probeCheckCodes: string[]; +} +export function validateClaudePrivateReceipt(value: unknown, expected: Pick): ClaudePrivateReceipt { + const r = value as ClaudePrivateReceipt; + if (!r || r.processIsolation !== true || r.testRunnerAttached !== false || r.apiStepRecorder !== false || r.operation !== expected.operation || r.companyId !== expected.companyId || r.environmentId !== expected.environmentId + || Object.keys(r).some(key => !["processIsolation", "testRunnerAttached", "apiStepRecorder", "operation", "companyId", "environmentId", "connectionId", "grantId", "agentId", "probeStatus", "probeCheckCodes"].includes(key)) + || ![r.companyId, r.environmentId, r.connectionId, r.grantId, ...(r.agentId ? [r.agentId] : [])].every(id => typeof id === "string" && /^[a-f0-9-]{36}$/i.test(id)) + || !["pass", "warn", "fail"].includes(r.probeStatus) || !Array.isArray(r.probeCheckCodes) || r.probeCheckCodes.length > 100 || !r.probeCheckCodes.every(code => /^[a-z0-9_.-]{1,100}$/i.test(code))) throw new Error("Invalid closed Claude qualification receipt; private diagnostics withheld"); + return r; +} +export type ClaudePrivateRequest = { operation: "prepare-account"; credential: string; harnessExecutable: string; model: string } | { operation: "validate-saved"; agentId: string; connectionId: string; grantId: string }; +/** Only the selected per-attempt key/session crosses the private0600 file. + * No Test request API, browser arguments, argv, env or raw child diagnostic gets it. */ +export async function privateClaudeOperation(input: ClaudePrivateRequest & { companyId: string; environmentId: string; baseURL: string; cookies: Cookie[] }, scanSecret = input.operation === "prepare-account" ? input.credential : undefined) { + if (!scanSecret || scanSecret.length < 12) throw new Error("Claude private operation requires the actual selected credential for private receipt scanning"); + const directory = await mkdtemp(path.join(tmpdir(), "paperclip-claude-private-")); + try { + const request = path.join(directory, "request.json"); await writeFile(request, JSON.stringify(input), { mode: 0o600 }); + const loader = fileURLToPath(new URL("../../../server/node_modules/tsx/dist/loader.mjs", import.meta.url)), worker = fileURLToPath(new URL("./runtime-claude-private-worker.ts", import.meta.url)); + let output: string; + try { output = (await promisify(execFile)(process.execPath, ["--import", loader, worker, request], { timeout: 180000, maxBuffer: 32768, + env: { PATH: process.env.PATH ?? "", HOME: directory, TMPDIR: directory, NODE_ENV: "test", DO_NOT_TRACK: "1" } })).stdout; } + catch { throw new Error("Selected Claude public setup/validation failed; private request and child diagnostics withheld"); } + if ([scanSecret, encodeURIComponent(scanSecret), Buffer.from(scanSecret).toString("base64")].some(value => output.includes(value))) throw new Error("Claude private child exceeded its secret-free receipt boundary; output withheld"); + let result: unknown; try { result = JSON.parse(output); } catch { throw new Error("Claude private child returned no closed qualification receipt"); } + return validateClaudePrivateReceipt(result, input); + } finally { await rm(directory, { recursive: true, force: true }); } +}