diff --git a/tests/runner-e2e/chat-catalog-domains/external-personal-secret-prerequisite.test.ts b/tests/runner-e2e/chat-catalog-domains/external-personal-secret-prerequisite.test.ts new file mode 100644 index 00000000000..14dad8dacc7 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-personal-secret-prerequisite.test.ts @@ -0,0 +1,74 @@ +import { afterEach, expect, it, vi } from "vitest"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { personalSecretExternalCases, personalSecretUnboundConfigHash, preparePersonalSecretWorker, provePersonalSecretMissingBoundary } from "./external-personal-secret.js"; +afterEach(() => { vi.useRealTimers(); vi.unstubAllGlobals(); }); +function fixture() { + vi.useFakeTimers(); vi.setSystemTime(100); + const config = { provider: "opencode", model: "openrouter/deepseek/deepseek-v4.1-flash-1017", env: { OPENROUTER_API_KEY: { type: "secret_ref", secretId: "controller", version: "latest" } } }; + const ready = { id: "morgan", companyId: "company", lifecycleState: "ready", lifecycleHolds: [] as string[], lifecycleError: null as string | null, status: "idle", adapterType: "paperclip_runner", adapterConfig: config }; + const agent = { ...ready, adapterConfig: { ...config, env: { ...config.env, PUBLISHING_API_KEY: { type: "user_secret_ref", key: "PERSONAL_PUBLISHING_KEY", version: "latest", required: true, allowMissingOverride: false } } } }; + const lead = { ...ready, id: "ceo" }, evidence = vi.fn(); + const get = vi.fn(async (route: string) => route === "/api/agents/ceo" ? lead : ready); + const ctx = { observationDeadlineAt: 10000, input: { api: { baseURL: "http://127.0.0.1:3100", get }, evidence, fixtures: { company: { id: "company" }, agent: { id: "ceo" } }, page: { context: () => ({ cookies: async () => [{ name: "session", value: "synthetic-private-session" }] }) } } } as unknown as ChatCatalogContext; + const state = { morganId: "morgan", morganReadyBeforeRequiredBinding: true, morganUnboundConfigHash: personalSecretUnboundConfigHash(config), definitionId: "definition", currentPersonId: "person" }; + const fetcher = vi.fn(async (_input: string | URL | Request, _init?: RequestInit) => new Response(JSON.stringify({ code: "user_secret_missing", details: { code: "user_secret_missing", definitionId: "definition", responsibleUserId: "person" } }), { status: 422 })); + vi.stubGlobal("fetch", fetcher); + return { ctx, ready, agent, lead, state, get, fetcher, evidence }; +} +it("preserves the exact missing-value scenario instead of demanding every configured worker ready", () => { + expect(personalSecretExternalCases[0]!.setupPolicy).toEqual({ readiness: "preserve-scenario", invariantCheckIds: ["current-owner-missing-required-personal-value"] }); + expect(personalSecretExternalCases[0]!.verifySetup).toBeTypeOf("function"); +}); +it("proves the worker ready before adding the missing required value and does no provider probe", async () => { + const f = fixture(); await expect(preparePersonalSecretWorker(f.ctx, "morgan")).resolves.toEqual(f.ready); + expect(f.fetcher).not.toHaveBeenCalled(); + f.ready.lifecycleError = "Authentication failed"; + await expect(preparePersonalSecretWorker(f.ctx, "morgan")).rejects.toThrow("unrelated setup failure"); +}); +it("requires exact current-person missing-secret resolution and ready CEO with unchanged worker configuration", async () => { + const f = fixture(); await provePersonalSecretMissingBoundary(f.ctx, f.state, f.agent); + expect(f.fetcher).toHaveBeenCalledOnce(); + expect(f.fetcher.mock.calls[0]?.[0]).toEqual(new URL("http://127.0.0.1:3100/api/companies/company/adapters/paperclip_runner/test-environment")); + expect(f.evidence).toHaveBeenCalledWith("personal-secret-missing-binding-prerequisite.json", expect.objectContaining({ status: 422, code: "user_secret_missing", prospectiveResolutionOnly: true })); + expect(JSON.stringify(f.evidence.mock.calls)).not.toContain("synthetic-private-session"); +}); +it.each(["bad CEO", "bad worker", "budget hold", "wrong company", "changed model", "optional", "override", "wrong definition", "never ready"])("rejects unrelated %s before any resolution request", async invalid => { + const f = fixture(); + if (invalid === "bad CEO") f.lead.lifecycleError = "Authentication failed"; + if (invalid === "bad worker") f.agent.lifecycleError = "Authentication failed"; + if (invalid === "budget hold") f.agent.lifecycleHolds = ["budget"]; + if (invalid === "wrong company") f.agent.companyId = "foreign"; + if (invalid === "changed model") f.agent.adapterConfig.model = "other"; + if (invalid === "optional") f.agent.adapterConfig.env.PUBLISHING_API_KEY.required = false; + if (invalid === "override") f.agent.adapterConfig.env.PUBLISHING_API_KEY.allowMissingOverride = true; + if (invalid === "wrong definition") f.agent.adapterConfig.env.PUBLISHING_API_KEY.key = "OTHER"; + if (invalid === "never ready") f.state.morganReadyBeforeRequiredBinding = false; + await expect(provePersonalSecretMissingBoundary(f.ctx, f.state, f.agent)).rejects.toThrow(); + expect(f.fetcher).not.toHaveBeenCalled(); expect(f.evidence).not.toHaveBeenCalled(); +}); +it.each(["other status", "other code", "other definition", "other person", "missing details"])("refuses %s rather than treating any failure as the intended prerequisite", async invalid => { + const f = fixture(), body = { code: "user_secret_missing", details: { code: "user_secret_missing", definitionId: "definition", responsibleUserId: "person" } }; + if (invalid === "other code") body.code = "harness_auth_failed"; + if (invalid === "other definition") body.details.definitionId = "other"; + if (invalid === "other person") body.details.responsibleUserId = "other"; + if (invalid === "missing details") delete (body as { details?: unknown }).details; + f.fetcher.mockImplementation(async () => new Response(JSON.stringify(body), { status: invalid === "other status" ? 500 : 422 })); + await expect(provePersonalSecretMissingBoundary(f.ctx, f.state, f.agent)).rejects.toThrow("exact current-person missing binding"); + expect(f.evidence).not.toHaveBeenCalled(); +}); +it("does not let a slow resolution reset the original observation clock", async () => { + const f = fixture(); f.fetcher.mockImplementation(async () => { vi.setSystemTime(10001); return new Response(JSON.stringify({ code: "user_secret_missing", details: { code: "user_secret_missing", definitionId: "definition", responsibleUserId: "person" } }), { status: 422 }); }); + await expect(provePersonalSecretMissingBoundary(f.ctx, f.state, f.agent)).rejects.toThrow("budget"); + expect(f.evidence).not.toHaveBeenCalled(); +}); + +it("accepts only the source-owned missing-binding lifecycle state once the precise resolution guard proves it", async () => { + const f = fixture(); f.agent.lifecycleState = "verifying"; f.agent.status = "paused"; + f.agent.lifecycleError = "Execution configuration check failed. Paperclip could not complete this step. It will retry automatically. If this continues, contact your instance administrator."; + await expect(provePersonalSecretMissingBoundary(f.ctx, f.state, f.agent)).resolves.toBeUndefined(); +}); +it("does not expose malformed private response contents", async () => { + const f = fixture(); f.fetcher.mockImplementation(async () => new Response("synthetic-private-invalid-json", { status: 422 })); + await expect(provePersonalSecretMissingBoundary(f.ctx, f.state, f.agent)).rejects.toThrow("contents withheld"); + expect(f.evidence).not.toHaveBeenCalled(); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts b/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts index 3fc4143e663..00d8be92c96 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-personal-secret.ts @@ -1,6 +1,6 @@ -import { randomBytes } from "node:crypto"; +import { createHash, randomBytes } from "node:crypto"; import { expect } from "@playwright/test"; -import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { chatCatalogAgentIsReady, type ChatCatalogContext } from "../chat-catalog-contract.js"; import type { CompanySecret, UserSecretDefinition, @@ -13,6 +13,7 @@ import { openStandingChat, probeAgentPermission, waitForReply, + remainingIdentityObservationBudget, } from "./identity-support.js"; import { createAuthenticatedPerson, @@ -47,6 +48,43 @@ interface PrivateFixture { endpoint: Awaited>; } const fixtures = new WeakMap(); +interface PersonalSetupAgent { id: string; companyId: string; lifecycleState: string; status: string; lifecycleError?: string | null; lifecycleHolds: string[]; adapterType: string; adapterConfig: Record } +const missingBindingLifecycleError = "Execution configuration check failed. Paperclip could not complete this step. It will retry automatically. If this continues, contact your instance administrator."; +export function personalSecretUnboundConfigHash(config: Record): string { + const env = { ...(config.env as Record | undefined) }; delete env.PUBLISHING_API_KEY; + return createHash("sha256").update(JSON.stringify({ ...config, env })).digest("hex"); +} +/** A configured worker must be healthy before introducing the intentional missing value. */ +export async function preparePersonalSecretWorker(ctx: ChatCatalogContext, id: string): Promise { + const deadlineAt = Date.now() + Math.min(45_000, remainingIdentityObservationBudget(ctx)); + const agent = await pollUntil({ label: "personal-secret worker ready before required binding", deadlineAt, intervalMs: 500, + load: () => ctx.input.api.get(`/api/agents/${id}`), + accept: value => chatCatalogAgentIsReady(value), reject: value => value.lifecycleError ? "Personal-secret worker has an unrelated setup failure before its required binding" : undefined }); + remainingIdentityObservationBudget(ctx); + if (agent.id !== id || agent.companyId !== ctx.input.fixtures.company.id || agent.adapterType !== "paperclip_runner" || Object.hasOwn(agent.adapterConfig.env ?? {}, "PUBLISHING_API_KEY")) throw new Error("Personal-secret prerequisite worker identity or initial binding differs"); + return agent; +} +/** This is a prospective saved-config resolution check. The exact missing user + * secret response is raised before adapter execution or environment acquisition. */ +export async function provePersonalSecretMissingBoundary(ctx: ChatCatalogContext, state: Record, agent: PersonalSetupAgent): Promise { + remainingIdentityObservationBudget(ctx); + const lead = await ctx.input.api.get(`/api/agents/${ctx.input.fixtures.agent.id}`); + if (!chatCatalogAgentIsReady(lead) || lead.id !== ctx.input.fixtures.agent.id || lead.companyId !== ctx.input.fixtures.company.id || lead.lifecycleError) throw new Error("Personal-secret CEO must remain independently ready"); + const ref = (agent.adapterConfig.env as Record | undefined)?.PUBLISHING_API_KEY as { type?: string; key?: string; required?: boolean; allowMissingOverride?: boolean } | undefined; + if (!["ready", "preparing", "verifying", "pausing"].includes(agent.lifecycleState) || !Array.isArray(agent.lifecycleHolds) || agent.lifecycleHolds.some(hold => hold !== "setup") || state.morganReadyBeforeRequiredBinding !== true || agent.id !== state.morganId || agent.companyId !== ctx.input.fixtures.company.id || agent.adapterType !== "paperclip_runner" || personalSecretUnboundConfigHash(agent.adapterConfig) !== state.morganUnboundConfigHash || ref?.type !== "user_secret_ref" || ref.key !== "PERSONAL_PUBLISHING_KEY" || ref.required !== true || ref.allowMissingOverride !== false || (agent.lifecycleError && agent.lifecycleError !== missingBindingLifecycleError)) throw new Error("Personal-secret worker has an unrelated lifecycle or configuration failure"); + const origin = ctx.input.api.baseURL, cookies = await ctx.input.page.context().cookies(origin); + if (!cookies.length) throw new Error("Personal-secret setup probe requires the real current-person session"); + const response = await fetch(new URL(`/api/companies/${ctx.input.fixtures.company.id}/adapters/paperclip_runner/test-environment`, origin), { method: "POST", redirect: "error", signal: AbortSignal.timeout(Math.min(15_000, remainingIdentityObservationBudget(ctx))), headers: { "content-type": "application/json", Origin: origin, Cookie: cookies.map(cookie => `${cookie.name}=${cookie.value}`).join("; ") }, body: JSON.stringify({ agentId: agent.id, adapterConfig: agent.adapterConfig }) }); + try { + let body: { code?: string; details?: { code?: string; definitionId?: string; responsibleUserId?: string } }; + try { body = await response.json(); } catch { throw new Error("Personal-secret setup response could not be verified; contents withheld"); } + if (!body || typeof body !== "object" || response.status !== 422 || body.code !== "user_secret_missing" || body.details?.code !== "user_secret_missing" || body.details.definitionId !== state.definitionId || body.details.responsibleUserId !== state.currentPersonId) throw new Error("Personal-secret setup did not prove the exact current-person missing binding"); + } finally { if (!response.bodyUsed) await response.body?.cancel(); } + remainingIdentityObservationBudget(ctx); + state.currentOwnerMissingBindingVerified = true; + await ctx.input.evidence("personal-secret-missing-binding-prerequisite.json", { agentId: agent.id, definitionId: state.definitionId, responsibleUserId: state.currentPersonId, status: 422, code: "user_secret_missing", unchangedWorkerConfiguration: true, controllerReady: true, prospectiveResolutionOnly: true }); +} + export function personalSecretGuidance(reply: string) { return ( /personal|your own|my secrets/i.test(reply) && @@ -235,9 +273,7 @@ export const personalSecretExternalCases = [ "Personal publishing definition must initially lack the current person's value", ); const morgan = await identityWorker(ctx), - agent = await ctx.input.api.get<{ - adapterConfig: Record; - }>(`/api/agents/${morgan.id}`); + agent = await preparePersonalSecretWorker(ctx, morgan.id); await ctx.input.api.patch(`/api/agents/${morgan.id}`, { adapterConfig: { ...agent.adapterConfig, @@ -277,6 +313,8 @@ export const personalSecretExternalCases = [ description: `This request concerns publishing task ${task.identifier ?? task.id}, assigned to Morgan. It is in backlog because the current responsible person has not set the required Personal publishing credential definition ${definition.id} (PERSONAL_PUBLISHING_KEY). Morgan's required PUBLISHING_API_KEY user-secret reference is already configured. Riley is another company member with their own separate personal value.`, }); return { + morganReadyBeforeRequiredBinding: true, + morganUnboundConfigHash: personalSecretUnboundConfigHash(agent.adapterConfig), definitionId: definition.id, currentPersonId: session.user.id, otherPersonId: other.id, @@ -307,6 +345,7 @@ export const personalSecretExternalCases = [ gradePersonalSecret, { authentication: "human-session", + setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["current-owner-missing-required-personal-value"] }, expectedRunCount: 3, minimumRunCount: 3, expectedHumanMessageCount: 2, @@ -340,9 +379,7 @@ export const personalSecretExternalCases = [ session = await ctx.input.api.get<{ user: { id: string } }>( "/api/auth/get-session", ), - agent = await ctx.input.api.get<{ - adapterConfig: { env?: Record }; - }>(`/api/agents/${state.morganId}`), + agent = await ctx.input.api.get(`/api/agents/${state.morganId}`), ref = agent.adapterConfig.env?.PUBLISHING_API_KEY as | { type?: string; @@ -351,6 +388,7 @@ export const personalSecretExternalCases = [ allowMissingOverride?: boolean; } | undefined; + await provePersonalSecretMissingBoundary(ctx, state, agent); state.requiredReferenceBefore = ref?.type === "user_secret_ref" && ref.key === "PERSONAL_PUBLISHING_KEY" &&