From 0d22c3cc1df41ae3e6614cab087ad7854a691e93 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 26 Oct 2026 17:52:29 -0500 Subject: [PATCH] test(e2e): cover secure keys, held resume and draft upload recovery --- .../runtime-agent-keys.test.ts | 24 ++++ .../runtime-agent-keys.ts | 85 ++++++++++++++ .../runtime-capability-audit.json | 6 +- .../runtime-resume.test.ts | 26 +++++ .../chat-catalog-domains/runtime-resume.ts | 86 ++++++++++++++ .../runtime-steering.test.ts | 29 +++++ .../chat-catalog-domains/runtime-steering.ts | 108 ++++++++++++++++++ .../chat-catalog-domains/runtime-support.ts | 2 +- 8 files changed, 362 insertions(+), 4 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-agent-keys.test.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-agent-keys.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-resume.test.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-resume.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-steering.test.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-steering.ts diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-agent-keys.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-agent-keys.test.ts new file mode 100644 index 00000000000..72c08f00940 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-agent-keys.test.ts @@ -0,0 +1,24 @@ +import { it, expect } from "vitest"; +import { gradeAgentKey } from "./runtime-agent-keys.js"; +function valid() { + const worker = { id: "morgan", companyId: "company", name: "Morgan", permissions: {}, adapterConfig: {} }; + const key = { id: "key", name: "local integration", revokedAt: null as string | null }; + return { scenarioId: "agents.agent-api-key", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", + state: { worker, agentCreateStatus: 403, keysBeforeHuman: 0, humanCreated: true, createdKey: key, createdKeyAgentId: "morgan", createdKeyCompanyId: "company", + secureUntracedContext: true, tokenShownOnce: true, tokenAbsentAfterReload: true, privateScanComplete: true, tokenLeaked: false, + revokedCredentialStatus: 401, finalKeys: [{ ...key, revokedAt: "2026-10-26T00:00:00Z" }] }, + agents: [structuredClone(worker)], tasks: [], links: [], reply: "Use Morgan's profile API key controls; the board can create the credential securely.", + activity: ["agent.key_created", "agent.key_revoked"].map(action => ({ id: action, action, entityId: "morgan", actorType: "user", details: { keyId: "key" } })), + runs: [], humanQueries: [] }; +} +it("accepts secure one-time human creation and actual credential revocation", () => expect(gradeAgentKey(valid()).passed).toBe(true)); +it("rejects missing human credential evidence", () => expect(gradeAgentKey(undefined).passed).toBe(false)); +it("rejects a key that authenticates a different agent", () => { + const o = valid(); o.state.createdKeyAgentId = "other"; expect(gradeAgentKey(o).passed).toBe(false); +}); +it("rejects revocation metadata while the credential still works", () => { + const o = valid(); o.state.revokedCredentialStatus = 200; expect(gradeAgentKey(o).passed).toBe(false); +}); +it("rejects credentials copied into conversation artifacts", () => { + const o = valid(); o.state.tokenLeaked = true; expect(gradeAgentKey(o).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-agent-keys.ts b/tests/runner-e2e/chat-catalog-domains/runtime-agent-keys.ts new file mode 100644 index 00000000000..625102b1e99 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-agent-keys.ts @@ -0,0 +1,85 @@ +import { expect } from "@playwright/test"; +import { checks } from "./internal-support.js"; +import { waitForReply, probeAgentPermission } from "./identity-support.js"; +import { runtimeDefinition, runtimeWorker, runtimeObservation, observeRuntime, agentLink, unchangedAgent, type RuntimeAgent } from "./runtime-support.js"; +interface KeyRow { id: string; name: string; revokedAt: string | null } +export function gradeAgentKey(value: unknown) { + const c = checks(); const o = runtimeObservation(value); + c.add("independent-human-key-evidence", !!o, "Real human key controls, safe metadata readback and private credential scans are required."); if (!o) return c.result(); + const worker = o.state.worker as RuntimeAgent; const current = o.agents.find(row => row.id === worker.id); + const key = o.state.createdKey as KeyRow | undefined; const revoked = (o.state.finalKeys as KeyRow[] | undefined)?.find(row => row.id === key?.id); + c.add("human-only-creation", o.state.agentCreateStatus === 403 && o.state.keysBeforeHuman === 0 && o.state.humanCreated === true + && !!key && o.state.createdKeyAgentId === worker.id && o.state.createdKeyCompanyId === o.companyId && key.name === "local integration" && !key.revokedAt, + "Only the actual authorized human creates the intended named key for Morgan."); + c.add("one-time-private-interface", o.state.secureUntracedContext === true && o.state.tokenShownOnce === true + && o.state.tokenAbsentAfterReload === true && o.state.privateScanComplete === true && o.state.tokenLeaked === false, + "The one-time display is observed privately, disappears on reload, and never appears in conversation evidence."); + c.add("revocation-preserves-agent-identity", !!revoked?.revokedAt && o.state.revokedCredentialStatus === 401 && !!current && unchangedAgent(worker, current), + "The same key is revoked through the human control and Morgan's identity, runtime and authority remain unchanged."); + c.add("attributed-key-lifecycle", !!key && ["agent.key_created", "agent.key_revoked"].every(action => o.activity.some(row => row.action === action + && row.entityId === worker.id && row.actorType === "user" && !row.agentId && row.details?.keyId === key.id)), + "Independent activity records attribute both credential operations to the human."); + c.add("honest-secure-human-path", /key|credential/i.test(o.reply) && /profile|control|you|human|board/i.test(o.reply) + && !/I(?:'ve| have)? (?:created|generated|issued)/i.test(o.reply), "The reply provides the supported secure human path without claiming an agent-created credential."); + return c.result(); +} +export const runtimeAgentKeyCases = [runtimeDefinition("agents.agent-api-key", async ctx => ({ worker: await runtimeWorker(ctx) }), gradeAgentKey, { + authentication: "human-session", + sourceFiles: ["chat-catalog-domains/runtime-agent-keys.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agents.yaml"], + async act(ctx, state) { + await waitForReply(ctx); const worker = state.worker as RuntimeAgent; + const secrets = ctx.input.privateSecrets; const browser = ctx.input.page.context().browser(); + if (!secrets || !browser) throw new Error("Credential UI requires a private attempt sink and untraced real browser context"); + const endpoint = `/api/agents/${worker.id}/keys`; + state.agentCreateStatus = await probeAgentPermission(ctx, endpoint, "POST", { name: "local integration" }); + state.keysBeforeHuman = (await ctx.input.api.get(endpoint)).length; + const o = await observeRuntime(ctx, state); const href = agentLink(o, worker); if (!href) throw new Error("Credential guidance lacks Morgan's exact profile"); + // This isolated context deliberately never starts tracing or screenshots. + // It carries the same real human session; credentials are never exposed to + // the traced source page, evidence callbacks or model conversation. + const secure = await browser.newContext({ baseURL: ctx.input.api.baseURL }); + await secure.addCookies(await ctx.input.page.context().cookies()); + const page = await secure.newPage(); + try { + await page.goto(new URL(href, ctx.input.api.baseURL).href, { waitUntil: "domcontentloaded" }); + await page.getByRole("link", { name: /API keys/i }).click(); + await page.getByPlaceholder("Key name (e.g. production)").fill("local integration"); + await page.getByRole("button", { name: "Create", exact: true }).click(); + await expect(page.getByText("API key created — copy it now, it will not be shown again.", { exact: true })).toBeVisible(); + const token = (await page.locator("code").filter({ hasText: /^pcp_/ }).first().textContent())?.trim(); + if (!token || token.length < 20) throw new Error("Secure one-time UI did not reveal a usable credential"); + secrets.push(token); + const identityResponse = await fetch(new URL("/api/agents/me", ctx.input.api.baseURL), { + redirect: "error", signal: AbortSignal.timeout(15_000), headers: { Authorization: `Bearer ${token}` }, + }); + if (!identityResponse.ok) { await identityResponse.body?.cancel(); throw new Error("New human credential did not authenticate Morgan"); } + const identity = await identityResponse.json() as { id: string; companyId: string }; + state.createdKeyAgentId = identity.id; state.createdKeyCompanyId = identity.companyId; + state.tokenShownOnce = true; state.secureUntracedContext = true; + const created = (await ctx.input.api.get(endpoint)).filter(row => row.name === "local integration" && !row.revokedAt); + if (created.length !== 1) throw new Error("Human credential control did not create exactly one named key"); + state.createdKey = created[0]; state.humanCreated = true; + await page.getByRole("button", { name: "Dismiss", exact: true }).click(); + await page.reload({ waitUntil: "domcontentloaded" }); + state.tokenAbsentAfterReload = !(await page.locator("body").textContent())?.includes(token); + const row = page.getByText("local integration", { exact: true }).locator("..").locator(".."); + await row.getByRole("button", { name: "Revoke", exact: true }).click(); + await expect(page.getByText("Revoked Keys", { exact: true })).toBeVisible(); + const denied = await fetch(new URL("/api/agents/me", ctx.input.api.baseURL), { + redirect: "error", signal: AbortSignal.timeout(15_000), headers: { Authorization: `Bearer ${token}` }, + }); + state.revokedCredentialStatus = denied.status; await denied.body?.cancel(); + const source = await observeRuntime(ctx, state); + state.privateScanComplete = true; state.tokenLeaked = source.reply.includes(token) + || source.comments.some(comment => comment.body?.includes(token)); + } finally { await secure.close(); } + }, + async observe(ctx, state) { + state.finalKeys = await ctx.input.api.get(`/api/agents/${(state.worker as RuntimeAgent).id}/keys`); + return observeRuntime(ctx, state); + }, + async cleanup(ctx, state) { + const key = state.createdKey as KeyRow | undefined; + if (key) await ctx.input.api.delete(`/api/agents/${(state.worker as RuntimeAgent).id}/keys/${key.id}`, { allowNotFound: true }); + }, +})]; diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-capability-audit.json b/tests/runner-e2e/chat-catalog-domains/runtime-capability-audit.json index 7f93c23949d..3751318aebc 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-capability-audit.json +++ b/tests/runner-e2e/chat-catalog-domains/runtime-capability-audit.json @@ -56,7 +56,7 @@ }, { "scenarioId": "agents.resume-agent", - "readinessClass": "local", + "readinessClass": "implemented", "needs": "Public manual pause plus independently preserved readiness; permitted resume and separate remaining-gate branch." }, { @@ -71,7 +71,7 @@ }, { "scenarioId": "agents.agent-api-key", - "readinessClass": "local", + "readinessClass": "implemented", "needs": "Human-session credential UI; one-time token privateSecrets sink, named-key public readback and revoke with identity retained." }, { @@ -236,7 +236,7 @@ }, { "scenarioId": "steering.preserve-draft", - "readinessClass": "source", + "readinessClass": "implemented", "needs": "Unsent canonical query and completed upload receipt must survive task switch/reload before send; shared flow sends original query immediately after setup." }, { diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-resume.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-resume.test.ts new file mode 100644 index 00000000000..3c696a4d809 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-resume.test.ts @@ -0,0 +1,26 @@ +import { it, expect } from "vitest"; +import { gradeResumeAgent } from "./runtime-resume.js"; +function valid() { + const worker = { id: "morgan", companyId: "company" }; + return { scenarioId: "agents.resume-agent", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", + state: { worker, firstResumedWorker: { ...worker, lifecycleState: "ready", status: "idle", lifecycleHolds: [] }, + firstReply: "Morgan is resumed and available for future work.", secondBudgetHardStop: true, secondBudgetAmount: 0, + secondReply: "The budget cap remains a hold; Morgan cannot pick up work before it is resolved." }, + agents: [{ ...worker, status: "paused", lifecycleHolds: ["budget"] }], tasks: [], links: ["/ACORN/agents/morgan"], reply: "", + activity: [{ id: "resume", action: "agent.resumed", entityId: "morgan", agentId: "ceo" }], + runs: [{ id: "first", agentId: "ceo", status: "succeeded" }, { id: "second", agentId: "ceo", status: "succeeded" }], humanQueries: [] }; +} +it("accepts both saved availability and independently bound held-phase evidence", () => expect(gradeResumeAgent(valid()).passed).toBe(true)); +it("rejects missing resume evidence", () => expect(gradeResumeAgent(undefined).passed).toBe(false)); +it("rejects first-phase prose borrowed to fake a successful saved resume", () => { + const o = valid(); o.state.firstResumedWorker.status = "paused"; expect(gradeResumeAgent(o).passed).toBe(false); +}); +it("rejects using the first reply instead of evidence from the held request", () => { + const o = valid(); o.state.secondReply = o.state.firstReply; expect(gradeResumeAgent(o).passed).toBe(false); +}); +it("rejects silently clearing the remaining budget gate", () => { + const o = valid(); o.agents[0]!.lifecycleHolds = []; expect(gradeResumeAgent(o).passed).toBe(false); +}); +it("rejects pretending a new worker execution already started", () => { + const o = valid(); o.runs.push({ id: "worker", agentId: "morgan", status: "succeeded" }); expect(gradeResumeAgent(o).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-resume.ts b/tests/runner-e2e/chat-catalog-domains/runtime-resume.ts new file mode 100644 index 00000000000..fc967297f02 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-resume.ts @@ -0,0 +1,86 @@ +import { pollUntil } from "../api.js"; +import { sendChatMessage } from "../chat-flow.js"; +import { checks, type Entity } from "./internal-support.js"; +import { runtimeDefinition, runtimeWorker, runtimeObservation, observeRuntime, agentLink, type RuntimeAgent } from "./runtime-support.js"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +interface PhaseComment extends Entity { } +export function gradeResumeAgent(value: unknown) { + const c = checks(); const o = runtimeObservation(value); + c.add("independent-resume-phases", !!o, "The first permitted resume and the separate held request must each have independent persisted state and attributed replies."); if (!o) return c.result(); + const worker = o.state.worker as RuntimeAgent; const first = o.state.firstResumedWorker as RuntimeAgent | undefined; + const current = o.agents.find(row => row.id === worker.id); + c.add("first-request-durably-resumed", first?.id === worker.id && first.companyId === o.companyId && first.lifecycleState === "ready" + && ["idle", "active"].includes(first.status ?? "") && !first.lifecycleHolds?.length + && o.activity.some(row => row.action === "agent.resumed" && row.entityId === worker.id && row.agentId === o.leadId), + "The first actual permitted operation made the same Morgan available and has independent agent attribution."); + c.add("first-reply-distinguishes-availability", typeof o.state.firstReply === "string" && /resum|available|ready/i.test(o.state.firstReply) + && !/started (?:a |the )?(?:run|task|work)|already (?:working|running)/i.test(o.state.firstReply), + "The unheld phase describes availability without inventing a new worker execution."); + c.add("real-budget-gate-preserved", current?.lifecycleHolds?.includes("budget") === true && current.status === "paused" + && o.state.secondBudgetHardStop === true && o.state.secondBudgetAmount === 0, + "The second branch retains its independently observed effective budget hold and saved hard stop."); + c.add("held-reply-bound-to-second-request", typeof o.state.secondReply === "string" && /budget|spend|cap|limit/i.test(o.state.secondReply) + && /hold|block|cannot|can't|remain|before|needs|not.*resum/i.test(o.state.secondReply) + && !/Morgan (?:is|has) (?:now )?(?:working|running)|I(?:'ve| have)? (?:started|resumed).*work/i.test(o.state.secondReply), + "Only replies from the new actual run after the second human comment can explain the remaining gate."); + c.add("no-fabricated-worker-start", o.runs.length === 2 && o.runs.every(run => run.agentId === o.leadId && run.status === "succeeded") && o.tasks.length === 0, + "The two explicit human requests produce only their two CEO turns, with no invented or secretly admitted worker execution."); + c.add("inspectable-same-agent", !!current && !!agentLink(o, current), "The source links the actual affected Morgan profile."); + return c.result(); +} +async function sourcePhase(ctx: ChatCatalogContext) { + await ctx.refreshIssue(); const source = ctx.issue(); if (!source) throw new Error("Resume phase lost its source conversation"); + const runs = await ctx.allRuns(); + const comments = await ctx.input.api.get(`/api/issues/${source.id}/comments?order=asc`); + return { runs, comments, source }; +} +export const runtimeResumeCases = [runtimeDefinition("agents.resume-agent", async ctx => { + const worker = await runtimeWorker(ctx); + await ctx.input.api.post(`/api/agents/${worker.id}/pause`, {}); + const paused = await pollUntil({ label: "Genuine manual Morgan pause", deadlineAt: Date.now() + 45_000, + load: () => ctx.input.api.get(`/api/agents/${worker.id}`), + accept: row => row.lifecycleState === "paused" && row.lifecycleHolds?.includes("manual") === true }); + return { worker: paused }; +}, gradeResumeAgent, { + sourceFiles: ["chat-catalog-domains/runtime-resume.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agents.yaml"], + setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["lead-ready", "manual-pause-only"] }, + expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 540_000, + async verifySetup(ctx, state) { + const c = checks(); const lead = await ctx.input.api.get(`/api/agents/${ctx.input.fixtures.agent.id}`); + const worker = await ctx.input.api.get(`/api/agents/${(state.worker as RuntimeAgent).id}`); + c.add("lead-ready", lead.lifecycleState === "ready" && ["idle", "active"].includes(lead.status ?? ""), "The real source agent can receive the query."); + c.add("manual-pause-only", worker.lifecycleState === "paused" && worker.status === "paused" + && worker.lifecycleHolds?.length === 1 && worker.lifecycleHolds[0] === "manual", "No budget, approval, setup or company hold replaces the catalog's first manual-only pause."); + return c.result(); + }, + async act(ctx, state) { + const worker = state.worker as RuntimeAgent; + const first = await pollUntil({ label: "Permitted resume phase settled and saved", deadlineAt: Date.now() + 150_000, + load: async () => ({ ...await sourcePhase(ctx), worker: await ctx.input.api.get(`/api/agents/${worker.id}`) }), + accept: value => value.source.conversationState === "waiting" && value.runs.length === 1 && value.runs[0]?.status === "succeeded" + && value.worker.lifecycleState === "ready" && ["idle", "active"].includes(value.worker.status ?? "") }); + state.firstResumedWorker = first.worker; + state.firstReply = first.comments.filter(row => row.authorAgentId === ctx.input.fixtures.agent.id && row.createdByRunId === first.runs[0]!.id).map(row => row.body ?? "").join("\n"); + state.firstRunIds = first.runs.map(run => run.id); state.firstCommentIds = first.comments.map(row => row.id); + await ctx.input.api.post(`/api/agents/${worker.id}/pause`, {}); + await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/budgets/policies`, { scopeType: "agent", scopeId: worker.id, + amount: 0, isActive: true, hardStopEnabled: true, unpricedUsagePolicy: "block" }); + await pollUntil({ label: "Independent effective budget hold before the second explicit request", deadlineAt: Date.now() + 45_000, + load: () => ctx.input.api.get(`/api/agents/${worker.id}`), accept: row => row.lifecycleHolds?.includes("budget") === true }); + await sendChatMessage(ctx.input.page, ctx.scenario.query); + }, + async observe(ctx, state) { + const phase = await sourcePhase(ctx); + if (Array.isArray(state.firstRunIds)) { + const oldComments = state.firstCommentIds as string[]; + const secondHuman = phase.comments.find(row => !row.authorAgentId && row.body === ctx.scenario.query && !oldComments.includes(row.id)); + state.secondReply = secondHuman ? phase.comments.filter(row => row.authorAgentId === ctx.input.fixtures.agent.id + && !oldComments.includes(row.id) && phase.runs.some(run => run.id === row.createdByRunId && !(state.firstRunIds as string[]).includes(run.id)) + && Date.parse(row.createdAt ?? "") >= Date.parse(secondHuman.createdAt ?? "")).map(row => row.body ?? "").join("\n") : ""; + } + const budget = await ctx.input.api.get<{ policies: Array<{ scopeId: string; scopeType: string; amount: number; hardStopEnabled: boolean }> }>(`/api/companies/${ctx.input.fixtures.company.id}/budgets/overview`); + const policy = budget.policies.find(row => row.scopeId === (state.worker as RuntimeAgent).id && row.scopeType === "agent"); + state.secondBudgetAmount = policy?.amount; state.secondBudgetHardStop = policy?.hardStopEnabled; + return observeRuntime(ctx, state); + }, +})]; diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-steering.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-steering.test.ts new file mode 100644 index 00000000000..9c805d1965a --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-steering.test.ts @@ -0,0 +1,29 @@ +import { createHash } from "node:crypto"; +import { it, expect } from "vitest"; +import { gradePreserveDraft } from "./runtime-steering.js"; +function valid() { + const briefBody = "# Real website brief\n"; const hash = createHash("sha256").update(briefBody).digest("hex"); + const attachment = { id: "upload", issueId: "source", companyId: "company", createdByUserId: "person", createdByAgentId: null, sha256: hash }; + const otherTask = { id: "other", title: "Other task", status: "backlog" }; + return { scenarioId: "steering.preserve-draft", companyId: "company", leadId: "ceo", + state: { originalQuery: "Build the page from this brief, but keep the copy short.", restoredDraft: "Build the page from this brief, but keep the copy short.", + restoredAttachmentVisible: true, beforeSendHumanCount: 0, otherTaskDraftIsolated: true, otherTaskAttachmentsEmpty: true, + briefAttachment: attachment, savedAttachment: { ...attachment, issueCommentId: "comment" }, sourceIssueId: "source", briefHash: hash, + briefBody, downloadedBrief: briefBody, transport: { commentId: "comment", replayedCommentId: "comment", clientRequestId: "request" }, + sourceComments: [{ id: "comment", clientRequestId: "request", authorUserId: "person" }], persistedCommentReviewedBeforeReplay: true, otherTask }, + agents: [], tasks: [otherTask], links: [], reply: "", activity: [], runs: [{ id: "run", contextSnapshot: { wakeCommentId: "comment" } }], humanQueries: [] }; +} +it("accepts retained draft, real source-bound upload and one reconciled send", () => expect(gradePreserveDraft(valid()).passed).toBe(true)); +it("rejects missing draft evidence", () => expect(gradePreserveDraft(undefined).passed).toBe(false)); +it("rejects leaking the unsent draft into the other task", () => { + const o = valid(); o.state.otherTaskDraftIsolated = false; expect(gradePreserveDraft(o).passed).toBe(false); +}); +it("rejects an upload attached to a different human comment", () => { + const o = valid(); o.state.savedAttachment.issueCommentId = "other-comment"; expect(gradePreserveDraft(o).passed).toBe(false); +}); +it("rejects duplicate native admission after lost acknowledgement replay", () => { + const o = valid(); o.runs.push({ id: "duplicate", contextSnapshot: { wakeCommentId: "comment" } }); expect(gradePreserveDraft(o).passed).toBe(false); +}); +it("rejects a receipt whose actual downloaded bytes changed", () => { + const o = valid(); o.state.downloadedBrief = "Different brief"; expect(gradePreserveDraft(o).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-steering.ts b/tests/runner-e2e/chat-catalog-domains/runtime-steering.ts new file mode 100644 index 00000000000..4a4143739fb --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-steering.ts @@ -0,0 +1,108 @@ +import { expect } from "@playwright/test"; +import { createHash } from "node:crypto"; +import { pollUntil } from "../api.js"; +import { dropChatSendAcknowledgement } from "../lost-send.js"; +import type { ChatQueryAttachmentReceipt } from "../chat-query-attachments.js"; +import { checks, sameTask, type Entity } from "./internal-support.js"; +import { openStandingChat } from "./identity-support.js"; +import { runtimeDefinition, runtimeWorker, runtimeObservation, observeRuntime } from "./runtime-support.js"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +type Interception = Awaited>; +const interceptions = new WeakMap(); +interface DraftComment extends Entity { clientRequestId?: string } +const editor = (ctx: ChatCatalogContext) => ctx.input.page.getByTestId("task-chat-composer-input").last().locator('[contenteditable="true"], textarea').first(); +async function draftText(ctx: ChatCatalogContext) { + return editor(ctx).evaluate(node => "value" in node ? String(node.value) : node.textContent ?? ""); +} +export function gradePreserveDraft(value: unknown) { + const c = checks(); const o = runtimeObservation(value); + c.add("real-draft-and-upload-evidence", !!o, "The unsent editor text, completed upload, source comment and actual transport receipt are independently required."); if (!o) return c.result(); + const attachment = o.state.briefAttachment as ChatQueryAttachmentReceipt; + const sourceComments = o.state.sourceComments as DraftComment[] | undefined; + const transport = o.state.transport as { commentId?: string; replayedCommentId?: string; clientRequestId?: string } | undefined; + const human = sourceComments?.filter(row => !row.authorAgentId); + c.add("draft-survived-switch-and-reload", o.state.restoredDraft === o.state.originalQuery && o.state.restoredAttachmentVisible === true + && o.state.beforeSendHumanCount === 0 && o.state.otherTaskDraftIsolated === true && o.state.otherTaskAttachmentsEmpty === true, + "The correct source retains the exact unsent text and completed upload while the other task receives neither."); + const canonical = human?.find(row => row.id === transport?.commentId); + const savedAttachment = o.state.savedAttachment as ChatQueryAttachmentReceipt | undefined; + c.add("one-message-with-original-upload", human?.length === 1 && !!canonical && !!attachment && savedAttachment?.id === attachment.id + && savedAttachment.issueCommentId === canonical.id && savedAttachment.issueId === o.state.sourceIssueId + && savedAttachment.companyId === o.companyId && savedAttachment.createdByUserId === canonical.authorUserId + && savedAttachment.createdByAgentId == null && savedAttachment.sha256 === o.state.briefHash, + "One actual human send carries the exact human-owned upload receipt for this source and company."); + c.add("actual-file-bytes-retained", o.state.downloadedBrief === o.state.briefBody && typeof o.state.briefBody === "string" + && createHash("sha256").update(o.state.briefBody).digest("hex") === savedAttachment?.sha256, + "Independent production download bytes match the original brief and receipt hash."); + const requestRuns = transport ? o.runs.filter(run => run.contextSnapshot?.wakeCommentId === transport.commentId + || Array.isArray(run.contextSnapshot?.wakeCommentIds) && run.contextSnapshot.wakeCommentIds.includes(transport.commentId)) : []; + c.add("uncertain-send-reconciled-without-duplication", !!transport?.commentId && transport.commentId === transport.replayedCommentId + && !!transport.clientRequestId && sourceComments?.filter(row => row.clientRequestId === transport.clientRequestId).map(row => row.id).join() === transport.commentId + && requestRuns.length === 1 && o.state.persistedCommentReviewedBeforeReplay === true, + "A real lost acknowledgement is reconciled against the persisted comment before an exact client-request replay; one request-bound run remains."); + c.add("other-task-unchanged", !!o.state.otherTask && sameTask(o.state.otherTask as Entity, o.tasks.find(row => row.id === (o.state.otherTask as Entity).id)), + "Draft navigation and the later send do not mutate the unrelated task."); + return c.result(); +} +export const runtimeSteeringCases = [runtimeDefinition("steering.preserve-draft", async ctx => { + const worker = await runtimeWorker(ctx); const source = await openStandingChat(ctx); + await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); await expect(editor(ctx)).toBeVisible(); + const otherTask = await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/issues`, { title: "Unrelated draft-isolation task", status: "backlog" }); + const sourceUrl = ctx.input.page.url(); const filename = "website-brief.md"; + const briefBody = `# Website brief\nUse short copy, an accessible single-page layout, and a contact section. Product reference: ${ctx.input.nonce}.\n`; + await editor(ctx).fill(ctx.scenario.query); + await ctx.input.page.locator('input[type="file"]').first().setInputFiles({ name: filename, mimeType: "text/markdown", buffer: Buffer.from(briefBody) }); + const briefAttachment = await pollUntil({ label: "Completed public brief upload before unsent navigation", deadlineAt: Date.now() + 30_000, + load: () => ctx.input.api.get(`/api/issues/${source.id}/attachments`), + accept: rows => rows.filter(row => row.originalFilename === filename).length === 1 }); + const receipt = briefAttachment.find(row => row.originalFilename === filename)!; + await expect(ctx.input.page.getByTestId("task-chat-composer-attachments")).toContainText(filename); + await expect(ctx.input.page.getByTestId("task-chat-composer-attachments")).not.toContainText("Uploading"); + await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${otherTask.identifier ?? otherTask.id}`, { waitUntil: "domcontentloaded" }); + await expect(editor(ctx)).toBeVisible(); await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); + await expect(editor(ctx)).toBeVisible(); + const otherTaskDraftIsolated = (await draftText(ctx)) !== ctx.scenario.query; + const otherTaskAttachmentsEmpty = (await ctx.input.api.get(`/api/issues/${otherTask.id}/attachments`)).length === 0 + && !(await ctx.input.page.getByTestId("task-chat-composer-attachments").isVisible()); + await ctx.input.page.goto(sourceUrl, { waitUntil: "domcontentloaded" }); + await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); await expect(editor(ctx)).toBeVisible(); + await expect.poll(() => draftText(ctx)).toBe(ctx.scenario.query); + const restoredDraft = await draftText(ctx); + await expect(ctx.input.page.getByTestId("task-chat-composer-attachments")).toContainText(filename); + const before = await ctx.input.api.get(`/api/issues/${source.id}/comments?order=asc`); + interceptions.set(ctx, await dropChatSendAcknowledgement(ctx.input.page, ctx.scenario.query)); + return { worker, otherTask, sourceIssueId: source.id, originalQuery: ctx.scenario.query, restoredDraft, + restoredAttachmentVisible: true, otherTaskDraftIsolated, otherTaskAttachmentsEmpty, beforeSendHumanCount: before.filter(row => !row.authorAgentId).length, + briefAttachment: receipt, briefBody, briefHash: createHash("sha256").update(briefBody).digest("hex") }; +}, gradePreserveDraft, { + authentication: "human-session", queryAttachments: [{ stateKey: "briefAttachment", filename: "website-brief.md" }], + sourceFiles: ["chat-catalog-domains/runtime-steering.ts", "chat-catalog-domains/runtime-support.ts", "lost-send.ts", "chat-scenarios/steering.yaml"], + expectedRunCount: 3, minimumRunCount: 1, attemptTimeoutMs: 540_000, + runPolicy: { kind: "source-settled-with-active-work", maximumActiveWorkerRuns: 1 }, + async act(ctx, state) { + const interception = interceptions.get(ctx); if (!interception) throw new Error("Lost-ack draft fixture is not armed"); + const sent = await interception.committed; + const comment = await ctx.input.api.get(`/api/issues/${state.sourceIssueId}/comments/${sent.commentId}`); + if (comment.id !== sent.commentId || comment.clientRequestId !== sent.data.clientRequestId) throw new Error("Uncertain send was not reconciled with its actual persisted comment"); + state.persistedCommentReviewedBeforeReplay = true; + await interception.dispose(); interceptions.delete(ctx); + const replayed = await ctx.input.api.post<{ id: string }>(sent.path, sent.data); + state.transport = { commentId: sent.commentId, replayedCommentId: replayed.id, clientRequestId: String(sent.data.clientRequestId) }; + await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); + }, + async observe(ctx, state) { + const source = String(state.sourceIssueId); + state.sourceComments = await ctx.input.api.get(`/api/issues/${source}/comments?order=asc`); + const attachment = (await ctx.input.api.get(`/api/issues/${source}/attachments`)).find(row => row.id === (state.briefAttachment as ChatQueryAttachmentReceipt).id); + state.savedAttachment = attachment; + if (attachment) state.downloadedBrief = Buffer.from(await ctx.input.api.getAttachmentContent(attachment.id)).toString("utf8"); + return observeRuntime(ctx, state); + }, + async verifyResult(ctx, state) { + const transport = state.transport as { commentId: string }; const receipt = state.savedAttachment as ChatQueryAttachmentReceipt; + const rendered = ctx.input.page.locator(`[id=${JSON.stringify(`comment-${transport.commentId}`)}]`); + await expect(rendered).toContainText(ctx.scenario.query); + await expect(rendered.locator(`a[href=${JSON.stringify(receipt.contentPath)}]`).first()).toBeVisible(); + }, + async cleanup(ctx) { const interception = interceptions.get(ctx); interceptions.delete(ctx); await interception?.dispose(); }, +})]; diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-support.ts b/tests/runner-e2e/chat-catalog-domains/runtime-support.ts index 434e840b243..d08d849c2b8 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-support.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-support.ts @@ -7,7 +7,7 @@ import { observeInternal, type Entity, type InternalObservation } from "./intern export interface RuntimeAgent extends Entity { reportsTo?: string | null; adapterType?: string; adapterConfig?: Record; runtimeConfig?: Record; permissions?: Record; - defaultEnvironmentId?: string | null; lifecycleState?: string; + defaultEnvironmentId?: string | null; lifecycleState?: string; lifecycleHolds?: string[]; pauseReason?: string | null; } export interface InstructionFile { path: string; content: string; revision: { id: string; parentRevisionId?: string | null;