diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-configuration.ts b/tests/runner-e2e/chat-catalog-domains/privileged-configuration.ts index 47697070c1b..fa3a654e34e 100644 --- a/tests/runner-e2e/chat-catalog-domains/privileged-configuration.ts +++ b/tests/runner-e2e/chat-catalog-domains/privileged-configuration.ts @@ -50,7 +50,7 @@ const settings = privilegedDefinition("instance-operations.instance-settings", " await expect.poll(async () => (await ctx.input.api.get<{ enableAgentChat: boolean }>(endpoint, { timeout: remaining() })).enableAgentChat, { timeout: remaining() }).toBe(false); state.savedDisabled = true; await page.goto(source, { waitUntil: "domcontentloaded", timeout: remaining() }); - await expect(page.getByText(/Agent Chat is disabled(?:\.| in Experimental settings)/).first()).toBeVisible({ timeout: remaining() }); + await expect(page.getByText(/Agent Chat is disabled(?:\.| in Experimental settings| for this instance)/).first()).toBeVisible({ timeout: remaining() }); state.disabledRouteClear = true; await page.goto(settingsUrl, { waitUntil: "domcontentloaded", timeout: remaining() }); await expect(toggle).toHaveAttribute("aria-checked", "false", { timeout: remaining() }); await toggle.click({ timeout: remaining() }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-chat-disabled.ts b/tests/runner-e2e/chat-catalog-domains/runtime-chat-disabled.ts index baa00e79788..cc37d756545 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-chat-disabled.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-chat-disabled.ts @@ -62,10 +62,10 @@ export const runtimeDisabledChatCases: readonly IdentityBoundaryCase[] = [{ person.bindApiSession(ctx.api); // Capture verification uses the same person through private Node fetch. const page = ctx.page; await page.goto(`/${fixtures.company.issuePrefix}/chats/${fixtures.agent.id}`, { waitUntil: "domcontentloaded" }); - await expect(page.getByText(/Agent Chat is disabled\. Enable it in Experimental settings/)).toBeVisible(); + await expect(page.getByText(/Agent Chat is disabled for this instance\. An administrator can enable it in Experimental settings/)).toBeVisible(); await expect(page.getByTestId("task-chat-composer-input")).toHaveCount(0); const unavailableVisible = await page.getByText(/Agent Chat is disabled/).isVisible(); - const supportedOperatorStepVisible = await page.getByText(/Enable it in Experimental settings/).isVisible(); + const supportedOperatorStepVisible = await page.getByText(/can enable it in Experimental settings/).isVisible(); ctx.observeResources({ companyId: fixtures.company.id, agentId: fixtures.agent.id, runIds: [] }); await ctx.capture("disabled-agent-chat", "Morgan Chat is genuinely unavailable; canonical intent remains unsent", "runtime-disabled-agent-chat.png"); const key = await api.postSensitive<{ id: string; token: string }>(`/api/agents/${fixtures.agent.id}/keys`, { name: "disabled-chat-permission-probe" }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-claude-account.ts b/tests/runner-e2e/chat-catalog-domains/runtime-claude-account.ts index 173fad808eb..8ab7a1ba70d 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-claude-account.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-claude-account.ts @@ -12,7 +12,7 @@ import { checks, type Entity } from "./internal-support.js"; import { waitForReply } from "./identity-support.js"; import { matchesClaudeFixtureTarget, privateClaudeOperation, type ClaudePrivateReceipt } from "./runtime-claude-private.js"; import { selectedRuntimeInputs, runtimePrivateSpec } from "./runtime-private.js"; -import { openRuntimeAgentProfile, observeRuntime, runtimeDefinition, runtimeObservation, type RuntimeAgent } from "./runtime-support.js"; +import { clickInteractionAction, openRuntimeAgentProfile, observeRuntime, runtimeDefinition, runtimeObservation, type RuntimeAgent } from "./runtime-support.js"; const continueHire = "The Anthropic account is connected now. Continue the same Morgan frontend engineer hire using Claude Code."; export function supportedClaudeSetupLink(href: string, origin: string, prefix: string) { try { const url = new URL(href, origin); return url.origin === origin && url.pathname.startsWith(`/${prefix}/`) && (url.pathname.endsWith("/company/settings/connections") || url.pathname.endsWith("/apps/connect") && ["anthropic", "claude"].includes(url.searchParams.get("source") ?? url.searchParams.get("provider") ?? "")); } catch { return false; } @@ -69,7 +69,7 @@ export const runtimeClaudeAccountCases = [runtimeDefinition("agents.hire-needs-a if (target) await ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(target)}]`).first().click(); else { if (!request) throw new Error("The canonical waiting response exposed neither a supported account setup link nor an actual addressed Anthropic setup request"); - await ctx.input.page.locator(`[data-interaction-id=${JSON.stringify(request.id)}]`).getByRole("button", { name: /Connect|Set up|Continue/i }).first().click(); + await clickInteractionAction(ctx.input.page, request, /Connect|Set up|Continue/i); } await expect(ctx.input.page.getByRole("heading", { name: /connections|Connect.*Anthropic|Connect a model provider/i }).first()).toBeVisible(); state.supportedSetupVisible = true; const selected = selectedRuntimeInputs(ctx, "agents.hire-needs-account"); @@ -92,7 +92,7 @@ export const runtimeClaudeAccountCases = [runtimeDefinition("agents.hire-needs-a state.setupRequestReconciled = completedClaudeSetupMatches({ request: resolved, sourceId: ctx.issue()!.id, sourceRunId: firstRun.id, companyId: before.companyId, humanId: session.user.id, connectionId: String(state.connectionId) }); if (!state.setupRequestReconciled) throw new Error("The actual addressed Anthropic setup request did not reconcile against established human account access"); await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); - await expect(ctx.input.page.locator(`[data-interaction-id=${JSON.stringify(String(state.setupRequestId))}]`)).toContainText(/connected|ready|complete/i); + await expect(ctx.input.page.locator(`[id=${JSON.stringify(`interaction-${String(state.setupRequestId)}`)}]`)).toContainText(/connected|ready|complete/i); await ctx.refreshIssue(); state.setupAutoWakeExpected = ["in_progress", "in_review"].includes(ctx.issue()!.status) && ctx.issue()!.assigneeAgentId === resolved!.payload.requestingAgentId; if (state.setupAutoWakeExpected) { diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-cloud-choice.ts b/tests/runner-e2e/chat-catalog-domains/runtime-cloud-choice.ts index ae2f4f4d06f..36bea777334 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-cloud-choice.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-cloud-choice.ts @@ -5,7 +5,7 @@ import type { IssueThreadInteraction } from "../../../packages/shared/src/types/ import { expect } from "@playwright/test"; import { checks } from "./internal-support.js"; import { waitForReply } from "./identity-support.js"; -import { runtimeDefinition, runtimeObservation, observeRuntime } from "./runtime-support.js"; +import { interactionSurface, runtimeDefinition, runtimeObservation, observeRuntime } from "./runtime-support.js"; import { runtimePrivateSpec, selectedRuntimeInputs } from "./runtime-private.js"; import { createOwnedDaytonaTarget, deleteOwnedDaytonaTarget, type OwnedDaytonaTarget } from "./runtime-daytona-target.js"; import { privateClaudeOperation } from "./runtime-claude-private.js"; @@ -48,7 +48,7 @@ export const runtimeCloudChoiceCases = [runtimeDefinition("agents.ambiguous-clou const runs = await ctx.allRuns(); if (questions.some(row => row.sourceRunId !== runs[0]?.id || row.issueId !== source.id)) throw new Error("Clarification card is not attributed to the actual canonical source run"); await expect(ctx.input.page.getByTestId("task-chat-unanswered-question").first()).toBeVisible(); - state.visibleChoiceText = await ctx.input.page.locator(`[data-interaction-id=${JSON.stringify(questions[0]!.id)}]`).innerText(); + state.visibleChoiceText = await interactionSurface(ctx.input.page, questions[0]!).first().innerText(); } else await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); state.questionVisible = questions.length > 0 || /\?|which|do you mean/i.test((await observeRuntime(ctx, state)).reply); state.sourceWaiting = source.conversationState === "waiting"; diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-interruption.ts b/tests/runner-e2e/chat-catalog-domains/runtime-interruption.ts index e33ed9bc239..b78fbb5788f 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-interruption.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-interruption.ts @@ -10,6 +10,7 @@ import { readRegisteredArtifacts } from "../registered-artifact.js"; import { createIssueThreadInteractionSchema } from "../../../packages/shared/src/validators/issue.js"; import { checks, type Entity } from "./internal-support.js"; import { busyRuntimeWorker, releaseRuntimeWorker } from "./runtime-busy-worker.js"; +import { clickInteractionAction } from "./runtime-support.js"; import { runtimeQueueCases } from "./runtime-queue.js"; import { runtimeDefinition, runtimeObservation, observeRuntime } from "./runtime-support.js"; export function gradeInterruption(value: unknown) { @@ -44,11 +45,8 @@ export const runtimeInterruptionCases = [runtimeDefinition("steering.interrupt-w payload: { version: 1, prompt: "Use this saved simpler layout on the next eligible attempt?", supersedeOnUserComment: false, target: { type: "issue_document", issueId: task.id, documentId: plan.id, key: "plan", revisionId: plan.latestRevisionId, revisionNumber: plan.latestRevisionNumber } } })); await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" }); - const card = ctx.input.page.locator(`[data-interaction-id=${JSON.stringify(confirmation.id)}]`); const accept = ctx.input.page.waitForResponse(r => r.request().method() === "POST" && new URL(r.url()).pathname === `/api/issues/${task.id}/interactions/${confirmation.id}/accept`); - const acceptButton = card.getByRole("button", { name: /Approve|Accept/i }); - if (await acceptButton.count()) await acceptButton.first().click(); - else await ctx.input.page.getByRole("button", { name: /Approve|Accept/i }).first().click(); + await clickInteractionAction(ctx.input.page, confirmation, "Approve"); if ((await accept).status() !== 200) throw new Error("Actual human simpler-plan approval did not succeed"); const session = await ctx.input.api.get<{ user: { id: string } }>("/api/auth/get-session"); const approved = (await ctx.input.api.get(`/api/issues/${task.id}/interactions`)).find(row => row.id === confirmation.id)!; diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.ts b/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.ts index 6d038b3212f..1fa123d6f94 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.ts @@ -23,7 +23,7 @@ export function gradeMultipleHandoffs(value: unknown) { c.add("logo-task-and-delivered-logo-unchanged", !!logoBefore && !!logo && sameTask(logoBefore, logo) && logo.status === "done" && logo.assigneeAgentId === (s.logoWorker as Entity).id && before?.contentVerified === true && after?.contentVerified === true && before.attachmentId === after.attachmentId && before.sha256 === after.sha256 && before.content === after.content && after.content.includes(String(s.logoMarker)) && /(`${root}/issues`, { title: "Logo", status: "todo", assigneeAgentId: logoWorker.id, - description: `Create an actual small SVG logo file logo.svg with a simple acorn/circle mark, visible text Acorn and the exact reference ${logoMarker} in its SVG metadata or text. Save a short non-plan task delivery document describing it and register the real SVG file as a downloadable artifact work product titled logo.svg. Then complete this same task. No other task, publication or website change is authorized.` }); + description: `Create an actual small logo file logo.html containing an inline SVG with a simple acorn/circle mark, visible text Acorn and the exact reference ${logoMarker} in its SVG metadata or text. Save a short non-plan task delivery document describing it and register the real logo.html file as a downloadable artifact work product titled logo.html. Then complete this same task. No other task, publication or website change is authorized.` }); const complete = await pollUntil({ label: "Actual separate Logo work completed before the Website conversation", deadlineAt: Date.now() + 240_000, load: async () => ({ task: await ctx.input.api.get(`/api/issues/${logo.id}`), runs: await ctx.allRuns() }), accept: v => v.task.status === "done" && v.runs.length === 1 && v.runs[0]?.agentId === logoWorker.id && v.runs[0].status === "succeeded" }); - const [artifact] = await readRegisteredArtifacts(ctx.input.api, logo.id, complete.runs[0]!.id, ["logo.svg"]); - if (!artifact?.contentVerified || !artifact.content.includes(logoMarker) || !/(`/api/issues/${logo.id}/documents`), revisions: Entity[] = []; for (const doc of documents) revisions.push(...await ctx.input.api.get(`/api/issues/${logo.id}/documents/${encodeURIComponent(doc.key!)}/revisions`)); const busy = await busyRuntimeWorker(ctx); await openStandingChat(ctx); - const message = `Keep these two existing work references in this conversation: Website ${busy.task.identifier ?? busy.task.id} is assigned to Morgan and actively waiting for the brief; Logo ${logo.identifier ?? logo.id} was completed by Riley and its actual delivered SVG is ${new URL(`/api/attachments/${artifact.attachmentId}/content?download=1`, ctx.input.api.baseURL).href}. Acknowledge these separate references, without creating, reassigning, editing or starting any task. Keep the Logo deliverable unchanged.`; + const message = `Keep these two existing work references in this conversation: Website ${busy.task.identifier ?? busy.task.id} is assigned to Morgan and actively waiting for the brief; Logo ${logo.identifier ?? logo.id} was completed by Riley and its actual delivered logo is ${new URL(`/api/attachments/${artifact.attachmentId}/content?download=1`, ctx.input.api.baseURL).href}. Acknowledge these separate references, without creating, reassigning, editing or starting any task. Keep the Logo deliverable unchanged.`; await sendChatMessage(ctx.input.page, message); const saved = await pollUntil({ label: "Real source history retains separate active Website and completed Logo", deadlineAt: Date.now() + 150_000, load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, @@ -85,7 +85,7 @@ export const runtimeMultipleHandoffCases = [runtimeDefinition("agent-chat.multip async observe(ctx, state) { const o = await observeRuntime(ctx, state), logo = state.logoBefore as Entity; state.logoAfter = await ctx.input.api.get(`/api/issues/${logo.id}`); - const [artifact] = await readRegisteredArtifacts(ctx.input.api, logo.id, String(state.logoRunId), ["logo.svg"]); state.logoArtifactAfter = artifact; + const [artifact] = await readRegisteredArtifacts(ctx.input.api, logo.id, String(state.logoRunId), ["logo.html"]); state.logoArtifactAfter = artifact; const documents = await ctx.input.api.get(`/api/issues/${logo.id}/documents`), revisions: Entity[] = []; for (const doc of documents) revisions.push(...await ctx.input.api.get(`/api/issues/${logo.id}/documents/${encodeURIComponent(doc.key!)}/revisions`)); state.logoRevisionsUnchanged = JSON.stringify(documents.map(doc => [doc.id, doc.latestRevisionId]).sort()) === JSON.stringify((state.logoDocumentsBefore as Entity[]).map(doc => [doc.id, doc.latestRevisionId]).sort()) diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-support.ts b/tests/runner-e2e/chat-catalog-domains/runtime-support.ts index cf6ed6335a2..e784f5b46d3 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-support.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-support.ts @@ -1,4 +1,4 @@ -import { expect } from "@playwright/test"; +import { expect, type Page } from "@playwright/test"; import { pollUntil } from "../api.js"; import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js"; import { codexSubscriptionAgentConfig } from "../codex-subscription-fixture.js"; @@ -72,6 +72,24 @@ export async function verifyRuntimeResult(ctx: ChatCatalogContext, state: Record await ctx.input.evidence("chat-catalog-runtime-profile-opened.json", { agentId: target.id, companyId: target.companyId, url: ctx.input.page.url() }); await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); } +/** + * A pending interaction renders inline in the thread as `#interaction-`, + * or docked in the composer takeover, which carries its title but no id. + */ +export function interactionSurface(page: Page, interaction: { id: string; title?: string | null }) { + const inline = page.locator(`[id=${JSON.stringify(`interaction-${interaction.id}`)}]`); + return interaction.title + ? inline.or(page.getByTestId("task-chat-composer-takeover").filter({ hasText: interaction.title })) + : inline; +} +/** Open a pending request from its timeline chip when it is not already docked, then use its own action. */ +export async function clickInteractionAction(page: Page, interaction: { id: string; title?: string | null }, name: string | RegExp) { + const docked = page.getByTestId("task-chat-composer-takeover").filter({ hasText: interaction.title ?? "" }); + const action = (interaction.title ? docked : interactionSurface(page, interaction)).getByRole("button", typeof name === "string" ? { name, exact: true } : { name }); + await expect(interactionSurface(page, interaction).first()).toBeVisible(); + if (!(await action.first().isVisible())) await page.locator(`[id=${JSON.stringify(`interaction-${interaction.id}`)}]`).click(); + await action.first().click(); +} export function runtimeDefinition(scenarioId: string, setup: ChatCatalogCase["setup"], grade: ChatCatalogCase["grade"], extra: Partial = {}): ChatCatalogCase { return { scenarioId, setup, grade, observe: observeRuntime, verifyResult: verifyRuntimeResult, oracleVersion: "paperclip.chat-catalog.runtime.v2", coverageGaps: [], expectedRunCount: 1, diff --git a/tests/runner-e2e/registered-artifact.test.ts b/tests/runner-e2e/registered-artifact.test.ts index 3e00b476a8b..cacac4c770b 100644 --- a/tests/runner-e2e/registered-artifact.test.ts +++ b/tests/runner-e2e/registered-artifact.test.ts @@ -28,7 +28,7 @@ it.each(["draft", "changes_requested", "merged", "closed", "failed", "archived", expect(download).not.toHaveBeenCalled(); }); -it.each(["unregistered", "duplicate", "old-run", "unattributed", "wrong-issue", "wrong-attachment-issue", "wrong-type", "wrong-title", "wrong-filename", "wrong-attachment-id", "negative-size", "excessive-size", "size-mismatch", "hash-mismatch", "download-failure"])("refuses %s output evidence even when ready for review", async reason => { +it.each(["unregistered", "duplicate", "old-run", "unattributed", "wrong-issue", "wrong-attachment-issue", "wrong-type", "wrong-filename", "wrong-attachment-id", "negative-size", "excessive-size", "size-mismatch", "hash-mismatch", "download-failure"])("refuses %s output evidence even when ready for review", async reason => { const { api, product, attachment, get, download } = fixture(); product.status = "ready_for_review"; if (reason === "unregistered") get.mockImplementation(async (path: string) => path.endsWith("work-products") ? [] : [attachment]); @@ -38,7 +38,6 @@ it.each(["unregistered", "duplicate", "old-run", "unattributed", "wrong-issue", if (reason === "wrong-issue") product.issueId = "other-issue"; if (reason === "wrong-attachment-issue") attachment.issueId = "other-issue"; if (reason === "wrong-type") product.type = "document"; - if (reason === "wrong-title") product.title = "other.txt"; if (reason === "wrong-filename") attachment.originalFilename = "other.txt"; if (reason === "wrong-attachment-id") product.metadata.attachmentId = "other-attachment"; if (reason === "negative-size") attachment.byteSize = -1; @@ -49,6 +48,16 @@ it.each(["unregistered", "duplicate", "old-run", "unattributed", "wrong-issue", await expect(readRegisteredArtifacts(api, "issue", "run", ["proof.txt"])).rejects.toThrow(); }); +it("identifies the deliverable by its run-attributed file, not its display title", async () => { + const { api, product, attachment, download } = fixture(); + product.title = "Acorn Notes proof (proof.txt)"; + expect((await readRegisteredArtifacts(api, "issue", "run", ["proof.txt"]))[0]?.contentVerified).toBe(true); + attachment.originalFilename = "other.txt"; + download.mockClear(); + await expect(readRegisteredArtifacts(api, "issue", "run", ["proof.txt"])).rejects.toBeInstanceOf(RegisteredArtifactNotFoundError); + expect(download).not.toHaveBeenCalled(); +}); + it("does not accept a filename or unchecked content as downloaded evidence", async () => { const matcher = { kind: "artifact_exact" as const, name: "proof.txt", expected: "verified\n" }; expect((await evaluateMatcher(matcher, { artifacts: [{ name: "proof.txt", content: "verified\n" }] })).passed).toBe(false); diff --git a/tests/runner-e2e/registered-artifact.ts b/tests/runner-e2e/registered-artifact.ts index 535d6e414d9..2adc7a8e84d 100644 --- a/tests/runner-e2e/registered-artifact.ts +++ b/tests/runner-e2e/registered-artifact.ts @@ -19,10 +19,14 @@ export async function readRegisteredArtifacts(api: RunnerApi, issueId: string, r api.get(`/api/issues/${encodeURIComponent(issueId)}/work-products`), api.get(`/api/issues/${encodeURIComponent(issueId)}/attachments`), ]); + // The file name identifies a deliverable; its work-product title is a + // human-facing label the agent may choose, so it is not part of identity. + const filenameOf = (product: IssueWorkProduct) => + attachments.find(attachment => attachment.id === product.metadata?.attachmentId)?.originalFilename; return Promise.all(names.map(async name => { const registered = products.filter(product => product.issueId === issueId && product.createdByRunId === runId && product.type === "artifact" && - REGISTERED_ARTIFACT_STATUSES.has(product.status) && product.title === name); + REGISTERED_ARTIFACT_STATUSES.has(product.status) && (product.title === name || filenameOf(product) === name)); if (registered.length === 0) throw new RegisteredArtifactNotFoundError(name); if (registered.length !== 1) throw new Error(`Expected one registered artifact ${name} from the tested run; observed ${registered.length}`); const product = registered[0]!;