diff --git a/tests/runner-e2e/PRODUCT-JOURNEYS.md b/tests/runner-e2e/PRODUCT-JOURNEYS.md index c5276b4a0e8..ed8c2d6e55b 100644 --- a/tests/runner-e2e/PRODUCT-JOURNEYS.md +++ b/tests/runner-e2e/PRODUCT-JOURNEYS.md @@ -36,6 +36,7 @@ never start a server or depend on a tenant filesystem. | `create-skill-studio` | Native agent with skill-write permission; one work run | Create a uniquely named company skill, open its activity card, edit in Studio, reload and verify task-card readback | | `stop-redirect` | Native agent; three work runs; qualified Docker oracle | Download a verified checkpoint, observe an active native command with current-turn usage, Stop it, verify cancellation and settled accounting, then send one new direction and complete without original-run activity | | `followup-while-running` | Native agent; a maximum of two work runs | Save a follow-up while an independently observed native shell command is still active, then require one attributed final reply with the late requirement after reload; in-turn steering and one normal successor are accepted | +| `hire-delegate-reuse` | Native lead with a fresh user chat and scoped hiring permission; Agent Chat enabled; direct hiring policy; seven work runs | Hire one teammate, delegate a saved checklist, reuse the same teammate for a saved review, verify native run and revision authorship, preserve the original, and follow the lead's saved result link after reload | | `cloud-onboarding` | Signed-in admitted Cloud account; fresh workspace; qualified bounded provider route | Create workspace in the portal, connect API key and native Codex agent in the wizard, complete a task, download and independently check its artifact | The deployment-targeted Stop journey asks the revision to finish one harmless @@ -87,6 +88,67 @@ The shared Studio helper waits for the original saved file body before typing. An editor control can mount before the file query completes; editing that empty draft can otherwise lose the original content when the file arrives. +The hiring journey uses the existing Agent Chat prompts and native hiring oracle. +It gives each disposable project, agent, and task a campaign name. The first +turn only hires. The target provider can then verify the native creation audit +and prepare that exact teammate's budget before the next turn delegates work. +Both handoffs ask for the task to be created already assigned to that teammate. +They use the normal composer agent and project mention formats, with the exact +IDs from the independently verified hire and owned project. Missing, foreign or +mismatched context fails before constructing the handoff. The mentions provide +the intended resources without prescribing a tool call or bypassing normal delegation. +The handoff selects these references through the normal rich-editor mention menu +and verifies each chip target before sending once. Typing Markdown link syntax +as plain rich-editor text escapes it and is not a mention. The saved user body +must still match the requested body exactly. +Creating work for the lead and later reassigning it starts additional native work +and fails the existing assignee and run checks. +When the lead uses a delegated personal connection, normal hiring selects the +responsible user's default rather than copying the manager's identity. Before +registering any chat or child-creation intent or sending the first turn, the +journey reads the actual signed-in workflow user's provider default and requires +the exact approved connection and grant, connected and owned by that user. +After hiring it independently checks the exact child installation. The underlying +connection's creator is not the owner of the user's default selection. Missing, +different or ambiguous authority fails closed; no credential, default or grant is +changed. A responsible-user hire stores the canonical `api_key` placeholder even +when the approved default uses a subscription. The resolved native method must +still match that independently approved default. Every successful native run must also retain the approved connection, +grant and responsible-user attribution in its canonical execution metadata. +Those references are compared privately and omitted from observation evidence. +The first independently approved child binding becomes an immutable expected +snapshot for the final shared hiring oracle; later binding changes fail. Only +typed transient observation GET failures may poll again within the original +deadline. Approval, authentication and ownership failures are never replayed. +The checklist and review each require one successful child run and a current +document revision authored by that run. The lead uses three requested turns and +two separate server completion callbacks, +for seven work runs in total. Each callback must identify exactly one of the two +completed child tasks with a distinct server delivery receipt; arbitrary extra +lead turns fail. The journey waits for each callback and its exact run-authored +saved reply before the next user turn. A terminal run can precede its final +comment; missing materialization polls within the existing deadline, while a +wrong author or binding fails immediately. +The lead has a maximum of five work runs and the teammate has two. +A 500-cent case allowance is split into 300 cents for the lead and 200 for the +teammate; the allocation must not duplicate the declared case allowance. +Setup probes and unpriced usage remain separate accounting +uncertainties. The journey does not change company hiring policy or instance +features. The provider must own and clean up the hired agent, both child tasks, +the chat, and the project. Before the first send, a fresh-chat intent records the +company, user and owned lead after a normal GET confirms no existing chat. +Recovery can attach only that exact fresh conversation. Saved replies and links +are allowed to render after reload within the journey deadline. Cleanup includes +unexpected campaign-owned children after +failure. A new deployment must qualify the journey before scheduled admission. +Child discovery reads the exact native creation audit after the run snapshot, +then fetches each audited agent/task ID directly. It never depends on the first +page of a shared company list. Each proven child is recorded before the next +lookup can fail. Observation retries are limited to typed transient GET transport failures within +the existing deadline. Ownership callbacks and grading failures are never +retried. Audited same-company children are recorded before manager/project +checks fail; foreign or malformed rows remain unowned and fail the journey. + The task-document journey uses the normal default-chat Artifacts pane and the shared document editor. Classic mode retains its editor in the task body. The general editor excludes Plans, which retain their conversation and approval diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 2a588e44e2b..8ce668acff0 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -118,7 +118,7 @@ describe("runner E2E catalog", () => { }); it("exports deployment journeys separately from the model matrix and default selection", () => { - expect(productJourneys.map(journey => journey.id)).toEqual(["task-create", "project-task-context", "ask-question", "question-answer-resume", "task-properties", "task-document-persistence", "task-search-filter", "plan-revise-accept", "build-revise", "protected-actions", "create-skill-studio", "stop-redirect", "followup-while-running", "cloud-onboarding"]); + expect(productJourneys.map(journey => journey.id)).toEqual(["task-create", "project-task-context", "ask-question", "question-answer-resume", "task-properties", "task-document-persistence", "task-search-filter", "plan-revise-accept", "build-revise", "protected-actions", "create-skill-studio", "stop-redirect", "followup-while-running", "hire-delegate-reuse", "cloud-onboarding"]); expect(runnerSuites.some(suite => suite.id === "product-journeys")).toBe(false); expect(runnerMatrix.some(execution => execution.suite.id === "product-journeys")).toBe(false); expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(execution => execution.suite.id === "product-journeys")).toBe(false); diff --git a/tests/runner-e2e/hire-delegate-composer.fixture.ts b/tests/runner-e2e/hire-delegate-composer.fixture.ts new file mode 100644 index 00000000000..6158683a375 --- /dev/null +++ b/tests/runner-e2e/hire-delegate-composer.fixture.ts @@ -0,0 +1,66 @@ +import { createRequire } from "node:module"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import type { IncomingMessage, ServerResponse } from "node:http"; + +export const composerResources = { + companyId: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", leadId: "33333333-3333-4333-8333-333333333333", + agentId: "11111111-1111-4111-8111-111111111111", projectId: "22222222-2222-4222-8222-222222222222", + campaignId: "journey-1-deadbeef", +}; + +export async function startHireComposerFixture() { + const root = fileURLToPath(new URL("../../", import.meta.url)), ui = path.join(root, "ui"); + const require = createRequire(path.join(ui, "package.json")); + const { createServer } = await import(require.resolve("vite")); + const { default: react } = await import(require.resolve("@vitejs/plugin-react")); + const cacheDir = await mkdtemp(path.join(tmpdir(), "hire-composer-vite-")); + const saved = new Map>(); + const entry = `import React from "react"; +import {createRoot} from "react-dom/client"; +import {TaskChatComposer} from "/src/components/task-chat/TaskChatComposer.tsx"; +import {TaskChatPresentationProvider} from "/src/components/task-chat/presentation-mode.tsx"; +import {buildMarkdownMentionOptions} from "/src/lib/company-members.ts"; +const r=${JSON.stringify(composerResources)}, q=new URLSearchParams(location.search); +const agents=[{id:q.get("variant")==="wrong-agent"?"44444444-4444-4444-8444-444444444444":r.agentId,name:"Morgan Reviewer "+r.campaignId,status:"idle",icon:"code"}]; +if(q.get("variant")==="duplicate")agents.push({...agents[0],id:"55555555-5555-4555-8555-555555555555"}); +const projects=q.get("variant")==="missing-project"?[]:[{id:r.projectId,name:"Journey QA hire-delegate-reuse "+r.campaignId,color:"#336699"}]; +const mentions=buildMarkdownMentionOptions({agents,projects}); +createRoot(document.getElementById("root")).render(React.createElement(TaskChatPresentationProvider,{mode:q.get("mode")||"production"},React.createElement(TaskChatComposer,{workMode:"standard",conversationMode:true,mentions,onAdd:async body=>{const response=await fetch("/__saved/"+encodeURIComponent(q.get("case")),{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({body})});if(!response.ok)throw Error("fixture_save_failed");}})));`; + const server = await createServer({ + configFile: false, root: ui, cacheDir, logLevel: "error", + plugins: [react(), { + name: "hire-production-composer-fixture", + resolveId(id: string) { if (id === "/__hire-entry.tsx") return "\0hire-entry.tsx"; }, + load(id: string) { if (id === "\0hire-entry.tsx") return entry; }, + configureServer(instance: any) { + instance.middlewares.use(async (req: IncomingMessage, res: ServerResponse, next: () => void) => { + const pathname = new URL(req.url ?? "/", "http://fixture.local").pathname; + if (pathname === "/__hire.html") { + res.setHeader("Content-Type", "text/html"); + res.end(await instance.transformIndexHtml(pathname, '
')); + } else if (pathname.startsWith("/__saved/")) { + const key = decodeURIComponent(pathname.slice("/__saved/".length)); + const rows = saved.get(key) ?? []; + if (req.method === "POST") { + let body = ""; for await (const chunk of req) body += chunk; + rows.push({ id: String(rows.length + 1), body: JSON.parse(body).body }); saved.set(key, rows); + } + res.setHeader("Content-Type", "application/json"); res.end(JSON.stringify(rows)); + } else next(); + }); + }, + }], + resolve: { alias: [ + { find: "@", replacement: path.join(ui, "src") }, + { find: /^@paperclipai\/shared$/, replacement: path.join(root, "packages/shared/src/index.ts") }, + { find: /^@paperclipai\/shared\/(.+)$/, replacement: path.join(root, "packages/shared/src/$1") }, + { find: "lexical", replacement: path.join(ui, "node_modules/lexical/dist/Lexical.mjs") }, + ] }, + server: { host: "127.0.0.1", port: 0, fs: { allow: [root] } }, + }); + await server.listen(); + return { url: server.resolvedUrls.local[0] as string, async close() { await server.close(); await rm(cacheDir, { recursive: true, force: true }); } }; +} diff --git a/tests/runner-e2e/hire-delegate-composer.spec.ts b/tests/runner-e2e/hire-delegate-composer.spec.ts new file mode 100644 index 00000000000..1fd0aa3140d --- /dev/null +++ b/tests/runner-e2e/hire-delegate-composer.spec.ts @@ -0,0 +1,49 @@ +import { expect, test, type Page } from "@playwright/test"; +import { sendChatMessage } from "./chat-flow.js"; +import { hireHandoffMentions } from "./product-journeys/hire-delegate-flow.js"; +import { sendHireHandoffMessage } from "./product-journeys/hire-delegate-composer.js"; +import { composerResources as r, startHireComposerFixture } from "./hire-delegate-composer.fixture.js"; + +test.setTimeout(45_000); +let fixture: Awaited>; +test.beforeAll(async () => { fixture = await startHireComposerFixture(); }); +test.afterAll(async () => { await fixture?.close(); }); +const references = hireHandoffMentions({ companyId: r.companyId, leadId: r.leadId, projectId: r.projectId, campaignId: r.campaignId }, r.agentId, + { id: r.agentId, companyId: r.companyId, reportsTo: r.leadId, name: `Morgan Reviewer ${r.campaignId}`, icon: "code" }, + { id: r.projectId, companyId: r.companyId, name: `Journey QA hire-delegate-reuse ${r.campaignId}`, color: "#336699" }); +const message = (review = false) => `Have the existing ${references.teammate} ${review ? "review the checklist on QA-1 and write a separate saved review" : "write a concise launch checklist as a saved Paperclip document"}. Create exactly one task in ${references.project}, already assigned to that teammate when it is created. Link the result here.`; +async function open(page: Page, key: string, variant = "valid", mode = "production") { + const url = new URL("/__hire.html", fixture.url); url.searchParams.set("case", key); url.searchParams.set("variant", variant); url.searchParams.set("mode", mode); + await page.goto(url.href); + await expect(page.getByTestId("task-chat-composer-input").locator('[contenteditable="true"]')).toBeVisible({ timeout: 30_000 }); +} +async function saved(page: Page, key: string) { return (await page.request.get(new URL(`/__saved/${encodeURIComponent(key)}`, fixture.url).href)).json() as Promise>; } + +test("raw fill escapes mention syntax in the real production composer", async ({ page }) => { + await open(page, "raw"); const text = message(); await sendChatMessage(page, text); + await expect.poll(async () => (await saved(page, "raw")).length).toBe(1); + const [row] = await saved(page, "raw"); expect(row!.body).not.toBe(text); expect(row!.body).toContain("\\[@"); +}); +for (const [review, mode] of [[false, "production"], [true, "streamlined"]] as const) { + test(`normal mention controls preserve the exact ${review ? "review" : "checklist"} body in ${mode} composer`, async ({ page }) => { + const key = mode; await open(page, key, "valid", mode); const text = message(review); + await sendHireHandoffMessage(page, text, references, Date.now() + 10_000); + await expect.poll(async () => (await saved(page, key)).length).toBe(1); + expect(await saved(page, key)).toEqual([{ id: "1", body: text }]); + }); +} +for (const variant of ["wrong-agent", "duplicate", "missing-project"] as const) { + test(`normal mention controls reject ${variant} without sending`, async ({ page }) => { + await open(page, variant, variant); + const send = sendHireHandoffMessage(page, message(), references, Date.now() + (variant === "missing-project" ? 5_000 : 10_000)); + if (variant !== "missing-project") await expect(send).rejects.toMatchObject({ + code: variant === "wrong-agent" ? "hire_handoff_mention_mismatch" : "hire_handoff_mention_ambiguous", + }); + else { + await expect(send).rejects.toThrow(); + await expect(page.locator('a[data-mention-kind="agent"]')).toHaveCount(1); + await expect(page.locator('a[data-mention-kind="project"]')).toHaveCount(0); + } + expect(await saved(page, variant)).toEqual([]); + }); +} diff --git a/tests/runner-e2e/journey-features.ts b/tests/runner-e2e/journey-features.ts index 73ec8eb1de4..8e93e71f946 100644 --- a/tests/runner-e2e/journey-features.ts +++ b/tests/runner-e2e/journey-features.ts @@ -21,6 +21,7 @@ function associate(suites: readonly string[], cases: Record(deadlineAt: number, load: () => Promise, acc export async function runAgentJourney(id: AgentJourney, context: JourneyContext, session: TargetSession, campaignId: string, evidence: JourneyEvidence, options: Options) { + if (id === "hire-delegate-reuse") return hireDelegateReuse(context, session, campaignId, evidence, options); const agent = context.agent; if (!agent || agent.runtimeMode !== "native" || !context.environmentId) throw new JourneyError("unsupported", "native_agent_and_environment_required"); const nonce = campaignId.replaceAll("-", "_"); diff --git a/tests/runner-e2e/product-journeys/catalog.ts b/tests/runner-e2e/product-journeys/catalog.ts index 365a3ac1794..13d423d5740 100644 --- a/tests/runner-e2e/product-journeys/catalog.ts +++ b/tests/runner-e2e/product-journeys/catalog.ts @@ -15,6 +15,7 @@ export const productJourneys = [ { id: "create-skill-studio", label: "Create a company skill and persist a Skill Studio edit", requires: ["company", "board-user", "task-write", "agent", "skill-write"], capabilities: ["skills/authoring"], paid: true }, { id: "stop-redirect", label: "Stop active work and send a new direction once", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["steering/stop-and-pause"], paid: true }, { id: "followup-while-running", label: "Send a follow-up during active work and verify its saved result", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["steering/queue"], paid: true }, + { id: "hire-delegate-reuse", label: "Hire, delegate and reuse one teammate", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["agents/hire", "documents-artifacts/documents"], paid: true }, { id: "cloud-onboarding", label: "Create a Cloud workspace and complete a first task", requires: ["cloud-account", "fresh-workspace", "provider-key"], capabilities: ["onboarding/first-task"], paid: true }, ] as const; diff --git a/tests/runner-e2e/product-journeys/hire-delegate-composer.ts b/tests/runner-e2e/product-journeys/hire-delegate-composer.ts new file mode 100644 index 00000000000..d94986723a6 --- /dev/null +++ b/tests/runner-e2e/product-journeys/hire-delegate-composer.ts @@ -0,0 +1,46 @@ +import { expect, type Page } from "@playwright/test"; +import { parseAgentMentionHref, parseProjectMentionHref } from "../../../packages/shared/src/project-mentions.js"; +import { JourneyError } from "./types.js"; + +export async function sendHireHandoffMessage(page: Page, text: string, references: Readonly<{ teammate: string; project: string }>, deadlineAt: number) { + const timeout = () => { + const remaining = Math.min(5_000, deadlineAt - Date.now()); + if (remaining <= 0) throw new JourneyError("failed", "hire_handoff_editor_not_ready"); + return remaining; + }; + const tokens = [references.teammate, references.project].map(markdown => { + const match = /^\[@([^\]]+)\]\(([^)]+)\)$/.exec(markdown); + const href = match?.[2], agent = href && parseAgentMentionHref(href), project = href && parseProjectMentionHref(href); + const at = text.indexOf(markdown); + if (!match || !href || (!agent && !project) || at < 0 || text.indexOf(markdown, at + markdown.length) >= 0) { + throw new JourneyError("failed", "hire_handoff_context_unverified"); + } + return { markdown, href, name: match[1]!, kind: agent ? "agent" : "project", at }; + }).sort((left, right) => left.at - right.at); + if (tokens[0]!.kind === tokens[1]!.kind) throw new JourneyError("failed", "hire_handoff_context_unverified"); + const composer = page.getByTestId("task-chat-composer-input").last(); + const editable = composer.locator('[contenteditable="true"]').first(); + await expect(editable).toBeVisible({ timeout: timeout() }); + await editable.fill(""); + let offset = 0; + for (const token of tokens) { + const prefix = text.slice(offset, token.at); + if (prefix) await editable.pressSequentially(prefix); + await editable.pressSequentially(`@${token.name}`); + const menu = page.getByTestId("mention-autocomplete-menu"); + const option = menu.getByRole("button").filter({ has: page.getByText(token.name, { exact: true }) }); + await expect(option.first()).toBeVisible({ timeout: timeout() }); + if (await option.count() !== 1) throw new JourneyError("failed", "hire_handoff_mention_ambiguous"); + await option.click({ timeout: timeout() }); + const chip = editable.locator(`a[data-mention-kind="${token.kind}"]`).last(); + await expect(chip).toBeVisible({ timeout: timeout() }); + if (await chip.getAttribute("href") !== token.href) throw new JourneyError("failed", "hire_handoff_mention_mismatch"); + await page.evaluate(() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(() => resolve())))); + offset = token.at + token.markdown.length; + } + const suffix = text.slice(offset); + if (suffix) await editable.pressSequentially(suffix); + const hrefs = await editable.locator("a[data-mention-kind]").evaluateAll(nodes => nodes.map(node => node.getAttribute("href"))); + if (JSON.stringify(hrefs) !== JSON.stringify(tokens.map(token => token.href))) throw new JourneyError("failed", "hire_handoff_mention_mismatch"); + await page.getByTestId("task-chat-composer-send").last().click({ timeout: timeout() }); +} diff --git a/tests/runner-e2e/product-journeys/hire-delegate-evidence.test.ts b/tests/runner-e2e/product-journeys/hire-delegate-evidence.test.ts new file mode 100644 index 00000000000..eb10aeefe09 --- /dev/null +++ b/tests/runner-e2e/product-journeys/hire-delegate-evidence.test.ts @@ -0,0 +1,239 @@ +import { describe, expect, it } from "vitest"; +import { assertHireAuditCoverage, assertHireBinding, assertHireDocument, assertHireRuns, assertHireTaskCreation, hireDelegateNames, observedHireResources, readHireConnectionChoice, type HireScope } from "./hire-delegate-evidence.js"; +import { defaultAiConnectionForHire } from "../../../server/src/services/agent-ai-connection-default.js"; +import { agentJourneyRuns, journeyRequiredChecks } from "./success-contract.js"; + +type Row = Record; +const scope: HireScope = { companyId: "company", leadId: "lead", chatId: "chat", projectId: "project", campaignId: "journey-1-deadbeef", + origin: "https://qa.example.test", issuePrefix: "TEST", + model: "fixed-model", provider: "opencode", environmentId: "daytona", connectionBinding: { connectionId: "connection" } }; +function fixture() { + const names = hireDelegateNames(scope.campaignId); + const agent = { id: "hire", companyId: scope.companyId, name: names.hire, reportsTo: scope.leadId, adapterType: "paperclip_runner", + adapterConfig: { model: scope.model }, runtimeConfig: { aiConnection: scope.connectionBinding } }; + const task = { id: "task", title: names.checklist, identifier: "TEST-1", companyId: scope.companyId, projectId: scope.projectId, status: "done", assigneeAgentId: agent.id }; + const run = (id: string, issueId: string, agentId: string) => ({ id, companyId: scope.companyId, agentId, issueId, status: "succeeded", runtimeMode: "native", + contextSnapshot: { issueId, paperclipEnvironment: { id: scope.environmentId } }, runnerProfileJson: { nativeExecutionInput: { provider: { kind: scope.provider, model: scope.model } } } }); + const leadRun = run("lead-run", scope.chatId, scope.leadId), taskRun = run("task-run", task.id, agent.id); + const event = (action: string, entityId: string): Row => ({ companyId: scope.companyId, actorType: "agent", actorId: scope.leadId, agentId: scope.leadId, runId: leadRun.id, action, entityId, entityType: action.startsWith("agent.") ? "agent" : "issue", + details: { assigneeAgentId: agent.id, assigneeUserId: null, status: "todo" } }); + const state = { agents: [agent], tasks: [task], leadRuns: [leadRun], activity: [event("agent.hire_created", agent.id), event("issue.created", task.id)] }; + return { agent, task, leadRun, taskRun, state, event, run }; +} +function completionFixture() { + const f = fixture(), second = { ...f.task, id: "review", identifier: "TEST-2" }; + const requested = ["message-1", "message-2", "message-3"].map((id, i) => ({ ...f.run(`lead-${i}`, scope.chatId, scope.leadId), + contextSnapshot: { ...f.leadRun.contextSnapshot, source: "issue.comment", wakeCommentId: id } })); + const callbacks = [f.task, second].map((task, i) => ({ ...f.run(`completion-${i}`, scope.chatId, scope.leadId), invocationSource: "automation", triggerDetail: "system", + contextSnapshot: { ...f.leadRun.contextSnapshot, wakeReason: "chat_task_completed", wakeSource: "automation", wakeTriggerDetail: "system", + chatCompletionDeliveryIds: [`delivery-${i}`], chatCompletionUpdates: [{ id: task.id, status: "done" }] } })); + const runs = [...requested, ...callbacks, f.taskRun, f.run("review-run", second.id, f.agent.id)]; + const comments = callbacks.map(run => ({ companyId: scope.companyId, issueId: scope.chatId, authorAgentId: scope.leadId, createdByRunId: run.id, body: `Completed [saved result](/TEST/issues/${[f.task, second][callbacks.indexOf(run)]!.identifier})` })); + const grade = (value = runs, replies = comments) => assertHireRuns(scope, f.agent.id, [f.task, second], value, 7, ["message-1", "message-2", "message-3"], replies); + return { f, second, requested, callbacks, runs, comments, grade }; +} +describe("deployment hiring qualification", () => { + it("uses seven bounded native runs and has closed retained-result success requirements", () => { + expect(agentJourneyRuns["hire-delegate-reuse"]).toBe(7); + expect(journeyRequiredChecks["hire-delegate-reuse"]).toContain("review-authored-by-same-hired-run"); + }); + it("accepts old activity only when the page covers the exact current lead runs", () => { + const activity = Array.from({ length: 500 }, () => ({ createdAt: "2026-10-29T00:00:00Z" })); + expect(() => assertHireAuditCoverage(activity, [{ startedAt: "2026-10-30T00:00:00Z" }])).not.toThrow(); + expect(() => assertHireAuditCoverage(activity, [{ startedAt: "2026-10-28T00:00:00Z" }])).toThrow(); + expect(() => assertHireAuditCoverage(activity, [{}])).toThrow(); + expect(() => assertHireAuditCoverage(activity, [])).not.toThrow(); + }); + it("trusts the native actor's actual hire/task receipts, not matching names in a shared company", () => { + const f = fixture(); + expect(observedHireResources(scope, f.state).hires).toEqual([f.agent]); + for (const property of ["companyId", "actorType", "actorId", "agentId", "runId"]) { + const changed = structuredClone(f.state); changed.activity[0]![property as keyof typeof changed.activity[0]] = "foreign"; + const observed = observedHireResources(scope, changed); + expect(() => assertHireBinding(scope, observed.agents)).toThrow(); + } + const unrelated = { ...f.agent, id: "unrelated", name: "Other work" }; + expect(observedHireResources(scope, { ...f.state, agents: [...f.state.agents, unrelated] }).agents).toEqual([f.agent]); + expect(observedHireResources(scope, { ...f.state, activity: [] }).agents).toEqual([]); + }); + it("rejects duplicate hires, altered runtime/account binding and a replaced teammate", () => { + const f = fixture(); expect(assertHireBinding(scope, [f.agent]).id).toBe("hire"); + expect(() => assertHireBinding(scope, [f.agent], "old-hire")).toThrow(); + expect(() => assertHireBinding(scope, [f.agent, { ...f.agent, id: "duplicate" }])).toThrow(); + for (const change of [{ companyId: "other" }, { reportsTo: "other" }, { adapterType: "legacy" }, + { adapterConfig: { model: "other" } }, { runtimeConfig: { aiConnection: { connectionId: "other" } } }]) { + expect(() => assertHireBinding(scope, [{ ...f.agent, ...change }])).toThrow(); + } + }); + it("accepts the canonical responsible-user fallback only for the independently approved same credential choice", () => { + const f = fixture(), delegated = { provider: "openrouter", method: "api_key", mode: "delegated", connectionId: "approved", grantId: "grant" }; + const scoped = { ...scope, connectionBinding: delegated, + connectionChoice: { connectionId: "approved", grantId: "grant", provider: "openrouter", method: "api_key", responsibleUserId: "user" } }; + Object.assign(f.agent.runtimeConfig, { aiConnection: { provider: "openrouter", method: "api_key", mode: "responsible_user" } }); + expect(() => assertHireBinding(scoped, [f.agent])).not.toThrow(); + expect(() => assertHireBinding({ ...scope, connectionBinding: delegated }, [f.agent])).toThrow(); + expect(() => assertHireBinding({ ...scoped, connectionChoice: { ...scoped.connectionChoice, grantId: "foreign" } }, [f.agent])).toThrow(); + }); + it("uses the production delegated fallback and the actual requesting user's same approved personal default", async () => { + const binding = { provider: "openrouter", method: "api_key", mode: "delegated", connectionId: "00000000-0000-4000-8000-000000000001", grantId: "00000000-0000-4000-8000-000000000002" }; + expect(defaultAiConnectionForHire("paperclip_runner", { provider: "opencode", model: "openrouter/deepseek/deepseek-v4.1-flash-1017" }, binding)) + .toEqual({ provider: "openrouter", method: "api_key", mode: "responsible_user" }); + const selected = { id: binding.connectionId, grantId: binding.grantId, companyId: scope.companyId, provider: binding.provider, + method: binding.method, ownership: "personal", ownerUserId: "user", isDefault: true, status: "connected" }; + const listed = { currentUserId: "user", connections: [selected] }; + const installed = { connectionId: binding.connectionId, installs: [{ companyId: scope.companyId, targetType: "agent", targetId: "hire" }] }; + const api = { async get(route: string): Promise { return (route.endsWith("/installs") ? installed : listed) as T; } }; + const input = { companyId: scope.companyId, responsibleUserId: "user", binding, hiredId: "hire" }; + await expect(readHireConnectionChoice(api, input)).resolves.toMatchObject({ connectionId: binding.connectionId, grantId: binding.grantId, responsibleUserId: "user" }); + for (const mutate of [() => { listed.currentUserId = "operator"; }, () => { selected.grantId = "foreign"; }, + () => { selected.ownerUserId = "operator"; }, () => { selected.status = "revoked"; }, + () => { selected.isDefault = false; }, () => { installed.installs[0]!.targetId = "foreign"; }]) { + const before = structuredClone({ listed, installed }); mutate(); + await expect(readHireConnectionChoice(api, input)).rejects.toMatchObject({ code: "hire_connection_choice_mismatch" }); + Object.assign(listed, before.listed); Object.assign(selected, before.listed.connections[0]); listed.connections = [selected]; Object.assign(installed, before.installed); + } + }); + it("distinguishes the subscription hire placeholder from the approved resolved native method", async () => { + const f = fixture(), binding = { provider: "openai", method: "subscription", mode: "delegated", + connectionId: "00000000-0000-4000-8000-000000000001", grantId: "00000000-0000-4000-8000-000000000002" }; + const placeholder = defaultAiConnectionForHire("paperclip_runner", { provider: "codex", model: "fixed-model" }, binding); + expect(placeholder).toEqual({ provider: "openai", method: "api_key", mode: "responsible_user" }); + const selected = { id: binding.connectionId, grantId: binding.grantId, companyId: scope.companyId, provider: binding.provider, + method: binding.method, ownership: "personal", ownerUserId: "user", isDefault: true, status: "connected" }; + const installed = { connectionId: binding.connectionId, installs: [{ companyId: scope.companyId, targetType: "agent", targetId: f.agent.id }] }; + const api = { async get(route: string): Promise { return (route.endsWith("/installs") ? installed : { currentUserId: "user", connections: [selected] }) as T; } }; + const input = { companyId: scope.companyId, responsibleUserId: "user", binding, hiredId: f.agent.id }; + const connectionChoice = await readHireConnectionChoice(api, input); + const scoped = { ...scope, provider: "codex", connectionBinding: binding, connectionChoice }; + const hired = { ...f.agent, runtimeConfig: { aiConnection: placeholder } }; + expect(() => assertHireBinding(scoped, [hired])).not.toThrow(); + expect(() => assertHireBinding({ ...scoped, connectionChoice: { ...connectionChoice!, grantId: "unapproved" } }, [hired])).toThrow("hire_native_binding_mismatch"); + for (const altered of [{ ...placeholder, provider: "anthropic" }, { ...placeholder, mode: "delegated" }, { ...placeholder, method: "subscription" }]) { + expect(() => assertHireBinding(scoped, [{ ...hired, runtimeConfig: { aiConnection: altered } }])).toThrow("hire_native_binding_mismatch"); + } + selected.grantId = "unapproved"; + await expect(readHireConnectionChoice(api, input)).rejects.toMatchObject({ code: "hire_connection_choice_mismatch" }); + const run = { ...f.taskRun, runnerProfileJson: { nativeExecutionInput: { provider: { kind: "codex", model: scope.model } } }, + contextSnapshot: { ...f.taskRun.contextSnapshot, aiConnection: { ...binding, mode: "responsible_user", responsibleUserId: "user" } } }; + expect(() => assertHireRuns(scoped, hired.id, [f.task], [run])).not.toThrow(); + expect(() => assertHireRuns(scoped, hired.id, [f.task], [{ ...run, + contextSnapshot: { ...run.contextSnapshot, aiConnection: { ...run.contextSnapshot.aiConnection, method: "api_key" } } }])).toThrow("hire_run_credential_mismatch"); + }); + it("requires actual execution attribution to keep the approved credential and responsible user", () => { + const f = fixture(), connectionChoice = { connectionId: "approved", grantId: "grant", provider: "openrouter", method: "api_key", responsibleUserId: "user" }; + const scoped = { ...scope, connectionChoice }, run = { ...f.taskRun, contextSnapshot: { ...f.taskRun.contextSnapshot, + aiConnection: { ...connectionChoice, mode: "responsible_user" } } }; + expect(() => assertHireRuns(scoped, f.agent.id, [f.task], [run])).not.toThrow(); + for (const field of ["connectionId", "grantId", "provider", "method", "responsibleUserId", "mode"]) { + expect(() => assertHireRuns(scoped, f.agent.id, [f.task], [{ ...run, + contextSnapshot: { ...run.contextSnapshot, aiConnection: { ...run.contextSnapshot.aiConnection, [field]: "foreign" } } }])).toThrow("hire_run_credential_mismatch"); + } + }); + it("keeps unexpected authored children visible for failure cleanup without grading them as the requested work", () => { + const f = fixture(), extra = { ...f.task, id: "extra", title: `Unexpected ${scope.campaignId}` }; + const observed = observedHireResources(scope, { ...f.state, tasks: [...f.state.tasks, extra], activity: [...f.state.activity, f.event("issue.created", extra.id)] }); + expect(observed.tasks).toHaveLength(2); expect(observed.expectedTasks).toEqual([f.task]); + const mismatch = observedHireResources(scope, { ...f.state, tasks: [{ ...f.task, projectId: "other" }] }); + expect(mismatch.tasks).toHaveLength(1); expect(mismatch.scopeMismatch).toBe(true); + }); + it("binds lead and worker runs to their exact tasks, provider, model and environment", () => { + const f = fixture(); expect(() => assertHireRuns(scope, f.agent.id, [f.task], [f.leadRun, f.taskRun])).not.toThrow(); + for (const change of [{ companyId: "other" }, { agentId: scope.leadId }, { issueId: "other" }, { status: "failed" }, + { runtimeMode: "legacy" }, { contextSnapshot: { paperclipEnvironment: { id: "local" } } }, + { runnerProfileJson: { nativeExecutionInput: { provider: { kind: "opencode", model: "other" } } } }]) { + expect(() => assertHireRuns(scope, f.agent.id, [f.task], [f.leadRun, { ...f.taskRun, ...change }])).toThrow(); + } + expect(() => assertHireRuns(scope, f.agent.id, [f.task], [f.taskRun, f.taskRun])).toThrow(); + expect(() => assertHireRuns(scope, f.agent.id, [f.task], [f.leadRun, f.taskRun], 5)).toThrow(); + }); + it("rejects a lead task run from create-self-then-reassign even after the saved assignee is correct", () => { + const f = fixture(); + const delegation = { ...f.run("delegate-chat", scope.chatId, scope.leadId), status: "running" }; + const extra = { ...f.run("self-assigned-task", f.task.id, scope.leadId), status: "cancelled", errorCode: "issue_reassigned" }; + const child = { ...f.taskRun, status: "running" }; + expect(f.task.assigneeAgentId).toBe(f.agent.id); + expect(() => assertHireRuns(scope, f.agent.id, [f.task], [f.leadRun, delegation, extra, child])).toThrow("hire_run_scope_mismatch"); + expect(() => assertHireRuns(scope, f.agent.id, [f.task], [f.leadRun, f.taskRun])).not.toThrow(); + }); + it("accounts for three requested lead turns, two exact completion callbacks and two child runs", () => { + const { f, second, callbacks, runs, comments, grade } = completionFixture(); + expect(() => grade()).not.toThrow(); + expect(() => grade(runs, [])).toThrow("hire_completion_reply_missing"); + expect(assertHireRuns(scope, f.agent.id, [f.task, second], runs, undefined, ["message-1", "message-2", "message-3"], [])).toBe(false); + expect(() => assertHireRuns(scope, f.agent.id, [f.task, second], runs, undefined, ["message-1", "message-2", "message-3"], + comments.map(comment => ({ ...comment, authorAgentId: "foreign" })))).toThrow("hire_completion_reply_mismatch"); + for (const change of [{ invocationSource: "on_demand" }, { contextSnapshot: { ...callbacks[0]!.contextSnapshot, chatCompletionUpdates: [{ id: "foreign", status: "done" }] } }, + { contextSnapshot: { ...callbacks[0]!.contextSnapshot, chatCompletionDeliveryIds: ["delivery-1"] } }, + { contextSnapshot: { ...callbacks[0]!.contextSnapshot, wakeReason: "arbitrary_extra_turn" } }]) { + const corrupted = runs.map(row => row.id === "completion-0" ? { ...row, ...change } : row); + expect(() => grade(corrupted)).toThrow(); + } + expect(() => grade([...runs, f.run("extra", scope.chatId, scope.leadId)])).toThrow(); + }); + it.each([0, 1])("requires callback %s to save its own completed task link", index => { + const { runs, comments, grade } = completionFixture(); + const changed = comments.map((comment, i) => i === index ? { ...comment, body: "Completed saved result" } : comment); + expect(() => grade(runs, changed)).toThrow("hire_completion_reply_missing"); + }); + it.each(["foreign", "code", "fenced-code", "wrong-task", "earlier-handoff-only"])("rejects a %s completion link despite seven successful native runs", variant => { + const { f, requested, runs, comments, grade } = completionFixture(); + const changed = comments.map((comment, i) => ({ ...comment, body: variant === "foreign" + ? `[TEST-${i + 1}](https://foreign.test/TEST/issues/TEST-${i + 1})` + : variant === "code" ? `\`[saved result](/TEST/issues/TEST-${i + 1})\`` + : variant === "fenced-code" ? `\`\`\`markdown\n[saved result](/TEST/issues/TEST-${i + 1})\n\`\`\`` + : variant === "wrong-task" ? "[other result](/TEST/issues/TEST-999)" : "Completed saved result" })); + if (variant === "earlier-handoff-only") changed.push({ ...comments[0]!, createdByRunId: requested[1]!.id, + body: `[first](/TEST/issues/${f.task.identifier}) and [second](/TEST/issues/TEST-2)` }); + expect(runs).toHaveLength(7); + expect(() => grade(runs, changed)).toThrow("hire_completion_reply_missing"); + }); + it("requires immutable native creation ownership independently of current assignment and run count", () => { + const f = fixture(); + expect(() => assertHireTaskCreation(scope, f.agent.id, [f.task], f.state.activity, [f.leadRun])).not.toThrow(); + for (const mutate of [ + (event: Row) => { event.details.assigneeAgentId = scope.leadId; event.details.status = "backlog"; }, + (event: Row) => { delete event.details.assigneeAgentId; }, + (event: Row) => { event.details.assigneeUserId = "user"; }, + (event: Row) => { event.actorId = "foreign"; }, + (event: Row) => { event.companyId = "foreign"; }, + (event: Row) => { event.runId = "foreign"; }, + ]) { + const activity = structuredClone(f.state.activity); mutate(activity[1]!); + expect(f.task.assigneeAgentId).toBe(f.agent.id); + expect(() => assertHireTaskCreation(scope, f.agent.id, [f.task], activity, [f.leadRun])).toThrow("hire_initial_assignee_mismatch"); + } + }); + it("accepts the actual UI's task auto-links and reference-style links in each callback", () => { + const { runs, comments, grade } = completionFixture(); + expect(() => grade(runs, comments.map((comment, i) => ({ ...comment, body: `Completed TEST-${i + 1}.` })))).not.toThrow(); + expect(() => grade(runs, comments.map((comment, i) => ({ ...comment, + body: `Completed [saved result][task].\n\n[task]: /TEST/issues/TEST-${i + 1}` })))).not.toThrow(); + }); + it.each(["heading", "list", "table", "fence", "table-link"])("accepts callback task links in %s replies without grading unrelated Markdown", shape => { + const { runs, comments, grade } = completionFixture(); + const changed = comments.map((comment, i) => { + const link = `[saved result](/TEST/issues/TEST-${i + 1})`; + const table = "| Check | Result |\n| --- | --- |\n| File | Saved |"; + return { ...comment, body: shape === "heading" ? `## Completed\n\n${link}` + : shape === "list" ? `- Completed\n - ${link}` + : shape === "table" ? `${link}\n\n${table}` + : shape === "fence" ? `\`\`\`markdown\n[not a link](/TEST/issues/TEST-999)\n\`\`\`\n\n${link}` + : `| Result |\n| --- |\n| ${link} |` }; + }); + expect(() => grade(runs, changed)).not.toThrow(); + }); + it("requires the saved document revision to come from the successful hired-agent work run", () => { + const f = fixture(), body = "Reference: independent-marker"; + const document = { id: "doc", issueId: f.task.id, companyId: scope.companyId, key: "checklist", body, latestRevisionId: "revision" }; + const revision: Row = { id: "revision", body, createdByAgentId: f.agent.id, createdByRunId: f.taskRun.id }; + const grade = (doc = document, revisions = [revision], runs = [f.taskRun]) => assertHireDocument(scope, f.task, doc, revisions, runs, "independent-marker", f.agent.id); + expect(() => grade()).not.toThrow(); + for (const change of [{ body: "old output" }, { createdByAgentId: "board" }, { createdByRunId: f.leadRun.id }, { id: "stale-revision" }]) { + expect(() => grade(document, [{ ...revision, ...change }])).toThrow(); + } + expect(() => grade({ ...document, issueId: "other" })).toThrow(); + expect(() => grade({ ...document, key: "plan" })).toThrow(); + expect(() => grade(document, [])).toThrow(); + expect(() => grade(document, [revision], [])).toThrow(); + }); +}); diff --git a/tests/runner-e2e/product-journeys/hire-delegate-evidence.ts b/tests/runner-e2e/product-journeys/hire-delegate-evidence.ts new file mode 100644 index 00000000000..93ae767e4b0 --- /dev/null +++ b/tests/runner-e2e/product-journeys/hire-delegate-evidence.ts @@ -0,0 +1,209 @@ +import { isDeepStrictEqual } from "node:util"; +import { createRequire } from "node:module"; +import { applyCompanyPrefix } from "../../../ui/src/lib/company-routes.js"; +import { parseIssueReferenceFromHref, remarkLinkIssueReferences } from "../../../ui/src/lib/issue-reference.js"; +import { JourneyError } from "./types.js"; + +type Row = Record; +const uiRequire = createRequire(new URL("../../../ui/package.json", import.meta.url)); +const [markdown, gfm, react] = await Promise.all([import(uiRequire.resolve("react-markdown")), + import(uiRequire.resolve("remark-gfm")), import(uiRequire.resolve("react"))]); +const Markdown = markdown.default as (input: { children: string; remarkPlugins: unknown[] }) => unknown; +const isValidElement = react.isValidElement as (node: unknown) => node is { type: unknown; props: { href?: unknown; children?: unknown } }; +function hireReplyLinks(body: string, issuePrefix: string): string[] { + const nodes: unknown[] = [Markdown({ children: body, remarkPlugins: [gfm.default, [remarkLinkIssueReferences, { knownPrefixes: [issuePrefix] }]] })]; + const links: string[] = []; + while (nodes.length) { + const node = nodes.pop(); + if (Array.isArray(node)) { nodes.push(...node); continue; } + if (!isValidElement(node) || node.type === "pre" || node.type === "code") continue; + if (node.type === "a" && typeof node.props.href === "string") links.push(node.props.href); + nodes.push(node.props.children); + } + return links; +} +export const HIRE_DELEGATE_JOURNEY = "hire-delegate-reuse" as const; +export function hireDelegateNames(campaignId: string) { + return { project: `Journey QA hire-delegate-reuse ${campaignId}`, hire: `Morgan Reviewer ${campaignId}`, + checklist: `Checklist ${campaignId}`, review: `Review ${campaignId}` }; +} +export interface HireScope { + companyId: string; leadId: string; chatId: string; projectId: string; campaignId: string; + origin: string; issuePrefix: string; + model: string; provider: string; environmentId: string; connectionBinding: unknown; + connectionChoice?: HireConnectionChoice; +} +export interface HireConnectionChoice { + connectionId: string; grantId: string; provider: string; method: string; responsibleUserId: string; +} +export async function readHireConnectionChoice(api: { get(route: string): Promise }, + input: { companyId: string; responsibleUserId: string; binding: unknown; hiredId?: string }): Promise { + const binding = input.binding as Row | undefined; + if (binding?.mode !== "delegated") return; + const fail = (): never => { throw new JourneyError("failed", "hire_connection_choice_mismatch"); }; + if (![binding.connectionId, binding.grantId, binding.provider, binding.method, input.responsibleUserId] + .every(value => typeof value === "string" && value)) fail(); + const listed = await api.get(`/api/companies/${input.companyId}/ai-connections`); + if (listed.currentUserId !== input.responsibleUserId || !Array.isArray(listed.connections) + || listed.connections.some((row: Row) => !row || row.companyId !== input.companyId)) fail(); + const defaults = listed.connections.filter((row: Row) => row.provider === binding.provider && row.isDefault === true); + const selected = defaults[0]; + if (defaults.length !== 1 || selected.companyId !== input.companyId || selected.id !== binding.connectionId + || selected.grantId !== binding.grantId || selected.method !== binding.method || selected.status !== "connected" + || selected.ownership !== "personal" || selected.ownerUserId !== input.responsibleUserId || selected.unavailableReason) fail(); + if (input.hiredId) { + const installed = await api.get(`/api/tool-connections/${binding.connectionId}/installs`); + if (installed.connectionId !== binding.connectionId || !Array.isArray(installed.installs) + || installed.installs.some((row: Row) => row.companyId !== input.companyId) + || installed.installs.filter((row: Row) => row.targetType === "agent" && row.targetId === input.hiredId).length !== 1) fail(); + } + return { connectionId: binding.connectionId, grantId: binding.grantId, provider: binding.provider, + method: binding.method, responsibleUserId: input.responsibleUserId }; +} + +function approvedFallback(scope: HireScope, hiredBinding: unknown) { + const binding = scope.connectionBinding as Row | undefined, choice = scope.connectionChoice; + return binding?.mode === "delegated" && choice && choice.connectionId === binding.connectionId && choice.grantId === binding.grantId + && choice.provider === binding.provider && choice.method === binding.method && typeof choice.responsibleUserId === "string" + && Boolean(choice.responsibleUserId) && isDeepStrictEqual(hiredBinding, + { provider: choice.provider, method: "api_key", mode: "responsible_user" }); +} +export const hireRunIssueId = (run: Row) => run.issueId ?? run.nativeIssueId ?? run.contextSnapshot?.issueId ?? run.contextSnapshot?.taskId; + +export function assertHireAuditCoverage(activity: Row[], leadRuns: Row[]) { + if (!Array.isArray(activity) || activity.length > 500) throw new JourneyError("failed", "hire_activity_coverage_unavailable"); + if (activity.length < 500) return; + if (!leadRuns.length) return; + const starts = leadRuns.map(run => Date.parse(run.startedAt ?? run.createdAt)); + const oldest = Math.min(...activity.map(event => Date.parse(event.createdAt))); + if (!starts.every(Number.isFinite) || !Number.isFinite(oldest) || oldest > Math.min(...starts)) { + throw new JourneyError("failed", "hire_activity_coverage_unavailable"); + } +} + +export function observedHireResources(scope: HireScope, state: { agents: A[]; tasks: T[]; activity: Row[]; leadRuns: Row[] }) { + const names = hireDelegateNames(scope.campaignId); + const authored = (row: Row, actions: string[]) => state.activity.some(event => event.companyId === scope.companyId + && event.actorType === "agent" && event.actorId === scope.leadId && event.agentId === scope.leadId + && event.entityId === row.id && actions.includes(event.action) + && state.leadRuns.some(run => run.id === event.runId && run.agentId === scope.leadId + && run.companyId === scope.companyId && hireRunIssueId(run) === scope.chatId)); + const agents = state.agents.filter(agent => authored(agent, ["agent.hire_created", "agent.created"])); + const tasks = state.tasks.filter(task => authored(task, ["issue.created", "issue.child_created"])); + const ownable = (row: Row, name: unknown) => row.companyId === scope.companyId && typeof row.id === "string" && row.id + && typeof name === "string" && Boolean(name); + const ownedAgents = agents.filter(agent => ownable(agent, agent.name)), ownedTasks = tasks.filter(task => ownable(task, task.title)); + return { agents: ownedAgents, tasks: ownedTasks, hires: ownedAgents.filter(agent => agent.name === names.hire), + expectedTasks: ownedTasks.filter(task => [names.checklist, names.review].includes(task.title)), + scopeMismatch: state.activity.some(event => [...agents, ...tasks].some(row => row.id === event.entityId) + && state.leadRuns.some(run => run.id === event.runId && run.contextSnapshot?.wakeReason === "chat_task_completed")) + || agents.some(agent => !ownable(agent, agent.name) || agent.reportsTo !== scope.leadId) + || tasks.some(task => !ownable(task, task.title) || task.projectId !== scope.projectId) }; +} +export function assertHireBinding(scope: HireScope, agents: Row[], hiredId?: string) { + const names = hireDelegateNames(scope.campaignId); + const hires = agents.filter(agent => agent.name === names.hire); + if (agents.length !== 1 || hires.length !== 1) throw new JourneyError("failed", "exactly_one_native_hire_required"); + const hired = hires[0]!; + if (hired.id === scope.leadId || hiredId && hired.id !== hiredId || hired.companyId !== scope.companyId + || hired.reportsTo !== scope.leadId || hired.adapterType !== "paperclip_runner" || hired.adapterConfig?.model !== scope.model + || !scope.connectionBinding || (!isDeepStrictEqual(hired.runtimeConfig?.aiConnection, scope.connectionBinding) + && !approvedFallback(scope, hired.runtimeConfig?.aiConnection))) { + throw new JourneyError("failed", "hire_native_binding_mismatch"); + } + return hired; +} +export function assertHireTaskCreation(scope: HireScope, hiredId: string | undefined, tasks: Row[], activity: Row[], leadRuns: Row[]) { + for (const task of tasks) { + const created = activity.filter(event => event.companyId === scope.companyId && event.actorType === "agent" + && event.actorId === scope.leadId && event.agentId === scope.leadId && event.entityType === "issue" + && event.entityId === task.id && ["issue.created", "issue.child_created"].includes(event.action) + && leadRuns.some(run => run.id === event.runId && run.companyId === scope.companyId && run.agentId === scope.leadId + && run.runtimeMode === "native" && hireRunIssueId(run) === scope.chatId)); + if (!hiredId || task.companyId !== scope.companyId || !created.length + || created.some(event => event.details?.assigneeAgentId !== hiredId || event.details?.assigneeUserId !== null)) { + throw new JourneyError("failed", "hire_initial_assignee_mismatch"); + } + } +} +export function hireResultLinkMatches(scope: Pick, task: Row, href: string, source = false) { + try { + const reference = source ? parseIssueReferenceFromHref(href) : null; + const url = new URL(reference ? applyCompanyPrefix(reference.href, scope.issuePrefix) : href, scope.origin); + return url.origin === scope.origin && [task.id, task.identifier].some(id => typeof id === "string" + && url.pathname === `/${scope.issuePrefix}/issues/${id}`); + } catch { return false; } +} +export function assertHireRuns(scope: HireScope, hiredId: string | undefined, tasks: Row[], runs: Row[], count?: number, requestedCommentIds?: string[], comments?: Row[]) { + const bindings = new Map([[scope.chatId, scope.leadId], ...tasks.map(task => [task.id, hiredId] as [string, string | undefined])]); + if (runs.length > 7 || count !== undefined && runs.length !== count || new Set(runs.map(run => run.id)).size !== runs.length) { + throw new JourneyError("failed", "hire_run_limit_mismatch"); + } + const requests = new Set(), deliveries = new Set(), completedTasks = new Set(); + let callbackCount = 0, completionRepliesReady = true; + for (const run of runs) { + const issueId = hireRunIssueId(run), provider = run.runnerProfileJson?.nativeExecutionInput?.provider; + if (!bindings.has(issueId) || !bindings.get(issueId) || run.companyId !== scope.companyId || run.agentId !== bindings.get(issueId)) { + throw new JourneyError("failed", "hire_run_scope_mismatch"); + } + if (requestedCommentIds && run.agentId === scope.leadId) { + const context = run.contextSnapshot ?? {}; + if (context.wakeReason === "chat_task_completed") { + callbackCount++; + const ids = context.chatCompletionDeliveryIds, updates = context.chatCompletionUpdates; + if (callbackCount > 2 || run.invocationSource !== "automation" || run.triggerDetail !== "system" + || context.wakeSource !== "automation" || context.wakeTriggerDetail !== "system" || context.wakeCommentId + || !Array.isArray(ids) || ids.length !== 1 || typeof ids[0] !== "string" || !ids[0] || deliveries.has(ids[0])) { + throw new JourneyError("failed", "hire_completion_binding_mismatch"); + } + deliveries.add(ids[0]); + if (run.status === "succeeded" || updates !== undefined) { + if (!Array.isArray(updates) || updates.length !== 1 || updates[0]?.status !== "done" + || !tasks.some(task => task.id === updates[0]?.id && task.status === "done") || completedTasks.has(updates[0].id)) { + throw new JourneyError("failed", "hire_completion_binding_mismatch"); + } + if (run.status === "succeeded" && comments) { + const replies = comments.filter(comment => comment.createdByRunId === run.id); + if (replies.some(comment => comment.companyId !== scope.companyId || comment.issueId !== scope.chatId + || comment.authorAgentId !== scope.leadId || comment.authorUserId)) throw new JourneyError("failed", "hire_completion_reply_mismatch"); + const task = tasks.find(task => task.id === updates[0].id)!; + if (!replies.some(comment => typeof comment.body === "string" + && hireReplyLinks(comment.body, scope.issuePrefix).some(href => hireResultLinkMatches(scope, task, href, true)))) { + completionRepliesReady = false; + } + } + completedTasks.add(updates[0].id); + } + } else { + if (context.source !== "issue.comment" || !requestedCommentIds.includes(context.wakeCommentId) + || requests.has(context.wakeCommentId) || requests.size >= 3) throw new JourneyError("failed", "hire_requested_turn_mismatch"); + requests.add(context.wakeCommentId); + } + } + if (!["queued", "running", "succeeded"].includes(run.status)) throw new JourneyError("failed", "hire_native_run_failed"); + if (run.status === "succeeded" && (run.runtimeMode !== "native" || provider?.kind !== scope.provider || provider?.model !== scope.model + || run.contextSnapshot?.paperclipEnvironment?.id !== scope.environmentId)) throw new JourneyError("failed", "hire_run_runtime_mismatch"); + if (run.status === "succeeded" && scope.connectionChoice) { + const actual = run.contextSnapshot?.aiConnection, expected = scope.connectionChoice; + if (actual?.connectionId !== expected.connectionId || actual.grantId !== expected.grantId + || actual.provider !== expected.provider || actual.method !== expected.method || actual.responsibleUserId !== expected.responsibleUserId + || !(run.agentId === scope.leadId ? actual.mode === "delegated" : ["delegated", "responsible_user"].includes(actual.mode))) { + throw new JourneyError("failed", "hire_run_credential_mismatch"); + } + } + } + if (requestedCommentIds && count !== undefined && (requests.size !== requestedCommentIds.length + || callbackCount !== tasks.length || completedTasks.size !== tasks.length)) throw new JourneyError("failed", "hire_turn_accounting_mismatch"); + if (count !== undefined && !completionRepliesReady) throw new JourneyError("failed", "hire_completion_reply_missing"); + return completionRepliesReady; +} +export function assertHireDocument(scope: HireScope, task: Row, document: Row, revisions: Row[], runs: Row[], marker: string, hiredId: string) { + const work = runs.filter(run => hireRunIssueId(run) === task.id); + const revision = revisions.find(row => row.id === document.latestRevisionId); + if (task.companyId !== scope.companyId || task.assigneeAgentId !== hiredId || task.status !== "done" || task.projectId !== scope.projectId + || work.length !== 1 || work[0]?.status !== "succeeded" || document.issueId !== task.id || document.companyId !== scope.companyId + || document.key === "plan" || !document.body?.includes(marker) || !revision || revision.body !== document.body + || revision.createdByAgentId !== hiredId || revision.createdByRunId !== work[0]?.id) { + throw new JourneyError("failed", "hired_document_provenance_mismatch"); + } +} diff --git a/tests/runner-e2e/product-journeys/hire-delegate-flow.test.ts b/tests/runner-e2e/product-journeys/hire-delegate-flow.test.ts new file mode 100644 index 00000000000..c954a0083dd --- /dev/null +++ b/tests/runner-e2e/product-journeys/hire-delegate-flow.test.ts @@ -0,0 +1,357 @@ +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, expect, it, vi } from "vitest"; +import { parseAgentMentionHref, parseProjectMentionHref } from "../../../packages/shared/src/project-mentions.js"; +import { ConnectionFailure } from "../connection-config.js"; +import { sendChatMessage, readChatOutputDocument } from "../chat-flow.js"; +import { hireDelegateReuse, hireHandoffMentions, openSavedHireResult } from "./hire-delegate-flow.js"; +import { hireDelegateNames } from "./hire-delegate-evidence.js"; +import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; + +vi.mock("@playwright/test", () => ({ expect: (value: unknown) => Object.assign(expect(value), { toBeVisible: async () => {}, toHaveURL: async (predicate: (url: URL) => boolean) => { expect(predicate(new URL((value as Row).currentUrl ?? "https://qa.example.test/QA/issues/QA-2"))).toBe(true); } }) })); +vi.mock("../chat-flow.js", () => ({ sendChatMessage: vi.fn(), readChatOutputDocument: vi.fn() })); +vi.mock("./hire-delegate-composer.js", () => ({ sendHireHandoffMessage: (page: any, text: string) => sendChatMessage(page, text) })); +type Row = Record; +const directories: string[] = []; +afterEach(async () => { vi.mocked(sendChatMessage).mockReset(); await Promise.all(directories.splice(0).map(directory => rm(directory, { recursive: true, force: true }))); }); + +async function fixture(options: { readError?: Error; mutate?(agents: Row[], tasks: Row[]): void; ownError?: Error; firstSendError?: Error; connectionList?: Row; connectionReadError?: Error } = {}) { + const campaignId = "journey-1-deadbeef", names = hireDelegateNames(campaignId), owned: Row[] = [], comments: Row[] = []; + const binding = { connectionId: "approved" }, lead = { id: "lead", companyId: "company", name: `Lead ${campaignId}`, permissions: { canCreateAgents: true }, runtimeConfig: { aiConnection: binding } }; + const hire = { id: "hired", companyId: "company", name: names.hire, reportsTo: lead.id, adapterType: "paperclip_runner", + adapterConfig: { model: "fixed-model" }, runtimeConfig: { aiConnection: binding } }; + const agents: Row[] = [lead, hire], tasks: Row[] = []; + const chat = { id: "chat", companyId: "company", title: `Chat with ${lead.name}`, assigneeAgentId: lead.id, conversationAgentId: lead.id, + conversationUserId: "user", createdByUserId: "user", status: "in_review", conversationState: "waiting" }; + const run = { id: "lead-run", companyId: "company", agentId: lead.id, issueId: chat.id, status: "succeeded", runtimeMode: "native", + contextSnapshot: { source: "issue.comment", wakeCommentId: "user-comment", paperclipEnvironment: { id: "environment" } }, runnerProfileJson: { nativeExecutionInput: { provider: { kind: "opencode", model: "fixed-model" } } } }; + options.mutate?.(agents, tasks); + if ((lead.runtimeConfig.aiConnection as Row).mode === "delegated") Object.assign(run.contextSnapshot, + { aiConnection: { ...lead.runtimeConfig.aiConnection, responsibleUserId: "user" } }); + const audit = (action: string, entityId: string) => ({ companyId: "company", actorType: "agent", actorId: lead.id, agentId: lead.id, runId: run.id, action, entityId, entityType: action.startsWith("agent.") ? "agent" : "issue" }); + const activity = [audit("agent.hire_created", hire.id), ...tasks.map(task => audit("issue.created", task.id))]; + let sent = false, failedRead = false, failedConnectionRead = false, runReads = 0; + const stop = new JourneyError("failed", "test_stop_before_delegation"); + vi.mocked(sendChatMessage).mockImplementation(async (_page, text) => { + if (sent) throw stop; + sent = true; if (options.firstSendError) throw options.firstSendError; comments.push({ id: "user-comment", authorUserId: "user", body: text }); + }); + const api = { origin: "https://qa.example.test", async get(route: string) { + if (route === "/api/instance/settings/experimental") return { enableAgentChat: true }; + if (route === "/api/companies/company") return { requireBoardApprovalForNewAgents: false }; + if (route === "/api/agents/lead") return lead; + if (route === "/api/companies/company/ai-connections") { + if (sent && options.connectionReadError && !failedConnectionRead) { failedConnectionRead = true; throw options.connectionReadError; } + return options.connectionList ?? { currentUserId: "user", connections: [] }; + } + if (route.endsWith("/installs")) return { connectionId: "approved", installs: [{ companyId: "company", targetType: "agent", targetId: "hired" }] }; + if (route === "/api/companies/company/chats/lead") return sent ? chat : null; + if (route === "/api/issues/chat/runs") { + runReads++; + if (options.readError && !failedRead) { failedRead = true; throw options.readError; } + return [{ runId: run.id }]; + } + if (route === "/api/heartbeat-runs/lead-run") return run; + if (route === "/api/agents/hired") return agents.find(row => row.id === "hired"); + if (route.startsWith("/api/issues/") && tasks.some(row => route === `/api/issues/${row.id}`)) return tasks.find(row => route === `/api/issues/${row.id}`); + if (route === "/api/companies/company/agents") return agents; + if (route === "/api/companies/company/issues") return tasks; + if (route === "/api/companies/company/activity?agentId=lead&limit=500") return activity; + if (route === "/api/issues/chat/comments?order=asc") return structuredClone(comments); + throw new Error(`Unexpected read ${route}`); + }, async post(route: string) { + expect(route).toBe("/api/companies/company/projects"); return { id: "project", companyId: "company", name: names.project }; + } }; + const context = { api, page: { async goto() {}, getByTestId() { return {}; } }, user: { id: "user" }, company: { id: "company", issuePrefix: "QA" }, + agent: { id: lead.id, model: "fixed-model", provider: "opencode", runtimeMode: "native" }, environmentId: "environment" } as unknown as JourneyContext; + const session = { async own(row: Row) { if (row.id === hire.id && options.ownError) throw options.ownError; owned.push(row); } } as unknown as TargetSession; + const evidence = { checks: [] } as unknown as JourneyEvidence; + const directory = await mkdtemp(path.join(os.tmpdir(), "hire-observation-")); directories.push(directory); + return { owned, evidence, stop, directory, runReads: () => runReads, + run: () => hireDelegateReuse(context, session, campaignId, evidence, { deadlineAt: Date.now() + 10_000, directory, capture: async () => {} }) }; +} + +function handoffTargets(text: string) { + const hrefs = [...text.matchAll(/\]\(([^)]+)\)/g)].map(match => match[1]!); + return { agents: hrefs.map(parseAgentMentionHref).filter(value => value !== null).map(value => value.agentId), + projects: hrefs.map(parseProjectMentionHref).filter(value => value !== null).map(value => value.projectId) }; +} + +it("carries the independently verified teammate and project into the actual delegation message", async () => { + const f = await fixture(); + await expect(f.run()).rejects.toBe(f.stop); + expect(handoffTargets(vi.mocked(sendChatMessage).mock.calls[1]![1])).toEqual({ agents: ["hired"], projects: ["project"] }); + expect(sendChatMessage).toHaveBeenCalledTimes(2); +}); + +it.each(["missing-agent", "missing-project", "foreign-agent", "foreign-project", "wrong-agent", "wrong-project", "missing-scope", "malformed-id"])( + "rejects %s handoff context instead of emitting an unbound mention", mismatch => { + const scope = { companyId: "company", leadId: "lead", projectId: "project", campaignId: "journey-1-deadbeef" }; + const names = hireDelegateNames(scope.campaignId), hired = { id: "hired", companyId: "company", reportsTo: "lead", name: names.hire }; + const project = { id: "project", companyId: "company", name: names.project }; + if (mismatch === "foreign-agent") hired.companyId = "foreign"; + if (mismatch === "foreign-project") project.companyId = "foreign"; + if (mismatch === "wrong-agent") hired.id = "other"; + if (mismatch === "wrong-project") project.id = "other"; + if (mismatch === "missing-scope") scope.companyId = ""; + if (mismatch === "malformed-id") hired.id = "hired?i=code"; + expect(() => hireHandoffMentions(scope, mismatch === "malformed-id" ? hired.id : "hired", + mismatch === "missing-agent" ? undefined! : hired, mismatch === "missing-project" ? undefined! : project)) + .toThrow("hire_handoff_context_unverified"); + }, +); + +it.each(["api_status_503", "api_request_failed"])("recovers a transient %s read without replaying the first user turn", async code => { + const f = await fixture({ readError: new ConnectionFailure(code) }); + await expect(f.run()).rejects.toBe(f.stop); + expect(f.runReads()).toBe(2); + expect(f.evidence.checks).toContainEqual({ id: "one-attributed-native-hire", passed: true }); + expect(vi.mocked(sendChatMessage).mock.calls.filter(([, text]) => text.startsWith("Hire exactly one"))).toHaveLength(1); +}); +it.each([new JourneyError("failed", "api_status_503"), new ConnectionFailure("api_status_403"), new ConnectionFailure("api_status_500"), new Error("api_request_failed")])( + "never retries a candidate, identity, application or unknown error: %s", async readError => { + const f = await fixture({ readError }); await expect(f.run()).rejects.toBe(readError); + expect(f.runReads()).toBe(1); expect(sendChatMessage).toHaveBeenCalledTimes(1); + }, +); +it("never retries ownership callbacks, whose mutations may already have committed", async () => { + const ownError = new ConnectionFailure("api_request_failed"), f = await fixture({ ownError }); + await expect(f.run()).rejects.toBe(ownError); expect(f.runReads()).toBe(1); expect(sendChatMessage).toHaveBeenCalledTimes(1); +}); +it.each(["manager", "project", "foreign-company"])("owns proven same-company children before rejecting a %s scope mismatch", async mismatch => { + const f = await fixture({ mutate(agents, tasks) { + if (mismatch === "manager") agents[1]!.reportsTo = "other"; + else tasks.push({ id: "unexpected-task", companyId: mismatch === "foreign-company" ? "foreign" : "company", + title: "Unexpected task", projectId: "other-project" }); + } }); + await expect(f.run()).rejects.toMatchObject({ code: "hire_resource_scope_mismatch" }); + expect(f.owned.some(row => row.id === "hired")).toBe(true); + expect(f.owned.some(row => row.id === "unexpected-task")).toBe(mismatch === "project"); + expect(sendChatMessage).toHaveBeenCalledTimes(1); +}); + +it("persists an exact fresh-chat intent before a first send can commit and fail", async () => { + const firstSendError = new ConnectionFailure("api_request_failed"), f = await fixture({ firstSendError }); + await expect(f.run()).rejects.toBe(firstSendError); + expect(f.owned).toContainEqual({ kind: "issue", name: "Chat with Lead journey-1-deadbeef", journeyId: "hire-delegate-reuse", + freshChat: { companyId: "company", userId: "user", leadAgentId: "lead", absent: true } }); + expect(sendChatMessage).toHaveBeenCalledTimes(1); +}); + +it("rejects missing actual-user readiness before any child or chat creation intent and before sending", async () => { + const f = await fixture({ mutate(agents) { + agents[0]!.runtimeConfig.aiConnection = { provider: "openrouter", method: "api_key", mode: "delegated", connectionId: "approved", grantId: "grant" }; + }, connectionList: { currentUserId: "operator", connections: [] } }); + await expect(f.run()).rejects.toMatchObject({ code: "hire_connection_choice_mismatch" }); + expect(f.owned).toEqual([]); expect(sendChatMessage).not.toHaveBeenCalled(); +}); + +it("validates the real-user fallback without retaining raw credential-choice references in observation evidence", async () => { + const f = await fixture({ mutate(agents) { + agents[0]!.runtimeConfig.aiConnection = { provider: "openrouter", method: "api_key", mode: "delegated", connectionId: "approved", grantId: "grant" }; + agents[1]!.runtimeConfig.aiConnection = { provider: "openrouter", method: "api_key", mode: "responsible_user" }; + }, connectionList: { currentUserId: "user", connections: [{ companyId: "company", id: "approved", grantId: "grant", provider: "openrouter", method: "api_key", + ownership: "personal", ownerUserId: "user", isDefault: true, status: "connected" }] } }); + await expect(f.run()).rejects.toBe(f.stop); + expect(f.evidence.checks).toContainEqual({ id: "one-attributed-native-hire", passed: true }); + const observed = await readFile(path.join(f.directory, "hire-delegate-observation.json"), "utf8"); + expect(observed).not.toMatch(/connectionId|grantId|responsibleUserId|approved/); +}); + +it.each(["api_status_503", "api_request_failed"])("retries a transient %s credential observation without replaying hire or ownership", async code => { + const f = await fixture({ connectionReadError: new ConnectionFailure(code), mutate(agents) { + agents[0]!.runtimeConfig.aiConnection = { provider: "openrouter", method: "api_key", mode: "delegated", connectionId: "approved", grantId: "grant" }; + agents[1]!.runtimeConfig.aiConnection = { provider: "openrouter", method: "api_key", mode: "responsible_user" }; + }, connectionList: { currentUserId: "user", connections: [{ companyId: "company", id: "approved", grantId: "grant", provider: "openrouter", method: "api_key", + ownership: "personal", ownerUserId: "user", isDefault: true, status: "connected" }] } }); + await expect(f.run()).rejects.toBe(f.stop); + expect(f.runReads()).toBe(2); expect(f.owned.filter(row => row.id === "hired")).toHaveLength(1); + expect(vi.mocked(sendChatMessage).mock.calls.filter(([, text]) => text.startsWith("Hire exactly one"))).toHaveLength(1); +}); + +it.each([new ConnectionFailure("api_status_403"), new JourneyError("failed", "hire_connection_choice_mismatch")])( + "never retries authentication or approval failure in credential observation: %s", async connectionReadError => { + const f = await fixture({ connectionReadError, mutate(agents) { + agents[0]!.runtimeConfig.aiConnection = { provider: "openrouter", method: "api_key", mode: "delegated", connectionId: "approved", grantId: "grant" }; + agents[1]!.runtimeConfig.aiConnection = { provider: "openrouter", method: "api_key", mode: "responsible_user" }; + }, connectionList: { currentUserId: "user", connections: [{ companyId: "company", id: "approved", grantId: "grant", provider: "openrouter", method: "api_key", + ownership: "personal", ownerUserId: "user", isDefault: true, status: "connected" }] } }); + await expect(f.run()).rejects.toBe(connectionReadError); + expect(f.runReads()).toBe(1); expect(sendChatMessage).toHaveBeenCalledTimes(1); + }, +); + +it("waits for reloaded replies and links, then clicks and reloads the exact saved child", async () => { + let visibleReads = 0, linkReads = 0, clicks = 0, reloads = 0; + const href = "/QA/issues/QA-2"; + const reply = { async isVisible() { return ++visibleReads > 2; }, locator(selector: string) { + if (selector === "a[href]") return { async evaluateAll() { return ++linkReads > 1 ? ["https://foreign.test/QA/issues/QA-2", href] : []; } }; + expect(selector).toBe(`a[href=${JSON.stringify(href)}]`); + return { first() { return { async click() { clicks++; } }; } }; + } }; + const page = { locator(selector: string) { expect(selector).toBe('[id="comment-saved"]'); return reply; }, + async reload() { reloads++; }, getByTestId(id: string) { expect(id).toBe("issue-detail-header"); return { getByRole(role: string, options: Row) { + expect(role).toBe("heading"); expect(options).toEqual({ name: "Saved review", exact: true }); return {}; } }; } }; + expect(await openSavedHireResult(page as any, "https://qa.example.test", "QA", [{ id: "saved", authorAgentId: "lead", companyId: "company", issueId: "chat", createdByRunId: "callback" }], "lead", + { id: "review", identifier: "QA-2", title: "Saved review" }, Date.now() + 3000, { companyId: "company", chatId: "chat", runId: "callback" })).toBe(true); + expect(visibleReads).toBe(4); expect(linkReads).toBe(2); expect(clicks).toBe(1); expect(reloads).toBe(1); +}); +it("ends a missing saved result at the original deadline without a click or replay", async () => { + const page = { locator() { return { async isVisible() { return false; } }; } }; + expect(await openSavedHireResult(page as any, "https://qa.example.test", "QA", [{ id: "saved", authorAgentId: "lead", companyId: "company", issueId: "chat", createdByRunId: "callback" }], "lead", + { id: "review", identifier: "QA-2", title: "Saved review" }, Date.now() + 10, { companyId: "company", chatId: "chat", runId: "callback" })).toBe(false); +}); + +it.each(["stale", "foreign", "truncated", "late-reply", "wrong-author", "activity-linked", "foreign-run", "contradictory-run", "extra-run", "fallback", "fallback-changed", "fallback-subscription", "fallback-subscription-wrong-method", "creation-backlog-reassigned"])("re-reads %s callback task state while preserving strict scope and seven-run accounting", async snapshot => { + const fallback = snapshot.startsWith("fallback"), subscription = snapshot.startsWith("fallback-subscription"); + const provider = subscription ? "codex" : "opencode", accountProvider = subscription ? "openai" : "openrouter", method = subscription ? "subscription" : "api_key"; + const campaignId = "journey-1-deadbeef", names = hireDelegateNames(campaignId), binding = fallback + ? { provider: accountProvider, method, mode: "delegated", connectionId: "approved", grantId: "grant" } : { connectionId: "approved" }; + const lead = { id: "lead", companyId: "company", name: `Lead ${campaignId}`, permissions: { canCreateAgents: true }, + adapterConfig: { model: "fixed-model" }, runtimeConfig: { aiConnection: binding } }; + const hired = { ...lead, id: "hired", name: names.hire, reportsTo: "lead", adapterType: "paperclip_runner" }; + if (fallback) Object.assign(hired, { runtimeConfig: { aiConnection: { provider: accountProvider, method: "api_key", mode: "responsible_user" } } }); + const chat = { id: "chat", companyId: "company", title: `Chat with ${lead.name}`, assigneeAgentId: "lead", conversationAgentId: "lead", + conversationUserId: "user", createdByUserId: "user", status: "in_review", conversationState: "waiting" }; + const runs: Row[] = [], tasks: Row[] = [], comments: Row[] = [], activity: Row[] = [], stages: number[] = []; + let sent = 0, callbackPoll = 0; + const replyPolls = new Map(), nativeReads = new Map(); + const run = (id: string, issueId: string, agentId: string, metadata: Row = {}) => { + const value = { id, companyId: "company", agentId, issueId, status: "succeeded", runtimeMode: "native", + contextSnapshot: { issueId, paperclipEnvironment: { id: "environment" }, aiConnection: fallback + ? { ...binding, ...(snapshot === "fallback-subscription-wrong-method" && agentId !== "lead" ? { method: "api_key" } : {}), + mode: agentId === "lead" ? "delegated" : "responsible_user", responsibleUserId: "user" } : binding, ...metadata }, + runnerProfileJson: { nativeExecutionInput: { provider: { kind: provider, model: "fixed-model" } } } }; + runs.push(value); return value; + }; + const audit = (action: string, entityId: string) => activity.push({ companyId: "company", actorType: "agent", actorId: "lead", agentId: "lead", runId: `lead-${sent}`, action, entityId, entityType: action.startsWith("agent.") ? "agent" : "issue", + details: { assigneeAgentId: snapshot === "creation-backlog-reassigned" && sent === 2 ? "lead" : "hired", assigneeUserId: null, + status: snapshot === "creation-backlog-reassigned" && sent === 2 ? "backlog" : "todo" } }); + vi.mocked(sendChatMessage).mockImplementation(async (_page, text) => { + stages.push(runs.length); sent++; callbackPoll = 0; + comments.push({ id: `user-${sent}`, companyId: "company", issueId: "chat", authorUserId: "user", body: text }); + run(`lead-${sent}`, "chat", "lead", { source: "issue.comment", wakeCommentId: `user-${sent}` }); + if (sent === 1) audit("agent.hire_created", "hired"); + else { + const task = { id: `task-${sent}`, identifier: `QA-${sent - 1}`, companyId: "company", title: sent === 2 ? names.checklist : names.review, + assigneeAgentId: "hired", projectId: "project", parentId: null, status: "done" }; + tasks.push(task); audit("issue.created", task.id); + if (snapshot === "creation-backlog-reassigned" && sent === 2) activity.push({ companyId: "company", actorType: "agent", actorId: "lead", agentId: "lead", + runId: `lead-${sent}`, action: "issue.reassigned", entityId: task.id, entityType: "issue", details: { previousAssigneeAgentId: "lead", assigneeAgentId: "hired" } }); + run(`child-${sent}`, task.id, "hired"); + if (snapshot === "extra-run" && sent === 3) run("extra-native", "chat", "lead", { source: "issue.comment", wakeCommentId: `user-${sent}` }); + } + }); + const marker = `HIRE${campaignId.replace(/[^a-z0-9]/gi, "")}`; + const document = (taskId: string) => ({ id: `doc-${taskId}`, issueId: taskId, companyId: "company", key: "result", updatedAt: "2020-01-01T00:00:00Z", latestRevisionId: `revision-${taskId}`, + body: `Reference: ${taskId === "task-2" ? marker : `REVIEW${marker}`}` }); + vi.mocked(readChatOutputDocument).mockImplementation(async (_api, taskId) => { + if (snapshot === "fallback-changed" && taskId === "task-3") Object.assign(hired, { runtimeConfig: { aiConnection: binding } }); + return document(taskId); + }); + const api = { origin: "https://qa.example.test", async get(route: string) { + if (route === "/api/instance/settings/experimental") return { enableAgentChat: true }; + if (route === "/api/companies/company") return { requireBoardApprovalForNewAgents: false }; + if (route === "/api/agents/lead") return lead; + if (route === "/api/companies/company/chats/lead") return sent ? chat : null; + if (route === "/api/companies/company/ai-connections") return { currentUserId: "user", connections: [{ companyId: "company", id: "approved", grantId: "grant", + provider: accountProvider, method, ownership: "personal", ownerUserId: "user", isDefault: true, status: "connected" }] }; + if (route.endsWith("/installs")) return { connectionId: "approved", installs: [{ companyId: "company", targetType: "agent", targetId: "hired" }] }; + if (route === "/api/agents/hired") return hired; + if (route === "/api/companies/company/agents") return sent ? [lead, hired] : [lead]; + if (route === "/api/companies/company/issues" && snapshot === "truncated") return Array.from({ length: 500 }, (_, i) => ({ id: `old-${i}`, companyId: "company", title: "Higher priority existing work", projectId: "other", status: "backlog" })); + if (route === "/api/companies/company/issues") return tasks.map(task => ({ ...task, + status: runs.some(row => row.id === `completion-${sent}`) && task.id === `task-${sent}` ? "in_progress" : task.status })); + if (route === "/api/companies/company/activity?agentId=lead&limit=500") return activity; + if (route === "/api/issues/chat/comments?order=asc") { + if (["late-reply", "wrong-author"].includes(snapshot)) for (const callback of runs.filter(run => run.id.startsWith("completion-"))) { + if (comments.some(comment => comment.createdByRunId === callback.id)) continue; + const count = (replyPolls.get(callback.id) ?? 0) + 1; replyPolls.set(callback.id, count); + if (snapshot === "wrong-author" || count > 1) comments.push({ id: `reply-${callback.id}`, companyId: "company", issueId: "chat", + authorAgentId: snapshot === "wrong-author" ? "foreign" : "lead", createdByRunId: callback.id, body: `Saved [result](/QA/issues/QA-${Number(callback.id.split("-")[1]) - 1})` }); + } + return structuredClone(comments); + } + if (route === "/api/issues/chat/runs" && sent > 1 && !runs.some(row => row.id === `completion-${sent}`) && ++callbackPoll > 1) { + const callback = run(`completion-${sent}`, "chat", "lead", { wakeReason: "chat_task_completed", wakeSource: "automation", wakeTriggerDetail: "system", + chatCompletionDeliveryIds: [`delivery-${sent}`], chatCompletionUpdates: [{ id: `task-${sent}`, status: "done" }] }); + Object.assign(callback, { invocationSource: "automation", triggerDetail: "system" }); + if (!["late-reply", "wrong-author"].includes(snapshot)) comments.push({ id: `reply-${sent}`, companyId: "company", issueId: "chat", authorAgentId: "lead", createdByRunId: callback.id, body: `Saved [result](/QA/issues/QA-${Number(callback.id.split("-")[1]) - 1})` }); + } + const issueRuns = /^\/api\/issues\/([^/]+)\/runs$/.exec(route); + if (issueRuns) return runs.filter(row => row.issueId === issueRuns[1] + || activity.some(event => event.entityType === "issue" && event.entityId === issueRuns[1] && event.runId === row.id)).map(row => ({ runId: row.id })); + if (route.startsWith("/api/heartbeat-runs/")) { + const row = runs.find(row => route.endsWith(`/${row.id}`))!; + const read = (nativeReads.get(row.id) ?? 0) + 1; nativeReads.set(row.id, read); + if (snapshot === "foreign-run" && row.id === "child-2") return { ...row, companyId: "foreign" }; + if (snapshot === "contradictory-run" && row.id === "lead-2" && read === 2) return { ...row, agentId: "hired", issueId: "task-2", + contextSnapshot: { ...row.contextSnapshot, issueId: "task-2" } }; + return row; + } + const output = /^\/api\/issues\/([^/]+)\/documents\/result(\/revisions)?$/.exec(route); + if (output) return output[2] ? [{ id: document(output[1]!).latestRevisionId, body: document(output[1]!).body, + createdByAgentId: "hired", createdByRunId: `child-${output[1]!.split("-")[1]}` }] : document(output[1]!); + if (route.startsWith("/api/issues/")) { + const task = tasks.find(row => route.endsWith(`/${row.id}`)); + return snapshot === "foreign" && task ? { ...task, projectId: "foreign" } : task; + } + throw new Error(route); + }, async post() { return { id: "project", companyId: "company", name: names.project }; } }; + const visited: string[] = []; + const page: Row = { currentUrl: "https://qa.example.test/QA/chats/lead", async goto(url: string) { page.currentUrl = url; }, async reload() {}, + locator(selector: string) { + const comment = comments.find(row => selector === `[id=${JSON.stringify(`comment-${row.id}`)}]`); + return { async isVisible() { return Boolean(comment); }, locator(selector: string) { + const links = [...String(comment?.body ?? "").matchAll(/\]\(([^)]+)\)/g)].map(match => match[1]!); + if (selector === "a[href]") return { async evaluateAll() { return links; } }; + const href = links.find(href => selector === `a[href=${JSON.stringify(href)}]`); + return { first() { return { async click() { expect(href).toBeDefined(); visited.push(href!); page.currentUrl = new URL(href!, api.origin).href; } }; } }; + } }; + }, getByTestId() { return { getByRole() { return {}; } }; } }; + const context = { api, page, user: { id: "user" }, company: { id: "company", issuePrefix: "QA" }, + agent: { id: "lead", model: "fixed-model", provider, runtimeMode: "native" }, environmentId: "environment" } as unknown as JourneyContext; + const evidence = { checks: [] } as unknown as JourneyEvidence, owned: Row[] = []; + const directory = await mkdtemp(path.join(os.tmpdir(), "hire-completion-")); directories.push(directory); + const execution = hireDelegateReuse(context, { async own(row: Row) { owned.push(row); } } as unknown as TargetSession, campaignId, evidence, + { deadlineAt: Date.now() + 10_000, directory, capture: async () => {} }); + if (["foreign", "wrong-author", "foreign-run", "contradictory-run", "extra-run", "fallback-changed", "fallback-subscription-wrong-method", "creation-backlog-reassigned"].includes(snapshot)) { + const code = snapshot === "creation-backlog-reassigned" ? "hire_initial_assignee_mismatch" : snapshot === "foreign" ? "hire_resource_scope_mismatch" : snapshot === "wrong-author" ? "hire_completion_reply_mismatch" + : snapshot === "extra-run" ? "hire_requested_turn_mismatch" : snapshot === "fallback-changed" ? "hire_native_binding_mismatch" + : snapshot === "fallback-subscription-wrong-method" ? "hire_run_credential_mismatch" : "hire_run_scope_mismatch"; + await expect(execution).rejects.toMatchObject({ code }); + expect(sendChatMessage).toHaveBeenCalledTimes(["extra-run", "fallback-changed"].includes(snapshot) ? 3 : 2); + if (snapshot === "creation-backlog-reassigned") { + expect(owned.some(row => row.id === "task-2")).toBe(true); + expect(runs.filter(run => run.issueId !== "chat").every(run => run.agentId === "hired")).toBe(true); + } + if (snapshot === "fallback-changed") { expect(stages).toEqual([0, 1, 4]); expect(runs).toHaveLength(7); } + return; + } + await execution; + if (snapshot === "stale") for (const [, text] of vi.mocked(sendChatMessage).mock.calls.slice(1)) { + expect(handoffTargets(text)).toEqual({ agents: ["hired"], projects: ["project"] }); + } + expect(stages).toEqual([0, 1, 4]); expect(runs).toHaveLength(7); expect(sendChatMessage).toHaveBeenCalledTimes(3); + expect(visited).toEqual(["/QA/issues/QA-1", "/QA/issues/QA-2"]); + expect(evidence.checks).toContainEqual({ id: "delegated-result-visible-after-reload", passed: true }); + expect(owned.find(row => row.id === "chat")?.freshChat).toEqual({ companyId: "company", userId: "user", leadAgentId: "lead", absent: true }); +}); + +it.each(["earlier-handoff", "foreign-company", "foreign-chat", "wrong-run", "foreign-origin"])("does not use %s as the callback's reloaded result", async mismatch => { + const href = mismatch === "foreign-origin" ? "https://foreign.test/QA/issues/QA-2" : "/QA/issues/QA-2"; + let clicked = false; + const comment = { id: "old", companyId: mismatch === "foreign-company" ? "foreign" : "company", + issueId: mismatch === "foreign-chat" ? "foreign" : "chat", authorAgentId: "lead", + createdByRunId: ["earlier-handoff", "wrong-run"].includes(mismatch) ? "handoff-run" : "callback" }; + const reply = { async isVisible() { return true; }, locator(selector: string) { + return selector === "a[href]" ? { async evaluateAll() { return [href]; } } + : { first() { return { async click() { clicked = true; } }; } }; + } }; + const page = { locator() { return reply; }, async reload() {}, getByTestId() { return { getByRole() { return {}; } }; } }; + expect(await openSavedHireResult(page as any, "https://qa.example.test", "QA", [comment], "lead", + { id: "review", identifier: "QA-2", title: "Review" }, Date.now() + 10, + { companyId: "company", chatId: "chat", runId: "callback" })).toBe(false); + expect(clicked).toBe(false); +}); diff --git a/tests/runner-e2e/product-journeys/hire-delegate-flow.ts b/tests/runner-e2e/product-journeys/hire-delegate-flow.ts new file mode 100644 index 00000000000..e6f61f494e0 --- /dev/null +++ b/tests/runner-e2e/product-journeys/hire-delegate-flow.ts @@ -0,0 +1,270 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { isDeepStrictEqual } from "node:util"; +import path from "node:path"; +import { expect, type Page } from "@playwright/test"; +import { buildAgentMentionHref, buildProjectMentionHref, parseAgentMentionHref, parseProjectMentionHref } from "../../../packages/shared/src/project-mentions.js"; +import { sendChatMessage, readChatOutputDocument, type ChatIssue, type ChatRun } from "../chat-flow.js"; +import { assertChatHire, mutableIssueSnapshot } from "../chat-hardening.js"; +import { ConnectionFailure } from "../connection-config.js"; +import { assertHireAuditCoverage, assertHireBinding, assertHireDocument, assertHireRuns, assertHireTaskCreation, hireDelegateNames, hireResultLinkMatches, hireRunIssueId, observedHireResources, readHireConnectionChoice, type HireScope } from "./hire-delegate-evidence.js"; +import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; +import { sendHireHandoffMessage } from "./hire-delegate-composer.js"; + +type Row = Record; +type Agent = Parameters[0]["agents"][number] & Row; +type Task = ChatIssue & Row; +type Run = ChatRun & Row; +interface Options { deadlineAt: number; directory: string; capture(id: string, label: string, file: string): Promise } +class HireReadTransportError extends Error {} + +export function hireHandoffMentions(scope: Pick, hiredId: string, hired: Row, project: Row) { + if (![scope?.companyId, scope?.leadId, scope?.projectId, scope?.campaignId].every(value => typeof value === "string" && value.length > 0)) { + throw new JourneyError("failed", "hire_handoff_context_unverified"); + } + const names = hireDelegateNames(scope.campaignId); + if (!hired || !project || typeof hiredId !== "string" || !hiredId || hired.id !== hiredId || hired.id === scope.leadId + || hired.companyId !== scope.companyId || hired.reportsTo !== scope.leadId || hired.name !== names.hire + || typeof project.id !== "string" || !project.id || project.id !== scope.projectId + || project.companyId !== scope.companyId || project.name !== names.project) { + throw new JourneyError("failed", "hire_handoff_context_unverified"); + } + const agentHref = buildAgentMentionHref(hiredId, hired.icon ?? null), projectHref = buildProjectMentionHref(project.id, project.color ?? null); + if (parseAgentMentionHref(agentHref)?.agentId !== hiredId || parseProjectMentionHref(projectHref)?.projectId !== project.id) { + throw new JourneyError("failed", "hire_handoff_context_unverified"); + } + return Object.freeze({ teammate: `[@${hired.name}](${agentHref})`, project: `[@${project.name}](${projectHref})` }); +} + +export async function hireDelegateReuse(context: JourneyContext, session: TargetSession, campaignId: string, evidence: JourneyEvidence, options: Options) { + const { api, page } = context, agent = context.agent; + if (!agent || agent.runtimeMode !== "native" || !context.environmentId) throw new JourneyError("unsupported", "native_agent_and_environment_required"); + const features = await api.get("/api/instance/settings/experimental"); + if (features.enableAgentChat !== true) throw new JourneyError("unsupported", "agent_chat_required"); + const company = await api.get(`/api/companies/${context.company.id}`); + if (company.requireBoardApprovalForNewAgents !== false) throw new JourneyError("unsupported", "automatic_hiring_policy_required"); + const lead = await api.get(`/api/agents/${agent.id}`), names = hireDelegateNames(campaignId); + if (lead.id !== agent.id || lead.companyId !== context.company.id || lead.permissions?.canCreateAgents !== true || !lead.runtimeConfig?.aiConnection) { + throw new JourneyError("unsupported", "owned_hiring_agent_required"); + } + const connectionChoice = await readHireConnectionChoice(api, { companyId: context.company.id, + responsibleUserId: context.user.id, binding: lead.runtimeConfig.aiConnection }); + const check = (id: string, passed = true) => { evidence.checks.push({ id, passed }); if (!passed) throw new JourneyError("failed", id.replaceAll("-", "_")); }; + await mkdir(options.directory, { recursive: true, mode: 0o700 }); + const retain = (name: string, value: unknown) => writeFile(path.join(options.directory, name), JSON.stringify(value, null, 2), { mode: 0o600 }); + await session.own({ kind: "project", name: names.project, journeyId: "hire-delegate-reuse" }); + const project = await api.post(`/api/companies/${context.company.id}/projects`, { name: names.project, description: "Disposable native hiring and delegation verification." }); + if (project.companyId !== context.company.id || !project.id || project.name !== names.project) throw new JourneyError("failed", "hire_project_binding_mismatch"); + await session.own({ kind: "project", name: names.project, id: project.id, journeyId: "hire-delegate-reuse" }); + const chatPath = `/api/companies/${context.company.id}/chats/${agent.id}`; + if (await api.get(chatPath)) throw new JourneyError("unsupported", "fresh_lead_chat_required"); + const freshChat = { companyId: context.company.id, userId: context.user.id, leadAgentId: agent.id, absent: true as const }; + await session.own({ kind: "issue", name: `Chat with ${lead.name}`, journeyId: "hire-delegate-reuse", freshChat }); + const chatUrl = `${api.origin}/${context.company.issuePrefix}/chats/${agent.id}`; + await page.goto(chatUrl, { waitUntil: "domcontentloaded" }); + await expect(page.getByTestId("task-chat-composer-input")).toBeVisible(); + let scope: HireScope | undefined, hiredId: string | undefined; + let hiredConnectionBinding: unknown; + const known = new Set(); + const taskNames = [names.checklist, names.review]; + const readObserved = async (route: string): Promise => { + try { return await api.get(route); } + catch (error) { + if (error instanceof ConnectionFailure && (error.message === "api_request_failed" + || /^api_status_(408|429|502|503|504)$/.test(error.message))) { + throw new HireReadTransportError("hire_observation_transport_unavailable"); + } + throw error; + } + }; + const readRuns = async (issueId: string) => { + const summaries = await readObserved(`/api/issues/${issueId}/runs`); + if (!Array.isArray(summaries) || summaries.length > 7) throw new JourneyError("failed", "hire_run_limit_mismatch"); + return Promise.all(summaries.map(async row => { + if (typeof row.runId !== "string" || !row.runId) throw new JourneyError("failed", "hire_run_scope_mismatch"); + const run = await readObserved(`/api/heartbeat-runs/${row.runId}`); + if (run.id !== row.runId) throw new JourneyError("failed", "hire_run_scope_mismatch"); + return run; + })); + }; + const observe = async () => { + const chat = await readObserved(chatPath); + if (!chat) return undefined; + if (chat.companyId !== context.company.id || chat.assigneeAgentId !== agent.id + || chat.conversationAgentId !== agent.id || chat.conversationUserId !== context.user.id || chat.createdByUserId !== context.user.id) throw new JourneyError("failed", "hire_chat_scope_mismatch"); + scope ??= { companyId: context.company.id, leadId: agent.id, chatId: chat.id, projectId: project.id, campaignId, + model: agent.model, provider: agent.provider, environmentId: context.environmentId!, connectionBinding: lead.runtimeConfig.aiConnection, connectionChoice, + origin: api.origin, issuePrefix: context.company.issuePrefix }; + if (chat.id !== scope.chatId) throw new JourneyError("failed", "hire_chat_replaced"); + if (!known.has(chat.id)) { await session.own({ kind: "issue", name: chat.title, id: chat.id, journeyId: "hire-delegate-reuse", freshChat }); known.add(chat.id); } + const leadProjection = await readRuns(chat.id); + const leadRuns = leadProjection.filter(run => run.companyId === scope!.companyId + && run.agentId === scope!.leadId && hireRunIssueId(run) === chat.id); + const activity = await readObserved(`/api/companies/${context.company.id}/activity?agentId=${agent.id}&limit=500`); + assertHireAuditCoverage(activity, leadRuns); + const agents: Agent[] = [], tasks: Task[] = [], seen = new Set(); + let scopeMismatch = false; + for (const event of activity) { + if (event.companyId !== scope.companyId || event.actorType !== "agent" || event.actorId !== agent.id || event.agentId !== agent.id + || !leadRuns.some(run => run.id === event.runId && run.companyId === scope!.companyId && run.agentId === agent.id + && hireRunIssueId(run) === chat.id && run.runtimeMode === "native")) continue; + const kind = ["agent.hire_created", "agent.created"].includes(event.action) ? "agent" + : ["issue.created", "issue.child_created"].includes(event.action) ? "issue" : undefined; + if (!kind || event.entityType !== kind) continue; + if (typeof event.entityId !== "string" || !event.entityId) throw new JourneyError("failed", "hire_resource_scope_mismatch"); + const key = `${kind}:${event.entityId}`; + if (seen.has(key)) continue; + if (seen.size >= 50) throw new JourneyError("failed", "hire_resource_coverage_exceeded"); + seen.add(key); + const row = await readObserved(`/api/${kind === "agent" ? "agents" : "issues"}/${encodeURIComponent(event.entityId)}`); + if (row.id !== event.entityId) throw new JourneyError("failed", "hire_resource_scope_mismatch"); + if (kind === "agent") agents.push(row as Agent); else tasks.push(row as Task); + const observed = observedHireResources(scope, { agents: kind === "agent" ? [row] : [], tasks: kind === "issue" ? [row] : [], activity, leadRuns }); + scopeMismatch ||= observed.scopeMismatch; + for (const resource of [...observed.agents.map(row => ({ kind: "agent" as const, id: row.id, name: row.name })), + ...observed.tasks.map(row => ({ kind: "issue" as const, id: row.id, name: row.title }))]) { + if (!known.has(resource.id)) { await session.own({ ...resource, journeyId: "hire-delegate-reuse" }); known.add(resource.id); } + } + } + const owned = observedHireResources(scope, { agents, tasks, activity, leadRuns }); + if (scopeMismatch || owned.scopeMismatch) throw new JourneyError("failed", "hire_resource_scope_mismatch"); + if (owned.agents.length) { + await readHireConnectionChoice({ get: readObserved }, { companyId: context.company.id, responsibleUserId: context.user.id, + binding: lead.runtimeConfig.aiConnection, hiredId: owned.agents[0]!.id }); + const verified = assertHireBinding(scope, owned.agents, hiredId); + if (hiredConnectionBinding === undefined) hiredConnectionBinding = Object.freeze(structuredClone(verified.runtimeConfig.aiConnection)); + else if (!isDeepStrictEqual(verified.runtimeConfig.aiConnection, hiredConnectionBinding)) throw new JourneyError("failed", "hire_native_binding_mismatch"); + hiredId = verified.id; + } + assertHireTaskCreation(scope, hiredId, owned.tasks, activity, leadRuns); + const projectedRuns = [...leadProjection, ...(await Promise.all(owned.tasks.map(task => readRuns(task.id)))).flat()]; + const canonical = new Map(); + for (const run of projectedRuns) { + const previous = canonical.get(run.id); + if (previous && (previous.companyId !== run.companyId || previous.agentId !== run.agentId + || hireRunIssueId(previous) !== hireRunIssueId(run))) throw new JourneyError("failed", "hire_run_scope_mismatch"); + if (!previous) canonical.set(run.id, run); + } + const runs = [...canonical.values()]; + const comments = await readObserved(`/api/issues/${chat.id}/comments?order=asc`); + const requestIds = comments.filter(row => row.authorUserId === context.user.id).map(row => row.id); + const completionRepliesReady = assertHireRuns(scope, hiredId, owned.tasks, runs, undefined, requestIds, comments); + for (const run of projectedRuns) assertHireRuns(scope, hiredId, owned.tasks, [run], undefined, requestIds, comments); + await retain("hire-delegate-observation.json", { scope: { ...scope, connectionBinding: undefined, connectionChoice: undefined }, chat, project, + agents: owned.agents.map(row => ({ id: row.id, name: row.name, reportsTo: row.reportsTo })), tasks: owned.tasks, + runs: runs.map(run => ({ id: run.id, issueId: hireRunIssueId(run), agentId: run.agentId, status: run.status })) }); + return { chat, ...owned, runs, requestIds, comments, completionRepliesReady, allAgents: [lead as Agent, ...agents], activity }; + }; + const waitForIdle = async (count: number) => { + let awaitingReply = false; + while (Date.now() < options.deadlineAt) { + try { + const state = await observe(); + awaitingReply = state?.completionRepliesReady === false; + if (state && state.completionRepliesReady && state.runs.length === count && state.runs.every(run => run.status === "succeeded") + && state.chat.status === "in_review" && state.chat.conversationState === "waiting") { + assertHireRuns(scope!, hiredId, state.tasks, state.runs, count, state.requestIds, state.comments); + return state; + } + } catch (error) { + if (!(error instanceof HireReadTransportError)) throw error; + } + await new Promise(resolve => setTimeout(resolve, 1000)); + } + throw new JourneyError("failed", awaitingReply ? "hire_completion_reply_missing" : "hire_chat_not_completed"); + }; + const turn = async (text: string, count: number, handoff?: ReturnType) => { + await page.goto(chatUrl, { waitUntil: "domcontentloaded" }); + const before = await api.get(chatPath); + const prior = before ? await api.get(`/api/issues/${before.id}/comments?order=asc`) : []; + if (handoff) await sendHireHandoffMessage(page, text, handoff, options.deadlineAt); + else await sendChatMessage(page, text); + const state = await waitForIdle(count); + const comments = await api.get(`/api/issues/${state.chat.id}/comments?order=asc`); + const added = comments.filter(row => row.authorUserId === context.user.id && !prior.some(old => old.id === row.id)); + check("hire-user-messages-stored-once", added.length === 1 && added[0].body === text); + return state; + }; + await session.own({ kind: "agent", name: names.hire, journeyId: "hire-delegate-reuse" }); + let state = await turn(`Hire exactly one teammate named ${names.hire}, reporting to you, using your native runner, model, and available AI connection. Do not create or start any tasks yet. Tell me when this teammate is ready.`, 1); + const hired = assertHireBinding(scope!, state.agents); + check("one-attributed-native-hire", state.tasks.length === 0); + const handoff = hireHandoffMentions(scope!, hiredId!, hired, project); + const marker = `HIRE${campaignId.replace(/[^a-z0-9]/gi, "")}`; + await session.own({ kind: "issue", name: names.checklist, journeyId: "hire-delegate-reuse" }); + state = await turn(`Have the existing ${handoff.teammate} write a concise launch checklist as a saved Paperclip document with the line "Reference: ${marker}". Create exactly one task titled ${names.checklist} in ${handoff.project}, already assigned to that teammate when it is created. Link the result here and let the teammate complete it. Do not hire another agent.`, 4, handoff); + const first = state.tasks.find(task => task.title === names.checklist); + check("first-delegated-task-complete", typeof first?.identifier === "string" && state.tasks.length === 1); + const readOutput = async (task: Row, token: string, runs: Row[]) => { + const document = await readChatOutputDocument(api, task.id, token); + const revisions = await api.get(`/api/issues/${task.id}/documents/${encodeURIComponent(document.key)}/revisions`); + assertHireDocument(scope!, task, document, revisions, runs, token, hired.id); + return document; + }; + const firstOutput = await readOutput(first!, marker, state.runs); + const originalTask = mutableIssueSnapshot(await api.get(`/api/issues/${first!.id}`)); + check("checklist-authored-by-hired-run"); + await session.own({ kind: "issue", name: names.review, journeyId: "hire-delegate-reuse" }); + state = await turn(`Have the existing ${handoff.teammate} review the checklist on ${first!.identifier} and write a separate saved review document with the line "Reference: REVIEW${marker}". Create exactly one task titled ${names.review} in ${handoff.project}, already assigned to that same teammate when it is created. Include the actual checklist in the handoff. Preserve the original checklist and task. Link the completed review here. Do not hire another agent.`, 7, handoff); + const second = state.tasks.find(task => task.title === names.review); + check("same-teammate-reused", typeof second?.identifier === "string" && state.tasks.length === 2 && state.tasks.every(task => taskNames.includes(task.title))); + const reviewOutput = await readOutput(second!, `REVIEW${marker}`, state.runs); + check("review-authored-by-same-hired-run"); + check("original-checklist-preserved", isDeepStrictEqual(await api.get(`/api/issues/${first!.id}/documents/${encodeURIComponent(firstOutput.key)}`), firstOutput) + && isDeepStrictEqual(mutableIssueSnapshot(await api.get(`/api/issues/${first!.id}`)), originalTask)); + if (hiredConnectionBinding === undefined || !isDeepStrictEqual( + state.agents.find(row => row.id === hired.id)?.runtimeConfig?.aiConnection, hiredConnectionBinding)) { + throw new JourneyError("failed", "hire_native_binding_mismatch"); + } + assertChatHire({ agents: state.allAgents, leadId: agent.id, hireName: names.hire, hiredId: hired.id, + connectionId: connectionChoice?.connectionId ?? lead.runtimeConfig.aiConnection.connectionId, binding: hiredConnectionBinding, + taskIds: [first!.id, second!.id], tasks: state.tasks, runs: state.runs }); + assertHireRuns(scope!, hired.id, state.tasks, state.runs, 7, state.requestIds, state.comments); + const comments = await api.get(`/api/issues/${state.chat.id}/comments?order=asc`); + assertHireRuns(scope!, hired.id, state.tasks, state.runs, 7, state.requestIds, comments); + let resultsVisible = true; + for (const task of [first!, second!]) { + const callback = state.runs.find(run => { + const updates = run.contextSnapshot?.chatCompletionUpdates; + return run.contextSnapshot?.wakeReason === "chat_task_completed" && Array.isArray(updates) && updates[0]?.id === task.id; + }); + if (!callback) throw new JourneyError("failed", "hire_completion_binding_mismatch"); + await page.goto(chatUrl, { waitUntil: "domcontentloaded" }); await page.reload(); + if (!await openSavedHireResult(page, api.origin, context.company.issuePrefix, comments, agent.id, task, + options.deadlineAt, { companyId: scope!.companyId, chatId: scope!.chatId, runId: callback.id })) { + resultsVisible = false; break; + } + } + check("delegated-result-visible-after-reload", resultsVisible); + check("configured-model-and-environment"); + await retain("hire-delegate-completed.json", { hiredId: hired.id, firstTaskId: first!.id, secondTaskId: second!.id, + firstDocument: firstOutput, reviewDocument: reviewOutput, runIds: state.runs.map(run => run.id), + turns: state.runs.map(run => ({ id: run.id, issueId: hireRunIssueId(run), agentId: run.agentId, + wakeReason: run.contextSnapshot?.wakeReason, wakeCommentId: run.contextSnapshot?.wakeCommentId, + chatCompletionDeliveryIds: run.contextSnapshot?.chatCompletionDeliveryIds, chatCompletionUpdates: run.contextSnapshot?.chatCompletionUpdates })), + completionReplies: state.comments.filter(comment => state.runs.some(run => run.id === comment.createdByRunId + && run.contextSnapshot?.wakeReason === "chat_task_completed")).map(comment => ({ id: comment.id, createdByRunId: comment.createdByRunId })) }); + await options.capture("hire-delegate-reuse", "Hired teammate completed and reused", "hire-delegate-reuse.png"); +} + +export async function openSavedHireResult(page: Page, origin: string, issuePrefix: string, comments: Row[], leadId: string, task: Row, deadlineAt: number, + callback: { companyId: string; chatId: string; runId: string }) { + while (Date.now() < deadlineAt) { + for (const comment of comments.filter(row => row.authorAgentId === leadId && !row.authorUserId + && row.companyId === callback.companyId && row.issueId === callback.chatId && row.createdByRunId === callback.runId)) { + const reply = page.locator(`[id=${JSON.stringify(`comment-${comment.id}`)}]`); + if (!await reply.isVisible()) continue; + const links = await reply.locator("a[href]").evaluateAll(elements => elements.map(element => element.getAttribute("href")!)); + const href = links.find(href => hireResultLinkMatches({ origin, issuePrefix }, task, href)); + if (!href) continue; + await reply.locator(`a[href=${JSON.stringify(href)}]`).first().click({ timeout: Math.max(1, deadlineAt - Date.now()) }); + await page.reload(); + await expect(page).toHaveURL(url => url.origin === origin + && [task.id, task.identifier].some(id => url.pathname === `/${issuePrefix}/issues/${id}`), { timeout: Math.max(1, deadlineAt - Date.now()) }); + await expect(page.getByTestId("issue-detail-header").getByRole("heading", { name: task.title, exact: true })) + .toBeVisible({ timeout: Math.max(1, deadlineAt - Date.now()) }); + return true; + } + await new Promise(resolve => setTimeout(resolve, Math.min(200, Math.max(1, deadlineAt - Date.now())))); + } + return false; +} diff --git a/tests/runner-e2e/product-journeys/launch.ts b/tests/runner-e2e/product-journeys/launch.ts index 9dfe00bd3d8..a6cc1027c36 100644 --- a/tests/runner-e2e/product-journeys/launch.ts +++ b/tests/runner-e2e/product-journeys/launch.ts @@ -108,8 +108,8 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri try { sha = execFileSync("git", ["rev-parse", "HEAD"], { cwd: repositoryRoot, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim(); } catch { /* Packaged harnesses may not include Git metadata. */ } const source = resolveRunnerE2ESource({ sha, ref: null, workflowRunUrl: null }); const definitionHash = createHash("sha256").update(JSON.stringify(productJourneys)); - for (const name of ["catalog.ts", "task-properties.ts", "task-search-filter.ts", "task-search-evidence.ts", "basic-flows.ts", "basic-evidence.ts", "task-document-flow.ts", "task-document-evidence.ts", "project-context-flow.ts", "project-context-readback.ts", "project-context-evidence.ts", "agent-flows.ts", "active-followup-flow.ts", "active-followup-evidence.ts", "stop-redirect-evidence.ts", "clarification-flow.ts", "clarification-evidence.ts", "../question-resume-ui.ts", "../question-resume-scoring.ts", "../runtime-question-readiness.ts", "../chat-flow.ts", "../continuation-readiness.ts", "../continuation-cases.ts", "success-contract.ts", "product-screenshot.ts", "cloud-onboarding.ts", "onboarding-connect.ts", "onboarding-first-task.ts", "workspace-deferral.ts", "onboarding-pre-acceptance.ts", "artifact-evidence.ts", "fixtures.ts", "launch.ts", "types.ts", "collection.ts", "local-target.ts"]) definitionHash.update(await readFile(path.join(import.meta.dirname, name))); - for (const file of ["catalog.ts", "api.ts", "run-observations.ts", "native-task-reply.ts", "native-event-envelope.ts", "user-actions.ts", "plan-revision-flow.ts", "skill-studio-flow.ts", "protected-action-flow.ts", "story-artifact.ts", "stop-redirect-flow.ts", "everyday-cases.ts", "everyday-delivery.ts", "everyday-decisions.ts", "connection-guidance-evidence.ts", "everyday-artifact.py"]) definitionHash.update(await readFile(path.join(import.meta.dirname, "..", file))); + for (const name of ["catalog.ts", "task-properties.ts", "task-search-filter.ts", "task-search-evidence.ts", "basic-flows.ts", "basic-evidence.ts", "task-document-flow.ts", "task-document-evidence.ts", "project-context-flow.ts", "project-context-readback.ts", "project-context-evidence.ts", "agent-flows.ts", "hire-delegate-flow.ts", "hire-delegate-composer.ts", "hire-delegate-evidence.ts", "active-followup-flow.ts", "active-followup-evidence.ts", "stop-redirect-evidence.ts", "clarification-flow.ts", "clarification-evidence.ts", "../question-resume-ui.ts", "../question-resume-scoring.ts", "../runtime-question-readiness.ts", "../chat-flow.ts", "../continuation-readiness.ts", "../continuation-cases.ts", "success-contract.ts", "product-screenshot.ts", "cloud-onboarding.ts", "onboarding-connect.ts", "onboarding-first-task.ts", "workspace-deferral.ts", "onboarding-pre-acceptance.ts", "artifact-evidence.ts", "fixtures.ts", "launch.ts", "types.ts", "collection.ts", "local-target.ts", "../../../ui/src/lib/markdown-reply-content.ts", "../../../ui/src/lib/issue-reference.ts", "../../../ui/src/lib/company-routes.ts", "../../../ui/package.json"]) definitionHash.update(await readFile(path.join(import.meta.dirname, name))); + for (const file of ["catalog.ts", "api.ts", "run-observations.ts", "native-task-reply.ts", "native-event-envelope.ts", "user-actions.ts", "plan-revision-flow.ts", "skill-studio-flow.ts", "protected-action-flow.ts", "story-artifact.ts", "stop-redirect-flow.ts", "everyday-cases.ts", "everyday-delivery.ts", "chat-hardening.ts", "everyday-decisions.ts", "connection-guidance-evidence.ts", "everyday-artifact.py"]) definitionHash.update(await readFile(path.join(import.meta.dirname, "..", file))); definitionHash.update(runnerSuites.flatMap(suite => suite.tasks).find(task => task.id === "ask-question")!.buildPrompt("definition")); const suiteDefinitionHash = definitionHash.digest("hex"); const expected = config.journeys.map(id => `product-journeys.${config.id}.${id === "cloud-onboarding" ? config.modelProfileId : isAgentJourney(id) ? "configured-agent" : "browser"}.${config.executionEnvironment}.${id}`); diff --git a/tests/runner-e2e/product-journeys/success-contract.ts b/tests/runner-e2e/product-journeys/success-contract.ts index baff0d5eec3..1ce269fac61 100644 --- a/tests/runner-e2e/product-journeys/success-contract.ts +++ b/tests/runner-e2e/product-journeys/success-contract.ts @@ -1,6 +1,6 @@ /** Closed, target-independent success requirements used by dispatch and retained evidence. */ export const agentJourneyRuns = { "ask-question": 1, "plan-revise-accept": 3, "build-revise": 2, - "followup-while-running": 2, "protected-actions": 4, "create-skill-studio": 1, "stop-redirect": 3, "question-answer-resume": 3 } as const; + "followup-while-running": 2, "protected-actions": 4, "create-skill-studio": 1, "stop-redirect": 3, "question-answer-resume": 3, "hire-delegate-reuse": 7 } as const; export function agentJourneyRunCountValid(id: string, count: unknown): boolean { if (!Object.hasOwn(agentJourneyRuns, id) || !Number.isSafeInteger(count)) return false; @@ -26,5 +26,6 @@ export const journeyRequiredChecks: Record = { "stop-redirect": ["checkpoint-artifact-verified", "active-work-stopped", "stop-native-terminal-retained", "new-direction-completed", "stopped-run-remained-terminal", "stopped-run-no-later-events", "stop-preserved-download", "redirect-consumed-once", "user-reply-stored-once", "configured-model-and-environment"], "followup-while-running": ["followup-native-command-active", "followup-stored-once", "followup-consumed-once", "followup-final-answer-persisted", "followup-one-or-two-native-runs", "configured-model-and-environment"], + "hire-delegate-reuse": ["one-attributed-native-hire", "first-delegated-task-complete", "checklist-authored-by-hired-run", "same-teammate-reused", "review-authored-by-same-hired-run", "original-checklist-preserved", "hire-user-messages-stored-once", "delegated-result-visible-after-reload", "configured-model-and-environment"], "cloud-onboarding": ["fresh-workspace", "ui-connection-and-native-agent", "first-task-complete", "downloadable-artifact-verified", "native-trace-verified"], }; diff --git a/tests/runner-e2e/product-journeys/types.ts b/tests/runner-e2e/product-journeys/types.ts index 3d39d6b37e3..df7289fb732 100644 --- a/tests/runner-e2e/product-journeys/types.ts +++ b/tests/runner-e2e/product-journeys/types.ts @@ -3,7 +3,7 @@ import type { ConnectionApi } from "../connection-target.js"; export type JourneyOutcome = "passed" | "failed" | "blocked" | "unsupported"; export type JourneyId = "task-properties" | "task-create" | "project-task-context" | "task-document-persistence" | "ask-question" | "cloud-onboarding" - | "question-answer-resume" | "task-search-filter" | "followup-while-running" | "plan-revise-accept" | "build-revise" | "protected-actions" | "create-skill-studio" | "stop-redirect"; + | "question-answer-resume" | "task-search-filter" | "followup-while-running" | "plan-revise-accept" | "build-revise" | "protected-actions" | "create-skill-studio" | "stop-redirect" | "hire-delegate-reuse"; export type TargetMode = "managed-self-hosted" | "existing-cloud" | "new-cloud"; export interface DeploymentIdentity { kind: "self-hosted" | "cloud"; @@ -111,7 +111,8 @@ export interface TargetSession { /** Publish the observed finalization failure without releasing resources or leases. */ checkpointEvidence?(input?: TargetLifecycleInput): Promise; /** Durable intent before creation; receipt immediately after creation. */ - own(resource: { kind: "issue" | "project" | "agent" | "connection" | "tool-connection" | "skill"; name: string; id?: string; journeyId?: JourneyId }): Promise; + own(resource: { kind: "issue" | "project" | "agent" | "connection" | "tool-connection" | "skill"; name: string; id?: string; journeyId?: JourneyId; + freshChat?: { companyId: string; userId: string; leadAgentId: string; absent: true } }): Promise; } export interface TargetProvider { version: 1;