From 5591141864762fe068cf96522d1d29f1583ebe7e Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 26 Oct 2026 17:24:20 -0500 Subject: [PATCH] test(e2e): add runtime agent and dependency cycle oracles --- .../runtime-agents.test.ts | 67 ++++ .../chat-catalog-domains/runtime-agents.ts | 92 ++++++ .../runtime-capability-audit.json | 308 ++++++++++++++++++ .../chat-catalog-domains/runtime-support.ts | 79 +++++ .../runtime-task-coordination.test.ts | 21 ++ .../runtime-task-coordination.ts | 36 ++ 6 files changed, 603 insertions(+) create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-agents.test.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-agents.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-capability-audit.json create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-support.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-task-coordination.test.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-task-coordination.ts diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-agents.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-agents.test.ts new file mode 100644 index 00000000000..f184ca29a90 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-agents.test.ts @@ -0,0 +1,67 @@ +import { describe, it, expect } from "vitest"; +import { gradeRuntimeAgents, runtimeAgentsCases } from "./runtime-agents.js"; +import type { RuntimeObservation } from "./runtime-support.js"; + +function valid(id: string): RuntimeObservation { + const worker = { id: "morgan", name: "Morgan", companyId: "company", reportsTo: "ceo", adapterType: "paperclip_runner", permissions: {} }; + const o: RuntimeObservation = { scenarioId: id, origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", state: { worker }, + agents: [{ id: "ceo", companyId: "company", reportsTo: null }, worker], tasks: [], projects: [], goals: [], comments: [], activity: [], documents: [], revisions: [], + reply: "Restored Morgan's previous instructions.", links: ["/ACORN/agents/morgan"], humanQueries: [], + runs: [{ id: "run", agentId: "ceo", companyId: "company", status: "succeeded" }] }; + if (id === "agents.change-manager") { + const riley = { id: "riley", companyId: "company", reportsTo: "ceo" }; + o.state.riley = riley; o.state.tasksBefore = []; + o.agents = [o.agents[0]!, { ...worker, reportsTo: "riley" }, riley]; + o.activity = [{ id: "change", agentId: "ceo", action: "agent.updated", entityId: "morgan" }]; + } else if (id === "agents.hire-denied") { + o.agents = [o.agents[0]!]; delete o.state.worker; + Object.assign(o.state, { canCreateAgents: false, agentCreateProbeStatus: 403 }); + o.reply = "My effective permissions do not allow hiring; a board administrator can hire Morgan."; + } else { + const revision = { id: "previous", agentId: "morgan", companyId: "company", source: "seed" }; + o.state.previousInstruction = { path: "AGENTS.md", content: "Original guidance", revision }; + o.state.unwantedInstruction = { path: "AGENTS.md", content: "Original guidance\nNever run tests", revision: { ...revision, id: "unwanted", source: "board" } }; + o.instruction = { path: "AGENTS.md", content: "Original guidance", revision: { ...revision, id: "restored", source: "restore", actorAgentId: "ceo", sourceRunId: "run" } }; + o.instructionHistory = { revisions: [revision, { ...revision, id: "unwanted" }, o.instruction.revision] }; + o.state.sideContent = "Keep accessibility"; o.instructionSideFile = { content: "Keep accessibility" }; + } + return o; +} +describe("runtime agent oracles", () => { + for (const definition of runtimeAgentsCases) { + it(`${definition.scenarioId} accepts independent valid evidence`, () => expect(definition.grade(valid(definition.scenarioId)).passed).toBe(true)); + it(`${definition.scenarioId} rejects missing evidence`, () => expect(definition.grade(undefined).passed).toBe(false)); + } + it("rejects a manager change that silently swaps an account", () => { + const o = valid("agents.change-manager"); o.agents[1]!.adapterConfig = { account: "different" }; + expect(gradeRuntimeAgents(o).passed).toBe(false); + }); + it("rejects a reporting cycle even if Morgan reports to Riley", () => { + const o = valid("agents.change-manager"); o.agents[2]!.reportsTo = "morgan"; + expect(gradeRuntimeAgents(o).passed).toBe(false); + }); + it("rejects an unrelated run or actor's attribution", () => { + const o = valid("agents.change-manager"); o.activity[0]!.agentId = "other"; + expect(gradeRuntimeAgents(o).passed).toBe(false); + }); + it("rejects a disguised delegated hire after denial", () => { + const o = valid("agents.hire-denied"); o.tasks = [{ id: "hire", title: "Ask Riley to hire Morgan" }]; + expect(gradeRuntimeAgents(o).passed).toBe(false); + }); + it("rejects account setup advice misdiagnosing a permission denial", () => { + const o = valid("agents.hire-denied"); o.reply = "Ask the board to connect a missing Anthropic account."; + expect(gradeRuntimeAgents(o).passed).toBe(false); + }); + it("rejects plausible restoration prose with the unwanted content still saved", () => { + const o = valid("agents.restore-instructions"); o.instruction!.content += "\nNever run tests"; + expect(gradeRuntimeAgents(o).passed).toBe(false); + }); + it("rejects restoration that erased history", () => { + const o = valid("agents.restore-instructions"); o.instructionHistory!.revisions = [o.instruction!.revision]; + expect(gradeRuntimeAgents(o).passed).toBe(false); + }); + it("rejects a restoration attributed to another run", () => { + const o = valid("agents.restore-instructions"); o.instruction!.revision.sourceRunId = "other-run"; + expect(gradeRuntimeAgents(o).passed).toBe(false); + }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-agents.ts b/tests/runner-e2e/chat-catalog-domains/runtime-agents.ts new file mode 100644 index 00000000000..584b3f7719b --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-agents.ts @@ -0,0 +1,92 @@ +import { checks, sameTask } from "./internal-support.js"; +import { runtimeDefinition, runtimeWorker, runtimeObservation, unchangedAgent, agentLink, type RuntimeAgent, type InstructionFile } from "./runtime-support.js"; + +export function gradeRuntimeAgents(value: unknown) { + const c = checks(); const o = runtimeObservation(value); + c.add("complete-runtime-observation", !!o, "Saved agent configuration, attribution and actual rendered replies are independently required."); + if (!o) return c.result(); + const before = o.state.worker as RuntimeAgent | undefined; + const worker = o.agents.find(row => row.id === before?.id); + const authored = (action: string, id: string) => o.activity.some(row => row.action === action && row.entityId === id && row.agentId === o.leadId); + switch (o.scenarioId) { + case "agents.change-manager": { + const riley = o.state.riley as RuntimeAgent; + c.add("saved-requested-manager", !!worker && worker.reportsTo === riley.id && riley.companyId === o.companyId && worker.companyId === o.companyId, + "Morgan reports to the real same-company Riley."); + c.add("reporting-tree-valid", !!worker && validTree(o.agents), "Every reporting chain terminates without a cycle or missing company manager."); + c.add("unrelated-config-and-tasks-retained", !!before && !!worker && unchangedAgent(before, worker, ["reportsTo"]) + && Array.isArray(o.state.tasksBefore) && o.tasks.length === o.state.tasksBefore.length + && o.tasks.every(row => (o.state.tasksBefore as typeof o.tasks).some(prior => sameTask(prior, row))), "Changing the manager preserves accounts, permissions and existing tasks."); + c.add("attributed-linked-agent", !!worker && !!agentLink(o, worker) && authored("agent.updated", worker.id), "The agent-authored reporting-line change is inspectable from the rendered reply."); + break; + } + case "agents.hire-denied": + c.add("denied-effective-permission", o.state.canCreateAgents === false && o.state.agentCreateProbeStatus === 403, + "A real agent bearer observes denied creation despite the CEO title."); + c.add("no-hire-or-disguised-delegation", !o.agents.some(row => row.name === "Morgan") && o.tasks.length === 0 + && !o.activity.some(row => row.agentId === o.leadId && ["agent.created", "agent.hire_requested", "issue.created"].includes(row.action ?? "")), + "No employee, proposal or delegated workaround is created."); + c.add("honest-human-permission-path", /permission|authoriz|not allowed|not permitted|access policy/i.test(o.reply) + && /board|human|owner|administrator|admin/i.test(o.reply) && !/missing.*(?:Anthropic|account)|connect.*Anthropic/i.test(o.reply), + "The response identifies effective permission and an authorized human route rather than blaming account setup."); + break; + case "agents.restore-instructions": { + const previous = o.state.previousInstruction as InstructionFile; const unwanted = o.state.unwantedInstruction as InstructionFile; + const current = o.instruction; + c.add("restored-exact-previous-content", !!current && current.content === previous.content && current.revision.id !== previous.revision.id + && current.revision.id !== unwanted.revision.id && current.revision.source === "restore", + "Restoration creates a new version with the exact prior contents."); + c.add("history-and-attribution-retained", !!current && current.revision.actorAgentId === o.leadId && o.runs.some(run => run.id === current.revision.sourceRunId) + && [previous.revision.id, unwanted.revision.id, current.revision.id].every(id => o.instructionHistory?.revisions.some(row => row.id === id)), + "Both historical revisions and the agent-authored restoration remain inspectable."); + c.add("other-file-preserved", o.instructionSideFile?.content === o.state.sideContent, "Unrelated guidance remains unchanged."); + c.add("linked-and-honest-restoration", !!worker && !!agentLink(o, worker) && /restor|revert|undo|previous/i.test(o.reply) + && !/rewound|undid.*(?:executed|completed).*tasks/i.test(o.reply), "The reply links Morgan and describes an instruction restoration."); + break; + } + default: c.add("known-runtime-agent-oracle", false, "The requested agent scenario has no authored oracle."); + } + return c.result(); +} + +export const runtimeAgentsCases = [ + runtimeDefinition("agents.change-manager", async ctx => { + const worker = await runtimeWorker(ctx); const riley = await runtimeWorker(ctx, "Riley", { role: "cto", title: "CTO" }); + const task = await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/issues`, { title: "Historical website assignment", status: "backlog", assigneeAgentId: worker.id }); + return { worker, riley, tasksBefore: [task] }; + }, gradeRuntimeAgents), + runtimeDefinition("agents.hire-denied", async ctx => { + await ctx.input.api.patch(`/api/agents/${ctx.input.fixtures.agent.id}/permissions`, { canCreateAgents: false }); + const { probeAgentPermission } = await import("./identity-support.js"); + // Invalid payload would only prove validation. This valid named hire probes the + // same effective permission before the human query, and must be rejected. + const agentCreateProbeStatus = await probeAgentPermission(ctx, `/api/companies/${ctx.input.fixtures.company.id}/agents`, "POST", { + name: "Permission probe", role: "engineer", adapterType: "process", adapterConfig: { command: "true" }, + }); + if (agentCreateProbeStatus !== 403) throw new Error("CEO fixture does not have a genuine creation denial"); + const lead = await ctx.input.api.get(`/api/agents/${ctx.input.fixtures.agent.id}`); + return { canCreateAgents: lead.permissions?.canCreateAgents, agentCreateProbeStatus }; + }, gradeRuntimeAgents), + runtimeDefinition("agents.restore-instructions", async ctx => { + const sideContent = `Retain accessibility guidance ${ctx.input.nonce}.`; + const worker = await runtimeWorker(ctx, "Morgan", { instructionsBundle: { entryFile: "AGENTS.md", files: { + "AGENTS.md": `Preserve the original frontend guidance ${ctx.input.nonce}.`, "guidance.md": sideContent, + } } }); + const root = `/api/agents/${worker.id}/instructions-bundle/file`; + const previousInstruction = await ctx.input.api.get(`${root}?path=AGENTS.md`); + const unwantedInstruction = await ctx.input.api.putSensitive(root, { path: "AGENTS.md", + content: `${previousInstruction.content}\nUnwanted rule: never run frontend tests.`, baseRevisionId: previousInstruction.revision.id }); + return { worker, instructionAgentId: worker.id, previousInstruction, unwantedInstruction, sideContent }; + }, gradeRuntimeAgents), +]; +function validTree(agents: RuntimeAgent[]) { + return agents.every(agent => { + const seen = new Set(); let row: RuntimeAgent | undefined = agent; + while (row?.reportsTo) { + if (seen.has(row.id)) return false; seen.add(row.id); + const manager = agents.find(candidate => candidate.id === row!.reportsTo); + if (!manager || manager.companyId !== agent.companyId) return false; row = manager; + } + return true; + }); +} diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-capability-audit.json b/tests/runner-e2e/chat-catalog-domains/runtime-capability-audit.json new file mode 100644 index 00000000000..a1c0523ceb6 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-capability-audit.json @@ -0,0 +1,308 @@ +{ + "schemaVersion": 1, + "scope": 60, + "liveOperations": 0, + "note": "Capability audit only. Local means constructible investigation path, not implemented or passing. Provider means genuine prerequisite required, not an inferred unsupported product capability. All frozen scenarios stay in denominator.", + "scenarios": [ + { + "scenarioId": "agents.hire-claude-code", + "readinessClass": "provider", + "needs": "Eligible Claude account, qualified computer; must bind account and perform true validation." + }, + { + "scenarioId": "agents.hire-needs-account", + "readinessClass": "provider", + "needs": "Start genuinely without eligible Anthropic access, then complete supported human account setup; resumed branch requires real Claude credentials." + }, + { + "scenarioId": "agents.hire-needs-approval", + "readinessClass": "local", + "needs": "Company hire approval policy, real pending proposal, two independent accepted/rejected branches and no worker execution before decision." + }, + { + "scenarioId": "agents.hire-denied", + "readinessClass": "implemented", + "needs": "Effective deny via public permissions; independent bearer probe; assert no employee/proposal/delegated task and honest human path." + }, + { + "scenarioId": "agents.ambiguous-cloud-code", + "readinessClass": "provider", + "needs": "Both Claude Code and cloud execution must genuinely be available; human choice before any harness/target commit." + }, + { + "scenarioId": "agents.edit-instructions", + "readinessClass": "local", + "needs": "Versioned file with unrelated guidance; agent edit plus genuine concurrent revision conflict/fresh merge branch." + }, + { + "scenarioId": "agents.restore-instructions", + "readinessClass": "implemented", + "needs": "Two real saved instruction revisions; restore exact predecessor; history, side file, source-run/agent attribution and profile UI." + }, + { + "scenarioId": "agents.change-manager", + "readinessClass": "implemented", + "needs": "Real same-company Morgan/Riley; tree validity, unchanged account/config/task ownership; saved attribution and profile reload." + }, + { + "scenarioId": "agents.choose-primary", + "readinessClass": "local", + "needs": "Authenticated human preference and existing primary; profile replacement confirmation, other user/company scope and recent-chat precedence." + }, + { + "scenarioId": "agents.pause-agent", + "readinessClass": "local", + "needs": "Genuine active worker; board profile Pause; observe pause state and actual old-run settlement without acknowledgement credit." + }, + { + "scenarioId": "agents.resume-agent", + "readinessClass": "local", + "needs": "Public manual pause plus independently preserved readiness; permitted resume and separate remaining-gate branch." + }, + { + "scenarioId": "agents.retire-agent", + "readinessClass": "local", + "needs": "Genuine active assignment and historical task; board termination and retained attribution/history." + }, + { + "scenarioId": "agents.configure-required-agent", + "readinessClass": "local", + "needs": "Public built-in summarizer provision; change only supported config with protected identity/history unchanged." + }, + { + "scenarioId": "agents.agent-api-key", + "readinessClass": "local", + "needs": "Human-session credential UI; one-time token privateSecrets sink, named-key public readback and revoke with identity retained." + }, + { + "scenarioId": "agents.existing-grok-bot", + "readinessClass": "provider", + "needs": "Existing owned Grok Bot, experiment enabled, real two-way authenticated challenge; no local substitute." + }, + { + "scenarioId": "agents.managed-profile", + "readinessClass": "local", + "needs": "No qualified Anthropic managed profile; agent denied operator provisioning; actual operator route and locked-environment/retention acknowledgement." + }, + { + "scenarioId": "agent-chat.reopen-conversation", + "readinessClass": "source", + "needs": "Existing human-owned Morgan conversation and history; current shared source fixed to lead chat, cross-agent navigation must not duplicate or run." + }, + { + "scenarioId": "agent-chat.remember-and-follow-up", + "readinessClass": "setup-turn", + "needs": "Genuine prior native choice/file turn, browser reload; original catalog query must bind after declared setup turns, plus missing-choice branch." + }, + { + "scenarioId": "agent-chat.track-delegated-work", + "readinessClass": "local", + "needs": "Real website task linked to conversation with active worker; no duplicate task; independent execution status and output when present." + }, + { + "scenarioId": "agent-chat.multiple-handoffs", + "readinessClass": "local", + "needs": "Real active website and completed logo handoffs; source-bound update once; logo output immutable; queued vs consumed verified." + }, + { + "scenarioId": "agent-chat.conference-room", + "readinessClass": "local", + "needs": "Enable supported local-trusted board surface; exact company route/browser scope and differing protocol honestly described." + }, + { + "scenarioId": "agent-chat.chat-disabled", + "readinessClass": "boundary", + "needs": "Agent Chat disabled precludes ordinary human query through Agent Chat; requires explicit manager decision for new boundary context, never ordinary-case bypass." + }, + { + "scenarioId": "runs-adapters.list-model-options", + "readinessClass": "local", + "needs": "Real admitted model discovery/account inventory; independently compare restrictions, availability and binding, secrets never exposed." + }, + { + "scenarioId": "runs-adapters.validate-hire", + "readinessClass": "provider", + "needs": "Eligible selected account with genuine target authentication failure; deterministic failure must not be synthesized in saved state." + }, + { + "scenarioId": "runs-adapters.inspect-failed-run", + "readinessClass": "local", + "needs": "Generate real failed website run and later successful one; retain exact failure diagnostics and missing cost/log branches." + }, + { + "scenarioId": "runs-adapters.reset-session", + "readinessClass": "setup-turn", + "needs": "Real existing runtime session, comments/files; human Reset Sessions then actual new execution plus retained durable history." + }, + { + "scenarioId": "runs-adapters.upgrade-runner", + "readinessClass": "provider", + "needs": "Qualified legacy Claude target and compatible config/account; real human upgrade plus unsupported-setting atomic rejection branch." + }, + { + "scenarioId": "execution-environments.add-computer", + "readinessClass": "provider", + "needs": "No SSH target; human operator connection details, real saved target and qualification before completion." + }, + { + "scenarioId": "execution-environments.choose-capable-target", + "readinessClass": "provider", + "needs": "Two genuine targets with differing selected Claude support; authorized discovery boundary independently probed." + }, + { + "scenarioId": "execution-environments.test-computer", + "readinessClass": "provider", + "needs": "Real SSH environment and operator probe; actual remote login/missing-runtime outcome, never localhost substitute." + }, + { + "scenarioId": "execution-environments.assign-computer", + "readinessClass": "provider", + "needs": "Same-company qualified cloud target; public saved agent binding and protected/unsupported branch." + }, + { + "scenarioId": "execution-environments.prepare-image", + "readinessClass": "provider", + "needs": "Real sandbox provider administration, image preparation status, immutable pinned usable image; Daytona key/image absent." + }, + { + "scenarioId": "recovery.explain-stall", + "readinessClass": "local", + "needs": "Real retained recovery interruption/attempt counter/next action; contrast infrastructure, input, budget and deliberate stop causes." + }, + { + "scenarioId": "recovery.controller-restart", + "readinessClass": "setup-turn", + "needs": "Existing input.restart supports action; native saved file/instruction before canonical query requires declared setup-human boundary." + }, + { + "scenarioId": "recovery.source-action", + "readinessClass": "local", + "needs": "Real failed task and source recovery action, owner readback, separate independent repair only when actual work warrants it." + }, + { + "scenarioId": "recovery.retry-exhausted", + "readinessClass": "local", + "needs": "Genuine exhausted infrastructure attempts and human Retry; previous evidence retained and new admission gates independently read." + }, + { + "scenarioId": "recovery.workspace-repair", + "readinessClass": "local", + "needs": "Real export failure and newer destination edits; board reconciliation and uncertain-source hold; owned filesystem fixtures allowed." + }, + { + "scenarioId": "recovery.unknown-email-outcome", + "readinessClass": "provider", + "needs": "Authorized real email with lost response after possible acceptance and unresolved provider receipt; no automatic resend, independent duplicates check." + }, + { + "scenarioId": "recovery.post-stop-message", + "readinessClass": "local", + "needs": "Real run cleanup boundary; explicit Stop and saved new direction, single eligible continuation, no concurrent owner." + }, + { + "scenarioId": "recovery.browser-reconnect", + "readinessClass": "local", + "needs": "Page context network disconnect while actual worker progresses; persisted events/history restored with no duplicate send or restart." + }, + { + "scenarioId": "steering.queue-follow-up", + "readinessClass": "local", + "needs": "Real busy Morgan; save once correct task queue; order/wait state and actual later delivery." + }, + { + "scenarioId": "steering.edit-queued-message", + "readinessClass": "source", + "needs": "Human-owned undispatched queue; browser edit/reorder, stale and dispatched branches; source task UI selection separate from fixed lead source." + }, + { + "scenarioId": "steering.discard-queued-message", + "readinessClass": "source", + "needs": "Human-owned queued comment; browser discard eligible entry and separate dispatched cancellation-as-direction branch." + }, + { + "scenarioId": "steering.steer-active", + "readinessClass": "local", + "needs": "Real compatible active harness and explicit Steer; delivered acknowledgement and actual output; provider rejection continuation branch." + }, + { + "scenarioId": "steering.interrupt-with-direction", + "readinessClass": "local", + "needs": "Real active task with saved human direction; explicit interruption; old cancelled run, no concurrent owners, delivery once after cleanup." + }, + { + "scenarioId": "steering.stop-versus-pause", + "readinessClass": "local", + "needs": "Running task under parent; board tree pause, persisted pause versus turn stop and explicit later resume/schedule." + }, + { + "scenarioId": "steering.preserve-draft", + "readinessClass": "source", + "needs": "Unsent canonical query and completed upload receipt must survive task switch/reload before send; shared flow sends original query immediately after setup." + }, + { + "scenarioId": "steering.approval-delivery", + "readinessClass": "local", + "needs": "Real waiting question plus ordinary comment queue during turn settlement; human answer delivery distinct from editable queue, once only." + }, + { + "scenarioId": "task-coordination.delegate-with-context", + "readinessClass": "local", + "needs": "CEO-owned source launch task and actual attached brief; single Morgan child/linked work, real report incorporated without false parent attribution." + }, + { + "scenarioId": "task-coordination.block-on-design", + "readinessClass": "local", + "needs": "Real unfinished design dependency, actual website admission blocked then resumed only after authorized prerequisite completion and other gates clear." + }, + { + "scenarioId": "task-coordination.reject-dependency-cycle", + "readinessClass": "implemented", + "needs": "Real Design\u2192Website edge before query; reject reverse edge, unchanged records, both detail endpoint sets and actual task links." + }, + { + "scenarioId": "task-coordination.independent-review", + "readinessClass": "local", + "needs": "Real delivered Morgan result plus independent eligible Riley; governed completion and self-approval rejection." + }, + { + "scenarioId": "task-coordination.pause-tree", + "readinessClass": "local", + "needs": "Two genuinely running descendants and manual-paused child; board tree pause/resume with independent child pause and budget hold preserved." + }, + { + "scenarioId": "workspaces.locate-files", + "readinessClass": "local", + "needs": "Actual projectless task execution binding; independent read active binding and distinguish pending choice from agent defaults." + }, + { + "scenarioId": "workspaces.prepare-repository", + "readinessClass": "provider", + "needs": "Authorized acorn/site source currently unavailable; use auditable incidental fixture binding only if manager approves real owned Git fixture identity." + }, + { + "scenarioId": "workspaces.private-repository-no-access", + "readinessClass": "provider", + "needs": "Real private repository and absent eligible Git connection, followed by genuine human access grant and intended-source preparation." + }, + { + "scenarioId": "workspaces.preview-service", + "readinessClass": "local", + "needs": "Owned ready workspace with real website files; start supported service; independent readiness HTTP/endpoint view plus startup-failure branch." + }, + { + "scenarioId": "workspaces.inspect-git", + "readinessClass": "local", + "needs": "Owned task-bound Git fixture with two genuine uncommitted changes; independently compare actual diff/file paths and preserve both edits." + }, + { + "scenarioId": "workspaces.close-and-reopen", + "readinessClass": "local", + "needs": "Real retained task workspace no active run; close lifecycle state with files retained and actual retrieval/reopen proof." + }, + { + "scenarioId": "workspaces.recover-lost-sandbox", + "readinessClass": "provider", + "needs": "True historical export failure and destroyed exact sandbox, surviving attachments; board saved-source reconstruction and fresh admission, no false recovered lost files." + } + ] +} diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-support.ts b/tests/runner-e2e/chat-catalog-domains/runtime-support.ts new file mode 100644 index 00000000000..434e840b243 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-support.ts @@ -0,0 +1,79 @@ +import { expect } from "@playwright/test"; +import { pollUntil } from "../api.js"; +import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js"; +import { codexSubscriptionAgentConfig } from "../codex-subscription-fixture.js"; +import { observeInternal, type Entity, type InternalObservation } from "./internal-support.js"; + +export interface RuntimeAgent extends Entity { + reportsTo?: string | null; adapterType?: string; adapterConfig?: Record; + runtimeConfig?: Record; permissions?: Record; + defaultEnvironmentId?: string | null; lifecycleState?: string; +} +export interface InstructionFile { + path: string; content: string; revision: { id: string; parentRevisionId?: string | null; + companyId: string; agentId: string; actorAgentId?: string | null; sourceRunId?: string | null; source: string; restoredFromRevisionId?: string | null }; +} +export interface RuntimeObservation extends InternalObservation { + agents: RuntimeAgent[]; + instruction?: InstructionFile; + instructionHistory?: { revisions: InstructionFile["revision"][] }; + instructionSideFile?: { content: string }; +} +export async function runtimeWorker(ctx: ChatCatalogContext, name = "Morgan", extra: Record = {}) { + const { api, execution, fixtures, workspacePath, nonce } = ctx.input; + const base = execution.profile.buildAgent({ environmentId: fixtures.environment.id, environmentFixtureId: execution.environment.id, + workspacePath, secretRefs: fixtures.secretRefs, executionId: nonce }); + const payload = { ...base, name, role: "engineer", title: "Frontend engineer", reportsTo: fixtures.agent.id, + runtimeConfig: { ...(base.runtimeConfig as Record), heartbeat: { enabled: false, maxConcurrentRuns: 1 } }, ...extra }; + const agent = await api.post(`/api/companies/${fixtures.company.id}/agents`, execution.profile.credential === "CODEX_AUTH_JSON" + ? codexSubscriptionAgentConfig(payload, process.env, fixtures.company.id) : payload); + return pollUntil({ label: `${name} real runtime readiness`, deadlineAt: Date.now() + 45_000, + load: () => api.get(`/api/agents/${agent.id}`), accept: row => row.lifecycleState === "ready" }); +} +export async function observeRuntime(ctx: ChatCatalogContext, state: Record): Promise { + const o = await observeInternal(ctx, state); + const agents = await ctx.input.api.get(`/api/companies/${o.companyId}/agents`); + const result: RuntimeObservation = { ...o, agents }; + if (state.instructionAgentId) { + const root = `/api/agents/${state.instructionAgentId}/instructions-bundle`; + result.instruction = await ctx.input.api.get(`${root}/file?path=AGENTS.md`); + result.instructionHistory = await ctx.input.api.get(`${root}/history?path=AGENTS.md`); + result.instructionSideFile = await ctx.input.api.get(`${root}/file?path=guidance.md`); + } + return result; +} +export function runtimeObservation(value: unknown): RuntimeObservation | undefined { + const o = value as RuntimeObservation | undefined; + return o && o.state && typeof o.reply === "string" && typeof o.companyId === "string" && typeof o.leadId === "string" + && [o.agents, o.tasks, o.links, o.activity, o.runs, o.humanQueries].every(Array.isArray) ? o : undefined; +} +export function agentLink(o: RuntimeObservation, agent: RuntimeAgent): string | undefined { + return o.links.find(href => { + try { const url = new URL(href, o.origin); const parts = url.pathname.split("/").map(decodeURIComponent); const index = parts.indexOf("agents"); + return url.origin === new URL(o.origin).origin && index >= 0 && [agent.id, agent.urlKey].includes(parts[index + 1]); } + catch { return false; } + }); +} +export async function verifyRuntimeResult(ctx: ChatCatalogContext, state: Record, value: unknown) { + const o = runtimeObservation(value); if (!o) throw new Error("Missing independent runtime result"); + const target = o.agents.find(row => row.id === (state.worker as RuntimeAgent | undefined)?.id); + if (!target) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); return; } + const href = agentLink(o, target); if (!href) throw new Error("No actual rendered target-agent link"); + const source = ctx.input.page.url(); + await ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`).first().click(); + await expect(ctx.input.page.getByRole("heading", { name: target.name!, exact: true }).first()).toBeVisible(); + await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); + await expect(ctx.input.page.getByRole("heading", { name: target.name!, exact: true }).first()).toBeVisible(); + await ctx.input.evidence("chat-catalog-runtime-profile-opened.json", { agentId: target.id, companyId: target.companyId, url: ctx.input.page.url() }); + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); +} +export function runtimeDefinition(scenarioId: string, setup: ChatCatalogCase["setup"], grade: ChatCatalogCase["grade"], extra: Partial = {}): ChatCatalogCase { + return { scenarioId, setup, grade, observe: observeRuntime, verifyResult: verifyRuntimeResult, + oracleVersion: "paperclip.chat-catalog.runtime.v1", coverageGaps: [], expectedRunCount: 1, + turnTimeoutMs: 120_000, attemptTimeoutMs: 300_000, + sourceFiles: ["chat-catalog-domains/runtime-support.ts", `chat-catalog-domains/runtime-${scenarioId.split(".")[0]}.ts`, `chat-scenarios/${scenarioId.split(".")[0]}.yaml`], ...extra }; +} +export function unchangedAgent(before: RuntimeAgent, after: RuntimeAgent, except: readonly string[] = []) { + return ["id", "companyId", "name", "role", "adapterType", "adapterConfig", "runtimeConfig", "permissions", "defaultEnvironmentId", "reportsTo"] + .filter(key => !except.includes(key)).every(key => JSON.stringify(before[key as keyof RuntimeAgent]) === JSON.stringify(after[key as keyof RuntimeAgent])); +} diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-task-coordination.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-task-coordination.test.ts new file mode 100644 index 00000000000..6325be4e77a --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-task-coordination.test.ts @@ -0,0 +1,21 @@ +import { it, expect } from "vitest"; +import { gradeDependencyCycle } from "./runtime-task-coordination.js"; +function valid() { + const website = { id: "website", title: "Website", companyId: "company", status: "backlog", blockedBy: [] }; + const design = { id: "design", title: "Design", companyId: "company", status: "backlog", blockedBy: [{ id: "website" }] }; + return { scenarioId: "task-coordination.reject-dependency-cycle", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", + state: { task: website, design }, dependencyTasks: [structuredClone(website), structuredClone(design)], agents: [], tasks: [website, design], + links: ["/ACORN/issues/website", "/ACORN/issues/design"], reply: "Design already depends on Website. That would be circular. Choose which should go first.", + activity: [], runs: [], humanQueries: [] }; +} +it("accepts an independently saved acyclic rejection", () => expect(gradeDependencyCycle(valid()).passed).toBe(true)); +it("rejects missing dependency detail", () => expect(gradeDependencyCycle(undefined).passed).toBe(false)); +it("rejects a plausible explanation with the cycle actually saved", () => { + const o = valid(); o.dependencyTasks[0]!.blockedBy = [{ id: "design" }]; expect(gradeDependencyCycle(o).passed).toBe(false); +}); +it("rejects silently deleting the original prerequisite", () => { + const o = valid(); o.dependencyTasks[1]!.blockedBy = []; expect(gradeDependencyCycle(o).passed).toBe(false); +}); +it("rejects a conflict with no actionable order decision", () => { + const o = valid(); o.reply = "Website and Design form a cycle."; expect(gradeDependencyCycle(o).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-task-coordination.ts b/tests/runner-e2e/chat-catalog-domains/runtime-task-coordination.ts new file mode 100644 index 00000000000..ef340469622 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-task-coordination.ts @@ -0,0 +1,36 @@ +import { checks, project, task, sameTask, hasLink, verifyInternalResult, type Entity } from "./internal-support.js"; +import { runtimeDefinition, observeRuntime, runtimeObservation, type RuntimeObservation } from "./runtime-support.js"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; + +interface DependencyTask extends Entity { blockedByIssueIds?: string[]; blockedBy?: Array<{ id: string }> } +interface DependencyObservation extends RuntimeObservation { dependencyTasks: DependencyTask[] } +export function gradeDependencyCycle(value: unknown) { + const c = checks(); const o = runtimeObservation(value) as DependencyObservation | undefined; + c.add("complete-dependency-observation", !!o && Array.isArray(o.dependencyTasks), "Both dependency sets are read from their actual persisted detail endpoints."); + if (!o || !Array.isArray(o.dependencyTasks)) return c.result(); + const website = o.state.task as DependencyTask; const design = o.state.design as DependencyTask; + const currentWebsite = o.dependencyTasks.find(row => row.id === website.id); + const currentDesign = o.dependencyTasks.find(row => row.id === design.id); + const blockers = (row: DependencyTask | undefined) => row?.blockedBy?.map(edge => edge.id) ?? row?.blockedByIssueIds; + c.add("original-edge-retained-reverse-edge-absent", !!currentWebsite && !!currentDesign + && blockers(currentWebsite)?.length === 0 && blockers(currentDesign)?.length === 1 && blockers(currentDesign)?.[0] === website.id, + "Design still waits for Website; the requested reverse edge is rejected without removing the original prerequisite."); + c.add("same-company-unchanged-task-records", !!currentWebsite && !!currentDesign && sameTask(website, currentWebsite) + && sameTask(design, currentDesign) && [currentWebsite, currentDesign].every(row => row.companyId === o.companyId) && o.tasks.length === 2, + "Cycle rejection preserves the two task identities, ownership and ordinary properties."); + c.add("explains-actual-conflict-and-next-decision", /\bWebsite\b/i.test(o.reply) && /\bDesign\b/i.test(o.reply) + && /cycl|circular|already.*depend|deadlock/i.test(o.reply) && /choose|which|instead|remove|order|first/i.test(o.reply), + "The response explains the named dependency conflict and an actionable order decision."); + c.add("actual-task-result-links", hasLink(o, "issues", website) && hasLink(o, "issues", design), "Both actual dependency records are linked from chat."); + return c.result(); +} +async function observeDependencies(ctx: ChatCatalogContext, state: Record): Promise { + const o = await observeRuntime(ctx, state); + const dependencyTasks = await Promise.all([state.task as Entity, state.design as Entity].map(row => ctx.input.api.get(`/api/issues/${row.id}`))); + return { ...o, dependencyTasks }; +} +export const runtimeTaskCoordinationCases = [runtimeDefinition("task-coordination.reject-dependency-cycle", async ctx => { + const p = await project(ctx); const website = await task(ctx, "Website", p.id); + const design = await task(ctx, "Design", p.id, { blockedByIssueIds: [website.id] }); + return { project: p, task: website, design }; +}, gradeDependencyCycle, { observe: observeDependencies, verifyResult: verifyInternalResult })];