diff --git a/tests/runner-e2e/chat-catalog-domains/internal-timeline-verifier.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-timeline-verifier.test.ts new file mode 100644 index 00000000000..299e7fd0e26 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/internal-timeline-verifier.test.ts @@ -0,0 +1,113 @@ +import { createHash } from "node:crypto"; +import { afterEach, expect, it, vi } from "vitest"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; + +const port = vi.hoisted(() => ({ handler: undefined as any, providerToken: "", executed: [] as string[][], infrastructureFailure: false })); +vi.mock("node:http", () => ({ createServer: (handler: unknown) => { + port.handler = handler; + return { closeAllConnections: vi.fn(), close: (done: () => void) => done() }; +} })); +vi.mock("../../e2e/fetch-allowed-port.js", () => ({ listenOnFetchAllowedPort: async () => 43123 })); +// Execute only the owning verifier's unchanged bounded Node command on synthetic +// HTML; no server, browser, application or native agent process is started. +vi.mock("node:child_process", async original => { + const actual = await original(); + return { ...actual, execFile: (...args: any[]) => { + port.executed.push(args[1]); + if (port.infrastructureFailure) { queueMicrotask(() => args[3](Object.assign(Error("synthetic command unavailable"), { code: "ENOENT" }))); return; } + return (actual.execFile as any)(...args); + } }; +}); +import { startTimelineVerifier } from "./internal-timeline-verifier.js"; +afterEach(() => { vi.restoreAllMocks(); vi.unstubAllGlobals(); }); +const broken = '

Launch

'; +const repaired = '

Launch

'; +type Options = { wrongTask?: boolean; foreignCompany?: boolean; wrongWorker?: boolean; wrongRun?: boolean; + stopped?: boolean; infrastructureFailure?: boolean; changedAfterCommand?: boolean; missingFile?: boolean; wrongFile?: boolean; badEncoding?: boolean; oversized?: boolean; changedCheckout?: boolean }; +function fixture(options: Options = {}) { + port.executed.splice(0); port.providerToken = ""; port.infrastructureFailure = options.infrastructureFailure ?? false; + let body = broken, reads = 0, taskReads = 0; + const current = () => ({ id: options.wrongTask ? "other" : "task", companyId: options.foreignCompany ? "foreign" : "company", + assigneeAgentId: options.wrongWorker ? "foreign-worker" : "worker", executionRunId: options.wrongRun || options.changedCheckout && reads > 0 || options.changedAfterCommand && taskReads > 2 ? "other-run" : "native" }); + const agentDenied = vi.fn(); + vi.stubGlobal("fetch", vi.fn(async (url: URL | string, init: RequestInit) => { + if (String(url).includes("/file-resources/content")) { agentDenied(); return new Response("Board access required", { status: 403 }); } + expect(new Headers(init.headers).get("authorization")).toBe("Bearer synthetic-worker-key"); + taskReads++; return new Response(JSON.stringify(current())); + })); + const get = vi.fn(async (path: string) => { + expect(path).toBe("/api/issues/task/file-resources/content?workspace=execution&path=launch.html"); + reads++; + if (options.missingFile) throw Error("Owned file absent"); + return { resource: { kind: "file", title: options.wrongFile ? "foreign.html" : "launch.html", workspaceKind: "execution_workspace" }, + content: { encoding: options.badEncoding ? "unknown" : "utf8", data: options.oversized ? "x".repeat(262_145) : body } }; + }); + const postSensitive = vi.fn(async (path: string, value: any) => { + if (path.includes("/keys")) return { id: "worker-key", token: "synthetic-worker-key" }; + port.providerToken = value.credentialValues["credentials.authorization"]; + return { connectionId: "connection", actions: { readOnly: [{ catalogEntryId: "verify" }] } }; + }); + const ctx = { input: { privateSecrets: [] as string[], fixtures: { company: { id: "company" } }, + api: { baseURL: "https://fixture.test", get, postSensitive, post: vi.fn(async () => {}), putSensitive: vi.fn(async () => {}), delete: vi.fn(async () => {}) } }, + allRuns: vi.fn(async () => [{ id: "native", companyId: "company", agentId: "worker", status: options.stopped ? "cancelled" : "running", runtimeMode: "native", contextSnapshot: { issueId: "task" } }]), + } as unknown as ChatCatalogContext; + async function call(token = port.providerToken, arguments_: Record = {}) { + let status = 0, response = ""; + const req = { headers: { authorization: `Bearer ${token}` }, async *[Symbol.asyncIterator]() { yield Buffer.from(JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name: "verify_launch_html", arguments: arguments_ } })); } }; + const res = { headersSent: false, writeHead(code: number) { status = code; this.headersSent = true; }, end(text = "") { response = text; } }; + await port.handler(req, res); + return { status, body: response ? JSON.parse(response) : null }; + } + return { ctx, get, call, agentDenied, setBody: (value: string) => { body = value; } }; +} +it("actual verifier uses only authorized fixture read and executes genuine failed-then-repaired file checks", async () => { + const f = fixture(); const verifier = await startTimelineVerifier(f.ctx, "task", ["worker", "repair-worker"]); + try { + const failed = await f.call(); + expect(failed.status).toBe(200); expect(failed.body.result.isError).toBe(true); + expect(verifier.receipts).toHaveLength(1); expect(verifier.receipts[0]!.exitCode).toBe(1); + expect(verifier.receipts[0]!.sha256).toBe(createHash("sha256").update(broken).digest("hex")); + f.setBody(repaired); + const passed = await f.call(); + expect(passed.status).toBe(200); expect(passed.body.result.isError).toBe(false); + expect(verifier.receipts.map(row => row.exitCode)).toEqual([1, 0]); + expect(verifier.receipts[1]!.sha256).toBe(createHash("sha256").update(repaired).digest("hex")); + expect(port.executed).toHaveLength(2); expect(port.executed.every(argv => argv[0] === "-e")).toBe(true); + expect(f.get).toHaveBeenCalledTimes(2); expect(f.agentDenied).not.toHaveBeenCalled(); + } finally { await verifier.close(); } +}); +it.each([{ wrongTask: true }, { foreignCompany: true }, { wrongWorker: true }, { wrongRun: true }, { stopped: true }, + { missingFile: true }, { wrongFile: true }, { badEncoding: true }, { oversized: true }, { changedCheckout: true }])("actual verifier refuses invalid scope/custody/content %j without a fabricated exit receipt", async options => { + const f = fixture(options); const verifier = await startTimelineVerifier(f.ctx, "task", ["worker"]); + try { + expect((await f.call()).status).toBe(502); + expect(verifier.receipts).toEqual([]); + expect(port.executed).toHaveLength(0); + } finally { await verifier.close(); } +}); +it("provider authentication failure cannot invoke the fixture board reader", async () => { + const f = fixture(); const verifier = await startTimelineVerifier(f.ctx, "task", ["worker"]); + try { expect((await f.call("unrelated-token")).status).toBe(401); expect(f.get).not.toHaveBeenCalled(); expect(verifier.receipts).toEqual([]); } + finally { await verifier.close(); } +}); + +it("requested fake exit and path cannot replace the real fixed-file command outcome", async () => { + const f = fixture(); const verifier = await startTimelineVerifier(f.ctx, "task", ["worker"]); + try { + const result = await f.call(port.providerToken, { exitCode: 0, path: "foreign.html" }); + expect(result.status).toBe(200); + expect(verifier.receipts).toHaveLength(1); + expect(verifier.receipts[0]).toMatchObject({ taskId: "task", runId: "native", filename: "launch.html", exitCode: 1 }); + expect(f.get).toHaveBeenCalledExactlyOnceWith("/api/issues/task/file-resources/content?workspace=execution&path=launch.html"); + expect(JSON.stringify(result.body)).not.toContain("synthetic-worker-key"); + expect(JSON.stringify(result.body)).not.toContain(port.providerToken); + } finally { await verifier.close(); } +}); +it.each([{ infrastructureFailure: true }, { changedAfterCommand: true }])("command infrastructure or post-command custody failure never fabricates a result %j", async options => { + const f = fixture(options); const verifier = await startTimelineVerifier(f.ctx, "task", ["worker"]); + try { + expect((await f.call()).status).toBe(502); + expect(port.executed).toHaveLength(1); + expect(verifier.receipts).toEqual([]); + } finally { await verifier.close(); } +}); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-timeline-verifier.ts b/tests/runner-e2e/chat-catalog-domains/internal-timeline-verifier.ts index 81f8523bbe9..dcf5436870f 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-timeline-verifier.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-timeline-verifier.ts @@ -3,6 +3,7 @@ import { randomBytes, createHash } from "node:crypto"; import { execFile } from "node:child_process"; import { listenOnFetchAllowedPort } from "../../e2e/fetch-allowed-port.js"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import type { WorkspaceFileContent } from "../../../packages/shared/src/types/workspace-file-resource.js"; export interface VerificationReceipt { taskId: string; runId: string; filename: string; sha256: string; exitCode: number; checkedAt: string; command: string } /** Real owned verification service: reads actual product workspace bytes, then executes a real bounded command. */ export async function startTimelineVerifier(ctx: ChatCatalogContext, taskId: string, agentIds: string[]) { @@ -20,14 +21,35 @@ export async function startTimelineVerifier(ctx: ChatCatalogContext, taskId: str if (m.method === "tools/list") { reply(m.id, { tools: [{ name: "verify_launch_html", description: "Run the real accessible-heading and labeled-email-input check on the current launch.html task workspace file. Returns the actual command exit code and byte digest.", inputSchema: { type: "object", properties: {}, additionalProperties: false }, annotations: { readOnlyHint: true, destructiveHint: false } }] }); return; } if (m.method !== "tools/call" || m.params?.name !== "verify_launch_html") { res.writeHead(204); res.end(); return; } const get = async (path: string) => { const response = await fetch(new URL(path, ctx.input.api.baseURL), { headers: { Authorization: `Bearer ${key.token}` }, redirect: "error", signal: AbortSignal.timeout(15000) }); if (!response.ok) { await response.body?.cancel(); throw new Error("Verifier cannot read actual task workspace"); } return response.json(); }; - const current = await get(`/api/issues/${taskId}`) as { executionRunId?: string; assigneeAgentId?: string }; - if (!current.executionRunId || !agentIds.includes(current.assigneeAgentId ?? "")) throw new Error("No actual authorized native task checkout"); - const file = await get(`/api/issues/${taskId}/file-resources/content?workspace=execution&path=launch.html`) as { content: { encoding: string; data: string } }; + const checkout = async (expectedRunId?: string, expectedAgentId?: string) => { + // Worker credentials remain limited to ordinary task access. Only the + // trusted fixture reads the board-only file route after this live join. + const current = await get(`/api/issues/${taskId}`) as { id?: string; companyId?: string; executionRunId?: string; assigneeAgentId?: string }; + const runs = await ctx.allRuns(); + const live = runs.find(run => run.id === current.executionRunId); + if (current.id !== taskId || current.companyId !== ctx.input.fixtures.company.id || !current.executionRunId + || !agentIds.includes(current.assigneeAgentId ?? "") || !live || live.companyId !== current.companyId + || live.agentId !== current.assigneeAgentId || live.status !== "running" || live.runtimeMode !== "native" + || (live.contextSnapshot?.issueId !== taskId && live.contextSnapshot?.taskId !== taskId) + || expectedRunId !== undefined && current.executionRunId !== expectedRunId + || expectedAgentId !== undefined && current.assigneeAgentId !== expectedAgentId) + throw new Error("No actual authorized native task checkout"); + return { runId: current.executionRunId, agentId: current.assigneeAgentId! }; + }; + const current = await checkout(); + // Fixed owned task/file only: never pass board credentials to the model, + // provider, worker bearer, or a caller-selected URL/path. + const file = await ctx.input.api.get(`/api/issues/${taskId}/file-resources/content?workspace=execution&path=launch.html`); + if (file.resource?.kind !== "file" || file.resource.title !== "launch.html" || file.resource.workspaceKind !== "execution_workspace" + || !file.content || typeof file.content.data !== "string" || !["utf8", "base64"].includes(file.content.encoding)) + throw new Error("Verifier cannot read actual owned launch file"); + await checkout(current.runId, current.agentId); const bytes = file.content.encoding === "base64" ? Buffer.from(file.content.data, "base64") : Buffer.from(file.content.data); if (bytes.length > 262144) throw new Error("Oversized launch fixture"); const script = "const s=Buffer.from(process.argv[1],'base64').toString();process.exit(/]/i.test(s)&&/]*for=[\"']email[\"']/i.test(s)&&/]*id=[\"']email[\"']/i.test(s)?0:1)"; const exitCode = await new Promise((resolve, reject) => execFile(process.execPath, ["-e", script, bytes.toString("base64")], { timeout: 10000, maxBuffer: 65536 }, error => { if (!error) resolve(0); else if (typeof error.code === "number") resolve(error.code); else reject(error); })); - const receipt = { taskId, runId: current.executionRunId, filename: "launch.html", sha256: createHash("sha256").update(bytes).digest("hex"), exitCode, checkedAt: new Date().toISOString(), command: "node accessible-heading-and-email-label-check" }; + await checkout(current.runId, current.agentId); + const receipt = { taskId, runId: current.runId, filename: "launch.html", sha256: createHash("sha256").update(bytes).digest("hex"), exitCode, checkedAt: new Date().toISOString(), command: "node accessible-heading-and-email-label-check" }; receipts.push(receipt); reply(m.id, { content: [{ type: "text", text: JSON.stringify(receipt) }], isError: exitCode !== 0 }); } catch { if (!res.headersSent) res.writeHead(502); res.end(); } });