diff --git a/tests/runner-e2e/CHAT-FEATURES.md b/tests/runner-e2e/CHAT-FEATURES.md index 8473ab53833..73efea89367 100644 --- a/tests/runner-e2e/CHAT-FEATURES.md +++ b/tests/runner-e2e/CHAT-FEATURES.md @@ -45,6 +45,17 @@ private secondary-user credentials join the attempt's redaction sink. Scenarios whose stated context has no working chat require a genuine boundary journey; these cannot receive a native chat pass from seeing an onboarding or login gate. +Catalog lanes that already use the original native case clock use that same +deadline for the prerequisite worker's readiness. The separate 45-second cap +cannot cut short a normal 60-second local OpenCode hello probe or its owned +verification recovery. Manual configuration repair and stopped lifecycle states +still fail setup. Other lanes keep their authored readiness cap. Each worker's +scoped state transitions and recovery flags are retained without provider output. +The helper consumes the public lifecycle state and fixed automatic-recovery or +verification-paused diagnostic; private lifecycle operation fields are not exposed. +Tests pass actual lifecycle diagnostics through the agent response normalizer; +the case budget and native execution limits do not increase. + | Scenario | Actual user request | Independent outcome | | --- | --- | --- | diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index d17bd84f804..194621f7d0d 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -3956,11 +3956,11 @@ "chat-catalog-domains/internal-projects.ts", "chat-scenarios/projects.yaml", "chat-catalog-domains/internal-project-defaults.ts", - "chat-catalog-domains/internal-run-brief.ts", + "chat-catalog-domains/internal-native-run-brief.ts", "../e2e/fetch-allowed-port.ts", "run-observations.ts" ], - "oracleVersion": "paperclip.chat-catalog.internal.v1", + "oracleVersion": "paperclip.chat-catalog.project-defaults.v4", "coverageGaps": [], "expectedRunCount": 3, "minimumRunCount": 3, diff --git a/tests/runner-e2e/chat-catalog-domains/internal-definition.ts b/tests/runner-e2e/chat-catalog-domains/internal-definition.ts index ff471c9f4fc..26ebb904867 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-definition.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-definition.ts @@ -1,4 +1,4 @@ -import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js"; +import { chatCatalogOriginalPhaseDeadline, type ChatCatalogCase, type ChatCatalogContext } from "../chat-catalog-contract.js"; import { observeInternal, verifyInternalResult, type Entity, type InternalObservation } from "./internal-support.js"; import { codexSubscriptionAgentConfig } from "../codex-subscription-fixture.js"; import { pollUntil } from "../api.js"; @@ -12,6 +12,7 @@ export function makeDefinition(scenarioId: string, setup: ChatCatalogCase["setup } export async function fixtureWorker(ctx: ChatCatalogContext) { + const originalDeadline = chatCatalogOriginalPhaseDeadline(ctx); const { execution, fixtures, workspacePath, nonce, api } = ctx.input; const base = execution.profile.buildAgent({ environmentId: fixtures.environment.id, environmentFixtureId: execution.environment.id, workspacePath, secretRefs: fixtures.secretRefs, executionId: nonce }); @@ -19,9 +20,40 @@ export async function fixtureWorker(ctx: ChatCatalogContext) { capabilities: "Builds accessible websites.", runtimeConfig: { ...(base.runtimeConfig as Record), heartbeat: { enabled: false, maxConcurrentRuns: 1 } } }; const worker = await api.post(`/api/companies/${fixtures.company.id}/agents`, execution.profile.credential === "CODEX_AUTH_JSON" ? codexSubscriptionAgentConfig(payload, process.env, fixtures.company.id) : payload); - await pollUntil({ label: "Morgan runtime readiness", deadlineAt: Date.now() + 45_000, - load: () => api.get<{ lifecycleState: string; lifecycleError?: string }>(`/api/agents/${worker.id}`), - accept: row => row.lifecycleState === "ready", reject: row => row.lifecycleError ? "Morgan runtime setup failed" : undefined }); + const deadlineAt = originalDeadline ?? Date.now() + 45_000; + const observations: Record[] = []; + let previous: string | undefined; + try { + await pollUntil({ label: "Morgan runtime readiness", deadlineAt, + load: async () => { + const row = await api.get<{ id: string; companyId: string; lifecycleState: string; status?: string; lifecycleError?: string | null }>(`/api/agents/${worker.id}`, + { timeout: Math.max(1, Math.min(5_000, deadlineAt - Date.now())) }); + const state = { id: row.id, companyId: row.companyId, lifecycleState: row.lifecycleState, status: row.status, + errorPresent: !!row.lifecycleError, + automaticRecovery: /\b(?:Paperclip will|It will) retry automatically\./.test(row.lifecycleError ?? ""), + verificationPaused: row.lifecycleError?.includes("Automatic verification is paused.") ?? false }; + const current = JSON.stringify(state); + if (current !== previous) { observations.push({ ...state, observedAt: new Date().toISOString() }); previous = current; } + return row; + }, + accept: row => Date.now() < deadlineAt && row.id === worker.id && row.companyId === fixtures.company.id + && row.lifecycleState === "ready" && !row.lifecycleError, + reject: row => { + if (row.id !== worker.id || row.companyId !== fixtures.company.id) return "Morgan runtime identity changed"; + if (["pending_approval", "paused", "rejected", "terminated", "terminating", "cleaning_up"].includes(row.lifecycleState)) return "Morgan runtime setup stopped"; + // Public agent rows omit lifecycleOperation. The lifecycle owner exposes its + // automatic-recovery promise and manual-repair boundary in fixed diagnostics. + const recovering = originalDeadline !== undefined && row.lifecycleState === "verifying" + && /\b(?:Paperclip will|It will) retry automatically\./.test(row.lifecycleError ?? "") + && !row.lifecycleError?.includes("Automatic verification is paused."); + return row.lifecycleError && !recovering ? "Morgan runtime setup failed" : undefined; + } }); + } finally { + await ctx.input.evidence(`chat-catalog-worker-readiness-${worker.id}.json`, { + policy: originalDeadline === undefined ? "authored-45s" : "original-native-case-clock", + deadlineAt, agentId: worker.id, companyId: fixtures.company.id, observations, + }); + } return worker; } diff --git a/tests/runner-e2e/chat-catalog-domains/internal-native-run-brief.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-native-run-brief.test.ts new file mode 100644 index 00000000000..e02153c7d54 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/internal-native-run-brief.test.ts @@ -0,0 +1,63 @@ +import { spawn } from "node:child_process"; +import { once } from "node:events"; +import { readFile, stat } from "node:fs/promises"; +import { expect, it } from "vitest"; +import { startNativeRunBriefGate } from "./internal-native-run-brief.js"; + +const worker = "10000000-0000-4000-8000-000000000001"; +it("keeps the registered gate credential out of task and run evidence", async () => { + const secrets: string[] = []; + const gate = await startNativeRunBriefGate(worker, Date.now() + 120_000, secrets); + try { + expect(secrets).toHaveLength(1); + expect(gate.command).not.toContain(secrets[0]); + expect(gate.command).not.toContain("Bearer"); + expect((await stat(gate.scriptPath)).mode & 0o777).toBe(0o600); + expect(await readFile(gate.scriptPath, "utf8")).toContain(secrets[0]); + } finally { await gate.close(); } + await expect(stat(gate.scriptPath)).rejects.toMatchObject({ code: "ENOENT" }); +}); +it("rejects a read without the per-worker gate capability before recording a receipt", async () => { + const gate = await startNativeRunBriefGate(worker, Date.now() + 120_000); + try { + const script = await readFile(gate.scriptPath, "utf8"); + const url = JSON.parse(script.match(/get\(("[^"]+")/)![1]!); + for (const headers of [new Headers(), new Headers({ authorization: "Bearer incorrect" })]) { + expect((await fetch(url, { headers })).status).toBe(401); + expect(gate.calls).toEqual([]); + } + } finally { await gate.close(); } +}); +it("holds an actual native command until the owned brief becomes available", async () => { + const gate = await startNativeRunBriefGate(worker, Date.now() + 120_000); + const child = spawn("sh", ["-c", gate.command], { stdio: ["ignore", "pipe", "pipe"] }); + let output = ""; child.stdout.on("data", chunk => { output += chunk; }); child.stderr.resume(); + const exited = once(child, "exit"); + try { + await expect.poll(() => gate.calls.length).toBe(1); + await new Promise(resolve => setTimeout(resolve, 60)); + expect(child.exitCode).toBeNull(); expect(output).toBe(""); + expect(gate.calls[0]).toMatchObject({ workerId: worker, workerBound: true }); + expect(await readFile(gate.scriptPath, "utf8")).toContain("- Date.now()" ); + gate.release(); + expect((await exited)[0]).toBe(0); expect(output).toContain("The website brief is now available"); + } finally { child.kill("SIGTERM"); await gate.close(); await exited; } +}); +it("cleanup stops a pending native read without inventing a successful brief", async () => { + const gate = await startNativeRunBriefGate(worker, Date.now() + 120_000); + const child = spawn("sh", ["-c", gate.command], { stdio: "ignore" }); const exited = once(child, "exit"); + try { + await expect.poll(() => gate.calls.length).toBe(1); + await gate.close(); expect((await exited)[0]).not.toBe(0); + } finally { child.kill("SIGTERM"); await gate.close(); await exited; } +}); + +it("does not give a delayed native command a fresh hold allowance", async () => { + const gate = await startNativeRunBriefGate(worker, Date.now() + 150); + await new Promise(resolve => setTimeout(resolve, 180)); + const child = spawn("sh", ["-c", gate.command], { stdio: "ignore" }); const exited = once(child, "exit"); + try { expect((await exited)[0]).not.toBe(0); } finally { child.kill("SIGTERM"); await gate.close(); } +}); +it.each([0, -1, NaN, Infinity])("rejects an absent or exhausted original hold clock: %s", async deadline => { + await expect(startNativeRunBriefGate(worker, deadline)).rejects.toThrow(/deadline/); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-native-run-brief.ts b/tests/runner-e2e/chat-catalog-domains/internal-native-run-brief.ts new file mode 100644 index 00000000000..ba487efc7f5 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/internal-native-run-brief.ts @@ -0,0 +1,58 @@ +import { createServer, type ServerResponse } from "node:http"; +import { randomBytes } from "node:crypto"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { listenOnFetchAllowedPort } from "../../e2e/fetch-allowed-port.js"; + +const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`; + +/** Keep a real prerequisite worker active without the app gateway's short call deadline. */ +export async function startNativeRunBriefGate(workerId: string, deadlineAt: number, secrets: string[] = []) { + if (!/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i.test(workerId)) throw new Error("Invalid prerequisite worker identity"); + if (!Number.isFinite(deadlineAt) || deadlineAt <= Date.now()) throw new Error("Original prerequisite observation deadline is exhausted or missing"); + const route = `/website-brief/${workerId}`; + const token = randomBytes(32).toString("hex"); + secrets.push(token); + const authorization = `Bearer ${token}`; + const calls: Array<{ workerId: string; workerBound: boolean; receivedAt: string }> = []; + const pending = new Set(); + let released = false; + const reply = (response: ServerResponse) => { + if (response.destroyed) return; + response.writeHead(200, { "content-type": "text/plain" }); + response.end("The website brief is now available: Acorn Notes needs a semantic heading and an accessible signup link. Record a short implementation note and complete this prerequisite task. No website file or delivery is requested yet."); + }; + const server = createServer((request, response) => { + if (request.method !== "GET" || request.url !== route) { response.writeHead(404); response.end(); return; } + if (request.headers.authorization !== authorization) { response.writeHead(401); response.end(); return; } + calls.push({ workerId, workerBound: true, receivedAt: new Date().toISOString() }); + if (released) reply(response); + else { pending.add(response); response.once("close", () => pending.delete(response)); } + }); + const directory = await mkdtemp(join(tmpdir(), "runner-native-brief-")); + try { + const port = await listenOnFetchAllowedPort(server); + // fetch has a separate five-minute header timeout. The prerequisite must + // follow only its original absolute observation deadline, including a wait + // for headers. A one-use HTTP agent also avoids inherited socket timeouts. + const script = `import {get} from "node:http"; const brief = await new Promise((resolve, reject) => { const request = get(${JSON.stringify(`http://127.0.0.1:${port}${route}`)}, {agent: false, timeout: 0, headers: {authorization: ${JSON.stringify(authorization)}}, signal: AbortSignal.timeout(Math.max(1, ${deadlineAt} - Date.now()))}, response => { if (response.statusCode !== 200) { response.resume(); reject(new Error("Website brief read failed")); return; } response.setEncoding("utf8"); let body = ""; response.on("data", chunk => { body += chunk; }); response.on("end", () => resolve(body)); response.on("error", reject); }); request.on("error", reject); }); process.stdout.write(brief);`; + // The task and native tool arguments contain only a file path. Keep the + // capability in this owned 0700 directory / 0600 script, outside every + // evidence tree, and register it so an unexpected disclosure still fails. + const scriptPath = join(directory, "read-brief.mjs"); + await writeFile(scriptPath, script, { mode: 0o600, flag: "wx" }); + return { + calls, + scriptPath, + command: `exec ${quote(process.execPath)} ${quote(scriptPath)}`, + release() { released = true; for (const response of pending) reply(response); pending.clear(); }, + async close() { for (const response of pending) response.destroy(); pending.clear(); server.closeAllConnections(); await new Promise(resolve => server.close(() => resolve())); await rm(directory, { recursive: true, force: true }); }, + }; + } catch (error) { + server.closeAllConnections(); + await new Promise(resolve => server.close(() => resolve())); + await rm(directory, { recursive: true, force: true }); + throw error; + } +} diff --git a/tests/runner-e2e/chat-catalog-domains/internal-project-defaults-setup-cleanup.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-project-defaults-setup-cleanup.test.ts new file mode 100644 index 00000000000..d4145a287d2 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/internal-project-defaults-setup-cleanup.test.ts @@ -0,0 +1,45 @@ +import { readFile, stat } from "node:fs/promises"; +import { expect, it, vi } from "vitest"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { projectDefaultsCases } from "./internal-project-defaults.js"; + +vi.mock("./internal-definition.js", async original => ({ + ...await original(), + fixtureWorker: async () => ({ id: "10000000-0000-4000-8000-000000000001" }), +})); +vi.mock("../api.js", async original => ({ + ...await original(), + pollUntil: async () => { throw new Error("controlled checkout failure"); }, +})); + +it.each(["creation", "checkout"])("cleans the private prerequisite after %s fails before setup returns state", async phase => { + let scriptPath = "", url = ""; + const context = { + observationDeadlineAt: Date.now() + 120_000, + input: { + privateSecrets: [], fixtures: { company: { id: "company" }, agent: { id: "lead" } }, + api: { + patch: async () => ({}), + post: async (_path: string, body: Record) => { + if (body.title === "Active launch research") { + scriptPath = String(body.description).match(/'([^']*read-brief\.mjs)'/)![1]!; + const script = await readFile(scriptPath, "utf8"); + url = JSON.parse(script.match(/get\(("[^"]+")/)![1]!); + if (phase === "creation") throw new Error("controlled creation failure"); + } + return { ...body, id: "created-resource" }; + }, + }, + }, + } as unknown as ChatCatalogContext; + const definition = projectDefaultsCases[0]!; + try { + await expect(definition.setup(context)).rejects.toThrow(`controlled ${phase} failure`); + expect(scriptPath).not.toBe(""); + await expect(stat(scriptPath)).rejects.toMatchObject({ code: "ENOENT" }); + await expect(fetch(url, { signal: AbortSignal.timeout(1000) })).rejects.toThrow(); + } finally { + // Also close the old implementation after its regression assertion fails. + await definition.cleanup?.(context, {}); + } +}); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-project-defaults.ts b/tests/runner-e2e/chat-catalog-domains/internal-project-defaults.ts index 2af4fc8fa71..7a0065f007d 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-project-defaults.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-project-defaults.ts @@ -6,25 +6,26 @@ import { collectRunEvents } from "../run-observations.js"; import type { ChatCatalogCase } from "../chat-catalog-contract.js"; import { pollUntil } from "../api.js"; import { fixtureWorker, makeDefinition } from "./internal-definition.js"; -import { startRunBriefGate } from "./internal-run-brief.js"; +import { startNativeRunBriefGate } from "./internal-native-run-brief.js"; import { checks, entity, goal, observation, observeInternal, project, sameTask, task, type Entity, type InternalObservation } from "./internal-support.js"; interface DefaultsObservation extends InternalObservation { futureContext: { project?: Entity; goal?: Entity }; futureComments: Entity[]; futureProjectReads?: { runId: string; projectId: string; callId: string }[] } interface PreparedAgent extends Entity { lifecycleState?: string; adapterConfig?: { timeoutSec?: number } } -export function gradeProjectDefaultsSetup(value: { lead: PreparedAgent; future: PreparedAgent; held: PreparedAgent; task: Entity & { executionRunId?: string }; runs: Entity[]; activeRunId: string; gateCalls: { authorized?: boolean }[] }) { +export function gradeProjectDefaultsSetup(value: { lead: PreparedAgent; future: PreparedAgent; held: PreparedAgent; task: Entity & { executionRunId?: string }; runs: Entity[]; activeRunId: string; gateCalls: { workerBound?: boolean; workerId?: string }[] }) { const c = checks(); const ready = (agent: PreparedAgent) => agent.lifecycleState === "ready" && ["idle", "active"].includes(agent.status ?? ""); c.add("lead-ready", ready(value.lead) && value.lead.adapterConfig?.timeoutSec === CHAT_NATIVE_TIMEOUT_POLICY.savedAgentTimeoutSec, "The canonical lead is independently ready under the production-default native timeout policy and unchanged finite scenario budget."); c.add("future-worker-ready", ready(value.future) && value.future.adapterConfig?.timeoutSec === CHAT_NATIVE_TIMEOUT_POLICY.savedAgentTimeoutSec, "The future-context worker is independently ready under the production-default native timeout policy and unchanged finite scenario budget."); - c.add("existing-work-actively-owned", value.held.lifecycleState === "ready" && value.held.status === "running" && value.held.adapterConfig?.timeoutSec === 420 + c.add("existing-work-actively-owned", value.held.lifecycleState === "ready" && value.held.status === "running" && value.held.adapterConfig?.timeoutSec === CHAT_NATIVE_TIMEOUT_POLICY.savedAgentTimeoutSec && value.task.assigneeAgentId === value.held.id && value.task.executionRunId === value.activeRunId && value.runs.some(run => run.id === value.activeRunId && run.agentId === value.held.id && run.status === "running") - && value.gateCalls.some(call => call.authorized), "Only the deliberately waiting prerequisite has the declared 420-second timeout, spanning the subsequent bounded 150+180-second stages with exact live ownership and authorized gate proof."); + && value.gateCalls.some(call => call.workerBound === true && call.workerId === value.held.id), "The prerequisite uses the same production native timeout policy; its brief read is bounded by the unchanged original observation clock, with exact live ownership and a worker-bound receipt."); return c.result(); } function canonical(value: unknown): string { if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`; if (value && typeof value === "object") return `{${Object.entries(value).filter(([, v]) => v !== undefined).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0).map(([k, v]) => `${JSON.stringify(k)}:${canonical(v)}`).join(",")}}`; return JSON.stringify(value) ?? "null"; } /** Public native history intentionally redacts API bodies but retains exact input digests, call identity and HTTP status. */ -export function projectReadCalls(events: unknown[], runId: string, projectId: string) { +export function projectReadCalls(events: unknown[], runId: string, projectId: string, expectedDescription?: string) { const digest = createHash("sha256").update(canonical({ operationId: "GET /api/projects/{id}", pathParams: { id: projectId } })).digest("hex"); const inputs = new Set(); const results = new Set(); + const listInputs = new Set(); const listResults = new Set(); // Codex emits these exact structured dynamic-tool events before/after the native // handler (codex-session-server-requests.ts). Never parse assistant prose as proof. for (const value of events) { @@ -39,12 +40,28 @@ export function projectReadCalls(events: unknown[], runId: string, projectId: st const input = item.input as Record | undefined; if (input?.apiPayloadRedacted === true && input.apiPayloadSha256 === digest) inputs.add(item.id); } + if (event.eventType === "item.started" && ["tool_use", "tool_call"].includes(String(item.type)) && item.name === "list_projects") listInputs.add(item.id); if (event.eventType === "item.completed" && item.type === "tool_result" && item.tool_use_id === item.id && item.is_error !== true) { const result = item.result as Record | undefined; if (result?.apiOperationId === "GET /api/projects/{id}" && result.status === 200 && result.ok === true) results.add(item.id); + // A dedicated read is equally valid when it delivered the actual full + // saved defaults. Summaries, stale/truncated text and assistant prose are + // insufficient; retain exact run, envelope and tool-call correlation. + if (expectedDescription && Array.isArray(result?.projects) && result.projects.some(project => { + if (!project || typeof project !== "object") return false; + const row = project as Record; + return row.id === projectId && row.description === expectedDescription && row.descriptionTruncated === false; + })) listResults.add(item.id); } } - return [...inputs].filter(callId => results.has(callId)).map(callId => ({ runId, projectId, callId })); + return [...new Set([...inputs].filter(callId => results.has(callId)).concat([...listInputs].filter(callId => listResults.has(callId))))] + .map(callId => ({ runId, projectId, callId })); +} +function changesGatedConfiguration(row: { action?: string; entityType?: unknown }) { + const action = row.action ?? ""; + // Normal execution acquires/releases leases without editing environment settings. + if (row.entityType === "environment_lease" && (action === "environment.lease_acquired" || action === "environment.lease_released")) return false; + return /budget\.|environment\./.test(action); } export function gradeProjectDefaults(value: unknown) { const c = checks(); const o = observation(value) as DefaultsObservation | undefined; @@ -69,43 +86,53 @@ export function gradeProjectDefaults(value: unknown) { && o.runs.some(row => row.id === o.state.activeRunId && row.status === "running" && row.agentId === active.assigneeAgentId) && JSON.stringify((p as Entity & { env?: unknown })?.env ?? null) === JSON.stringify(o.state.originalEnv) && !o.activity.some(row => row.agentId === worker.id && row.entityId === p?.id && row.action === "project.updated") - && !o.activity.some(row => row.agentId === o.leadId && /budget\.|environment\./.test(row.action ?? "")), "The existing checkout/task and separately governed environment/budget are not rewritten by future defaults; the probe cannot create its own defaults."); + && !o.activity.some(row => row.agentId === o.leadId && changesGatedConfiguration(row)), "The existing checkout/task and separately governed environment/budget are not rewritten by future defaults; the probe cannot create its own defaults."); const first = String(o.state.defaultsReply ?? ""); - c.add("future-defaults-distinguished-from-active-work", /\b(?:future|new|subsequent)\b/i.test(first) && /\b(?:existing|active|current|running)\b/i.test(first) + c.add("future-defaults-distinguished-from-active-work", /\b(?:future|new(?:ly)?|subsequent)\b/i.test(first) && /\b(?:existing|active|current|running)\b/i.test(first) && /\b(?:unchanged|not|does not|doesn't|won't|will not|only)\b/i.test(first), "The canonical reply distinguishes future task defaults from changes to already active work."); return c.result(); } function defaults(): ChatCatalogCase { - let gate: Awaited> | undefined; + let gate: Awaited> | undefined; const base = makeDefinition("projects.project-defaults", async ctx => { const worker = await fixtureWorker(ctx); const futureWorker = await fixtureWorker(ctx); await ctx.input.api.patch(`/api/agents/${futureWorker.id}`, { name: "Riley" }); - const original = await ctx.input.api.get(`/api/agents/${worker.id}`); - await ctx.input.api.patch(`/api/agents/${worker.id}`, { adapterConfig: { ...original.adapterConfig, timeoutSec: 420 } }); - await pollUntil({ label: "Declared long-wait prerequisite configuration ready", deadlineAt: Date.now() + 45000, - load: () => ctx.input.api.get(`/api/agents/${worker.id}`), accept: row => row.lifecycleState === "ready" && row.adapterConfig?.timeoutSec === 420 }); const p = await project(ctx); const g = await goal(ctx, "Growth: reach 1,000 signups"); const checklist = await task(ctx, "Accessibility checklist", p.id, { description: "All Launch work must support keyboard navigation, sufficient color contrast, and explicitly labeled controls." }); const root = `/api/companies/${ctx.input.fixtures.company.id}`; for (const scope of [{ scopeType: "company", scopeId: ctx.input.fixtures.company.id }, ...[worker, futureWorker].map(row => ({ scopeType: "agent", scopeId: row.id }))]) await ctx.input.api.post(`${root}/budgets/policies`, { ...scope, amount: 500, hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" }); - gate = await startRunBriefGate(ctx, worker.id); - const t = await task(ctx, "Active launch research", p.id, { status: "todo", assigneeAgentId: worker.id, description: "Read the authorized Website Brief app's read_website_brief before proceeding; wait for the actual result. Do not change this brief or other tasks." }); - const held = await pollUntil({ label: "Real prior active Launch checkout", deadlineAt: Date.now() + 150000, - load: async () => ({ task: await ctx.input.api.get(`/api/issues/${t.id}`), runs: await ctx.allRuns() }), - accept: row => gate!.calls.length > 0 && row.runs.length === 1 && row.runs[0]?.status === "running" && row.task.executionRunId === row.runs[0]?.id }); - await ctx.input.evidence("chat-catalog-project-defaults-prerequisite-timeout.json", { heldWorkerId: worker.id, heldWorkerTimeoutSec: 420, canonicalTimeoutSec: CHAT_NATIVE_TIMEOUT_POLICY.savedAgentTimeoutSec, futureWorkerTimeoutSec: CHAT_NATIVE_TIMEOUT_POLICY.savedAgentTimeoutSec, canonicalStageBoundMs: 150000, futureProbeBoundMs: 180000, attemptBoundMs: 660000 }); - return { project: p, goal: g, checklist, task: held.task, worker, futureWorker, activeRunId: held.runs[0]!.id, originalEnv: (p as Entity & { env?: unknown }).env ?? null }; + const holdDeadlineAt = ctx.observationDeadlineAt!; + gate = await startNativeRunBriefGate(worker.id, holdDeadlineAt, ctx.input.privateSecrets); + try { + const t = await task(ctx, "Active launch research", p.id, { status: "todo", assigneeAgentId: worker.id, description: `Read the pending Website Brief by running this native shell command with a ${Math.max(1, holdDeadlineAt - Date.now())} ms tool timeout. Wait for its actual result before proceeding. Do not change this brief or other tasks. Command: ${gate.command}` }); + const held = await pollUntil({ label: "Real prior active Launch checkout", deadlineAt: Date.now() + 150000, + load: async () => ({ task: await ctx.input.api.get(`/api/issues/${t.id}`), runs: await ctx.allRuns() }), + accept: row => gate!.calls.length > 0 && row.runs.length === 1 && row.runs[0]?.status === "running" && row.task.executionRunId === row.runs[0]?.id }); + await ctx.input.evidence("chat-catalog-project-defaults-prerequisite-timeout.json", { heldWorkerId: worker.id, heldWorkerTimeoutSec: CHAT_NATIVE_TIMEOUT_POLICY.savedAgentTimeoutSec, holdDeadlineAt, remainingHoldBudgetMs: Math.max(0, holdDeadlineAt - Date.now()), canonicalTimeoutSec: CHAT_NATIVE_TIMEOUT_POLICY.savedAgentTimeoutSec, futureWorkerTimeoutSec: CHAT_NATIVE_TIMEOUT_POLICY.savedAgentTimeoutSec, authoredLegacyCanonicalStageMs: 150000, authoredLegacyFutureProbeStageMs: 180000, attemptBoundMs: 660000 }); + return { project: p, goal: g, checklist, task: held.task, worker, futureWorker, activeRunId: held.runs[0]!.id, originalEnv: (p as Entity & { env?: unknown }).env ?? null }; + } catch (error) { + // The flow has no state yet, so its ordinary case cleanup cannot run. + try { await gate.close(); } + catch (cleanupError) { throw new AggregateError([error, cleanupError], "Native brief setup and cleanup failed"); } + finally { gate = undefined; } + throw error; + } }, gradeProjectDefaults); - return { ...base, expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 660000, runPolicy: { kind: "source-settled-with-active-work", maximumActiveWorkerRuns: 1 }, + return { ...base, oracleVersion: "paperclip.chat-catalog.project-defaults.v4", expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 660000, runPolicy: { kind: "source-settled-with-active-work", maximumActiveWorkerRuns: 1 }, setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["lead-ready", "future-worker-ready", "existing-work-actively-owned"] }, async verifySetup(ctx, state) { const [lead, future, held, t, runs] = await Promise.all([ ctx.input.api.get(`/api/agents/${ctx.input.fixtures.agent.id}`), ctx.input.api.get(`/api/agents/${entity(state, "futureWorker").id}`), ctx.input.api.get(`/api/agents/${entity(state, "worker").id}`), ctx.input.api.get(`/api/issues/${entity(state, "task").id}`), ctx.allRuns() ]); return gradeProjectDefaultsSetup({ lead, future, held, task: t, runs, activeRunId: String(state.activeRunId), gateCalls: gate?.calls ?? [] }); }, - sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-project-defaults.ts", "chat-catalog-domains/internal-run-brief.ts", "../e2e/fetch-allowed-port.ts", "run-observations.ts"], + sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-project-defaults.ts", "chat-catalog-domains/internal-native-run-brief.ts", "../e2e/fetch-allowed-port.ts", "run-observations.ts"], async act(ctx, state) { - const settled = await pollUntil({ label: "Canonical project-defaults turn settles", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150000), load: () => observeInternal(ctx, state), - accept: o => chatCatalogNativePhaseIsTimely(ctx) && o.runs.some(row => row.agentId === o.leadId && row.status === "succeeded") && ctx.issue()?.conversationState === "waiting" }); + // This boundary depends only on the canonical run and conversation. + // Rebuilding domain/DOM evidence on each wait poll adds unrelated load + // and trace volume; retain the complete observation after settlement. + await pollUntil({ label: "Canonical project-defaults turn settles", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 150000), + load: async () => { await ctx.refreshIssue(); return ctx.allRuns(); }, + accept: runs => chatCatalogNativePhaseIsTimely(ctx) && runs.some(row => row.agentId === ctx.input.fixtures.agent.id && row.status === "succeeded") && ctx.issue()?.conversationState === "waiting" }); + const settled = await observeInternal(ctx, state); state.defaultsReply = settled.reply; state.futureTask = await task(ctx, "Verify inherited Launch defaults", entity(state, "project").id, { status: "todo", assigneeAgentId: entity(state, "futureWorker").id, description: "Use the current saved project/task context to report which project checklist applies to this new work and the associated goal. Do not change project defaults or any other task. Complete after recording your factual findings, including if no defaults or goal are present." }); @@ -113,7 +140,7 @@ function defaults(): ChatCatalogCase { }, async observe(ctx, state): Promise { const o = await observeInternal(ctx, state); const run = state.futureTask ? o.runs.find(row => row.agentId === entity(state, "futureWorker").id && (row.contextSnapshot?.issueId === entity(state, "futureTask").id || row.contextSnapshot?.taskId === entity(state, "futureTask").id)) : undefined; const events = run ? await collectRunEvents((after, limit) => ctx.input.api.get(`/api/heartbeat-runs/${run.id}/events?afterSeq=${after}&limit=${limit}`)) : []; - return { ...o, futureProjectReads: run ? projectReadCalls(events, run.id, entity(state, "project").id) : [], + return { ...o, futureProjectReads: run ? projectReadCalls(events, run.id, entity(state, "project").id, o.projects.find(project => project.id === entity(state, "project").id)?.description ?? undefined) : [], futureContext: state.futureTask ? await ctx.input.api.get(`/api/issues/${entity(state, "futureTask").id}/heartbeat-context`) : {}, futureComments: state.futureTask ? await ctx.input.api.get(`/api/issues/${entity(state, "futureTask").id}/comments?order=asc`) : [] }; }, async verifyResult(ctx, state) { const source = ctx.input.page.url(); try { await ctx.input.page.goto(new URL(`/${ctx.input.fixtures.company.issuePrefix}/projects/${entity(state, "project").id}`, ctx.input.api.baseURL).href); await expect(ctx.input.page.getByRole("heading", { name: "Launch", exact: true }).first()).toBeVisible(); } finally { await ctx.input.page.goto(source); await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); } }, diff --git a/tests/runner-e2e/chat-catalog-domains/internal-project-read-producer.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-project-read-producer.test.ts new file mode 100644 index 00000000000..ad6dd368a28 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/internal-project-read-producer.test.ts @@ -0,0 +1,43 @@ +import { chmod, copyFile, mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it } from "vitest"; +import { OpenCodeServerDriver } from "../../../packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.js"; +import { projectReadCalls } from "./internal-project-defaults.js"; + +it("accepts project reads emitted by the actual OpenCode MCP event producer", async () => { + const root = await mkdtemp(join(tmpdir(), "catalog-project-read-producer-")); + const command = join(root, "fake-opencode.mjs"); + await copyFile(new URL("../../../packages/paperclip-runner/test/fixtures/fake-opencode-server.mjs", import.meta.url), command); + await chmod(command, 0o755); + const description = "Keyboard navigation; adequate contrast; labeled controls."; + const driver = new OpenCodeServerDriver({ + model: "openrouter/controlled-model", runtimeDirectory: join(root, "runtime"), command, + environment: { PATH: process.env.PATH, OPENROUTER_API_KEY: "controlled-fixture-key" }, + dynamicTools: [{ name: "list_projects", inputSchema: { type: "object", properties: {} } }], + dynamicToolHandler: async call => { + expect(call.tool).toBe("list_projects"); + return { projects: [{ id: "project", description, descriptionTruncated: false }], nextCursor: null }; + }, + }); + const session = await driver.openSession({ runId: "owned-read-run", normalizedSessionId: "owned-read-session", workingDirectory: root }); + try { + await session.startTurn({ message: { role: "user", text: "Read project defaults." } }); + const events = []; + for await (const event of session.events()) { + events.push(event); + if (["turn.completed", "turn.failed"].includes(event.eventType)) break; + } + expect(events.at(-1)?.eventType).toBe("turn.completed"); + expect(events.some(event => { + const item = event.payload.item as { type?: string; name?: string } | undefined; + return item?.type === "tool_call" && item.name === "list_projects"; + })).toBe(true); + expect(projectReadCalls(events, "owned-read-run", "project", description)).toHaveLength(1); + expect(projectReadCalls(events, "foreign-run", "project", description)).toEqual([]); + expect(projectReadCalls(events, "owned-read-run", "project", "stale defaults")).toEqual([]); + } finally { + await session.close({ reason: "project read producer check complete" }); + await rm(root, { recursive: true, force: true }); + } +}, 30_000); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-remaining.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-remaining.test.ts index 607a74c0639..8ff80e9d551 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-remaining.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-remaining.test.ts @@ -7,7 +7,7 @@ import { gradeArchivePartial } from "./internal-archive-partial.js"; function base(scenarioId: string) { return { scenarioId, origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: {} as Record, tasks: [] as any[], projects: [] as any[], goals: [] as any[], comments: [] as any[], activity: [] as any[], documents: [], revisions: [], links: [] as string[], reply: "", humanQueries: [], runs: [] as any[] }; } it("preserves the exact active prerequisite while independently proving ready canonical and future agents", () => { const ready = { id: "ceo", lifecycleState: "ready", status: "idle", adapterConfig: { timeoutSec: 0 } }; - const prepared = { lead: ready, future: { ...ready, id: "riley" }, held: { ...ready, id: "morgan", status: "running", adapterConfig: { timeoutSec: 420 } }, task: { id: "task", assigneeAgentId: "morgan", executionRunId: "held" }, runs: [{ id: "held", agentId: "morgan", status: "running" }], activeRunId: "held", gateCalls: [{ authorized: true }] }; + const prepared = { lead: ready, future: { ...ready, id: "riley" }, held: { ...ready, id: "morgan", status: "running", adapterConfig: { timeoutSec: 0 } }, task: { id: "task", assigneeAgentId: "morgan", executionRunId: "held" }, runs: [{ id: "held", agentId: "morgan", status: "running" }], activeRunId: "held", gateCalls: [{ workerBound: true, workerId: "morgan" }] }; expect(gradeProjectDefaultsSetup(prepared).passed).toBe(true); expect(gradeProjectDefaultsSetup({ ...prepared, lead: { ...prepared.lead, adapterConfig: { timeoutSec: 120 } } }).passed).toBe(false); expect(gradeProjectDefaultsSetup({ ...prepared, future: { ...prepared.future, adapterConfig: { timeoutSec: 120 } } }).passed).toBe(false); @@ -15,6 +15,7 @@ it("preserves the exact active prerequisite while independently proving ready ca expect(gradeProjectDefaultsSetup({ ...prepared, future: { ...prepared.future, status: "error" } }).passed).toBe(false); expect(gradeProjectDefaultsSetup({ ...prepared, held: { ...prepared.held, adapterConfig: { timeoutSec: 120 } } }).passed).toBe(false); expect(gradeProjectDefaultsSetup({ ...prepared, gateCalls: [] }).passed).toBe(false); + expect(gradeProjectDefaultsSetup({ ...prepared, gateCalls: [{ workerBound: true, workerId: "unrelated" }] }).passed).toBe(false); }); function defaults() { const o = base("projects.project-defaults"); const task = { id: "active", status: "in_progress", projectId: "p", assigneeAgentId: "morgan", executionRunId: "prior" }; const future = { id: "future", status: "done", projectId: "p" }; const p = { id: "p", description: "Keyboard navigation; adequate contrast; labeled controls", goalIds: ["growth"], env: null }; @@ -24,9 +25,30 @@ function defaults() { } it("accepts genuine saved defaults reaching a new native task while active work stays fixed", () => expect(gradeProjectDefaults(defaults()).passed).toBe(true)); it("accepts natural future-default wording", () => { const o = defaults(); o.state.defaultsReply = "These defaults apply only to subsequent Launch tasks; your running work is not changed."; expect(gradeProjectDefaults(o).passed).toBe(true); }); +it("accepts newly admitted work while still requiring the existing-work distinction", () => { + const o = defaults(); o.state.defaultsReply = "Newly admitted work inherits these defaults. Existing items were left unchanged."; + expect(gradeProjectDefaults(o).passed).toBe(true); + o.state.defaultsReply = "Newly admitted work inherits these defaults."; + expect(gradeProjectDefaults(o).passed).toBe(false); + o.state.defaultsReply = "Existing items were left unchanged."; + expect(gradeProjectDefaults(o).passed).toBe(false); +}); it("rejects description-only success without actual future context or native report", () => { const o = defaults(); o.futureContext.project.description = ""; expect(gradeProjectDefaults(o).passed).toBe(false); const wrong = defaults(); wrong.futureComments[0]!.createdByRunId = "prior"; expect(gradeProjectDefaults(wrong).passed).toBe(false); }); it("rejects silently changing active ownership or gated environment", () => { const o = defaults(); o.tasks[0].assigneeAgentId = "ceo"; expect(gradeProjectDefaults(o).passed).toBe(false); const wrong = defaults(); wrong.projects[0].env = { TOKEN: "changed" }; expect(gradeProjectDefaults(wrong).passed).toBe(false); }); it("rejects wrong goal and an unacknowledged future/active distinction", () => { const o = defaults(); o.projects[0].goalIds = ["wrong"]; expect(gradeProjectDefaults(o).passed).toBe(false); const wrong = defaults(); wrong.state.defaultsReply = "All work updated."; expect(gradeProjectDefaults(wrong).passed).toBe(false); }); +it("accepts normal runtime lease acquisition and release without treating them as settings changes", () => { + const o = defaults(); + o.activity.push(...["environment.lease_acquired", "environment.lease_released"].map(action => ({ action, entityType: "environment_lease", agentId: "ceo" }))); + expect(gradeProjectDefaults(o).passed).toBe(true); +}); +it.each([ + ["environment.created", "environment"], ["environment.updated", "environment"], ["environment.deleted", "environment"], + ["budget.policy_upserted", "budget_policy"], ["environment.lease_acquired", "environment"], + ["environment.lease_released", "environment"], ["environment.lease_unknown", "environment_lease"], +])("rejects configuration changes and unrecognized lease activity: %s / %s", (action, entityType) => { + const o = defaults(); o.activity.push({ action, entityType, agentId: "ceo" }); + expect(gradeProjectDefaults(o).passed).toBe(false); +}); it("accepts consumed supported same-project API reads without requiring a particular context DTO field", () => { const o = defaults(); o.futureContext.project.description = ""; const digest = createHash("sha256").update(JSON.stringify({ operationId: "GET /api/projects/{id}", pathParams: { id: "p" } })).digest("hex"); @@ -41,6 +63,26 @@ it("accepts consumed supported same-project API reads without requiring a partic expect(projectReadCalls([{ runId: "new", eventType: "item.completed", payload: { kind: "agentMessage", text: JSON.stringify(events) } }], "new", "p")).toEqual([]); }); it("accepts a persisted checklist reference only when the new native context actually contains it", () => { const o = defaults(); Object.assign(o.state, { checklist: { id: "checklist", title: "Accessibility checklist" } }); o.projects[0].description = "Future work must follow Accessibility checklist."; o.futureContext.project.description = o.projects[0].description; expect(gradeProjectDefaults(o).passed).toBe(true); o.futureContext.project.description = ""; expect(gradeProjectDefaults(o).passed).toBe(false); }); +it("accepts a correlated dedicated project read only with the exact saved defaults", () => { + const o = defaults(); o.futureContext.project.description = ""; + const description = o.projects[0].description; + const started = { runId: "new", eventType: "item.started", itemId: "read-p", payload: { kind: "dynamicToolCall", item: { type: "tool_use", id: "read-p", name: "list_projects", input: {} } } }; + const completed = { runId: "new", eventType: "item.completed", itemId: "read-p", payload: { kind: "dynamicToolCall", item: { type: "tool_result", id: "read-p", tool_use_id: "read-p", is_error: false, result: { projects: [{ id: "p", description, descriptionTruncated: false }], nextCursor: null } } } }; + const read = (events: unknown[]) => projectReadCalls(events, "new", "p", description); + expect(read([started, completed])).toEqual([{ runId: "new", projectId: "p", callId: "read-p" }]); + Object.assign(o, { futureProjectReads: read([started, completed]) }); expect(gradeProjectDefaults(o).passed).toBe(true); + for (const mutate of [ + (row: typeof completed) => { row.runId = "foreign"; }, + (row: typeof completed) => { row.itemId = "foreign"; }, + (row: typeof completed) => { row.payload.item.tool_use_id = "foreign"; }, + (row: typeof completed) => { row.payload.item.is_error = true; }, + (row: typeof completed) => { row.payload.item.result.projects[0]!.id = "foreign"; }, + (row: typeof completed) => { row.payload.item.result.projects[0]!.description = "stale defaults"; }, + (row: typeof completed) => { row.payload.item.result.projects[0]!.descriptionTruncated = true; }, + ]) { const wrong = structuredClone(completed); mutate(wrong); expect(read([started, wrong])).toEqual([]); } + expect(read([completed])).toEqual([]); + expect(projectReadCalls([started, completed], "new", "p")).toEqual([]); +}); function routine() { const o = base("activity.routine-audit"); const r = { id: "routine", companyId: "company", title: "Daily launch", description: "Launch audit", updatedAt: "2026-10-25T14:00:00Z" }; o.state.routine = structuredClone(r); o.activity = [{ id: "edit", entityId: r.id, action: "routine.updated", createdAt: r.updatedAt, actorType: "user" }]; o.links = ["/RUN/routines/routine"]; o.reply = "The definition was edited yesterday; there is no successful execution this morning and no resulting work. The enabled schedule alone is not proof of a run."; return { ...o, routine: r, routineRuns: [] as any[], routineRevisions: [{ id: "rev" }], now: "2026-10-26T15:00:00Z", timezone: "UTC" }; } it("accepts real prior-day definition history with absent morning execution", () => expect(gradeRoutineAudit(routine()).passed).toBe(true)); it("accepts a natural evidence-grounded missing-run explanation", () => { const o = routine(); o.reply = "Yesterday's schedule edit is saved, but I found no completed task or successful execution this morning."; expect(gradeRoutineAudit(o).passed).toBe(true); }); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-worker-readiness.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-worker-readiness.test.ts new file mode 100644 index 00000000000..0d6a39273f8 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/internal-worker-readiness.test.ts @@ -0,0 +1,85 @@ +import { afterEach, expect, it, vi } from "vitest"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { fixtureWorker } from "./internal-definition.js"; +import { agentRecordQueries } from "../../../server/src/lib/agent-records.js"; +import { lifecycleFailureMessage } from "../../../server/src/modules/agent-lifecycle/application/failure.js"; + +afterEach(() => vi.useRealTimers()); +const selected = "chat-features.runner-opencode.local.projects.project-defaults"; +const { normalizeAgentRows } = agentRecordQueries(undefined as unknown as Parameters[0]); +const recoveryError = lifecycleFailureMessage({ stage: "host", phase: "verifying" }, { code: "harness_probe_timed_out" }); +const repairError = lifecycleFailureMessage({ stage: "host", phase: "verifying" }, { code: "harness_auth_failed" }); +function fixture(state: (now: number) => Record, executionId = selected, deadlineAt = 120_000) { + const worker = { id: "worker", companyId: "company", name: "Morgan" }; + const get = vi.fn(async () => normalizeAgentRows([{ + ...worker, status: "paused", reportsTo: null, permissions: {}, appearance: null, ...state(Date.now()), + } as unknown as Parameters[0][number]])[0]); + const evidence = vi.fn(async () => {}); + const ctx = { observationDeadlineAt: deadlineAt, input: { + execution: { id: executionId, environment: { id: "local" }, profile: { buildAgent: () => ({}), credential: "OPENROUTER_API_KEY" } }, + fixtures: { company: { id: "company" }, agent: { id: "lead" }, environment: { id: "environment" }, secretRefs: {} }, + workspacePath: "/fixture", nonce: "fixture", api: { get, post: async () => worker }, evidence, + } } as unknown as ChatCatalogContext; + return { ctx, get, evidence, worker }; +} +function outcome(ctx: ChatCatalogContext) { + return fixtureWorker(ctx).then(value => ({ value, error: undefined, settledAt: Date.now() }), + error => ({ value: undefined, error, settledAt: Date.now() })); +} + +it("allows a valid readiness result after the unrelated 45-second cap within the original case clock", async () => { + vi.useFakeTimers({ now: 0 }); + const f = fixture(now => ({ lifecycleState: now >= 50_000 ? "ready" : "verifying" })); + const result = outcome(f.ctx); + await vi.advanceTimersByTimeAsync(50_000); + expect(await result).toEqual({ value: f.worker, error: undefined, settledAt: 50_000 }); +}); + +it("waits for the lifecycle owner's automatic verification recovery", async () => { + vi.useFakeTimers({ now: 0 }); + const f = fixture(now => now >= 50_000 ? { lifecycleState: "ready", lifecycleError: null } + : { lifecycleState: "verifying", lifecycleError: recoveryError, + lifecycleOperation: { id: "verification", attempts: 1, retryRequired: false, retryAt: new Date(20_000).toISOString() } }); + const result = outcome(f.ctx); + await vi.advanceTimersByTimeAsync(50_000); + expect(await result).toEqual({ value: f.worker, error: undefined, settledAt: 50_000 }); + for (const row of await Promise.all(f.get.mock.results.map(result => result.value))) { + expect(row).not.toHaveProperty("lifecycleOperation"); + } +}); + +it.each(["Authentication failed.", "Unknown verification failure.", + `${recoveryError} Automatic verification is paused. Update Configuration or choose Retry now.`])( + "rejects an error without the public automatic recovery promise: %s", async lifecycleError => { + vi.useFakeTimers({ now: 0 }); + const f = fixture(() => ({ lifecycleState: "verifying", lifecycleError })); + await expect(fixtureWorker(f.ctx)).rejects.toThrow(/setup/i); + expect(f.get).toHaveBeenCalledOnce(); + }); + +it.each(["verifying", "ready"])("rejects %s verification that requires configuration repair", async lifecycleState => { + vi.useFakeTimers({ now: 0 }); + const f = fixture(() => ({ lifecycleState, lifecycleError: repairError, lifecycleOperation: { id: "verification", retryRequired: true } })); + await expect(fixtureWorker(f.ctx)).rejects.toThrow(/setup|verification/i); + expect(f.get).toHaveBeenCalledOnce(); +}); + +it("cannot extend the original case clock while automatic recovery remains pending", async () => { + vi.useFakeTimers({ now: 0 }); + const f = fixture(() => ({ lifecycleState: "verifying", lifecycleError: recoveryError, lifecycleOperation: { id: "verification", retryRequired: false } }), selected, 2_000); + let settledAt: number | undefined; + const result = outcome(f.ctx).then(value => { settledAt = value.settledAt; return value; }); + await vi.advanceTimersByTimeAsync(2_000); + const atDeadline = settledAt; + await vi.advanceTimersByTimeAsync(45_000); + expect((await result).error).toBeInstanceOf(Error); + expect(atDeadline).toBe(2_000); +}); + +it("retains the authored setup cap in lanes that have not opted into the original clock", async () => { + vi.useFakeTimers({ now: 0 }); + const f = fixture(now => ({ lifecycleState: now >= 50_000 ? "ready" : "verifying" }), "chat-features.runner-codex-subscription.local.delegate-announcement"); + const result = outcome(f.ctx); + await vi.advanceTimersByTimeAsync(120_000); + expect(await result).toMatchObject({ error: expect.any(Error), settledAt: 46_000 }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-instance.ts b/tests/runner-e2e/chat-catalog-domains/privileged-instance.ts index 1cbf8a9c359..a3e91b43089 100644 --- a/tests/runner-e2e/chat-catalog-domains/privileged-instance.ts +++ b/tests/runner-e2e/chat-catalog-domains/privileged-instance.ts @@ -38,6 +38,11 @@ export const privilegedInstanceCases = [privilegedDefinition("instance-operation const initial = await settlePrivilegedChat(ctx), source = ctx.issue()!; const history = await ctx.input.api.get(`/api/issues/${source.id}/comments?order=asc`); const url = ctx.input.page.url(); + const reply = history.filter(row => row.authorAgentId === ctx.input.fixtures.agent.id).at(-1)!; + const replyBody = () => ctx.input.page.locator(`[id=${JSON.stringify(`comment-${reply.id}`)}]`).locator(".paperclip-markdown").first(); + await expect(replyBody()).toBeVisible(); + const renderedReplyBefore = await replyBody().innerText(); + if (!renderedReplyBefore.trim()) throw new Error("Pre-restart reply has no visible rendered body"); state.sourceBefore = source.id; state.runBefore = initial.runs[0]!.id; state.expectedRequestId = `chat-${ctx.input.nonce}`; state.restartStartedAt = Date.now(); const logPath = process.env.PAPERCLIP_RUNNER_E2E_SERVER_LOG!; @@ -57,8 +62,8 @@ export const privilegedInstanceCases = [privilegedDefinition("instance-operation await ctx.refreshIssue(); state.sourceAfter = ctx.issue()?.id; const after = await ctx.input.api.get(`/api/issues/${source.id}/comments?order=asc`); state.persistedHistoryRetained = after.length === history.length && history.every(row => after.some(saved => saved.id === row.id && saved.body === row.body && saved.createdByRunId === row.createdByRunId)); - const reply = history.filter(row => row.authorAgentId === ctx.input.fixtures.agent.id).at(-1)!; - await expect(ctx.input.page.locator(`[id=${JSON.stringify(`comment-${reply.id}`)}]`)).toContainText(reply.body!); + await expect(replyBody()).toBeVisible(); + await expect(replyBody()).toHaveText(renderedReplyBefore, { useInnerText: true }); state.browserHistoryVisible = true; await ctx.input.evidence("chat-catalog-owned-service-restart.json", { ...state, retainedCommentIds: after.map(row => row.id) }); }, grade: gradeRestartService, diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-restart-history.test.ts b/tests/runner-e2e/chat-catalog-domains/privileged-restart-history.test.ts new file mode 100644 index 00000000000..927ed3589cf --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/privileged-restart-history.test.ts @@ -0,0 +1,49 @@ +import { afterEach, expect, it, vi } from "vitest"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +const fixture = vi.hoisted(() => ({ log: "", receipt: {} })); +vi.mock("node:fs/promises", async original => ({ ...await original(), readFile: async (file: string) => file.endsWith("server.log") ? fixture.log : JSON.stringify(fixture.receipt) })); +vi.mock("./privileged-support.js", async original => ({ ...await original(), settlePrivilegedChat: async () => ({ runs: [{ id: "run" }] }) })); +vi.mock("@playwright/test", () => ({ expect: (node: { text: string; visible: boolean }) => ({ + toBeVisible: async () => expect(node.visible).toBe(true), + toContainText: async (text: string) => expect(node.text).toContain(text), + toHaveText: async (text: string, options: unknown) => { expect(options).toEqual({ useInnerText: true }); expect(node.text).toBe(text); }, +}) })); +import { privilegedInstanceCases, gradeRestartService } from "./privileged-instance.js"; +afterEach(() => vi.unstubAllEnvs()); +const rendered = "Restart the service.\n\nHistory stays intact."; +async function run(afterText = rendered, beforeText = rendered, afterVisible = true, changedStoredBody = false) { + vi.stubEnv("PAPERCLIP_RUNNER_E2E_TEMP_ROOT", "/tmp/owned-restart-history"); + vi.stubEnv("PAPERCLIP_RUNNER_E2E_SERVER_LOG", "/tmp/owned-restart-history/server.log"); + fixture.log = ""; fixture.receipt = {}; + let restarted = false; + const body = { text: beforeText, visible: true, innerText: vi.fn(async () => { expect(restarted).toBe(false); return body.text; }), first: () => body }; + const comment = { id: "reply", authorAgentId: "lead", body: "**Restart** the service.\n\nHistory stays `intact`.", createdByRunId: "run" }; + const ctx = { input: { nonce: "nonce", fixtures: { agent: { id: "lead" } }, + api: { get: async () => [{ ...comment, ...(restarted && changedStoredBody ? { body: "Changed persisted reply" } : {}) }] }, + page: { url: () => "https://fixture.test/RUN/chats/lead", goto: async () => {}, getByTestId: () => ({ visible: true }), + locator: (selector: string) => { expect(selector).toBe('[id="comment-reply"]'); return { get text() { return "CEO" + body.text + "10:00 AM"; }, visible: true, locator: (nested: string) => { expect(nested).toBe(".paperclip-markdown"); return body; } }; } }, + restart: async () => { restarted = true; body.text = afterText; body.visible = afterVisible; + fixture.log = "Restart request chat-nonce: stopping Paperclip\nRestart request chat-nonce: starting Paperclip\nRestart request chat-nonce: Paperclip is healthy"; + fixture.receipt = { requestId: "chat-nonce", status: "ready", completedAt: new Date().toISOString() }; }, evidence: vi.fn(), + }, issue: () => ({ id: "source" }), refreshIssue: async () => {} } as unknown as ChatCatalogContext; + const state: Record = {}; + await privilegedInstanceCases[0]!.act!(ctx, state); + expect(body.innerText).toHaveBeenCalledTimes(1); + return state; +} +it("checks the same attributed rendered body across restart while preserving raw Markdown separately", async () => { + expect(await run()).toMatchObject({ persistedHistoryRetained: true, browserHistoryVisible: true, stopStartLogVerified: true }); +}); +it.each(["Changed visible reply", ""])("rejects changed or missing rendered history: %j", async text => { + await expect(run(text)).rejects.toThrow(); +}); +it("requires a visible nonempty body before restart and a visible body afterward", async () => { + await expect(run(rendered, "")).rejects.toThrow(/no visible/); + await expect(run(rendered, rendered, false)).rejects.toThrow(); +}); +it("keeps independently changed persisted bytes failing despite unchanged rendered text", async () => { + const state = await run(rendered, rendered, true, true); + expect(state).toMatchObject({ persistedHistoryRetained: false, browserHistoryVisible: true }); + expect(gradeRestartService({ state, origin: "https://fixture.test", companyId: "company", leadId: "lead", reply: "The operator should restart the server.", + tasks: [], projects: [], goals: [], comments: [], activity: [], documents: [], revisions: [], links: [], humanQueries: [], runs: [{ id: "run", status: "succeeded" }] }).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/verify-native-brief-clock.ts b/tests/runner-e2e/chat-catalog-domains/verify-native-brief-clock.ts new file mode 100644 index 00000000000..fbb3a3d8659 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/verify-native-brief-clock.ts @@ -0,0 +1,29 @@ +/** Explicit slow control: node --import ./cli/node_modules/tsx/dist/loader.mjs tests/runner-e2e/chat-catalog-domains/verify-native-brief-clock.ts --five-minute-control */ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { once } from "node:events"; +import { setTimeout as delay } from "node:timers/promises"; +import { startNativeRunBriefGate } from "./internal-native-run-brief.js"; + +if (!process.argv.includes("--five-minute-control")) throw new Error("Use --five-minute-control to run the 310-second native lifetime check"); +const started = Date.now(), holdMs = 310_000, deadlineMs = 340_000; +const gate = await startNativeRunBriefGate("10000000-0000-4000-8000-000000000001", started + deadlineMs); +const child = spawn("sh", ["-c", gate.command], { stdio: ["ignore", "pipe", "pipe"] }); +const exited = once(child, "exit"); +let output = ""; child.stdout.on("data", chunk => { output += chunk; }); child.stderr.resume(); +try { + while (!gate.calls.length && Date.now() - started < 5000 && child.exitCode === null) await delay(20); + assert.equal(gate.calls.length, 1, "The actual native worker must reach the owned brief gate"); + console.log(JSON.stringify({ stage: "native-read-held", node: process.version, holdMs, deadlineMs })); + await delay(holdMs); + assert.equal(child.exitCode, null, "The native read must survive the five-minute header timeout"); + assert.equal(output, "", "The pending brief must not produce a premature result"); + gate.release(); + assert.equal((await exited)[0], 0); + assert.ok(output.includes("The website brief is now available")); + console.log(JSON.stringify({ status: "passed", elapsedMs: Date.now() - started, holdMs, deadlineMs })); +} finally { + child.kill("SIGTERM"); + await gate.close(); + await exited; +} diff --git a/tests/runner-e2e/chat-catalog-flow.ts b/tests/runner-e2e/chat-catalog-flow.ts index 84b50020f56..5787989efc6 100644 --- a/tests/runner-e2e/chat-catalog-flow.ts +++ b/tests/runner-e2e/chat-catalog-flow.ts @@ -180,7 +180,7 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) { expect(persisted.grade.passed, "Independent oracle survives browser reload").toBe(true); await implementation.verifyResult(context, state, observation); for (const check of persisted.grade.checks) input.check?.(check.id, check.passed, check.detail); - await input.capture("chat-catalog-result", scenario.id, "chat-catalog-result.png"); + await input.capture("final-state", scenario.id, "final-state.png"); expect(chatCatalogNativePhaseIsTimely(context), "Original selected native observation budget includes complete final proof").toBe(true); return createChatCatalogFinalRunVerifier({ implementation, companyId: input.fixtures.company.id, leadId: input.fixtures.agent.id, observation }); @@ -189,10 +189,13 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) { throw error; } finally { try { - await conversation(); + const finalConversation = await conversation(); if (state) await refresh(); const runs = await context.allRuns(); - await input.evidence("chat-catalog-runs.json", await Promise.all(runs.map(run => collectChatRunEvidence(chatRunEvidenceApi(input), run)))); + const runEvidence = await Promise.all(runs.map(run => collectChatRunEvidence(chatRunEvidenceApi(input), run))); + await input.evidence("chat-catalog-runs.json", runEvidence); + await input.evidence("api-state.json", { capturePhase: failed ? "failure" : "final", + ...finalConversation, runs, runEvidence }); } catch { await input.evidence("chat-catalog-evidence-error.json", { incomplete: true, scenarioId: scenario.id }); if (!failed) throw new Error("Scenario evidence collection failed"); diff --git a/tests/runner-e2e/launch.ts b/tests/runner-e2e/launch.ts index 9cd0a80339e..39a8bf7a211 100644 --- a/tests/runner-e2e/launch.ts +++ b/tests/runner-e2e/launch.ts @@ -1,3 +1,4 @@ +import { appendPrivateAttemptDiagnostic, readPrivateAttemptObservation, type PrivateAttemptDiagnosticInput } from "./private-attempt-diagnostic.js"; import { withholdPrivateScreenshots } from "./screenshot-privacy.js"; import { copyHealthHelloDetail } from "./chat-readiness-detail.js"; import { copyRetainedSourceSdk } from "./source-sdk-evidence.js"; @@ -112,6 +113,7 @@ const localEnvPath = path.join(repositoryRoot, ".env.runner-e2e.local"); let resultsRoot = path.join(repositoryRoot, "tests/runner-e2e/results"); const manualEvidenceSecrets = new Set(); let manualPrivateHandoff: ManualPrivateHandoff | undefined; +let manualDiagnostic: PrivateAttemptDiagnosticInput | undefined; const activeProcessGroups = new Set(); const activeProcessCleanup = new Map>(); const activeProcessTerminators = new Map void>(); @@ -616,7 +618,7 @@ async function runAttempt(input: { PAPERCLIP_RUNNER_E2E_TEMP_ROOT: temporaryRoot, PAPERCLIP_RUNNER_E2E_PRIVATE_DIR: privateDir, PAPERCLIP_RUNNER_E2E_WORKSPACE: workspace, - PAPERCLIP_RUNNER_E2E_SERVER_LOG: path.join(privateDir, "server.log"), + PAPERCLIP_RUNNER_E2E_SERVER_LOG: path.join(temporaryRoot, "artifacts-private", "server.log"), PAPERCLIP_RUNNER_BINARY: runnerBinary, PAPERCLIP_RUNNER_REMOTE_BINARY_PATH: installedCli ? undefined : resolvePaperclipRemoteRunnerBinaryForHarness(executions, runnerBinary), @@ -793,7 +795,12 @@ async function runAttempt(input: { persistedStateError = error; } } - const reporterPrivacy = await sanitizePrivateReporterArtifacts(privateDir, credentials); + const ownedServerLog = path.join(temporaryRoot, "artifacts-private", "server.log"); + if (manualPrivateHandoff && await access(ownedServerLog).then(() => true, () => false)) { + await cp(ownedServerLog, path.join(privateDir, "server.log"), { errorOnExist: true, force: false }); + } + const reporterPrivacy = await sanitizePrivateReporterArtifacts(privateDir, credentials, manualPrivateHandoff + ? { owner: await privateCellEvidenceOwner(manualPrivateHandoff, execution.id) } : undefined); if (processResult.privateDiagnosticLeak) reporterPrivacy.push({ file: "playwright process stdout/stderr", disposition: "redacted", reason: "credential detected before diagnostic emission" }); for (const [index, candidate] of executions.entries()) { let result: RunnerE2EResult = { @@ -888,6 +895,7 @@ async function runAttempt(input: { }); const resultPath = path.join(casePrivateDir, "result.json"); if (manualPrivateHandoff) { + manualDiagnostic = { stage: "package-original", result, observation: await readPrivateAttemptObservation(casePrivateDir) }; await preserveOriginalFrameworkResult(casePrivateDir); await writeFile( resultPath, @@ -902,6 +910,7 @@ async function runAttempt(input: { privateCell: { owner: await privateCellEvidenceOwner(manualPrivateHandoff, candidate.id), layout: "original" as const }, }); const originalEvidence = evidence, originalFeaturePassed = result.status === "passed"; + manualDiagnostic!.evidence = originalEvidence; if (evidence.leaks.length > 0 || evidence.missing.length > 0) { result = withholdPrivateScreenshots(result, { packagedEvidenceSecretFinding: evidence.leaks.length > 0 }); result = { @@ -929,6 +938,7 @@ async function runAttempt(input: { privateCell: { owner: await privateCellEvidenceOwner(manualPrivateHandoff, candidate.id), layout: "original" as const }, }); } + manualDiagnostic!.stage = "seal-original"; await sealStandardOriginalEvidence({ context: manualPrivateHandoff, executionId: candidate.id, source: casePrivateDir, evidence: originalEvidence, secrets: credentials, expectPassScreenshot: originalFeaturePassed }); } else { @@ -959,15 +969,20 @@ async function runAttempt(input: { } return [...publishedResults]; } catch (error) { + if (manualDiagnostic) manualDiagnostic.error = error; if (error instanceof PublicOriginCleanupError) processCleanupFailed = true; // Packaging/diagnostic failures must not erase an actually launched attempt // or its primary startup failure. Retain only closed sanitized metadata; // raw reports stay private when their normal packaging path failed. for (const candidate of executions) { if (publishedResultPaths.has(candidate.id)) continue; - const casePrivateDir = path.join(temporaryRoot, "artifacts-private", "cases", candidate.task.id); + const casePrivateDir = path.join(privateDir, "cases", candidate.task.id); const fallback = syntheticResult(candidate, attempt, startedAtMs, "Attempt terminated before a worker result was retained"); const original = await readResult(path.join(casePrivateDir, "result.json"), fallback); + if (manualPrivateHandoff) { + manualDiagnostic ??= { stage: "execution", result: original, observation: await readPrivateAttemptObservation(casePrivateDir) }; + manualDiagnostic.error ??= error; + } const result = abnormalFinalizationResult(original, error); if (databaseLane === "owned-docker-postgres") result.databaseIdentity = dockerDatabase ? publicDockerDatabaseIdentity(dockerDatabase.receipt) : (await dockerDatabaseLaunchState(databaseLane, temporaryRoot)).identity; const uploadDir = path.join(resultsRoot, campaignId, candidate.suite.id, candidate.profile.id, candidate.environment.id, candidate.task.id, `attempt-${attempt}`); @@ -1188,6 +1203,7 @@ async function main() { } const executions = selectRunnerExecutions(options, runnerMatrix); manualPrivateHandoff = await readManualPrivateHandoff(process.env, executions.map(execution => execution.id)); + if (manualPrivateHandoff) await appendPrivateAttemptDiagnostic({ stage: "execution" }); resultsRoot = manualPrivateHandoff ? path.join(manualPrivateHandoff.root, "results") : resolveRunnerE2EResultsRoot(repositoryRoot); if (options.installedStartupOnly) { assertInstalledStartupOnly(process.env, executions, options); @@ -1384,7 +1400,13 @@ async function main() { await main().catch((error) => { console.error(error instanceof Error ? error.message : String(error)); + if (manualPrivateHandoff) manualDiagnostic ??= { stage: "execution", error }; process.exitCode = 1; }).finally(async () => { if (manualPrivateHandoff) await sealManualResultEvidence(manualPrivateHandoff, [...manualEvidenceSecrets]); -}).catch(() => { process.stderr.write("Private evidence finalization failed; no public fallback\n"); process.exitCode = 1; }); +}).catch((error) => { + if (manualDiagnostic && !manualDiagnostic.error) manualDiagnostic.error = error; + process.stderr.write("Private evidence finalization failed; no public fallback\n"); process.exitCode = 1; +}).finally(async () => { + if (manualPrivateHandoff && manualDiagnostic) await appendPrivateAttemptDiagnostic(manualDiagnostic).catch(() => {}); +}); diff --git a/tests/runner-e2e/manual-private-cell-report-assets.test.ts b/tests/runner-e2e/manual-private-cell-report-assets.test.ts index d132a3e8b25..528aa845fc8 100644 --- a/tests/runner-e2e/manual-private-cell-report-assets.test.ts +++ b/tests/runner-e2e/manual-private-cell-report-assets.test.ts @@ -1,3 +1,4 @@ +import { readPrivateAttemptObservation } from "./private-attempt-diagnostic.js"; import { afterAll, afterEach, beforeAll, expect, it, vi } from "vitest"; import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; @@ -27,6 +28,7 @@ import { MANUAL_PRIVATE_EXECUTIONS, MANUAL_PRIVATE_PROTOCOL, manualCellKind } fr import { archivePrivateEvidence, retainPrivateTransfer } from "./private-history.js"; import { BoundedHistoryPublisher } from "./history-budget.js"; import { sanitizeJson } from "./redaction.js"; +import { sanitizePrivateReporterArtifacts } from "./private-reporter-artifacts.js"; import type { RunnerE2EResult } from "./types.js"; const hash = (value: string | Buffer) => createHash("sha256").update(value).digest("hex"); const roots: string[] = []; @@ -114,6 +116,21 @@ it("retains the real failed trace/report through actual copy, private package, b expect((await validateSealedInventory(path.join(restored, "sealed-results/html-report"), "../result-evidence-inventory.json", undefined, "standard", downloaded)).complete).toBe(true); expect(await readFile(path.join(restored, "sealed-results/html-report", f.prefix, "html-report/trace/playwright-logo.svg"))).toEqual(await readFile(canonicalGeneratedReportAsset("html/trace/playwright-logo.svg")!)); }, 60_000); +it("preserves canonical trace assets through reporter sanitization before private packaging", async () => { + const f = await fixture(); + const before = await readFile(path.join(f.cell, "html-report/trace/index.html")); + await sanitizePrivateReporterArtifacts(f.cell, [], { owner: f.owner }); + expect(await readFile(path.join(f.cell, "html-report/trace/index.html"))).toEqual(before); + await expect(packageEvidence({ privateDir: f.cell, uploadDir: f.upload, secrets: [], expectPassScreenshot: false, + privateCell: { owner: f.owner, layout: "original" } })).resolves.toMatchObject({ leaks: [] }); +}); +it("keeps rejecting contaminated or modified canonical assets during reporter sanitization", async () => { + const f = await fixture(); + await expect(sanitizePrivateReporterArtifacts(f.cell, ["Playwright"], { owner: f.owner })).rejects.toThrow(/Secret leak/); + const file = path.join(f.cell, "html-report/trace/index.html"), original = await readFile(file, "utf8"); + await writeFile(file, original.replace("Playwright", "Xlaywright")); + await expect(sanitizePrivateReporterArtifacts(f.cell, [], { owner: f.owner })).rejects.toThrow(/canonical producer/); +}); it.each(["wrong-case", "wrong-run", "wrong-attempt", "double-wrapper", "extra-nesting", "watch", "xterm", "tampered-html", "tampered-svg", "shape"])("actual package rejects %s in the reserved cell namespace", async kind => { const f = await fixture(); let relative = "html-report/trace/index.html"; if (kind === "watch") relative = "html-report/trace/watch.js"; @@ -275,13 +292,13 @@ it.each([0, 1])("actual launcher packaging call %s retains the verified private async function actualLauncherEvidenceBranch(packageCall = packageEvidence) { const source = await readFile(new URL("./launch.ts", import.meta.url), "utf8"); - const start = source.indexOf(" if (manualPrivateHandoff) {\n await preserveOriginalFrameworkResult(casePrivateDir);"); const parser = createRequire(import.meta.url)("../../node_modules/.pnpm/@babel+parser@7.29.8/node_modules/@babel/parser"); const ast = parser.parse(source, { sourceType: "module", plugins: ["typescript"] }); - let end = -1; + let start = -1, end = -1; const visit = (node: any) => { if (!node || typeof node !== "object") return; - if (node.type === "IfStatement" && node.start === start + 6) end = node.end; + if (node.type === "IfStatement" && node.test?.name === "manualPrivateHandoff" && Array.isArray(node.consequent?.body) && node.consequent.body.some((statement: any) => + statement.expression?.argument?.callee?.name === "preserveOriginalFrameworkResult")) { start = node.start; end = node.end; } for (const value of Object.values(node)) { if (Array.isArray(value)) value.forEach(visit); else if (value && typeof value === "object") visit(value); @@ -289,10 +306,10 @@ async function actualLauncherEvidenceBranch(packageCall = packageEvidence) { }; visit(ast); expect(start).toBeGreaterThan(0); expect(end).toBeGreaterThan(start); - const code = stripTypeScriptTypes(`async function actualBranch(manualPrivateHandoff, candidate, casePrivateDir, resultsRoot, uploadDir, credentials, result, resultPath) { ${source.slice(start, end)} return result; }`, { mode: "strip" }); + const code = stripTypeScriptTypes(`async function actualBranch(manualPrivateHandoff, candidate, casePrivateDir, resultsRoot, uploadDir, credentials, result, resultPath) { let manualDiagnostic; ${source.slice(start, end)} return result; }`, { mode: "strip" }); const publisher = vi.fn(publishRunnerE2EAttemptEvidence); - const branch = Function("preserveOriginalFrameworkResult", "writeFile", "sanitizeJson", "packageEvidence", "privateCellEvidenceOwner", "sealStandardOriginalEvidence", "publishRunnerE2EAttemptEvidence", `${code}; return actualBranch;`)( - preserveOriginalFrameworkResult, writeFile, sanitizeJson, packageCall, privateCellEvidenceOwner, sealStandardOriginalEvidence, publisher, + const branch = Function("preserveOriginalFrameworkResult", "writeFile", "sanitizeJson", "packageEvidence", "privateCellEvidenceOwner", "sealStandardOriginalEvidence", "publishRunnerE2EAttemptEvidence", "readPrivateAttemptObservation", `${code}; return actualBranch;`)( + preserveOriginalFrameworkResult, writeFile, sanitizeJson, packageCall, privateCellEvidenceOwner, sealStandardOriginalEvidence, publisher, readPrivateAttemptObservation, ) as (context: ManualPrivateHandoff | undefined, candidate: { id: string }, source: string, root: string, target: string, secrets: string[], result: RunnerE2EResult, resultPath: string) => Promise; return { branch, publisher }; } diff --git a/tests/runner-e2e/private-attempt-diagnostic.test.ts b/tests/runner-e2e/private-attempt-diagnostic.test.ts new file mode 100644 index 00000000000..8336fff4bde --- /dev/null +++ b/tests/runner-e2e/private-attempt-diagnostic.test.ts @@ -0,0 +1,72 @@ +import { mkdtemp, mkdir, readFile, rm, symlink, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { expect, it } from "vitest"; +import { appendPrivateAttemptDiagnostic, privateAttemptDiagnostic, readPrivateAttemptObservation } from "./private-attempt-diagnostic.js"; +import { canonicalProviderEventsFromOpenCodePart } from "../../packages/paperclip-runner/src/provider-events.js"; +it("projects only fixed outcomes from private result, run and carrier evidence", () => { + const diagnostic = privateAttemptDiagnostic({ stage: "seal-original", result: { status: "failed", error: "Actual new native task reads inherited project context: private-value", runIds: ["private-id"] }, + error: new Error("Credential-contaminated original evidence cannot be retained"), evidence: { leaks: [{ file: "private-path", reason: "zip inspection exceeded its expanded-byte bound" }] }, + observation: { observation: { leadId: "lead", state: { futureWorker: { id: "future" }, activeRunId: "held" }, runs: [{ agentId: "lead", status: "succeeded" }, { agentId: "future", status: "running" }, { id: "held", status: "timed_out" }], reply: "private-value" } } }); + expect(diagnostic).toMatchObject({ finalization: "credential-contamination", phase: "future-work-timeout", runs: 3, source: "succeeded", future: "running", held: "timed_out", carrier: "zip-byte-bound", runStatusCounts: expect.objectContaining({ succeeded: 1, running: 1, timed_out: 1 }) }); + expect(JSON.stringify(diagnostic)).not.toMatch(/private-value|private-id|private-path/); +}); +it("never copies unknown error, stage, failure or status text", () => { + expect(JSON.stringify(privateAttemptDiagnostic({ stage: "private-value", result: { status: "private-value", failureClass: "private-value", error: "private-value" }, error: new Error("private-value") }))).not.toContain("private-value"); +}); +it("counts a finish from the actual OpenCode event producer and recovers the pre-scan phase", () => { + const sourceEvents = ["pending", "running"].flatMap(status => canonicalProviderEventsFromOpenCodePart({ id: "private-call", type: "tool", tool: "paperclip_paperclip_finish", state: { status, input: {} } })); + const diagnostic = privateAttemptDiagnostic({ stage: "package-original", result: { error: "Secret leak in persisted Paperclip home state at private-path" }, + observation: { sourceEvents, persistedState: { scanError: "private-error", originalResult: { error: "Canonical project-defaults turn settles: private-value" } } } }); + expect(diagnostic).toMatchObject({ phase: "canonical-turn-timeout", privacy: { persistedState: true }, sourceProgress: { tools: { paperclip_finish: { started: 1, progressed: 1, completed: 0 }, other: { started: 0, completed: 0 } } } }); + expect(JSON.stringify(diagnostic)).not.toMatch(/private-(?:call|path|error|value)/); +}); +it("distinguishes an unfinished source tool and preserves a timeout hidden by privacy finalization", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "closed-eval-progress-")); + try { + const snapshots = path.join(root, "snapshots"); await mkdir(snapshots); + await writeFile(path.join(snapshots, "chat-catalog-observation.json"), JSON.stringify({ observation: { leadId: "private-lead", runs: [{ id: "private-run", agentId: "private-lead", status: "running" }] } })); + await writeFile(path.join(snapshots, "chat-catalog-runs.json"), JSON.stringify([ + { runId: "private-other", events: [{ eventType: "run.result.proposed" }] }, + { runId: "private-run", events: [ + { payload: { prpEvent: { eventType: "turn.started" } } }, + { payload: { prpEvent: { eventType: "tool.execution.started", payload: { name: "paperclip_call_api", output: "private-value" } } } }, + { payload: { prpEvent: { eventType: "tool.execution.started", payload: { name: "private-tool" } } } }, + { payload: { prpEvent: { eventType: "tool.execution.completed", payload: { name: "private-tool" } } } }, + ] }, + ])); + await writeFile(path.join(snapshots, "diagnostic-privacy.json"), JSON.stringify({ originalResult: { error: "Canonical project-defaults turn settles: private-value" }, issues: [{ file: "private-path" }] })); + const diagnostic = privateAttemptDiagnostic({ stage: "package-original", result: { failureClass: "secret_leak" }, + error: new Error("Secret leak in snapshots/private.json: exact secret value"), observation: await readPrivateAttemptObservation(root) }); + expect(diagnostic).toMatchObject({ phase: "canonical-turn-timeout", finalization: "text-secret-finding", privacy: { reporter: 1, packaging: "snapshot" }, + sourceProgress: { events: 4, turns: 1, finishProposals: 0, tools: { paperclip_call_api: { started: 1, completed: 0 }, other: { started: 1, completed: 1 } } } }); + expect(JSON.stringify(diagnostic)).not.toMatch(/private-(?:lead|run|value|path|tool|other)/); + await rm(path.join(snapshots, "chat-catalog-runs.json")); + const external = path.join(root, "outside.json"); await writeFile(external, JSON.stringify([{ runId: "private-run", events: [{ eventType: "run.result.proposed" }] }])); + await symlink(external, path.join(snapshots, "chat-catalog-runs.json")); + expect(privateAttemptDiagnostic({ stage: "execution", observation: await readPrivateAttemptObservation(root) }).sourceProgress.events).toBe(0); + } finally { await rm(root, { recursive: true, force: true }); } +}); +it("reports recognized tool and API outcomes without copying arbitrary operation names", () => { + const diagnostic = privateAttemptDiagnostic({ stage: "package-original", error: new Error("Private cell evidence exceeds byte allowance"), + observation: { privacy: { issues: [{ reason: "zip inspection exceeded its expanded-byte bound" }] }, sourceEvents: [ + { eventType: "tool.execution.completed", payload: { name: "paperclip_search_api", status: "completed" } }, + { eventType: "item.completed", payload: { kind: "dynamicToolCall", item: { result: { apiOperationId: "PATCH /api/projects/{id}", status: 403, ok: false, body: "private-value" } } } }, + { eventType: "item.completed", payload: { kind: "dynamicToolCall", item: { result: { apiOperationId: "private-operation", status: 200, ok: true } } } }, + ] } }); + expect(diagnostic).toMatchObject({ finalization: "byte-bound", privacy: { reporterReasons: { "zip inspection exceeded its expanded-byte bound": 1 } }, + sourceProgress: { tools: { paperclip_search_api: { completed: 1 } }, lastTools: [{ name: "paperclip_search_api", phase: "completed", status: "completed" }], + apiResults: [{ operation: "update-project", status: 403, ok: false }, { operation: "other", status: 200, ok: true }] } }); + expect(JSON.stringify(diagnostic)).not.toMatch(/private-(?:value|operation)/); +}); +it("writes one closed variable to the actual owned runner environment file and rejects symlinks", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "closed-eval-diagnostic-")); + try { + const commands = path.join(root, "_runner_file_commands"); await mkdir(commands); const file = path.join(commands, "set_env_test"); await writeFile(file, ""); + const env = { GITHUB_ACTIONS: "true", RUNNER_TEMP: root, GITHUB_ENV: file }; + await appendPrivateAttemptDiagnostic({ stage: "execution", error: new Error("private-value") }, env); + const text = await readFile(file, "utf8"); expect(text).toMatch(/^PAPERCLIP_E2E_CLOSED_DIAGNOSTIC_V1=\{/); expect(text).not.toContain("private-value"); expect(text.trim().split("\n")).toHaveLength(1); + await symlink(file, path.join(commands, "set_env_symlink")); + await expect(appendPrivateAttemptDiagnostic({ stage: "execution" }, { ...env, GITHUB_ENV: path.join(commands, "set_env_symlink") })).rejects.toThrow(); + } finally { await rm(root, { recursive: true, force: true }); } +}); diff --git a/tests/runner-e2e/private-attempt-diagnostic.ts b/tests/runner-e2e/private-attempt-diagnostic.ts new file mode 100644 index 00000000000..22f3a2be4ed --- /dev/null +++ b/tests/runner-e2e/private-attempt-diagnostic.ts @@ -0,0 +1,132 @@ +import { constants } from "node:fs"; +import { lstat, open, readFile, realpath } from "node:fs/promises"; +import path from "node:path"; +import { PAPERCLIP_PROTOCOL_ACTIONS } from "../../packages/paperclip-runner/src/protocol-actions/index.js"; + +export interface PrivateAttemptDiagnosticInput { stage: unknown; result?: unknown; error?: unknown; observation?: unknown; evidence?: unknown } +const record = (value: unknown): Record => value && typeof value === "object" ? value as Record : {}; +const member = (value: unknown, choices: string[]) => typeof value === "string" && choices.includes(value) ? value : "unknown"; +const failures = new Map([ + ["Credential-contaminated original evidence cannot be retained", "credential-contamination"], + ["Unsupported original evidence carrier cannot be retained", "unsupported-carrier"], + ["Complete original evidence exceeds byte allowance", "byte-bound"], + ["Complete original evidence exceeds entry allowance", "entry-bound"], + ["Private cell evidence exceeds byte allowance", "byte-bound"], + ["Private cell evidence exceeds entry allowance", "entry-bound"], + ["Unsupported private cell evidence carrier", "unsupported-carrier"], + ["Unsupported private cell trace owner", "trace-owner"], + ["Invalid private cell trace asset", "trace-asset"], + ["Private cell trace content differs from canonical producer", "trace-producer-mismatch"], + ["Private cell report producer differs from trusted preflight", "report-producer-mismatch"], + ["Canonical report source changed", "report-source-changed"], + ["Invalid canonical report source", "report-source"], +]); +const phases = [["Real prior active Launch checkout", "held-work-timeout"], ["Canonical project-defaults turn settles", "canonical-turn-timeout"], ["Actual new native task reads inherited project context", "future-work-timeout"]]; +const toolNames = ["bash", "paperclip_call_api", "paperclip_finish", "paperclip_get_workspace", "paperclip_get_run", "paperclip_list_runs"]; +const recognizedTools = new Set([...toolNames, "read", "glob", "grep", "task", "todowrite", "question", + ...PAPERCLIP_PROTOCOL_ACTIONS.map(action => `paperclip_${action.id}`)]); +const toolName = (value: unknown) => typeof value === "string" && recognizedTools.has(value) ? value : "other"; +const apiOperations = new Map([ + ["GET /api/projects/{id}", "read-project"], ["PATCH /api/projects/{id}", "update-project"], + ["GET /api/companies/{companyId}/projects", "list-projects"], ["GET /api/companies/{companyId}/goals", "list-goals"], + ["GET /api/issues/{id}", "read-task"], ["PATCH /api/issues/{id}", "update-task"], + ["GET /api/companies/{companyId}/issues", "list-tasks"], ["GET /api/companies/{companyId}/agents", "list-agents"], +]); +function sourceProgress(value: unknown) { + const rows = Array.isArray(value) ? value : []; + const events = rows.map(row => record(record(row).payload).prpEvent ?? row).map(record); + const tools = events.filter(event => ["tool.execution.started", "tool.execution.progressed", "tool.execution.completed"].includes(event.eventType)); + const names = [...new Set([...toolNames, ...tools.map(event => toolName(record(event.payload).name)), "other"])]; + return { + events: Math.min(events.length, 100000), + turns: Math.min(events.filter(event => event.eventType === "turn.started").length, 1000), + finishProposals: Math.min(events.filter(event => event.eventType === "run.result.proposed").length, 1000), + finishFeedback: Object.fromEntries(([ + ["timeout", /MCP error -32001|timed out/i], ["contract", /Stale completionClaim\.contractRevision/i], + ["criteria", /completionClaim\.criteria must contain exactly/i], ["incomplete", /done report includes unfinished work/i], + ["approval", /requires a persisted human decision|Required human quality review is incomplete/i], + ] as const).map(([label, pattern]) => [label, Math.min(tools.filter(event => event.eventType === "tool.execution.completed" + && record(event.payload).name === "paperclip_finish" && pattern.test(String(record(event.payload).output ?? ""))).length, 1000)])), + tools: Object.fromEntries(names.map(name => [name, Object.fromEntries(["started", "progressed", "completed"].map(status => + [status, Math.min(tools.filter(event => event.eventType === `tool.execution.${status}` && + toolName(record(event.payload).name) === name).length, 10000)]))])), + lastTools: tools.filter(event => event.eventType !== "tool.execution.progressed").slice(-12).map(event => ({ + name: toolName(record(event.payload).name), phase: event.eventType === "tool.execution.started" ? "started" : "completed", + status: member(record(event.payload).status, ["running", "completed", "failed", "cancelled"]), + })), + apiResults: events.filter(event => event.eventType === "item.completed" && record(event.payload).kind === "dynamicToolCall") + .map(event => record(record(record(event.payload).item).result)).filter(result => typeof result.apiOperationId === "string").slice(-12) + .map(result => ({ operation: apiOperations.get(result.apiOperationId) ?? "other", + status: Number.isSafeInteger(result.status) && result.status >= 100 && result.status <= 599 ? result.status : null, ok: result.ok === true })), + }; +} +/** Closed labels/counts only; never copy an error, identity, path, body or provider value. */ +export function privateAttemptDiagnostic(input: PrivateAttemptDiagnosticInput) { + const result = record(input.result), envelope = record(input.observation), observation = record(envelope.observation), state = record(observation.state); + const runs = Array.isArray(observation.runs) ? observation.runs.map(record) : []; + const status = (run: Record | undefined) => member(run?.status, ["queued", "running", "succeeded", "failed", "cancelled", "timed_out"]); + const message = input.error instanceof Error ? input.error.message : "", productError = typeof result.error === "string" ? result.error : ""; + const privacy = record(envelope.privacy), original = record(privacy.originalResult), persistedState = record(envelope.persistedState); + const errors = [productError, original.error, persistedState.scanError, record(persistedState.originalResult).error].filter(value => typeof value === "string") as string[]; + const evidence = record(input.evidence), leaks = Array.isArray(evidence.leaks) ? evidence.leaks : []; + return { + schema: "paperclip.private-attempt-diagnostic/v1", + stage: member(input.stage, ["execution", "package-original", "seal-original"]), + status: member(result.status, ["passed", "failed", "skipped"]), + failureClass: member(result.failureClass, ["candidate_failure", "transient_infrastructure", "permanent_infrastructure", "secret_leak", "cleanup_failure"]), + finalization: failures.get(message) ?? (record(input.error).code === "ENOENT" ? "missing-file" + : /^Secret leak in [\s\S]*: (?:exact secret value|secret-shaped value)$/.test(message) ? "text-secret-finding" : "unclassified"), + phase: phases.find(([label]) => [...errors, message].some(error => error.includes(label!)))?.[1] ?? "unclassified", + privacy: { persistedState: errors.some(error => error.startsWith("Secret leak in persisted ")), + reporter: Array.isArray(privacy.issues) ? Math.min(privacy.issues.length, 1000) : 0, + reporterReasons: Object.fromEntries(["zip inspection exceeded its expanded-byte bound", "zip inspection exceeded its time bound", "exact secret value", "secret-shaped value"].map(reason => + [reason, Array.isArray(privacy.issues) ? Math.min(privacy.issues.filter(issue => record(issue).reason === reason).length, 1000) : 0])), + packaging: /^Secret leak in /.test(message) ? (/^Secret leak in snapshots\//.test(message) ? "snapshot" : /^Secret leak in .*\.json:/.test(message) ? "json" : "text") : "unknown" }, + runStatusCounts: Object.fromEntries(["queued", "running", "succeeded", "failed", "cancelled", "timed_out"].map(label => + [label, Math.min(runs.filter(run => run.status === label).length, 1000)])), + runs: Math.min(runs.length || (Array.isArray(result.runIds) ? result.runIds.length : 0), 1000), + checks: Array.isArray(result.matcherResults) ? Math.min(result.matcherResults.length, 1000) : 0, + source: status(runs.find(run => typeof observation.leadId === "string" && run.agentId === observation.leadId)), + future: status(runs.find(run => typeof record(state.futureWorker).id === "string" && run.agentId === record(state.futureWorker).id)), + held: status(runs.find(run => typeof state.activeRunId === "string" && run.id === state.activeRunId)), + productGradePassed: record(envelope.grade).passed === true, + sourceProgress: sourceProgress(envelope.sourceEvents), + carrier: leaks.some(leak => record(leak).reason === "zip inspection exceeded its expanded-byte bound") ? "zip-byte-bound" + : leaks.some(leak => record(leak).reason === "zip inspection exceeded its time bound") ? "zip-time-bound" : leaks.length ? "other-finding" : "none", + }; +} +export async function readPrivateAttemptObservation(casePrivateDir: string) { + const read = async (name: string, limit = 8 * 1024 * 1024) => { + try { + const file = path.join(casePrivateDir, "snapshots", name), metadata = await lstat(file); + if (!metadata.isFile() || metadata.isSymbolicLink() || metadata.size > limit) return; + return JSON.parse(await readFile(file, "utf8")); + } catch { return; } + }; + const envelope = record(await read("chat-catalog-observation.json")); + const observation = record(envelope.observation); + const runs = Array.isArray(observation.runs) ? observation.runs.map(record) : []; + const source = runs.find(run => typeof observation.leadId === "string" && run.agentId === observation.leadId); + const evidence = await read("chat-catalog-runs.json", 64 * 1024 * 1024); + const sourceEvents = Array.isArray(evidence) ? record(evidence.find(row => source && record(row).runId === source.id)).events : undefined; + return { ...envelope, sourceEvents, privacy: await read("diagnostic-privacy.json"), persistedState: await read("persisted-state-scan.json") }; +} +/** + * GITHUB_ENV survives the paid step's private stdout redirect. SetEnvFileCommand + * updates the env context; ActionRunner passes it to ScriptHandler, whose + * PrintActionDetails prints it before the next trusted shell step runs. + * https://github.com/actions/runner/blob/main/src/Runner.Worker/Handlers/ScriptHandler.cs#L120-L126 + * The variable is not consumed as authority by the protected transfer. + */ +export async function appendPrivateAttemptDiagnostic(input: PrivateAttemptDiagnosticInput, env = process.env) { + if (env.GITHUB_ACTIONS !== "true") return; + if (!env.GITHUB_ENV || !env.RUNNER_TEMP) throw new Error("Closed private diagnostic channel is unavailable"); + const root = await realpath(env.RUNNER_TEMP), parent = await realpath(path.dirname(env.GITHUB_ENV)); + if (!parent.startsWith(`${root}${path.sep}`) || !path.basename(env.GITHUB_ENV).startsWith("set_env_")) throw new Error("Diagnostic environment file escaped runner commands"); + const file = await open(env.GITHUB_ENV, constants.O_WRONLY | constants.O_APPEND | constants.O_NOFOLLOW); + try { + const metadata = await file.stat(); + if (!metadata.isFile() || typeof process.getuid === "function" && metadata.uid !== process.getuid()) throw new Error("Invalid diagnostic environment file owner"); + await file.write(`PAPERCLIP_E2E_CLOSED_DIAGNOSTIC_V1=${JSON.stringify(privateAttemptDiagnostic(input))}\n`); + } finally { await file.close(); } +} diff --git a/tests/runner-e2e/private-reporter-artifacts.ts b/tests/runner-e2e/private-reporter-artifacts.ts index 1f0ffff12a9..99e1513816c 100644 --- a/tests/runner-e2e/private-reporter-artifacts.ts +++ b/tests/runner-e2e/private-reporter-artifacts.ts @@ -3,20 +3,30 @@ import path from "node:path"; import { inspectEmbeddedReport, inspectZip } from "./evidence.js"; import { diagnosticSecrets, type DiagnosticSecretExpansion } from "./private-diagnostics.js"; import { findSecretLeak, redactText, sanitizeJson } from "./redaction.js"; +import { cellTraceAsset, readPrivateCellTraceAsset, type PrivateCellEvidenceOwner } from "./manual-private-report-assets.js"; export interface ReporterPrivacyIssue { file: string; disposition: "redacted" | "omitted"; reason: string } const sources = ["server.log", "playwright.log", "junit.xml", "html-report", "blob-report", "playwright-output"]; const text = new Set([".log", ".txt", ".md", ".xml", ".html", ".json", ".js", ".css"]); /** Only reporter/diagnostic sinks are touched, after independent product oracles * finish. Product API snapshots and work-product bytes are never pre-sanitized. * Uninspectable/leaking compressed reports are omitted rather than fabricated. */ -export async function sanitizePrivateReporterArtifacts(root: string, values: readonly string[] | DiagnosticSecretExpansion): Promise { +export async function sanitizePrivateReporterArtifacts(root: string, values: readonly string[] | DiagnosticSecretExpansion, + privateCell?: { owner: PrivateCellEvidenceOwner }): Promise { const secrets = diagnosticSecrets(values).values; const issues: ReporterPrivacyIssue[] = []; const visit = async (relative: string): Promise => { const filename = path.join(root, relative); const info = await lstat(filename).catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return null; throw error; }); if (!info) return; const safeName = redactText(relative, secrets); - if (info.isSymbolicLink()) { await rm(filename); issues.push({ file: safeName, disposition: "omitted", reason: "reporter symlink cannot be inspected" }); return; } if (info.isDirectory()) { for (const name of await readdir(filename)) await visit(path.join(relative, name)); return; } + if (privateCell && cellTraceAsset(relative, privateCell.owner, "original").reserved) { + // These are immutable Playwright program files. Scan and authenticate + // them with the same guard used by packaging; diagnostic text rewriting + // can change clean JavaScript and invalidate its producer identity. + await readPrivateCellTraceAsset({ owner: privateCell.owner, layout: "original", relative, + file: filename, remaining: info.size, secrets }); + return; + } + if (info.isSymbolicLink()) { await rm(filename); issues.push({ file: safeName, disposition: "omitted", reason: "reporter symlink cannot be inspected" }); return; } if (!info.isFile()) return; if (safeName !== relative) { await rm(filename); issues.push({ file: safeName, disposition: "omitted", reason: "credential in reporter filename" }); return; } const extension = path.extname(filename).toLowerCase();