PaperClipAI/tests/runner-e2e/private-attempt-diagnostic.test.ts

import { mkdtemp, mkdir, readFile, rm, symlink, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { expect, it } from "vitest";
import { appendPrivateAttemptDiagnostic, privateAttemptDiagnostic, readPrivateAttemptObservation } from "./private-attempt-diagnostic.js";
import { canonicalProviderEventsFromOpenCodePart } from "../../packages/paperclip-runner/src/provider-events.js";
it("projects only fixed outcomes from private result, run and carrier evidence", () => {
  const diagnostic = privateAttemptDiagnostic({ stage: "seal-original", result: { status: "failed", error: "Actual new native task reads inherited project context: private-value", runIds: ["private-id"] },
    error: new Error("Credential-contaminated original evidence cannot be retained"), evidence: { leaks: [{ file: "private-path", reason: "zip inspection exceeded its expanded-byte bound" }] },
    observation: { observation: { leadId: "lead", state: { futureWorker: { id: "future" }, activeRunId: "held" }, runs: [{ agentId: "lead", status: "succeeded" }, { agentId: "future", status: "running" }, { id: "held", status: "timed_out" }], reply: "private-value" } } });
  expect(diagnostic).toMatchObject({ finalization: "credential-contamination", phase: "future-work-timeout", runs: 3, source: "succeeded", future: "running", held: "timed_out", carrier: "zip-byte-bound", runStatusCounts: expect.objectContaining({ succeeded: 1, running: 1, timed_out: 1 }) });
  expect(JSON.stringify(diagnostic)).not.toMatch(/private-value|private-id|private-path/);
});
it("never copies unknown error, stage, failure or status text", () => {
  expect(JSON.stringify(privateAttemptDiagnostic({ stage: "private-value", result: { status: "private-value", failureClass: "private-value", error: "private-value" }, error: new Error("private-value") }))).not.toContain("private-value");
});
it("counts a finish from the actual OpenCode event producer and recovers the pre-scan phase", () => {
  const sourceEvents = ["pending", "running"].flatMap(status => canonicalProviderEventsFromOpenCodePart({ id: "private-call", type: "tool", tool: "paperclip_paperclip_finish", state: { status, input: {} } }));
  const diagnostic = privateAttemptDiagnostic({ stage: "package-original", result: { error: "Secret leak in persisted Paperclip home state at private-path" },
    observation: { sourceEvents, persistedState: { scanError: "private-error", originalResult: { error: "Canonical project-defaults turn settles: private-value" } } } });
  expect(diagnostic).toMatchObject({ phase: "canonical-turn-timeout", privacy: { persistedState: true }, sourceProgress: { tools: { paperclip_finish: { started: 1, progressed: 1, completed: 0 }, other: { started: 0, completed: 0 } } } });
  expect(JSON.stringify(diagnostic)).not.toMatch(/private-(?:call|path|error|value)/);
});
it("distinguishes an unfinished source tool and preserves a timeout hidden by privacy finalization", async () => {
  const root = await mkdtemp(path.join(os.tmpdir(), "closed-eval-progress-"));
  try {
    const snapshots = path.join(root, "snapshots"); await mkdir(snapshots);
    await writeFile(path.join(snapshots, "chat-catalog-observation.json"), JSON.stringify({ observation: { leadId: "private-lead", runs: [{ id: "private-run", agentId: "private-lead", status: "running" }] } }));
    await writeFile(path.join(snapshots, "chat-catalog-runs.json"), JSON.stringify([
      { runId: "private-other", events: [{ eventType: "run.result.proposed" }] },
      { runId: "private-run", events: [
        { payload: { prpEvent: { eventType: "turn.started" } } },
        { payload: { prpEvent: { eventType: "tool.execution.started", payload: { name: "paperclip_call_api", output: "private-value" } } } },
        { payload: { prpEvent: { eventType: "tool.execution.started", payload: { name: "private-tool" } } } },
        { payload: { prpEvent: { eventType: "tool.execution.completed", payload: { name: "private-tool" } } } },
      ] },
    ]));
    await writeFile(path.join(snapshots, "diagnostic-privacy.json"), JSON.stringify({ originalResult: { error: "Canonical project-defaults turn settles: private-value" }, issues: [{ file: "private-path" }] }));
    const diagnostic = privateAttemptDiagnostic({ stage: "package-original", result: { failureClass: "secret_leak" },
      error: new Error("Secret leak in snapshots/private.json: exact secret value"), observation: await readPrivateAttemptObservation(root) });
    expect(diagnostic).toMatchObject({ phase: "canonical-turn-timeout", finalization: "text-secret-finding", privacy: { reporter: 1, packaging: "snapshot" },
      sourceProgress: { events: 4, turns: 1, finishProposals: 0, tools: { paperclip_call_api: { started: 1, completed: 0 }, other: { started: 1, completed: 1 } } } });
    expect(JSON.stringify(diagnostic)).not.toMatch(/private-(?:lead|run|value|path|tool|other)/);
    await rm(path.join(snapshots, "chat-catalog-runs.json"));
    const external = path.join(root, "outside.json"); await writeFile(external, JSON.stringify([{ runId: "private-run", events: [{ eventType: "run.result.proposed" }] }]));
    await symlink(external, path.join(snapshots, "chat-catalog-runs.json"));
    expect(privateAttemptDiagnostic({ stage: "execution", observation: await readPrivateAttemptObservation(root) }).sourceProgress.events).toBe(0);
  } finally { await rm(root, { recursive: true, force: true }); }
});
it("reports recognized tool and API outcomes without copying arbitrary operation names", () => {
  const diagnostic = privateAttemptDiagnostic({ stage: "package-original", error: new Error("Private cell evidence exceeds byte allowance"),
    observation: { privacy: { issues: [{ reason: "zip inspection exceeded its expanded-byte bound" }] }, sourceEvents: [
      { eventType: "tool.execution.completed", payload: { name: "paperclip_search_api", status: "completed" } },
      { eventType: "item.completed", payload: { kind: "dynamicToolCall", item: { result: { apiOperationId: "PATCH /api/projects/{id}", status: 403, ok: false, body: "private-value" } } } },
      { eventType: "item.completed", payload: { kind: "dynamicToolCall", item: { result: { apiOperationId: "private-operation", status: 200, ok: true } } } },
    ] } });
  expect(diagnostic).toMatchObject({ finalization: "byte-bound", privacy: { reporterReasons: { "zip inspection exceeded its expanded-byte bound": 1 } },
    sourceProgress: { tools: { paperclip_search_api: { completed: 1 } }, lastTools: [{ name: "paperclip_search_api", phase: "completed", status: "completed" }],
      apiResults: [{ operation: "update-project", status: 403, ok: false }, { operation: "other", status: 200, ok: true }] } });
  expect(JSON.stringify(diagnostic)).not.toMatch(/private-(?:value|operation)/);
});
it("writes one closed variable to the actual owned runner environment file and rejects symlinks", async () => {
  const root = await mkdtemp(path.join(os.tmpdir(), "closed-eval-diagnostic-"));
  try {
    const commands = path.join(root, "_runner_file_commands"); await mkdir(commands); const file = path.join(commands, "set_env_test"); await writeFile(file, "");
    const env = { GITHUB_ACTIONS: "true", RUNNER_TEMP: root, GITHUB_ENV: file };
    await appendPrivateAttemptDiagnostic({ stage: "execution", error: new Error("private-value") }, env);
    const text = await readFile(file, "utf8"); expect(text).toMatch(/^PAPERCLIP_E2E_CLOSED_DIAGNOSTIC_V1=\{/); expect(text).not.toContain("private-value"); expect(text.trim().split("\n")).toHaveLength(1);
    await symlink(file, path.join(commands, "set_env_symlink"));
    await expect(appendPrivateAttemptDiagnostic({ stage: "execution" }, { ...env, GITHUB_ENV: path.join(commands, "set_env_symlink") })).rejects.toThrow();
  } finally { await rm(root, { recursive: true, force: true }); }
});

it("classifies existing reporter and packaging carriers without exposing private filenames", () => {
  const reason = "zip inspection exceeded its expanded-byte bound";
  const issues = [
    { file: "blob-report/report.zip", reason },
    { file: "playwright-output/private-test/trace.zip", reason },
    { file: "html-report/data/private-hash.zip", reason },
    { file: "html-report/index.html", reason: "zip inspection exceeded its time bound" },
    { file: "snapshots/private-path.json", reason: "exact secret value" },
    { file: "private-path", reason: "private-reason" },
  ];
  const diagnostic = privateAttemptDiagnostic({ stage: "package-original", observation: { privacy: { issues } },
    evidence: { leaks: [{ file: "playwright-output\\private-test\\trace.zip", reason }, { file: "private-path", reason }] } });
  expect(diagnostic.carrierReasons).toEqual([
    { source: "reporter", carrier: "blob-zip", reason, count: 1 },
    { source: "reporter", carrier: "playwright-trace-zip", reason, count: 1 },
    { source: "reporter", carrier: "html-trace-zip", reason, count: 1 },
    { source: "reporter", carrier: "embedded-html", reason: "zip inspection exceeded its time bound", count: 1 },
    { source: "reporter", carrier: "other", reason: "exact secret value", count: 1 },
    { source: "packaging", carrier: "playwright-trace-zip", reason, count: 1 },
    { source: "packaging", carrier: "other", reason, count: 1 },
  ]);
  expect(JSON.stringify(diagnostic)).not.toMatch(/private-(?:test|hash|path|reason)/);
});


it("reports fixed embedded HTML and reporter text bounds while excluding unknown reasons", () => {
  const diagnostic = privateAttemptDiagnostic({ stage: "package-original", observation: { privacy: { issues: [
    { file: "html-report/index.html", reason: "embedded report exceeded its byte bound" },
    { file: "html-report/index.html", reason: "reporter text exceeds inspection bound" },
    { file: "html-report/index.html", reason: "zip could not be inspected: private-error" },
  ] } } });
  expect(diagnostic.carrierReasons).toEqual([
    { source: "reporter", carrier: "embedded-html", reason: "embedded report exceeded its byte bound", count: 1 },
    { source: "reporter", carrier: "embedded-html", reason: "reporter text exceeds inspection bound", count: 1 },
  ]);
  expect(JSON.stringify(diagnostic)).not.toMatch(/private-error|html-report/);
});


it("retains each remaining fixed scanner reason without dynamic text", () => {
  const reasons = ["embedded report encoding could not be inspected", "reporter symlink cannot be inspected",
    "credential in reporter filename", "reporter JSON could not be safely parsed"];
  const diagnostic = privateAttemptDiagnostic({ stage: "package-original", observation: { privacy: { issues: [
    ...reasons.map(reason => ({ file: "html-report/index.html", reason })),
    { file: "private-filename", reason: "zip could not be inspected: private-error" },
  ] } } });
  expect(diagnostic.carrierReasons).toEqual(reasons.map(reason => ({ source: "reporter", carrier: "embedded-html", reason, count: 1 })));
  expect(JSON.stringify(diagnostic)).not.toMatch(/private-filename|private-error|html-report/);
});

import { readFileSync } from "node:fs";
it("gives every fixed packaging, sealing and report-asset refusal a closed finalization code", () => {
  const messages = ["manual-private-handoff.ts", "evidence.ts", "manual-private-report-assets.ts", "private-framework-report.ts", "redaction.ts", "manual-private-retention.ts", "chat-readiness-detail.ts", "source-sdk-evidence.ts", "private-diagnostics.ts", "source-sdk-public-type.ts"].flatMap(file =>
    [...readFileSync(new URL(`./${file}`, import.meta.url), "utf8").matchAll(/(?:new Error|budgetFailure)\("([^"$`]+)"/g)].map(match => match[1]!));
  expect(messages.length).toBeGreaterThan(20);
  for (const message of messages) {
    const diagnostic = privateAttemptDiagnostic({ stage: "seal-original", error: new Error(message) });
    expect(diagnostic.finalization, message).not.toBe("unclassified");
  }
});
it("names other filesystem refusals by their closed error code", () => {
  const error = Object.assign(new Error("EEXIST: file already exists, open '/private/path'"), { code: "EEXIST" });
  expect(privateAttemptDiagnostic({ stage: "seal-original", error }).finalization).toBe("fs-EEXIST");
  expect(JSON.stringify(privateAttemptDiagnostic({ stage: "seal-original", error }))).not.toContain("/private/path");
});
it("names Node error codes, error types and non-error throws without their messages", () => {
  const node = Object.assign(new Error("File size (3) is greater than 2 GiB: /private/x"), { code: "ERR_FS_FILE_TOO_LARGE" });
  expect(privateAttemptDiagnostic({ stage: "attach-original", error: node }).finalization).toBe("fs-ERR_FS_FILE_TOO_LARGE");
  expect(privateAttemptDiagnostic({ stage: "attach-original", error: new TypeError("x of /private/y") }).finalization).toBe("error-type-TypeError");
  expect(privateAttemptDiagnostic({ stage: "attach-original", error: "raw /private/z" }).finalization).toBe("non-error-thrown");
  expect(JSON.stringify(privateAttemptDiagnostic({ stage: "attach-original", error: node }))).not.toContain("/private/");
});