PaperClipAI/tests/runner-e2e/paperclip-document.test.ts

import { describe, expect, it } from "vitest";
import { contextIntegrityScenario, PAPERCLIP_DOCUMENT_CASE } from "./context-integrity-cases.js";
import { gradeContextIntegrity, type ContextIntegrityCheckpoint } from "./context-integrity-scoring.js";

function recording() {
  const scenario = contextIntegrityScenario(PAPERCLIP_DOCUMENT_CASE, "probe");
  const initial: ContextIntegrityCheckpoint = {
    phase: "initial", issue: { id: "issue", identifier: "DOC-1", issuePrefix: "DOC", appOrigin: "https://paperclip.example", status: "in_progress" },
    documents: [], comments: [], runs: [{ id: "run", status: "running" }],
    assignedSkill: { key: scenario.skillKey, runtimeName: scenario.skillKey, versionId: "version", markdown: `Write ${scenario.marker}` },
    skillRequestText: `${scenario.prompt}\nUse /${scenario.skillKey}`,
  };
  const final: ContextIntegrityCheckpoint = {
    ...initial, phase: "final", issue: { ...initial.issue, status: "done" },
    documents: [{ key: "report", body: `Requested ${scenario.marker}`, latestRevisionId: "revision", latestRevisionNumber: 1 }],
    comments: [{ authorAgentId: "agent", body: "Saved [report](/DOC/issues/DOC-1#document-report)." }],
    runs: [{ id: "run", status: "succeeded" }],
  };
  return { scenario, initial, final };
}

describe("explicit Paperclip document delivery", () => {
  it("checks the actual saved content, revision and exact document link", () => {
    const { scenario, initial, final } = recording();
    expect(gradeContextIntegrity({ ...scenario, checkpoints: [initial, final] }).every(row => row.passed)).toBe(true);
    expect(scenario.prompt).toContain("Paperclip document");
    expect(scenario.prompt).not.toContain(scenario.marker);
  });
  it.each(["https://paperclip.example/DOC/issues/DOC-1#document-report", "<https://paperclip.example/DOC/issues/DOC-1#document-report>"])("accepts the same-app document link %s", target => {
    const { scenario, initial, final } = recording();
    final.comments[0]!.body = `Saved [report](${target}).`;
    expect(gradeContextIntegrity({ ...scenario, checkpoints: [initial, final] }).every(row => row.passed)).toBe(true);
  });
  it.each(["local-only", "missing-revision", "wrong-content", "local-link", "wrong-document-link", "other-app-link", "user-link-only"])(
    "rejects plausible %s delivery", variant => {
      const { scenario, initial, final } = recording();
      if (variant === "local-only") { final.documents = []; final.comments[0]!.body = "Saved report.md in the workspace."; }
      if (variant === "missing-revision") final.documents[0]!.latestRevisionId = null;
      if (variant === "wrong-content") final.documents[0]!.body = "A different report";
      if (variant === "local-link") final.comments[0]!.body = "Saved [report](./report.md).";
      if (variant === "wrong-document-link") final.comments[0]!.body = "Saved [report](/DOC/issues/DOC-1#document-other).";
      if (variant === "other-app-link") final.comments[0]!.body = "Saved [report](https://other.example/DOC/issues/DOC-1#document-report).";
      if (variant === "user-link-only") { delete final.comments[0]!.authorAgentId; final.comments[0]!.authorUserId = "user"; }
      expect(gradeContextIntegrity({ ...scenario, checkpoints: [initial, final] }).some(row => !row.passed)).toBe(true);
    },
  );
  it("preserves the original ambiguous request and assigned output procedure", () => {
    expect(contextIntegrityScenario("assigned-skill-explicit-invocation", "probe").prompt).toBe(
      "Use the assigned Context integrity output skill for this task, then follow its instructions and finish the task. Do not create child tasks or unrelated deliverables.",
    );
  });
});