mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-06 21:05:21 +02:00
DottaandPaperclip
fe9dc1e3c5
Use stock discovery metadata and an actual Markdown document-link example. Clarify the future explicit UI-link fixture and retain the original task request and all measured failures. Co-Authored-By: Paperclip <noreply@paperclip.ing>
57 lines
4.0 KiB
TypeScript
57 lines
4.0 KiB
TypeScript
import { describe, expect, it } from "vitest";
|
|
import { contextIntegrityScenario, PAPERCLIP_DOCUMENT_CASE } from "./context-integrity-cases.js";
|
|
import { gradeContextIntegrity, type ContextIntegrityCheckpoint } from "./context-integrity-scoring.js";
|
|
|
|
function recording() {
|
|
const scenario = contextIntegrityScenario(PAPERCLIP_DOCUMENT_CASE, "probe");
|
|
const initial: ContextIntegrityCheckpoint = {
|
|
phase: "initial", issue: { id: "issue", identifier: "DOC-1", issuePrefix: "DOC", appOrigin: "https://paperclip.example", status: "in_progress" },
|
|
documents: [], comments: [], runs: [{ id: "run", status: "running" }],
|
|
assignedSkill: { key: scenario.skillKey, runtimeName: scenario.skillKey, versionId: "version", markdown: `Write ${scenario.marker}` },
|
|
skillRequestText: `${scenario.prompt}\nUse /${scenario.skillKey}`,
|
|
};
|
|
const final: ContextIntegrityCheckpoint = {
|
|
...initial, phase: "final", issue: { ...initial.issue, status: "done" },
|
|
documents: [{ key: "report", body: `Requested ${scenario.marker}`, latestRevisionId: "revision", latestRevisionNumber: 1 }],
|
|
comments: [{ authorAgentId: "agent", body: "Saved [report](/DOC/issues/DOC-1#document-report)." }],
|
|
runs: [{ id: "run", status: "succeeded" }],
|
|
};
|
|
return { scenario, initial, final };
|
|
}
|
|
|
|
describe("explicit Paperclip document delivery", () => {
|
|
it("checks the actual saved content, revision and exact document link", () => {
|
|
const { scenario, initial, final } = recording();
|
|
expect(gradeContextIntegrity({ ...scenario, checkpoints: [initial, final] }).every(row => row.passed)).toBe(true);
|
|
expect(scenario.prompt).toContain("Paperclip document");
|
|
expect(scenario.prompt).toContain("clickable Markdown link");
|
|
expect(scenario.prompt).toContain("Paperclip task UI");
|
|
expect(scenario.prompt).not.toContain(scenario.marker);
|
|
});
|
|
it.each(["https://paperclip.example/DOC/issues/DOC-1#document-report", "<https://paperclip.example/DOC/issues/DOC-1#document-report>"])("accepts the same-app document link %s", target => {
|
|
const { scenario, initial, final } = recording();
|
|
final.comments[0]!.body = `Saved [report](${target}).`;
|
|
expect(gradeContextIntegrity({ ...scenario, checkpoints: [initial, final] }).every(row => row.passed)).toBe(true);
|
|
});
|
|
it.each(["local-only", "missing-revision", "wrong-content", "local-link", "wrong-document-link", "other-app-link", "user-link-only", "bare-path", "code-formatted-path"])(
|
|
"rejects plausible %s delivery", variant => {
|
|
const { scenario, initial, final } = recording();
|
|
if (variant === "local-only") { final.documents = []; final.comments[0]!.body = "Saved report.md in the workspace."; }
|
|
if (variant === "missing-revision") final.documents[0]!.latestRevisionId = null;
|
|
if (variant === "wrong-content") final.documents[0]!.body = "A different report";
|
|
if (variant === "bare-path") final.comments[0]!.body = "Saved /DOC/issues/DOC-1#document-report.";
|
|
if (variant === "code-formatted-path") final.comments[0]!.body = "Saved `/DOC/issues/DOC-1#document-report`.";
|
|
if (variant === "local-link") final.comments[0]!.body = "Saved [report](./report.md).";
|
|
if (variant === "wrong-document-link") final.comments[0]!.body = "Saved [report](/DOC/issues/DOC-1#document-other).";
|
|
if (variant === "other-app-link") final.comments[0]!.body = "Saved [report](https://other.example/DOC/issues/DOC-1#document-report).";
|
|
if (variant === "user-link-only") { delete final.comments[0]!.authorAgentId; final.comments[0]!.authorUserId = "user"; }
|
|
expect(gradeContextIntegrity({ ...scenario, checkpoints: [initial, final] }).some(row => !row.passed)).toBe(true);
|
|
},
|
|
);
|
|
it("preserves the original ambiguous request and assigned output procedure", () => {
|
|
expect(contextIntegrityScenario("assigned-skill-explicit-invocation", "probe").prompt).toBe(
|
|
"Use the assigned Context integrity output skill for this task, then follow its instructions and finish the task. Do not create child tasks or unrelated deliverables.",
|
|
);
|
|
});
|
|
});
|