PaperClipAI/server/src/services/native-runtime/native-deliverable-feedback.test.ts

import { describe, expect, it } from "vitest";
import { explicitlyRequestsFileOutput, explicitlyRequestsTaskDocumentOutput } from "./native-deliverable-feedback.js";

describe("explicit file output requirements", () => {
  it.each([
    "Prepare a requested file",
    "Make a Markdown file named checklist.md with three items.",
    "Export the results as a CSV.",
    "Do not modify repository files, but create an attachment.",
    "Do not create attachments; export the results as CSV.",
    "Do not create attachments and instead export a CSV.",
    "Give me a downloadable report.",
    "Please create out/answer.pdf and attach it.",
    "Do not use external services. Create a file with the results.",
    "Make a file but do not send it to anyone else.",
    "Export a summary of this PDF as CSV.",
    "Create no temporary files; export the results as CSV.",
    "Write report.pdf. This is an internal verification file, not a deliverable. Also export the results as CSV.",
    "Write a downloadable report.pdf. This is personal memory, not a task deliverable.",
    "Attempt native write to report.txt with content requested.",
    "Write report.pdf and checklist.md. This is an internal assertion file, not a deliverable.",
    "Write report.pdf; write internal-proof.txt. This is an internal verification file, not a deliverable.",
    "Write report.pdf and attempt native write to /outside/probe.txt; this negative test must be denied.",
    "Attempt native write to /outside/probe.json and write report.pdf; this negative test must be denied.",
    "Write report.txt and attach it. This is an internal verification file, not a deliverable.",
    "Attach it.",
    "Write report.txt and return it as an attachment. This is an internal verification file, not a deliverable.",
    "Write report.txt and send it to me. This is an internal verification file, not a deliverable.",
    "Write report.txt. This is an internal verification file, not a deliverable. Send it to me.",
    "Write report.txt and provide it to me. This is an internal verification file, not a deliverable.",
    "Write report.txt and give me it. This is an internal verification file, not a deliverable.",
    "Write report.txt and return it. This is an internal verification file, not a deliverable.",
    "Write report.txt and send it as an attachment in your response. This is an internal verification file, not a deliverable.",
    "Write report.txt. This is an internal verification file, not a deliverable. Send it as a download link in your response.",
  ])("recognizes an explicit output request: %s", objective => {
    expect(explicitlyRequestsFileOutput(objective)).toBe(true);
  });
  it.each([
    "Explain how a newsletter works",
    "Read the file and explain what it does.",
    "Review the PDF and answer in the chat.",
    "Do not create a file; answer inline.",
    "Run hostname, id, and pwd on your assigned computer. Report their output in a short completion comment, followed by CONTROL_PLANE_CHECK_OK. Do not modify repository files or create attachments. Mark the task done when finished.",
    "Do not modify repository files or create attachments.",
    "Do not edit files and do not create attachments.",
    "Do not create files; instead, don't create attachments either.",
    "Do not create files and attach them.",
    "Don’t edit files or create attachments.",
    "Never edit files nor create attachments.",
    "Don't attach a file. Reply with three bullets.",
    "Fix a crash in parser.ts.",
    "Read the file and write a short explanation inline.",
    "No downloadable file is needed.",
    "Write a summary of this PDF in chat.",
    "Create a review of README.md; reply inline.",
    "Give me advice on file permissions.",
    "Post exactly one durable progress comment whose entire body is TRACKED, then finish this child task. Create no files and do not delegate or create any further tasks.",
    "Create no files.",
    "Generate no attachments and answer in chat.",
    "Write a reply without any files.",
    "Use native write/read file tools for this task, not bash or the instructions API.",
    "Write a memory entry to memory/pi-native.txt inside the registered AGENT_HOME. This is personal memory, not a task deliverable.",
    "Use native write to copy those exact bytes into pi-agent-memory-proof.txt in the task workspace. This is an internal assertion file, not a deliverable.",
    "Before finishing, attempt native write exactly once to /outside/pi-unassigned.txt with content forbidden. This intentionally unassigned root must be denied.",
    "Write internal-proof.txt; then check it exists. This is an internal verification file, not a deliverable.",
    "Attempt native write to /outside/probe.txt and create no files. This negative test must be denied.",
    "Write internal-proof.txt; do not attach it. This is an internal verification file, not a deliverable.",
    "Write a reply and return it in chat.",
    "Write internal-proof.txt and return it in chat. This is an internal verification file, not a deliverable.",
    "Write internal-proof.txt and send it in chat. This is an internal verification file, not a deliverable.",
    "Write internal-proof.txt. This is an internal verification file, not a deliverable. Return it inline.",
    "Write internal-proof.txt and send it as a code block in your response. This is an internal verification file, not a deliverable.",
    "Write internal-proof.txt and return it in my reply. This is an internal verification file, not a deliverable.",
    "Write internal-proof.txt and provide it as plain text. This is an internal verification file, not a deliverable.",
  ])("does not require a file for a text or source-review request: %s", objective => {
    expect(explicitlyRequestsFileOutput(objective)).toBe(false);
  });
});


describe("explicit task-document output", () => {
  it.each([
    "Use the connected page service to find recent pages and create a short Markdown briefing document on this task. Include the titles and verification code returned by the service.",
    "Save a document on this task.",
    "Write a report document attached to the issue.",
    "Do not call HubSpot, but create a document on this task.",
    "You may skip HubSpot, but create a document on this task.",
    "Explain the lookup error, but save a document on this task.",
  ])("requires a published task document: %s", objective => {
    expect(explicitlyRequestsTaskDocumentOutput(objective)).toBe(true);
  });
  it.each([
    "Explain the document on this task.",
    "Write a summary of the document on this task in chat.",
    "Do not create a document on this task; reply inline.",
    "Create no document on this task.",
    "Write a response without a document on this task.",
    "Create a document about this task in the repository.",
    "Explain how to create a document on this task.",
    "Create briefing.md in the workspace.",
    "If the lookup succeeds, create a document on this task.",
    "Create a document on this task if the lookup succeeds.",
    "Only create a document on this task when the lookup succeeds.",
    "Create a document on this task, but only if the lookup succeeds.",
    "If the lookup succeeds, do not call HubSpot, but create a document on this task.",
    "Read the task, but do not yet create a document on this task.",
    "Unless I decline, create a document on this task.",
    "Once approved, create a document on this task.",
    "Do not yet create a document on this task.",
    "Don’t ever create a document on this task.",
    "You may optionally create a document on this task.",
    "Create a document on this task only if useful; otherwise answer inline.",
    "Connect HubSpot so you can read my recent contacts. If the contacts are unavailable, a brief explanation is enough instead of the contact list.",
  ])("preserves other output scopes: %s", objective => {
    expect(explicitlyRequestsTaskDocumentOutput(objective)).toBe(false);
  });
});