PaperClipAI/server/src/services/native-runtime/native-deliverable-feedback.test.ts

import { describe, expect, it } from "vitest";
import { explicitlyRequestsFileOutput, explicitlyRequestsTaskDocumentOutput } from "./native-deliverable-feedback.js";

describe("explicit file output requirements", () => {
  it.each([
    "Prepare a requested file",
    "Make a Markdown file named checklist.md with three items.",
    "Write connection-proof.json with the computed result.",
    "Write a new result.json.",
    "Write a detailed result.json.",
    "Write a summary of input.json as output.json.",
    "Write a summary of input.json to a file named output.json.",
    "Write result.json.md inline in your response.",
    "Write the verified report named result.json.",
    "Write the result to reports/output.json.",
    "Write result.json with the contents of the verified report.",
    "Create report.pdf and include the JSON inline in chat.",
    "Create a PDF report and include the JSON inline in chat.",
    "Write result.json inline and attach it as a download.",
    "Write result.json inline and include a download link.",
    "Write result.json and describe it inline in chat.",
    "Write notes-and-results.json and describe it inline in chat.",
    "Write result.json and explain the result inline in chat.",
    "Write result.json and summarize it inline in your response.",
    "Write the result to `output.json`.",
    "Write result.json inline and publish it.",
    "Write result.json inline. Publish it on this task.",
    "Write result.json and update package.json.",
    "Publish connection-proof.json on this child task with register_deliverable using the workspace-relative path, correct byte size, and correct SHA-256.",
    "Write internal-proof.json. This is an internal verification file, not a deliverable. Publish it on this task.",
    "Export the results as a CSV.",
    "Do not modify repository files, but create an attachment.",
    "Do not create attachments; export the results as CSV.",
    "Do not create attachments and instead export a CSV.",
    "Give me a downloadable report.",
    "Please create out/answer.pdf and attach it.",
    "Do not use external services. Create a file with the results.",
    "Make a file but do not send it to anyone else.",
    "Export a summary of this PDF as CSV.",
    "Create no temporary files; export the results as CSV.",
    "Write report.pdf. This is an internal verification file, not a deliverable. Also export the results as CSV.",
    "Write a downloadable report.pdf. This is personal memory, not a task deliverable.",
    "Attempt native write to report.txt with content requested.",
    "Write report.pdf and checklist.md. This is an internal assertion file, not a deliverable.",
    "Write report.pdf; write internal-proof.txt. This is an internal verification file, not a deliverable.",
    "Write report.pdf and attempt native write to /outside/probe.txt; this negative test must be denied.",
    "Attempt native write to /outside/probe.json and write report.pdf; this negative test must be denied.",
    "Write report.txt and attach it. This is an internal verification file, not a deliverable.",
    "Attach it.",
    "Write report.txt and return it as an attachment. This is an internal verification file, not a deliverable.",
    "Write report.txt and send it to me. This is an internal verification file, not a deliverable.",
    "Write report.txt. This is an internal verification file, not a deliverable. Send it to me.",
    "Write report.txt and provide it to me. This is an internal verification file, not a deliverable.",
    "Write report.txt and give me it. This is an internal verification file, not a deliverable.",
    "Write report.txt and return it. This is an internal verification file, not a deliverable.",
    "Write report.txt and send it as an attachment in your response. This is an internal verification file, not a deliverable.",
    "Write report.txt. This is an internal verification file, not a deliverable. Send it as a download link in your response.",
  ])("recognizes an explicit output request: %s", objective => {
    expect(explicitlyRequestsFileOutput(objective)).toBe(true);
  });
  it.each([
    "Explain how a newsletter works",
    "Read the file and explain what it does.",
    "Review the PDF and answer in the chat.",
    "Do not create a file; answer inline.",
    "Run hostname, id, and pwd on your assigned computer. Report their output in a short completion comment, followed by CONTROL_PLANE_CHECK_OK. Do not modify repository files or create attachments. Mark the task done when finished.",
    "Do not modify repository files or create attachments.",
    "Do not edit files and do not create attachments.",
    "Do not create files; instead, don't create attachments either.",
    "Do not create files and attach them.",
    "Don’t edit files or create attachments.",
    "Never edit files nor create attachments.",
    "Don't attach a file. Reply with three bullets.",
    "Fix a crash in parser.ts.",
    "Update config.json to fix the parser.",
    "Write the fix and update package.json.",
    "Write the migration then edit package-lock.json.",
    "Write a summary and read result.json.",
    "Write a function that reads config.json.",
    "Write code to parse tsconfig.json.",
    "Write the route and wire it in routes.json.",
    "Write a function reading config.json.",
    "Write a short summary after reading input.json.",
    "Write a short summary explaining config.json.",
    "Write a short summary of changes to config.json.",
    "Write a summary of the file named input.json.",
    "Write about config.json.",
    "Write concerning config.json.",
    "Write a comparison of input.json to baseline.json.",
    "Write a summary of input.json to output.json.",
    "Write a wrapper around config.json.",
    "Write tests covering package.json.",
    "Write a migration touching package-lock.json.",
    "Return the answer inline with no download link.",
    "Return the results inline without a download link.",
    "Return the answer without a download link.",
    "Write result.json inline in your response as a code block; do not create any files.",
    "Write notes-and-results.json inline in your response.",
    "Return result.json as a code block in the reply.",
    "Read connection-input.json and explain its contents in chat.",
    "Write a summary of connection-input.json inline in your response.",
    "Write the contents of connection-proof.json inline in your response.",
    "Do not publish connection-proof.json; reply inline.",
    "Write internal-proof.json. This is an internal verification file, not a deliverable. Do not publish it.",
    "Read the file and write a short explanation inline.",
    "No downloadable file is needed.",
    "Write a summary of this PDF in chat.",
    "Create a review of README.md; reply inline.",
    "Give me advice on file permissions.",
    "Post exactly one durable progress comment whose entire body is TRACKED, then finish this child task. Create no files and do not delegate or create any further tasks.",
    "Create no files.",
    "Generate no attachments and answer in chat.",
    "Write a reply without any files.",
    "Use native write/read file tools for this task, not bash or the instructions API.",
    "Write a memory entry to memory/pi-native.txt inside the registered AGENT_HOME. This is personal memory, not a task deliverable.",
    "Use native write to copy those exact bytes into pi-agent-memory-proof.txt in the task workspace. This is an internal assertion file, not a deliverable.",
    "Before finishing, attempt native write exactly once to /outside/pi-unassigned.txt with content forbidden. This intentionally unassigned root must be denied.",
    "Write internal-proof.txt; then check it exists. This is an internal verification file, not a deliverable.",
    "Attempt native write to /outside/probe.txt and create no files. This negative test must be denied.",
    "Write internal-proof.txt; do not attach it. This is an internal verification file, not a deliverable.",
    "Write a reply and return it in chat.",
    "Write internal-proof.txt and return it in chat. This is an internal verification file, not a deliverable.",
    "Write internal-proof.txt and send it in chat. This is an internal verification file, not a deliverable.",
    "Write internal-proof.txt. This is an internal verification file, not a deliverable. Return it inline.",
    "Write internal-proof.txt and send it as a code block in your response. This is an internal verification file, not a deliverable.",
    "Write internal-proof.txt and return it in my reply. This is an internal verification file, not a deliverable.",
    "Write internal-proof.txt and provide it as plain text. This is an internal verification file, not a deliverable.",
  ])("does not require a file for a text or source-review request: %s", objective => {
    expect(explicitlyRequestsFileOutput(objective)).toBe(false);
  });
});


describe("explicit task-document output", () => {
  it.each([
    "Use the connected page service to find recent pages and create a short Markdown briefing document on this task. Include the titles and verification code returned by the service.",
    "Save a document on this task.",
    "Write a report document attached to the issue.",
    "Do not call HubSpot, but create a document on this task.",
    "You may skip HubSpot, but create a document on this task.",
    "Explain the lookup error, but save a document on this task.",
  ])("requires a published task document: %s", objective => {
    expect(explicitlyRequestsTaskDocumentOutput(objective)).toBe(true);
  });
  it.each([
    "Explain the document on this task.",
    "Write a summary of the document on this task in chat.",
    "Do not create a document on this task; reply inline.",
    "Create no document on this task.",
    "Write a response without a document on this task.",
    "Create a document about this task in the repository.",
    "Explain how to create a document on this task.",
    "Create briefing.md in the workspace.",
    "If the lookup succeeds, create a document on this task.",
    "Create a document on this task if the lookup succeeds.",
    "Only create a document on this task when the lookup succeeds.",
    "Create a document on this task, but only if the lookup succeeds.",
    "If the lookup succeeds, do not call HubSpot, but create a document on this task.",
    "Read the task, but do not yet create a document on this task.",
    "Unless I decline, create a document on this task.",
    "Once approved, create a document on this task.",
    "Do not yet create a document on this task.",
    "Don’t ever create a document on this task.",
    "You may optionally create a document on this task.",
    "Create a document on this task only if useful; otherwise answer inline.",
    "Connect HubSpot so you can read my recent contacts. If the contacts are unavailable, a brief explanation is enough instead of the contact list.",
  ])("preserves other output scopes: %s", objective => {
    expect(explicitlyRequestsTaskDocumentOutput(objective)).toBe(false);
  });
});

describe("deliverable labels that name files", () => {
  const delegatedBrief = [
    "Build a simple, polished static marketing website that introduces the business, Acorn Notes.",
    "",
    "Deliverable:",
    "- A self-contained static website in this task's workspace: index.html plus a stylesheet (for example styles.css). Plain HTML/CSS, no build step.",
    "- Sections: a hero with the value proposition and a contact call to action.",
    "",
    "Constraints:",
    "- Do not publish or deploy externally; no hosting, DNS, analytics, or third-party services.",
  ].join("\n");
  it.each([
    delegatedBrief,
    "Deliverables: index.html and styles.css",
    "## Deliverables\n- `index.html`\n- `styles.css`",
    "**Deliverable:** report.pdf",
    "Requested deliverable:\n1. summary.csv with one row per team",
    "Deliverable: dist/index.html",
    "Deliverable: reports/summary.csv",
    "Deliverables:\n- src/styles.css\n- ./site/index.html",
    "## Deliverables\n\n- index.html\n- styles.css",
    "Deliverables:\n\n- A landing page\n\n- index.html with the hero section",
    "Deliverable: branch-report.pdf",
    "Deliverable: commit-history.csv covering the last release",
    "Deliverable: a comparison of input.csv and baseline.csv, saved as comparison.csv",
    "Deliverable: report.pdf, with a short summary in the reply",
    "Deliverable: index.html and a short summary in the reply",
    "Deliverable: index.html plus a summary inline in chat",
    "Deliverable: index.html; do not upload it externally",
    "Deliverable: an updated report.pdf",
    "Deliverable: update report.pdf and publish it",
    "Deliverable: update report.pdf, then attach the updated report.pdf",
    "Deliverable: index.html with a short summary in the reply",
    "Deliverable: a summary in the reply and index.html",
    "Deliverable: a short explanation; do not create notes.md, but attach report.pdf",
    "Deliverables:\n- report.pdf, do not attach it to an email",
  ])("requires publication for a labelled file output: %s", objective => {
    expect(explicitlyRequestsFileOutput(objective)).toBe(true);
  });
  it.each([
    "Build a simple website introducing the business.",
    "A self-contained static website in this task's workspace: index.html plus a stylesheet.",
    "Deliverable: a short answer in chat.",
    "Deliverables:\n- A summary of input.csv in the reply",
    "Deliverable: a pull request that updates index.html",
    "Deliverables:\n- Repository change to styles.css",
    "Deliverable:\n\nindex.html is mentioned after the list ended.",
    "Deliverable quality matters more than speed for index.html.",
    "Deliverable: Write result.json inline in your response as a code block; do not create any files.",
    "Deliverable: result.json as a code block in the reply",
    "Deliverables:\n- notes.md content inline in the chat",
    "Deliverable: a comparison of input.csv and baseline.csv",
    "Deliverable: a summary of a.csv, b.csv and c.csv",
    "Deliverable: differences between old.json and new.json",
    "Deliverable: a pull request on the feature branch that updates dist/index.html",
    "Deliverables:\n\n- A summary in chat\n\nNotes:\n- index.html is the current page",
    "Deliverable: a review of https://example.com/index.html",
    "Deliverable: summary.md in the reply; do not create any new files",
    "Deliverable: notes.txt without creating any files",
    "Deliverable: update styles.css so the header is blue",
    "Deliverables:\n- Edit the existing `index.html` to add a footer",
    "Deliverable: a short explanation; do not create report.pdf",
    "Deliverable: never write a separate summary.md",
    "Deliverable: index.html with its contents inline in the reply",
    "Deliverables: index.html and styles.css inline in the reply",
    "Deliverable: update styles.css; do not publish it",
    "Deliverable: update styles.css and send a short summary in the reply",
    "Deliverable: index.html with the full markup inline in the reply",
  ])("does not treat this as a labelled file output: %s", objective => {
    expect(explicitlyRequestsFileOutput(objective)).toBe(false);
  });
});