Require publication for explicit JSON file outputs (#18228)

Extend the native file evidence rule to explicit JSON creation and publication requests, with conservative input and destination handling and a real PostgreSQL completion regression.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

doc/AGENT-ARTIFACTS.md

@@ -27,6 +27,14 @@ remote workspace. Supply a workspace-relative `contentRef`, basename `filename`,
 The tool verifies the file, stores an attachment and artifact work product, and
 binds it to the response. Generic API tools and a legacy API key are unnecessary.
 
+Named JSON file outputs and explicit publication requests require the same
+verified delivery evidence as other file outputs. A saved summary document does
+not replace a requested downloadable JSON file. Reading JSON inputs or discussing
+their contents inline does not by itself require a file attachment.
+JSON output intent is recognized from a direct filename or an explicit output
+destination. Ambiguous descriptions of input files joined by `to` remain text
+requests; use `as`, `into`, or an explicit output file when an artifact is required.
+
 The completion gate permits a narrow sentence-bound exception for one requested
 file when the following sentence explicitly qualifies it, for example:
 `Write memory.txt. This is personal memory, not a task deliverable.` or

server/src/services/native-runtime/native-completion-feedback.test.ts

@@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto";
 import { eq } from "drizzle-orm";
 import { afterAll, beforeAll, describe, expect, it } from "vitest";
 import { agents, companies, createDb, documentRevisions, executionWorkspaces, executionWorkspaceRepositories, heartbeatRuns, issues, issueThreadInteractions, issueTreeHolds, issueDocuments } from "@paperclipai/db";
-import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
+import { startRecoveryPostgres } from "../../__tests__/helpers/recovery-postgres.js";
 import type { PrpStructuredRunResult } from "../../vendor/paperclip-runner/index.js";
 import { documentService } from "../documents.js";
 import { nativeCompletionFeedback } from "./native-completion-feedback.js";
@@ -15,9 +15,9 @@ const done: PrpStructuredRunResult = {
 };
 
 describe("native final-response feedback", () => {
-  let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
+  let temporary: Awaited<ReturnType<typeof startRecoveryPostgres>>;
   let db: ReturnType<typeof createDb>;
-  beforeAll(async () => { temporary = await startEmbeddedPostgresTestDatabase("native-final-response-"); db = createDb(temporary.connectionString); });
+  beforeAll(async () => { temporary = await startRecoveryPostgres("native-final-response-"); db = createDb(temporary.connectionString); });
   afterAll(async () => { await temporary?.cleanup(); });
   async function fixture() {
     const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID();
@@ -129,6 +129,19 @@ describe("native final-response feedback", () => {
     } });
     await expect(nativeCompletionFeedback(db, value.runId, done)).resolves.toContain("Saved document");
   });
+  it("requires a requested JSON file even when the child task already has a saved result document", async () => {
+    const value = await fixture();
+    const objective = [
+      "Decode the supplied input into `input.json` in your workspace, then compute `proof.json`.",
+      "Requirements:",
+      "- Write `input.json` from the exact input bytes.",
+      "- Save a document on this child task describing the result and file link.",
+      "- Publish `proof.json` on the child task with `register_deliverable` using the workspace-relative path, correct byte size, and correct SHA-256.",
+    ].join("\n");
+    await db.update(issues).set({ title: "Compute the proof", description: objective }).where(eq(issues.id, value.issueId));
+    const report = { ...done, summary: "Computed the JSON and saved its task document.", evidence: [{ ref: value.saved.documentHref }] };
+    await expect(nativeCompletionFeedback(db, value.runId, report)).rejects.toThrow("register_deliverable");
+  });
   it("accepts a still-published document from an earlier run after a response continuation", async () => {
     const value = await fixture(), continuationRunId = randomUUID();
     const objective = "Save a document on this task.";

server/src/services/native-runtime/native-deliverable-feedback.test.ts

@@ -5,6 +5,29 @@ describe("explicit file output requirements", () => {
   it.each([
     "Prepare a requested file",
     "Make a Markdown file named checklist.md with three items.",
+    "Write connection-proof.json with the computed result.",
+    "Write a new result.json.",
+    "Write a detailed result.json.",
+    "Write a summary of input.json as output.json.",
+    "Write a summary of input.json to a file named output.json.",
+    "Write result.json.md inline in your response.",
+    "Write the verified report named result.json.",
+    "Write the result to reports/output.json.",
+    "Write result.json with the contents of the verified report.",
+    "Create report.pdf and include the JSON inline in chat.",
+    "Create a PDF report and include the JSON inline in chat.",
+    "Write result.json inline and attach it as a download.",
+    "Write result.json inline and include a download link.",
+    "Write result.json and describe it inline in chat.",
+    "Write notes-and-results.json and describe it inline in chat.",
+    "Write result.json and explain the result inline in chat.",
+    "Write result.json and summarize it inline in your response.",
+    "Write the result to `output.json`.",
+    "Write result.json inline and publish it.",
+    "Write result.json inline. Publish it on this task.",
+    "Write result.json and update package.json.",
+    "Publish connection-proof.json on this child task with register_deliverable using the workspace-relative path, correct byte size, and correct SHA-256.",
+    "Write internal-proof.json. This is an internal verification file, not a deliverable. Publish it on this task.",
     "Export the results as a CSV.",
     "Do not modify repository files, but create an attachment.",
     "Do not create attachments; export the results as CSV.",
@@ -49,6 +72,36 @@ describe("explicit file output requirements", () => {
     "Never edit files nor create attachments.",
     "Don't attach a file. Reply with three bullets.",
     "Fix a crash in parser.ts.",
+    "Update config.json to fix the parser.",
+    "Write the fix and update package.json.",
+    "Write the migration then edit package-lock.json.",
+    "Write a summary and read result.json.",
+    "Write a function that reads config.json.",
+    "Write code to parse tsconfig.json.",
+    "Write the route and wire it in routes.json.",
+    "Write a function reading config.json.",
+    "Write a short summary after reading input.json.",
+    "Write a short summary explaining config.json.",
+    "Write a short summary of changes to config.json.",
+    "Write a summary of the file named input.json.",
+    "Write about config.json.",
+    "Write concerning config.json.",
+    "Write a comparison of input.json to baseline.json.",
+    "Write a summary of input.json to output.json.",
+    "Write a wrapper around config.json.",
+    "Write tests covering package.json.",
+    "Write a migration touching package-lock.json.",
+    "Return the answer inline with no download link.",
+    "Return the results inline without a download link.",
+    "Return the answer without a download link.",
+    "Write result.json inline in your response as a code block; do not create any files.",
+    "Write notes-and-results.json inline in your response.",
+    "Return result.json as a code block in the reply.",
+    "Read connection-input.json and explain its contents in chat.",
+    "Write a summary of connection-input.json inline in your response.",
+    "Write the contents of connection-proof.json inline in your response.",
+    "Do not publish connection-proof.json; reply inline.",
+    "Write internal-proof.json. This is an internal verification file, not a deliverable. Do not publish it.",
     "Read the file and write a short explanation inline.",
     "No downloadable file is needed.",
     "Write a summary of this PDF in chat.",

server/src/services/native-runtime/native-deliverable-feedback.ts

@@ -82,9 +82,9 @@ export function explicitlyRequestsFileOutput(objective: string): boolean {
   const sentences = objective.replaceAll("\\_", "_").split(/(?:[.!?](?:\s|$)|\n)/iu);
   let precedingFileOutput = false;
   return sentences.some((sentence, index) => {
-    const file = /\b(?:files?|attachments?|downloads?|pdf|spreadsheets?|workbooks?|slide decks?|powerpoints?|docx|xlsx|csv)\b|\b[^\s/]+\.(?:md|txt|pdf|docx?|xlsx?|csv|pptx?|png|jpe?g|svg|zip)\b/giu;
+    const file = /\b(?:files?|attachments?|downloads?|pdf|spreadsheets?|workbooks?|slide decks?|powerpoints?|docx|xlsx|csv)\b|\b[^\s/]+\.(?:md|txt|json|pdf|docx?|xlsx?|csv|pptx?|png|jpe?g|svg|zip)\b/giu;
     const outputs = sentence.split(/(?:[;,]|\bbut\b)/iu).flatMap(clause => {
-      const creates = [...clause.matchAll(/\b(?:create|make|write|save|export|attach|send|generate|produce|prepare|provide|give|return|build)\b/giu)];
+      const creates = [...clause.matchAll(/\b(?:create|make|write|save|export|attach|send|generate|produce|prepare|provide|give|return|build|publish)\b/giu)];
       return creates.flatMap((create, createIndex) => {
         const before = clause.slice(0, create.index);
         if (/\b(?:do not|don['’]t|never|no need to)\s*$/iu.test(before)) return [];
@@ -96,33 +96,46 @@ export function explicitlyRequestsFileOutput(objective: string): boolean {
         // Bind each object to its own verb. A denied write or "create no files"
         // cannot suppress a separate requested report in this same clause.
         const output = clause.slice(create.index + create[0].length, creates[createIndex + 1]?.index);
+        // An explicit attachment/export request can refer to the file by
+        // pronoun. It still requires publication when its name is omitted.
+        const referencesOutput = /^(?:attach|export|send|provide|give|return|publish)$/iu.test(create[0])
+          && /^\s+(?:me\s+)?(?:it|them|this|that)\b/iu.test(output);
+        const referencedAttachment = referencesOutput && /^(?:attach|export)$/iu.test(create[0]);
+        const inlineDestination = /\b(?:inline|(?:in|within|inside|as|into)\s+(?:(?:a|an|the|my|your|our|final|plain|markdown|chat|fenced)\s+)*(?:chat|response|reply|message|comment|text|code block))\b/iu;
+        const inline = inlineDestination.test(output) || /\b(?:its|the) contents\b/iu.test(output);
+        const downloadable = referencedAttachment || (!/\b(?:no|without)\s+(?:downloadable|attached)/iu.test(output)
+          && /\b(?:downloadable|attached)\s+(?:file|report|document|checklist|draft)\b/iu.test(output));
+        const inlineOnly = inlineDestination.test(output.split(/\s+(?:and|then)\s+/iu, 1)[0]) && !downloadable && !/^(?:attach|export|publish)$/iu.test(create[0])
+          && !/\bas (?:an? )?attachment\b/iu.test(output);
         const objects = [...output.matchAll(file)].filter(match => {
           const prefix = output.slice(0, match.index);
           const suffix = output.slice(match.index + match[0].length);
           // "Create no files" is a prohibition, even though it contains a creation
           // verb. Negate this object only; another explicit output can still count.
-          if (/\b(?:no|zero|without(?:\s+any)?)\s+(?:(?:new|temporary|downloadable|attached|additional)\s+)*$/iu.test(prefix)) return false;
+          if (/\b(?:no|zero|without(?:\s+(?:any|a|an))?)\s+(?:(?:new|temporary|downloadable|attached|additional)\s+)*$/iu.test(prefix)) return false;
           // "Write a summary of this PDF" names input, not a requested file.
           // Explicit export destinations still count after such input references.
-          const destination = /\b(?:as|into|to)\s+(?:(?:a|an|the|new|separate|markdown|word|excel)\s+)*$/iu.test(prefix);
-          if (!destination && /\b(?:of|about|on|from|using|for|with)\b/iu.test(prefix)) return false;
+          const json = /\.json$/iu.test(match[0]);
+          const inputReference = /\b(?:of|about|on|from|using|for|with)\b/iu;
+          const destinationMatch = (json
+            ? /\b(as|into|to|named|called)\s+(?:(?:a|an|the|new|separate|markdown|word|excel)\s+)*["'`*]*(?:[^\s]*\/)?$/iu
+            : /\b(?:as|into|to)\s+(?:(?:a|an|the|new|separate|markdown|word|excel)\s+)*$/iu).exec(prefix);
+          const beforeDestination = prefix.slice(0, destinationMatch?.index ?? 0);
+          const destination = destinationMatch !== null && (!json || !/^(?:to|named|called)$/iu.test(destinationMatch[1])
+            || !inputReference.test(beforeDestination));
+          if (json) {
+            const directObject = /^\s*(?:(?:a|an|the|my|your|our|their|his|her|its|this|that)\s+(?:(?:new|separate|downloadable|attached)\s+)*([\p{L}\p{N}_-]+\s+)?|(?:(?:new|separate|downloadable|attached)\s+)*)["'`*]*(?:[^\s]*\/)?$/iu.exec(prefix);
+            if (!destination && (!directObject || inputReference.test(directObject[1] ?? ""))) return false;
+          } else if (!destination && inputReference.test(prefix)) return false;
           if (/^files?$/iu.test(match[0]) && /^\s+(?:tools?|permissions?|systems?|formats?|names?|paths?|types?|sizes?|descriptors?)\b/iu.test(suffix)) return false;
           return true;
         });
-        // An explicit attachment/export request can refer to the file by
-        // pronoun. It still requires publication when its name is omitted.
-        const referencesOutput = /^(?:attach|export|send|provide|give|return)$/iu.test(create[0])
-          && /^\s+(?:me\s+)?(?:it|them|this|that)\b/iu.test(output);
-        const referencedAttachment = referencesOutput && /^(?:attach|export)$/iu.test(create[0]);
-        const inline = /\b(?:inline|(?:in|within|inside|as|into)\s+(?:(?:a|an|the|my|your|our|final|plain|markdown|chat|fenced)\s+)*(?:chat|response|reply|message|comment|text|code block)|(?:its|the) contents)\b/iu.test(output);
-        const downloadable = referencedAttachment || (!/\b(?:no|without)\s+(?:downloadable|attached)/iu.test(output)
-          && /\b(?:downloadable|attached)\s+(?:file|report|document|checklist|draft)\b/iu.test(output));
-        const publication = /\b(?:downloadable|attached|attach|export|send|provide|return|give)\b/iu.test(create[0] + output);
+        const publication = /\b(?:downloadable|attached|attach|export|send|provide|return|give|publish)\b/iu.test(create[0] + output);
         const deniedAttempt = create[0].toLowerCase() === "write" && objects.length === 1
           && /\b(?:attempt|try)\s+(?:the\s+)?native\s*$/iu.test(before)
           && /\b(?:must be denied|denial is (?:the )?expected|(?:this|the) negative test)\b/iu.test(objective);
         if (objects.length === 0 && !downloadable && /^\s+(?:no|zero|without)\b/iu.test(output)) return [];
-        return [{ objects: objects.length, downloadable, publication, deniedAttempt,
+        return [{ namesFile: objects.length > 0, objects: objects.filter(match => !inlineOnly || !/\.json$/iu.test(match[0])).length, downloadable, publication, deniedAttempt,
           referencesOutput, publicationReference: referencesOutput && !inline }];
       });
     });
@@ -133,7 +146,7 @@ export function explicitlyRequestsFileOutput(objective: string): boolean {
       && /^\s*This is (?:an? )?(?:personal memory|internal (?:assertion|verification) file)\b[^.!?]*\bnot a (?:task )?deliverable\b/iu.test(sentences[index + 1] ?? "");
     return outputs.some(output => {
       const publicationReference = output.publicationReference && precedingFileOutput;
-      if (!output.referencesOutput) precedingFileOutput = output.objects > 0;
+      if (!output.referencesOutput) precedingFileOutput = output.namesFile;
       return (output.objects > 0 || output.downloadable || publicationReference)
         && (output.publication || (!internal && !output.deniedAttempt));
     });