diff --git a/doc/AGENT-ARTIFACTS.md b/doc/AGENT-ARTIFACTS.md index 2136fc5ca95..e380f34c3c5 100644 --- a/doc/AGENT-ARTIFACTS.md +++ b/doc/AGENT-ARTIFACTS.md @@ -27,6 +27,14 @@ remote workspace. Supply a workspace-relative `contentRef`, basename `filename`, The tool verifies the file, stores an attachment and artifact work product, and binds it to the response. Generic API tools and a legacy API key are unnecessary. +Named JSON file outputs and explicit publication requests require the same +verified delivery evidence as other file outputs. A saved summary document does +not replace a requested downloadable JSON file. Reading JSON inputs or discussing +their contents inline does not by itself require a file attachment. +JSON output intent is recognized from a direct filename or an explicit output +destination. Ambiguous descriptions of input files joined by `to` remain text +requests; use `as`, `into`, or an explicit output file when an artifact is required. + The completion gate permits a narrow sentence-bound exception for one requested file when the following sentence explicitly qualifies it, for example: `Write memory.txt. This is personal memory, not a task deliverable.` or diff --git a/server/src/services/native-runtime/native-completion-feedback.test.ts b/server/src/services/native-runtime/native-completion-feedback.test.ts index 04452de5cf9..df586909b15 100644 --- a/server/src/services/native-runtime/native-completion-feedback.test.ts +++ b/server/src/services/native-runtime/native-completion-feedback.test.ts @@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto"; import { eq } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { agents, companies, createDb, documentRevisions, executionWorkspaces, executionWorkspaceRepositories, heartbeatRuns, issues, issueThreadInteractions, issueTreeHolds, issueDocuments } from "@paperclipai/db"; -import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; +import { startRecoveryPostgres } from "../../__tests__/helpers/recovery-postgres.js"; import type { PrpStructuredRunResult } from "../../vendor/paperclip-runner/index.js"; import { documentService } from "../documents.js"; import { nativeCompletionFeedback } from "./native-completion-feedback.js"; @@ -15,9 +15,9 @@ const done: PrpStructuredRunResult = { }; describe("native final-response feedback", () => { - let temporary: Awaited>; + let temporary: Awaited>; let db: ReturnType; - beforeAll(async () => { temporary = await startEmbeddedPostgresTestDatabase("native-final-response-"); db = createDb(temporary.connectionString); }); + beforeAll(async () => { temporary = await startRecoveryPostgres("native-final-response-"); db = createDb(temporary.connectionString); }); afterAll(async () => { await temporary?.cleanup(); }); async function fixture() { const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID(); @@ -129,6 +129,19 @@ describe("native final-response feedback", () => { } }); await expect(nativeCompletionFeedback(db, value.runId, done)).resolves.toContain("Saved document"); }); + it("requires a requested JSON file even when the child task already has a saved result document", async () => { + const value = await fixture(); + const objective = [ + "Decode the supplied input into `input.json` in your workspace, then compute `proof.json`.", + "Requirements:", + "- Write `input.json` from the exact input bytes.", + "- Save a document on this child task describing the result and file link.", + "- Publish `proof.json` on the child task with `register_deliverable` using the workspace-relative path, correct byte size, and correct SHA-256.", + ].join("\n"); + await db.update(issues).set({ title: "Compute the proof", description: objective }).where(eq(issues.id, value.issueId)); + const report = { ...done, summary: "Computed the JSON and saved its task document.", evidence: [{ ref: value.saved.documentHref }] }; + await expect(nativeCompletionFeedback(db, value.runId, report)).rejects.toThrow("register_deliverable"); + }); it("accepts a still-published document from an earlier run after a response continuation", async () => { const value = await fixture(), continuationRunId = randomUUID(); const objective = "Save a document on this task."; diff --git a/server/src/services/native-runtime/native-deliverable-feedback.test.ts b/server/src/services/native-runtime/native-deliverable-feedback.test.ts index a033834de93..d3637c080bc 100644 --- a/server/src/services/native-runtime/native-deliverable-feedback.test.ts +++ b/server/src/services/native-runtime/native-deliverable-feedback.test.ts @@ -5,6 +5,29 @@ describe("explicit file output requirements", () => { it.each([ "Prepare a requested file", "Make a Markdown file named checklist.md with three items.", + "Write connection-proof.json with the computed result.", + "Write a new result.json.", + "Write a detailed result.json.", + "Write a summary of input.json as output.json.", + "Write a summary of input.json to a file named output.json.", + "Write result.json.md inline in your response.", + "Write the verified report named result.json.", + "Write the result to reports/output.json.", + "Write result.json with the contents of the verified report.", + "Create report.pdf and include the JSON inline in chat.", + "Create a PDF report and include the JSON inline in chat.", + "Write result.json inline and attach it as a download.", + "Write result.json inline and include a download link.", + "Write result.json and describe it inline in chat.", + "Write notes-and-results.json and describe it inline in chat.", + "Write result.json and explain the result inline in chat.", + "Write result.json and summarize it inline in your response.", + "Write the result to `output.json`.", + "Write result.json inline and publish it.", + "Write result.json inline. Publish it on this task.", + "Write result.json and update package.json.", + "Publish connection-proof.json on this child task with register_deliverable using the workspace-relative path, correct byte size, and correct SHA-256.", + "Write internal-proof.json. This is an internal verification file, not a deliverable. Publish it on this task.", "Export the results as a CSV.", "Do not modify repository files, but create an attachment.", "Do not create attachments; export the results as CSV.", @@ -49,6 +72,36 @@ describe("explicit file output requirements", () => { "Never edit files nor create attachments.", "Don't attach a file. Reply with three bullets.", "Fix a crash in parser.ts.", + "Update config.json to fix the parser.", + "Write the fix and update package.json.", + "Write the migration then edit package-lock.json.", + "Write a summary and read result.json.", + "Write a function that reads config.json.", + "Write code to parse tsconfig.json.", + "Write the route and wire it in routes.json.", + "Write a function reading config.json.", + "Write a short summary after reading input.json.", + "Write a short summary explaining config.json.", + "Write a short summary of changes to config.json.", + "Write a summary of the file named input.json.", + "Write about config.json.", + "Write concerning config.json.", + "Write a comparison of input.json to baseline.json.", + "Write a summary of input.json to output.json.", + "Write a wrapper around config.json.", + "Write tests covering package.json.", + "Write a migration touching package-lock.json.", + "Return the answer inline with no download link.", + "Return the results inline without a download link.", + "Return the answer without a download link.", + "Write result.json inline in your response as a code block; do not create any files.", + "Write notes-and-results.json inline in your response.", + "Return result.json as a code block in the reply.", + "Read connection-input.json and explain its contents in chat.", + "Write a summary of connection-input.json inline in your response.", + "Write the contents of connection-proof.json inline in your response.", + "Do not publish connection-proof.json; reply inline.", + "Write internal-proof.json. This is an internal verification file, not a deliverable. Do not publish it.", "Read the file and write a short explanation inline.", "No downloadable file is needed.", "Write a summary of this PDF in chat.", diff --git a/server/src/services/native-runtime/native-deliverable-feedback.ts b/server/src/services/native-runtime/native-deliverable-feedback.ts index 3970c173227..b761f61a249 100644 --- a/server/src/services/native-runtime/native-deliverable-feedback.ts +++ b/server/src/services/native-runtime/native-deliverable-feedback.ts @@ -82,9 +82,9 @@ export function explicitlyRequestsFileOutput(objective: string): boolean { const sentences = objective.replaceAll("\\_", "_").split(/(?:[.!?](?:\s|$)|\n)/iu); let precedingFileOutput = false; return sentences.some((sentence, index) => { - const file = /\b(?:files?|attachments?|downloads?|pdf|spreadsheets?|workbooks?|slide decks?|powerpoints?|docx|xlsx|csv)\b|\b[^\s/]+\.(?:md|txt|pdf|docx?|xlsx?|csv|pptx?|png|jpe?g|svg|zip)\b/giu; + const file = /\b(?:files?|attachments?|downloads?|pdf|spreadsheets?|workbooks?|slide decks?|powerpoints?|docx|xlsx|csv)\b|\b[^\s/]+\.(?:md|txt|json|pdf|docx?|xlsx?|csv|pptx?|png|jpe?g|svg|zip)\b/giu; const outputs = sentence.split(/(?:[;,]|\bbut\b)/iu).flatMap(clause => { - const creates = [...clause.matchAll(/\b(?:create|make|write|save|export|attach|send|generate|produce|prepare|provide|give|return|build)\b/giu)]; + const creates = [...clause.matchAll(/\b(?:create|make|write|save|export|attach|send|generate|produce|prepare|provide|give|return|build|publish)\b/giu)]; return creates.flatMap((create, createIndex) => { const before = clause.slice(0, create.index); if (/\b(?:do not|don['’]t|never|no need to)\s*$/iu.test(before)) return []; @@ -96,33 +96,46 @@ export function explicitlyRequestsFileOutput(objective: string): boolean { // Bind each object to its own verb. A denied write or "create no files" // cannot suppress a separate requested report in this same clause. const output = clause.slice(create.index + create[0].length, creates[createIndex + 1]?.index); + // An explicit attachment/export request can refer to the file by + // pronoun. It still requires publication when its name is omitted. + const referencesOutput = /^(?:attach|export|send|provide|give|return|publish)$/iu.test(create[0]) + && /^\s+(?:me\s+)?(?:it|them|this|that)\b/iu.test(output); + const referencedAttachment = referencesOutput && /^(?:attach|export)$/iu.test(create[0]); + const inlineDestination = /\b(?:inline|(?:in|within|inside|as|into)\s+(?:(?:a|an|the|my|your|our|final|plain|markdown|chat|fenced)\s+)*(?:chat|response|reply|message|comment|text|code block))\b/iu; + const inline = inlineDestination.test(output) || /\b(?:its|the) contents\b/iu.test(output); + const downloadable = referencedAttachment || (!/\b(?:no|without)\s+(?:downloadable|attached)/iu.test(output) + && /\b(?:downloadable|attached)\s+(?:file|report|document|checklist|draft)\b/iu.test(output)); + const inlineOnly = inlineDestination.test(output.split(/\s+(?:and|then)\s+/iu, 1)[0]) && !downloadable && !/^(?:attach|export|publish)$/iu.test(create[0]) + && !/\bas (?:an? )?attachment\b/iu.test(output); const objects = [...output.matchAll(file)].filter(match => { const prefix = output.slice(0, match.index); const suffix = output.slice(match.index + match[0].length); // "Create no files" is a prohibition, even though it contains a creation // verb. Negate this object only; another explicit output can still count. - if (/\b(?:no|zero|without(?:\s+any)?)\s+(?:(?:new|temporary|downloadable|attached|additional)\s+)*$/iu.test(prefix)) return false; + if (/\b(?:no|zero|without(?:\s+(?:any|a|an))?)\s+(?:(?:new|temporary|downloadable|attached|additional)\s+)*$/iu.test(prefix)) return false; // "Write a summary of this PDF" names input, not a requested file. // Explicit export destinations still count after such input references. - const destination = /\b(?:as|into|to)\s+(?:(?:a|an|the|new|separate|markdown|word|excel)\s+)*$/iu.test(prefix); - if (!destination && /\b(?:of|about|on|from|using|for|with)\b/iu.test(prefix)) return false; + const json = /\.json$/iu.test(match[0]); + const inputReference = /\b(?:of|about|on|from|using|for|with)\b/iu; + const destinationMatch = (json + ? /\b(as|into|to|named|called)\s+(?:(?:a|an|the|new|separate|markdown|word|excel)\s+)*["'`*]*(?:[^\s]*\/)?$/iu + : /\b(?:as|into|to)\s+(?:(?:a|an|the|new|separate|markdown|word|excel)\s+)*$/iu).exec(prefix); + const beforeDestination = prefix.slice(0, destinationMatch?.index ?? 0); + const destination = destinationMatch !== null && (!json || !/^(?:to|named|called)$/iu.test(destinationMatch[1]) + || !inputReference.test(beforeDestination)); + if (json) { + const directObject = /^\s*(?:(?:a|an|the|my|your|our|their|his|her|its|this|that)\s+(?:(?:new|separate|downloadable|attached)\s+)*([\p{L}\p{N}_-]+\s+)?|(?:(?:new|separate|downloadable|attached)\s+)*)["'`*]*(?:[^\s]*\/)?$/iu.exec(prefix); + if (!destination && (!directObject || inputReference.test(directObject[1] ?? ""))) return false; + } else if (!destination && inputReference.test(prefix)) return false; if (/^files?$/iu.test(match[0]) && /^\s+(?:tools?|permissions?|systems?|formats?|names?|paths?|types?|sizes?|descriptors?)\b/iu.test(suffix)) return false; return true; }); - // An explicit attachment/export request can refer to the file by - // pronoun. It still requires publication when its name is omitted. - const referencesOutput = /^(?:attach|export|send|provide|give|return)$/iu.test(create[0]) - && /^\s+(?:me\s+)?(?:it|them|this|that)\b/iu.test(output); - const referencedAttachment = referencesOutput && /^(?:attach|export)$/iu.test(create[0]); - const inline = /\b(?:inline|(?:in|within|inside|as|into)\s+(?:(?:a|an|the|my|your|our|final|plain|markdown|chat|fenced)\s+)*(?:chat|response|reply|message|comment|text|code block)|(?:its|the) contents)\b/iu.test(output); - const downloadable = referencedAttachment || (!/\b(?:no|without)\s+(?:downloadable|attached)/iu.test(output) - && /\b(?:downloadable|attached)\s+(?:file|report|document|checklist|draft)\b/iu.test(output)); - const publication = /\b(?:downloadable|attached|attach|export|send|provide|return|give)\b/iu.test(create[0] + output); + const publication = /\b(?:downloadable|attached|attach|export|send|provide|return|give|publish)\b/iu.test(create[0] + output); const deniedAttempt = create[0].toLowerCase() === "write" && objects.length === 1 && /\b(?:attempt|try)\s+(?:the\s+)?native\s*$/iu.test(before) && /\b(?:must be denied|denial is (?:the )?expected|(?:this|the) negative test)\b/iu.test(objective); if (objects.length === 0 && !downloadable && /^\s+(?:no|zero|without)\b/iu.test(output)) return []; - return [{ objects: objects.length, downloadable, publication, deniedAttempt, + return [{ namesFile: objects.length > 0, objects: objects.filter(match => !inlineOnly || !/\.json$/iu.test(match[0])).length, downloadable, publication, deniedAttempt, referencesOutput, publicationReference: referencesOutput && !inline }]; }); }); @@ -133,7 +146,7 @@ export function explicitlyRequestsFileOutput(objective: string): boolean { && /^\s*This is (?:an? )?(?:personal memory|internal (?:assertion|verification) file)\b[^.!?]*\bnot a (?:task )?deliverable\b/iu.test(sentences[index + 1] ?? ""); return outputs.some(output => { const publicationReference = output.publicationReference && precedingFileOutput; - if (!output.referencesOutput) precedingFileOutput = output.objects > 0; + if (!output.referencesOutput) precedingFileOutput = output.namesFile; return (output.objects > 0 || output.downloadable || publicationReference) && (output.publication || (!internal && !output.deniedAttempt)); });