From 2b461b2f5801dddb8b0a5bb4d194bc6f136b2c71 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Mon, 9 Nov 2026 07:15:11 -0600 Subject: [PATCH] fix(native): require publication for files named under a Deliverable label (#20231) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Native agents finish a task with a completion report, and a server gate checks that requested files are published to the task > - The gate finds a file request only from a creation verb, such as "Write report.pdf" > - Delegated briefs often list the output under a "Deliverable:" label instead, and the gate did not know `.html` or `.css` names > - So a worker can finish a website task with `index.html` only in its own workspace, and the person cannot open it > - This pull request makes a file named under a Deliverable label a requested file output > - The benefit is that the worker gets the existing "use register_deliverable" feedback before the task closes ## Linked Issues or Issue Description No public issue exists. Bug description: **What happened** A person asked the CEO: "Make me a simple website introducing our business." The CEO delegated a task. Its brief said "Deliverable: - A self-contained static website in this task's workspace: index.html plus a stylesheet (for example styles.css)". The worker wrote `index.html` and `styles.css` in its workspace. It did not call `register_deliverable`. It cited `index.html` and `styles.css` as completion evidence, and the gate accepted the report. The task was marked done with no attachment. The CEO then told the person that the files were "in the task workspace". **Expected behavior** A brief that names a file as its deliverable is a file request. The completion gate must reject a workspace-only file and tell the worker to publish it with `register_deliverable`. **Steps to reproduce** 1. Call `explicitlyRequestsFileOutput` with the brief above. 2. See that it returns `false`, so `validateNativeDeliverableEvidence` accepts the bare `index.html` evidence ref. ## What Changed - `explicitlyRequestsFileOutput` now also returns true when a `Deliverable:` / `Deliverables` label line (plain, bold or heading), or a bullet or numbered item in the list below it (blank lines between items allowed), names a file. Relative paths such as `dist/index.html` count. - Under the label, web page files (`.html`, `.htm`, `.css`) count with the existing file types. - Items that name an input ("a summary of input.csv", "a comparison of input.csv and baseline.csv"), describe a repository change, pull request, commit or branch, forbid the named file (`do not create report.pdf`), name a file to edit in place (`update styles.css`) without asking to publish it, send the file inline (`index.html and styles.css inline in the reply`), or forbid creating files, do not count. An inline or no-upload instruction about another output in the same item (`index.html and a short summary in the reply`, `index.html; do not upload it externally`) does not cancel the named file. The repository-change check ignores file names, so `branch-report.pdf` still counts. - The sentence-based detector is unchanged. - `doc/AGENT-ARTIFACTS.md` describes the label rule. ## Verification - `cd server && npx vitest run src/services/native-runtime/native-deliverable-feedback.test.ts src/services/native-runtime/native-runner-file-handoff.test.ts` (197 pass), then after the review fixes `native-deliverable-feedback.test.ts`, `native-completion-format-feedback.test.ts` and `chat-attachment-reuse.test.ts` (211 pass). The 13 first-round review cases fail on the original head. - New cases: the real delegated brief, inline labels, heading lists, numbered lists and bold labels require publication. Unlabelled text, inline-chat deliverables, input references, repository changes, text after the list ends, and "Deliverable quality ..." prose do not. - The chat eval `onboarding.first-work` exercises this path end to end on its candidate branch. ## Risks - Low risk. The change only adds a true result for labelled file outputs; an inline-only or no-file label item stays false, as the sentence detector already decided. - A labelled brief that names a file which is not a repository change now needs `register_deliverable` or an accessible work product. That is the documented delivery rule. ## Model Used - Anthropic Claude Opus 5.5 (`claude-opus-5-5`), tool use and code execution in Claude Code. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- doc/AGENT-ARTIFACTS.md | 12 +++ .../native-deliverable-feedback.test.ts | 73 +++++++++++++++++++ .../native-deliverable-feedback.ts | 60 +++++++++++++++ 3 files changed, 145 insertions(+) diff --git a/doc/AGENT-ARTIFACTS.md b/doc/AGENT-ARTIFACTS.md index e380f34c3c5..f5c12fdaf9c 100644 --- a/doc/AGENT-ARTIFACTS.md +++ b/doc/AGENT-ARTIFACTS.md @@ -34,6 +34,18 @@ their contents inline does not by itself require a file attachment. JSON output intent is recognized from a direct filename or an explicit output destination. Ambiguous descriptions of input files joined by `to` remain text requests; use `as`, `into`, or an explicit output file when an artifact is required. +A `Deliverable:` or `Deliverables` label also states the output. A file named +on that line, or in the list directly below it (blank lines between items are +allowed), is a requested file. This includes relative paths such as +`dist/index.html` and web page files such as `index.html` and `styles.css`. +Items that describe a repository change, pull request, commit, or branch do +not count, nor do files named as inputs (`a comparison of input.csv and +baseline.csv`), files the item forbids (`do not create report.pdf`), files to +be edited in place (`update styles.css`) unless the item also asks to publish +them, files to be written inline in the reply (`index.html and styles.css inline +in the reply`), or items that say no files are created. An inline or no-upload +instruction about another output in the same item does not cancel a named file +(`index.html and a short summary in the reply`). The completion gate permits a narrow sentence-bound exception for one requested file when the following sentence explicitly qualifies it, for example: diff --git a/server/src/services/native-runtime/native-deliverable-feedback.test.ts b/server/src/services/native-runtime/native-deliverable-feedback.test.ts index d3637c080bc..e73cf2d7983 100644 --- a/server/src/services/native-runtime/native-deliverable-feedback.test.ts +++ b/server/src/services/native-runtime/native-deliverable-feedback.test.ts @@ -168,3 +168,76 @@ describe("explicit task-document output", () => { expect(explicitlyRequestsTaskDocumentOutput(objective)).toBe(false); }); }); + +describe("deliverable labels that name files", () => { + const delegatedBrief = [ + "Build a simple, polished static marketing website that introduces the business, Acorn Notes.", + "", + "Deliverable:", + "- A self-contained static website in this task's workspace: index.html plus a stylesheet (for example styles.css). Plain HTML/CSS, no build step.", + "- Sections: a hero with the value proposition and a contact call to action.", + "", + "Constraints:", + "- Do not publish or deploy externally; no hosting, DNS, analytics, or third-party services.", + ].join("\n"); + it.each([ + delegatedBrief, + "Deliverables: index.html and styles.css", + "## Deliverables\n- `index.html`\n- `styles.css`", + "**Deliverable:** report.pdf", + "Requested deliverable:\n1. summary.csv with one row per team", + "Deliverable: dist/index.html", + "Deliverable: reports/summary.csv", + "Deliverables:\n- src/styles.css\n- ./site/index.html", + "## Deliverables\n\n- index.html\n- styles.css", + "Deliverables:\n\n- A landing page\n\n- index.html with the hero section", + "Deliverable: branch-report.pdf", + "Deliverable: commit-history.csv covering the last release", + "Deliverable: a comparison of input.csv and baseline.csv, saved as comparison.csv", + "Deliverable: report.pdf, with a short summary in the reply", + "Deliverable: index.html and a short summary in the reply", + "Deliverable: index.html plus a summary inline in chat", + "Deliverable: index.html; do not upload it externally", + "Deliverable: an updated report.pdf", + "Deliverable: update report.pdf and publish it", + "Deliverable: update report.pdf, then attach the updated report.pdf", + "Deliverable: index.html with a short summary in the reply", + "Deliverable: a summary in the reply and index.html", + "Deliverable: a short explanation; do not create notes.md, but attach report.pdf", + "Deliverables:\n- report.pdf, do not attach it to an email", + ])("requires publication for a labelled file output: %s", objective => { + expect(explicitlyRequestsFileOutput(objective)).toBe(true); + }); + it.each([ + "Build a simple website introducing the business.", + "A self-contained static website in this task's workspace: index.html plus a stylesheet.", + "Deliverable: a short answer in chat.", + "Deliverables:\n- A summary of input.csv in the reply", + "Deliverable: a pull request that updates index.html", + "Deliverables:\n- Repository change to styles.css", + "Deliverable:\n\nindex.html is mentioned after the list ended.", + "Deliverable quality matters more than speed for index.html.", + "Deliverable: Write result.json inline in your response as a code block; do not create any files.", + "Deliverable: result.json as a code block in the reply", + "Deliverables:\n- notes.md content inline in the chat", + "Deliverable: a comparison of input.csv and baseline.csv", + "Deliverable: a summary of a.csv, b.csv and c.csv", + "Deliverable: differences between old.json and new.json", + "Deliverable: a pull request on the feature branch that updates dist/index.html", + "Deliverables:\n\n- A summary in chat\n\nNotes:\n- index.html is the current page", + "Deliverable: a review of https://example.com/index.html", + "Deliverable: summary.md in the reply; do not create any new files", + "Deliverable: notes.txt without creating any files", + "Deliverable: update styles.css so the header is blue", + "Deliverables:\n- Edit the existing `index.html` to add a footer", + "Deliverable: a short explanation; do not create report.pdf", + "Deliverable: never write a separate summary.md", + "Deliverable: index.html with its contents inline in the reply", + "Deliverables: index.html and styles.css inline in the reply", + "Deliverable: update styles.css; do not publish it", + "Deliverable: update styles.css and send a short summary in the reply", + "Deliverable: index.html with the full markup inline in the reply", + ])("does not treat this as a labelled file output: %s", objective => { + expect(explicitlyRequestsFileOutput(objective)).toBe(false); + }); +}); diff --git a/server/src/services/native-runtime/native-deliverable-feedback.ts b/server/src/services/native-runtime/native-deliverable-feedback.ts index 3cba6e59d5c..bad131e5af9 100644 --- a/server/src/services/native-runtime/native-deliverable-feedback.ts +++ b/server/src/services/native-runtime/native-deliverable-feedback.ts @@ -99,6 +99,66 @@ async function hasCurrentPublicationReceipt(db: Db, companyId: string, receipts: * an output requirement or erase a user's request for a file. */ export function explicitlyRequestsFileOutput(objective: string): boolean { + return deliverableLabelNamesFile(objective) || explicitlyRequestsFileOutputBySentence(objective); +} + +function deliverableLabelNamesFile(objective: string): boolean { + const lines = objective.replaceAll("\\_", "_").split(/\r?\n/u); + const named = /(?:^|[\s(`'"*:])((?:\.{0,2}\/)?(?:[\w.-]+\/)*[^\s/()`'"*:,;]+\.(?:html?|css|md|txt|json|pdf|docx?|xlsx?|csv|pptx?|png|jpe?g|svg|zip))\b/giu; + const repositoryChange = /\b(?:[Rr]epo(?:sitory)?|[Pp]ull [Rr]equest|PR|[Cc]ommit|[Bb]ranch)\b/u; + const inputReference = /\b(?:of|about|from|using|reading|comparing|between|no|without)\s+(?:(?:the|a|an|this|that)\s+)?[`'"*]*(?:[^\s,;]+[`'"*]*(?:\s*,\s*(?:(?:and|or)\s+)?|\s+(?:and|or|vs\.?|versus)\s+)(?:(?:the|a|an|this|that)\s+)?[`'"*]*)*$/iu; + const inlineDestination = /\b(?:inline|(?:in|within|inside|as|into)\s+(?:(?:a|an|the|my|your|our|final|plain|markdown|chat|fenced)\s+)*(?:chat|response|reply|message|comment|text|code block))\b/iu; + const noFiles = /\b(?:(?:do not|don['’]t|never)\s+(?:create|make|write|save|produce)\s+(?:any\s+)?(?:new\s+)?files?|(?:no|without(?:\s+(?:any|creating(?:\s+any)?))?)\s+(?:new\s+)?files?)\b/iu; + const negated = /\b(?:do not|don['’]t|never|no need to)\s+(?:\w+\s+)?$/iu; + const forbiddenFile = /\b(?:do not|don['’]t|never|no need to)\s+(?:create|make|write|save|produce|generate|build|export|prepare|attach|upload|publish)\s+(?:(?:a|an|the|any|new|separate|extra|additional)\s+)*[`'"*]*$/iu; + const editedFile = /\b(?:update|edit|modify|fix|change|refactor|patch)\s+(?:(?:the|existing|current)\s+)*[`'"*]*$/iu; + const publicationVerb = /\b(?:publish|attach|export|upload|send|share|deliver|register)\b/giu; + const conjunction = /\b(?:and|plus|then|alongside|as well as|with)\b/giu; + const sameOutput = /^\s*(?:(?:its|their|the)\s+(?:full\s+)?(?:contents?|source|code|markup|text)|it|them)\s*$/iu; + const namesFile = (text: string) => { + const matches = [...text.matchAll(named)]; + if (matches.length === 0 || noFiles.test(text) || repositoryChange.test(text.replace(named, " "))) return false; + const publishes = (name: string) => { + const object = new RegExp(`^\\s+(?:(?:it|them|both)\\b|(?:(?:the|a|an|updated|edited|new|final)\\s+)*(?:files?\\b|[\`'"*]*${name.replace(/[.*+?^${}()|[\]\\/]/gu, "\\$&")}))`, "iu"); + return [...text.matchAll(publicationVerb)].some(verb => !negated.test(text.slice(0, verb.index)) + && object.test(text.slice(verb.index! + verb[0].length))); + }; + const ownsInline = (between: string) => { + const last = [...between.matchAll(conjunction)].at(-1); + const rest = between.slice(last ? last.index! + last[0].length : 0); + return !last || sameOutput.test(rest) || [...rest.matchAll(named)].length > 0; + }; + return matches.some(match => { + const start = match.index! + match[0].indexOf(match[1]!); + const before = text.slice(0, start); + if (inputReference.test(before) || forbiddenFile.test(before)) return false; + if (editedFile.test(before) && !publishes(match[1]!)) return false; + const end = start + match[1]!.length; + const clauseStart = Math.max(text.lastIndexOf(",", start), text.lastIndexOf(";", start)) + 1; + const next = text.slice(end).search(/[,;]/u); + const clauseEnd = next < 0 ? text.length : end + next; + const after = inlineDestination.exec(text.slice(end, clauseEnd)); + if (after && ownsInline(text.slice(end, end + after.index))) return false; + const lead = text.slice(clauseStart, start); + const leading = [...lead.matchAll(new RegExp(inlineDestination.source, "giu"))].at(-1); + return !(leading && ownsInline(lead.slice(leading.index! + leading[0].length))); + }); + }; + for (let index = 0; index < lines.length; index += 1) { + const label = /^\s*(?:#{1,6}\s*)?(?:\*\*|__)?(?:requested\s+)?deliverables?(?:\*\*|__)?\s*(?::(?:\*\*|__)?\s*(.*)|(?:\*\*|__)?\s*)$/iu.exec(lines[index]!); + if (!label) continue; + if (namesFile(label[1] ?? "")) return true; + for (let item = index + 1; item < lines.length; item += 1) { + if (!lines[item]!.trim()) continue; + const bullet = /^\s*(?:[-*+]|\d+[.)])\s+(.*)$/u.exec(lines[item]!); + if (!bullet) break; + if (namesFile(bullet[1]!)) return true; + } + } + return false; +} + +function explicitlyRequestsFileOutputBySentence(objective: string): boolean { const sentences = objective.replaceAll("\\_", "_").split(/(?:[.!?](?:\s|$)|\n)/iu); let precedingFileOutput = false; return sentences.some((sentence, index) => {