From dc6e4be04d1c5c438a27915dbbb7d4f477d00c7f Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 7 Nov 2026 20:04:03 -0600 Subject: [PATCH] test(chat): accept an exact inline brief handoff in tasks.attachment-brief The grader required either a copied attachment or an attachment URL in the delegated task, so it rejected the agent copying the complete brief verbatim into the task description, which the chat guidance itself recommends ("copy the relevant existing content into the handoff") and which gave Morgan everything needed to build the site. - Accept the complete exact brief text (setup-chosen literal, tolerant only of line endings, indentation, blank lines and quote markers) in the full saved task description or its documents, and require Morgan's own bearer to read it there. - Read the saved task in full; company issue listings shorten descriptions to 1200 characters, which could also hide a reference. - Drop the website/landing-page keyword check on the model-written title. - Bump the case oracle version. Co-Authored-By: Claude Opus 5.5 --- .../runner-e2e/chat-catalog-descriptors.json | 2 +- .../internal-task-attachment.ts | 50 +++++++++++++------ .../chat-catalog-domains/internal.test.ts | 19 +++++++ 3 files changed, 56 insertions(+), 15 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 3ca9fc32d6c..6e1d095274f 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -5784,7 +5784,7 @@ "chat-catalog-domains/identity-support.ts", "chat-query-attachments.ts" ], - "oracleVersion": "paperclip.chat-catalog.internal.v1", + "oracleVersion": "paperclip.chat-catalog.attachment-brief.v2", "coverageGaps": [], "expectedRunCount": 3, "minimumRunCount": 2, diff --git a/tests/runner-e2e/chat-catalog-domains/internal-task-attachment.ts b/tests/runner-e2e/chat-catalog-domains/internal-task-attachment.ts index 8c7ecd0e10e..a88da8a23f7 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-task-attachment.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-task-attachment.ts @@ -11,8 +11,16 @@ interface BriefObservation extends InternalObservation { sourceId?: string; originalBytesHash: string; sourceAttachments: ChatQueryAttachmentReceipt[]; workAttachments: ChatQueryAttachmentReceipt[]; workDocuments: Entity[]; workerTaskStatus?: number; workerDownloadStatus?: number; workerDownloadedHash?: string; durablePath?: string; + workDescription?: string | null; workerReadInlineBrief?: boolean; } const hash = (bytes: Uint8Array) => createHash("sha256").update(bytes).digest("hex"); +// Exact brief text, tolerant only of line endings, indentation, trailing spaces, blank lines and Markdown quote markers. +const briefLines = (text: string) => text.replace(/\r\n?/g, "\n").split("\n").map(line => line.trim().replace(/^(?:>\s?)+/, "").trim()).filter(Boolean); +export function containsExactBrief(context: string | null | undefined, brief: unknown): boolean { + if (typeof brief !== "string" || typeof context !== "string") return false; + const want = briefLines(brief); const have = briefLines(context); + return want.length > 0 && have.some((_, start) => want.every((line, offset) => have[start + offset] === line)); +} const workerProbeKeys = new WeakMap(); export function gradeAttachmentBrief(value: unknown) { const c = checks(); const o = observation(value) as BriefObservation | undefined; @@ -25,23 +33,24 @@ export function gradeAttachmentBrief(value: unknown) { && row.createdByUserId === receipt.createdByUserId && row.createdByAgentId == null && row.issueCommentId && row.sha256 === o.state.briefHash) && o.originalBytesHash === o.state.briefHash, "The original signed-in person's uploaded brief and its exact bytes remain attached to the actual source human comment."); c.add("one-contextual-ordinary-work-task", work.length === 1 && !!saved && saved.companyId === o.companyId && saved.assigneeAgentId === worker.id - && /\b(?:website|landing.page|web page)\b/i.test(`${saved.title} ${saved.description ?? ""}`) && o.activity.some(row => row.action === "issue.created" && row.entityId === saved.id && row.agentId === o.leadId) && hasLink(o, "issues", saved), "One actual CEO-created website task is delegated to the intended Morgan and linked in chat."); - const context = `${saved?.description ?? ""}\n${o.workDocuments.map(row => row.body ?? "").join("\n")}`; + const context = `${o.workDescription ?? saved?.description ?? ""}\n${o.workDocuments.map(row => row.body ?? "").join("\n")}`; const copied = o.workAttachments.some(row => row.issueId === saved?.id && row.companyId === o.companyId && row.sha256 === o.state.briefHash && o.durablePath === row.contentPath); const originalReference = !!receipt && context.includes(receipt.contentPath) && o.durablePath === receipt.contentPath; - c.add("durable-brief-in-work-context", copied || originalReference, - "The delegated task owns an exact-byte copy or a durable actual attachment reference; an unavailable filename or source-chat-only attachment fails."); - c.add("morgan-can-read-the-real-brief", o.workerTaskStatus === 200 && o.workerDownloadStatus === 200 && o.workerDownloadedHash === o.state.briefHash, - "An independent real Morgan bearer reads the work task and downloads the actual referenced brief bytes from its authorized context."); + const inline = containsExactBrief(context, o.state.briefBody); + c.add("durable-brief-in-work-context", copied || originalReference || inline, + "The delegated task owns an exact-byte copy, a durable actual attachment reference, or the complete exact brief text; an unavailable filename, a paraphrase or a source-chat-only attachment fails."); + const downloaded = o.workerDownloadStatus === 200 && o.workerDownloadedHash === o.state.briefHash; + c.add("morgan-can-read-the-real-brief", o.workerTaskStatus === 200 && (downloaded || (inline && o.workerReadInlineBrief === true)), + "An independent real Morgan bearer reads the work task and either downloads the actual referenced brief bytes or reads the complete exact brief in its own task context."); const runs = saved ? o.runs.filter(row => row.contextSnapshot?.issueId === saved.id || row.contextSnapshot?.taskId === saved.id) : []; c.add("separate-real-worker-admission", runs.length === 1 && runs[0]?.agentId === worker.id && ["queued", "running", "succeeded"].includes(runs[0]?.status ?? ""), "Morgan's real admitted run belongs to the new work task, separate from the source conversation."); return c.result(); } -async function workerReads(ctx: ChatCatalogContext, workerId: string, taskId: string, path: string | undefined) { +async function workerReads(ctx: ChatCatalogContext, workerId: string, taskId: string, path: string | undefined, brief: string) { const key = workerProbeKeys.get(ctx); if (!key || key.workerId !== workerId) throw new Error("Attempt-private Morgan probe credential is unavailable"); const get = (value: string) => { @@ -49,8 +58,18 @@ async function workerReads(ctx: ChatCatalogContext, workerId: string, taskId: st if (url.origin !== ctx.input.api.baseURL || !/^\/api\/(?:issues|attachments)\//.test(url.pathname)) throw new Error("Worker brief read escaped isolated resources"); return fetch(url, { redirect: "error", signal: AbortSignal.timeout(15_000), headers: { Authorization: `Bearer ${key.token}` } }); }; - const taskRead = await get(`/api/issues/${taskId}`); const workerTaskStatus = taskRead.status; await taskRead.body?.cancel(); - if (!path) return { workerTaskStatus }; + const taskRead = await get(`/api/issues/${taskId}`); const workerTaskStatus = taskRead.status; + if (!path) { + if (!taskRead.ok) { await taskRead.body?.cancel(); return { workerTaskStatus }; } + const parts = [((await taskRead.json()) as Entity).description ?? ""]; + const list = await get(`/api/issues/${taskId}/documents`); + if (list.ok) for (const summary of (await list.json()) as Entity[]) { + const doc = await get(`/api/issues/${taskId}/documents/${encodeURIComponent(summary.key!)}`); + if (doc.ok) parts.push(((await doc.json()) as Entity).body ?? ""); else await doc.body?.cancel(); + } else await list.body?.cancel(); + return { workerTaskStatus, workerReadInlineBrief: containsExactBrief(parts.join("\n"), brief) }; + } + await taskRead.body?.cancel(); const response = await get(path); const workerDownloadStatus = response.status; if (!response.ok) { await response.body?.cancel(); return { workerTaskStatus, workerDownloadStatus }; } const bytes = new Uint8Array(await response.arrayBuffer()); @@ -71,9 +90,9 @@ const base = makeDefinition("tasks.attachment-brief", async ctx => { if (!ctx.input.privateSecrets) throw new Error("Worker brief probe requires private secret redaction"); const key = await ctx.input.api.postSensitive<{ id: string; token: string }>(`/api/agents/${worker.id}/keys`, { name: "delegated-brief-read-control" }); ctx.input.privateSecrets.push(key.token); workerProbeKeys.set(ctx, { ...key, workerId: worker.id }); - return { worker, sourceId: source.id, briefAttachment: receipt, briefHash: hash(Buffer.from(briefBody)), workerProbeKeyId: key.id }; + return { worker, sourceId: source.id, briefAttachment: receipt, briefBody, briefHash: hash(Buffer.from(briefBody)), workerProbeKeyId: key.id }; }, gradeAttachmentBrief); -export const taskAttachmentCases: readonly ChatCatalogCase[] = [{ ...base, authentication: "human-session", +export const taskAttachmentCases: readonly ChatCatalogCase[] = [{ ...base, authentication: "human-session", oracleVersion: "paperclip.chat-catalog.attachment-brief.v2", queryAttachments: [{ stateKey: "briefAttachment", filename: "website-brief.md" }], expectedRunCount: 3, minimumRunCount: 2, runPolicy: { kind: "source-settled-with-active-work", maximumActiveWorkerRuns: 1 }, sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-task-attachment.ts", "chat-catalog-domains/identity-support.ts", "chat-query-attachments.ts"], @@ -83,16 +102,19 @@ export const taskAttachmentCases: readonly ChatCatalogCase[] = [{ ...base, authe const receipt = state.briefAttachment as ChatQueryAttachmentReceipt; const originalBytesHash = hash(await ctx.input.api.getAttachmentContent(receipt.id)); const saved = o.tasks.filter(row => row.id !== source?.id)[0]; const workAttachments: ChatQueryAttachmentReceipt[] = []; const workDocuments: Entity[] = []; + let workDescription: string | null | undefined; if (saved) { + // Company issue listings shorten descriptions; read the saved task in full. + workDescription = (await ctx.input.api.get(`/api/issues/${saved.id}`)).description; workAttachments.push(...await ctx.input.api.get(`/api/issues/${saved.id}/attachments`)); for (const summary of await ctx.input.api.get(`/api/issues/${saved.id}/documents`)) workDocuments.push(await ctx.input.api.get(`/api/issues/${saved.id}/documents/${encodeURIComponent(summary.key!)}`)); } - const context = `${saved?.description ?? ""}\n${workDocuments.map(row => row.body ?? "").join("\n")}`; + const context = `${workDescription ?? ""}\n${workDocuments.map(row => row.body ?? "").join("\n")}`; const durablePath = workAttachments.find(row => row.sha256 === state.briefHash)?.contentPath ?? (context.includes(receipt.contentPath) ? receipt.contentPath : undefined); - const reads = saved ? await workerReads(ctx, entity(state, "worker").id, saved.id, durablePath) : {}; - return { ...o, sourceId: source?.id, sourceAttachments, originalBytesHash, workAttachments, workDocuments, durablePath, ...reads }; + const reads = saved ? await workerReads(ctx, entity(state, "worker").id, saved.id, durablePath, String(state.briefBody ?? "")) : {}; + return { ...o, sourceId: source?.id, sourceAttachments, originalBytesHash, workAttachments, workDocuments, workDescription, durablePath, ...reads }; }, async cleanup(ctx, state) { const key = workerProbeKeys.get(ctx); workerProbeKeys.delete(ctx); diff --git a/tests/runner-e2e/chat-catalog-domains/internal.test.ts b/tests/runner-e2e/chat-catalog-domains/internal.test.ts index 989f2df63fa..ba5af3b469a 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal.test.ts @@ -650,6 +650,25 @@ describe("delegated attachment context calibration", () => { const o = valid("tasks.attachment-brief"); Object.assign(o, { workerDownloadedHash: "b".repeat(64) }); expect(gradeInternal(o).passed).toBe(false); }); + const inlineBrief = "# Acorn Field Notes\nBuild a simple accessible website.\nUse headline: Research without the busywork.\nOffer early signup with invitation code ORCHID-n1.\n"; + const inlined = (description: string, workerReadInlineBrief = true) => { + const o = valid("tasks.attachment-brief"); o.state.briefBody = inlineBrief; o.tasks[0]!.description = description.slice(0, 40); + Object.assign(o, { workAttachments: [], durablePath: undefined, workerDownloadStatus: undefined, workerDownloadedHash: undefined, + workDescription: description, workerReadInlineBrief }); + return o; + }; + it("accepts the complete exact brief copied into the full saved task that Morgan reads", () => { + const filler = "Acceptance criteria follow. ".repeat(60); + expect(gradeInternal(inlined(`${filler}\n\nBRIEF (verbatim):\n${inlineBrief}`)).passed).toBe(true); + expect(gradeInternal(inlined(`Brief:\r\n${inlineBrief.split("\n").map(line => `> ${line}`).join("\r\n")}`)).passed).toBe(true); + expect(gradeInternal(inlined(`Ship a marketing site.\n\`\`\`md\n${inlineBrief}\`\`\``)).passed).toBe(true); + }); + it("rejects a paraphrased, partial or Morgan-unreadable inline brief", () => { + expect(gradeInternal(inlined("Build a site for research teams with headline Research without the busywork and code ORCHID-n1.")).passed).toBe(false); + expect(gradeInternal(inlined(inlineBrief.split("\n").slice(0, 3).join("\n"))).passed).toBe(false); + expect(gradeInternal(inlined(`Use website-brief.md from the chat.`)).passed).toBe(false); + expect(gradeInternal(inlined(inlineBrief, false)).passed).toBe(false); + }); }); describe("governed completion calibration", () => {