diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 3ca9fc32d6c..6e1d095274f 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -5784,7 +5784,7 @@ "chat-catalog-domains/identity-support.ts", "chat-query-attachments.ts" ], - "oracleVersion": "paperclip.chat-catalog.internal.v1", + "oracleVersion": "paperclip.chat-catalog.attachment-brief.v2", "coverageGaps": [], "expectedRunCount": 3, "minimumRunCount": 2, diff --git a/tests/runner-e2e/chat-catalog-domains/internal-task-attachment.ts b/tests/runner-e2e/chat-catalog-domains/internal-task-attachment.ts index 8c7ecd0e10e..a88da8a23f7 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-task-attachment.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-task-attachment.ts @@ -11,8 +11,16 @@ interface BriefObservation extends InternalObservation { sourceId?: string; originalBytesHash: string; sourceAttachments: ChatQueryAttachmentReceipt[]; workAttachments: ChatQueryAttachmentReceipt[]; workDocuments: Entity[]; workerTaskStatus?: number; workerDownloadStatus?: number; workerDownloadedHash?: string; durablePath?: string; + workDescription?: string | null; workerReadInlineBrief?: boolean; } const hash = (bytes: Uint8Array) => createHash("sha256").update(bytes).digest("hex"); +// Exact brief text, tolerant only of line endings, indentation, trailing spaces, blank lines and Markdown quote markers. +const briefLines = (text: string) => text.replace(/\r\n?/g, "\n").split("\n").map(line => line.trim().replace(/^(?:>\s?)+/, "").trim()).filter(Boolean); +export function containsExactBrief(context: string | null | undefined, brief: unknown): boolean { + if (typeof brief !== "string" || typeof context !== "string") return false; + const want = briefLines(brief); const have = briefLines(context); + return want.length > 0 && have.some((_, start) => want.every((line, offset) => have[start + offset] === line)); +} const workerProbeKeys = new WeakMap(); export function gradeAttachmentBrief(value: unknown) { const c = checks(); const o = observation(value) as BriefObservation | undefined; @@ -25,23 +33,24 @@ export function gradeAttachmentBrief(value: unknown) { && row.createdByUserId === receipt.createdByUserId && row.createdByAgentId == null && row.issueCommentId && row.sha256 === o.state.briefHash) && o.originalBytesHash === o.state.briefHash, "The original signed-in person's uploaded brief and its exact bytes remain attached to the actual source human comment."); c.add("one-contextual-ordinary-work-task", work.length === 1 && !!saved && saved.companyId === o.companyId && saved.assigneeAgentId === worker.id - && /\b(?:website|landing.page|web page)\b/i.test(`${saved.title} ${saved.description ?? ""}`) && o.activity.some(row => row.action === "issue.created" && row.entityId === saved.id && row.agentId === o.leadId) && hasLink(o, "issues", saved), "One actual CEO-created website task is delegated to the intended Morgan and linked in chat."); - const context = `${saved?.description ?? ""}\n${o.workDocuments.map(row => row.body ?? "").join("\n")}`; + const context = `${o.workDescription ?? saved?.description ?? ""}\n${o.workDocuments.map(row => row.body ?? "").join("\n")}`; const copied = o.workAttachments.some(row => row.issueId === saved?.id && row.companyId === o.companyId && row.sha256 === o.state.briefHash && o.durablePath === row.contentPath); const originalReference = !!receipt && context.includes(receipt.contentPath) && o.durablePath === receipt.contentPath; - c.add("durable-brief-in-work-context", copied || originalReference, - "The delegated task owns an exact-byte copy or a durable actual attachment reference; an unavailable filename or source-chat-only attachment fails."); - c.add("morgan-can-read-the-real-brief", o.workerTaskStatus === 200 && o.workerDownloadStatus === 200 && o.workerDownloadedHash === o.state.briefHash, - "An independent real Morgan bearer reads the work task and downloads the actual referenced brief bytes from its authorized context."); + const inline = containsExactBrief(context, o.state.briefBody); + c.add("durable-brief-in-work-context", copied || originalReference || inline, + "The delegated task owns an exact-byte copy, a durable actual attachment reference, or the complete exact brief text; an unavailable filename, a paraphrase or a source-chat-only attachment fails."); + const downloaded = o.workerDownloadStatus === 200 && o.workerDownloadedHash === o.state.briefHash; + c.add("morgan-can-read-the-real-brief", o.workerTaskStatus === 200 && (downloaded || (inline && o.workerReadInlineBrief === true)), + "An independent real Morgan bearer reads the work task and either downloads the actual referenced brief bytes or reads the complete exact brief in its own task context."); const runs = saved ? o.runs.filter(row => row.contextSnapshot?.issueId === saved.id || row.contextSnapshot?.taskId === saved.id) : []; c.add("separate-real-worker-admission", runs.length === 1 && runs[0]?.agentId === worker.id && ["queued", "running", "succeeded"].includes(runs[0]?.status ?? ""), "Morgan's real admitted run belongs to the new work task, separate from the source conversation."); return c.result(); } -async function workerReads(ctx: ChatCatalogContext, workerId: string, taskId: string, path: string | undefined) { +async function workerReads(ctx: ChatCatalogContext, workerId: string, taskId: string, path: string | undefined, brief: string) { const key = workerProbeKeys.get(ctx); if (!key || key.workerId !== workerId) throw new Error("Attempt-private Morgan probe credential is unavailable"); const get = (value: string) => { @@ -49,8 +58,18 @@ async function workerReads(ctx: ChatCatalogContext, workerId: string, taskId: st if (url.origin !== ctx.input.api.baseURL || !/^\/api\/(?:issues|attachments)\//.test(url.pathname)) throw new Error("Worker brief read escaped isolated resources"); return fetch(url, { redirect: "error", signal: AbortSignal.timeout(15_000), headers: { Authorization: `Bearer ${key.token}` } }); }; - const taskRead = await get(`/api/issues/${taskId}`); const workerTaskStatus = taskRead.status; await taskRead.body?.cancel(); - if (!path) return { workerTaskStatus }; + const taskRead = await get(`/api/issues/${taskId}`); const workerTaskStatus = taskRead.status; + if (!path) { + if (!taskRead.ok) { await taskRead.body?.cancel(); return { workerTaskStatus }; } + const parts = [((await taskRead.json()) as Entity).description ?? ""]; + const list = await get(`/api/issues/${taskId}/documents`); + if (list.ok) for (const summary of (await list.json()) as Entity[]) { + const doc = await get(`/api/issues/${taskId}/documents/${encodeURIComponent(summary.key!)}`); + if (doc.ok) parts.push(((await doc.json()) as Entity).body ?? ""); else await doc.body?.cancel(); + } else await list.body?.cancel(); + return { workerTaskStatus, workerReadInlineBrief: containsExactBrief(parts.join("\n"), brief) }; + } + await taskRead.body?.cancel(); const response = await get(path); const workerDownloadStatus = response.status; if (!response.ok) { await response.body?.cancel(); return { workerTaskStatus, workerDownloadStatus }; } const bytes = new Uint8Array(await response.arrayBuffer()); @@ -71,9 +90,9 @@ const base = makeDefinition("tasks.attachment-brief", async ctx => { if (!ctx.input.privateSecrets) throw new Error("Worker brief probe requires private secret redaction"); const key = await ctx.input.api.postSensitive<{ id: string; token: string }>(`/api/agents/${worker.id}/keys`, { name: "delegated-brief-read-control" }); ctx.input.privateSecrets.push(key.token); workerProbeKeys.set(ctx, { ...key, workerId: worker.id }); - return { worker, sourceId: source.id, briefAttachment: receipt, briefHash: hash(Buffer.from(briefBody)), workerProbeKeyId: key.id }; + return { worker, sourceId: source.id, briefAttachment: receipt, briefBody, briefHash: hash(Buffer.from(briefBody)), workerProbeKeyId: key.id }; }, gradeAttachmentBrief); -export const taskAttachmentCases: readonly ChatCatalogCase[] = [{ ...base, authentication: "human-session", +export const taskAttachmentCases: readonly ChatCatalogCase[] = [{ ...base, authentication: "human-session", oracleVersion: "paperclip.chat-catalog.attachment-brief.v2", queryAttachments: [{ stateKey: "briefAttachment", filename: "website-brief.md" }], expectedRunCount: 3, minimumRunCount: 2, runPolicy: { kind: "source-settled-with-active-work", maximumActiveWorkerRuns: 1 }, sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-task-attachment.ts", "chat-catalog-domains/identity-support.ts", "chat-query-attachments.ts"], @@ -83,16 +102,19 @@ export const taskAttachmentCases: readonly ChatCatalogCase[] = [{ ...base, authe const receipt = state.briefAttachment as ChatQueryAttachmentReceipt; const originalBytesHash = hash(await ctx.input.api.getAttachmentContent(receipt.id)); const saved = o.tasks.filter(row => row.id !== source?.id)[0]; const workAttachments: ChatQueryAttachmentReceipt[] = []; const workDocuments: Entity[] = []; + let workDescription: string | null | undefined; if (saved) { + // Company issue listings shorten descriptions; read the saved task in full. + workDescription = (await ctx.input.api.get(`/api/issues/${saved.id}`)).description; workAttachments.push(...await ctx.input.api.get(`/api/issues/${saved.id}/attachments`)); for (const summary of await ctx.input.api.get(`/api/issues/${saved.id}/documents`)) workDocuments.push(await ctx.input.api.get(`/api/issues/${saved.id}/documents/${encodeURIComponent(summary.key!)}`)); } - const context = `${saved?.description ?? ""}\n${workDocuments.map(row => row.body ?? "").join("\n")}`; + const context = `${workDescription ?? ""}\n${workDocuments.map(row => row.body ?? "").join("\n")}`; const durablePath = workAttachments.find(row => row.sha256 === state.briefHash)?.contentPath ?? (context.includes(receipt.contentPath) ? receipt.contentPath : undefined); - const reads = saved ? await workerReads(ctx, entity(state, "worker").id, saved.id, durablePath) : {}; - return { ...o, sourceId: source?.id, sourceAttachments, originalBytesHash, workAttachments, workDocuments, durablePath, ...reads }; + const reads = saved ? await workerReads(ctx, entity(state, "worker").id, saved.id, durablePath, String(state.briefBody ?? "")) : {}; + return { ...o, sourceId: source?.id, sourceAttachments, originalBytesHash, workAttachments, workDocuments, workDescription, durablePath, ...reads }; }, async cleanup(ctx, state) { const key = workerProbeKeys.get(ctx); workerProbeKeys.delete(ctx); diff --git a/tests/runner-e2e/chat-catalog-domains/internal.test.ts b/tests/runner-e2e/chat-catalog-domains/internal.test.ts index 989f2df63fa..ba5af3b469a 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal.test.ts @@ -650,6 +650,25 @@ describe("delegated attachment context calibration", () => { const o = valid("tasks.attachment-brief"); Object.assign(o, { workerDownloadedHash: "b".repeat(64) }); expect(gradeInternal(o).passed).toBe(false); }); + const inlineBrief = "# Acorn Field Notes\nBuild a simple accessible website.\nUse headline: Research without the busywork.\nOffer early signup with invitation code ORCHID-n1.\n"; + const inlined = (description: string, workerReadInlineBrief = true) => { + const o = valid("tasks.attachment-brief"); o.state.briefBody = inlineBrief; o.tasks[0]!.description = description.slice(0, 40); + Object.assign(o, { workAttachments: [], durablePath: undefined, workerDownloadStatus: undefined, workerDownloadedHash: undefined, + workDescription: description, workerReadInlineBrief }); + return o; + }; + it("accepts the complete exact brief copied into the full saved task that Morgan reads", () => { + const filler = "Acceptance criteria follow. ".repeat(60); + expect(gradeInternal(inlined(`${filler}\n\nBRIEF (verbatim):\n${inlineBrief}`)).passed).toBe(true); + expect(gradeInternal(inlined(`Brief:\r\n${inlineBrief.split("\n").map(line => `> ${line}`).join("\r\n")}`)).passed).toBe(true); + expect(gradeInternal(inlined(`Ship a marketing site.\n\`\`\`md\n${inlineBrief}\`\`\``)).passed).toBe(true); + }); + it("rejects a paraphrased, partial or Morgan-unreadable inline brief", () => { + expect(gradeInternal(inlined("Build a site for research teams with headline Research without the busywork and code ORCHID-n1.")).passed).toBe(false); + expect(gradeInternal(inlined(inlineBrief.split("\n").slice(0, 3).join("\n"))).passed).toBe(false); + expect(gradeInternal(inlined(`Use website-brief.md from the chat.`)).passed).toBe(false); + expect(gradeInternal(inlined(inlineBrief, false)).passed).toBe(false); + }); }); describe("governed completion calibration", () => {