diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-queue.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-queue.test.ts index 613c6cb3ca5..f9f87a9b51e 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-queue.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-queue.test.ts @@ -1,5 +1,18 @@ import { it, expect } from "vitest"; -import { gradeQueuedFollowup, promptCarriesComment } from "./runtime-queue.js"; +import { deflateRawSync } from "node:zlib"; +import { gradeQueuedFollowup, promptCarriesComment, zipEntryTexts } from "./runtime-queue.js"; +/** A minimal zip with one deflated and one stored entry, laid out as zip tools write it. */ +function zip(entries: Array<{ name: string; text: string; deflate: boolean }>) { + const locals: Buffer[] = [], centrals: Buffer[] = []; let offset = 0; + for (const entry of entries) { + const name = Buffer.from(entry.name), raw = Buffer.from(entry.text), data = entry.deflate ? deflateRawSync(raw) : raw; + const local = Buffer.alloc(30); local.writeUInt32LE(0x04034b50, 0); local.writeUInt16LE(entry.deflate ? 8 : 0, 8); local.writeUInt32LE(data.length, 18); local.writeUInt32LE(raw.length, 22); local.writeUInt16LE(name.length, 26); + const central = Buffer.alloc(46); central.writeUInt32LE(0x02014b50, 0); central.writeUInt16LE(entry.deflate ? 8 : 0, 10); central.writeUInt32LE(data.length, 20); central.writeUInt32LE(raw.length, 24); central.writeUInt16LE(name.length, 28); central.writeUInt32LE(offset, 42); + locals.push(local, name, data); centrals.push(central, name); offset += 30 + name.length + data.length; + } + const directory = Buffer.concat(centrals), end = Buffer.alloc(22); end.writeUInt32LE(0x06054b50, 0); end.writeUInt16LE(entries.length, 8); end.writeUInt16LE(entries.length, 10); end.writeUInt32LE(directory.length, 12); end.writeUInt32LE(offset, 16); + return Buffer.concat([...locals, directory, end]); +} function valid() { return { scenarioId: "steering.queue-follow-up", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", state: { task: { id: "website", identifier: "PAP-7" }, worker: { id: "morgan" }, workerStatusAtReply: "running", canonicalRunId: "controller", queuedAtReply: { issueId: "website", state: "queued", entries: [{ position: 0, comment: { id: "contact", body: "Add a contact section", authorAgentId: "ceo", createdByRunId: "controller" } }] }, @@ -19,6 +32,14 @@ it("rejects a successor without the actual saved comment in its native input", ( it("rejects duplicate requirement admission", () => { const o = valid(); o.state.queuedAtReply.entries.push({ ...o.state.queuedAtReply.entries[0]!, position: 1 }); expect(gradeQueuedFollowup(o).passed).toBe(false); }); it("rejects losing the original worker's saved work", () => { const o = valid(); o.state.initialBriefRevisionRetained = false; expect(gradeQueuedFollowup(o).passed).toBe(false); }); it("rejects output attributed to the wrong native revision", () => { const o = valid(); o.state.contactResultRevision = false; expect(gradeQueuedFollowup(o).passed).toBe(false); }); +it("accepts the contact section delivered as the successor's registered build instead of a document", () => { const o = valid(); o.state.contactResultRevision = false; Object.assign(o.state, { contactResultDeliverable: true }); expect(gradeQueuedFollowup(o).passed).toBe(true); }); +it("rejects a successor deliverable without the contact section", () => { const o = valid(); o.state.contactResultRevision = false; Object.assign(o.state, { contactResultDeliverable: false }); expect(gradeQueuedFollowup(o).passed).toBe(false); }); +it("reads stored and deflated zip entries and treats other bytes as no entries", () => { + const bytes = zip([{ name: "site/index.html", text: '

Contact

', deflate: true }, { name: "site/styles.css", text: "body{margin:0}", deflate: false }]); + expect(zipEntryTexts(bytes)).toEqual(['

Contact

', "body{margin:0}"]); + expect(zipEntryTexts(Buffer.from("Contact"))).toEqual([]); + expect(zipEntryTexts(bytes.subarray(0, bytes.length - 30))).toEqual([]); +}); it("finds the saved comment in a native prompt raw or JSON-escaped, and only with its id and body", () => { const comment = { id: "c-1", body: "Add a \"Contact us\" section\nwith email." }; const prompt = `{"newComments":[{"id":"c-1","body":${JSON.stringify(comment.body)}}]}`; @@ -28,3 +49,11 @@ it("finds the saved comment in a native prompt raw or JSON-escaped, and only wit expect(promptCarriesComment("continue the website task c-1", comment)).toBe(false); expect(promptCarriesComment(undefined, comment)).toBe(false); }); +it("finds the saved comment when a continuation envelope nests the events JSON as a string", () => { + const comment = { id: "c-1", body: "Add a \"Contact\" section.\n\n- nav link" }; + const prompt = JSON.stringify({ schema: "paperclip.native-continuation.v1", events: JSON.stringify({ messages: [{ id: "c-1", body: comment.body }, { id: "c-2", body: "Done." }] }) }); + expect(prompt.includes(comment.body) || prompt.includes(JSON.stringify(comment.body).slice(1, -1))).toBe(false); + expect(promptCarriesComment(prompt, comment)).toBe(true); + expect(promptCarriesComment(JSON.stringify({ events: JSON.stringify({ messages: [{ id: "c-1", body: "Add a section." }] }) }), comment)).toBe(false); + expect(promptCarriesComment(JSON.stringify({ events: JSON.stringify({ messages: [{ id: "c-2", body: comment.body }], note: "c-1" }) }), comment)).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-queue.ts b/tests/runner-e2e/chat-catalog-domains/runtime-queue.ts index 5ef32e3ed4d..e81e173eb88 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-queue.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-queue.ts @@ -1,15 +1,46 @@ import { chatCatalogNativePhaseDeadline, chatCatalogNativePhaseIsTimely } from "../chat-catalog-contract.js"; import { expect } from "@playwright/test"; +import { createHash } from "node:crypto"; import { writeFile } from "node:fs/promises"; +import { inflateRawSync } from "node:zlib"; import { pollUntil } from "../api.js"; import { checks, type Entity } from "./internal-support.js"; import { busyRuntimeWorker, releaseRuntimeWorker } from "./runtime-busy-worker.js"; import { runtimeDefinition, runtimeObservation, observeRuntime, type RuntimeAgent } from "./runtime-support.js"; -import type { IssueQueuedCommentQueue } from "../../../packages/shared/src/types/issue.js"; -/** Native continuations embed new comments as JSON, so a body with quotes or newlines appears JSON-escaped. */ +import type { IssueAttachment, IssueQueuedCommentQueue } from "../../../packages/shared/src/types/issue.js"; +import type { IssueWorkProduct } from "../../../packages/shared/src/types/work-product.js"; +const MAX_DELIVERABLE_BYTES = 262_144; +/** Text of each stored or deflated entry in a zip deliverable, read from its central directory; [] when it is not a readable zip. */ +export function zipEntryTexts(bytes: Buffer): string[] { + try { + const eocd = bytes.lastIndexOf(Buffer.from([0x50, 0x4b, 0x05, 0x06])); if (eocd < 0 || eocd + 22 > bytes.length) return []; + const texts: string[] = []; let at = bytes.readUInt32LE(eocd + 16); + for (let index = 0, count = bytes.readUInt16LE(eocd + 10); index < count && at + 46 <= bytes.length && bytes.readUInt32LE(at) === 0x02014b50; index++) { + const method = bytes.readUInt16LE(at + 10), size = bytes.readUInt32LE(at + 20), local = bytes.readUInt32LE(at + 42); + const start = local + 30 + bytes.readUInt16LE(local + 26) + bytes.readUInt16LE(local + 28), data = bytes.subarray(start, start + size); + if (method === 0) texts.push(data.toString("utf8")); + else if (method === 8) texts.push(inflateRawSync(data, { maxOutputLength: 4 * MAX_DELIVERABLE_BYTES }).toString("utf8")); + at += 46 + bytes.readUInt16LE(at + 28) + bytes.readUInt16LE(at + 30) + bytes.readUInt16LE(at + 32); + } + return texts; + } catch { return []; } +} +/** Find a message object with the exact comment id and body, descending into JSON nested as string values. */ +function jsonCarriesComment(value: unknown, comment: { id: string; body: string }, depth = 0): boolean { + if (depth > 6) return false; + if (typeof value === "string") { + const text = value.trim(); if (!/^[[{]/.test(text)) return false; + try { return jsonCarriesComment(JSON.parse(text), comment, depth + 1); } catch { return false; } + } + if (Array.isArray(value)) return value.some(item => jsonCarriesComment(item, comment, depth + 1)); + if (!value || typeof value !== "object") return false; + const row = value as Record; + return (row.id === comment.id && row.body === comment.body) || Object.values(row).some(item => jsonCarriesComment(item, comment, depth + 1)); +} +/** Native continuations embed new comments as JSON (an envelope may nest it again as a string), so a body with quotes or newlines appears escaped. */ export function promptCarriesComment(prompt: string | undefined, comment: { id: string; body: string }) { return typeof prompt === "string" && prompt.includes(comment.id) - && (prompt.includes(comment.body) || prompt.includes(JSON.stringify(comment.body).slice(1, -1))); + && (prompt.includes(comment.body) || prompt.includes(JSON.stringify(comment.body).slice(1, -1)) || jsonCarriesComment(prompt, comment)); } /** A task is named by its setup-assigned identifier (which chat links) or by a link to its page. */ export function replyNamesTask(reply: string, task: Entity) { @@ -37,9 +68,9 @@ export function gradeQueuedFollowup(value: unknown) { && o.runs.length === 3 && o.runs.every(run => run.status === "succeeded") && o.runs.filter(run => run.agentId === (o.state.worker as Entity).id).length === 2, "The same saved requirement is consumed by one ordinary successor native turn after the real prior owner settles."); - c.add("earlier-work-and-contact-result-retained", o.state.initialBriefRevisionRetained === true && o.state.contactResultRevision === true + c.add("earlier-work-and-contact-result-retained", o.state.initialBriefRevisionRetained === true && (o.state.contactResultRevision === true || o.state.contactResultDeliverable === true) && o.tasks.length === 1 && o.tasks[0]?.id === task.id && o.tasks[0]?.assigneeAgentId === (o.state.worker as Entity).id && o.tasks[0]?.status === "done", - "The original worker's saved brief revision remains inspectable and the successor saves an actual contact-section result on the same task."); + "The original worker's saved brief revision remains inspectable and the successor saves an actual contact-section result on the same task, as a document revision or a registered deliverable."); return c.result(); } export const runtimeQueueCases = [runtimeDefinition("steering.queue-follow-up", busyRuntimeWorker, gradeQueuedFollowup, { @@ -91,6 +122,18 @@ export const runtimeQueueCases = [runtimeDefinition("steering.queue-follow-up", const successorIds = o.runs.filter(run => run.agentId === (state.worker as Entity).id && run.id !== state.workerRunId).map(run => run.id); state.contactResultRevision = o.revisions.some(revision => successorIds.includes(revision.createdByRunId ?? "") && /contact/i.test(revision.body ?? "") && revision.createdByAgentId === (state.worker as Entity).id && o.documents.some(doc => doc.id === revision.documentId && doc.latestRevisionId === revision.id)); + // A website build may be delivered as a registered file (often a zip) rather than a document; its file name is the model's, so read every successor deliverable's verified bytes. + const taskId = (state.task as Entity).id; + const [products, attachments] = await Promise.all([ctx.input.api.get(`/api/issues/${taskId}/work-products`), ctx.input.api.get(`/api/issues/${taskId}/attachments`)]); + const delivered = attachments.filter(attachment => attachment.issueId === taskId && successorIds.includes(attachment.originatingRunId ?? "") && attachment.byteSize <= MAX_DELIVERABLE_BYTES + && products.some(product => product.type === "artifact" && product.createdByRunId === attachment.originatingRunId && product.metadata?.attachmentId === attachment.id)); + const texts: string[] = []; + for (const attachment of delivered) { + const bytes = await ctx.input.api.getAttachmentContent(attachment.id); + if (bytes.length !== attachment.byteSize || createHash("sha256").update(bytes).digest("hex") !== attachment.sha256) continue; + const entries = zipEntryTexts(bytes); texts.push(...(entries.length > 0 ? entries : [bytes.toString("utf8")])); + } + state.contactResultDeliverable = texts.some(text => /contact/i.test(text)); return o; }, async verifyResult(ctx, state) {