import { withRestoreConcurrency, identifiesRevision } from "./internal-restore-concurrency.js"; import { projectMarkdownToText, resolveProjectionRange, createDocumentAnchorSelector } from "../../../packages/shared/src/document-anchors.js"; import type { ChatCatalogCase } from "../chat-catalog-contract.js"; import { privateDownloadCases } from "./internal-private-download.js"; import { project, task, goal, putDocument, entity, observation, checks, hasLink, sameTask, type Entity } from "./internal-support.js"; import { makeDefinition, retryDefinition, fixtureWorker } from "./internal-definition.js"; import { selectDocumentSource } from "./internal-document-context.js"; import type { IssueDocument } from "../../../packages/shared/src/types/issue.js"; import { factualContextQueryAttachment } from "../chat-query-attachments.js"; export function grade_documents_artifacts(value: unknown) { const c = checks(); const o = observation(value); c.add("complete-independent-observation", !!o, "Missing API, document, audit or rendered-link evidence cannot pass."); if (!o) return c.result(); const before = o.state.task ? entity(o.state, "task") : undefined; const current = before && o.tasks.find(row => row.id === before.id); const savedProject = o.state.project ? entity(o.state, "project") : undefined; const projectNow = savedProject && o.projects.find(row => row.id === savedProject.id); const authored = (action: string, id: string) => o.activity.some(row => row.action === action && row.entityId === id && row.agentId === o.leadId); const linkedTask = () => !!current && hasLink(o, "issues", current); switch (o.scenarioId) { case "documents-artifacts.write-plan": { const plans = o.documents.filter(row => row.key === "plan"); const plan = plans[0]; c.add("saved-implementation-plan", plans.length === 1 && !!plan && !!plan.latestRevisionId && planCoversBrief(plan.body ?? ""), "A durable plan grounded in the website task's brief, rather than chat prose or a generic plan."); c.add("versioned-author", !!plan && o.revisions.some(row => row.id === plan.latestRevisionId && row.body === plan.body && row.createdByAgentId === o.leadId && o.runs.some(run => run.id === row.createdByRunId)), "The current document revision is attributed to this real agent run."); c.add("plan-audit", !!before && authored("issue.document_created", before.id), "The API audit records the CEO creating the plan after the request."); c.add("task-context-and-link", !!before && sameTask(before, current) && linkedTask(), "Saving a plan preserves the task and makes the saved plan reachable through its task."); break; } case "documents-artifacts.resolve-feedback": { const target = entity(o.state, "pricingThread"); const sibling = entity(o.state, "siblingThread"); const annotation = o.annotations?.find(row => row.id === target.id); const unrelated = o.annotations?.find(row => row.id === sibling.id); const plan = o.documents.find(row => row.key === "plan"); c.add("substantive-pricing-fix", !!plan && monthlyTeamPricing(plan.body ?? "") && (plan.body ?? "").includes(String(o.state.supportText)), "The actual current document unambiguously preserves $19 monthly total for the team and unrelated support content."); c.add("correct-thread-resolved", annotation?.status === "resolved" && annotation.resolvedByAgentId === o.leadId && unrelated?.status === "open" && annotation.documentId === plan?.id && annotation.originalRevisionId === o.state.originalRevisionId, "Only the anchored pricing annotation is resolved by the CEO, preserving its original document and revision relationship."); const edited = o.activity.find(row => row.action === "issue.document_updated" && row.agentId === o.leadId); const resolved = o.activity.find(row => row.action === "issue.document_annotation_thread_resolved" && row.agentId === o.leadId && row.details?.threadId === target.id); c.add("edit-before-resolution", !!edited && !!resolved && Date.parse(edited.createdAt ?? "") <= Date.parse(resolved.createdAt ?? "") && !!plan && o.revisions.some(row => row.id === plan.latestRevisionId && row.createdByAgentId === o.leadId && row.body === plan.body), "Saved revision and independent audit events establish the substantive edit before resolution."); c.add("task-and-document-access", !!before && sameTask(before, current) && linkedTask(), "Chat links the changed document's task without modifying task execution."); break; } case "documents-artifacts.restore-document": { const plan = o.documents.find(row => row.key === "plan"); const revisions = o.state.originalRevisions as Entity[]; const previous = revisions?.[1]; c.add("restored-exact-previous-bytes", !!previous && !!plan && plan.body === previous.body && plan.latestRevisionId !== previous.id && plan.latestRevisionId !== revisions[2]?.id, "Restoration creates a new current revision with the exact second revision's contents."); c.add("retained-history", Array.isArray(revisions) && revisions.length === 3 && o.revisions.length === 4 && revisions.every(original => o.revisions.some(row => row.id === original.id && row.body === original.body)), "All three historical revisions remain unmodified alongside the restoration."); c.add("restore-path-attribution", !!before && o.activity.some(row => row.action === "issue.document_restored" && row.entityId === before.id && row.agentId === o.leadId && row.details?.restoredFromRevisionId === previous?.id), "An independently observed restore audit identifies the exact source revision."); c.add("no-execution-rewind", !!before && sameTask(before, current) && !!plan && identifiesRevision(o.reply, plan), "Chat identifies the resulting current document revision and leaves task execution unchanged."); break; } default: c.add("implemented-scenario", false, "No independent oracle is implemented for this scenario."); } return c.result(); } const define = (id: string, setup: ChatCatalogCase["setup"]): ChatCatalogCase => { const base = makeDefinition(id, setup, grade_documents_artifacts); return id === "documents-artifacts.restore-document" || id === "documents-artifacts.resolve-feedback" ? { ...base, authentication: "human-session", queryAttachments: factualContextQueryAttachment("selected-source.md") } : base; }; export const documents_artifactsCases: readonly ChatCatalogCase[] = [ ...privateDownloadCases, resolveFeedback(), { ...define("documents-artifacts.write-plan", async ctx => { const p = await project(ctx); const t = await task(ctx, "Website task", p.id, { assigneeAgentId: ctx.input.fixtures.agent.id, description: WEBSITE_PLAN_BRIEF }); return { project: p, task: t }; }), oracleVersion: "paperclip.chat-catalog.write-plan.brief-coverage.v2" }, withRestoreConcurrency(define("documents-artifacts.restore-document", async ctx => { const p = await project(ctx); const t = await task(ctx, "Website task", p.id, { description: "The website task's plan has three saved revisions. The previous version before the last edit is the desired content." }); const revisions: Entity[] = []; for (const body of [`# Plan\nBuild a responsive website for ${ctx.input.nonce}.`, `# Plan\nBuild accessible signup for ${ctx.input.nonce}.\nVerify keyboard operation.`, `# Plan\nReplace signup with an unrelated pricing experiment for ${ctx.input.nonce}.`]) { const saved = await putDocument(ctx, t.id, body, revisions.at(-1)?.id ?? null); revisions.push({ id: saved.latestRevisionId!, body }); } const document = await ctx.input.api.get(`/api/issues/${t.id}/documents/plan`); const selectedSource = await selectDocumentSource(ctx, { task: t, document }); return { project: p, task: t, originalRevisions: revisions, selectedSource, factualContextAttachment: selectedSource.factualContextAttachment }; })), ]; function resolveFeedback(): ChatCatalogCase { const base = define("documents-artifacts.resolve-feedback", async ctx => { const p = await project(ctx); const t = await task(ctx, "Website task", p.id); const pricing = "Pricing is $19 every month and $19 per user every month."; const supportText = `Support is provided by the launch team ${ctx.input.nonce}.`; const body = `${pricing}\n\n${supportText}`; const doc = await putDocument(ctx, t.id, body); const createAnnotation = (quote: string, feedback: string) => { const projection = projectMarkdownToText(body); const start = projection.text.indexOf(quote); const range = resolveProjectionRange(projection, start, start + quote.length); if (!range || range.text !== quote) throw new Error("Fixture annotation quote did not resolve through production text projection"); return ctx.input.api.post(`/api/issues/${t.id}/documents/plan/annotations`, { baseRevisionId: doc.latestRevisionId, baseRevisionNumber: doc.latestRevisionNumber, selector: createDocumentAnchorSelector(projection, range), body: feedback, }); }; const pricingThread = await createAnnotation(pricing, "Clarify that $19 is the monthly total for the team, not per person. Keep the amount and billing cadence."); const siblingThread = await createAnnotation(supportText, "Please add a support contact later; leave this unrelated comment open for now."); const selectedSource = await selectDocumentSource(ctx, { task: t, document: { id: doc.id, key: "plan", latestRevisionId: doc.latestRevisionId! }, annotationId: pricingThread.id }); return { project: p, task: t, pricingThread, siblingThread, supportText, originalRevisionId: doc.latestRevisionId, selectedSource, factualContextAttachment: selectedSource.factualContextAttachment }; }); return { ...base, sourceFiles: [...base.sourceFiles, "../../packages/shared/src/document-anchors.ts"], async observe(ctx, state) { const o = await observeInternal(ctx, state); return { ...o, annotations: await ctx.input.api.get(`/api/issues/${entity(state, "task").id}/documents/plan/annotations?status=all&includeComments=true`) }; } }; } import { observeInternal } from "./internal-support.js"; /** The website brief the write-plan setup saves on the task; the grader measures the plan against this same text. */ export const WEBSITE_PLAN_BRIEF = "Build a single-page website for Orchid, with accessible signup and responsive layout. Verify keyboard navigation and mobile presentation."; /** * Whether a saved plan is grounded in the task's brief. Free-form plans paraphrase, so this does not * require any particular word: it counts how many of the brief's content words the plan covers * (matching inflections such as verify/verification and joined forms such as "sign up"/"signup") * and accepts a plan that covers at least half of them. Generic or foreign plans cover few. */ export function planCoversBrief(body: string, brief = WEBSITE_PLAN_BRIEF): boolean { const stop = new Set(["a", "an", "and", "for", "of", "the", "to", "with"]); const words = (text: string) => text.toLowerCase().split(/[^a-z0-9]+/).filter(Boolean); const required = [...new Set(words(brief).filter(word => !stop.has(word)))]; const tokens = words(body); const present = new Set([...tokens, ...tokens.slice(1).map((word, index) => tokens[index] + word)]); const stem = (word: string) => word.length > 5 ? word.slice(0, 5) : word; const stems = new Set([...present].map(stem)); const covered = required.filter(word => present.has(word) || stems.has(stem(word))); return covered.length * 2 >= required.length; } export function monthlyTeamPricing(body: string): boolean { const pricingClauses = body.split(/(?<=[.!?])\s+|[\n;]|\bbut\b/i); if (!pricingClauses.some(clause => /\$19(?:\.00)?(?![\d.])/.test(clause) && /\b(?:monthly|month)\b/i.test(clause) && /\bteam\b/i.test(clause) && /\b(?:total|entire|whole|per team|for the team)\b/i.test(clause))) return false; const negated = (index: number) => /\b(?:not|never|no|rather than|instead of)(?:\s+(?:charged|billed|priced|calculated|applied))?(?:\s+(?:a|an|the))?(?:\s+\$[\d,.]+)?\s*$/i.test(body.slice(Math.max(0, index - 80), index)); for (const amount of body.matchAll(/\$([\d,]+(?:\.\d+)?)/g)) { if (Number(amount[1]!.replaceAll(",", "")) !== 19 && !negated(amount.index!)) return false; if (Number(amount[1]!.replaceAll(",", "")) === 19 && negated(amount.index!) && !/\bper[- ](?:user|person|seat)\b/i.test(body.slice(amount.index! + amount[0].length).split(/[.;\n]/)[0]!)) return false; } for (const scope of body.matchAll(/\b(?:per[- ]|each\s+|every\s+)(?:user|person|seat)\b/gi)) { const suffix = body.slice(scope.index! + scope[0].length); const deniedAfter = /^\s+(?:(?:pricing|billing|charge)\s+)?(?:is|would be)\s+(?:wrong|incorrect|not (?:correct|applicable|required))\b/i.test(suffix); if (!negated(scope.index!) && !deniedAfter) return false; } for (const cadence of body.matchAll(/\b(?:yearly|annually|annual|per year|one[- ]time)\b/gi)) if (!negated(cadence.index!)) return false; return !/\b(?:not|never|no)\s+(?:monthly|per month|(?:the\s+)?total|(?:(?:for|per)\s+(?:the\s+)?)?(?:(?:entire|whole)\s+)?team)\b/i.test(body); }