import { isAssignedSkillContext, PAPERCLIP_DOCUMENT_CASE, type ContextIntegrityCase } from "./context-integrity-cases.js"; export interface ContextIntegrityCheckpoint { phase: "initial" | "comment-1" | "comment-2" | "comment-3" | "final"; issue: { id: string; status: string; identifier?: string; issuePrefix?: string; appOrigin?: string }; comments: Array>; queuedComments?: Record; documents: Array<{ key: string; body?: string | null; latestRevisionId?: string | null; latestRevisionNumber?: number | null }>; runs: Array>; assignedSkill?: { key: string; runtimeName?: string; versionId?: string | null; markdown?: string }; skillRequestText?: string; runEvents?: Array>; runLogs?: unknown[]; skillInvocationEvidence?: boolean; } function userComments(checkpoint: ContextIntegrityCheckpoint) { return checkpoint.comments .filter( (comment) => !comment.createdByRunId && (comment.authorType === "user" || comment.authorUserId), ) .map((comment) => String(comment.body ?? "")); } function humanCommentRows(checkpoint: ContextIntegrityCheckpoint) { return checkpoint.comments.filter( (comment) => !comment.createdByRunId && (comment.authorType === "user" || comment.authorUserId), ); } function oneOutput(checkpoint: ContextIntegrityCheckpoint, marker: string, requireMarker: boolean) { const outputs = checkpoint.documents.filter((document) => !requireMarker || String(document.body ?? "").includes(marker), ); return outputs.length === 1 ? outputs[0] : undefined; } function wakeCommentIds(run: Record): string[] { const context = (run.contextSnapshot ?? {}) as Record; const paperclipWake = (context.paperclipWake ?? {}) as Record; const continuation = (context.executionContinuation ?? {}) as Record; for (const candidate of [paperclipWake.commentIds, paperclipWake.wakeCommentIds, continuation.commentIds, continuation.wakeCommentIds]) { if (Array.isArray(candidate)) return candidate.map(String); } return []; } export function gradeContextIntegrity(input: { id: ContextIntegrityCase; marker: string; comments: readonly string[]; checkpoints: readonly ContextIntegrityCheckpoint[]; }) { const checks: Array<{ id: string; passed: boolean; detail: string }> = []; const check = (id: string, passed: boolean, detail: string) => checks.push({ id, passed, detail }); const initial = input.checkpoints.find((checkpoint) => checkpoint.phase === "initial"); const final = input.checkpoints.find((checkpoint) => checkpoint.phase === "final"); const finalComments = final ? userComments(final) : []; if (input.id === "ordered-comment-continuation") { const queued = input.checkpoints.find((checkpoint) => checkpoint.phase === "comment-3")?.queuedComments; const queuedEntries = Array.isArray(queued?.entries) ? queued.entries : []; const queuedRows = queuedEntries.map((entry) => { const comment = (entry as Record).comment as Record | undefined; return { id: String(comment?.id ?? ""), body: String(comment?.body ?? "") }; }); check("comments-queued-as-batch", queuedRows.length >= input.comments.length && input.comments.every((body, index) => queuedRows[index]?.body === body) && queuedRows.slice(0, input.comments.length).every((row) => Boolean(row.id)) && new Set(queuedRows.slice(0, input.comments.length).map((row) => row.id)).size === input.comments.length, "The three public comments must be present in one ordered deferred queue with distinct durable IDs before the initial run closes."); const initialUserComments = (initial?.comments ?? []).filter((comment) => !comment.createdByRunId && (comment.authorType === "user" || comment.authorUserId)); check("initial-run-started", initial?.runs.some((run) => run.status === "running") === true && initialUserComments.length === 0, "The initial public run must be running before follow-up comments are submitted, with no follow-up user comments in the initial checkpoint."); check( "ordered-comments", input.comments.every((body, index) => finalComments[index] === body) && finalComments.length >= input.comments.length, "The three user comments must remain distinct and ordered, including the intentional repeated comment.", ); const humanRows = final ? humanCommentRows(final) : []; const ids = humanRows.map((comment) => String(comment.id ?? "")); check( "distinct-comment-identities", ids.length === new Set(ids).size && ids.every(Boolean), "Each delivered user comment must retain its own durable identity, including repeated wording.", ); check( "changed-scope-preserved", finalComments.at(2) === input.comments[2] && finalComments.at(0) === finalComments.at(1), "The final changed-scope comment must follow two identical earlier comments.", ); const report = final?.documents.find((document) => /passport/i.test(String(document.body ?? "")) && /charger/i.test(String(document.body ?? ""))); const body = String(report?.body ?? ""); const orderedTerms = [input.comments[0], input.comments[1], input.comments[2]]; let cursor = -1; const ordered = orderedTerms.every((term) => { const position = body.indexOf(term, cursor + 1); if (position < 0) return false; cursor = position + term.length - 1; return true; }); check( "packing-report-order", /passport/i.test(body) && /charger/i.test(body) && ordered, "The durable packing report must retain both initial items and each verbatim request in order, including the repeated request and final scope.", ); check( "final-scope-applied", ["launch checklist", "the launch checklist"].includes( body.match(/(?:^|\n)##[ \t]+Final scope[ \t]*\r?\n([\s\S]*?)(?=\n#{1,6}[ \t]+|$)/i)?.[1]?.trim().toLowerCase() ?? "", ), "The report must apply the final Launch checklist scope in a separate final-scope section outside the quoted request ledger.", ); const succeededRunIds = (final?.runs ?? []).filter((run) => run.status === "succeeded").map((run) => String(run.id ?? "")).filter(Boolean); check("continuation-run-count", new Set(succeededRunIds).size === 2, "The initial run and one distinct successful deferred continuation run must both be recorded."); const continuation = (final?.runs ?? []) .filter((run) => run.status === "succeeded") .sort((a, b) => Date.parse(String(a.startedAt ?? "")) - Date.parse(String(b.startedAt ?? "")))[1]; const expectedCommentIds = humanRows.slice(0, input.comments.length).map((comment) => String(comment.id ?? "")); check("continuation-wake-comment-ids", Boolean(continuation) && JSON.stringify(wakeCommentIds(continuation)) === JSON.stringify(expectedCommentIds), "The deferred continuation wake must carry all public comment IDs in arrival order."); } if (isAssignedSkillContext(input.id)) { const runtimeName = String(initial?.assignedSkill?.runtimeName ?? initial?.assignedSkill?.key ?? ""); const requestText = String(initial?.skillRequestText ?? ""); const references = requestText.split(/\s+/).map((word) => word.replace(/[.,]$/, "")); check("assigned-skill-present", Boolean(initial?.assignedSkill?.key && initial.assignedSkill.versionId), "The task run must receive one pinned skill version through the public assignment state."); check("skill-request-explicit", Boolean(runtimeName && references.some((word) => word === `/${runtimeName}` || word === `$${runtimeName}`)), "The task request must explicitly name the assigned skill with a supported slash or dollar reference."); check("skill-source-marker", Boolean(initial?.assignedSkill?.markdown?.includes(input.marker)), "The assigned pinned skill source must contain the output marker."); check("marker-not-in-request", !requestText.includes(input.marker) && !finalComments.some((comment) => comment.includes(input.marker)), "The output marker must originate from the assigned skill, not the task request or comments."); } const output = final ? oneOutput(final, input.marker, isAssignedSkillContext(input.id)) : undefined; check("single-durable-output", Boolean(output) && final!.documents.length === 1, isAssignedSkillContext(input.id) ? "Exactly one durable task document must contain the skill's marker." : "Exactly one durable packing report document must be saved."); if (input.id === PAPERCLIP_DOCUMENT_CASE) { check("saved-document-revision", Boolean(output?.latestRevisionId) && Number.isInteger(output?.latestRevisionNumber) && Number(output?.latestRevisionNumber) > 0, "The public saved document must have a persisted revision and the requested content."); const route = output && final?.issue.identifier && final.issue.issuePrefix ? `/${final.issue.issuePrefix}/issues/${final.issue.identifier}#document-${output.key}` : undefined; const agentComments = (final?.comments ?? []).filter(comment => comment.authorType === "agent" || Boolean(comment.authorAgentId)); const origin = final?.issue.appOrigin; const usableLink = Boolean(route && origin && agentComments.some(comment => { const links = String(comment.body ?? "").matchAll(/\[[^\]]*\]\((?:<([^>]+)>|([^\s)]+))(?:\s+["'][^"']*["'])?\)/g); return [...links].some(link => { try { const url = new URL(link[1] ?? link[2]!, origin); return url.origin === origin && `${url.pathname}${url.hash}` === route && !url.search; } catch { return false; } }); })); check("saved-document-link", usableLink, "An agent completion comment must link the exact saved document on this task; a local path or claimed URL does not suffice."); } check("completed-task", final?.issue.status === "done", `Final task status: ${final?.issue.status ?? "missing"}.`); check("successful-runs", Boolean(final?.runs.length) && final!.runs.every((run) => run.status === "succeeded"), "All recorded context-integrity runs must succeed."); return checks; }