test(evals): accept first-work's CEO owning and running the linked website task itself

The expectation is linked work under the available execution policy, not
delegation to Morgan. A run where the CEO created COM-2 for itself, ran it in
a separate task run and delivered index.html was rejected only for the
assignee. The task's assignee (the CEO or Morgan) must now be the agent that
uploaded the file from a run bound to the task.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DottaandClaude Opus 5.5 committed 2026-11-09 03:43:11 -06:00
1 parent 9375ac95cc
commit 61264662c6
3 files changed
+16 -6

No files matched your search

@@ -3712,7 +3712,7 @@
"chat-catalog-domains/internal-support.ts",
"chat-scenarios/onboarding.yaml"
],
"oracleVersion": "paperclip.chat-catalog.identity.first-work.v2",
"oracleVersion": "paperclip.chat-catalog.identity.first-work.v3",
"expectedRunCount": 3,
"minimumRunCount": 2,
"turnTimeoutMs": 120000,
@@ -25,16 +25,19 @@ async function actualWebsite(ctx: ChatCatalogContext, state: Record<string, unkn
}
export function gradeFirstWork(value: unknown) {
const c = checks(); const o = identityObservation(value); c.add("complete-first-website-work", !!o, "Actual first request, ordinary linked execution, authorized artifact bytes and browser rendering are required."); if (!o) return c.result();
const work = o.tasks[0]; const worker = entity(o.state, "worker"); const websites = o.state.websites as Website[] | undefined;
// The expectation is linked work under the available execution policy, not
// a particular assignee: the CEO may delegate to Morgan or own the task and
// run it itself. Either way the task's assignee must deliver the file.
const work = o.tasks[0]; const worker = entity(o.state, "worker"); const executor = [worker.id, o.leadId].includes(work?.assigneeAgentId ?? "") ? work!.assigneeAgentId! : undefined; const websites = o.state.websites as Website[] | undefined;
const website = websites?.find(file => file.attachment.id === o.state.reviewedAttachmentId);
c.add("actual-first-request-context", o.state.beforeSourceHumanComments === 0 && o.state.beforeSourceRuns === 0 && o.humanQueries.length === 1
&& o.humanQueries[0]?.trim() === "Make me a simple website introducing our business.", "The canonical request is the first real human instruction in this ready company's conversation.");
c.add("separate-requested-ordinary-work", o.tasks.length === 1 && work?.companyId === o.companyId && work.assigneeAgentId === worker.id && work.id !== o.state.sourceId && work.parentId == null
c.add("separate-requested-ordinary-work", o.tasks.length === 1 && work?.companyId === o.companyId && !!executor && work.id !== o.state.sourceId && work.parentId == null
&& work.status === "done" && hasLink(o, "issues", work) && o.state.sourceStillConversation === true
&& o.activity.some(row => row.action === "issue.created" && row.entityId === work.id && row.agentId === o.leadId), "The first chat creates one linked ordinary completed task; its standing conversation stays separate. Its delivered HTML (graded below), not its model-written title, proves it is the business website.");
c.add("actual-run-attributed-website", !!work && !!website && website.attachment.issueId === work.id && website.attachment.companyId === o.companyId
&& website.sha256 === website.attachment.sha256 && website.attachment.createdByAgentId === worker.id
&& o.runs.some(run => run.id === website.attachment.originatingRunId && run.agentId === worker.id && run.status === "succeeded"
&& website.sha256 === website.attachment.sha256 && !!executor && website.attachment.createdByAgentId === executor
&& o.runs.some(run => run.id === website.attachment.originatingRunId && run.agentId === executor && run.status === "succeeded"
&& (run.contextSnapshot?.issueId === work.id || (run as unknown as { nativeIssueId?: string }).nativeIssueId === work.id))
&& /<html\b/i.test(website.body) && /<body\b/i.test(website.body) && /<h[1-6]\b/i.test(website.body)
&& /Acorn Notes/i.test(website.body) && /research|science/i.test(website.body), "Authorized exact HTML bytes match the immutable actual upload hash and originating worker run, introducing the real business rather than an empty onboarding task.");
@@ -49,7 +52,7 @@ export const firstWorkIdentityCases: readonly ChatCatalogCase[] = [identityDefin
const comments = await ctx.input.api.get<Entity[]>(`/api/issues/${source.id}/comments`);
return { sourceId: source.id, worker, project: p, beforeSourceHumanComments: comments.filter(comment => comment.authorUserId).length, beforeSourceRuns: (await ctx.allRuns()).length };
}, gradeFirstWork, {
oracleVersion: "paperclip.chat-catalog.identity.first-work.v2", authentication: "human-session", expectedRunCount: 3, minimumRunCount: 2, attemptTimeoutMs: 600_000,
oracleVersion: "paperclip.chat-catalog.identity.first-work.v3", authentication: "human-session", expectedRunCount: 3, minimumRunCount: 2, attemptTimeoutMs: 600_000,
sourceFiles: ["chat-catalog-domains/identity-first-work.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/onboarding.yaml"],
async act(ctx, state) {
const o = await pollUntil({ label: "first real completed website and delivered HTML", deadlineAt: Date.now() + 360_000,
@@ -31,6 +31,13 @@ describe("human company creation and genuine first completed website", () => {
const file = (o.state.websites as Array<{ body: string }>)[0]!; file.body = "<html><body><header><a class=\"brand\">Acorn Notes</a></header><h1>Capture every observation</h1><p>A notebook for small science teams.</p></body></html>";
expect(gradeFirstWork(o).passed).toBe(true);
});
it("accepts the CEO owning and running the linked task itself, but not an unrelated agent", () => {
const own = website(); own.tasks[0]!.assigneeAgentId = "ceo"; own.runs[0]!.agentId = "ceo"; (own.state.websites as Array<{ attachment: Record<string, unknown> }>)[0]!.attachment.createdByAgentId = "ceo";
expect(gradeFirstWork(own).passed).toBe(true);
const other = website(); other.tasks[0]!.assigneeAgentId = "stranger"; other.runs[0]!.agentId = "stranger"; (other.state.websites as Array<{ attachment: Record<string, unknown> }>)[0]!.attachment.createdByAgentId = "stranger";
expect(gradeFirstWork(other).passed).toBe(false);
const mismatch = website(); (mismatch.state.websites as Array<{ attachment: Record<string, unknown> }>)[0]!.attachment.createdByAgentId = "ceo"; expect(gradeFirstWork(mismatch).passed).toBe(false);
});
it("rejects a rendered page without a visible heading or without the business name", () => {
const heading = website(); heading.state.renderedHeading = false; expect(gradeFirstWork(heading).passed).toBe(false);
const name = website(); name.state.renderedBusinessName = false; expect(gradeFirstWork(name).passed).toBe(false);