test(evals): observe team install reply within original clock

Bind the actual human and canonical native source before independent installed-runtime readback. Preserve installer, partial-outcome, repeat, runtime, query and original480s contract.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/chat-catalog-domains/external-team-install-original-deadline.test.ts

@@ -0,0 +1,41 @@
+import { afterEach, expect, it, vi } from "vitest";
+import type { ChatCatalogContext } from "../chat-catalog-contract.js";
+vi.mock("../codex-subscription-fixture.js", () => ({ codexSubscriptionAgentConfig: (value: unknown) => value }));
+import { teamInstallExternalCases } from "./external-team-install.js";
+afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); });
+const start = 10_000, deadline = start + 450_000;
+type Fault = "failed" | "cancelled" | "foreign actor" | "foreign company" | "foreign source" | "foreign native comment" | "foreign human" | "foreign reply" | "missing reply" | "late read" | "no native";
+function fixture(fault?: Fault, budget: number | "missing" = deadline) {
+  vi.useFakeTimers(); vi.setSystemTime(start); let readbackReached = false;
+  const query = "Install that website team in this company.";
+  const run = { id: "initial", companyId: fault === "foreign company" ? "other" : "company", agentId: fault === "foreign actor" ? "morgan" : "ceo", runtimeMode: "native",
+    contextSnapshot: { issueId: fault === "foreign source" ? "other" : "source", commentId: fault === "foreign native comment" ? "other" : "human" } };
+  const comments = [{ id: "human", companyId: "company", issueId: "source", authorUserId: fault === "foreign human" ? "riley" : "person", body: query },
+    ...(fault === "missing reply" ? [] : [{ id: "reply", companyId: "company", issueId: "source", authorAgentId: fault === "foreign reply" ? "morgan" : "ceo", createdByRunId: "initial", body: "Installed the graph; a required skill remains blocked by the local conflict." }])];
+  const ctx = { ...(budget === "missing" ? {} : { observationDeadlineAt: budget }), scenario: { id: "teams.install-team", query },
+    input: { fixtures: { company: { id: "company" }, agent: { id: "ceo" } }, api: { get: async (url: string) => {
+      if (url === "/api/issues/source/comments?order=asc") { if (fault === "late read") vi.setSystemTime(deadline + 1); return comments; }
+      if (url === "/api/companies/company/agents") { readbackReached = true; throw Error("NEXT_INDEPENDENT_RUNTIME_READBACK"); }
+      return [];
+    } } }, issue: () => ({ id: "source", companyId: "company", conversationState: "waiting" }), refreshIssue: async () => {},
+    allRuns: async () => fault === "no native" ? [] : [{ ...run, status: fault === "failed" || fault === "cancelled" ? fault : Date.now() < (fault ? start : start + 190_000) ? "running" : "succeeded" }],
+  } as unknown as ChatCatalogContext;
+  async function act(ms = 191_000) {
+    const pending = teamInstallExternalCases[0]!.act!(ctx, { sourceIssueId: "source", humanId: "person" }).then(() => undefined, (error: Error) => error);
+    await vi.advanceTimersByTimeAsync(ms); return pending;
+  }
+  return { ctx, act, readbackReached: () => readbackReached };
+}
+it("actual installer act observes a canonical response after190s in its original budget before existing independent readback", async () => {
+  const f = fixture(); expect((await f.act())?.message).toBe("NEXT_INDEPENDENT_RUNTIME_READBACK"); expect(f.readbackReached()).toBe(true);
+  expect(f.ctx.observationDeadlineAt).toBe(deadline); expect(teamInstallExternalCases[0]!.attemptTimeoutMs).toBe(480_000);
+});
+it.each(["failed", "cancelled", "foreign actor", "foreign company", "foreign source", "foreign native comment", "foreign human", "foreign reply", "missing reply", "late read"] as const)("rejects %s before independent readback or repeat", async fault => {
+  const f = fixture(fault); expect(await f.act()).toBeInstanceOf(Error); expect(f.readbackReached()).toBe(false);
+});
+it.each(["missing", NaN, Infinity, start, start - 1] as const)("rejects invalid original deadline%s", async budget => {
+  const f = fixture(undefined, budget); expect(await f.act(1000)).toBeInstanceOf(Error); expect(f.readbackReached()).toBe(false);
+});
+it("does not invent a native response when no source work exists", async () => {
+  const f = fixture("no native"); expect(await f.act(451_000)).toBeInstanceOf(Error); expect(f.readbackReached()).toBe(false);
+});

tests/runner-e2e/chat-catalog-domains/external-team-install.ts

@@ -22,7 +22,8 @@ import {
   checks,
   verifyRenderedReply,
 } from "./external-support.js";
-import { waitForReply } from "./identity-support.js";
+import { remainingIdentityObservationBudget, waitForReply } from "./identity-support.js";
+import type { Entity } from "./internal-support.js";
 interface Resource {
   id: string;
   companyId: string;
@@ -315,7 +316,11 @@ export const teamInstallExternalCases = [
       await ctx.input.api.patch(`/api/issues/${ctx.issue()!.id}`, {
         description: `The accepted website team preview is ${team.name}, catalog key ${team.key}, content hash ${team.contentHash}. Install the complete selected package under the current CEO using this company's existing eligible runtime account. No team resource is installed yet. The selected runtime uses ${actor.adapterType} in environment ${ctx.input.fixtures.environment.id}. The existing eligible runtime is OpenCode/OpenRouter ${selectedRuntime.model}; its credential stays in the existing same-company secret reference. The supported installer takes these per-slug adapterOverrides for this accepted runtime selection: ${JSON.stringify(adapterOverrides)}.`,
       });
+      const session = await ctx.input.api.get<{ user: { id: string } }>("/api/auth/get-session");
+      if (!session.user?.id) throw new Error("Team installer requires its actual responsible human session");
       return {
+        sourceIssueId: ctx.issue()!.id,
+        humanId: session.user.id,
         ...conditional,
         selectedRuntime,
         team,
@@ -349,7 +354,28 @@ export const teamInstallExternalCases = [
         "chat-scenarios/teams.yaml",
       ],
       async act(ctx, state) {
-        await waitForReply(ctx);
+        const remaining = () => remainingIdentityObservationBudget(ctx);
+        remaining(); await waitForReply(ctx, ctx.observationDeadlineAt); remaining();
+        const source = ctx.issue();
+        if (!source || source.id !== state.sourceIssueId || source.companyId !== ctx.input.fixtures.company.id)
+          throw new Error("Team install initial response has the wrong conversation source");
+        const [runs, comments] = await Promise.all([
+          ctx.allRuns(), ctx.input.api.get<Entity[]>(`/api/issues/${source.id}/comments?order=asc`, { timeout: remaining() }),
+        ]);
+        remaining();
+        const human = comments.filter(row => row.companyId === source.companyId && row.issueId === source.id
+          && row.authorUserId === state.humanId && !row.authorAgentId && !row.createdByRunId
+          && row.body?.trim() === ctx.scenario.query.trim());
+        const run = runs[0];
+        if (ctx.issue()?.id !== source.id || source.conversationState !== "waiting" || runs.length !== 1 || !run
+          || run.companyId !== source.companyId || run.agentId !== ctx.input.fixtures.agent.id || run.status !== "succeeded"
+          || run.runtimeMode !== "native" || run.contextSnapshot?.issueId !== source.id || human.length !== 1
+          || run.contextSnapshot?.commentId !== human[0]?.id
+          || (run.contextSnapshot.taskId !== undefined && run.contextSnapshot.taskId !== source.id)
+          || (run.nativeIssueId != null && run.nativeIssueId !== source.id)
+          || !comments.some(row => row.companyId === source.companyId && row.issueId === source.id
+            && row.authorAgentId === run.agentId && row.createdByRunId === run.id && !!row.body?.trim()))
+          throw new Error("Team install initial reply is not bound to its canonical human and native request");
         state.first = await observeTeamResources(ctx);
         state.runtimeReadback = await verifyInstalledTeamRuntime(ctx, state.team as CatalogTeam, state.first as TeamResourceSnapshot, state.selectedRuntime as TeamInstallRuntime);
         state.initialRunId = (await ctx.allRuns())[0]?.id;