diff --git a/tests/runner-e2e/chat-catalog-contract.ts b/tests/runner-e2e/chat-catalog-contract.ts index 23d5f5ebfcc..fc8e4a6ed1a 100644 --- a/tests/runner-e2e/chat-catalog-contract.ts +++ b/tests/runner-e2e/chat-catalog-contract.ts @@ -48,7 +48,7 @@ export interface ChatSlackIngressReceipt { /** One concrete scenario, with production API setup and a case-specific independent oracle. */ export interface NativeStopBoundaryReceipt { runId: string; issueId: string; intentId: string; intentRecordedAt: string } export interface ChatCatalogCase { - /** Only post-stop-message: actual human Stop after bookkeeping, before unchanged canonical send. */ + /** Only post-stop-message: Stop the genuine same-source setup turn after bookkeeping, then send the unchanged canonical query. */ canonicalBoundary?: "native-stop-cleanup"; beforeCanonicalSend?(context: ChatCatalogContext, state: Record): Promise; scenarioId: string; diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-native-fault.ts b/tests/runner-e2e/chat-catalog-domains/runtime-native-fault.ts index 38fe91b794c..6baf96619c6 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-native-fault.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-native-fault.ts @@ -10,7 +10,7 @@ import { defaultCapabilityRunnerdBinary } from "../../../server/src/vendor/paper import { pollUntil } from "../api.js"; // The public full-run route selects these persisted native fields; the legacy // shared HeartbeatRun DTO does not yet declare them. Keep this exact projection. -export type NativeFaultRun = HeartbeatRun & Pick; +export type NativeFaultRun = HeartbeatRun & Pick; export type NativeFaultBinding = Pick; export interface OwnedNativeIdentity { pid: number; uid: number; group: number; startedAt: string; executable: string; commandRunId: string; commandRunnerId: string; runtimeRoot: string; @@ -42,7 +42,7 @@ async function processIdentity(run: NativeFaultRun): Promise ctx.input.api.get(`/api/heartbeat-runs/${binding.runId}`); // Deliberately signal only the verified Runner PID. Its provider/tools remain // subject to the production cleanup/quarantine rules; never signal a group. process.kill(confirmed.pid, "SIGKILL"); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-post-stop.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-post-stop.test.ts new file mode 100644 index 00000000000..2fe371e7feb --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-post-stop.test.ts @@ -0,0 +1,20 @@ +import { expect, it } from "vitest"; +import { gradePostStop, sameSourcePostStopChronology } from "./runtime-post-stop.js"; +function chronology() { + const old = { id: "old", nativeIssueId: "source", companyId: "company", agentId: "ceo", runtimeMode: "native", status: "cancelled", finishedAt: "2026-10-26T12:00:02Z", resultJson: { nativeCancellation: { schema: "paperclip.native-cancellation.v1", intentId: "intent", intentAuditId: "audit-intent", acknowledgementAuditId: "audit-ack", issueId: "source", companyId: "company", runId: "old", scope: "run", recordedAt: "2026-10-26T12:00:00Z", acknowledgedAt: "2026-10-26T12:00:02Z", dispatchState: "acknowledged", dispatched: true } } }; + return { old, next: { id: "new", nativeIssueId: "source", companyId: "company", agentId: "ceo", runtimeMode: "native", status: "succeeded", startedAt: "2026-10-26T12:00:03Z" }, boundary: { runId: "old", issueId: "source", intentId: "intent", intentRecordedAt: "2026-10-26T12:00:00Z" }, human: { id: "canonical", body: "Now make the website a single page instead.", issueId: "source", companyId: "company", authorAgentId: null, authorUserId: "person", createdAt: "2026-10-26T12:00:01Z" }, query: "Now make the website a single page instead.", companyId: "company", agentId: "ceo", humanId: "person", prompt: "canonical\nNow make the website a single page instead.", wakeCommentIds: ["canonical"] } as unknown as Parameters[0]; +} +function valid() { + return { companyId: "company", leadId: "ceo", agents: [], links: [], humanQueries: ["initial", "canonical"], reply: "", tasks: [], activity: [], runs: [{ id: "old", agentId: "ceo", status: "cancelled" }, { id: "new", agentId: "ceo", status: "succeeded" }], state: { stopStatus: 200, chronologyBound: true, providerExited: true, initialHumanRetained: true, singleSuccessor: true, sameCurrentAssignment: true, oldEvidenceRetained: true, actualSimplePage: true, artifactRunBound: true, canonicalReply: "The single-page website is built and saved here." } }; +} +it("binds the real unchanged canonical human instruction between same-source Stop intent and cleanup ACK", () => expect(sameSourcePostStopChronology(chronology())).toBe(true)); +it("rejects wrong-source, early, late, duplicate, resumed or unacknowledged deliveries", () => { + for (const change of [(v: ReturnType) => { v.human.issueId = "other"; }, (v: ReturnType) => { v.old.nativeIssueId = "other"; }, (v: ReturnType) => { v.next.agentId = "other"; }, (v: ReturnType) => { v.human.createdAt = "2026-10-26T12:00:03Z" as never; }, (v: ReturnType) => { v.next.startedAt = "2026-10-26T12:00:01Z" as never; }, (v: ReturnType) => { v.wakeCommentIds.push("initial"); }, (v: ReturnType) => { v.prompt = "a different instruction"; }, (v: ReturnType) => { v.old.status = "succeeded"; }, (v: ReturnType) => { (v.old.resultJson!.nativeCancellation as Record).dispatchState = "pending"; }]) { const v = chronology(); change(v); expect(sameSourcePostStopChronology(v)).toBe(false); } +}); +it("accepts real post-cleanup delivery and a natural substantive result paraphrase", () => { + expect(gradePostStop(valid()).passed).toBe(true); const o = valid(); o.state.canonicalReply = "Acorn Notes is ready, with its signup action and all the content together. The artifact is saved in this conversation."; expect(gradePostStop(o).passed).toBe(true); +}); +it("rejects polite completion without actual same-source cleanup, delivery or rendered output", () => { + for (const key of ["chronologyBound", "providerExited", "initialHumanRetained", "singleSuccessor", "sameCurrentAssignment", "oldEvidenceRetained", "actualSimplePage", "artifactRunBound"] as const) { const o = valid(); o.state[key] = false; expect(gradePostStop(o).passed).toBe(false); } + const o = valid(); o.state.canonicalReply = "The website is still queued and not built."; expect(gradePostStop(o).passed).toBe(false); expect(gradePostStop(undefined).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-post-stop.ts b/tests/runner-e2e/chat-catalog-domains/runtime-post-stop.ts new file mode 100644 index 00000000000..901181f8008 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-post-stop.ts @@ -0,0 +1,113 @@ +import { expect, type Page } from "@playwright/test"; +import { readFile, writeFile } from "node:fs/promises"; +import type { ChatCatalogContext, NativeStopBoundaryReceipt } from "../chat-catalog-contract.js"; +import type { IssueComment } from "../../../packages/shared/src/types/issue.js"; +import { pollUntil } from "../api.js"; +import { sendChatMessage } from "../chat-flow.js"; +import { collectRunEvents } from "../run-observations.js"; +import { readRegisteredArtifacts } from "../registered-artifact.js"; +import { checks } from "./internal-support.js"; +import { openStandingChat } from "./identity-support.js"; +import { prepareRuntimeWorkerGate } from "./runtime-busy-worker.js"; +import { suspendOwnedProviderForCleanup, releaseOwnedCleanupProvider, ownedCleanupProviderHasExited } from "./runtime-provider-cleanup.js"; +import type { NativeFaultRun } from "./runtime-native-fault.js"; +import { hasSimplerWebsite, renderRuntimeArtifact } from "./runtime-rendered-artifact.js"; +import { observeRuntime, runtimeDefinition, runtimeObservation, type RuntimeAgent } from "./runtime-support.js"; +interface StopIntent { schema?: string; intentId?: string; intentAuditId?: string; companyId?: string; runId?: string; issueId?: string; scope?: string; dispatchState?: string; dispatched?: boolean; recordedAt?: string; acknowledgedAt?: string; acknowledgementAuditId?: string } +interface StopHandle { page: Page; completed?: Promise } +const handles = new WeakMap(); +export function sameSourcePostStopChronology(input: { boundary: NativeStopBoundaryReceipt; old: NativeFaultRun; next: NativeFaultRun; human: Pick; query: string; companyId: string; agentId: string; humanId: string; prompt: string; wakeCommentIds: string[] }) { + const { old, next, human, boundary: b } = input, intent = old.resultJson?.nativeCancellation as StopIntent | undefined; + const before = Date.parse(b.intentRecordedAt), sent = Date.parse(String(human.createdAt)), ack = Date.parse(intent?.acknowledgedAt ?? ""), started = Date.parse(String(next.startedAt)), ended = Date.parse(String(old.finishedAt)); + return old.id === b.runId && old.nativeIssueId === b.issueId && old.companyId === input.companyId && old.agentId === input.agentId && old.runtimeMode === "native" && old.status === "cancelled" + && intent?.schema === "paperclip.native-cancellation.v1" && intent.intentId === b.intentId && intent.issueId === b.issueId && intent.companyId === input.companyId && intent.runId === old.id && intent.scope === "run" + && !!intent.intentAuditId && !!intent.acknowledgementAuditId && intent.recordedAt === b.intentRecordedAt && intent.dispatchState === "acknowledged" && intent.dispatched === true + && human.issueId === b.issueId && human.companyId === input.companyId && !human.authorAgentId && human.authorUserId === input.humanId && human.body === input.query + && before <= sent && sent < ack && next.id !== old.id && next.nativeIssueId === b.issueId && next.companyId === input.companyId && next.agentId === input.agentId && next.runtimeMode === "native" && next.status === "succeeded" + && started >= ack && started >= ended && input.wakeCommentIds.length === 1 && input.wakeCommentIds[0] === human.id && input.prompt.includes(human.id) && input.prompt.includes(input.query); +} +export function gradePostStop(value: unknown) { + const c = checks(), o = runtimeObservation(value); + c.add("actual-same-source-stop-continuation-evidence", !!o, "A real setup human request, actual native provider cleanup, canonical human instruction and normal continuation must be independently observed."); if (!o) return c.result(); + const s = o.state; + c.add("human-stop-cleanup-and-exact-message-chronology", s.stopStatus === 200 && s.chronologyBound === true && s.providerExited === true && s.initialHumanRetained === true, + "The same actual source run has a real Stop intent and ACK; the canonical human message persisted in between, unchanged, and only its eligible successor consumes it after cleanup."); + c.add("one-eligible-current-owner-with-no-revival-or-duplicate", o.runs.length === 2 && o.runs.filter(row => row.status === "cancelled").length === 1 && o.runs.filter(row => row.status === "succeeded").length === 1 + && o.runs.every(row => row.agentId === o.leadId) && s.singleSuccessor === true && s.sameCurrentAssignment === true && s.oldEvidenceRetained === true && o.humanQueries.length === 2 && o.tasks.length === 0, + "The original stopped execution remains cancelled, one native source continuation owns the new direction under current assignment, and no duplicate task or user message appears."); + c.add("actual-visible-single-page-result-from-continuation", s.actualSimplePage === true && s.artifactRunBound === true && /single[ -]page|one page|Acorn|website|site|page/i.test(String(s.canonicalReply)) && /ready|built|created|updated|implemented|saved|now/i.test(String(s.canonicalReply)) + && !/still (?:queued|waiting)|not (?:yet )?(?:built|implemented|ready)|could not (?:build|implement)|kept.*(?:tabs|multi[ -]page)/i.test(String(s.canonicalReply)), + "The subsequent native reply reports the real saved simple single page; registered bytes and visible design independently prove the result."); + return c.result(); +} +export const runtimePostStopCases = [runtimeDefinition("recovery.post-stop-message", async ctx => { + const source = await openStandingChat(ctx), lead = await ctx.input.api.get(`/api/agents/${ctx.input.fixtures.agent.id}`); + const { gate } = await prepareRuntimeWorkerGate(ctx, { worker: lead }); + const secondary = await ctx.input.page.context().newPage(); handles.set(ctx, { page: secondary }); + try { + const first = `Build an Acorn Notes website with a visible heading, product paragraph and labelled signup action. Initially use several panels and tabs. Work directly in this conversation; save and register the actual website.html artifact here, without delegating another task. First run the bounded command node ${gate.scriptPath} once and wait for its output before implementing the website. Do not replace the command or invent its output. Preserve later human changes to this same website.`; + await sendChatMessage(ctx.input.page, first); + const running = await pollUntil({ label: "Genuine initial source Website request executing its bounded native tool", deadlineAt: Date.now() + 150000, + load: async () => ({ runs: await ctx.allRuns(), waiting: await readFile(gate.ready, "utf8").catch(() => "") }), + accept: v => v.waiting === "waiting" && v.runs.length === 1 && v.runs[0]?.agentId === lead.id && v.runs[0].status === "running", + reject: v => v.runs.some(row => row.status !== "running" && row.status !== "queued") ? "The genuine initial source execution ended before Stop setup" : undefined }); + const original = await ctx.input.api.get(`/api/heartbeat-runs/${running.runs[0]!.id}`); + if (original.nativeIssueId !== source.id || original.runtimeMode !== "native") throw new Error("The initial native execution is not bound to the actual canonical source chat"); + const people = await ctx.input.api.get<{ user: { id: string } }>("/api/auth/get-session"); + const comments = await ctx.input.api.get(`/api/issues/${source.id}/comments?order=asc`), initial = comments.filter(row => !row.authorAgentId); + if (initial.length !== 1 || initial[0]?.body !== first || initial[0].authorUserId !== people.user.id) throw new Error("The initial Website request is not the exact genuine signed-in human setup turn"); + await secondary.goto(ctx.input.page.url(), { waitUntil: "domcontentloaded" }); await expect(secondary.getByTestId("task-chat-composer-stop")).toBeEnabled(); + return { gate, sourceId: source.id, workerRunId: original.id, initialHuman: initial[0], humanId: people.user.id }; + } catch (error) { await writeFile(gate.gate, "The pending fixture tool has been released.", "utf8"); await secondary.close(); handles.delete(ctx); throw error; } +}, gradePostStop, { + canonicalBoundary: "native-stop-cleanup", authentication: "human-session", setupHumanMessageCount: 1, expectedHumanMessageCount: 2, expectedRunCount: 2, attemptTimeoutMs: 720000, + runPolicy: { kind: "terminal-counts", terminalStatusCounts: { cancelled: 1, succeeded: 1 } }, + sourceFiles: ["chat-catalog-domains/runtime-post-stop.ts", "chat-catalog-domains/runtime-provider-cleanup.ts", "chat-catalog-domains/runtime-native-fault.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-rendered-artifact.ts", "chat-catalog-domains/runtime-support.ts", "chat-native-stop-boundary.ts", "../../server/src/services/native-runtime/native-session-executor.ts", "../../server/src/services/explicit-native-continuation.ts", "../../packages/paperclip-runner/src/live/runnerd-codex-transport.ts", "../../packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs", "../../packages/paperclip-runner/runner/crates/runner-core/src/process_supervisor.rs", "registered-artifact.ts", "chat-scenarios/recovery.yaml"], + setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["actual-source-native-owner"] }, + async verifySetup(ctx, state) { const c = checks(), run = await ctx.input.api.get(`/api/heartbeat-runs/${state.workerRunId}`); + c.add("actual-source-native-owner", run.status === "running" && run.nativeIssueId === ctx.issue()?.id && run.agentId === ctx.input.fixtures.agent.id && run.runtimeMode === "native", "The actual canonical conversation already has its genuine native setup owner, without fabricated status or response history."); return c.result(); }, + async beforeCanonicalSend(ctx, state) { + const handle = handles.get(ctx); if (!handle || !ctx.issue()) throw new Error("Actual same-source human Stop tab is unavailable"); + await suspendOwnedProviderForCleanup(ctx, { runId: String(state.workerRunId), issueId: ctx.issue()!.id, agentId: ctx.input.fixtures.agent.id }); + const endpoint = `/api/heartbeat-runs/${state.workerRunId}/cancel`; + const response = handle.page.waitForResponse(r => r.request().method() === "POST" && new URL(r.url()).pathname === endpoint, { timeout: 30000 }); + handle.completed = response.then(r => r.status()); void handle.completed.catch(() => undefined); + await handle.page.getByTestId("task-chat-composer-stop").click(); + const run = await pollUntil({ label: "Actual public human Stop durably records pending native cleanup intent", deadlineAt: Date.now() + 1000, intervalMs: 25, + load: () => ctx.input.api.get(`/api/heartbeat-runs/${state.workerRunId}`), + accept: row => (row.resultJson?.nativeCancellation as StopIntent | undefined)?.dispatchState === "pending" }); + const intent = run.resultJson!.nativeCancellation as StopIntent; + if (!intent.intentId || !intent.recordedAt || !intent.intentAuditId || intent.issueId !== ctx.issue()!.id) throw new Error("The actual pending Stop lacks its native source/audit identity"); + return { runId: run.id, issueId: ctx.issue()!.id, intentId: intent.intentId, intentRecordedAt: intent.recordedAt }; + }, + async act(ctx, state) { + const settled = await pollUntil({ label: "One normal same-source continuation after actual stopped-provider cleanup", deadlineAt: Date.now() + 300000, + load: async () => { await ctx.refreshIssue(); return { runs: await ctx.allRuns(), source: ctx.issue() }; }, + accept: v => v.source?.conversationState === "waiting" && v.runs.length === 2 && v.runs.find(row => row.id === state.workerRunId)?.status === "cancelled" && v.runs.find(row => row.id !== state.workerRunId)?.status === "succeeded", + reject: v => v.runs.length > 2 || v.runs.some(row => row.id !== state.workerRunId && ["failed", "timed_out", "cancelled"].includes(row.status)) ? "The actual continuation duplicated or failed" : undefined }); + state.stopStatus = await handles.get(ctx)!.completed; + const old = await ctx.input.api.get(`/api/heartbeat-runs/${state.workerRunId}`), successor = settled.runs.find(row => row.id !== old.id)!; + const next = await ctx.input.api.get(`/api/heartbeat-runs/${successor.id}`), comments = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`); + const human = comments.filter(row => !row.authorAgentId), canonical = human[1]; + const native = next.runnerProfileJson?.nativeExecutionInput as { task?: { prompt?: string } } | undefined; + state.chronologyBound = !!canonical && sameSourcePostStopChronology({ boundary: state.nativeStopBoundary as NativeStopBoundaryReceipt, old, next, human: canonical, query: ctx.scenario.query, companyId: ctx.input.fixtures.company.id, agentId: ctx.input.fixtures.agent.id, humanId: String(state.humanId), prompt: native?.task?.prompt ?? "", wakeCommentIds: (next.contextSnapshot?.wakeCommentIds as string[] | undefined) ?? [] }); + state.initialHumanRetained = human[0]?.id === (state.initialHuman as IssueComment).id && human[0]?.body === (state.initialHuman as IssueComment).body; + state.singleSuccessor = settled.runs.filter(row => row.id !== old.id).length === 1; + state.sameCurrentAssignment = ctx.issue()?.assigneeAgentId === next.agentId || (ctx.issue() as { conversationAgentId?: string }).conversationAgentId === next.agentId; + const events = await collectRunEvents<{ seq?: number; eventType?: string; createdAt?: string; payload?: { schema?: string; runId?: string; nativeSessionId?: string } }>((afterSeq, limit) => ctx.input.api.get(`/api/heartbeat-runs/${old.id}/events?afterSeq=${afterSeq}&limit=${limit}`)); + state.oldEvidenceRetained = events.length > 0 && old.status === "cancelled"; + state.providerExited = await ownedCleanupProviderHasExited(ctx); + if (old.errorCode === "native_session_cleanup_quarantined") state.chronologyBound = state.chronologyBound === true && events.some(event => event.eventType === "native.stopped_conversation_verified" && event.payload?.schema === "paperclip.stopped_native_conversation.v1" && event.payload.runId === old.id && event.payload.nativeSessionId === old.nativeSessionId && Date.parse(event.createdAt ?? "") <= Date.parse(String(next.startedAt))); + state.chronologyBound = state.chronologyBound === true && !!next.nativeSessionId && next.nativeSessionId !== old.nativeSessionId; + state.canonicalReply = comments.filter(row => row.createdByRunId === next.id && row.authorAgentId === next.agentId).map(row => row.body).join("\n"); state.successorRunId = next.id; + await releaseOwnedCleanupProvider(ctx); + }, + async observe(ctx, state) { + const o = await observeRuntime(ctx, state); + if (state.successorRunId) { const [artifact] = await readRegisteredArtifacts(ctx.input.api, String(state.sourceId), String(state.successorRunId), ["website.html"]); + state.artifactRunBound = artifact?.contentVerified === true; state.actualSimplePage = !!artifact?.contentVerified && hasSimplerWebsite(await renderRuntimeArtifact(ctx, artifact.content)); } + return o; + }, + async verifyResult(ctx) { await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); }, + async cleanup(ctx, state) { try { await releaseOwnedCleanupProvider(ctx); } finally { if (state.gate) await writeFile((state.gate as { gate: string }).gate, "The pending fixture tool has been released.", "utf8"); const handle = handles.get(ctx); if (handle) { await handle.completed?.catch(() => undefined); await handle.page.close(); handles.delete(ctx); } } }, +})]; diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-provider-cleanup.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-provider-cleanup.test.ts new file mode 100644 index 00000000000..8b916f47c6f --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-provider-cleanup.test.ts @@ -0,0 +1,12 @@ +import { expect, it } from "vitest"; +import { matchesOwnedProvider } from "./runtime-provider-cleanup.js"; +function fixture() { + return { identity: { pid: 987654, parentPid: 987653, uid: 501, startedAt: "2026-10-26T12:00:01Z", executable: "/owned/bin/codex", digest: "a".repeat(64), appServer: true }, expected: { providerPid: 987654, providerStartedAt: "2026-10-26T12:00:01Z", runnerPid: 987653, uid: 501, runStartedAt: "2026-10-26T12:00:00Z" } }; +} +it("qualifies only the exact checkpoint provider with the actual direct owned Runner parent", () => { + expect(matchesOwnedProvider(fixture())).toBe(true); + for (const patch of [{ pid: process.pid }, { pid: 987655 }, { parentPid: 987652 }, { uid: 502 }, { startedAt: "2026-10-26T11:00:00Z" }, { executable: "/owned/bin/unrelated" }, { digest: "unknown" }, { appServer: false }]) { + const f = fixture(); expect(matchesOwnedProvider({ ...f, identity: { ...f.identity, ...patch } })).toBe(false); + } + const f = fixture(); f.expected.providerStartedAt = "2026-10-26T12:00:00Z"; expect(matchesOwnedProvider(f)).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-provider-cleanup.ts b/tests/runner-e2e/chat-catalog-domains/runtime-provider-cleanup.ts new file mode 100644 index 00000000000..015da824404 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-provider-cleanup.ts @@ -0,0 +1,63 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { createHash } from "node:crypto"; +import { readFile, realpath } from "node:fs/promises"; +import path from "node:path"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { collectRunEvents } from "../run-observations.js"; +import { readProcessStartedAt } from "../../../server/src/services/hot-restart.js"; +import { verifyOwnedNativeRunner } from "./runtime-native-fault.js"; +interface ProviderIdentity { pid: number; parentPid: number; uid: number; startedAt: string; executable: string; digest: string; appServer: boolean } +interface SuspendedOwner { identity: ProviderIdentity; runId: string; timer: ReturnType } +const suspended = new WeakMap(); +const retainedProvider = new WeakMap(); +export function matchesOwnedProvider(input: { identity: ProviderIdentity; expected: { providerPid: number; providerStartedAt: string; runnerPid: number; uid: number; runStartedAt: string } }) { + const { identity: p, expected: e } = input; + return Number.isSafeInteger(p.pid) && p.pid > 1 && p.pid !== process.pid && p.pid === e.providerPid && p.pid !== e.runnerPid && p.parentPid === e.runnerPid && p.uid === e.uid + && p.startedAt === e.providerStartedAt && Date.parse(p.startedAt) + 1_000 >= Date.parse(e.runStartedAt) && p.appServer === true && path.basename(p.executable) === "codex" + && /^[a-f0-9]{64}$/.test(p.digest); +} +async function providerIdentity(pid: number): Promise { + try { + const ps = async (field: string) => (await promisify(execFile)("/bin/ps", ["-p", String(pid), "-o", `${field}=`], { timeout: 5_000, maxBuffer: 65_536 })).stdout.trim(); + const [uid, parent, executable, command, startedAt] = await Promise.all([ps("uid"), ps("ppid"), ps("comm"), ps("args"), readProcessStartedAt(pid)]); + const canonical = await realpath(executable); if (!startedAt) throw new Error("missing start identity"); + return { pid, parentPid: Number(parent), uid: Number(uid), startedAt, executable: canonical, + digest: createHash("sha256").update(await readFile(canonical)).digest("hex"), appServer: /(?:^|\s)app-server(?=\s|$)/.test(command) }; + } catch { throw new Error("Provider cleanup fixture could not qualify the exact owned process; private diagnostics withheld"); } +} +/** This does not signal any group or control ACK timing. Actual public Stop must immediately follow. */ +export async function suspendOwnedProviderForCleanup(ctx: ChatCatalogContext, binding: { runId: string; issueId: string; agentId: string }) { + if (ctx.scenario.id !== "recovery.post-stop-message" || suspended.has(ctx)) throw new Error("Bounded provider cleanup suspension is exclusive to the selected post-stop scenario"); + const { run, identity: runner } = await verifyOwnedNativeRunner(ctx, binding); + const checkpoint = run.runnerProfileJson?.sessionCheckpoint as { process?: { providerPid?: number; providerProcessStartedAt?: string } } | undefined; + const provider = run.runnerProfileJson?.nativeExecutionInput as { provider?: { kind?: string } } | undefined; + const pid = checkpoint?.process?.providerPid, started = checkpoint?.process?.providerProcessStartedAt; + if (provider?.provider?.kind !== "codex" || !Number.isSafeInteger(pid) || !pid || !started) throw new Error("Actual public native checkpoint lacks a qualified local Codex provider PID/start identity"); + const events = await collectRunEvents<{ seq?: number; payload?: { prpEvent?: { runId?: string; normalizedSessionId?: string; sourceKind?: string; sourceInstanceId?: string; eventType?: string; payload?: { processId?: number; providerSessionId?: string; driverSessionId?: string } } } }>((afterSeq, limit) => ctx.input.api.get(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=${limit}`)); + if (!events.some(row => { const e = row.payload?.prpEvent; return e?.runId === run.id && e.sourceKind === "runner" && e.sourceInstanceId === run.runnerInstanceId && e.normalizedSessionId === run.nativeSessionId + && ["session.started", "session.resumed"].includes(e.eventType ?? "") && e.payload?.processId === pid && !!(e.payload.providerSessionId ?? e.payload.driverSessionId); })) throw new Error("Provider suspension lacks the exact durable native session/PID receipt"); + const expected = { providerPid: pid, providerStartedAt: started, runnerPid: runner.pid, uid: runner.uid, runStartedAt: String(run.startedAt) }; + const first = await providerIdentity(pid), second = await providerIdentity(pid); + if (!matchesOwnedProvider({ identity: first, expected }) || !matchesOwnedProvider({ identity: second, expected }) || JSON.stringify(first) !== JSON.stringify(second)) throw new Error("Provider cleanup suspension rejected changed or unowned process identity"); + process.kill(pid, "SIGSTOP"); retainedProvider.set(ctx, second); + const timer = setTimeout(() => { void releaseOwnedCleanupProvider(ctx).catch(() => undefined); }, 8_000); timer.unref(); suspended.set(ctx, { identity: second, runId: run.id, timer }); + await ctx.input.evidence("runtime-cleanup-owned-provider.json", { runId: run.id, companyId: run.companyId, issueId: run.nativeIssueId, agentId: run.agentId, providerPid: pid, providerStartedAt: started, + sameUid: true, directOwnedRunnerParent: true, executableSha256: second.digest, durableSessionReceipt: true, identityReadTwice: true, signal: "SIGSTOP", groupSignalled: false, maximumSuspensionMs: 8_000 }); + return { providerPid: pid, providerStartedAt: started, runId: run.id }; +} +export async function releaseOwnedCleanupProvider(ctx: ChatCatalogContext) { + const owned = suspended.get(ctx); if (!owned) return; clearTimeout(owned.timer); + try { process.kill(owned.identity.pid, 0); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ESRCH") { suspended.delete(ctx); return; } throw new Error("Owned provider cleanup liveness could not be verified"); } + const fresh = await providerIdentity(owned.identity.pid); + if (fresh.pid !== owned.identity.pid || fresh.uid !== owned.identity.uid || fresh.startedAt !== owned.identity.startedAt || fresh.executable !== owned.identity.executable || fresh.digest !== owned.identity.digest) + throw new Error("Provider cleanup refuses signalling a reused or changed PID"); + process.kill(fresh.pid, "SIGCONT"); suspended.delete(ctx); +} + +export async function ownedCleanupProviderHasExited(ctx: ChatCatalogContext) { + const identity = retainedProvider.get(ctx); if (!identity) throw new Error("No retained exact provider identity for cleanup verification"); + try { process.kill(identity.pid, 0); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ESRCH") return true; return false; } + const startedAt = await readProcessStartedAt(identity.pid); + return !!startedAt && startedAt !== identity.startedAt; +} diff --git a/tests/runner-e2e/chat-native-stop-boundary.test.ts b/tests/runner-e2e/chat-native-stop-boundary.test.ts index 3b81fa167a4..8d0570fdba7 100644 --- a/tests/runner-e2e/chat-native-stop-boundary.test.ts +++ b/tests/runner-e2e/chat-native-stop-boundary.test.ts @@ -3,18 +3,18 @@ import type { ChatCatalogCase, ChatCatalogContext, NativeStopBoundaryReceipt } f import { loadChatCatalogImplementation, type ChatCatalogDescriptor } from "./chat-catalog-registry.js"; import { assertNativeStopCallback, nativeStopPendingReceiptMatches, prepareNativeStopCanonicalBoundary, validNativeStopDeclaration } from "./chat-native-stop-boundary.js"; const receipt: NativeStopBoundaryReceipt = { runId: "11111111-1111-4111-8111-111111111111", issueId: "22222222-2222-4222-8222-222222222222", intentId: "33333333-3333-4333-8333-333333333333", intentRecordedAt: "2026-10-26T12:00:00.000Z" }; -const metadata: ChatCatalogDescriptor = { scenarioId: "recovery.post-stop-message", canonicalBoundary: "native-stop-cleanup", authentication: "human-session", sourceFiles: ["runtime-post-stop.ts"], modulePath: "./chat-catalog-domains/runtime-post-stop.js", exportName: "cases", oracleVersion: "control", expectedRunCount: 3, expectedHumanMessageCount: 1, turnTimeoutMs: 120000, attemptTimeoutMs: 600000 }; +const metadata: ChatCatalogDescriptor = { scenarioId: "recovery.post-stop-message", canonicalBoundary: "native-stop-cleanup", authentication: "human-session", sourceFiles: ["runtime-post-stop.ts"], modulePath: "./chat-catalog-domains/runtime-post-stop.js", exportName: "cases", oracleVersion: "control", expectedRunCount: 3, expectedHumanMessageCount: 2, setupHumanMessageCount: 1, turnTimeoutMs: 120000, attemptTimeoutMs: 600000 }; function fixture() { - const run = { id: receipt.runId, companyId: "company", nativeIssueId: receipt.issueId, status: "running", resultJson: { nativeCancellation: { schema: "paperclip.native-cancellation.v1", companyId: "company", runId: receipt.runId, issueId: receipt.issueId, scope: "run", intentId: receipt.intentId, recordedAt: receipt.intentRecordedAt, dispatchState: "pending", dispatched: false } } }; - const comments: Array<{ id: string; authorAgentId?: string }> = [], runs = [{ id: receipt.runId }]; let url = "http://fixture/source"; - const ctx = { scenario: { id: metadata.scenarioId, query: "Make the page simpler." }, issue: () => ({ id: "source" }), allRuns: async () => runs, - input: { fixtures: { company: { id: "company" } }, page: { url: () => url }, evidence: vi.fn(async () => {}), api: { get: async (path: string) => path.startsWith("/api/heartbeat-runs/") ? run : comments } } } as unknown as ChatCatalogContext; + const run = { id: receipt.runId, companyId: "company", nativeIssueId: receipt.issueId, agentId: "lead", runtimeMode: "native", status: "running", resultJson: { nativeCancellation: { schema: "paperclip.native-cancellation.v1", companyId: "company", runId: receipt.runId, issueId: receipt.issueId, scope: "run", intentId: receipt.intentId, recordedAt: receipt.intentRecordedAt, dispatchState: "pending", dispatched: false } } }; + const comments: Array<{ id: string; body: string; authorAgentId?: string }> = [{ id: "setup-human", body: "Build the original website after the bounded wait." }], runs = [{ id: receipt.runId }]; let url = "http://fixture/source"; + const ctx = { scenario: { id: metadata.scenarioId, query: "Make the page simpler." }, issue: () => ({ id: receipt.issueId, assigneeAgentId: "lead" }), allRuns: async () => runs, + input: { fixtures: { company: { id: "company" } }, page: { url: () => url }, evidence: vi.fn(async () => {}), api: { get: async (path: string) => path.startsWith("/api/heartbeat-runs/") ? run : path.endsWith("/comments?order=asc") ? structuredClone(comments) : { id: receipt.issueId, companyId: "company", conversationAgentId: "lead" } } } } as unknown as ChatCatalogContext; const implementation = { ...metadata, setup: async () => ({}), observe: async () => ({}), grade: () => ({ passed: false, checks: [] }), verifyResult: async () => {}, beforeCanonicalSend: vi.fn(async () => ({ ...receipt })) } as unknown as ChatCatalogCase; - return { run, comments, runs, ctx, implementation, move: () => { url += "/other"; }, setup: { humanCommentIds: [], runIds: [receipt.runId] } }; + return { run, comments, runs, ctx, implementation, move: () => { url += "/other"; }, setup: { humanCommentIds: ["setup-human"], runIds: [receipt.runId] } }; } it("admits only the exact unchanged human post-stop boundary declaration", () => { expect(validNativeStopDeclaration(metadata)).toBe(true); - for (const patch of [{ scenarioId: "recovery.explain-stall" }, { journeyKind: "browser-boundary" }, { authentication: "local-trusted" }, { expectedHumanMessageCount: 2 }, { setupHumanMessageCount: 1 }, { queryOutcome: "rejected" }, { queryAttachments: [{ stateKey: "file", filename: "brief.md" }] }]) expect(validNativeStopDeclaration({ ...metadata, ...patch } as typeof metadata)).toBe(false); + for (const patch of [{ scenarioId: "recovery.explain-stall" }, { journeyKind: "browser-boundary" }, { authentication: "local-trusted" }, { expectedHumanMessageCount: 1 }, { setupHumanMessageCount: 0 }, { queryOutcome: "rejected" }, { queryAttachments: [{ stateKey: "file", filename: "brief.md" }] }]) expect(validNativeStopDeclaration({ ...metadata, ...patch } as typeof metadata)).toBe(false); expect(() => assertNativeStopCallback({ ...fixture().implementation, beforeCanonicalSend: undefined })).toThrow("callback"); expect(() => assertNativeStopCallback({ ...fixture().implementation, canonicalBoundary: undefined })).toThrow("callback"); }); @@ -26,15 +26,15 @@ it("preflight rejects declaration drift and absent or hidden callbacks before an await expect(load({ ...f.implementation, canonicalBoundary: undefined }, { ...metadata, canonicalBoundary: undefined })).rejects.toThrow("callback"); }); it("requires actual same-company admitted running setup work and a pending unacknowledged persisted intent", () => { - const input = () => { const f = fixture(); return { receipt: { ...receipt }, run: f.run, companyId: "company", setupRunIds: [receipt.runId] }; }; + const input = () => { const f = fixture(); return { receipt: { ...receipt }, run: f.run, companyId: "company", sourceIssueId: receipt.issueId, assignedAgentId: "lead", setupRunIds: [receipt.runId] }; }; expect(nativeStopPendingReceiptMatches(input())).toBe(true); - for (const change of [(v: ReturnType) => { v.run.companyId = "foreign"; }, (v: ReturnType) => { v.run.status = "cancelled"; }, (v: ReturnType) => { v.setupRunIds = []; }, (v: ReturnType) => { v.run.resultJson.nativeCancellation.dispatchState = "acknowledged"; }, (v: ReturnType) => { v.run.resultJson.nativeCancellation.dispatched = true; }, (v: ReturnType) => { v.receipt.intentId = receipt.runId; }, (v: ReturnType) => { v.receipt.intentRecordedAt = "invalid"; }]) { const value = input(); change(value); expect(nativeStopPendingReceiptMatches(value)).toBe(false); } + for (const change of [(v: ReturnType) => { v.run.companyId = "foreign"; }, (v: ReturnType) => { v.run.status = "cancelled"; }, (v: ReturnType) => { v.sourceIssueId = "another-source"; }, (v: ReturnType) => { v.run.agentId = "another-agent"; }, (v: ReturnType) => { v.run.runtimeMode = "legacy"; }, (v: ReturnType) => { v.setupRunIds = []; }, (v: ReturnType) => { v.run.resultJson.nativeCancellation.dispatchState = "acknowledged"; }, (v: ReturnType) => { v.run.resultJson.nativeCancellation.dispatched = true; }, (v: ReturnType) => { v.receipt.intentId = receipt.runId; }, (v: ReturnType) => { v.receipt.intentRecordedAt = "invalid"; }]) { const value = input(); change(value); expect(nativeStopPendingReceiptMatches(value)).toBe(false); } }); it("keeps the canonical human message untouched and rejects hidden turns, source navigation or extra admissions", async () => { const success = fixture(), state = {}; await prepareNativeStopCanonicalBoundary(success.ctx, success.implementation, state, success.ctx.scenario.query, success.setup); expect(state).toEqual({ nativeStopBoundary: receipt }); expect(success.implementation.beforeCanonicalSend).toHaveBeenCalledOnce(); - for (const change of [(f: ReturnType) => { f.comments.push({ id: "hidden-human" }); }, (f: ReturnType) => { f.runs.push({ id: "extra-run" }); }, (f: ReturnType) => { f.move(); }, (f: ReturnType) => { f.ctx.scenario.query += " Hurry"; }]) { + for (const change of [(f: ReturnType) => { f.comments.push({ id: "hidden-human", body: "Hidden direction" }); }, (f: ReturnType) => { f.runs.push({ id: "extra-run" }); }, (f: ReturnType) => { f.comments[0]!.body = "Altered setup direction"; }, (f: ReturnType) => { f.move(); }, (f: ReturnType) => { f.ctx.scenario.query += " Hurry"; }]) { const f = fixture(); f.implementation.beforeCanonicalSend = async () => { change(f); return receipt; }; await expect(prepareNativeStopCanonicalBoundary(f.ctx, f.implementation, {}, f.ctx.scenario.query, f.setup)).rejects.toThrow("changed source/query/history"); } @@ -45,7 +45,7 @@ it("does no boundary work in an ordinary undeclared catalog scenario", async () expect(f.ctx.input.evidence).not.toHaveBeenCalled(); }); it("preflights the finite Slack distractor callback only for its declared reply thread", async () => { - const descriptor: ChatCatalogDescriptor = { ...metadata, scenarioId: "chat-channels.reply-in-thread", canonicalBoundary: undefined, journeyKind: "native-slack-thread", slackIngress: { kind: "native-slack-thread", maximumProviderCalls: 20, ingressTimeoutMs: 120000, setupThread: "same-agent-distractor" }, expectedRunCount: 4, minimumRunCount: 2 }; + const descriptor: ChatCatalogDescriptor = { ...metadata, scenarioId: "chat-channels.reply-in-thread", canonicalBoundary: undefined, journeyKind: "native-slack-thread", slackIngress: { kind: "native-slack-thread", maximumProviderCalls: 20, ingressTimeoutMs: 120000, setupThread: "same-agent-distractor" }, expectedRunCount: 4, minimumRunCount: 2, expectedHumanMessageCount: 1, setupHumanMessageCount: 0 }; const row = { ...fixture().implementation, ...descriptor, beforeCanonicalSend: undefined, dispatchSlackQuery: async () => ({}), dispatchSlackOtherThread: async () => ({}) } as unknown as ChatCatalogCase; const load = (implementation: ChatCatalogCase, selected = descriptor) => loadChatCatalogImplementation(selected, async () => ({ cases: [implementation] })); await expect(load(row)).resolves.toBe(row); diff --git a/tests/runner-e2e/chat-native-stop-boundary.ts b/tests/runner-e2e/chat-native-stop-boundary.ts index a9059095dcf..7f612942201 100644 --- a/tests/runner-e2e/chat-native-stop-boundary.ts +++ b/tests/runner-e2e/chat-native-stop-boundary.ts @@ -1,33 +1,41 @@ import type { ChatCatalogCase, ChatCatalogContext, ChatCatalogSetupBoundary, NativeStopBoundaryReceipt } from "./chat-catalog-contract.js"; export function validNativeStopDeclaration(value: Omit, "journeyKind"> & { journeyKind?: string }) { return value.canonicalBoundary === "native-stop-cleanup" && value.scenarioId === "recovery.post-stop-message" && value.authentication === "human-session" - && (value.journeyKind ?? "agent-chat") === "agent-chat" && (value.expectedHumanMessageCount ?? 1) === 1 && (value.setupHumanMessageCount ?? 0) === 0 + && (value.journeyKind ?? "agent-chat") === "agent-chat" && value.expectedHumanMessageCount === 2 && value.setupHumanMessageCount === 1 && (value.queryOutcome ?? "persisted") === "persisted" && !value.queryBindings?.length && !value.queryAttachments?.length; } export function assertNativeStopCallback(implementation: ChatCatalogCase) { if (implementation.canonicalBoundary ? !validNativeStopDeclaration(implementation) || typeof implementation.beforeCanonicalSend !== "function" : implementation.beforeCanonicalSend !== undefined) throw new Error("Missing, wrong-case or undeclared native Stop canonical-boundary callback"); } -export function nativeStopPendingReceiptMatches(input: { receipt: NativeStopBoundaryReceipt; run: { id?: string; companyId?: string; nativeIssueId?: string; status?: string; resultJson?: { nativeCancellation?: Record } }; companyId: string; setupRunIds: string[] }) { +// Production native-session-executor writes this intent in its cancellation transaction, +// then only writes dispatchState=acknowledged/acknowledgedAt after actual cleanup. +// A request receipt or a cancelled status alone does not establish this interval. +export function nativeStopPendingReceiptMatches(input: { receipt: NativeStopBoundaryReceipt; run: { id?: string; companyId?: string; nativeIssueId?: string; agentId?: string; runtimeMode?: string; status?: string; resultJson?: { nativeCancellation?: Record } }; companyId: string; sourceIssueId: string; assignedAgentId: string; setupRunIds: string[] }) { const { receipt: r, run } = input, intent = run.resultJson?.nativeCancellation; return !!r && Object.keys(r).every(key => ["runId", "issueId", "intentId", "intentRecordedAt"].includes(key)) && [r.runId, r.issueId, r.intentId].every(id => typeof id === "string" && /^[a-f0-9-]{36}$/i.test(id)) && Number.isFinite(Date.parse(r.intentRecordedAt)) && input.setupRunIds.includes(r.runId) && run.id === r.runId && run.companyId === input.companyId && run.nativeIssueId === r.issueId - && run.status === "running" && intent?.schema === "paperclip.native-cancellation.v1" && intent.companyId === input.companyId && intent.runId === r.runId && intent.issueId === r.issueId + && r.issueId === input.sourceIssueId && run.agentId === input.assignedAgentId && run.runtimeMode === "native" && run.status === "running" && intent?.schema === "paperclip.native-cancellation.v1" && intent.companyId === input.companyId && intent.runId === r.runId && intent.issueId === r.issueId && intent.intentId === r.intentId && intent.recordedAt === r.intentRecordedAt && intent.scope === "run" && intent.dispatchState === "pending" && intent.dispatched === false && intent.acknowledgedAt == null; } export async function prepareNativeStopCanonicalBoundary(ctx: ChatCatalogContext, implementation: ChatCatalogCase, state: Record, canonicalQuery: string, setup: ChatCatalogSetupBoundary) { assertNativeStopCallback(implementation); if (!implementation.canonicalBoundary) return; - if (canonicalQuery !== ctx.scenario.query || setup.humanCommentIds.length) throw new Error("Native Stop boundary cannot alter the canonical query or introduce setup chat turns"); + if (canonicalQuery !== ctx.scenario.query || setup.humanCommentIds.length !== 1 || setup.runIds.length !== 1) throw new Error("Native Stop boundary cannot alter the canonical query or introduce setup chat turns"); const source = ctx.issue(); if (!source) throw new Error("Native Stop boundary requires the existing actual source chat"); const sourceURL = ctx.input.page.url(), originalQuery = ctx.scenario.query; + const before = await ctx.input.api.get>(`/api/issues/${source.id}/comments?order=asc`); + const retainedHuman = before.filter(row => !row.authorAgentId); + if (retainedHuman.length !== 1 || retainedHuman[0]?.id !== setup.humanCommentIds[0]) throw new Error("Native Stop boundary requires exactly the declared genuine setup human message"); const receipt = await implementation.beforeCanonicalSend!(ctx, state); - const [run, comments, runs] = await Promise.all([ + const [run, comments, runs, actualSource] = await Promise.all([ ctx.input.api.get[0]["run"]>(`/api/heartbeat-runs/${receipt.runId}`), - ctx.input.api.get>(`/api/issues/${source.id}/comments?order=asc`), ctx.allRuns(), + ctx.input.api.get>(`/api/issues/${source.id}/comments?order=asc`), ctx.allRuns(), + ctx.input.api.get<{ id: string; companyId: string; assigneeAgentId?: string | null; conversationAgentId?: string | null }>(`/api/issues/${source.id}`), ]); - if (!nativeStopPendingReceiptMatches({ receipt, run, companyId: ctx.input.fixtures.company.id, setupRunIds: setup.runIds }) || ctx.issue()?.id !== source.id || ctx.input.page.url() !== sourceURL - || ctx.scenario.query !== originalQuery || canonicalQuery !== originalQuery || comments.some(row => !row.authorAgentId) + const assignedAgentId = actualSource.conversationAgentId ?? actualSource.assigneeAgentId; + if (actualSource.id !== source.id || actualSource.companyId !== ctx.input.fixtures.company.id || !assignedAgentId || !nativeStopPendingReceiptMatches({ receipt, run, companyId: ctx.input.fixtures.company.id, sourceIssueId: source.id, assignedAgentId, setupRunIds: setup.runIds }) || ctx.issue()?.id !== source.id || ctx.input.page.url() !== sourceURL + || ctx.scenario.query !== originalQuery || canonicalQuery !== originalQuery || JSON.stringify(comments.filter(row => !row.authorAgentId)) !== JSON.stringify(retainedHuman) || runs.length !== setup.runIds.length || runs.some(row => !setup.runIds.includes(row.id))) throw new Error("Native Stop boundary changed source/query/history or missed the genuine pending cleanup interval"); state.nativeStopBoundary = receipt; - await ctx.input.evidence("chat-catalog-native-stop-boundary.json", { kind: implementation.canonicalBoundary, sourceIssueId: source.id, ...receipt, canonicalQueryUnchanged: true, extraHumanMessages: 0, extraRuns: 0 }); + await ctx.input.evidence("chat-catalog-native-stop-boundary.json", { kind: implementation.canonicalBoundary, sourceIssueId: source.id, ...receipt, canonicalQueryUnchanged: true, retainedSetupHumanCommentId: retainedHuman[0]!.id, extraHumanMessages: 0, extraRuns: 0 }); }