diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index cca89921afd..022b2176f6c 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -4415,7 +4415,7 @@ "registered-artifact.ts", "chat-scenarios/recovery.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime.v2", + "oracleVersion": "paperclip.chat-catalog.post-stop.v3", "coverageGaps": [], "expectedRunCount": 2, "setupHumanMessageCount": 1, diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-post-stop.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-post-stop.test.ts index c7a557717de..bef431e3891 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-post-stop.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-post-stop.test.ts @@ -1,31 +1,43 @@ -import { expect, it, vi } from "vitest"; -import { gradePostStop, sameSourcePostStopChronology, releasePostStopGateAfterCleanup } from "./runtime-post-stop.js"; -function chronology() { - const old = { id: "old", nativeIssueId: "source", companyId: "company", agentId: "ceo", runtimeMode: "native", status: "cancelled", finishedAt: "2026-10-26T12:00:02Z", resultJson: { nativeCancellation: { schema: "paperclip.native-cancellation.v1", intentId: "intent", intentAuditId: "audit-intent", acknowledgementAuditId: "audit-ack", issueId: "source", companyId: "company", runId: "old", scope: "run", recordedAt: "2026-10-26T12:00:00Z", acknowledgedAt: "2026-10-26T12:00:02Z", dispatchState: "acknowledged", dispatched: true } } }; - return { old, next: { id: "new", nativeIssueId: "source", companyId: "company", agentId: "ceo", runtimeMode: "native", status: "succeeded", startedAt: "2026-10-26T12:00:03Z" }, boundary: { runId: "old", issueId: "source", intentId: "intent", intentRecordedAt: "2026-10-26T12:00:00Z" }, human: { id: "canonical", body: "Now make the website a single page instead.", issueId: "source", companyId: "company", authorAgentId: null, authorUserId: "person", createdAt: "2026-10-26T12:00:01Z" }, query: "Now make the website a single page instead.", companyId: "company", agentId: "ceo", humanId: "person", prompt: "canonical\nNow make the website a single page instead.", wakeCommentIds: ["canonical"] } as unknown as Parameters[0]; +import { expect, it } from "vitest"; +import { gradePostStop, sameSourcePostStopChronology } from "./runtime-post-stop.js"; + +const query = "Now make the website a single page instead."; +const ids = { company: "company", agent: "lead", human: "user", issue: "11111111-1111-1111-1111-111111111111", old: "22222222-2222-2222-2222-222222222222", next: "33333333-3333-3333-3333-333333333333", comment: "comment-2" }; +const intentId = "native-cancellation:44444444-4444-4444-4444-444444444444"; +function input(times: { sent: string; ack: string; released: string; started: string; ended?: string }) { + const boundary = { runId: ids.old, issueId: ids.issue, intentId, intentRecordedAt: "2026-11-07T10:00:00.000Z" }; + const old = { id: ids.old, nativeIssueId: ids.issue, companyId: ids.company, agentId: ids.agent, runtimeMode: "native", status: "cancelled", finishedAt: times.ended ?? times.ack, + resultJson: { nativeCancellation: { schema: "paperclip.native-cancellation.v1", intentId, issueId: ids.issue, companyId: ids.company, runId: ids.old, scope: "run", intentAuditId: "a1", acknowledgementAuditId: "a2", + recordedAt: boundary.intentRecordedAt, dispatchState: "acknowledged", dispatched: true, acknowledgedAt: times.ack } } }; + const next = { id: ids.next, nativeIssueId: ids.issue, companyId: ids.company, agentId: ids.agent, runtimeMode: "native", status: "succeeded", startedAt: times.started }; + const human = { id: ids.comment, body: query, issueId: ids.issue, companyId: ids.company, authorAgentId: null, authorUserId: ids.human, createdAt: times.sent }; + return { boundary, old, next, human, query, companyId: ids.company, agentId: ids.agent, humanId: ids.human, prompt: `${ids.comment} ${query}`, wakeCommentIds: [ids.comment], heldReleasedAt: times.released } as unknown as Parameters[0]; } -function valid() { - return { companyId: "company", leadId: "ceo", agents: [], links: [], humanQueries: ["initial", "canonical"], reply: "", tasks: [], activity: [], runs: [{ id: "old", agentId: "ceo", status: "cancelled" }, { id: "new", agentId: "ceo", status: "succeeded" }], state: { stopStatus: 200, gateReleasedAfterCleanup: true, chronologyBound: true, providerExited: true, initialHumanRetained: true, singleSuccessor: true, sameCurrentAssignment: true, oldEvidenceRetained: true, actualSimplePage: true, artifactRunBound: true, canonicalReply: "The single-page website is built and saved here." } }; -} -it("binds the real unchanged canonical human instruction between same-source Stop intent and cleanup ACK", () => expect(sameSourcePostStopChronology(chronology())).toBe(true)); -it("rejects wrong-source, early, late, duplicate, resumed or unacknowledged deliveries", () => { - for (const change of [(v: ReturnType) => { v.human.issueId = "other"; }, (v: ReturnType) => { v.old.nativeIssueId = "other"; }, (v: ReturnType) => { v.next.agentId = "other"; }, (v: ReturnType) => { v.human.createdAt = "2026-10-26T12:00:03Z" as never; }, (v: ReturnType) => { v.next.startedAt = "2026-10-26T12:00:01Z" as never; }, (v: ReturnType) => { v.wakeCommentIds.push("initial"); }, (v: ReturnType) => { v.prompt = "a different instruction"; }, (v: ReturnType) => { v.old.status = "succeeded"; }, (v: ReturnType) => { (v.old.resultJson!.nativeCancellation as Record).dispatchState = "pending"; }]) { const v = chronology(); change(v); expect(sameSourcePostStopChronology(v)).toBe(false); } -}); -it("accepts real post-cleanup delivery and a natural substantive result paraphrase", () => { - expect(gradePostStop(valid()).passed).toBe(true); const o = valid(); o.state.canonicalReply = "Acorn Notes is ready, with its signup action and all the content together. The artifact is saved in this conversation."; expect(gradePostStop(o).passed).toBe(true); -}); -it("rejects polite completion without actual same-source cleanup, delivery or rendered output", () => { - for (const key of ["gateReleasedAfterCleanup", "chronologyBound", "providerExited", "initialHumanRetained", "singleSuccessor", "sameCurrentAssignment", "oldEvidenceRetained", "actualSimplePage", "artifactRunBound"] as const) { const o = valid(); o.state[key] = false; expect(gradePostStop(o).passed).toBe(false); } - const o = valid(); o.state.canonicalReply = "The website is still queued and not built."; expect(gradePostStop(o).passed).toBe(false); expect(gradePostStop(undefined).passed).toBe(false); + +it("accepts a message saved while the old provider is still held in cleanup, even after the Stop ACK", () => { + // The product ACKs a Stop after a bounded cleanup grace while the provider may still be closing. + expect(sameSourcePostStopChronology(input({ ack: "2026-11-07T10:00:02.000Z", sent: "2026-11-07T10:00:02.300Z", released: "2026-11-07T10:00:03.000Z", started: "2026-11-07T10:00:04.000Z" }))).toBe(true); + expect(sameSourcePostStopChronology(input({ sent: "2026-11-07T10:00:01.000Z", ack: "2026-11-07T10:00:02.000Z", released: "2026-11-07T10:00:03.000Z", started: "2026-11-07T10:00:04.000Z" }))).toBe(true); }); -it("releases the neutral command gate after real cleanup without waiting for successor success", async () => { - const release = vi.fn(async () => {}), old = chronology().old; - await expect(releasePostStopGateAfterCleanup(old, false, release)).rejects.toThrow("cleanup"); - await expect(releasePostStopGateAfterCleanup({ ...old, status: "running" }, true, release)).rejects.toThrow("cleanup"); - const pending = chronology().old; (pending.resultJson!.nativeCancellation as Record).dispatchState = "pending"; - await expect(releasePostStopGateAfterCleanup(pending, true, release)).rejects.toThrow("cleanup"); expect(release).not.toHaveBeenCalled(); - // No successor outcome is an input: it may still be running and retrying the - // legitimate initial command. Releasing only its neutral result avoids a cycle. - await releasePostStopGateAfterCleanup(old, true, release); expect(release).toHaveBeenCalledOnce(); +it("rejects a message sent after cleanup was released and a successor started during cleanup", () => { + expect(sameSourcePostStopChronology(input({ ack: "2026-11-07T10:00:02.000Z", released: "2026-11-07T10:00:03.000Z", sent: "2026-11-07T10:00:03.500Z", started: "2026-11-07T10:00:04.000Z" }))).toBe(false); + expect(sameSourcePostStopChronology(input({ ack: "2026-11-07T10:00:02.000Z", sent: "2026-11-07T10:00:02.300Z", started: "2026-11-07T10:00:02.600Z", released: "2026-11-07T10:00:03.000Z" }))).toBe(false); + expect(sameSourcePostStopChronology(input({ sent: "2026-11-07T09:59:59.000Z", ack: "2026-11-07T10:00:02.000Z", released: "2026-11-07T10:00:03.000Z", started: "2026-11-07T10:00:04.000Z" }))).toBe(false); +}); + +function observation(state: Record) { + return { reply: "", companyId: ids.company, agents: [], links: [], activity: [], leadId: ids.agent, humanQueries: [{}, {}], tasks: [], runs: [{ status: "cancelled", agentId: ids.agent }, { status: "succeeded", agentId: ids.agent }], + state: { stopStatus: 200, messageAcceptedDuringHeldCleanup: true, noSuccessorWhileHeld: true, gateReleasedAfterCleanup: true, chronologyBound: true, providerExited: true, initialHumanRetained: true, + singleSuccessor: true, sameCurrentAssignment: true, oldEvidenceRetained: true, actualSimplePage: true, artifactRunBound: true, canonicalReply: "Done: [website.html](/PAP/issues/PAP-1#document-website)", ...state } }; +} + +it("grades the saved page, not the reply wording", () => { + const passed = (state: Record) => gradePostStop(observation(state)).checks.every(check => check.passed); + expect(passed({})).toBe(true); + expect(passed({ canonicalReply: "Switched it over — one scrolling layout, no tabs anymore." })).toBe(true); + expect(passed({ actualSimplePage: false })).toBe(false); + expect(passed({ canonicalReply: "" })).toBe(false); + expect(passed({ messageAcceptedDuringHeldCleanup: false })).toBe(false); + expect(passed({ noSuccessorWhileHeld: false })).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-post-stop.ts b/tests/runner-e2e/chat-catalog-domains/runtime-post-stop.ts index 6bbb70862fb..539a64e5f31 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-post-stop.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-post-stop.ts @@ -9,22 +9,22 @@ import { readRegisteredArtifacts } from "../registered-artifact.js"; import { checks } from "./internal-support.js"; import { openStandingChat } from "./identity-support.js"; import { prepareRuntimeWorkerGate, readRuntimeTaskGate } from "./runtime-busy-worker.js"; -import { suspendOwnedProviderForCleanup, releaseOwnedCleanupProvider, ownedCleanupProviderHasExited } from "./runtime-provider-cleanup.js"; +import { suspendOwnedProviderForCleanup, releaseOwnedCleanupProvider, ownedCleanupProviderHasExited, ownedCleanupProviderIsHeld } from "./runtime-provider-cleanup.js"; import type { NativeFaultRun } from "./runtime-native-fault.js"; import { hasSimplerWebsite, renderRuntimeArtifact } from "./runtime-rendered-artifact.js"; import { observeRuntime, runtimeDefinition, runtimeObservation, type RuntimeAgent } from "./runtime-support.js"; interface StopIntent { schema?: string; intentId?: string; intentAuditId?: string; companyId?: string; runId?: string; issueId?: string; scope?: string; dispatchState?: string; dispatched?: boolean; recordedAt?: string; acknowledgedAt?: string; acknowledgementAuditId?: string } interface StopHandle { page: Page; completed?: Promise } const handles = new WeakMap(); -export function sameSourcePostStopChronology(input: { boundary: NativeStopBoundaryReceipt; old: NativeFaultRun; next: NativeFaultRun; human: Pick; query: string; companyId: string; agentId: string; humanId: string; prompt: string; wakeCommentIds: string[] }) { +export function sameSourcePostStopChronology(input: { boundary: NativeStopBoundaryReceipt; old: NativeFaultRun; next: NativeFaultRun; human: Pick; query: string; companyId: string; agentId: string; humanId: string; prompt: string; wakeCommentIds: string[]; heldReleasedAt: string }) { const { old, next, human, boundary: b } = input, intent = old.resultJson?.nativeCancellation as StopIntent | undefined; - const before = Date.parse(b.intentRecordedAt), sent = Date.parse(String(human.createdAt)), ack = Date.parse(intent?.acknowledgedAt ?? ""), started = Date.parse(String(next.startedAt)), ended = Date.parse(String(old.finishedAt)); + const before = Date.parse(b.intentRecordedAt), sent = Date.parse(String(human.createdAt)), ack = Date.parse(intent?.acknowledgedAt ?? ""), started = Date.parse(String(next.startedAt)), ended = Date.parse(String(old.finishedAt)), released = Date.parse(input.heldReleasedAt); return old.id === b.runId && old.nativeIssueId === b.issueId && old.companyId === input.companyId && old.agentId === input.agentId && old.runtimeMode === "native" && old.status === "cancelled" && intent?.schema === "paperclip.native-cancellation.v1" && intent.intentId === b.intentId && intent.issueId === b.issueId && intent.companyId === input.companyId && intent.runId === old.id && intent.scope === "run" && !!intent.intentAuditId && !!intent.acknowledgementAuditId && intent.recordedAt === b.intentRecordedAt && intent.dispatchState === "acknowledged" && intent.dispatched === true && human.issueId === b.issueId && human.companyId === input.companyId && !human.authorAgentId && human.authorUserId === input.humanId && human.body === input.query - && before <= sent && sent < ack && next.id !== old.id && next.nativeIssueId === b.issueId && next.companyId === input.companyId && next.agentId === input.agentId && next.runtimeMode === "native" && next.status === "succeeded" - && started >= ack && started >= ended && input.wakeCommentIds.length === 1 && input.wakeCommentIds[0] === human.id && input.prompt.includes(human.id) && input.prompt.includes(input.query); + && before <= sent && sent < released && Number.isFinite(ack) && next.id !== old.id && next.nativeIssueId === b.issueId && next.companyId === input.companyId && next.agentId === input.agentId && next.runtimeMode === "native" && next.status === "succeeded" + && started >= ack && started >= ended && started >= released && input.wakeCommentIds.length === 1 && input.wakeCommentIds[0] === human.id && input.prompt.includes(human.id) && input.prompt.includes(input.query); } export async function releasePostStopGateAfterCleanup(old: Pick, providerExited: boolean, release: () => Promise) { const intent = old.resultJson?.nativeCancellation as StopIntent | undefined; @@ -36,13 +36,12 @@ export function gradePostStop(value: unknown) { const c = checks(), o = runtimeObservation(value); c.add("actual-same-source-stop-continuation-evidence", !!o, "A real setup human request, actual native provider cleanup, canonical human instruction and normal continuation must be independently observed."); if (!o) return c.result(); const s = o.state; - c.add("human-stop-cleanup-and-exact-message-chronology", s.stopStatus === 200 && s.gateReleasedAfterCleanup === true && s.chronologyBound === true && s.providerExited === true && s.initialHumanRetained === true, + c.add("human-stop-cleanup-and-exact-message-chronology", s.stopStatus === 200 && s.messageAcceptedDuringHeldCleanup === true && s.noSuccessorWhileHeld === true && s.gateReleasedAfterCleanup === true && s.chronologyBound === true && s.providerExited === true && s.initialHumanRetained === true, "The same actual source run has a real Stop intent and ACK; the owned protocol endpoint has retired (the OpenCode proxy, not SDK server health), and only its eligible successor consumes the unchanged canonical message after cleanup."); c.add("one-eligible-current-owner-with-no-revival-or-duplicate", o.runs.length === 2 && o.runs.filter(row => row.status === "cancelled").length === 1 && o.runs.filter(row => row.status === "succeeded").length === 1 && o.runs.every(row => row.agentId === o.leadId) && s.singleSuccessor === true && s.sameCurrentAssignment === true && s.oldEvidenceRetained === true && o.humanQueries.length === 2 && o.tasks.length === 0, "The original stopped execution remains cancelled, one native source continuation owns the new direction under current assignment, and no duplicate task or user message appears."); - c.add("actual-visible-single-page-result-from-continuation", s.actualSimplePage === true && s.artifactRunBound === true && /single[ -]page|one page|Acorn|website|site|page/i.test(String(s.canonicalReply)) && /ready|built|created|updated|implemented|saved|now/i.test(String(s.canonicalReply)) - && !/still (?:queued|waiting)|not (?:yet )?(?:built|implemented|ready)|could not (?:build|implement)|kept.*(?:tabs|multi[ -]page)/i.test(String(s.canonicalReply)), + c.add("actual-visible-single-page-result-from-continuation", s.actualSimplePage === true && s.artifactRunBound === true && String(s.canonicalReply ?? "").trim().length > 0, "The subsequent native reply reports the real saved simple single page; registered bytes and visible design independently prove the result."); return c.result(); } @@ -66,7 +65,7 @@ export const runtimePostStopCases = [runtimeDefinition("recovery.post-stop-messa return { gate, sourceId: source.id, workerRunId: original.id, initialHuman: initial[0], humanId: people.user.id }; } catch (error) { if (gate.gate) await writeFile(gate.gate, "The pending fixture tool has been released.", "utf8"); await secondary.close(); handles.delete(ctx); throw error; } }, gradePostStop, { - canonicalBoundary: "native-stop-cleanup", authentication: "human-session", setupHumanMessageCount: 1, expectedHumanMessageCount: 2, expectedRunCount: 2, attemptTimeoutMs: 720000, + oracleVersion: "paperclip.chat-catalog.post-stop.v3", canonicalBoundary: "native-stop-cleanup", authentication: "human-session", setupHumanMessageCount: 1, expectedHumanMessageCount: 2, expectedRunCount: 2, attemptTimeoutMs: 720000, runPolicy: { kind: "terminal-counts", terminalStatusCounts: { cancelled: 1, succeeded: 1 } }, sourceFiles: ["chat-catalog-domains/runtime-post-stop.ts", "chat-catalog-domains/runtime-provider-cleanup.ts", "chat-catalog-domains/runtime-native-fault.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-rendered-artifact.ts", "chat-catalog-domains/runtime-support.ts", "chat-native-stop-boundary.ts", "../../server/src/services/native-runtime/native-session-executor.ts", "../../server/src/services/explicit-native-continuation.ts", "../../packages/paperclip-runner/src/live/runnerd-codex-transport.ts", "../../packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs", "../../packages/paperclip-runner/runner/crates/runner-core/src/process_supervisor.rs", "registered-artifact.ts", "chat-scenarios/recovery.yaml"], setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["actual-source-native-owner"] }, @@ -87,6 +86,18 @@ export const runtimePostStopCases = [runtimeDefinition("recovery.post-stop-messa return { runId: run.id, issueId: ctx.issue()!.id, intentId: intent.intentId, intentRecordedAt: intent.recordedAt }; }, async act(ctx, state) { + // Deterministic cleanup gate: the old provider stays stopped (so its cleanup cannot finish) + // until the follow-up message is durably accepted. Only then is the hold released. + const accepted = await pollUntil({ label: "Follow-up message durably accepted while the stopped run's provider is still held in cleanup", deadlineAt: Date.now() + 60000, intervalMs: 100, + load: async () => ({ comments: await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`), held: await ownedCleanupProviderIsHeld(ctx) }), + accept: v => v.comments.filter(row => !row.authorAgentId).length === 2 && v.comments.filter(row => !row.authorAgentId)[1]?.body === ctx.scenario.query, + reject: v => !v.held ? "The old provider left the fixture's cleanup hold before the follow-up message was accepted" : undefined }); + const runsWhileHeld = await ctx.allRuns(); + state.noSuccessorWhileHeld = runsWhileHeld.filter(row => row.id !== state.workerRunId).every(row => row.status === "queued" && !(row as { startedAt?: string | null }).startedAt); + state.messageAcceptedDuringHeldCleanup = accepted.held && await ownedCleanupProviderIsHeld(ctx); + if (state.messageAcceptedDuringHeldCleanup !== true) throw new Error("The follow-up message was not accepted inside the held cleanup interval"); + state.heldReleasedAt = new Date().toISOString(); + await releaseOwnedCleanupProvider(ctx); const cleaned = await pollUntil({ label: "Actual old source cancellation ACK and owned provider endpoint retirement before neutral gate release", deadlineAt: Date.now() + 90000, load: async () => ({ old: await ctx.input.api.get(`/api/heartbeat-runs/${state.workerRunId}`), providerExited: await ownedCleanupProviderHasExited(ctx) }), accept: v => v.old.status === "cancelled" && (v.old.resultJson?.nativeCancellation as StopIntent | undefined)?.dispatchState === "acknowledged" && v.providerExited }); @@ -101,7 +112,7 @@ export const runtimePostStopCases = [runtimeDefinition("recovery.post-stop-messa const next = await ctx.input.api.get(`/api/heartbeat-runs/${successor.id}`), comments = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`); const human = comments.filter(row => !row.authorAgentId), canonical = human[1]; const native = next.runnerProfileJson?.nativeExecutionInput as { task?: { prompt?: string } } | undefined; - state.chronologyBound = !!canonical && sameSourcePostStopChronology({ boundary: state.nativeStopBoundary as NativeStopBoundaryReceipt, old, next, human: canonical, query: ctx.scenario.query, companyId: ctx.input.fixtures.company.id, agentId: ctx.input.fixtures.agent.id, humanId: String(state.humanId), prompt: native?.task?.prompt ?? "", wakeCommentIds: (next.contextSnapshot?.wakeCommentIds as string[] | undefined) ?? [] }); + state.chronologyBound = !!canonical && sameSourcePostStopChronology({ boundary: state.nativeStopBoundary as NativeStopBoundaryReceipt, old, next, human: canonical, query: ctx.scenario.query, companyId: ctx.input.fixtures.company.id, agentId: ctx.input.fixtures.agent.id, humanId: String(state.humanId), prompt: native?.task?.prompt ?? "", wakeCommentIds: (next.contextSnapshot?.wakeCommentIds as string[] | undefined) ?? [], heldReleasedAt: String(state.heldReleasedAt) }); state.initialHumanRetained = human[0]?.id === (state.initialHuman as IssueComment).id && human[0]?.body === (state.initialHuman as IssueComment).body; state.singleSuccessor = settled.runs.filter(row => row.id !== old.id).length === 1; state.sameCurrentAssignment = ctx.issue()?.assigneeAgentId === next.agentId || (ctx.issue() as { conversationAgentId?: string }).conversationAgentId === next.agentId; diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-provider-cleanup.ts b/tests/runner-e2e/chat-catalog-domains/runtime-provider-cleanup.ts index 729a361db2a..4cbf20d32b2 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-provider-cleanup.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-provider-cleanup.ts @@ -69,6 +69,9 @@ async function soleProviderDescendant(ctx: ChatCatalogContext, runnerPid: number await ctx.input.evidence("runtime-cleanup-provider-search.json", { kind, descendants: seen, qualifying: providers.length }); return providers.length === 1 ? providers[0] : undefined; } +/** Upper bound on the hold, only so an aborted attempt never leaves a stopped process behind. The + * case releases the hold itself, deterministically, once the follow-up message is durably accepted. */ +const MAXIMUM_SUSPENSION_MS = 120_000; /** This does not signal any group or control ACK timing. Actual public Stop must immediately follow. */ export async function suspendOwnedProviderForCleanup(ctx: ChatCatalogContext, binding: { runId: string; issueId: string; agentId: string }) { if (ctx.scenario.id !== "recovery.post-stop-message" || suspended.has(ctx)) throw new Error("Bounded provider cleanup suspension is exclusive to the selected post-stop scenario"); @@ -98,9 +101,9 @@ export async function suspendOwnedProviderForCleanup(ctx: ChatCatalogContext, bi const first = await providerIdentity(pid), second = await providerIdentity(pid); if (!matchesOwnedProvider({ identity: first, expected }) || !matchesOwnedProvider({ identity: second, expected }) || JSON.stringify(first) !== JSON.stringify(second)) throw new Error("Provider cleanup suspension rejected changed or unowned process identity"); process.kill(pid, "SIGSTOP"); retainedProvider.set(ctx, second); - const timer = setTimeout(() => { void releaseOwnedCleanupProvider(ctx).catch(() => undefined); }, 8_000); timer.unref(); suspended.set(ctx, { identity: second, runId: run.id, timer }); + const timer = setTimeout(() => { void releaseOwnedCleanupProvider(ctx).catch(() => undefined); }, MAXIMUM_SUSPENSION_MS); timer.unref(); suspended.set(ctx, { identity: second, runId: run.id, timer }); await ctx.input.evidence("runtime-cleanup-owned-provider.json", { runId: run.id, companyId: run.companyId, issueId: run.nativeIssueId, agentId: run.agentId, providerPid: pid, providerStartedAt: started, - providerKind: kind, cleanupBoundary: kind === "opencode" ? viaProxy ? "owned-protocol-proxy-retirement" : "owned-opencode-serve-retirement" : "owned-app-server-retirement", sdkServerExitRequired: false, sameUid: true, directOwnedRunnerParent: true, proxyScriptSha256: proxyDigest, executableSha256: second.digest, durableSessionReceipt: true, identityReadTwice: true, signal: "SIGSTOP", groupSignalled: false, maximumSuspensionMs: 8_000 }); + providerKind: kind, cleanupBoundary: kind === "opencode" ? viaProxy ? "owned-protocol-proxy-retirement" : "owned-opencode-serve-retirement" : "owned-app-server-retirement", sdkServerExitRequired: false, sameUid: true, directOwnedRunnerParent: true, proxyScriptSha256: proxyDigest, executableSha256: second.digest, durableSessionReceipt: true, identityReadTwice: true, signal: "SIGSTOP", groupSignalled: false, maximumSuspensionMs: MAXIMUM_SUSPENSION_MS, releasedBy: "follow-up-message-accepted" }); return { providerPid: pid, providerStartedAt: started, runId: run.id }; } export async function releaseOwnedCleanupProvider(ctx: ChatCatalogContext) { @@ -112,6 +115,16 @@ export async function releaseOwnedCleanupProvider(ctx: ChatCatalogContext) { process.kill(fresh.pid, "SIGCONT"); suspended.delete(ctx); } +/** True only while this fixture still holds the exact owned provider stopped: same PID, start + * identity, executable digest, and a kernel state of stopped (T). The old provider's cleanup cannot + * finish while this holds, so anything observed under it happened during that cleanup. */ +export async function ownedCleanupProviderIsHeld(ctx: ChatCatalogContext) { + const owned = suspended.get(ctx); if (!owned) return false; + try { process.kill(owned.identity.pid, 0); } catch { return false; } + const state = await promisify(execFile)("/bin/ps", ["-p", String(owned.identity.pid), "-o", "stat="], { timeout: 5_000, maxBuffer: 4_096 }).then(out => out.stdout.trim(), () => ""); + const fresh = await providerIdentity(owned.identity.pid).catch(() => undefined); + return state.startsWith("T") && !!fresh && fresh.startedAt === owned.identity.startedAt && fresh.uid === owned.identity.uid && fresh.executable === owned.identity.executable && fresh.digest === owned.identity.digest; +} export async function ownedCleanupProviderHasExited(ctx: ChatCatalogContext) { const identity = retainedProvider.get(ctx); if (!identity) throw new Error("No retained exact provider identity for cleanup verification"); try { process.kill(identity.pid, 0); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ESRCH") return true; return false; }