test(evals): author genuine owned native infrastructure recovery fixtures

This commit is contained in:
Dotta committed 2026-10-26 22:09:30 -05:00
1 parent 02c0b64b72
commit 0a97b24b64
5 files changed
+205 -4

No files matched your search

@@ -166,8 +166,8 @@
},
{
"scenarioId": "recovery.explain-stall",
"readinessClass": "local",
"needs": "Real retained recovery interruption/attempt counter/next action; contrast infrastructure, input, budget and deliberate stop causes."
"readinessClass": "implemented",
"needs": "Actual local native process-fault fixture verifies public PID/company/task/actor, OS UID/group/start/executable and attempt-owned realpath runtime root twice before signalling only the exact owned Runner PID. Requires three distinct actual coordinator process attempts on the same heartbeat and genuine terminal native_session_retry_exhausted source-scoped board recovery action with retained run/cause/maxAttempts/nextAction. Unexpected quarantine, permanent/model failure or replacement heartbeat blocks setup concretely, never relabelled. Canonical reply must preserve original task/action/owner and failure, no duplicate repair; diagnosis additionally binds actual three-attempt infrastructure facts. All paths authored; no process signals or live qualification performed."
},
{
"scenarioId": "recovery.controller-restart",
@@ -176,8 +176,8 @@
},
{
"scenarioId": "recovery.source-action",
"readinessClass": "local",
"needs": "Real failed task and source recovery action, owner readback, separate independent repair only when actual work warrants it."
"readinessClass": "implemented",
"needs": "Actual local native process-fault fixture verifies public PID/company/task/actor, OS UID/group/start/executable and attempt-owned realpath runtime root twice before signalling only the exact owned Runner PID. Requires three distinct actual coordinator process attempts on the same heartbeat and genuine terminal native_session_retry_exhausted source-scoped board recovery action with retained run/cause/maxAttempts/nextAction. Unexpected quarantine, permanent/model failure or replacement heartbeat blocks setup concretely, never relabelled. Canonical reply must preserve original task/action/owner and failure, no duplicate repair; diagnosis additionally binds actual three-attempt infrastructure facts. All paths authored; no process signals or live qualification performed."
},
{
"scenarioId": "recovery.retry-exhausted",
@@ -0,0 +1,24 @@
import { expect, it } from "vitest";
import { gradeInfrastructureRecovery } from "./runtime-infrastructure.js";
function valid(id = "recovery.explain-stall") {
const recovery = { id: "recovery", companyId: "company", sourceIssueId: "website", kind: "active_run_watchdog", cause: "native_session_retry_exhausted", ownerType: "board", ownerAgentId: null,
returnOwnerAgentId: "morgan", recoveryIssueId: null, evidence: { runId: "run", coordinatorAttempt: 3, recoveryDisposition: "native_session_retry_exhausted" }, maxAttempts: 3, wakePolicy: null, nextAction: "Inspect the provider trace and explicitly choose a replacement run or provider configuration; automatic provider work is stopped." };
return { scenarioId: id, companyId: "company", leadId: "ceo", agents: [], links: [], humanQueries: [], reply: "", tasks: [{ id: "website" }], activity: [],
runs: [{ id: "run", agentId: "morgan", status: "failed" }, { id: "source", agentId: "ceo", status: "succeeded" }],
state: { task: { id: "website" }, worker: { id: "morgan" }, failure: { id: "run", status: "failed", nativePhase: "terminal_failure" }, recoveryBefore: recovery, recoveryAfter: { ...recovery },
setupFaults: [1, 2, 3].map(pid => ({ runId: "run", pid, errorCode: "native_runner_process_exited" })), sourceRecoveryVisible: true,
queryReply: "Website stopped after three native Runner infrastructure attempts exhausted recovery. The original task's recovery action belongs to the board: an operator must inspect its provider trace and explicitly choose a replacement run or provider configuration." } };
}
it("accepts actual bounded process failure and truthful source-scoped diagnosis", () => expect(gradeInfrastructureRecovery(valid()).passed).toBe(true));
it("accepts a natural factual explanation", () => { const o = valid(); o.state.queryReply = "The Website's native session hit its limit of 3 resume attempts after process loss. You can open the run details and inspect the provider trace before authorizing a retry; this needs operator action."; expect(gradeInfrastructureRecovery(o).passed).toBe(true); });
it("accepts actual original source recovery identification", () => { const o = valid("recovery.source-action"); o.state.queryReply = "Use the recovery action on the original Website task. Its board owner must inspect the run details and choose an explicit retry. A separate repair task is unnecessary."; expect(gradeInfrastructureRecovery(o).passed).toBe(true); });
it("rejects relabelled unavailable-model failure as infrastructure loss", () => { const o = valid(); o.state.setupFaults[0]!.errorCode = "native_provider_model_rejected"; expect(gradeInfrastructureRecovery(o).passed).toBe(false); });
it("rejects an invented exhaustion counter", () => { const o = valid(); o.state.recoveryBefore.evidence.coordinatorAttempt = 1; expect(gradeInfrastructureRecovery(o).passed).toBe(false); });
it("rejects three receipts for one reused PID", () => { const o = valid(); o.state.setupFaults[1]!.pid = 1; expect(gradeInfrastructureRecovery(o).passed).toBe(false); });
it("rejects silently transferring the action owner", () => { const o = valid(); o.state.recoveryAfter.ownerType = "agent"; expect(gradeInfrastructureRecovery(o).passed).toBe(false); });
it("rejects a foreign source action", () => { const o = valid(); o.state.recoveryAfter.sourceIssueId = "other"; expect(gradeInfrastructureRecovery(o).passed).toBe(false); });
it("rejects falsely diagnosing a clear budget as the cause", () => { const o = valid(); o.state.queryReply += " Website stopped because the budget was exhausted."; expect(gradeInfrastructureRecovery(o).passed).toBe(false); });
it("rejects falsely calling a filed repair success", () => { const o = valid(); o.state.queryReply += " Website is now complete."; expect(gradeInfrastructureRecovery(o).passed).toBe(false); });
it("rejects duplicate repair chains", () => { const o = valid(); o.tasks.push({ id: "repair" }); expect(gradeInfrastructureRecovery(o).passed).toBe(false); });
it("rejects concealing resumed work as still failed", () => { const o = valid(); o.runs[0]!.status = "running"; expect(gradeInfrastructureRecovery(o).passed).toBe(false); });
it("rejects missing actual source recovery evidence", () => expect(gradeInfrastructureRecovery(undefined).passed).toBe(false));
@@ -0,0 +1,93 @@
import { expect } from "@playwright/test";
import type { Issue, IssueRecoveryAction } from "../../../packages/shared/src/types/issue.js";
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
import { pollUntil } from "../api.js";
import { checks, type Entity } from "./internal-support.js";
import { busyRuntimeWorker, releaseRuntimeWorker } from "./runtime-busy-worker.js";
import { faultOwnedNativeProcess, type NativeFaultRun } from "./runtime-native-fault.js";
import { runtimeDefinition, runtimeObservation, observeRuntime, type RuntimeAgent } from "./runtime-support.js";
export function isExhaustedNativeRecovery(action: IssueRecoveryAction | undefined, run: NativeFaultRun, companyId: string, issueId: string, agentId: string) {
return !!action && action.companyId === companyId && action.sourceIssueId === issueId && action.kind === "active_run_watchdog" && action.cause === "native_session_retry_exhausted"
&& action.ownerType === "board" && action.ownerAgentId == null && action.returnOwnerAgentId === agentId && action.recoveryIssueId == null
&& action.evidence.runId === run.id && action.evidence.coordinatorAttempt === 3 && action.evidence.recoveryDisposition === "native_session_retry_exhausted"
&& action.maxAttempts === 3 && action.wakePolicy == null && action.nextAction.trim().length > 0 && run.status === "failed" && run.nativePhase === "terminal_failure";
}
/** Three actual coordinator attempts on the SAME native heartbeat; never status patches or fake event rows. */
export async function exhaustedNativeInfrastructure(ctx: ChatCatalogContext) {
const state = await busyRuntimeWorker(ctx), task = state.task as Entity;
const faults: Array<{ runId: string; pid: number | null; errorCode: string | null }> = [];
let previousPid: number | null = null;
for (let attempt = 1; attempt <= 3; attempt++) {
const active = await pollUntil({ label: `Real native coordinator attempt ${attempt} on the owned original heartbeat`, deadlineAt: Date.now() + 120_000,
load: () => ctx.input.api.get<NativeFaultRun>(`/api/heartbeat-runs/${state.workerRunId}`),
accept: run => run.status === "running" && !!run.processPid && run.processPid !== previousPid,
reject: run => run.nativePhase === "terminal_failure" ? "Native infrastructure became terminal before three real bounded attempts; this is not retry exhaustion" : undefined });
const failed = await faultOwnedNativeProcess(ctx, { runId: active.id, issueId: task.id, agentId: state.worker.id });
faults.push({ runId: failed.id, pid: failed.processPid, errorCode: failed.errorCode }); previousPid = active.processPid;
await ctx.input.evidence(`runtime-infrastructure-attempt-${attempt}.json`, { runId: active.id, attempt, processPid: active.processPid, errorCode: failed.errorCode, status: failed.status, nativePhase: failed.nativePhase });
if ((await ctx.allRuns()).length !== 1) throw new Error("Infrastructure setup created an undeclared replacement heartbeat instead of same-run coordinator recovery");
}
const result = await pollUntil({ label: "Actual exhausted source recovery action with no automatic wake", deadlineAt: Date.now() + 30_000,
load: async () => ({ task: await ctx.input.api.get<Issue>(`/api/issues/${task.id}`), run: await ctx.input.api.get<NativeFaultRun>(`/api/heartbeat-runs/${state.workerRunId}`),
recovery: await ctx.input.api.get<{ active?: IssueRecoveryAction; actions: IssueRecoveryAction[] }>(`/api/issues/${task.id}/recovery-actions`) }),
accept: v => v.task.status === "blocked" && isExhaustedNativeRecovery(v.recovery.active, v.run, ctx.input.fixtures.company.id, task.id, state.worker.id) });
return { ...state, task: result.task, failure: result.run, recoveryBefore: result.recovery.active!, setupFaults: faults };
}
export function gradeInfrastructureRecovery(value: unknown) {
const c = checks(), o = runtimeObservation(value);
c.add("real-infrastructure-and-source-action-evidence", !!o, "Three actual verified native Runner losses and the resulting public source-scoped recovery action are required."); if (!o) return c.result();
const s = o.state, task = s.task as Issue, run = s.failure as NativeFaultRun, before = s.recoveryBefore as IssueRecoveryAction, after = s.recoveryAfter as IssueRecoveryAction;
c.add("genuine-bounded-infrastructure-exhaustion", !!run && isExhaustedNativeRecovery(before, run, o.companyId, task.id, (s.worker as Entity).id)
&& Array.isArray(s.setupFaults) && s.setupFaults.length === 3 && new Set(s.setupFaults.map(row => (row as { pid: number }).pid)).size === 3
&& (s.setupFaults as Array<{ runId: string; errorCode: string }>).every(row => row.runId === run.id && ["native_runner_process_exited", "provider_transport_failed", "provider_stdout_closed", "provider_process_output_closed", "native_session_cleanup_quarantined"].includes(row.errorCode)),
"The actual original native run exhausts three coordinator attempts with real process loss evidence; model failure, cancellation or a fabricated attempt counter cannot qualify.");
c.add("retained-source-owner-cause-and-next-action", !!after && after.id === before.id && after.sourceIssueId === task.id && after.recoveryIssueId == null
&& after.ownerType === "board" && after.cause === before.cause && after.nextAction === before.nextAction && after.evidence.runId === run.id
&& s.sourceRecoveryVisible === true && o.tasks.length === 1 && o.runs.length === 2
&& o.runs.find(row => row.id === run.id)?.status === "failed" && o.runs.filter(row => row.agentId === o.leadId).every(row => row.status === "succeeded"),
"The original task retains its exact recovery action, responsible operator and failure; explanatory chat creates no duplicate repair chain or premature worker.");
const reply = String(s.queryReply ?? o.reply);
c.add("concrete-honest-operator-next-step", /website|Launch/i.test(reply) && /operator|human|board|you (?:can|need|must|should)/i.test(reply)
&& /retry|inspect|provider trace|configuration|reconcile|recovery action|run details/i.test(reply)
&& !/website (?:is|has been) (?:now )?(?:done|complete|finished)|successfully (?:recovered|repaired)|I (?:created|opened).{0,30}repair task/i.test(reply),
"The actual controller reply identifies the responsible supported next step and preserves failure instead of calling a filed repair or queued retry success.");
if (o.scenarioId === "recovery.explain-stall") {
c.add("recorded-infrastructure-cause-and-actual-attempt-count", /runner|process|native session|infrastructure/i.test(reply) && /exhaust|attempt|retr(?:y|ies)|resume/i.test(reply)
&& /\b3\b|\bthree\b/i.test(reply) && !/(?:stopped|blocked) (?:because|due to).{0,35}(?:budget|missing (?:brief|input)|you (?:paused|stopped))/i.test(reply),
"Diagnosis uses the real exhausted native infrastructure evidence and three attempts, distinguishing it from invented budget, input or deliberate-stop causes.");
} else c.add("source-recovery-action-identified", reply.includes(before.id) || /(?:source|original|same|website).{0,60}(?:recovery|repair|retry|action)|(?:recovery|repair).{0,60}(?:source|original|same|website)/i.test(reply),
"The response identifies the recovery on the actual original task; no independent repair issue is invented where ordinary harness/operator recovery is sufficient.");
return c.result();
}
function infrastructureCase(id: "recovery.explain-stall" | "recovery.source-action") {
return runtimeDefinition(id, exhaustedNativeInfrastructure, gradeInfrastructureRecovery, {
expectedRunCount: 2, attemptTimeoutMs: 1_200_000,
runPolicy: { kind: "terminal-counts", terminalStatusCounts: { failed: 1, succeeded: 1 } },
setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["lead-ready", "real-native-infrastructure-exhaustion", "original-source-action-owner"] },
async verifySetup(ctx, state) { const c = checks(), lead = await ctx.input.api.get<RuntimeAgent>(`/api/agents/${ctx.input.fixtures.agent.id}`);
const run = await ctx.input.api.get<NativeFaultRun>(`/api/heartbeat-runs/${state.workerRunId}`), task = state.task as Issue;
const recovery = await ctx.input.api.get<{ active?: IssueRecoveryAction }>(`/api/issues/${task.id}/recovery-actions`);
c.add("lead-ready", lead.lifecycleState === "ready" && ["active", "idle"].includes(lead.status ?? ""), "The actual controller remains independently admitted.");
c.add("real-native-infrastructure-exhaustion", isExhaustedNativeRecovery(recovery.active, run, ctx.input.fixtures.company.id, task.id, (state.worker as Entity).id), "The saved native run really exhausted its own three attempts.");
c.add("original-source-action-owner", recovery.active?.sourceIssueId === task.id && recovery.active.ownerType === "board" && recovery.active.recoveryIssueId == null, "The original task exposes the real operator-owned recovery without a separate repair task."); return c.result(); },
sourceFiles: ["chat-catalog-domains/runtime-infrastructure.ts", "chat-catalog-domains/runtime-native-fault.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "../../server/src/services/hot-restart.ts", "../../server/src/vendor/paperclip-runner/index.ts", "chat-scenarios/recovery.yaml"],
async act(ctx, state) {
await pollUntil({ label: "Canonical source explanation while native infrastructure remains failed", deadlineAt: Date.now() + 150_000,
load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; },
accept: v => v.source?.conversationState === "waiting" && v.runs.length === 2 && v.runs.some(row => row.id === state.workerRunId && row.status === "failed") && v.runs.some(row => row.agentId === ctx.input.fixtures.agent.id && row.status === "succeeded") });
state.queryReply = (await observeRuntime(ctx, state)).reply;
const source = ctx.input.page.url(), task = state.task as Issue;
await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" });
await expect(ctx.input.page.getByRole("heading", { name: task.title, exact: true }).first()).toBeVisible();
await expect(ctx.input.page.getByText(/recovery|retry|failed|interrupted/i).first()).toBeVisible(); state.sourceRecoveryVisible = true;
await ctx.input.capture("actual-exhausted-source-recovery", "Original Website retains actual exhausted native infrastructure and operator recovery", "runtime-infrastructure-recovery.png");
await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
},
async observe(ctx, state) { const recovery = await ctx.input.api.get<{ active?: IssueRecoveryAction }>(`/api/issues/${(state.task as Entity).id}/recovery-actions`); state.recoveryAfter = recovery.active; return observeRuntime(ctx, state); },
async verifyResult(ctx, state) { const source = ctx.input.page.url(), task = state.task as Issue;
await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" }); await ctx.input.page.reload({ waitUntil: "domcontentloaded" });
await expect(ctx.input.page.getByRole("heading", { name: task.title, exact: true }).first()).toBeVisible(); await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); },
cleanup: async (_ctx, state) => releaseRuntimeWorker(state),
});
}
export const runtimeInfrastructureCases = [infrastructureCase("recovery.explain-stall"), infrastructureCase("recovery.source-action")];
@@ -0,0 +1,17 @@
import { expect, it } from "vitest";
import { matchesOwnedNativeIdentity, type NativeFaultBinding } from "./runtime-native-fault.js";
function valid() {
return { identity: { pid: 940123, uid: 501, group: 940123, startedAt: "2026-10-26T12:00:00Z", executable: "/qualified/runnerd", commandRunId: "run", commandRunnerId: "runner", runtimeRoot: "/owned/attempt/state/runner" },
run: { id: "run", companyId: "company", agentId: "morgan", status: "running", runtimeMode: "native", nativeIssueId: "website", runnerInstanceId: "runner", processPid: 940123, processGroupId: 940123, processStartedAt: new Date("2026-10-26T12:00:00.123Z") } as NativeFaultBinding,
expected: { companyId: "company", agentId: "morgan", issueId: "website", uid: 501, executable: "/qualified/runnerd", root: "/owned/attempt" } };
}
it("accepts exact public native binding and separately observed owned process identity", () => expect(matchesOwnedNativeIdentity(valid())).toBe(true));
it.each(["uid", "pid", "group"] as const)("rejects mismatched %s", field => { const x = valid(); x.identity[field]++; expect(matchesOwnedNativeIdentity(x)).toBe(false); });
it.each(["commandRunId", "commandRunnerId", "executable", "runtimeRoot"] as const)("rejects foreign %s", field => { const x = valid(); x.identity[field] = "/other"; expect(matchesOwnedNativeIdentity(x)).toBe(false); });
it("rejects a sibling path sharing the attempt prefix", () => { const x = valid(); x.identity.runtimeRoot = "/owned/attempt-foreign/state"; expect(matchesOwnedNativeIdentity(x)).toBe(false); });
it("rejects PID reuse with changed start identity", () => { const x = valid(); x.identity.startedAt = "2026-10-26T13:00:00Z"; expect(matchesOwnedNativeIdentity(x)).toBe(false); });
it.each(["companyId", "agentId", "nativeIssueId"] as const)("rejects changed public run %s", field => { const x = valid(); x.run[field] = "foreign"; expect(matchesOwnedNativeIdentity(x)).toBe(false); });
it("rejects a missing retained process start", () => { const x = valid(); x.run.processStartedAt = null; expect(matchesOwnedNativeIdentity(x)).toBe(false); });
it("rejects terminal public run state", () => { const x = valid(); x.run.status = "succeeded"; expect(matchesOwnedNativeIdentity(x)).toBe(false); });
it("rejects legacy runtime", () => { const x = valid(); x.run.runtimeMode = "legacy"; expect(matchesOwnedNativeIdentity(x)).toBe(false); });
@@ -0,0 +1,67 @@
import { execFile } from "node:child_process";
import { promisify } from "node:util";
import path from "node:path";
import { realpath } from "node:fs/promises";
import type { heartbeatRuns } from "../../../packages/db/src/schema/heartbeat_runs.js";
import type { HeartbeatRun } from "../../../packages/shared/src/types/heartbeat.js";
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
import { readProcessStartedAt } from "../../../server/src/services/hot-restart.js";
import { defaultCapabilityRunnerdBinary } from "../../../server/src/vendor/paperclip-runner/index.js";
import { pollUntil } from "../api.js";
// The public full-run route selects these persisted native fields; the legacy
// shared HeartbeatRun DTO does not yet declare them. Keep this exact projection.
export type NativeFaultRun = HeartbeatRun & Pick<typeof heartbeatRuns.$inferSelect, "runtimeMode" | "nativeIssueId" | "runnerInstanceId" | "nativePhase">;
export type NativeFaultBinding = Pick<NativeFaultRun, "id" | "companyId" | "agentId" | "status" | "runtimeMode" | "nativeIssueId" | "runnerInstanceId" | "processPid" | "processGroupId" | "processStartedAt">;
export interface OwnedNativeIdentity {
pid: number; uid: number; group: number; startedAt: string; executable: string; commandRunId: string; commandRunnerId: string; runtimeRoot: string;
}
export function matchesOwnedNativeIdentity(input: { identity: OwnedNativeIdentity; run: NativeFaultBinding;
expected: { companyId: string; agentId: string; issueId: string; uid: number; executable: string; root: string } }) {
const { identity: p, run, expected: e } = input;
return run.status === "running" && run.runtimeMode === "native" && run.companyId === e.companyId && run.agentId === e.agentId && run.nativeIssueId === e.issueId
&& Number.isSafeInteger(p.pid) && p.pid > 1 && p.pid !== process.pid && p.pid === run.processPid && p.group === run.processGroupId
&& p.uid === e.uid && p.commandRunId === run.id && p.commandRunnerId === run.runnerInstanceId
&& p.executable === e.executable && path.isAbsolute(p.runtimeRoot) && p.runtimeRoot.startsWith(`${e.root}${path.sep}`) && path.resolve(p.runtimeRoot) === p.runtimeRoot
&& !!run.processStartedAt && Number.isFinite(Date.parse(p.startedAt)) && Math.abs(Date.parse(p.startedAt) - Date.parse(String(run.processStartedAt))) < 2_000;
}
async function processIdentity(run: NativeFaultRun): Promise<OwnedNativeIdentity> {
const pid = run.processPid;
if (!Number.isSafeInteger(pid) || !pid || pid <= 1) throw new Error("Native infrastructure fault lacks an actual public process identity");
const read = async (field: string) => (await promisify(execFile)("/bin/ps", ["-p", String(pid), "-o", `${field}=`], { maxBuffer: 65_536, timeout: 5_000 })).stdout.trim();
try {
const [uid, group, executable, command, startedAt] = await Promise.all([read("uid"), read("pgid"), read("comm"), read("args"), readProcessStartedAt(pid)]);
if (!startedAt) throw new Error("missing process start");
// Source buildNativeRunnerArguments has exactly these finite identity flags.
// Never retain or print the complete command line or transport URL.
const argument = (name: string) => {
const matches = [...command.matchAll(new RegExp(`(?:^|\\s)--${name}\\s+([^\\s]+)(?=\\s|$)`, "g"))];
if (matches.length !== 1) throw new Error("ambiguous process argument identity");
return matches[0]![1]!;
};
return { pid, uid: Number(uid), group: Number(group), startedAt, executable: await realpath(executable), commandRunId: argument("run-id"), commandRunnerId: argument("runner-id"), runtimeRoot: await realpath(argument("state-dir")) };
} catch { throw new Error("Native infrastructure fault could not independently verify the actual process; private diagnostics withheld"); }
}
/** Authoring primitive only: this function is invoked solely inside the coordinator's selected live attempt. */
export async function faultOwnedNativeProcess(ctx: ChatCatalogContext, binding: { runId: string; issueId: string; agentId: string }) {
if (ctx.input.execution.environment.id !== "local" || typeof process.getuid !== "function") throw new Error("Owned infrastructure fault requires a supported local UID/process identity");
const isolated = process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT; if (!isolated) throw new Error("Owned infrastructure fault requires the explicit isolated attempt root");
const expected = { companyId: ctx.input.fixtures.company.id, agentId: binding.agentId, issueId: binding.issueId, uid: process.getuid(), executable: await realpath(defaultCapabilityRunnerdBinary()), root: await realpath(isolated) };
const load = () => ctx.input.api.get<NativeFaultRun>(`/api/heartbeat-runs/${binding.runId}`);
const first = await load(), observed = await processIdentity(first);
if (!matchesOwnedNativeIdentity({ identity: observed, run: first, expected })) throw new Error("Owned infrastructure fault rejected a foreign or mismatched native process identity");
const current = await load(), confirmed = await processIdentity(current);
if (!matchesOwnedNativeIdentity({ identity: confirmed, run: current, expected }) || JSON.stringify(confirmed) !== JSON.stringify(observed)) throw new Error("Owned infrastructure fault rejected process reuse or changed ownership");
// Deliberately signal only the verified Runner PID. Its provider/tools remain
// subject to the production cleanup/quarantine rules; never signal a group.
process.kill(confirmed.pid, "SIGKILL");
await ctx.input.evidence("runtime-owned-native-process-fault.json", { runId: current.id, companyId: current.companyId, agentId: current.agentId,
issueId: current.nativeIssueId, runnerInstanceId: current.runnerInstanceId, processPid: confirmed.pid, processGroupId: confirmed.group, processStartedAt: confirmed.startedAt,
uidMatches: true, executableMatches: true, runtimeRootWithinAttempt: true, identityReadTwice: true, signal: "SIGKILL", processGroupSignalled: false });
const failed = await pollUntil({ label: "Real owned native infrastructure failure after verified Runner process loss", deadlineAt: Date.now() + 150_000,
load, accept: run => run.status === "failed" && !!run.finishedAt && !!run.errorCode,
reject: run => ["cancelled", "succeeded", "timed_out"].includes(run.status) ? "The real process fault did not produce a failed native infrastructure run" : undefined });
if (!["native_runner_process_exited", "provider_transport_failed", "provider_stdout_closed", "provider_process_output_closed", "native_session_cleanup_quarantined"].includes(failed.errorCode ?? ""))
throw new Error("The owned process fault did not retain a source-qualified infrastructure/cleanup failure; actual classification is not relabelled");
await ctx.input.evidence("runtime-owned-native-process-failure.json", { runId: failed.id, status: failed.status, errorCode: failed.errorCode, finishedAt: failed.finishedAt, nativePhase: (failed as NativeFaultRun & { nativePhase?: string }).nativePhase ?? null });
return failed;
}