fix: retain diagnostic reasons for native runner failures (#14481)

Retain bounded reasons for runner identity, harness recovery, and provider-pack read failures. Preserve existing ownership and cleanup proofs and compatibility with receipt-gated chat recovery.

Verified with executor, recovery, diagnostic privacy, typecheck, build, and full CI checks.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Devin FoleyandPaperclip authored and GitHub committed 2026-09-28 16:16:19 -07:00
1 parent 4f6cf5b3ff
commit ad1f7e98ea
5 files changed
+117 -34

No files matched your search

@@ -56157,9 +56157,13 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
"leased",
"wrong_thread",
"source_edited",
])(
"retries only the original pre-provider Telegram request after exact cleanup: %s",
async (mode) => {
"different_error",
].flatMap((mode) => [
"runner_state_identity_mismatch",
"runner_state_identity_mismatch: prior_owner_active",
].map((errorMessage) => ({ mode, errorMessage }))))(
"retries only the original pre-provider Telegram request after exact cleanup: $mode ($errorMessage)",
async ({ mode, errorMessage }) => {
const context = await committedChatResponseRecoveryFixture("telegram");
const providerAccount =
mode === "null_account"
@@ -56286,7 +56290,9 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
agentId: context.fixture.assignedAgentId,
status: "failed",
errorCode: "adapter_failed",
error: "runner_state_identity_mismatch",
error: mode === "different_error"
? errorMessage.replace("runner_state_identity_mismatch", "runner_state_identity_mismatch_other")
: errorMessage,
finishedAt: new Date(),
wakeupRequestId: action.id,
runtimeMode: "native",
+5 -1
View File
@@ -12006,7 +12006,11 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {
run.nativeIssueId === issueId &&
run.status === "failed" &&
run.errorCode === "adapter_failed" &&
run.error === "runner_state_identity_mismatch" &&
// A diagnostic reason does not change this failure category. The exact
// checkpoint, cleanup receipt, and no-provider-work proofs below still
// decide whether the original request can be retried.
(run.error === "runner_state_identity_mismatch" ||
run.error?.startsWith("runner_state_identity_mismatch: ")) &&
run.nativePhase === "observed" &&
coordinator.phase === "observed" &&
coordinator.attempt === 0 &&
@@ -1,4 +1,5 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { readFileSync } from "node:fs";
import {
access,
cp,
@@ -56,6 +57,11 @@ import { buildNativeHeartbeatPreparationSpans } from "./native-run-trace.js";
import { NativeRunnerOwnershipUnverifiedError } from "./native-runner-ownership.js";
import type { AdapterRuntimeEvent } from "../../adapters/index.js";
vi.mock("node:fs", async (importOriginal) => {
const actual = await importOriginal<typeof import("node:fs")>();
return { ...actual, readFileSync: vi.fn(actual.readFileSync) };
});
const githubAccess = vi.hoisted(() => ({
activate: vi.fn((_binding: { runId: string }) => vi.fn()),
stop: vi.fn(async () => undefined),
@@ -1171,6 +1177,54 @@ describe("remote provider pack manifest", () => {
});
});
describe("provider pack read diagnostics", () => {
it.each([
["EACCES", "permission_denied"],
["EPERM", "permission_denied"],
["EIO", "io_error"],
])("reports %s without exposing the underlying filesystem message", (code, reason) => {
const root = join(tmpdir(), "private-provider-pack");
const manifestPath = join(root, "provider-pack.json");
const cause = Object.assign(new Error(`${code}: cannot read ${manifestPath}`), {
code,
path: manifestPath,
});
vi.mocked(readFileSync).mockImplementationOnce(() => { throw cause; });
let failure: Error | undefined;
try { readRemoteProviderPackManifest(root); } catch (error) { failure = error as Error; }
expect(readFileSync).toHaveBeenLastCalledWith(manifestPath, "utf8");
expect(failure?.message).toBe(`runner_remote_provider_artifact_incompatible: provider-pack.json is unreadable (${reason})`);
expect(failure?.message).not.toContain(root);
expect(failure?.message).not.toContain(code);
expect(failure?.cause).toBe(cause);
});
it("classifies a JSON null manifest as incompatible instead of a TypeError", async () => {
const root = await mkdtemp(join(tmpdir(), "paperclip-null-pack-"));
try {
await writeFile(join(root, "provider-pack.json"), "null");
expect(() => readRemoteProviderPackManifest(root)).toThrow(
"runner_remote_provider_artifact_incompatible: provider pack pins or source revision do not match",
);
} finally { await rm(root, { recursive: true, force: true }); }
});
it.each(["missing", "invalid_json", "invalid_path_type"])("reports %s without putting the path in the terminal message", async (reason) => {
const root = await mkdtemp(join(tmpdir(), "paperclip-private-pack-"));
try {
const manifestPath = join(root, "provider-pack.json");
if (reason === "invalid_json") await writeFile(manifestPath, "{ private-invalid-json");
if (reason === "invalid_path_type") await mkdir(manifestPath);
let failure: Error | undefined;
try { readRemoteProviderPackManifest(root); } catch (error) { failure = error as Error; }
expect(failure?.message).toBe(`runner_remote_provider_artifact_incompatible: provider-pack.json is unreadable (${reason})`);
expect(failure?.message).not.toContain(root);
expect(failure?.message).not.toContain("private-invalid-json");
expect(failure?.cause).toBeDefined();
} finally { await rm(root, { recursive: true, force: true }); }
});
});
describe("native harness persistence profiles", () => {
const profile = (provider: Record<string, unknown>, driverKind: string) =>
resolveNativeHarnessPersistenceProfile({
@@ -1477,7 +1531,7 @@ describe("verified native harness backups", () => {
},
sourceProviderLeaseId: "sandbox-1",
}),
).toThrow("runner_harness_state_mismatch");
).toThrow("runner_harness_state_mismatch: backup_provider_identity_missing");
} finally {
await rm(root, { recursive: true, force: true });
}
@@ -9119,7 +9173,7 @@ describe("runnerd provider runtime wiring", () => {
execution: currentExecution,
runnerInstanceId: "runner-after-running-prior-scope",
}),
).rejects.toThrow("runner_state_identity_mismatch");
).rejects.toThrow("runner_state_identity_mismatch: prior_owner_active");
await expect(access(scopedRoot)).resolves.toBeUndefined();
await expect(access(join(stateBase, "quarantine"))).rejects.toThrow();
expect(state.createBackend).not.toHaveBeenCalled();
@@ -10691,7 +10745,7 @@ describe("runnerd provider runtime wiring", () => {
}
// Ambiguous ordinary recovery must still fail closed. Only the runtime's
// explicitly admitted replacement may retire these prior-session backups.
await expect(prepareReplacement()).rejects.toThrow("runner_harness_state_mismatch");
await expect(prepareReplacement()).rejects.toThrow("runner_harness_state_mismatch: backup_without_reusable_lease");
await expect(backend.openReplacementSession!({
identity: { runId: execution.binding.runId }, workingDirectory: execution.workspace.cwd,
} as never, {} as never)).resolves.toBe(replacement);
@@ -1691,7 +1691,7 @@ function migrateLegacyRunnerdStateRoot(input: {
// A legacy path does not encode the full session scope. A mismatch may be
// valid live state owned by another agent/workspace, so refusing the claim
// is safe but moving that ambiguous directory is not.
throw new Error("runner_state_identity_mismatch");
throw new Error("runner_state_identity_mismatch: legacy_owner_unverified");
}
if (
exactRun &&
@@ -1704,7 +1704,7 @@ function migrateLegacyRunnerdStateRoot(input: {
)
) {
quarantineRunnerdStateRoot(input.legacy, "identity_indeterminate");
throw new Error("runner_state_identity_mismatch");
throw new Error("runner_state_identity_mismatch: legacy_authority_indeterminate");
}
try {
renameSync(input.legacy, input.scoped);
@@ -1728,7 +1728,7 @@ function migrateLegacyRunnerdStateRoot(input: {
durableIdentityMatchesSession(scopedIdentity, input.execution),
);
if (!exactScopedRun && !sameVerifiedPriorRun) {
throw new Error("runner_state_identity_mismatch");
throw new Error("runner_state_identity_mismatch: migration_destination_owner_changed");
}
}
return input.scoped;
@@ -4573,7 +4573,7 @@ async function migrateRunnerdStateRootForExecution(input: {
await recoverQuiescentRunnerdState({ ...input, scoped });
}
if (input.restartRecovery?.kind === "reattach_remote_runner" && !existsSync(scoped)) {
throw new Error("runner_state_identity_mismatch");
throw new Error("runner_state_identity_mismatch: remote_reattach_root_missing");
}
if (existsSync(scoped)) {
if (!isSafeNativeStateDirectory(scoped)) {
@@ -4593,14 +4593,14 @@ async function migrateRunnerdStateRootForExecution(input: {
input.restartRecovery?.kind !== "reattach_remote_runner") {
quarantineRunnerdStateRoot(scoped, "identity_indeterminate");
}
throw new Error("runner_state_identity_mismatch");
throw new Error("runner_state_identity_mismatch: durable_identity_unreadable");
}
if (!durableIdentityMatchesSession(identity, input.execution)) {
if (input.restartRecovery?.kind !== "reattach_existing_runner" &&
input.restartRecovery?.kind !== "reattach_remote_runner") {
quarantineRunnerdStateRoot(scoped, "identity_mismatch");
}
throw new Error("runner_state_identity_mismatch");
throw new Error("runner_state_identity_mismatch: session_scope_mismatch");
}
if (input.restartRecovery?.kind === "bootstrap_incomplete") {
if (runnerdStateProvesIncompleteBootstrap(scoped)) {
@@ -4610,7 +4610,7 @@ async function migrateRunnerdStateRootForExecution(input: {
// Database evidence alone cannot distinguish a never-connected runner
// from a partially-persisted provider bootstrap. Only the durable PRP
// root can authorize a fresh bootstrap; anything else stays fail-closed.
throw new Error("runner_state_identity_mismatch");
throw new Error("runner_state_identity_mismatch: bootstrap_not_proven_incomplete");
}
if (input.restartRecovery?.kind === "reattach_remote_runner") {
await verifyRemoteRunnerReattachment({
@@ -4631,7 +4631,7 @@ async function migrateRunnerdStateRootForExecution(input: {
if (input.restartRecovery?.kind !== "reattach_existing_runner") {
quarantineRunnerdStateRoot(scoped, "identity_indeterminate");
}
throw new Error("runner_state_identity_mismatch");
throw new Error("runner_state_identity_mismatch: authority_indeterminate");
}
} else {
const verification = await verifyPriorRunnerdStateForSessionScope({
@@ -4654,7 +4654,7 @@ async function migrateRunnerdStateRootForExecution(input: {
: "identity_indeterminate",
);
}
throw new Error("runner_state_identity_mismatch");
throw new Error(`runner_state_identity_mismatch: prior_owner_${verification}`);
}
}
return;
@@ -4672,7 +4672,7 @@ async function migrateRunnerdStateRootForExecution(input: {
// Unlike the full-scope target above, this legacy name can legitimately
// belong to another scope. Leave it in place for its owner and fail the
// attempted migration visibly.
throw new Error("runner_state_identity_mismatch");
throw new Error("runner_state_identity_mismatch: legacy_session_scope_mismatch");
}
let verifiedPriorRunId: string | undefined;
if (!durableIdentityMatchesExecution(identity, input.execution)) {
@@ -4695,7 +4695,7 @@ async function migrateRunnerdStateRootForExecution(input: {
// untouched because the legacy name may still belong to them.
quarantineRunnerdStateRoot(legacy, "identity_indeterminate");
}
throw new Error("runner_state_identity_mismatch");
throw new Error(`runner_state_identity_mismatch: legacy_prior_owner_${verification}`);
}
verifiedPriorRunId = identity.runId;
}
@@ -4745,7 +4745,7 @@ async function recoverQuiescentRunnerdState(input: {
) {
// Do not roll back to an older valid checkpoint when a newer quarantined
// root contains unconfirmed work, even if the newer root is unreadable.
throw new Error("runner_state_identity_mismatch");
throw new Error("runner_state_identity_mismatch: quarantine_candidates_ambiguous");
}
const verified: Array<{
root: string;
@@ -4932,7 +4932,7 @@ async function recoverQuiescentRunnerdState(input: {
) {
// Known provider history is not permission to start a replacement when
// recovery cannot prove a unique, settled owner.
throw new Error("runner_state_identity_mismatch");
throw new Error("runner_state_identity_mismatch: quarantine_owner_unverified");
}
return;
}
@@ -4951,14 +4951,14 @@ async function recoverQuiescentRunnerdState(input: {
"recovery_state_too_large",
).toString("utf8") !== expected
) {
throw new Error("runner_state_identity_mismatch");
throw new Error("runner_state_identity_mismatch: recovery_evidence_changed");
}
}
if (
!localProcessDefinitelyGone(candidate.processPid) ||
!localProcessDefinitelyGone(candidate.processGroupId, true)
) {
throw new Error("runner_state_identity_mismatch");
throw new Error("runner_state_identity_mismatch: recovery_process_not_gone");
}
if (candidate.root !== input.scoped) {
if (existsSync(input.scoped)) {
@@ -5673,12 +5673,12 @@ export function buildNativeHarnessBackupManifest(input: {
completedAt?: string;
}): NativeHarnessBackupManifest {
if (!providerSessionIdentityIsPresent(input.providerSessionIdentity)) {
throw new Error("runner_harness_state_mismatch");
throw new Error("runner_harness_state_mismatch: backup_provider_identity_missing");
}
const profile = resolveNativeHarnessPersistenceProfile(input.execution);
const directories = profile.directories.map((directory) => {
const path = resolve(input.backupRoot, directory.name);
if (!existsSync(path)) throw new Error("runner_harness_state_mismatch");
if (!existsSync(path)) throw new Error("runner_harness_state_mismatch: backup_directory_missing");
return { name: directory.name, ...digestBackupDirectory(path) };
});
return {
@@ -9364,14 +9364,22 @@ export function readRemoteProviderPackManifest(
readFileSync(resolve(packRoot, "provider-pack.json"), "utf8"),
) as RemoteProviderPackManifest;
} catch (error) {
// The terminal run report keeps the outer message, not the cause chain.
// Keep a bounded reason there; raw filesystem errors include private paths.
const code = (error as NodeJS.ErrnoException | null)?.code;
const reason = error instanceof SyntaxError ? "invalid_json"
: code === "ENOENT" ? "missing"
: code === "EACCES" || code === "EPERM" ? "permission_denied"
: code === "EISDIR" || code === "ENOTDIR" ? "invalid_path_type"
: "io_error";
throw new Error(
"runner_remote_provider_artifact_incompatible: provider-pack.json is unreadable",
`runner_remote_provider_artifact_incompatible: provider-pack.json is unreadable (${reason})`,
{ cause: error },
);
}
const payload = manifest?.payload;
if (
manifest.schema !== REMOTE_PROVIDER_PACK_SCHEMA ||
manifest?.schema !== REMOTE_PROVIDER_PACK_SCHEMA ||
!payload ||
canonicalJson(payload.pins) !== canonicalJson(REMOTE_PROVIDER_PACK_PINS) ||
canonicalJson(payload.acpxProfileDigests) !==
@@ -11543,7 +11551,7 @@ async function createRunnerdBackendWithinSessionClaim(
async () => {
for (const directory of persistenceProfile.directories) {
const targetPath = remotePersistencePath(directory);
if (!targetPath) throw new Error("runner_harness_state_mismatch");
if (!targetPath) throw new Error("runner_harness_state_mismatch: restore_target_unavailable");
await stageRemoteRunnerDirectory({
target: remoteTarget,
runner: remoteCommandRunner,
@@ -11571,7 +11579,7 @@ async function createRunnerdBackendWithinSessionClaim(
canonicalJson(restored.providerSessionIdentity) !==
canonicalJson(backup.manifest.providerSessionIdentity)
) {
throw new Error("runner_harness_state_mismatch");
throw new Error("runner_harness_state_mismatch: restored_provider_identity_changed");
}
// A deliberately non-reusable environment receives a fresh provider lease
// for every turn. Stamp that new lease as soon as the verified host backup
@@ -11586,7 +11594,7 @@ async function createRunnerdBackendWithinSessionClaim(
for (const directory of persistenceProfile.directories) {
if (directory.location !== "filesystem") continue;
const targetPath = remotePersistencePath(directory);
if (!targetPath) throw new Error("runner_harness_state_mismatch");
if (!targetPath) throw new Error("runner_harness_state_mismatch: bootstrap_target_unavailable");
const escapedTarget = targetPath.replaceAll("'", "'\\''");
const created = await remoteCommandRunner.execute({
command: "sh",
@@ -11766,7 +11774,7 @@ async function createRunnerdBackendWithinSessionClaim(
// A continuation that has a durable backup but no recorded reusable
// lease was not provider-confirmed lost. Never silently create a new
// provider session from that ambiguous state.
throw new Error("runner_harness_state_mismatch");
throw new Error("runner_harness_state_mismatch: backup_without_reusable_lease");
}
}
} else if (remoteTarget && remoteCommandRunner) {
@@ -11950,7 +11958,7 @@ async function createRunnerdBackendWithinSessionClaim(
!verified.runnerState ||
!verified.providerSessionIdentity
) {
throw new Error("runner_harness_state_mismatch");
throw new Error("runner_harness_state_mismatch: checkpoint_identity_incomplete");
}
const providerSessionIdentity = verified.providerSessionIdentity;
@@ -11965,7 +11973,7 @@ async function createRunnerdBackendWithinSessionClaim(
for (const directory of persistenceProfile.directories) {
const sourcePath = remotePersistencePath(directory);
if (!sourcePath)
throw new Error("runner_harness_state_mismatch");
throw new Error("runner_harness_state_mismatch: checkpoint_source_unavailable");
const targetPath = resolve(pendingRoot, directory.name);
await syncRemoteRunnerDirectoryOut({
runner: remoteCommandRunner,
@@ -11975,7 +11983,7 @@ async function createRunnerdBackendWithinSessionClaim(
excludeEntries: directory.excludeEntries,
});
if (!existsSync(targetPath)) {
throw new Error("runner_harness_state_mismatch");
throw new Error("runner_harness_state_mismatch: checkpoint_directory_missing");
}
}
const manifest = buildNativeHarnessBackupManifest({