diff --git a/doc/observability.md b/doc/observability.md index fb2a8b15ee..491e8e35c8 100644 --- a/doc/observability.md +++ b/doc/observability.md @@ -486,6 +486,17 @@ These fields contain build identifiers; they add no tenant or user identity. `errorCode`, and `agentAdapter`. The server redacts the error message and the error code before it sends the event. +Native runner identity and harness failures retain their existing error prefixes. +Their terminal messages now include a bounded guard reason, such as +`session_scope_mismatch`, `durable_identity_unreadable`, or +`backup_without_reusable_lease`. Provider-pack read failures distinguish a missing +file, invalid JSON, permission denial, invalid path type, and other I/O errors. +These reasons contain no session identifiers, provider output, or filesystem +paths. They help diagnose recurrence; they do not authorize a retry, quarantine, +replacement, or a weaker identity check. Existing chat recovery recognizes the +same failure category with or without a reason suffix; it still requires the +exact cleanup receipt, checkpoint, and absence of provider work. + **Server events the default integrations add** - `OnUncaughtException` — each uncaught exception on the main thread, at diff --git a/server/src/__tests__/chat-channels.integration.test.ts b/server/src/__tests__/chat-channels.integration.test.ts index 85d968cbc1..d04154ee97 100644 --- a/server/src/__tests__/chat-channels.integration.test.ts +++ b/server/src/__tests__/chat-channels.integration.test.ts @@ -56157,9 +56157,13 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { "leased", "wrong_thread", "source_edited", - ])( - "retries only the original pre-provider Telegram request after exact cleanup: %s", - async (mode) => { + "different_error", + ].flatMap((mode) => [ + "runner_state_identity_mismatch", + "runner_state_identity_mismatch: prior_owner_active", + ].map((errorMessage) => ({ mode, errorMessage }))))( + "retries only the original pre-provider Telegram request after exact cleanup: $mode ($errorMessage)", + async ({ mode, errorMessage }) => { const context = await committedChatResponseRecoveryFixture("telegram"); const providerAccount = mode === "null_account" @@ -56286,7 +56290,9 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { agentId: context.fixture.assignedAgentId, status: "failed", errorCode: "adapter_failed", - error: "runner_state_identity_mismatch", + error: mode === "different_error" + ? errorMessage.replace("runner_state_identity_mismatch", "runner_state_identity_mismatch_other") + : errorMessage, finishedAt: new Date(), wakeupRequestId: action.id, runtimeMode: "native", diff --git a/server/src/services/chat-channels.ts b/server/src/services/chat-channels.ts index dd3694dc6a..a4c67d20b3 100644 --- a/server/src/services/chat-channels.ts +++ b/server/src/services/chat-channels.ts @@ -12006,7 +12006,11 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { run.nativeIssueId === issueId && run.status === "failed" && run.errorCode === "adapter_failed" && - run.error === "runner_state_identity_mismatch" && + // A diagnostic reason does not change this failure category. The exact + // checkpoint, cleanup receipt, and no-provider-work proofs below still + // decide whether the original request can be retried. + (run.error === "runner_state_identity_mismatch" || + run.error?.startsWith("runner_state_identity_mismatch: ")) && run.nativePhase === "observed" && coordinator.phase === "observed" && coordinator.attempt === 0 && diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index d533986d90..b6d7316911 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -1,4 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { readFileSync } from "node:fs"; import { access, cp, @@ -56,6 +57,11 @@ import { buildNativeHeartbeatPreparationSpans } from "./native-run-trace.js"; import { NativeRunnerOwnershipUnverifiedError } from "./native-runner-ownership.js"; import type { AdapterRuntimeEvent } from "../../adapters/index.js"; +vi.mock("node:fs", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, readFileSync: vi.fn(actual.readFileSync) }; +}); + const githubAccess = vi.hoisted(() => ({ activate: vi.fn((_binding: { runId: string }) => vi.fn()), stop: vi.fn(async () => undefined), @@ -1171,6 +1177,54 @@ describe("remote provider pack manifest", () => { }); }); +describe("provider pack read diagnostics", () => { + it.each([ + ["EACCES", "permission_denied"], + ["EPERM", "permission_denied"], + ["EIO", "io_error"], + ])("reports %s without exposing the underlying filesystem message", (code, reason) => { + const root = join(tmpdir(), "private-provider-pack"); + const manifestPath = join(root, "provider-pack.json"); + const cause = Object.assign(new Error(`${code}: cannot read ${manifestPath}`), { + code, + path: manifestPath, + }); + vi.mocked(readFileSync).mockImplementationOnce(() => { throw cause; }); + let failure: Error | undefined; + try { readRemoteProviderPackManifest(root); } catch (error) { failure = error as Error; } + expect(readFileSync).toHaveBeenLastCalledWith(manifestPath, "utf8"); + expect(failure?.message).toBe(`runner_remote_provider_artifact_incompatible: provider-pack.json is unreadable (${reason})`); + expect(failure?.message).not.toContain(root); + expect(failure?.message).not.toContain(code); + expect(failure?.cause).toBe(cause); + }); + + it("classifies a JSON null manifest as incompatible instead of a TypeError", async () => { + const root = await mkdtemp(join(tmpdir(), "paperclip-null-pack-")); + try { + await writeFile(join(root, "provider-pack.json"), "null"); + expect(() => readRemoteProviderPackManifest(root)).toThrow( + "runner_remote_provider_artifact_incompatible: provider pack pins or source revision do not match", + ); + } finally { await rm(root, { recursive: true, force: true }); } + }); + + it.each(["missing", "invalid_json", "invalid_path_type"])("reports %s without putting the path in the terminal message", async (reason) => { + const root = await mkdtemp(join(tmpdir(), "paperclip-private-pack-")); + try { + const manifestPath = join(root, "provider-pack.json"); + if (reason === "invalid_json") await writeFile(manifestPath, "{ private-invalid-json"); + if (reason === "invalid_path_type") await mkdir(manifestPath); + let failure: Error | undefined; + try { readRemoteProviderPackManifest(root); } catch (error) { failure = error as Error; } + expect(failure?.message).toBe(`runner_remote_provider_artifact_incompatible: provider-pack.json is unreadable (${reason})`); + expect(failure?.message).not.toContain(root); + expect(failure?.message).not.toContain("private-invalid-json"); + expect(failure?.cause).toBeDefined(); + } finally { await rm(root, { recursive: true, force: true }); } + }); +}); + describe("native harness persistence profiles", () => { const profile = (provider: Record, driverKind: string) => resolveNativeHarnessPersistenceProfile({ @@ -1477,7 +1531,7 @@ describe("verified native harness backups", () => { }, sourceProviderLeaseId: "sandbox-1", }), - ).toThrow("runner_harness_state_mismatch"); + ).toThrow("runner_harness_state_mismatch: backup_provider_identity_missing"); } finally { await rm(root, { recursive: true, force: true }); } @@ -9119,7 +9173,7 @@ describe("runnerd provider runtime wiring", () => { execution: currentExecution, runnerInstanceId: "runner-after-running-prior-scope", }), - ).rejects.toThrow("runner_state_identity_mismatch"); + ).rejects.toThrow("runner_state_identity_mismatch: prior_owner_active"); await expect(access(scopedRoot)).resolves.toBeUndefined(); await expect(access(join(stateBase, "quarantine"))).rejects.toThrow(); expect(state.createBackend).not.toHaveBeenCalled(); @@ -10691,7 +10745,7 @@ describe("runnerd provider runtime wiring", () => { } // Ambiguous ordinary recovery must still fail closed. Only the runtime's // explicitly admitted replacement may retire these prior-session backups. - await expect(prepareReplacement()).rejects.toThrow("runner_harness_state_mismatch"); + await expect(prepareReplacement()).rejects.toThrow("runner_harness_state_mismatch: backup_without_reusable_lease"); await expect(backend.openReplacementSession!({ identity: { runId: execution.binding.runId }, workingDirectory: execution.workspace.cwd, } as never, {} as never)).resolves.toBe(replacement); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 7b682232f8..1e75231a0b 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -1691,7 +1691,7 @@ function migrateLegacyRunnerdStateRoot(input: { // A legacy path does not encode the full session scope. A mismatch may be // valid live state owned by another agent/workspace, so refusing the claim // is safe but moving that ambiguous directory is not. - throw new Error("runner_state_identity_mismatch"); + throw new Error("runner_state_identity_mismatch: legacy_owner_unverified"); } if ( exactRun && @@ -1704,7 +1704,7 @@ function migrateLegacyRunnerdStateRoot(input: { ) ) { quarantineRunnerdStateRoot(input.legacy, "identity_indeterminate"); - throw new Error("runner_state_identity_mismatch"); + throw new Error("runner_state_identity_mismatch: legacy_authority_indeterminate"); } try { renameSync(input.legacy, input.scoped); @@ -1728,7 +1728,7 @@ function migrateLegacyRunnerdStateRoot(input: { durableIdentityMatchesSession(scopedIdentity, input.execution), ); if (!exactScopedRun && !sameVerifiedPriorRun) { - throw new Error("runner_state_identity_mismatch"); + throw new Error("runner_state_identity_mismatch: migration_destination_owner_changed"); } } return input.scoped; @@ -4573,7 +4573,7 @@ async function migrateRunnerdStateRootForExecution(input: { await recoverQuiescentRunnerdState({ ...input, scoped }); } if (input.restartRecovery?.kind === "reattach_remote_runner" && !existsSync(scoped)) { - throw new Error("runner_state_identity_mismatch"); + throw new Error("runner_state_identity_mismatch: remote_reattach_root_missing"); } if (existsSync(scoped)) { if (!isSafeNativeStateDirectory(scoped)) { @@ -4593,14 +4593,14 @@ async function migrateRunnerdStateRootForExecution(input: { input.restartRecovery?.kind !== "reattach_remote_runner") { quarantineRunnerdStateRoot(scoped, "identity_indeterminate"); } - throw new Error("runner_state_identity_mismatch"); + throw new Error("runner_state_identity_mismatch: durable_identity_unreadable"); } if (!durableIdentityMatchesSession(identity, input.execution)) { if (input.restartRecovery?.kind !== "reattach_existing_runner" && input.restartRecovery?.kind !== "reattach_remote_runner") { quarantineRunnerdStateRoot(scoped, "identity_mismatch"); } - throw new Error("runner_state_identity_mismatch"); + throw new Error("runner_state_identity_mismatch: session_scope_mismatch"); } if (input.restartRecovery?.kind === "bootstrap_incomplete") { if (runnerdStateProvesIncompleteBootstrap(scoped)) { @@ -4610,7 +4610,7 @@ async function migrateRunnerdStateRootForExecution(input: { // Database evidence alone cannot distinguish a never-connected runner // from a partially-persisted provider bootstrap. Only the durable PRP // root can authorize a fresh bootstrap; anything else stays fail-closed. - throw new Error("runner_state_identity_mismatch"); + throw new Error("runner_state_identity_mismatch: bootstrap_not_proven_incomplete"); } if (input.restartRecovery?.kind === "reattach_remote_runner") { await verifyRemoteRunnerReattachment({ @@ -4631,7 +4631,7 @@ async function migrateRunnerdStateRootForExecution(input: { if (input.restartRecovery?.kind !== "reattach_existing_runner") { quarantineRunnerdStateRoot(scoped, "identity_indeterminate"); } - throw new Error("runner_state_identity_mismatch"); + throw new Error("runner_state_identity_mismatch: authority_indeterminate"); } } else { const verification = await verifyPriorRunnerdStateForSessionScope({ @@ -4654,7 +4654,7 @@ async function migrateRunnerdStateRootForExecution(input: { : "identity_indeterminate", ); } - throw new Error("runner_state_identity_mismatch"); + throw new Error(`runner_state_identity_mismatch: prior_owner_${verification}`); } } return; @@ -4672,7 +4672,7 @@ async function migrateRunnerdStateRootForExecution(input: { // Unlike the full-scope target above, this legacy name can legitimately // belong to another scope. Leave it in place for its owner and fail the // attempted migration visibly. - throw new Error("runner_state_identity_mismatch"); + throw new Error("runner_state_identity_mismatch: legacy_session_scope_mismatch"); } let verifiedPriorRunId: string | undefined; if (!durableIdentityMatchesExecution(identity, input.execution)) { @@ -4695,7 +4695,7 @@ async function migrateRunnerdStateRootForExecution(input: { // untouched because the legacy name may still belong to them. quarantineRunnerdStateRoot(legacy, "identity_indeterminate"); } - throw new Error("runner_state_identity_mismatch"); + throw new Error(`runner_state_identity_mismatch: legacy_prior_owner_${verification}`); } verifiedPriorRunId = identity.runId; } @@ -4745,7 +4745,7 @@ async function recoverQuiescentRunnerdState(input: { ) { // Do not roll back to an older valid checkpoint when a newer quarantined // root contains unconfirmed work, even if the newer root is unreadable. - throw new Error("runner_state_identity_mismatch"); + throw new Error("runner_state_identity_mismatch: quarantine_candidates_ambiguous"); } const verified: Array<{ root: string; @@ -4932,7 +4932,7 @@ async function recoverQuiescentRunnerdState(input: { ) { // Known provider history is not permission to start a replacement when // recovery cannot prove a unique, settled owner. - throw new Error("runner_state_identity_mismatch"); + throw new Error("runner_state_identity_mismatch: quarantine_owner_unverified"); } return; } @@ -4951,14 +4951,14 @@ async function recoverQuiescentRunnerdState(input: { "recovery_state_too_large", ).toString("utf8") !== expected ) { - throw new Error("runner_state_identity_mismatch"); + throw new Error("runner_state_identity_mismatch: recovery_evidence_changed"); } } if ( !localProcessDefinitelyGone(candidate.processPid) || !localProcessDefinitelyGone(candidate.processGroupId, true) ) { - throw new Error("runner_state_identity_mismatch"); + throw new Error("runner_state_identity_mismatch: recovery_process_not_gone"); } if (candidate.root !== input.scoped) { if (existsSync(input.scoped)) { @@ -5673,12 +5673,12 @@ export function buildNativeHarnessBackupManifest(input: { completedAt?: string; }): NativeHarnessBackupManifest { if (!providerSessionIdentityIsPresent(input.providerSessionIdentity)) { - throw new Error("runner_harness_state_mismatch"); + throw new Error("runner_harness_state_mismatch: backup_provider_identity_missing"); } const profile = resolveNativeHarnessPersistenceProfile(input.execution); const directories = profile.directories.map((directory) => { const path = resolve(input.backupRoot, directory.name); - if (!existsSync(path)) throw new Error("runner_harness_state_mismatch"); + if (!existsSync(path)) throw new Error("runner_harness_state_mismatch: backup_directory_missing"); return { name: directory.name, ...digestBackupDirectory(path) }; }); return { @@ -9364,14 +9364,22 @@ export function readRemoteProviderPackManifest( readFileSync(resolve(packRoot, "provider-pack.json"), "utf8"), ) as RemoteProviderPackManifest; } catch (error) { + // The terminal run report keeps the outer message, not the cause chain. + // Keep a bounded reason there; raw filesystem errors include private paths. + const code = (error as NodeJS.ErrnoException | null)?.code; + const reason = error instanceof SyntaxError ? "invalid_json" + : code === "ENOENT" ? "missing" + : code === "EACCES" || code === "EPERM" ? "permission_denied" + : code === "EISDIR" || code === "ENOTDIR" ? "invalid_path_type" + : "io_error"; throw new Error( - "runner_remote_provider_artifact_incompatible: provider-pack.json is unreadable", + `runner_remote_provider_artifact_incompatible: provider-pack.json is unreadable (${reason})`, { cause: error }, ); } const payload = manifest?.payload; if ( - manifest.schema !== REMOTE_PROVIDER_PACK_SCHEMA || + manifest?.schema !== REMOTE_PROVIDER_PACK_SCHEMA || !payload || canonicalJson(payload.pins) !== canonicalJson(REMOTE_PROVIDER_PACK_PINS) || canonicalJson(payload.acpxProfileDigests) !== @@ -11543,7 +11551,7 @@ async function createRunnerdBackendWithinSessionClaim( async () => { for (const directory of persistenceProfile.directories) { const targetPath = remotePersistencePath(directory); - if (!targetPath) throw new Error("runner_harness_state_mismatch"); + if (!targetPath) throw new Error("runner_harness_state_mismatch: restore_target_unavailable"); await stageRemoteRunnerDirectory({ target: remoteTarget, runner: remoteCommandRunner, @@ -11571,7 +11579,7 @@ async function createRunnerdBackendWithinSessionClaim( canonicalJson(restored.providerSessionIdentity) !== canonicalJson(backup.manifest.providerSessionIdentity) ) { - throw new Error("runner_harness_state_mismatch"); + throw new Error("runner_harness_state_mismatch: restored_provider_identity_changed"); } // A deliberately non-reusable environment receives a fresh provider lease // for every turn. Stamp that new lease as soon as the verified host backup @@ -11586,7 +11594,7 @@ async function createRunnerdBackendWithinSessionClaim( for (const directory of persistenceProfile.directories) { if (directory.location !== "filesystem") continue; const targetPath = remotePersistencePath(directory); - if (!targetPath) throw new Error("runner_harness_state_mismatch"); + if (!targetPath) throw new Error("runner_harness_state_mismatch: bootstrap_target_unavailable"); const escapedTarget = targetPath.replaceAll("'", "'\\''"); const created = await remoteCommandRunner.execute({ command: "sh", @@ -11766,7 +11774,7 @@ async function createRunnerdBackendWithinSessionClaim( // A continuation that has a durable backup but no recorded reusable // lease was not provider-confirmed lost. Never silently create a new // provider session from that ambiguous state. - throw new Error("runner_harness_state_mismatch"); + throw new Error("runner_harness_state_mismatch: backup_without_reusable_lease"); } } } else if (remoteTarget && remoteCommandRunner) { @@ -11950,7 +11958,7 @@ async function createRunnerdBackendWithinSessionClaim( !verified.runnerState || !verified.providerSessionIdentity ) { - throw new Error("runner_harness_state_mismatch"); + throw new Error("runner_harness_state_mismatch: checkpoint_identity_incomplete"); } const providerSessionIdentity = verified.providerSessionIdentity; @@ -11965,7 +11973,7 @@ async function createRunnerdBackendWithinSessionClaim( for (const directory of persistenceProfile.directories) { const sourcePath = remotePersistencePath(directory); if (!sourcePath) - throw new Error("runner_harness_state_mismatch"); + throw new Error("runner_harness_state_mismatch: checkpoint_source_unavailable"); const targetPath = resolve(pendingRoot, directory.name); await syncRemoteRunnerDirectoryOut({ runner: remoteCommandRunner, @@ -11975,7 +11983,7 @@ async function createRunnerdBackendWithinSessionClaim( excludeEntries: directory.excludeEntries, }); if (!existsSync(targetPath)) { - throw new Error("runner_harness_state_mismatch"); + throw new Error("runner_harness_state_mismatch: checkpoint_directory_missing"); } } const manifest = buildNativeHarnessBackupManifest({