diff --git a/packages/adapters/codex-local/src/server/codex-home.test.ts b/packages/adapters/codex-local/src/server/codex-home.test.ts index 840db456916..a6e6aa86b4c 100644 --- a/packages/adapters/codex-local/src/server/codex-home.test.ts +++ b/packages/adapters/codex-local/src/server/codex-home.test.ts @@ -55,6 +55,54 @@ describe("codex managed home", () => { } }); + it("still throws on EEXIST when a raced-in auth symlink points elsewhere", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-codex-home-")); + const sharedCodexHome = path.join(root, "shared-codex-home"); + const paperclipHome = path.join(root, "paperclip-home"); + const managedCodexHome = path.join( + paperclipHome, + "instances", + "default", + "companies", + "company-1", + "codex-home", + ); + const sharedAuth = path.join(sharedCodexHome, "auth.json"); + const wrongAuth = path.join(sharedCodexHome, "other-auth.json"); + const managedAuth = path.join(managedCodexHome, "auth.json"); + + await fs.mkdir(sharedCodexHome, { recursive: true }); + await fs.writeFile(sharedAuth, '{"token":"shared"}\n', "utf8"); + await fs.writeFile(wrongAuth, '{"token":"other"}\n', "utf8"); + + const originalSymlink = fs.symlink.bind(fs); + vi.spyOn(fs, "symlink").mockImplementationOnce(async (_source, target, type) => { + await originalSymlink(wrongAuth, target, type); + const error = new Error("file already exists") as NodeJS.ErrnoException; + error.code = "EEXIST"; + throw error; + }); + + try { + await expect( + prepareManagedCodexHome( + { + CODEX_HOME: sharedCodexHome, + PAPERCLIP_HOME: paperclipHome, + PAPERCLIP_INSTANCE_ID: "default", + }, + async () => {}, + "company-1", + ), + ).rejects.toMatchObject({ code: "EEXIST" }); + + expect((await fs.lstat(managedAuth)).isSymbolicLink()).toBe(true); + expect(await fs.readlink(managedAuth)).toBe(wrongAuth); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + // Regression for #5028: older Paperclip versions copied auth.json into the // managed home instead of symlinking. After upgrading to the symlink-based // logic, the stale regular file at the target stayed in place and every diff --git a/packages/adapters/gemini-local/src/server/execute.ts b/packages/adapters/gemini-local/src/server/execute.ts index df20937e3ca..12ec7ad0337 100644 --- a/packages/adapters/gemini-local/src/server/execute.ts +++ b/packages/adapters/gemini-local/src/server/execute.ts @@ -51,8 +51,9 @@ import { DEFAULT_GEMINI_LOCAL_MODEL, SANDBOX_INSTALL_COMMAND } from "../index.js import { describeGeminiFailure, detectGeminiAuthRequired, + isGeminiTransientNetworkError, isGeminiTurnLimitResult, - isGeminiUnknownSessionError, + isGeminiSessionUnrecoverableError, parseGeminiJsonl, } from "./parse.js"; import { firstNonEmptyLine } from "./utils.js"; @@ -569,6 +570,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise { it("collects assistant text from message events with string content", () => { @@ -131,3 +135,69 @@ describe("parseGeminiJsonl", () => { expect(result.errorMessage).toBe("boom"); }); }); + +describe("isGeminiSessionUnrecoverableError", () => { + it("matches 'unknown session'", () => { + expect(isGeminiSessionUnrecoverableError("", "Error: unknown session 'abc-123'")).toBe(true); + }); + + it("matches 'session ... not found'", () => { + expect(isGeminiSessionUnrecoverableError("", "Resumed session abc-123 not found on disk")).toBe(true); + }); + + it("matches 'exceeds the maximum number of tokens' (compression overflow)", () => { + const stderr = + '_ApiError: {"error":{"code":400,"message":"The input token count exceeds the maximum number of tokens allowed 1048576","status":"INVALID_ARGUMENT"}} at ChatCompressionService.compress'; + expect(isGeminiSessionUnrecoverableError("", stderr)).toBe(true); + }); + + it("matches 'input token count exceeds'", () => { + expect( + isGeminiSessionUnrecoverableError("", "input token count exceeds maximum"), + ).toBe(true); + }); + + it("does not match unrelated stderr", () => { + expect(isGeminiSessionUnrecoverableError("", "Some other error")).toBe(false); + }); + + it("does not match transient network errors (those go to isGeminiTransientNetworkError)", () => { + expect( + isGeminiSessionUnrecoverableError( + "", + "_GaxiosError: getaddrinfo ENOTFOUND oauth2.googleapis.com", + ), + ).toBe(false); + }); +}); + +describe("isGeminiTransientNetworkError", () => { + it("matches DNS failure on oauth2.googleapis.com", () => { + const stderr = + "_GaxiosError: request to https://oauth2.googleapis.com/token failed, reason: getaddrinfo ENOTFOUND oauth2.googleapis.com"; + expect(isGeminiTransientNetworkError("", stderr)).toBe(true); + }); + + it("matches EAI_AGAIN", () => { + expect( + isGeminiTransientNetworkError("", "Error: getaddrinfo EAI_AGAIN sts.googleapis.com"), + ).toBe(true); + }); + + it("matches _UserRefreshClient ENOTFOUND", () => { + const stderr = + "at _UserRefreshClient.refreshTokenNoCache (.../google-auth-library/...)\n" + + " caused by: ENOTFOUND oauth2.googleapis.com"; + expect(isGeminiTransientNetworkError("", stderr)).toBe(true); + }); + + it("does not match unrelated stderr", () => { + expect(isGeminiTransientNetworkError("", "Some other error")).toBe(false); + }); + + it("does not match unknown-session errors (those go to isGeminiSessionUnrecoverableError)", () => { + expect( + isGeminiTransientNetworkError("", "Error: unknown session 'abc-123'"), + ).toBe(false); + }); +}); diff --git a/packages/adapters/gemini-local/src/server/parse.ts b/packages/adapters/gemini-local/src/server/parse.ts index d309c8c4a43..93686b619f8 100644 --- a/packages/adapters/gemini-local/src/server/parse.ts +++ b/packages/adapters/gemini-local/src/server/parse.ts @@ -199,14 +199,26 @@ export function parseGeminiJsonl(stdout: string) { }; } -export function isGeminiUnknownSessionError(stdout: string, stderr: string): boolean { +export function isGeminiSessionUnrecoverableError(stdout: string, stderr: string): boolean { const haystack = `${stdout}\n${stderr}` .split(/\r?\n/) .map((line) => line.trim()) .filter(Boolean) .join("\n"); - return /unknown\s+session|session\s+.*\s+not\s+found|resume\s+.*\s+not\s+found|checkpoint\s+.*\s+not\s+found|cannot\s+resume|failed\s+to\s+resume/i.test( + return /unknown\s+session|session\s+.*\s+not\s+found|resume\s+.*\s+not\s+found|checkpoint\s+.*\s+not\s+found|cannot\s+resume|failed\s+to\s+resume|exceeds\s+the\s+maximum\s+number\s+of\s+tokens|input\s+token\s+count\s+exceeds/i.test( + haystack, + ); +} + +export function isGeminiTransientNetworkError(stdout: string, stderr: string): boolean { + const haystack = `${stdout}\n${stderr}` + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean) + .join("\n"); + + return /ENOTFOUND\s+oauth2\.googleapis\.com|ENOTFOUND\s+sts\.googleapis\.com|EAI_AGAIN|_GaxiosError.*ENOTFOUND|_UserRefreshClient.*ENOTFOUND/i.test( haystack, ); } diff --git a/packages/adapters/gemini-local/vitest.config.ts b/packages/adapters/gemini-local/vitest.config.ts new file mode 100644 index 00000000000..f624398e8de --- /dev/null +++ b/packages/adapters/gemini-local/vitest.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + environment: "node", + }, +}); diff --git a/server/src/__tests__/adapter-session-codecs.test.ts b/server/src/__tests__/adapter-session-codecs.test.ts index 10905e5193f..fd5fa60e7ae 100644 --- a/server/src/__tests__/adapter-session-codecs.test.ts +++ b/server/src/__tests__/adapter-session-codecs.test.ts @@ -7,7 +7,7 @@ import { } from "@paperclipai/adapter-cursor-local/server"; import { sessionCodec as geminiSessionCodec, - isGeminiUnknownSessionError, + isGeminiSessionUnrecoverableError, } from "@paperclipai/adapter-gemini-local/server"; import { sessionCodec as opencodeSessionCodec, @@ -220,19 +220,19 @@ describe("cursor resume recovery detection", () => { describe("gemini resume recovery detection", () => { it("detects unknown session errors from gemini output", () => { expect( - isGeminiUnknownSessionError( + isGeminiSessionUnrecoverableError( "", "unknown session id abc", ), ).toBe(true); expect( - isGeminiUnknownSessionError( + isGeminiSessionUnrecoverableError( "", "checkpoint latest not found", ), ).toBe(true); expect( - isGeminiUnknownSessionError( + isGeminiSessionUnrecoverableError( "{\"type\":\"result\",\"subtype\":\"success\"}", "", ), diff --git a/server/src/__tests__/gemini-local-adapter.test.ts b/server/src/__tests__/gemini-local-adapter.test.ts index 96e2e8ff1bc..bc2d4643d8b 100644 --- a/server/src/__tests__/gemini-local-adapter.test.ts +++ b/server/src/__tests__/gemini-local-adapter.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it, vi } from "vitest"; import { isGeminiTurnLimitResult, - isGeminiUnknownSessionError, + isGeminiSessionUnrecoverableError, parseGeminiJsonl, } from "@paperclipai/adapter-gemini-local/server"; import { parseGeminiStdoutLine } from "@paperclipai/adapter-gemini-local/ui"; @@ -76,8 +76,8 @@ describe("gemini_local parser", () => { describe("gemini_local stale session detection", () => { it("treats missing session messages as an unknown session error", () => { - expect(isGeminiUnknownSessionError("", "unknown session id abc")).toBe(true); - expect(isGeminiUnknownSessionError("", "checkpoint latest not found")).toBe(true); + expect(isGeminiSessionUnrecoverableError("", "unknown session id abc")).toBe(true); + expect(isGeminiSessionUnrecoverableError("", "checkpoint latest not found")).toBe(true); }); }); diff --git a/server/src/__tests__/heartbeat-workspace-session.test.ts b/server/src/__tests__/heartbeat-workspace-session.test.ts index 309212833ea..874edb83352 100644 --- a/server/src/__tests__/heartbeat-workspace-session.test.ts +++ b/server/src/__tests__/heartbeat-workspace-session.test.ts @@ -21,6 +21,9 @@ import { shouldDeferFollowupWakeForSameIssue, stripHostWorkspaceProvisionForLowTrustSandbox, stripWorkspaceRuntimeFromExecutionRunConfig, + shouldResetTaskSessionForModelChange, + stripConfiguredModelFromSessionParams, + normalizeSessionParams, shouldResetTaskSessionForWake, type ResolvedWorkspaceForRun, } from "../services/heartbeat.ts"; @@ -926,6 +929,113 @@ describe("shouldDeferFollowupWakeForSameIssue", () => { }); }); +describe("shouldResetTaskSessionForModelChange", () => { + it("resets when configured model differs from persisted session model", () => { + expect( + shouldResetTaskSessionForModelChange({ + configuredModel: "gpt-5.4-mini", + taskSessionParams: { + sessionId: "thread-1", + __paperclipConfiguredModel: "opencode/mimo-v2-pro-free", + }, + }), + ).toBe(true); + }); + + it("does not reset when models match", () => { + expect( + shouldResetTaskSessionForModelChange({ + configuredModel: "gpt-5.4-mini", + taskSessionParams: { + sessionId: "thread-1", + __paperclipConfiguredModel: "gpt-5.4-mini", + }, + }), + ).toBe(false); + }); + + it("does not reset when persisted session model is missing", () => { + expect( + shouldResetTaskSessionForModelChange({ + configuredModel: "gpt-5.4-mini", + taskSessionParams: { + sessionId: "thread-1", + }, + }), + ).toBe(false); + }); + + it("does not reset when configured model is missing", () => { + expect( + shouldResetTaskSessionForModelChange({ + configuredModel: null, + taskSessionParams: { + sessionId: "thread-1", + __paperclipConfiguredModel: "gpt-5.4-mini", + }, + }), + ).toBe(false); + }); + + it("does not reset when task session params are missing", () => { + expect( + shouldResetTaskSessionForModelChange({ + configuredModel: "gpt-5.4-mini", + taskSessionParams: null, + }), + ).toBe(false); + }); +}); + +describe("stripConfiguredModelFromSessionParams", () => { + it("removes the internal model key from persisted session params", () => { + expect( + stripConfiguredModelFromSessionParams({ + sessionId: "thread-1", + __paperclipConfiguredModel: "gpt-5.4-mini", + }), + ).toEqual({ sessionId: "thread-1" }); + }); + + it("returns null when session params are missing", () => { + expect(stripConfiguredModelFromSessionParams(null)).toBeNull(); + expect(stripConfiguredModelFromSessionParams(undefined)).toBeNull(); + }); + + it("returns a copy without mutating the input", () => { + const input = { sessionId: "thread-1", __paperclipConfiguredModel: "gpt-5.4-mini" }; + const result = stripConfiguredModelFromSessionParams(input); + expect(result).not.toBe(input); + expect(input.__paperclipConfiguredModel).toBe("gpt-5.4-mini"); + }); + + it("returns an empty object when only the internal model key is present (caller must normalize)", () => { + const stripped = stripConfiguredModelFromSessionParams({ + __paperclipConfiguredModel: "gpt-5.4-mini", + }); + expect(stripped).toEqual({}); + // Callers that forward params to adapters must normalize {} back to null so + // the pre-PR null contract is preserved (adapters distinguishing {} from null). + expect(normalizeSessionParams(stripped)).toBeNull(); + }); +}); + +describe("normalizeSessionParams", () => { + it("collapses an empty object to null", () => { + expect(normalizeSessionParams({})).toBeNull(); + }); + + it("returns null for null or undefined inputs", () => { + expect(normalizeSessionParams(null)).toBeNull(); + expect(normalizeSessionParams(undefined)).toBeNull(); + }); + + it("preserves a non-empty object", () => { + const params = { sessionId: "thread-1" }; + expect(normalizeSessionParams(params)).toBe(params); + }); +}); + describe("deriveTaskKeyWithHeartbeatFallback", () => { it("returns explicit taskKey when present", () => { expect(deriveTaskKeyWithHeartbeatFallback({ taskKey: "issue-123" }, null)).toBe("issue-123"); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index b988dcb32ec..b88d6995a96 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -2131,6 +2131,49 @@ export function shouldDeferFollowupWakeForSameIssue(input: { return false; } +const SESSION_CONFIGURED_MODEL_KEY = "__paperclipConfiguredModel"; + +function readConfiguredModelFromAdapterConfig( + adapterConfig: Record | null | undefined, +) { + return readNonEmptyString(adapterConfig?.model); +} + +function attachConfiguredModelToSessionParams( + sessionParams: Record | null | undefined, + configuredModel: string | null, +) { + if (!configuredModel) return sessionParams ?? null; + const next = { ...(sessionParams ?? {}) }; + next[SESSION_CONFIGURED_MODEL_KEY] = configuredModel; + return next; +} + +function readConfiguredModelFromSessionParams( + sessionParams: Record | null | undefined, +) { + return readNonEmptyString(sessionParams?.[SESSION_CONFIGURED_MODEL_KEY]); +} + +export function shouldResetTaskSessionForModelChange(input: { + configuredModel: string | null; + taskSessionParams: Record | null | undefined; +}) { + const { configuredModel, taskSessionParams } = input; + if (!configuredModel || !taskSessionParams) return false; + const sessionModel = readConfiguredModelFromSessionParams(taskSessionParams); + return !!sessionModel && sessionModel !== configuredModel; +} + +export function stripConfiguredModelFromSessionParams( + sessionParams: Record | null | undefined, +) { + if (!sessionParams) return null; + const next = { ...sessionParams }; + delete next[SESSION_CONFIGURED_MODEL_KEY]; + return next; +} + function shouldAutoCheckoutIssueForWake(input: { contextSnapshot: Record | null | undefined; issueStatus: string | null; @@ -2846,7 +2889,7 @@ function getAdapterSessionCodec(adapterType: string) { return adapter.sessionCodec ?? defaultSessionCodec; } -function normalizeSessionParams(params: Record | null | undefined) { +export function normalizeSessionParams(params: Record | null | undefined) { if (!params) return null; return Object.keys(params).length > 0 ? params : null; } @@ -7833,11 +7876,27 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) } : null, }); + const config = parseObject(agent.adapterConfig); + const configuredModel = readConfiguredModelFromAdapterConfig(config); const taskSession = taskKey ? await getTaskSession(agent.companyId, agent.id, agent.adapterType, taskKey) : null; - const resetTaskSession = shouldResetTaskSessionForWake(context); - const sessionResetReason = describeSessionResetReason(context); + const taskSessionDecodedParams = normalizeSessionParams( + sessionCodec.deserialize(taskSession?.sessionParamsJson ?? null), + ); + const modelChangedSinceTaskSession = shouldResetTaskSessionForModelChange({ + configuredModel, + taskSessionParams: taskSessionDecodedParams, + }); + const resetTaskSession = shouldResetTaskSessionForWake(context) || modelChangedSinceTaskSession; + const wakeSessionResetReason = describeSessionResetReason(context); + const taskSessionConfiguredModel = readConfiguredModelFromSessionParams(taskSessionDecodedParams); + const modelSessionResetReason = modelChangedSinceTaskSession && taskSessionConfiguredModel + ? `configured model changed from "${taskSessionConfiguredModel}" to "${configuredModel}"` + : null; + const sessionResetReason = [modelSessionResetReason, wakeSessionResetReason] + .filter((value): value is string => Boolean(value)) + .join("; ") || null; const taskSessionForRun = resetTaskSession ? null : taskSession; const explicitResumeSessionParams = normalizeResumeParamsForAdapter( agent.adapterType, @@ -7855,9 +7914,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) : null) ?? normalizeResumeParamsForAdapter( agent.adapterType, - sessionCodec.deserialize(taskSessionForRun?.sessionParamsJson ?? null), + stripConfiguredModelFromSessionParams( + sessionCodec.deserialize(taskSessionForRun?.sessionParamsJson ?? null), + ), ); - const config = parseObject(agent.adapterConfig); const resolvedExecutionWorkspaceMode = resolveExecutionWorkspaceMode({ projectPolicy: projectExecutionWorkspacePolicy, issueSettings: issueExecutionWorkspaceSettings, @@ -8490,7 +8550,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) : runtimeSessionDisplayId; let runtimeSessionIdForAdapter = readNonEmptyString(runtimeSessionParams?.sessionId) ?? runtimeSessionFallback; - let runtimeSessionParamsForAdapter = runtimeSessionParams; + let runtimeSessionParamsForAdapter = normalizeSessionParams( + stripConfiguredModelFromSessionParams(runtimeSessionParams), + ); const sessionCompaction = await evaluateSessionCompaction({ agent, @@ -9159,7 +9221,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) agentId: agent.id, adapterType: agent.adapterType, taskKey, - sessionParamsJson: nextSessionState.params, + sessionParamsJson: attachConfiguredModelToSessionParams(nextSessionState.params, configuredModel), sessionDisplayId: nextSessionState.displayId, lastRunId: finalizedRun.id, lastError: outcome === "succeeded" ? null : (adapterResult.errorMessage ?? "run_failed"), @@ -9242,7 +9304,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) agentId: agent.id, adapterType: agent.adapterType, taskKey, - sessionParamsJson: previousSessionParams, + sessionParamsJson: attachConfiguredModelToSessionParams(previousSessionParams, configuredModel), sessionDisplayId: previousSessionDisplayId, lastRunId: failedRun.id, lastError: message,