fix(codex): read post-run credential telemetry from remote CODEX_HOME

When executionTargetIsRemote, Codex runs with CODEX_HOME set to the
remote sandbox home. Any token rotation that occurs inside the sandbox
updates only the remote copy, so the prior post-run snapshot read from
the local effectiveCodexHome would always report rotationsDetected:false.

- Add readPostRunCodexCredentialTelemetrySnapshot: for remote targets,
  shell-cats the remote auth.json via runAdapterExecutionTargetShellCommand
  so the post-run fingerprint reflects the actual remote token state
- Add parseCodexCredentialTelemetrySnapshot as an exported function so
  the remote reader can parse the raw JSON without touching the local fs
- Add shellQuote and missingCodexCredentialTelemetrySnapshot helpers
- Tighten CODEX_REFRESH_TOKEN_INVALIDATED_RE (remove broad 401/unauthorized
  to avoid false positives) and add CODEX_OAUTH_INVALID_GRANT_RE plus
  CODEX_CONTEXTUAL_REFRESH_AUTH_INVALIDATED_RE for more precise matching
- Add execute.remote.test coverage: reads post-run telemetry from remote
  CODEX_HOME and verifies rotationsDetected:true + no token material leaked

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Harold KimandPaperclip committed 2026-07-09 09:33:19 +00:00
1 parent 0d3f2eebb7
commit d8bc224137
3 files changed
+159 -14

No files matched your search

@@ -35,12 +35,22 @@ export interface CodexCredentialTelemetrySnapshot {
const ONE_HOUR_MS = 60 * 60 * 1000;
const EIGHT_DAYS_MS = 8 * 24 * ONE_HOUR_MS;
function missingCodexCredentialTelemetrySnapshot(): CodexCredentialTelemetrySnapshot {
return {
refreshTokenFingerprint: null,
lastRefreshAgeBucket: "missing",
};
}
const CODEX_REFRESH_TOKEN_REUSED_RE =
/(?:refresh[_\s-]?token[_\s-]?reused|refresh token (?:has )?already been used|token reuse detected)/i;
const CODEX_REFRESH_TOKEN_EXPIRED_RE =
/(?:refresh[_\s-]?token[_\s-]?expired|refresh token (?:has )?expired|expired refresh token)/i;
const CODEX_REFRESH_TOKEN_INVALIDATED_RE =
/(?:refresh[_\s-]?token[_\s-]?invalidated|refresh token (?:has been )?(?:invalidated|revoked|invalid)|invalid refresh token|invalid[_\s-]?grant|unauthorized|missing bearer|\b401\b)/i;
/(?:refresh[_\s-]?token[_\s-]?(?:invalidated|revoked|invalid)|refresh token (?:has been )?(?:invalidated|revoked|invalid)|invalid refresh token|missing bearer)/i;
const CODEX_OAUTH_INVALID_GRANT_RE = /\binvalid_grant\b/i;
const CODEX_CONTEXTUAL_REFRESH_AUTH_INVALIDATED_RE =
/(?:(?:oauth|refresh|access[_\s-]?token|bearer|credential).{0,80}(?:\b401\b|unauthori[sz]ed|\binvalid[\s-]grant\b)|(?:\b401\b|unauthori[sz]ed|\binvalid[\s-]grant\b).{0,80}(?:oauth|refresh|access[_\s-]?token|bearer|credential))/i;
function isPlainObject(value: unknown): value is Record<string, unknown> {
if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
@@ -85,26 +95,23 @@ export async function readCodexCredentialTelemetrySnapshot(
try {
raw = await fs.readFile(authPath, "utf8");
} catch {
return {
refreshTokenFingerprint: null,
lastRefreshAgeBucket: "missing",
};
return missingCodexCredentialTelemetrySnapshot();
}
return parseCodexCredentialTelemetrySnapshot(raw, now);
}
export function parseCodexCredentialTelemetrySnapshot(
raw: string,
now = new Date(),
): CodexCredentialTelemetrySnapshot {
let parsed: unknown;
try {
parsed = JSON.parse(raw);
} catch {
return {
refreshTokenFingerprint: null,
lastRefreshAgeBucket: "missing",
};
return missingCodexCredentialTelemetrySnapshot();
}
if (!isPlainObject(parsed)) {
return {
refreshTokenFingerprint: null,
lastRefreshAgeBucket: "missing",
};
return missingCodexCredentialTelemetrySnapshot();
}
return {
@@ -135,6 +142,8 @@ export function classifyCodexAuthRefreshFailure(input: {
if (CODEX_REFRESH_TOKEN_REUSED_RE.test(haystack)) return "refresh_token_reused";
if (CODEX_REFRESH_TOKEN_EXPIRED_RE.test(haystack)) return "refresh_token_expired";
if (CODEX_REFRESH_TOKEN_INVALIDATED_RE.test(haystack)) return "refresh_token_invalidated";
if (CODEX_OAUTH_INVALID_GRANT_RE.test(haystack)) return "refresh_token_invalidated";
if (CODEX_CONTEXTUAL_REFRESH_AUTH_INVALIDATED_RE.test(haystack)) return "refresh_token_invalidated";
return null;
}
@@ -9,6 +9,7 @@ const {
resolveCommandForLogs,
prepareWorkspaceForSshExecution,
restoreWorkspaceFromSshExecution,
runSshCommand,
syncDirectoryToSsh,
startAdapterExecutionTargetPaperclipBridge,
} = vi.hoisted(() => ({
@@ -25,6 +26,7 @@ const {
resolveCommandForLogs: vi.fn(async () => "/usr/bin/codex"),
prepareWorkspaceForSshExecution: vi.fn(async () => ({ gitBacked: false })),
restoreWorkspaceFromSshExecution: vi.fn(async () => undefined),
runSshCommand: vi.fn(async (_spec: unknown, _command: string) => ({ stdout: "", stderr: "" })),
syncDirectoryToSsh: vi.fn(async () => undefined),
startAdapterExecutionTargetPaperclipBridge: vi.fn(async () => ({
env: {
@@ -66,6 +68,7 @@ vi.mock("@paperclipai/adapter-utils/execution-target", async () => {
);
return {
...actual,
runAdapterExecutionTargetShellCommand: runSshCommand,
startAdapterExecutionTargetPaperclipBridge,
};
});
@@ -419,4 +422,86 @@ describe("codex remote execution", () => {
expect(call?.[3].env.CODEX_HOME).toBe(`${managedRemoteWorkspace}/.paperclip-runtime/codex/home`);
expect(call?.[3].remoteExecution?.remoteCwd).toBe(managedRemoteWorkspace);
});
it("reads post-run credential telemetry from the remote CODEX_HOME", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-codex-remote-telemetry-"));
cleanupDirs.push(rootDir);
const workspaceDir = path.join(rootDir, "workspace");
const codexHomeDir = path.join(rootDir, "codex-home");
const managedRemoteWorkspace = "/remote/workspace/.paperclip-runtime/runs/run-telemetry/workspace";
const remoteAuthPath = `${managedRemoteWorkspace}/.paperclip-runtime/codex/home/auth.json`;
await mkdir(workspaceDir, { recursive: true });
await mkdir(codexHomeDir, { recursive: true });
await writeFile(
path.join(codexHomeDir, "auth.json"),
JSON.stringify({
tokens: { refresh_token: "seed-refresh-token-secret" },
last_refresh: "2999-01-01T00:00:00.000Z",
}),
"utf8",
);
runSshCommand.mockImplementationOnce(async (_spec: unknown, command: string) => {
expect(command).toContain(remoteAuthPath);
return {
stdout: JSON.stringify({
tokens: { refresh_token: "remote-rotated-refresh-token-secret" },
last_refresh: "2999-01-01T00:05:00.000Z",
}),
stderr: "",
};
});
const result = await execute({
runId: "run-telemetry",
agent: {
id: "agent-1",
companyId: "company-1",
name: "CodexCoder",
adapterType: "codex_local",
adapterConfig: {},
},
runtime: {
sessionId: null,
sessionParams: null,
sessionDisplayId: null,
taskKey: null,
},
config: {
command: "codex",
env: {
CODEX_HOME: codexHomeDir,
},
},
context: {
paperclipWorkspace: {
cwd: workspaceDir,
source: "project_primary",
},
},
executionTransport: {
remoteExecution: {
host: "127.0.0.1",
port: 2222,
username: "fixture",
remoteWorkspacePath: "/remote/workspace",
remoteCwd: "/remote/workspace",
privateKey: "PRIVATE KEY",
knownHosts: "[127.0.0.1]:2222 ssh-ed25519 AAAA",
strictHostKeyChecking: true,
},
},
onLog: async () => {},
});
const telemetry = (result.resultJson as Record<string, unknown>).codexCredentialTelemetry;
expect(telemetry).toEqual({
seedSource: "snapshot_file",
lastRefreshAgeBucket: "lt_1h",
rotationsDetected: true,
});
const serialized = JSON.stringify(telemetry);
expect(serialized).not.toContain("seed-refresh-token-secret");
expect(serialized).not.toContain("remote-rotated-refresh-token-secret");
expect(runSshCommand).toHaveBeenCalledTimes(1);
});
});
@@ -17,7 +17,9 @@ import {
resolveAdapterExecutionTargetTimeoutSec,
resolveAdapterExecutionTargetCommandForLogs,
runAdapterExecutionTargetProcess,
runAdapterExecutionTargetShellCommand,
startAdapterExecutionTargetPaperclipBridge,
type AdapterExecutionTarget,
} from "@paperclipai/adapter-utils/execution-target";
import {
asString,
@@ -58,9 +60,11 @@ import {
CODEX_CREDENTIAL_TELEMETRY_RESULT_KEY,
buildCodexCredentialTelemetryDimensions,
classifyCodexAuthRefreshFailure,
parseCodexCredentialTelemetrySnapshot,
readCodexCredentialTelemetrySnapshot,
type CodexAuthRefreshFailureClass,
type CodexCredentialSeedSource,
type CodexCredentialTelemetrySnapshot,
} from "./credential-telemetry.js";
import { prepareCodexRuntimeConfig } from "./runtime-config.js";
import { resolveCodexDesiredSkillNames } from "./skills.js";
@@ -83,6 +87,47 @@ const executeCodexAcp = createCodexAcpExecutor();
const CODEX_ROLLOUT_NOISE_RE =
/^\d{4}-\d{2}-\d{2}T[^\s]+\s+ERROR\s+codex_core::rollout::list:\s+state db missing rollout path for thread\s+[a-z0-9-]+$/i;
function shellQuote(value: string): string {
return `'${value.replace(/'/g, "'\\''")}'`;
}
function missingCodexCredentialTelemetrySnapshot(): CodexCredentialTelemetrySnapshot {
return { refreshTokenFingerprint: null, lastRefreshAgeBucket: "missing" };
}
async function readPostRunCodexCredentialTelemetrySnapshot(input: {
runId: string;
target: AdapterExecutionTarget | null | undefined;
localCodexHome: string;
remoteCodexHome: string | null;
cwd: string;
}): Promise<CodexCredentialTelemetrySnapshot> {
if (!input.remoteCodexHome || input.target?.kind !== "remote") {
return await readCodexCredentialTelemetrySnapshot(input.localCodexHome);
}
const authPath = path.posix.join(input.remoteCodexHome, "auth.json");
try {
const result = await runAdapterExecutionTargetShellCommand(
input.runId,
input.target,
`if [ -f ${shellQuote(authPath)} ]; then cat ${shellQuote(authPath)}; fi`,
{
cwd: input.cwd,
env: {},
timeoutSec: 15,
graceSec: 5,
},
);
if (result.timedOut || result.exitCode !== 0) {
return missingCodexCredentialTelemetrySnapshot();
}
return parseCodexCredentialTelemetrySnapshot(result.stdout);
} catch {
return missingCodexCredentialTelemetrySnapshot();
}
}
function stripCodexRolloutNoise(text: string): string {
const parts = text.split(/\r?\n/);
const kept: string[] = [];
@@ -986,7 +1031,13 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
[CODEX_CREDENTIAL_TELEMETRY_RESULT_KEY]: buildCodexCredentialTelemetryDimensions({
seedSource: codexCredentialSeedSource,
seedSnapshot: codexCredentialSeedSnapshot,
postRunSnapshot: await readCodexCredentialTelemetrySnapshot(effectiveCodexHome),
postRunSnapshot: await readPostRunCodexCredentialTelemetrySnapshot({
runId,
target: runtimeExecutionTarget,
localCodexHome: effectiveCodexHome,
remoteCodexHome,
cwd: effectiveExecutionCwd,
}),
failureClass,
}),
});