fix(codex-local): backfill auth for already-isolated codex_local homes

Phase 1 seeds Paperclip-managed CODEX_HOME at execute time, which fixes
new strandings. Agents already carrying a persisted (but unseeded)
per-agent CODEX_HOME from the #8272 isolation guard (e.g. CTO) only
recover the next time they run. Add a proactive startup backfill so a
stranded agent's auth.json is seeded without waiting to run.

- Add idempotent reconcileManagedCodexHome to the codex-local adapter,
  reusing the Phase 1 managed-home detection (isManagedCodexHomePath) and
  seeding (seedManagedCodexHome). Genuine external/user CODEX_HOME
  overrides are left untouched; a home with valid auth is a no-op.
- Add a server startup reconciliation that enumerates codex_local agents
  and seeds any persisted managed home. Only literal (plain) env values
  are used, so a secret-bound OPENAI_API_KEY is never written verbatim
  into auth.json (falls back to the subscription symlink).
- Wire the reconciliation into server boot alongside the other
  reconcile*OnStartup steps. Safe to re-run on every boot.

Adds unit tests for the reconcile primitive (seed/no-op/external/api-key)
and the server service (status accounting, secret-binding safety).

Co-Authored-By: Paperclip <noreply@paperclip.ing>

packages/adapters/codex-local/src/server/codex-home.test.ts

@@ -7,6 +7,7 @@ import {
   ensureSymlink,
   isManagedCodexHomePath,
   prepareManagedCodexHome,
+  reconcileManagedCodexHome,
   seedManagedCodexHome,
 } from "./codex-home.js";
 
@@ -314,3 +315,110 @@ describe("seedManagedCodexHome", () => {
     }
   });
 });
+
+// PAPA-910 Phase 2: backfill seeding for already-isolated agent homes.
+describe("reconcileManagedCodexHome", () => {
+  async function makeFixture() {
+    const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-codex-reconcile-"));
+    const sharedCodexHome = path.join(root, "shared-codex-home");
+    const paperclipHome = path.join(root, "paperclip-home");
+    const agentHome = path.join(
+      paperclipHome,
+      "instances",
+      "default",
+      "companies",
+      "company-1",
+      "agents",
+      "agent-7",
+      "codex-home",
+    );
+    const sharedAuth = path.join(sharedCodexHome, "auth.json");
+    const agentAuth = path.join(agentHome, "auth.json");
+    await fs.mkdir(sharedCodexHome, { recursive: true });
+    await fs.writeFile(sharedAuth, '{"token":"shared"}', "utf8");
+    const env = {
+      CODEX_HOME: sharedCodexHome,
+      PAPERCLIP_HOME: paperclipHome,
+      PAPERCLIP_INSTANCE_ID: "default",
+    } satisfies NodeJS.ProcessEnv;
+    return { root, sharedCodexHome, sharedAuth, agentHome, agentAuth, env };
+  }
+
+  it("seeds a previously-stranded managed home and is a no-op on re-run", async () => {
+    const fx = await makeFixture();
+    try {
+      // The isolation guard created the per-agent home with no auth.json.
+      expect(await codexHomeHasUsableAuth(fx.agentHome)).toBe(false);
+
+      const first = await reconcileManagedCodexHome({
+        companyId: "company-1",
+        configuredCodexHome: fx.agentHome,
+        env: fx.env,
+      });
+      expect(first.status).toBe("seeded");
+      expect(first.home).toBe(fx.agentHome);
+      expect((await fs.lstat(fx.agentAuth)).isSymbolicLink()).toBe(true);
+      expect(await fs.realpath(fx.agentAuth)).toBe(await fs.realpath(fx.sharedAuth));
+
+      const second = await reconcileManagedCodexHome({
+        companyId: "company-1",
+        configuredCodexHome: fx.agentHome,
+        env: fx.env,
+      });
+      expect(second.status).toBe("already_seeded");
+      expect((await fs.lstat(fx.agentAuth)).isSymbolicLink()).toBe(true);
+      expect(await fs.realpath(fx.agentAuth)).toBe(await fs.realpath(fx.sharedAuth));
+    } finally {
+      await fs.rm(fx.root, { recursive: true, force: true });
+    }
+  });
+
+  it("leaves a genuine external override untouched", async () => {
+    const fx = await makeFixture();
+    try {
+      const external = path.join(fx.root, "user-codex");
+      await fs.mkdir(external, { recursive: true });
+
+      const result = await reconcileManagedCodexHome({
+        companyId: "company-1",
+        configuredCodexHome: external,
+        env: fx.env,
+      });
+      expect(result.status).toBe("external_override");
+      expect(await codexHomeHasUsableAuth(external)).toBe(false);
+    } finally {
+      await fs.rm(fx.root, { recursive: true, force: true });
+    }
+  });
+
+  it("reports no_managed_home when no CODEX_HOME is configured", async () => {
+    const fx = await makeFixture();
+    try {
+      const result = await reconcileManagedCodexHome({
+        companyId: "company-1",
+        configuredCodexHome: null,
+        env: fx.env,
+      });
+      expect(result).toEqual({ status: "no_managed_home", home: null });
+    } finally {
+      await fs.rm(fx.root, { recursive: true, force: true });
+    }
+  });
+
+  it("writes an API-key auth.json into a managed home when an apiKey is supplied", async () => {
+    const fx = await makeFixture();
+    try {
+      const result = await reconcileManagedCodexHome({
+        companyId: "company-1",
+        configuredCodexHome: fx.agentHome,
+        apiKey: "sk-reconcile-1",
+        env: fx.env,
+      });
+      expect(result.status).toBe("seeded");
+      const written = JSON.parse(await fs.readFile(fx.agentAuth, "utf8"));
+      expect(written).toEqual({ OPENAI_API_KEY: "sk-reconcile-1" });
+    } finally {
+      await fs.rm(fx.root, { recursive: true, force: true });
+    }
+  });
+});

packages/adapters/codex-local/src/server/codex-home.ts

@@ -219,3 +219,58 @@ export async function prepareManagedCodexHome(
   await seedManagedCodexHome(targetHome, env, onLog, options);
   return targetHome;
 }
+
+export type ReconcileManagedCodexHomeStatus =
+  | "no_managed_home"
+  | "external_override"
+  | "already_seeded"
+  | "seeded";
+
+export interface ReconcileManagedCodexHomeInput {
+  companyId: string | undefined;
+  configuredCodexHome: string | null | undefined;
+  apiKey?: string | null;
+  env?: NodeJS.ProcessEnv;
+  onLog?: AdapterExecutionContext["onLog"];
+}
+
+export interface ReconcileManagedCodexHomeResult {
+  status: ReconcileManagedCodexHomeStatus;
+  home: string | null;
+}
+
+const noopOnLog: AdapterExecutionContext["onLog"] = async () => {};
+
+/**
+ * Idempotently reconciles a persisted `codex_local` agent home. Phase 1 seeds
+ * managed homes at execute time; this is the backfill for agents that already
+ * carry a persisted (but unseeded) per-agent `CODEX_HOME` and have not run
+ * since the seeding fix landed. Shares the managed-home detection
+ * (`isManagedCodexHomePath`) and seeding (`seedManagedCodexHome`) logic so a
+ * genuine external/user override is never touched. Safe to re-run: when a valid
+ * `auth.json` is already present (and no API-key rewrite is requested) it is a
+ * no-op and reports `already_seeded`.
+ */
+export async function reconcileManagedCodexHome(
+  input: ReconcileManagedCodexHomeInput,
+): Promise<ReconcileManagedCodexHomeResult> {
+  const env = input.env ?? process.env;
+  const configured = nonEmpty(input.configuredCodexHome ?? undefined);
+  if (!configured) return { status: "no_managed_home", home: null };
+
+  const resolved = path.resolve(configured);
+  if (!isManagedCodexHomePath(env, input.companyId, resolved)) {
+    return { status: "external_override", home: resolved };
+  }
+
+  const apiKey = nonEmpty(input.apiKey ?? undefined);
+  const hadUsableAuth = await codexHomeHasUsableAuth(resolved);
+  await seedManagedCodexHome(resolved, env, input.onLog ?? noopOnLog, { apiKey });
+
+  // Seeding always (re)writes when an API key is supplied; otherwise it only
+  // changes disk state when auth was missing. Report accordingly so callers can
+  // distinguish a real backfill from a verified no-op.
+  const status: ReconcileManagedCodexHomeStatus =
+    !apiKey && hadUsableAuth ? "already_seeded" : "seeded";
+  return { status, home: resolved };
+}

packages/adapters/codex-local/src/server/index.ts

@@ -1,4 +1,11 @@
 export { execute, ensureCodexSkillsInjected } from "./execute.js";
+export {
+  reconcileManagedCodexHome,
+  isManagedCodexHomePath,
+  type ReconcileManagedCodexHomeInput,
+  type ReconcileManagedCodexHomeResult,
+  type ReconcileManagedCodexHomeStatus,
+} from "./codex-home.js";
 export { listCodexSkills, syncCodexSkills } from "./skills.js";
 export { testEnvironment } from "./test.js";
 export { parseCodexJsonl, isCodexTransientUpstreamError, isCodexUnknownSessionError } from "./parse.js";

server/src/__tests__/codex-auth-reconciliation.test.ts

@@ -0,0 +1,147 @@
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import type { Db } from "@paperclipai/db";
+import { reconcileCodexLocalManagedHomesOnStartup } from "../services/codex-auth-reconciliation.js";
+
+type AgentRow = {
+  id: string;
+  companyId: string;
+  adapterConfig: Record<string, unknown>;
+};
+
+function makeDb(rows: AgentRow[]): Db {
+  return {
+    select: () => ({
+      from: () => ({
+        where: async () => rows,
+      }),
+    }),
+  } as unknown as Db;
+}
+
+describe("reconcileCodexLocalManagedHomesOnStartup", () => {
+  let root: string;
+  let paperclipHome: string;
+  let sharedCodexHome: string;
+  const savedEnv: Record<string, string | undefined> = {};
+
+  function managedAgentHome(companyId: string, agentId: string): string {
+    return path.join(
+      paperclipHome,
+      "instances",
+      "default",
+      "companies",
+      companyId,
+      "agents",
+      agentId,
+      "codex-home",
+    );
+  }
+
+  beforeEach(async () => {
+    root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-codex-startup-"));
+    paperclipHome = path.join(root, "paperclip-home");
+    sharedCodexHome = path.join(root, "shared-codex-home");
+    await fs.mkdir(sharedCodexHome, { recursive: true });
+    await fs.writeFile(path.join(sharedCodexHome, "auth.json"), '{"token":"shared"}', "utf8");
+
+    for (const key of ["PAPERCLIP_HOME", "PAPERCLIP_INSTANCE_ID", "CODEX_HOME"]) {
+      savedEnv[key] = process.env[key];
+    }
+    process.env.PAPERCLIP_HOME = paperclipHome;
+    process.env.PAPERCLIP_INSTANCE_ID = "default";
+    process.env.CODEX_HOME = sharedCodexHome;
+  });
+
+  afterEach(async () => {
+    for (const [key, value] of Object.entries(savedEnv)) {
+      if (value === undefined) delete process.env[key];
+      else process.env[key] = value;
+    }
+    await fs.rm(root, { recursive: true, force: true });
+  });
+
+  it("seeds a stranded managed home and is a no-op on the next boot", async () => {
+    const agentHome = managedAgentHome("company-1", "agent-7");
+    const rows: AgentRow[] = [
+      { id: "agent-7", companyId: "company-1", adapterConfig: { env: { CODEX_HOME: agentHome, OPENAI_API_KEY: "" } } },
+    ];
+
+    const first = await reconcileCodexLocalManagedHomesOnStartup(makeDb(rows));
+    expect(first).toMatchObject({ scanned: 1, seeded: 1, alreadySeeded: 0, failed: 0 });
+    expect(first.seededAgentIds).toEqual(["agent-7"]);
+    const agentAuth = path.join(agentHome, "auth.json");
+    expect((await fs.lstat(agentAuth)).isSymbolicLink()).toBe(true);
+    expect(await fs.realpath(agentAuth)).toBe(
+      await fs.realpath(path.join(sharedCodexHome, "auth.json")),
+    );
+
+    const second = await reconcileCodexLocalManagedHomesOnStartup(makeDb(rows));
+    expect(second).toMatchObject({ scanned: 1, seeded: 0, alreadySeeded: 1, failed: 0 });
+  });
+
+  it("classifies external overrides and unconfigured homes without seeding", async () => {
+    const external = path.join(root, "user-codex");
+    await fs.mkdir(external, { recursive: true });
+    const rows: AgentRow[] = [
+      { id: "ext", companyId: "company-1", adapterConfig: { env: { CODEX_HOME: external } } },
+      { id: "none", companyId: "company-1", adapterConfig: { env: {} } },
+    ];
+
+    const summary = await reconcileCodexLocalManagedHomesOnStartup(makeDb(rows));
+    expect(summary).toMatchObject({
+      scanned: 2,
+      seeded: 0,
+      externalOverride: 1,
+      noManagedHome: 1,
+      failed: 0,
+    });
+    expect(await fs.access(path.join(external, "auth.json")).then(() => true).catch(() => false)).toBe(
+      false,
+    );
+  });
+
+  it("does not write a secret-bound OPENAI_API_KEY placeholder as the API key", async () => {
+    const agentHome = managedAgentHome("company-2", "agent-9");
+    // A secret binding must never be written verbatim into auth.json; the home
+    // should fall back to the seeded subscription symlink instead.
+    const rows: AgentRow[] = [
+      {
+        id: "agent-9",
+        companyId: "company-2",
+        adapterConfig: {
+          env: {
+            CODEX_HOME: agentHome,
+            OPENAI_API_KEY: { type: "secret", secretId: "sec-123" },
+          },
+        },
+      },
+    ];
+
+    const summary = await reconcileCodexLocalManagedHomesOnStartup(makeDb(rows));
+    expect(summary).toMatchObject({ scanned: 1, seeded: 1, failed: 0 });
+    const agentAuth = path.join(agentHome, "auth.json");
+    expect((await fs.lstat(agentAuth)).isSymbolicLink()).toBe(true);
+    expect(await fs.realpath(agentAuth)).toBe(
+      await fs.realpath(path.join(sharedCodexHome, "auth.json")),
+    );
+  });
+
+  it("writes a plain OPENAI_API_KEY into the managed home", async () => {
+    const agentHome = managedAgentHome("company-3", "agent-5");
+    const rows: AgentRow[] = [
+      {
+        id: "agent-5",
+        companyId: "company-3",
+        adapterConfig: { env: { CODEX_HOME: agentHome, OPENAI_API_KEY: "sk-plain-1" } },
+      },
+    ];
+
+    const summary = await reconcileCodexLocalManagedHomesOnStartup(makeDb(rows));
+    expect(summary).toMatchObject({ scanned: 1, seeded: 1, failed: 0 });
+    const written = JSON.parse(await fs.readFile(path.join(agentHome, "auth.json"), "utf8"));
+    expect(written).toEqual({ OPENAI_API_KEY: "sk-plain-1" });
+  });
+});

server/src/index.ts

@@ -41,6 +41,7 @@ import {
   heartbeatService,
   instanceSettingsService,
   reconcileCloudUpstreamRunsOnStartup,
+  reconcileCodexLocalManagedHomesOnStartup,
   reconcilePersistedRuntimeServicesOnStartup,
   routineService,
 } from "./services/index.js";
@@ -728,7 +729,23 @@ export async function startServer(): Promise<StartedServer> {
     .catch((err) => {
       logger.error({ err }, "startup reconciliation of cloud upstream runs failed");
     });
-  
+
+  // Backfill auth.json into any already-isolated codex_local managed home that
+  // was created by the #8272 isolation guard before the Phase 1 seeding fix.
+  // Idempotent; the Phase 1 execute-time seeding covers new strandings.
+  void reconcileCodexLocalManagedHomesOnStartup(db as any)
+    .then((result) => {
+      if (result.seeded > 0 || result.failed > 0) {
+        logger.warn(
+          { seeded: result.seeded, failed: result.failed, scanned: result.scanned },
+          "reconciled codex_local managed homes (backfilled missing auth)",
+        );
+      }
+    })
+    .catch((err) => {
+      logger.error({ err }, "startup reconciliation of codex_local managed homes failed");
+    });
+
   // Force the instance onto the Kubernetes sandbox provider when configured via
   // env (PAPERCLIP_EXECUTION_MODE=kubernetes). Runs BEFORE the heartbeat resumes
   // queued runs so the policy + managed k8s environments are in place. A bad

server/src/services/codex-auth-reconciliation.ts

@@ -0,0 +1,116 @@
+import type { Db } from "@paperclipai/db";
+import { agents } from "@paperclipai/db";
+import { reconcileManagedCodexHome } from "@paperclipai/adapter-codex-local/server";
+import { eq } from "drizzle-orm";
+import { logger } from "../middleware/logger.js";
+
+export interface CodexAuthReconciliationSummary {
+  scanned: number;
+  seeded: number;
+  alreadySeeded: number;
+  externalOverride: number;
+  noManagedHome: number;
+  failed: number;
+  seededAgentIds: string[];
+}
+
+function asRecord(value: unknown): Record<string, unknown> | null {
+  return typeof value === "object" && value !== null && !Array.isArray(value)
+    ? (value as Record<string, unknown>)
+    : null;
+}
+
+/**
+ * Extracts a literal (non-secret) env value. Adapter env bindings persist either
+ * as a bare string or as a `{ type: "plain", value }` object; secret bindings
+ * are intentionally NOT resolved here (we never write an unresolved secret
+ * placeholder into auth.json). Mirrors the server-side env-binding extraction in
+ * routes/agents.ts.
+ */
+function readPlainEnvValue(value: unknown): string | null {
+  if (typeof value === "string") {
+    const trimmed = value.trim();
+    return trimmed.length > 0 ? trimmed : null;
+  }
+  const record = asRecord(value);
+  if (record?.type !== "plain") return null;
+  return readPlainEnvValue(record.value);
+}
+
+/**
+ * Startup backfill for PAPA-910 Phase 2: seed `auth.json` into any already-
+ * isolated `codex_local` managed home that was created (by the #8272 isolation
+ * guard) before the Phase 1 seeding fix landed. Phase 1 seeds at execute time;
+ * this repairs persisted homes proactively so a stranded agent recovers without
+ * waiting to run and without a manual symlink. Idempotent and safe to re-run on
+ * every boot: a home that already has valid auth is a no-op.
+ */
+export async function reconcileCodexLocalManagedHomesOnStartup(
+  db: Db,
+): Promise<CodexAuthReconciliationSummary> {
+  const summary: CodexAuthReconciliationSummary = {
+    scanned: 0,
+    seeded: 0,
+    alreadySeeded: 0,
+    externalOverride: 0,
+    noManagedHome: 0,
+    failed: 0,
+    seededAgentIds: [],
+  };
+
+  const rows = await db
+    .select({
+      id: agents.id,
+      companyId: agents.companyId,
+      adapterConfig: agents.adapterConfig,
+    })
+    .from(agents)
+    .where(eq(agents.adapterType, "codex_local"));
+
+  for (const row of rows) {
+    summary.scanned += 1;
+    const env = asRecord(asRecord(row.adapterConfig)?.env);
+    const configuredCodexHome = env ? readPlainEnvValue(env.CODEX_HOME) : null;
+    const apiKey = env ? readPlainEnvValue(env.OPENAI_API_KEY) : null;
+
+    try {
+      const result = await reconcileManagedCodexHome({
+        companyId: row.companyId,
+        configuredCodexHome,
+        apiKey,
+      });
+      switch (result.status) {
+        case "seeded":
+          summary.seeded += 1;
+          summary.seededAgentIds.push(row.id);
+          logger.info(
+            { agentId: row.id, companyId: row.companyId, home: result.home },
+            "seeded auth into already-isolated codex_local managed home",
+          );
+          break;
+        case "already_seeded":
+          summary.alreadySeeded += 1;
+          break;
+        case "external_override":
+          summary.externalOverride += 1;
+          break;
+        case "no_managed_home":
+          summary.noManagedHome += 1;
+          break;
+      }
+    } catch (err) {
+      summary.failed += 1;
+      logger.warn(
+        {
+          agentId: row.id,
+          companyId: row.companyId,
+          home: configuredCodexHome,
+          err: err instanceof Error ? err.message : String(err),
+        },
+        "failed to reconcile codex_local managed home on startup",
+      );
+    }
+  }
+
+  return summary;
+}

server/src/services/index.ts

@@ -80,5 +80,9 @@ export { workProductService } from "./work-products.js";
 export { logActivity, type LogActivityInput } from "./activity-log.js";
 export { notifyHireApproved, type NotifyHireApprovedInput } from "./hire-hook.js";
 export { publishLiveEvent, subscribeCompanyLiveEvents } from "./live-events.js";
+export {
+  reconcileCodexLocalManagedHomesOnStartup,
+  type CodexAuthReconciliationSummary,
+} from "./codex-auth-reconciliation.js";
 export { reconcilePersistedRuntimeServicesOnStartup, restartDesiredRuntimeServicesOnStartup } from "./workspace-runtime.js";
 export { createStorageServiceFromConfig, getStorageService } from "../storage/index.js";