Merge current master lifecycle and CI fixes into Claude behavior

Co-Authored-By: Paperclip <noreply@paperclip.ing>

* codex/claude-runner-protocol:
  fix(grok): avoid routine wakes for proactive admission (#16422)
  feat(ci): profile full CODEOWNER PR suites in Honeycomb (#16378)
  test: bound command snapshot forwarding checks (#16421)
  fix(agents): report actionable lifecycle failure steps (#16418)

server/src/__tests__/agent-harness-verification.test.ts

@@ -8,6 +8,22 @@ describe("saved harness verification", () => {
     adapterType: "test", status, testedAt: new Date().toISOString(),
     checks: codes.map(code => ({ code, level: "info", message: code })),
   });
+  it.each([
+    ["opencode_hello_probe_auth_required", "harness_auth_failed"],
+    ["opencode_hello_probe_model_unavailable", "harness_model_unavailable"],
+    ["opencode_command_unresolvable", "harness_command_unavailable"],
+    ["opencode_cwd_invalid", "harness_cwd_invalid"],
+    ["opencode_hello_probe_timed_out", "harness_probe_timed_out"],
+    ["opencode_hello_probe_unexpected_output", "harness_probe_unexpected_output"],
+    ["private provider error", "harness_test_failed"],
+  ])("classifies %s without keeping provider output", (checkCode, code) => {
+    try {
+      assertHarnessTestPassed(result("fail", [checkCode]));
+      expect.fail("Expected verification to fail");
+    } catch (error) {
+      expect(error).toMatchObject({ code, message: "The harness test failed" });
+    }
+  });
   it.each([
     ["fail", []], ["warn", ["claude_hello_probe_auth_required"]],
     ["warn", ["gemini_hello_probe_timed_out"]], ["pass", ["claude_hello_probe_skipped_custom_command"]],

server/src/__tests__/dot-runner.test.ts

@@ -243,6 +243,9 @@ describe("durable Dot Runner integration", () => {
     try {
       const requests = await Promise.all([1, 2, 3].map(() => f.broker.requestTurn(f.principal, "Create hello and assign it to my owner", requestId)));
       expect(requests[1]).toEqual(requests[0]); expect(requests[2]).toEqual(requests[0]);
+      expect(requests[0]!.instruction).toContain(provider === "grok_bot"
+        ? "Queued admission does not start another Grok routine"
+        : "Events also notify queued work");
       const intake = await db.select().from(issues).where(eq(issues.id, requests[0]!.issueId));
       expect(intake).toHaveLength(1);
       expect(intake[0]).toMatchObject({ assigneeAgentId: f.agent.id, createdByAgentId: f.agent.id, responsibleUserId: f.userId });

server/src/__tests__/grok-bot-routine.test.ts

@@ -289,6 +289,28 @@ describe("durable Grok routine qualification", () => {
     expect((await dotRunnerBroker(db).read(f.principal, work.id)).toolUsageInstructions).toContain("paperclip_grok_operation_status");
   });
 
+  it.each(["new offer", "prepared before restart"] as const)("does not echo delayed Bot-initiated work: %s", async scenario => {
+    const f = await fixture(); const { work, run } = await admittedWork(f, "offered");
+    await db.update(heartbeatRuns).set({ contextSnapshot: { grokProactiveRequest: {
+      requestId: randomUUID(), bindingId: f.pairing.bindingId, bindingGeneration: 1,
+    } } }).where(eq(heartbeatRuns.id, run.id));
+    await db.update(dotRunnerAssignments).set({ createdAt: new Date(Date.now() - 60_000) }).where(eq(dotRunnerAssignments.id, work.id));
+    const legacy = scenario === "prepared before restart" ? await f.service.reserve({
+      companyId: f.company.id, agentId: f.agent.id, bindingId: f.pairing.bindingId,
+      assignmentId: work.id, kind: "mailbox_updated", sourceEventId: `proactive-admission:${work.id}`,
+    }) : null;
+    f.delivery.mockClear(); const dispatcher = createGrokRoutineDispatcher(db, () => true, f.delivery);
+    await dispatcher.tick(); await dispatcher.tick();
+    expect(f.delivery).not.toHaveBeenCalled();
+    const wakes = await db.select().from(grokBotDispatches).where(eq(grokBotDispatches.assignmentId, work.id));
+    if (legacy) expect(wakes).toEqual([expect.objectContaining({ id: legacy.id, status: "cancelled" })]);
+    else expect(wakes).toEqual([]);
+    // The original conversation still owns and can accept the polled offer.
+    await expect(dotRunnerBroker(db).read(f.principal, work.id)).resolves.toBeDefined();
+    await (await acceptance(f, work))();
+    expect((await db.select().from(dotRunnerAssignments).where(eq(dotRunnerAssignments.id, work.id)))[0]?.status).toBe("accepted");
+  });
+
   it("cancels a queued wake when its assignment was already accepted", async () => {
     const f = await fixture(); const { work } = await admittedWork(f, "offered");
     const queued = await f.service.reserve({ companyId: f.company.id, agentId: f.agent.id,

server/src/modules/agent-lifecycle/__tests__/diagnostics.test.ts

@@ -5,6 +5,7 @@ import type { LifecycleAgent, LifecycleStore, LifecycleDriver } from "../applica
 import { startAgentLifecycle } from "../index.js";
 import { createAgentLifecycleEffects } from "../../../services/agent-lifecycle.js";
 import { logger } from "../../../middleware/logger.js";
+import { lifecycleFailureMessage } from "../application/failure.js";
 
 const agent: LifecycleAgent = { id: "agent", companyId: "company", lifecycleState: "verifying", lifecycleVersion: 3,
   lifecycleRequiredPluginIds: null, lifecycleError: null,
@@ -66,6 +67,30 @@ describe("lifecycle readiness handoff", () => {
 });
 
 describe("lifecycle failure diagnostics", () => {
+  it.each([
+    ["host", "harness_model_unavailable", "Execution configuration check failed. The selected model is unavailable."],
+    ["plugin", "required_plugin_unavailable", "Execution configuration check: required plugin step failed. A required plugin is unavailable."],
+    ["select_plugins", "unknown private code", "Execution configuration check: finding required plugins failed. Paperclip could not complete this step."],
+    ["transition", "40P01", "Execution configuration check: saving progress failed. Paperclip could not complete this step."],
+  ] as const)("persists safe actionable errors for %s", async (stage, code, expected) => {
+    const { store, driver, worker } = fixture();
+    const error = Object.assign(new Error("secret provider output"), { code });
+    if (stage === "host") driver.runHost.mockRejectedValue(error);
+    if (stage === "plugin") driver.runPlugin.mockRejectedValue(error);
+    if (stage === "select_plugins") driver.requiredPluginIds.mockRejectedValue(error);
+    if (stage === "transition") store.change.mockRejectedValue(error);
+    await worker.process(agent.id);
+    expect(store.defer).toHaveBeenCalledWith(agent, expect.any(String), expect.stringContaining(expected), expect.any(Date));
+    expect(JSON.stringify(store.defer.mock.calls)).not.toContain("secret provider output");
+    expect(JSON.stringify(store.defer.mock.calls)).not.toContain("unknown private code");
+    await worker.stop();
+  });
+
+  it("does not treat prototype properties as allowed failure codes", () => {
+    expect(lifecycleFailureMessage({ stage: "host", phase: "verifying" }, { code: "toString" }))
+      .toContain("Paperclip could not complete this step");
+  });
+
   it.each(["host", "plugin", "claim", "renew", "scan", "defer"] as const)("reports %s failures with the available context", async stage => {
     const { store, driver, report, worker } = fixture();
     const error = new Error("private provider detail");
@@ -100,6 +125,7 @@ describe("lifecycle failure diagnostics", () => {
   it.each([
     [{ code: "required_plugin_unavailable" }, "required_plugin_unavailable"],
     [{ code: "harness_test_failed" }, "harness_test_failed"],
+    [{ code: "harness_auth_failed" }, "harness_auth_failed"],
     [{ cause: { code: "40P01", message: "private query" } }, "40P01"],
     [{ code: "private credential value" }, "unclassified"],
   ])("logs known codes without provider error contents", (details, failureCode) => {

server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts

@@ -140,7 +140,7 @@ const support = await getEmbeddedPostgresTestSupport();
       const work = worker(createLifecycleDriver(db, {} as never).runHost);
       await work.process(agent.id);
       expect(probe).toHaveBeenCalled();
-      expect(await current(agent.id)).toMatchObject({ lifecycleState: "verifying", lifecycleError: "The lifecycle step failed. Retry the operation." });
+      expect(await current(agent.id)).toMatchObject({ lifecycleState: "verifying", lifecycleError: expect.stringContaining("Execution configuration check failed. Authentication failed.") });
     } finally { probe.mockRestore(); }
   });
 

server/src/modules/agent-lifecycle/application/failure.ts

@@ -0,0 +1,40 @@
+import type { LifecycleFailure } from "./ports.js";
+
+// Only fixed text may leave this boundary. Provider errors can contain secrets.
+const reasons = {
+  harness_test_failed: "The execution test did not pass. Open Configuration and run the agent test to inspect the failed checks.",
+  harness_auth_failed: "Authentication failed. Check the agent's AI connection or sign in to its execution tool, then retry.",
+  harness_model_unavailable: "The selected model is unavailable. Choose a model supported by the agent's AI connection in Configuration, then retry.",
+  harness_command_unavailable: "The execution command was not found. Check the command and its installation in the agent's environment, then retry.",
+  harness_cwd_invalid: "The working directory is unavailable. Check the working directory in Configuration, then retry.",
+  harness_probe_timed_out: "The execution test timed out. Paperclip will retry automatically. If this continues, run the agent test in Configuration.",
+  harness_probe_unexpected_output: "The execution test did not return the expected response. Run the agent test in Configuration to inspect the failed checks.",
+  test_environment_unavailable: "The test environment is unavailable. Check the agent's environment in Configuration. Paperclip will retry automatically.",
+  required_plugin_unavailable: "A required plugin is unavailable. Check the plugin's status in Settings. Paperclip will retry automatically.",
+  invalid_lifecycle_result: "A required plugin returned an invalid result. Check the plugin's status in Settings. Paperclip will retry automatically.",
+} as const;
+
+const transientCodes = new Set(["ECONNREFUSED", "ECONNRESET", "ETIMEDOUT", "40001", "40P01", "53300", "57P01"]);
+
+export function lifecycleFailureCode(error: unknown): string | undefined {
+  const candidate = error as { code?: unknown; cause?: { code?: unknown } } | null;
+  return [candidate?.code, candidate?.cause?.code].find((code): code is string =>
+    typeof code === "string" && (Object.hasOwn(reasons, code) || transientCodes.has(code)));
+}
+
+export function lifecycleFailureMessage(context: LifecycleFailure, error: unknown): string {
+  const phases = {
+    preparing: "Agent preparation", verifying: "Execution configuration check", pausing: "Stopping agent execution",
+    resuming: "Starting agent resources", terminating: "Stopping agent execution", cleaning_up: "Agent resource cleanup",
+  };
+  const phase = context.phase && Object.hasOwn(phases, context.phase)
+    ? phases[context.phase as keyof typeof phases] : "Agent setup";
+  const step = context.stage === "plugin" ? `${phase}: required plugin step`
+    : context.stage === "select_plugins" ? `${phase}: finding required plugins`
+    : ["save_plugins", "complete_host", "complete_plugin", "transition"].includes(context.stage)
+      ? `${phase}: saving progress` : phase;
+  const code = lifecycleFailureCode(error);
+  const reason = code && Object.hasOwn(reasons, code) ? reasons[code as keyof typeof reasons]
+    : "Paperclip could not complete this step. It will retry automatically. If this continues, contact your instance administrator.";
+  return `${step} failed. ${reason}`;
+}

server/src/modules/agent-lifecycle/application/worker.ts

@@ -1,6 +1,7 @@
 import { randomUUID } from "node:crypto";
 import type { LifecycleDriver, LifecycleStore, LifecycleFailure } from "./ports.js";
 import { isLifecycleWorkPending } from "../domain/policy.js";
+import { lifecycleFailureMessage } from "./failure.js";
 
 /** Database leases protect each attempt across server processes. */
 export function createLifecycleWorker(store: LifecycleStore, driver: LifecycleDriver, reportFailure: (context: LifecycleFailure, error: unknown) => void, canRun = () => true) {
@@ -20,10 +21,13 @@ export function createLifecycleWorker(store: LifecycleStore, driver: LifecycleDr
       if (!agent || !isLifecycleWorkPending(agent.lifecycleState)) return;
       const context = { agentId: agent.id, companyId: agent.companyId, phase: agent.lifecycleState,
         operationId: agent.lifecycleOperation?.id, version: agent.lifecycleVersion };
-      const runStep = <T>(stage: LifecycleFailure["stage"], work: () => Promise<T>, pluginId?: string) =>
-        attempt({ ...context, stage, ...(pluginId ? { pluginId } : {}) }, work);
+      let failureContext: LifecycleFailure = { ...context, stage: "host" };
+      const runStep = <T>(stage: LifecycleFailure["stage"], work: () => Promise<T>, pluginId?: string) => {
+        const stepContext = { ...context, stage, ...(pluginId ? { pluginId } : {}) };
+        return attempt(stepContext, work).catch(error => { failureContext = stepContext; throw error; });
+      };
       const renewal = setInterval(() => {
-        void runStep("renew", () => store.renew(agent, owner, new Date())).catch(() => {});
+        void attempt({ ...context, stage: "renew" }, () => store.renew(agent, owner, new Date())).catch(() => {});
       }, 30_000);
       renewal.unref?.();
       try {
@@ -65,8 +69,9 @@ export function createLifecycleWorker(store: LifecycleStore, driver: LifecycleDr
           async () => { await driver.onReady?.(changed); }).catch(() => {});
         }
         if (changed && !isLifecycleWorkPending(changed.lifecycleState)) return;
-      } catch {
-        if (!await runStep("defer", () => store.defer(agent, owner, "The lifecycle step failed. Retry the operation.", new Date()))) continue;
+      } catch (error) {
+        const message = lifecycleFailureMessage(failureContext, error);
+        if (!await runStep("defer", () => store.defer(agent, owner, message, new Date()))) continue;
         return;
       } finally {
         clearInterval(renewal);

server/src/modules/agent-lifecycle/index.ts

@@ -142,3 +142,5 @@ export function startAgentLifecycle(db: Db, effects: LifecycleEffects, driver: L
     },
   };
 }
+
+export { lifecycleFailureCode } from "./application/failure.js";

server/src/services/agent-harness-verification.ts

@@ -19,7 +19,17 @@ export function assertHarnessTestPassed(result: AdapterEnvironmentTestResult) {
   const authenticationFailed = result.checks.some(check =>
     check.code === ADAPTER_AUTH_MISSING_CHECK_CODE || /_hello_probe_auth_required$/.test(check.code));
   const incomplete = !verified && (result.status === "warn" || result.checks.some(check => check.code.includes("hello_probe")));
-  if (result.status === "fail" || authenticationFailed || incomplete) throw Object.assign(new Error("The harness test failed"), { code: "harness_test_failed" });
+  if (result.status === "fail" || authenticationFailed || incomplete) {
+    const has = (suffix: string) => result.checks.some(check => check.code.endsWith(suffix));
+    const code = authenticationFailed ? "harness_auth_failed"
+      : has("_hello_probe_model_unavailable") || has("_model_invalid") ? "harness_model_unavailable"
+      : has("_command_unresolvable") ? "harness_command_unavailable"
+      : has("_cwd_invalid") ? "harness_cwd_invalid"
+      : has("_hello_probe_timed_out") ? "harness_probe_timed_out"
+      : has("_hello_probe_unexpected_output") ? "harness_probe_unexpected_output"
+      : "harness_test_failed";
+    throw Object.assign(new Error("The harness test failed"), { code });
+  }
 }
 
 export function agentHarnessVerificationService(db: Db, manager: PluginWorkerManager) {

server/src/services/agent-lifecycle.ts

@@ -16,11 +16,7 @@ export type { LifecycleDriver, LifecycleAgent } from "../modules/agent-lifecycle
 export function createAgentLifecycleEffects(hooks: BudgetServiceHooks = {}): lifecycle.LifecycleEffects {
   return {
     reportFailure(context, error) {
-      // Only known codes are safe; provider errors can contain credentials or private data.
-      const allowedCodes = new Set(["required_plugin_unavailable", "invalid_lifecycle_result", "harness_test_failed",
-        "test_environment_unavailable", "ECONNREFUSED", "ECONNRESET", "ETIMEDOUT", "40001", "40P01", "53300", "57P01"]);
-      const candidate = error as { code?: unknown; cause?: { code?: unknown } } | null;
-      const code = [candidate?.code, candidate?.cause?.code].find(value => typeof value === "string" && allowedCodes.has(value));
+      const code = lifecycle.lifecycleFailureCode(error);
       logger.warn({ ...context, failureCode: code ?? "unclassified" }, "Agent lifecycle operation failed");
     },
     transaction: withAccountingTransaction,

server/src/services/dot-runner-broker.ts

@@ -249,6 +249,16 @@ function createBroker(db: Db) {
       if (options.requireOffered && authority.assignment.status !== "offered") {
         throw fail("This work was already accepted. Read its feedback and operation receipts without another routine wake.");
       }
+      if (options.requireOffered && authority.binding.providerKind === "grok_bot") {
+        // A prepared intent from an older dispatcher must not start a second
+        // interaction while the original caller is waiting for admission.
+        const [run] = await db.select({ contextSnapshot: heartbeatRuns.contextSnapshot }).from(heartbeatRuns)
+          .where(and(eq(heartbeatRuns.id, authority.assignment.runId), eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.agentId, agentId)));
+        const proactive = run?.contextSnapshot?.grokProactiveRequest as Record<string, unknown> | undefined;
+        if (proactive?.bindingId === authority.binding.id && proactive.bindingGeneration === authority.binding.generation) {
+          throw fail("This Bot requested the work. Continue polling its original request instead of starting another routine.");
+        }
+      }
       return authority;
     },
     async authorizeBinding(principal: McpPrincipal, companyId: string, bindingId: string) {
@@ -390,13 +400,16 @@ function createBroker(db: Db) {
         if (!assignment) await new Promise(resolve => setTimeout(resolve, 50));
       } while (!assignment && Date.now() < deadline);
       return { ...wake, issueId: task.id, identifier: task.identifier, assignment: assignment ?? null,
-        instruction: wake.status === "admission_failed" ? wake.message : "Drain the inbox now. If assignment is present, read and accept it, then use its catalog. If admission is still pending, retry this provider’s request_turn tool with exactly the same requestId and prompt to check this intake; never create a second request. Events also notify queued work. This intake supplies normal run authority; no human needs to create a preliminary task." };
+        instruction: wake.status === "admission_failed" ? wake.message : `Drain the inbox now. If assignment is present, read and accept it, then use its catalog. If admission is still pending, retry this provider’s request_turn tool with exactly the same requestId and prompt to check this intake; never create a second request. ${b.providerKind === "grok_bot" ? "Queued admission does not start another Grok routine. Keep checking this request from the current conversation until it is admitted or fails." : "Events also notify queued work."} This intake supplies normal run authority; no human needs to create a preliminary task.` };
     },
     async requestWork(principal: McpPrincipal, issueId: string, requestId: string) {
       const b = await principalBinding(principal);
       if (!z.uuid().safeParse(requestId).success) throw fail("Use a stable UUID requestId.");
       if (!await enabled(b.providerKind) || !(await this.bindingForAgent(b.companyId, b.agentId))?.wakeVerified) throw fail("External agent admission is unavailable.");
       const key = `dot-work:${b.id}:${b.generation}:${requestId}`;
+      const admissionMessage = b.providerKind === "grok_bot"
+        ? "Normal admission determines when this task can run. Keep checking the original request with the same request ID and arguments, then read and accept its inbox assignment. Queued admission does not start another Grok routine."
+        : "Normal admission determines when this task can run. Read the mailbox after its event.";
       const receipt = async () => (await db.select().from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, b.companyId), eq(agentWakeupRequests.agentId, b.agentId), eq(agentWakeupRequests.idempotencyKey, key))).limit(1))[0];
       const replay = async (row: typeof agentWakeupRequests.$inferSelect) => {
         if (row.payload?.issueId !== issueId) throw fail("requestId was reused for another task.");
@@ -407,7 +420,7 @@ function createBroker(db: Db) {
           return { status: "admission_failed", runId: row.runId, admissionStatus: run.status,
             message: "The previous admission ended without an active assignment. Stop waiting for its mailbox event. Inspect the task's run and use normal task recovery after resolving the failure. This request ID retains its original receipt." };
         }
-        return { status: "requested", runId: row.runId, message: "Normal admission determines when this task can run. Read the mailbox after its event." };
+        return { status: "requested", runId: row.runId, message: admissionMessage };
       };
       // A retry is a receipt read, including after the task yielded or finished.
       const old = await receipt();
@@ -424,7 +437,7 @@ function createBroker(db: Db) {
           payload: { issueId, dotRequestId: requestId }, contextSnapshot: { issueId, ...(b.providerKind === "grok_bot" ? { grokProactiveRequest: { requestId, bindingId: b.id, bindingGeneration: b.generation } } : {}) }, idempotencyKey: key, requestedByActorType: "agent", requestedByActorId: b.agentId,
           allowRunCoalescing: false, durableDotRequest: { id: receiptId, companyId: b.companyId, agentId: b.agentId, issueId, requestId, idempotencyKey: key, requestedAt: new Date() } });
         const reserved = await receipt();
-        return reserved ? replay(reserved) : { status: "requested", runId: run?.id ?? null, message: "Normal admission determines when this task can run. Read the mailbox after its event." };
+        return reserved ? replay(reserved) : { status: "requested", runId: run?.id ?? null, message: admissionMessage };
       } catch (error) {
         // A concurrent reservation can lose the PK race on a different task.
         // Replay only that durable receipt; do not mask unrelated failures.

server/src/services/grok-bot/dispatcher.ts

@@ -1,5 +1,5 @@
-import { and, asc, eq, isNull, lt, sql } from "drizzle-orm";
-import { grokBotDispatches, grokBotRoutines, dotAgentBindings, dotRunnerAssignments, heartbeatRuns, type Db } from "@paperclipai/db";
+import { and, asc, eq } from "drizzle-orm";
+import { grokBotDispatches, grokBotRoutines, type Db } from "@paperclipai/db";
 import { grokRoutineService } from "./routine.js";
 import type { deliverGrokRoutineWake } from "./webhook.js";
 import { externalBotEnabled } from "../external-bot-authority.js";
@@ -14,17 +14,9 @@ export function createGrokRoutineDispatcher(db: Db, isBackgroundWorkEnabled: ()
     running = true;
     try {
       const service = grokRoutineService(db, deliver);
-      // A proactive caller receives an immediate assignment without an echo.
-      // Queued admission gets one delayed wake only while still unaccepted.
-      const proactive = await db.select({ assignment: dotRunnerAssignments }).from(dotRunnerAssignments)
-        .innerJoin(dotAgentBindings, eq(dotAgentBindings.id, dotRunnerAssignments.bindingId))
-        .innerJoin(heartbeatRuns, eq(heartbeatRuns.id, dotRunnerAssignments.runId))
-        .where(and(eq(dotAgentBindings.providerKind, "grok_bot"), isNull(dotAgentBindings.revokedAt), eq(dotRunnerAssignments.status, "offered"),
-          lt(dotRunnerAssignments.createdAt, new Date(Date.now() - 10_000)), sql`${heartbeatRuns.contextSnapshot}->'grokProactiveRequest'->>'bindingId' = ${dotAgentBindings.id}::text`)).limit(10);
-      for (const { assignment } of proactive) {
-        try { await service.reserve({ companyId: assignment.companyId, agentId: assignment.agentId, bindingId: assignment.bindingId,
-          assignmentId: assignment.id, kind: "mailbox_updated", sourceEventId: `proactive-admission:${assignment.id}` }); } catch { /* Current authority is rechecked on reservation and send. */ }
-      }
+      // Bot-initiated callers poll their durable request while admission is
+      // pending. Delay does not prove that the originating conversation ended.
+      // Starting a webhook here would create a competing Grok interaction.
       // An accepted session already has a routine handling it. Quiet MCP
       // traffic does not prove that routine ended. Keep feedback and receipts
       // durable for the Bot to read instead of starting overlapping work.