Merge origin/master and retain both Cloud skill fixture contracts

Co-Authored-By: Paperclip <noreply@paperclip.ing>

server/src/modules/agent-lifecycle/__tests__/diagnostics.test.ts

@@ -5,7 +5,7 @@ import type { LifecycleAgent, LifecycleStore, LifecycleDriver } from "../applica
 import { startAgentLifecycle } from "../index.js";
 import { createAgentLifecycleEffects } from "../../../services/agent-lifecycle.js";
 import { logger } from "../../../middleware/logger.js";
-import { lifecycleFailureMessage } from "../application/failure.js";
+import { lifecycleFailureMessage, lifecycleVerificationNeedsRetry } from "../application/failure.js";
 
 const agent: LifecycleAgent = { id: "agent", companyId: "company", lifecycleState: "verifying", lifecycleVersion: 3,
   lifecycleRequiredPluginIds: null, lifecycleError: null,
@@ -67,6 +67,20 @@ describe("lifecycle readiness handoff", () => {
 });
 
 describe("lifecycle failure diagnostics", () => {
+  it.each(["harness_auth_failed", "harness_model_unavailable", "harness_command_unavailable", "harness_cwd_invalid", "harness_probe_unexpected_output"])("waits for repair only for a host verification failure: %s", code => {
+    const error = { code };
+    expect(lifecycleVerificationNeedsRetry({ stage: "host", phase: "verifying" }, error)).toBe(true);
+    expect(lifecycleVerificationNeedsRetry({ stage: "plugin", phase: "verifying" }, error)).toBe(false);
+    expect(lifecycleVerificationNeedsRetry({ stage: "host", phase: "cleaning_up" }, error)).toBe(false);
+  });
+
+  it("allows two automatic retries of a generic failed test before waiting for repair", () => {
+    for (const verificationFailures of [1, 2, 3, 8_000]) {
+      const context = { stage: "host" as const, phase: "verifying" as const, attempts: 8_000, verificationFailures };
+      expect(lifecycleVerificationNeedsRetry(context, { code: "harness_test_failed" })).toBe(verificationFailures >= 3);
+    }
+  });
+
   it.each([
     ["host", "harness_model_unavailable", "Execution configuration check failed. The selected model is unavailable."],
     ["plugin", "required_plugin_unavailable", "Execution configuration check: required plugin step failed. A required plugin is unavailable."],
@@ -80,7 +94,8 @@ describe("lifecycle failure diagnostics", () => {
     if (stage === "select_plugins") driver.requiredPluginIds.mockRejectedValue(error);
     if (stage === "transition") store.change.mockRejectedValue(error);
     await worker.process(agent.id);
-    expect(store.defer).toHaveBeenCalledWith(agent, expect.any(String), expect.stringContaining(expected), expect.any(Date));
+    expect(store.defer).toHaveBeenCalledWith(agent, expect.any(String), expect.stringContaining(expected), expect.any(Date),
+      ...(stage === "host" ? [{ retryRequired: true }] : []));
     expect(JSON.stringify(store.defer.mock.calls)).not.toContain("secret provider output");
     expect(JSON.stringify(store.defer.mock.calls)).not.toContain("unknown private code");
     await worker.stop();

server/src/modules/agent-lifecycle/__tests__/verification-retry.test.ts

@@ -0,0 +1,182 @@
+import { randomUUID } from "node:crypto";
+import { agents, companies, createDb, applyPendingMigrations, closeRegisteredClients, type Db } from "@paperclipai/db";
+import { eq, sql } from "drizzle-orm";
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
+import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../../__tests__/helpers/embedded-postgres.js";
+import { createAgentLifecycle, configureAgentLifecycle } from "../../../services/agent-lifecycle.js";
+import { createAgentLifecycleEffects } from "../../../services/agent-lifecycle.js";
+import { startAgentLifecycle } from "../index.js";
+import { createLifecycleStore } from "../adapters/postgres.js";
+import { createLifecycleWorker } from "../application/worker.js";
+import type { LifecycleAgent } from "../application/ports.js";
+import { resolveExternalTestDatabaseUrl } from "../../../__tests__/helpers/external-test-database.js";
+
+const external = resolveExternalTestDatabaseUrl("PAPERCLIP_LIFECYCLE_VERIFICATION_RETRY_TEST_DATABASE_URL");
+const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport();
+(support.supported ? describe : describe.skip)("failed lifecycle verification retries", () => {
+  let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
+  let db: Db;
+  let companyId: string;
+  const workers: Array<{ stop(): Promise<void> }> = [];
+  beforeAll(async () => {
+    if (external) { await applyPendingMigrations(external); db = createDb(external); }
+    else { database = await startEmbeddedPostgresTestDatabase("paperclip-verification-retry-"); db = createDb(database.connectionString); }
+  }, 90_000);
+  afterAll(async () => { if (external) await closeRegisteredClients(external); else await database?.cleanup(); });
+  beforeEach(async () => {
+    companyId = randomUUID();
+    await db.insert(companies).values({ id: companyId, name: "Verification retry", issuePrefix: `V${companyId.slice(0, 7)}` });
+  });
+  afterEach(async () => {
+    for (const worker of workers.splice(0)) await worker.stop();
+    // Retire only this test's synthetic records so later coordinator reads stay isolated.
+    await db.update(agents).set({ lifecycleState: "terminated", status: "terminated" }).where(eq(agents.companyId, companyId));
+  });
+  const current = async (id: string) => (await db.select().from(agents).where(eq(agents.id, id)))[0]!;
+  async function failed(code = "harness_test_failed") {
+    const runHost = vi.fn(async (agent: LifecycleAgent) => {
+      if (agent.lifecycleState === "verifying") throw Object.assign(new Error("private provider detail"), { code });
+      return "complete" as const;
+    });
+    const driver = { requiredPluginIds: async () => [], runHost, runPlugin: async () => "complete" as const };
+    const work = configureAgentLifecycle(db, driver); workers.push(work);
+    const agent = await createAgentLifecycle(db).requestHire(companyId, { name: "Retry fixture", adapterType: "process", adapterConfig: { command: "missing-command" } });
+    await work.process(agent.id);
+    if (code === "harness_test_failed") {
+      for (let retry = 0; retry < 2; retry++) {
+        expect((await current(agent.id)).lifecycleOperation?.retryRequired).toBeUndefined();
+        await db.update(agents).set({ lifecycleOperation: sql`${agents.lifecycleOperation} - 'retryAt'` }).where(eq(agents.id, agent.id));
+        await work.process(agent.id);
+      }
+    }
+    return { agent: await current(agent.id), work, driver, runHost };
+  }
+
+  it("persists a failed test across worker restarts without scanning, claiming, or probing it again", async () => {
+    const { agent, work, driver, runHost } = await failed();
+    expect(agent).toMatchObject({ lifecycleState: "verifying", status: "paused", lifecycleOperation: { retryRequired: true } });
+    expect(agent.lifecycleOperation).not.toHaveProperty("retryAt");
+    expect(agent.lifecycleOperation).not.toHaveProperty("leaseOwner");
+    expect(agent.lifecycleError).toContain("Automatic verification is paused");
+    expect(agent.lifecycleError).not.toContain("private provider detail");
+    await work.stop();
+    const restarted = startAgentLifecycle(db, createAgentLifecycleEffects(), driver, () => true); workers.push(restarted);
+    const calls = runHost.mock.calls.length;
+    expect(await restarted.hasPending()).toBe(false);
+    expect(restarted.nextRunAt()).toBeNull();
+    await restarted.sweep();
+    const store = createLifecycleStore(db, createAgentLifecycleEffects());
+    const tomorrow = new Date(Date.now() + 86_400_000);
+    expect(await store.pending(100, tomorrow)).not.toContain(agent.id);
+    expect(await store.claim(agent.id, "new-owner", tomorrow)).toBeNull();
+    await work.process(agent.id);
+    expect(runHost).toHaveBeenCalledTimes(calls);
+    expect((await current(agent.id)).lifecycleOperation).toEqual(agent.lifecycleOperation);
+  });
+
+  it("permits an explicit retry of the same operation and completes verification before readiness", async () => {
+    const { agent, work, runHost } = await failed();
+    runHost.mockResolvedValue("complete");
+    await createAgentLifecycle(db).retry(agent.id);
+    expect((await current(agent.id)).lifecycleOperation).not.toHaveProperty("retryRequired");
+    expect((await current(agent.id)).lifecycleOperation).not.toHaveProperty("verificationFailures");
+    expect((await current(agent.id)).lifecycleOperation!.id).toBe(agent.lifecycleOperation!.id);
+    await work.process(agent.id);
+    expect(await current(agent.id)).toMatchObject({ lifecycleState: "ready", status: "idle", lifecycleError: null });
+  });
+
+  it("does not spend verification retries on claims deferred before the host test", async () => {
+    const [agent] = await db.insert(agents).values({ companyId, name: "Deferred verification", status: "paused",
+      lifecycleState: "verifying", lifecycleRequiredPluginIds: [],
+      lifecycleOperation: { id: randomUUID(), hostComplete: false, completedPluginIds: [], attempts: 0 } }).returning();
+    const store = createLifecycleStore(db, createAgentLifecycleEffects());
+    let allowed = true;
+    let deferBeforeHost = true;
+    const runHost = vi.fn(async () => { throw Object.assign(new Error("test failed"), { code: "harness_test_failed" }); });
+    const work = createLifecycleWorker({ ...store, async claim(...args) {
+      const claimed = await store.claim(...args);
+      if (claimed && deferBeforeHost) allowed = false;
+      return claimed;
+    } }, { requiredPluginIds: async () => [], runPlugin: async () => "complete", runHost }, () => {}, () => allowed);
+    workers.push(work);
+    const due = async () => {
+      allowed = true;
+      await db.update(agents).set({ lifecycleOperation: sql`${agents.lifecycleOperation} - 'retryAt'` }).where(eq(agents.id, agent.id));
+      await work.process(agent.id);
+    };
+    await due(); await due();
+    expect(runHost).not.toHaveBeenCalled();
+    expect((await current(agent.id)).lifecycleOperation).toMatchObject({ attempts: 2 });
+    expect((await current(agent.id)).lifecycleOperation).not.toHaveProperty("verificationFailures");
+    deferBeforeHost = false;
+    await due();
+    expect((await current(agent.id)).lifecycleOperation).toMatchObject({ attempts: 3, verificationFailures: 1 });
+    expect((await current(agent.id)).lifecycleOperation?.retryRequired).toBeUndefined();
+    // Another deferred claim preserves the already recorded failed test.
+    deferBeforeHost = true; await due();
+    expect((await current(agent.id)).lifecycleOperation?.verificationFailures).toBe(1);
+    deferBeforeHost = false;
+    await due();
+    expect((await current(agent.id)).lifecycleOperation?.retryRequired).toBeUndefined();
+    await due();
+    expect(runHost).toHaveBeenCalledTimes(3);
+    expect((await current(agent.id)).lifecycleOperation).toMatchObject({ verificationFailures: 3, retryRequired: true });
+  });
+
+  it("recovers from a generic failed test before the automatic retry budget is exhausted", async () => {
+    const { agent, work, runHost } = await failed("harness_probe_timed_out");
+    runHost.mockRejectedValueOnce(Object.assign(new Error("temporary test failure"), { code: "harness_test_failed" }));
+    await db.update(agents).set({ lifecycleOperation: sql`${agents.lifecycleOperation} - 'retryAt'` }).where(eq(agents.id, agent.id));
+    await work.process(agent.id);
+    expect((await current(agent.id)).lifecycleOperation?.retryRequired).toBeUndefined();
+    runHost.mockResolvedValue("complete");
+    await db.update(agents).set({ lifecycleOperation: sql`${agents.lifecycleOperation} - 'retryAt'` }).where(eq(agents.id, agent.id));
+    await work.process(agent.id);
+    expect((await current(agent.id)).lifecycleState).toBe("ready");
+  });
+
+  it("rechecks changed execution configuration but leaves a name edit waiting", async () => {
+    const { agent, work, runHost } = await failed();
+    const lifecycle = createAgentLifecycle(db);
+    await lifecycle.updateConfiguration(agent.id, { name: "Renamed fixture" });
+    expect((await current(agent.id)).lifecycleOperation!.retryRequired).toBe(true);
+    runHost.mockResolvedValue("complete");
+    await lifecycle.updateConfiguration(agent.id, { adapterConfig: { command: process.execPath } });
+    const changed = await current(agent.id);
+    expect(changed.lifecycleOperation).not.toHaveProperty("retryRequired");
+    expect(changed.lifecycleOperation).not.toHaveProperty("verificationFailures");
+    expect(changed.lifecycleOperation!.id).not.toBe(agent.lifecycleOperation!.id);
+    await work.process(agent.id);
+    expect((await current(agent.id)).lifecycleState).toBe("ready");
+  });
+
+  it("does not let a stale verification failure park newly saved configuration", async () => {
+    const { agent, work } = await failed("harness_auth_failed");
+    await work.stop();
+    const store = createLifecycleStore(db, createAgentLifecycleEffects());
+    await store.change(agent.id, "retry");
+    const claimed = await store.claim(agent.id, "old-verification", new Date());
+    expect(claimed).not.toBeNull();
+    await createAgentLifecycle(db).updateConfiguration(agent.id, { adapterConfig: { command: process.execPath } });
+    expect(await store.defer(claimed!, "old-verification", "Old failure", new Date(), { retryRequired: true })).toBe(false);
+    expect((await current(agent.id)).lifecycleOperation?.retryRequired).toBeUndefined();
+  });
+
+  it.each(["harness_probe_timed_out", "test_environment_unavailable", "ECONNRESET", "unclassified"])("keeps automatic recovery for %s", async code => {
+    const { agent } = await failed(code);
+    expect(agent.lifecycleOperation).not.toHaveProperty("retryRequired");
+    expect(Date.parse(agent.lifecycleOperation!.retryAt!)).toBeGreaterThan(Date.now());
+    const store = createLifecycleStore(db, createAgentLifecycleEffects());
+    expect(await store.pending(100, new Date(Date.now() + 120_000))).toContain(agent.id);
+    expect(await store.claim(agent.id, "automatic-retry", new Date(Date.now() + 120_000))).not.toBeNull();
+  });
+
+  it("still completes termination and cleanup when verification needs repair", async () => {
+    const { agent, work, runHost } = await failed();
+    runHost.mockResolvedValue("complete");
+    await createAgentLifecycle(db).terminateAgent(agent.id);
+    await work.process(agent.id);
+    expect((await current(agent.id)).lifecycleState).toBe("terminated");
+    expect(runHost.mock.calls.map(([a]) => a.lifecycleState)).toContain("cleaning_up");
+  });
+});

server/src/modules/agent-lifecycle/adapters/configuration.ts

@@ -37,7 +37,8 @@ export async function updateAgentConfiguration(tx: Db, effects: LifecycleEffects
     ...(!prepareSetup && current.lifecycleHolds.includes("setup") && ["preparing", "verifying"].includes(current.lifecycleState)
       ? { lifecycleState: "pausing" as const } : {}),
     lifecycleOperation: { ...current.lifecycleOperation!, id: randomUUID(), hostComplete: false,
-      completedPluginIds: [], leaseOwner: undefined, leaseUntil: undefined, retryAt: undefined },
+      completedPluginIds: [], attempts: 0, leaseOwner: undefined, leaseUntil: undefined,
+      retryAt: undefined, retryRequired: undefined, verificationFailures: undefined },
   } : {}) }).where(eq(agents.id, id)).returning();
   await effects.completeConfiguration(tx, current, updated, patch, options, publications);
   return agentRecordQueries(tx).getById(id);

server/src/modules/agent-lifecycle/adapters/postgres.ts

@@ -17,6 +17,7 @@ type ChangeOptions = { version?: number; owner?: string; reason?: string };
 export const pendingLifecycleWork = and(
   inArray(agents.lifecycleState, ["preparing", "verifying", "pausing", "resuming", "terminating", "cleaning_up"]),
   sql`not (${agents.lifecycleState} = 'verifying' and ${agents.lifecycleHolds} @> '["setup"]'::jsonb)`,
+  sql`not (${agents.lifecycleState} = 'verifying' and coalesce(${agents.lifecycleOperation}->>'retryRequired', 'false') = 'true')`,
 );
 
 /** The caller owns the accounting transaction and has locked this agent. */
@@ -33,7 +34,7 @@ async function changeLockedAgent(tx: Db, effects: LifecycleEffects, agent: Lifec
     if (agent.lifecycleOperation?.leaseUntil && Date.parse(agent.lifecycleOperation.leaseUntil) > Date.now()) return agent;
     await effects.notifyWork(tx);
     const [updated] = await tx.update(agents).set({ lifecycleError: null,
-      lifecycleOperation: sql`${agents.lifecycleOperation} - 'retryAt' - 'leaseOwner' - 'leaseUntil'`, updatedAt: new Date() }).where(eq(agents.id, agent.id)).returning();
+      lifecycleOperation: sql`${agents.lifecycleOperation} - 'retryAt' - 'retryRequired' - 'verificationFailures' - 'leaseOwner' - 'leaseUntil'`, updatedAt: new Date() }).where(eq(agents.id, agent.id)).returning();
     return updated;
   }
   if (command === "resume" && !["ready", "paused", "pausing", "resuming"].includes(agent.lifecycleState)) transition(agent.lifecycleState, command);
@@ -177,11 +178,17 @@ export function createLifecycleStore(db: Db, effects: LifecycleEffects): Lifecyc
         sql`${agents.lifecycleOperation}->>'leaseOwner' = ${owner}`)).returning({ id: agents.id });
       return rows.length > 0;
     },
-    async defer(agent, owner, error, now) {
+    async defer(agent, owner, error, now, options) {
+      const retry: Partial<AgentLifecycleOperation> = options?.retryRequired && agent.lifecycleState === "verifying"
+        ? { retryRequired: true }
+        : { retryAt: new Date(now.getTime() + (error ? 60_000 : 2_000)).toISOString() };
+      if (options?.verificationFailures !== undefined && agent.lifecycleState === "verifying") {
+        retry.verificationFailures = options.verificationFailures;
+      }
       const rows = await db.update(agents).set({
         lifecycleError: error,
-        lifecycleOperation: sql`(${agents.lifecycleOperation} - 'leaseOwner' - 'leaseUntil')
-          || jsonb_build_object('retryAt', ${new Date(now.getTime() + (error ? 60_000 : 2_000)).toISOString()}::text)`,
+        lifecycleOperation: sql`(${agents.lifecycleOperation} - 'leaseOwner' - 'leaseUntil' - 'retryAt' - 'retryRequired')
+          || ${JSON.stringify(retry)}::jsonb`,
         updatedAt: now,
       }).where(and(eq(agents.id, agent.id), eq(agents.lifecycleVersion, agent.lifecycleVersion),
         sql`${agents.lifecycleOperation}->>'leaseOwner' = ${owner}`)).returning({ id: agents.id });

server/src/modules/agent-lifecycle/application/failure.ts

@@ -15,6 +15,16 @@ const reasons = {
 } as const;
 
 const transientCodes = new Set(["ECONNREFUSED", "ECONNRESET", "ETIMEDOUT", "40001", "40P01", "53300", "57P01"]);
+const configurationFailures = new Set([
+  "harness_auth_failed", "harness_model_unavailable",
+  "harness_command_unavailable", "harness_cwd_invalid", "harness_probe_unexpected_output",
+]);
+
+export function lifecycleVerificationNeedsRetry(context: LifecycleFailure, error: unknown): boolean {
+  const code = lifecycleFailureCode(error) ?? "";
+  return context.phase === "verifying" && context.stage === "host"
+    && (configurationFailures.has(code) || (code === "harness_test_failed" && (context.verificationFailures ?? 0) >= 3));
+}
 
 export function lifecycleFailureCode(error: unknown): string | undefined {
   const candidate = error as { code?: unknown; cause?: { code?: unknown } } | null;
@@ -36,5 +46,6 @@ export function lifecycleFailureMessage(context: LifecycleFailure, error: unknow
   const code = lifecycleFailureCode(error);
   const reason = code && Object.hasOwn(reasons, code) ? reasons[code as keyof typeof reasons]
     : "Paperclip could not complete this step. It will retry automatically. If this continues, contact your instance administrator.";
-  return `${step} failed. ${reason}`;
+  return `${step} failed. ${reason}${lifecycleVerificationNeedsRetry(context, error)
+    ? " Automatic verification is paused. Update Configuration or choose Retry now." : ""}`;
 }

server/src/modules/agent-lifecycle/application/ports.ts

@@ -20,7 +20,7 @@ export interface LifecycleStore {
   setRequiredPlugins(agent: LifecycleAgent, owner: string, pluginIds: string[]): Promise<boolean>;
   completeHost(agent: LifecycleAgent, owner: string): Promise<boolean>;
   completePlugin(agent: LifecycleAgent, owner: string, pluginId: string): Promise<boolean>;
-  defer(agent: LifecycleAgent, owner: string, error: string | null, now: Date): Promise<boolean>;
+  defer(agent: LifecycleAgent, owner: string, error: string | null, now: Date, options?: { retryRequired: boolean; verificationFailures?: number }): Promise<boolean>;
   pending(limit: number, now: Date): Promise<string[]>;
 }
 
@@ -40,5 +40,7 @@ export interface LifecycleFailure {
   phase?: AgentLifecycleState;
   operationId?: string;
   version?: number;
+  attempts?: number;
+  verificationFailures?: number;
   pluginId?: string;
 }

server/src/modules/agent-lifecycle/application/worker.ts

@@ -1,7 +1,7 @@
 import { randomUUID } from "node:crypto";
 import type { LifecycleDriver, LifecycleStore, LifecycleFailure } from "./ports.js";
 import { isLifecycleWorkPending } from "../domain/policy.js";
-import { lifecycleFailureMessage } from "./failure.js";
+import { lifecycleFailureCode, lifecycleFailureMessage, lifecycleVerificationNeedsRetry } from "./failure.js";
 
 /** Database leases protect each attempt across server processes. */
 export function createLifecycleWorker(store: LifecycleStore, driver: LifecycleDriver, reportFailure: (context: LifecycleFailure, error: unknown) => void, canRun = () => true) {
@@ -20,7 +20,8 @@ export function createLifecycleWorker(store: LifecycleStore, driver: LifecycleDr
       const agent = await attempt({ stage: "claim", agentId: id }, () => store.claim(id, owner, new Date()));
       if (!agent || !isLifecycleWorkPending(agent.lifecycleState)) return;
       const context = { agentId: agent.id, companyId: agent.companyId, phase: agent.lifecycleState,
-        operationId: agent.lifecycleOperation?.id, version: agent.lifecycleVersion };
+        operationId: agent.lifecycleOperation?.id, version: agent.lifecycleVersion,
+        attempts: agent.lifecycleOperation?.attempts };
       let failureContext: LifecycleFailure = { ...context, stage: "host" };
       const runStep = <T>(stage: LifecycleFailure["stage"], work: () => Promise<T>, pluginId?: string) => {
         const stepContext = { ...context, stage, ...(pluginId ? { pluginId } : {}) };
@@ -70,8 +71,16 @@ export function createLifecycleWorker(store: LifecycleStore, driver: LifecycleDr
         }
         if (changed && !isLifecycleWorkPending(changed.lifecycleState)) return;
       } catch (error) {
+        // Claims can defer before runHost. Only a failed host test spends this budget.
+        const verificationFailures = failureContext.stage === "host" && agent.lifecycleState === "verifying"
+          && lifecycleFailureCode(error) === "harness_test_failed"
+          ? (agent.lifecycleOperation?.verificationFailures ?? 0) + 1 : undefined;
+        failureContext = { ...failureContext, verificationFailures };
         const message = lifecycleFailureMessage(failureContext, error);
-        if (!await runStep("defer", () => store.defer(agent, owner, message, new Date()))) continue;
+        const retryRequired = lifecycleVerificationNeedsRetry(failureContext, error);
+        if (!await runStep("defer", () => retryRequired || verificationFailures !== undefined
+          ? store.defer(agent, owner, message, new Date(), { retryRequired, ...(verificationFailures !== undefined ? { verificationFailures } : {}) })
+          : store.defer(agent, owner, message, new Date()))) continue;
         return;
       } finally {
         clearInterval(renewal);