From bf9eebe5bd3d53049d1bfefee0407ce73612c133 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 31 Oct 2026 00:29:28 -0500 Subject: [PATCH 1/2] Exercise saved legacy input through its eligibility producer Co-Authored-By: Paperclip --- .../2026-10-30-deferred-execution-sleep.md | 20 ++++++++++++ tests/e2e/legacy-failure-continuation.spec.ts | 31 ++++++++++++++----- 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/doc/plans/2026-10-30-deferred-execution-sleep.md b/doc/plans/2026-10-30-deferred-execution-sleep.md index 3d2cb88ce40..9c90025abe6 100644 --- a/doc/plans/2026-10-30-deferred-execution-sleep.md +++ b/doc/plans/2026-10-30-deferred-execution-sleep.md @@ -32,3 +32,23 @@ Project budget admission locks and re-reads the company-scoped project before pu Native receipt checks share an opaque dispatcher created by the existing queue owner for exactly one bounded worker pass. Its canonical company/agent/issue/review scope map is the same map used by the manual dispatcher; the native worker explicitly admits at most100due entries per pass, matching the existing default batch while retaining company fairness. The next pass creates a fresh map. This restores same-pass grouping without introducing global or cross-invocation deduplication, changing claim authority, or consuming later notices. Each receipt still re-reads its own moved claim/error deadline before dispatch. Actual PostgreSQL controls reconstruct two saved native intents into one dispatch and shared run with real process stdout, cover the100-intent boundary, and keep another issue and company separate. A real issue-row lock holds the first admission while a separate transaction commits a new notice whose requestedAt is explicitly older than admission. The fixed current batch cannot consume that new identity; a fresh pass creates a distinct run, preserving the late notice even though its ordering metadata is old. Once all executions and stored deadlines settle,25virtual minutes produce zero driver SQL. The canonical dispatch body, including parsing, claims, deterministic receipt recovery and error handling, is unchanged apart from moving its map into the owner-created batch closure. + + +### Browser fixture producer boundary + +The legacy recovery browser fixture used to insert a deferred request through a +separate SQL connection after server startup, then wait for the removed poll. +That is neither saved startup history nor a supported in-process producer. The +automatic browser case now seeds while the company is paused, verifies the saved +request and sole failed source run, and reactivates through the ordinary company +API. No Retry or duplicate message is sent. It requires exactly one new run, +one persisted agent response, the original queued comment in the process prompt, +and the response after page reload. Startup reconstruction remains covered by +the PostgreSQL case that persists history before worker registration. + +The corrected automatic path passes three consecutive built-UI/browser runs +(2.1 minutes total). All six legacy continuation browser paths pass together +(1.6 minutes), including task/inbox Retry, Retry after reload, ordinary message, +queued Retry and automatic reactivation. The fixture also reads the configured +external PostgreSQL connection when present, so local proof can reuse a dedicated +test database instead of starting another database process. diff --git a/tests/e2e/legacy-failure-continuation.spec.ts b/tests/e2e/legacy-failure-continuation.spec.ts index ed998017eae..46ce639e5b5 100644 --- a/tests/e2e/legacy-failure-continuation.spec.ts +++ b/tests/e2e/legacy-failure-continuation.spec.ts @@ -11,14 +11,15 @@ async function json(response: APIResponse) { return response.json(); } -for (const action of ["task_retry", "retry_after_reload", "inbox_retry", "message", "queued_retry", "automatic_message"] as const) { +for (const action of ["task_retry", "retry_after_reload", "inbox_retry", "message", "queued_retry", "automatic_message_on_reactivation"] as const) { test(`legacy startup hold: ${action} reaches a new agent response`, async ({ page, request }) => { test.setTimeout(120_000); const root = await mkdtemp(path.join(os.tmpdir(), "legacy-recovery-browser-")); const config = JSON.parse(await readFile(process.env.PAPERCLIP_E2E_SERVER_CONFIG!, "utf8")); // Use the running test server's actual port, including fallback allocation. - const pid = await readFile(path.join(config.database.embeddedPostgresDataDir, "postmaster.pid"), "utf8"); - const url = `postgres://paperclip:paperclip@127.0.0.1:${pid.split("\n")[3]}/paperclip`; + const url = config.database.mode === "postgres" ? config.database.connectionString + : `postgres://paperclip:paperclip@127.0.0.1:${(await readFile( + path.join(config.database.embeddedPostgresDataDir, "postmaster.pid"), "utf8")).split("\n")[3]}/paperclip`; const db = createDb(url); const company = await json(await request.post("/api/companies", { data: { name: `Legacy recovery ${action} ${Date.now()}` } })); try { @@ -37,8 +38,11 @@ for (const action of ["task_retry", "retry_after_reload", "inbox_retry", "messag title: "Continue after startup failure", description: "Answer the pending follow-up once.", status: "backlog", assigneeAgentId: agent.id, } })); + if (action === "automatic_message_on_reactivation") { + await json(await request.patch(`/api/companies/${company.id}`, { data: { status: "paused" } })); + } const sourceRunId = randomUUID(); - // Seed the historical incident, then exercise all recovery through the UI. + // Seed the historical incident, then exercise ordinary recovery entry points. // No adapter.invoke or new dispatch identity exists on this pre-upgrade run. await db.insert(heartbeatRuns).values({ id: sourceRunId, companyId: company.id, agentId: agent.id, status: "failed", runtimeMode: "legacy", processPid: action === "queued_retry" ? process.pid : 999999999, @@ -58,7 +62,7 @@ for (const action of ["task_retry", "retry_after_reload", "inbox_retry", "messag await db.insert(companyMemberships).values({ companyId: company.id, principalType: "user", principalId: "original-board", membershipRole: "operator", status: "active" }); } - if (action === "queued_retry" || action === "automatic_message") { + if (action === "queued_retry" || action === "automatic_message_on_reactivation") { const commentId = randomUUID(); // Reproduce a real user comment saved while the failed run was active, // including queues originally created by a system wake. @@ -104,8 +108,19 @@ for (const action of ["task_retry", "retry_after_reload", "inbox_retry", "messag retry.click(), ]); expect(response.ok(), `${response.status()} ${await response.text()}`).toBe(true); - } else if (action === "automatic_message") { - // No Retry, duplicate message, or status change: the saved input runs. + } else if (action === "automatic_message_on_reactivation") { + // This fixture writes history through a separate SQL connection after boot. + // Reactivation is the supported producer that makes saved work eligible; + // startup reconstruction itself is covered before registration in the PG suite. + const pending = await db.select().from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.companyId, company.id), eq(agentWakeupRequests.agentId, agent.id), + eq(agentWakeupRequests.status, "deferred_issue_execution"), + )); + expect(pending).toHaveLength(1); + const before = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, company.id)); + expect(before).toHaveLength(1); + await json(await request.patch(`/api/companies/${company.id}`, { data: { status: "active" } })); + // No Retry or duplicate message: the existing saved input runs once. } else if (action === "message") { await page.getByRole("textbox", { name: "editable markdown" }).fill("Please continue the pending follow-up."); await page.getByRole("button", { name: "Send", exact: true }).click(); @@ -133,7 +148,7 @@ for (const action of ["task_retry", "retry_after_reload", "inbox_retry", "messag expect(runs.filter(run => run.id !== sourceRunId)).toHaveLength(1); expect(runs.find(run => run.id === sourceRunId)).toMatchObject({ status: "failed", resultJson: null }); const prompts = await readFile(path.join(root, "prompts"), "utf8"); - if (action === "queued_retry" || action === "automatic_message") { + if (action === "queued_retry" || action === "automatic_message_on_reactivation") { expect(prompts).toContain("Approved"); await expect(page.getByRole("button", { name: "Interrupt", exact: true })).toHaveCount(0); } From 26275cae7f301e8206ce9b3b226f64f7b985123c Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 31 Oct 2026 00:32:36 -0500 Subject: [PATCH 2/2] test: hold lifecycle verification before retry identity assertions Co-Authored-By: Paperclip --- .../__tests__/verification-retry.test.ts | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/server/src/modules/agent-lifecycle/__tests__/verification-retry.test.ts b/server/src/modules/agent-lifecycle/__tests__/verification-retry.test.ts index cc76c705d32..93cc524613d 100644 --- a/server/src/modules/agent-lifecycle/__tests__/verification-retry.test.ts +++ b/server/src/modules/agent-lifecycle/__tests__/verification-retry.test.ts @@ -19,8 +19,8 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu let companyId: string; const workers: Array<{ stop(): Promise }> = []; beforeAll(async () => { - if (external) { await applyPendingMigrations(external); db = createDb(external); } - else { database = await startEmbeddedPostgresTestDatabase("paperclip-verification-retry-"); db = createDb(database.connectionString); } + if (external) { await applyPendingMigrations(external); db = createDb(external, { maxConnections: 6 }); } + else { database = await startEmbeddedPostgresTestDatabase("paperclip-verification-retry-"); db = createDb(database.connectionString, { maxConnections: 6 }); } }, 90_000); afterAll(async () => { if (external) await closeRegisteredClients(external); else await database?.cleanup(); }); beforeEach(async () => { @@ -76,12 +76,22 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu it("permits an explicit retry of the same operation and completes verification before readiness", async () => { const { agent, work, runHost } = await failed(); - runHost.mockResolvedValue("complete"); - await createAgentLifecycle(db).retry(agent.id); - expect((await current(agent.id)).lifecycleOperation).not.toHaveProperty("retryRequired"); - expect((await current(agent.id)).lifecycleOperation).not.toHaveProperty("verificationFailures"); - expect((await current(agent.id)).lifecycleOperation!.id).toBe(agent.lifecycleOperation!.id); - await work.process(agent.id); + let release!: () => void; + const verification = new Promise(resolve => { release = resolve; }); + runHost.mockImplementation(async () => { await verification; return "complete"; }); + let processing: Promise | undefined; + try { + await createAgentLifecycle(db).retry(agent.id); + processing = work.process(agent.id); + const retried = await current(agent.id); + expect(retried).toMatchObject({ lifecycleState: "verifying", status: "paused" }); + expect(retried.lifecycleOperation).not.toHaveProperty("retryRequired"); + expect(retried.lifecycleOperation).not.toHaveProperty("verificationFailures"); + expect(retried.lifecycleOperation!.id).toBe(agent.lifecycleOperation!.id); + } finally { + release(); + await processing; + } expect(await current(agent.id)).toMatchObject({ lifecycleState: "ready", status: "idle", lifecycleError: null }); });