diff --git a/doc/native-workspace-finalization-recovery.md b/doc/native-workspace-finalization-recovery.md index 4ab1801bced..2a56876bbf1 100644 --- a/doc/native-workspace-finalization-recovery.md +++ b/doc/native-workspace-finalization-recovery.md @@ -454,6 +454,10 @@ delay that sweep until the claim expires. Neither an unconfirmed stop nor a stale receipt grants admission, and a changed lease or competing sandbox owner prevents cleanup from taking ownership. Retry export after the lease has a new confirmed stopped receipt. No provider turn is created by this recovery. +Once recovery has saved the files and queued destruction of the stopped sandbox, +the cleanup worker admits its first attempt without the five-minute retry delay. +The shared worker spacing and any in-flight claim deadline still apply. A failed +first attempt keeps the existing retry delay, attempt limit and ownership checks. Cleanup and its readiness probe use the plugin ID recorded on that lease. Another plugin with the same provider name cannot take over; an unavailable original plugin defers cleanup without consuming an attempt. Stop-only cleanup requires the exact plugin worker to advertise `environmentStopLease`. Older plugins without that hook receive neither release nor destroy; resume admission also defers until safe compensation is available. diff --git a/server/src/__tests__/environment-cleanup-work.test.ts b/server/src/__tests__/environment-cleanup-work.test.ts index d7bec03701c..071576b2a10 100644 --- a/server/src/__tests__/environment-cleanup-work.test.ts +++ b/server/src/__tests__/environment-cleanup-work.test.ts @@ -10,7 +10,7 @@ import { idleWorkSnapshot } from "../services/task-admission.js"; import { randomUUID } from "node:crypto"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { eq, sql } from "drizzle-orm"; -import { companies, createDb, sandboxCreationOperations, environmentLeases, environments, type Db } from "@paperclipai/db"; +import { agents, heartbeatRuns, companies, createDb, sandboxCreationOperations, environmentLeases, environments, type Db } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; @@ -20,6 +20,9 @@ import { environmentService } from "../services/environments.js"; import { historicalCleanupVerificationService } from "../services/historical-cleanup-verification.js"; import { heartbeatService } from "../services/heartbeat.js"; import { environmentRuntimeService } from "../services/environment-runtime.js"; +import { writeEnvironmentCleanup } from "../services/environment-cleanup-notifications.js"; +import { remoteTerminationReceipt } from "../services/remote-execution-termination.js"; +import { agentExecutionsHaveStopped } from "../services/agent-execution-stop.js"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; @@ -364,6 +367,52 @@ describe.skipIf(!support.supported)("environment cleanup scheduling", () => { expect((await environmentService(db).getLeaseById(lease.id))?.status).toBe("expired"); await quiet(); }); + async function recoveredWorkspaceCleanup() { + const [agent] = await db.insert(agents).values({ companyId, name: "Stopped workspace owner" }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: agent.id, status: "cancelled", + runtimeMode: "native", invocationSource: "on_demand", startedAt: new Date(Date.now() - 60_000), finishedAt: new Date() }).returning(); + const lease = { id: randomUUID(), companyId, environmentId, heartbeatRunId: run.id, provider: "fixture", + providerLeaseId: randomUUID(), leasePolicy: "ephemeral", status: "pending_cleanup", cleanupStatus: "failed", + releasedAt: new Date(), updatedAt: new Date() }; + const metadata = { driver: "sandbox", workspaceRecoveryCleanup: { runId: run.id, snapshotId: randomUUID(), epoch: 1, + stopReceipt: remoteTerminationReceipt(lease, { providerLeaseId: lease.providerLeaseId, state: "stopped" }) }, + pendingCleanupRetryAttempts: 0, pendingCleanupInFlight: false, pendingCleanupRetryAfterMs: 0 }; + await writeEnvironmentCleanup(db, tx => tx.insert(environmentLeases).values({ ...lease, metadata }), { companyId, runId: run.id }); + return { agent, run, lease, metadata }; + } + it("destroys a recovered workspace on its first scheduled pass, before agent stop can finish", async () => { + const destroy = vi.fn(async ({ lease }: { lease: { providerLeaseId: string } }) => + ({ providerLeaseId: lease.providerLeaseId, state: "destroyed" as const })); + const runtime = environmentRuntimeService(db, { drivers: [{ driver: "sandbox", retryPendingSandboxTeardown: destroy } as any] }); + const heartbeat = heartbeatService(db, { environmentRuntime: runtime }); + await start(false, backoffMs => heartbeat.sweepPendingCleanupLeases({ backoffMs })); await quiet(); + draining = true; + const { agent, lease } = await recoveredWorkspaceCleanup(); + expect(await nextEnvironmentCleanupAt(db)).toBeLessThanOrEqual(Date.now()); + expect(await agentExecutionsHaveStopped(db, [agent.id])).toBe(false); + draining = false; + await settle(); + expect(destroy).toHaveBeenCalledTimes(1); + expect((await environmentService(db).getLeaseById(lease.id))?.status).toBe("expired"); + expect(await agentExecutionsHaveStopped(db, [agent.id])).toBe(true); + await quiet(); + }); + it("keeps normal retry backoff after a recovered workspace's first teardown fails", async () => { + await start(); await quiet(); + failing = true; + const { lease } = await recoveredWorkspaceCleanup(); + await settle(); + expect(releases).toHaveBeenCalledTimes(1); + expect((await environmentService(db).getLeaseById(lease.id))?.status).toBe("pending_cleanup"); + expect(await nextEnvironmentCleanupAt(db)).toBeGreaterThan(Date.now() + 240_000); + await settle(60_000); + expect(releases).toHaveBeenCalledTimes(1); + failing = false; + await settle(240_000); + expect(releases).toHaveBeenCalledTimes(2); + expect((await environmentService(db).getLeaseById(lease.id))?.status).toBe("expired"); + await quiet(); + }); it("does not rescan pending cleanup from an otherwise idle run-reaper pass", async () => { const lease = await pending(); await db.update(environmentLeases).set({ updatedAt: new Date(Date.now() - 600_000) }).where(eq(environmentLeases.id, lease.id)); diff --git a/server/src/services/environment-cleanup-state.ts b/server/src/services/environment-cleanup-state.ts index d7c8aefc405..e0a31303680 100644 --- a/server/src/services/environment-cleanup-state.ts +++ b/server/src/services/environment-cleanup-state.ts @@ -57,16 +57,11 @@ export function recoverableSandboxCreationCondition() { )); } -/** A run lease the ownership settle moved to cleanup (an orphaned active - * lease, or a retained orphan sandbox nothing references any more), or a warm - * sandbox whose warm window ended, before its first cleanup attempt. Its - * holder is proven gone, so the first stop or teardown is - * due now regardless of how recently the lease was used; a failed attempt - * counts and the normal retry backoff applies from then on. */ -export function recoveredOrphanFirstAttemptSql() { +export function recoveredCleanupFirstAttemptSql() { return sql`(${environmentLeases.failureReason} in ('orphaned_active_lease_recovered', 'retained_orphan_sandbox_released') or ${environmentLeases.metadata}->'sandboxStopAndRetain'->>'recoveredOrphanedRunLease' = 'true' - or ${environmentLeases.metadata}->'sandboxStopAndRetain'->>'warmWindowExpired' = 'true') + or ${environmentLeases.metadata}->'sandboxStopAndRetain'->>'warmWindowExpired' = 'true' + or ${environmentLeases.metadata}->'workspaceRecoveryCleanup'->>'runId' = ${environmentLeases.heartbeatRunId}::text) and coalesce(${environmentLeases.metadata}->>'pendingCleanupRetryAttempts', '0') = '0' and coalesce(${environmentLeases.metadata}->>'pendingCleanupInFlight', 'false') <> 'true'`; } @@ -84,7 +79,7 @@ export function pendingCleanupDeadlineSql(now = Date.now()) { } export async function nextEnvironmentCleanupAt(db: Db, backoffMs = ENVIRONMENT_CLEANUP_RETRY_MS): Promise { - const [row] = await db.select({ next: sql`min(case when ${recoveredOrphanFirstAttemptSql()} + const [row] = await db.select({ next: sql`min(case when ${recoveredCleanupFirstAttemptSql()} then ${pendingCleanupDeadlineSql()} else greatest( extract(epoch from ${environmentLeases.updatedAt}) * 1000 + ${backoffMs}, ${pendingCleanupDeadlineSql()}) end)` }).from(environmentLeases).where(eq(environmentLeases.status, "pending_cleanup")); diff --git a/server/src/services/heartbeat/recovery.ts b/server/src/services/heartbeat/recovery.ts index 932a5376f95..186f855e7e7 100644 --- a/server/src/services/heartbeat/recovery.ts +++ b/server/src/services/heartbeat/recovery.ts @@ -21,7 +21,7 @@ import { pendingCleanupAttemptsSql, pendingCleanupDeadlineSql, pendingCleanupMetadataObjectSql, - recoveredOrphanFirstAttemptSql, + recoveredCleanupFirstAttemptSql, } from "../environment-cleanup-state.js"; import { currentControllerProcess, readControllerProcess } from "../controller-process-identity.js"; import { writeEnvironmentCleanup } from "../environment-cleanup-notifications.js"; @@ -1872,7 +1872,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD opts?.explicitRetry ? eq(environmentLeases.heartbeatRunId, opts.explicitRetry.runId) : undefined, opts?.environmentRetry ? eq(environmentLeases.environmentId, opts.environmentRetry.environmentId) : undefined, pendingCleanupRetryDueSql(manualRetry), - backoffMs > 0 ? or(lte(environmentLeases.updatedAt, cutoff), recoveredOrphanFirstAttemptSql()) : undefined, + backoffMs > 0 ? or(lte(environmentLeases.updatedAt, cutoff), recoveredCleanupFirstAttemptSql()) : undefined, ), ) .orderBy(asc(environmentLeases.updatedAt)) diff --git a/server/src/services/recovery/ownership-registry.ts b/server/src/services/recovery/ownership-registry.ts index 66c5f07b43f..66c768dc06a 100644 --- a/server/src/services/recovery/ownership-registry.ts +++ b/server/src/services/recovery/ownership-registry.ts @@ -62,7 +62,7 @@ import { ENVIRONMENT_LEASE_CONTROLLER_KEY } from "../environment-lease-controlle import { ENVIRONMENT_CLEANUP_RETRY_MS, pendingCleanupDeadlineSql, - recoveredOrphanFirstAttemptSql, + recoveredCleanupFirstAttemptSql, } from "../environment-cleanup-state.js"; import { settleEnvironmentLoginProbeLease } from "../environment-login-probe-lease-settlement.js"; import { settleEnvironmentRunLease } from "../environment-run-lease-settlement.js"; @@ -515,7 +515,7 @@ function pendingCleanupKind(): OwnableKind { // by the settle; a row from before that is due since its last write). // Built per listing: the deadline reader bounds saved deadlines relative to // the current time, so it must not be frozen at module load. - const dueMs = () => sql`case when ${recoveredOrphanFirstAttemptSql()} + const dueMs = () => sql`case when ${recoveredCleanupFirstAttemptSql()} then coalesce(case when jsonb_typeof(${environmentLeases.metadata}->'pendingCleanupRecoveredAtMs') = 'number' then (${environmentLeases.metadata}->>'pendingCleanupRecoveredAtMs')::numeric end, extract(epoch from ${environmentLeases.updatedAt}) * 1000) diff --git a/tests/runner-e2e/PRODUCT-JOURNEYS.md b/tests/runner-e2e/PRODUCT-JOURNEYS.md index eec3849f077..abe5558510c 100644 --- a/tests/runner-e2e/PRODUCT-JOURNEYS.md +++ b/tests/runner-e2e/PRODUCT-JOURNEYS.md @@ -461,6 +461,12 @@ not evidence that its Cloud journey passed. ## Reuse and daily QA promotion +Task property and document journeys open the current side panel or the classic +Properties control before they act. Property reads exclude inert and accessibility-hidden +panels, select the Properties tab when needed, and check the row in the viewport. +Document navigation waits for the available authoring controls after each reload. +These browser regressions do not establish a new deployed QA pass. + The classic matrix and deployment-targeted stories share Plan revision actions, artifact download/oracle code, Skill Studio editing and protected-action decisions. Their target adapters own login, agent/connection provisioning, environment diff --git a/tests/runner-e2e/current-ui-controls.spec.ts b/tests/runner-e2e/current-ui-controls.spec.ts index 1071f312a60..c904acfe790 100644 --- a/tests/runner-e2e/current-ui-controls.spec.ts +++ b/tests/runner-e2e/current-ui-controls.spec.ts @@ -1,9 +1,51 @@ import { expect, test } from "@playwright/test"; import { newProjectButton } from "./product-journeys/project-context-flow.js"; import { storyArtifactDownloadLink } from "./story-artifact.js"; +import { taskProperties } from "./product-journeys/task-properties.js"; +import { showPropertyRow } from "./product-journeys/task-panel.js"; +import type { JourneyContext, JourneyEvidence, TargetSession } from "./product-journeys/types.js"; const id = "11111111-2222-4333-8444-555555555555"; +test("property reads select Properties from an open panel and exclude inactive duplicate rows", async ({ page }) => { + const row = (value: string) => `
Project${value}
`; + await page.setContent(` + `); + const property = await showPropertyRow(page, "Project"); + await expect(property.locator("[data-property-value]")).toHaveText("Saved project"); +}); + +test("property reads cannot use an inert row when the panel has no open control", async ({ page }) => { + await page.setContent(''); + await expect(showPropertyRow(page, "Project")).rejects.toThrow(); +}); + +test("assignment opens the current side panel again after reload and verifies the saved user", async ({ page }) => { + const origin = "https://assignment-controls.test", title = "Owned assignment task", userId = "owned-user"; + let assigned = false; + await page.route(`${origin}/**`, async route => { + const pathname = new URL(route.request().url()).pathname; + if (pathname === "/assign") { + assigned = true; + await route.fulfill({ json: {} }); + return; + } + await route.fulfill({ contentType: "text/html", body: pathname === "/QA/issues" + ? `${title}` + : ` + + ` }); + }); + const context = { page, company: { issuePrefix: "QA" }, user: { id: userId }, capabilities: ["company", "board-user", "task-write"], + fixtures: { backlogTask: { id, title, identifier: "QA-201" } }, api: { origin, + get: async (route: string) => route.endsWith("/runs") ? [] : { assigneeUserId: assigned ? userId : null, assigneeAgentId: null, status: "backlog" } }, + } as unknown as JourneyContext; + const evidence: JourneyEvidence = { checks: [], outcome: "passed", startingState: "recipe", phases: [], resourceDisposition: "released", privacy: "private" }; + await taskProperties(context, {} as TargetSession, "owned-campaign", evidence); + expect(evidence.checks).toEqual([{ id: "assignment-persisted", passed: true }, { id: "no-agent-work", passed: true }]); + await expect(page.getByRole("button", { name: "You", exact: true })).toBeVisible(); +}); + test("finds the attachment download link in legacy and Artifacts V1 markup only", async ({ page }) => { const markup = { legacy: `Download`, diff --git a/tests/runner-e2e/product-journeys/task-panel.ts b/tests/runner-e2e/product-journeys/task-panel.ts index 7285dc2d47a..bbf80306b11 100644 --- a/tests/runner-e2e/product-journeys/task-panel.ts +++ b/tests/runner-e2e/product-journeys/task-panel.ts @@ -19,11 +19,14 @@ export async function openTaskPanelTab(page: Page, tabName: "Properties" | "Arti for (let step = 0; step < 4; step++) { await expect.poll(() => anyVisible(ready, tab, option, newTab, toggle, opener)).toBe(true); if (await ready.isVisible()) return; - if (await option.isVisible()) await option.click(); - else if (await tab.isVisible()) await tab.click(); - else if (await newTab.isVisible()) await newTab.click(); - else if (await toggle.isVisible()) await toggle.click(); - else await opener.click(); + if (await option.isVisible()) { await option.click(); await expect(ready).toBeVisible(); } + else if (await tab.isVisible()) { await tab.click(); await expect(ready).toBeVisible(); } + else if (await newTab.isVisible()) { await newTab.click(); await expect(option).toBeVisible(); } + else { + if (await toggle.isVisible()) await toggle.click(); + else await opener.click(); + await expect.poll(() => anyVisible(ready, tab, option, newTab)).toBe(true); + } } await expect(ready).toBeVisible(); } diff --git a/tests/runner-e2e/project-context-readback.spec.ts b/tests/runner-e2e/project-context-readback.spec.ts index fb04df62bea..255f0236306 100644 --- a/tests/runner-e2e/project-context-readback.spec.ts +++ b/tests/runner-e2e/project-context-readback.spec.ts @@ -7,7 +7,7 @@ const projectPath = "/QA/projects/owned-project-f962db91/issues", taskPath = "/Q const projectInput = { origin, issuePrefix: "QA", projectName, projectRef: "owned-project-f962db91" }; const taskInput = { origin, issuePrefix: "QA", projectName, taskTitle, prompt, taskRef: "QA-201" }; -async function fixture(page: Page, changes: { missingProject?: boolean; wrongProject?: boolean; missingTask?: boolean; wrongTask?: boolean; wrongProperty?: boolean } = {}) { +async function fixture(page: Page, changes: { missingProject?: boolean; wrongProject?: boolean; missingTask?: boolean; wrongTask?: boolean; wrongProperty?: boolean; sidePanel?: boolean } = {}) { const requests: string[] = []; await page.route(`${origin}/**`, async route => { const path = new URL(route.request().url()).pathname; @@ -15,9 +15,13 @@ async function fixture(page: Page, changes: { missingProject?: boolean; wrongPro // Layout can retain a hidden sidebar label. ProjectDetail renders its // production InlineEditor as h2; IssueRow has an overlay link with sr-only text. const sidebar = ``; + const propertyValue = changes.wrongProperty ? "Other project" : projectName; + const panel = changes.sidePanel + ? `` + : ``; const body = path === "/QA/projects" ? `${sidebar}
${changes.missingProject ? "" : `${projectName}`}
` : path.includes("/projects/") ? `${sidebar}

${projectName}

${changes.missingTask ? "" : ``}
` - : `${sidebar}

${taskTitle}

${prompt}

`; + : `${sidebar}

${taskTitle}

${prompt}

${panel}
`; await route.fulfill({ contentType: "text/html", body: `${body}` }); }); return requests; @@ -32,6 +36,13 @@ test("reopens and reloads the owned project and task despite hidden duplicate si await expect(page.locator("main#main-content [data-property-value]")).toContainText(projectName); }); +test("opens the current collapsed inert side panel before reading the saved project", async ({ page }) => { + await fixture(page, { sidePanel: true }); + await page.goto(`${origin}${projectPath}`); + await openProjectTaskThroughList({ ...taskInput, page }); + await expect(page.getByRole("button", { name: "Toggle side panel" })).toHaveAttribute("aria-pressed", "true"); +}); + for (const [name, changes, code] of [ ["missing project list row", { missingProject: true }, "project_context_project_open_failed"], ["wrong project route with a matching label", { wrongProject: true }, "project_context_project_open_failed"], diff --git a/tests/runner-e2e/task-document-navigation.spec.ts b/tests/runner-e2e/task-document-navigation.spec.ts index 8571290d779..4587da053fa 100644 --- a/tests/runner-e2e/task-document-navigation.spec.ts +++ b/tests/runner-e2e/task-document-navigation.spec.ts @@ -2,7 +2,7 @@ import { expect, test } from "@playwright/test"; import { reopenTaskDocument } from "./product-journeys/task-document-flow.js"; import type { JourneyContext } from "./product-journeys/types.js"; -for (const layout of ["collapsed", "artifacts", "launcher", "classic"] as const) { +for (const layout of ["collapsed", "side-panel", "artifacts", "launcher", "classic"] as const) { test(`document reopen waits for the ${layout} task controls after reload`, async ({ page }) => { const issue = { title: "Journey QA document navigation" }; let taskLoads = 0; @@ -20,7 +20,7 @@ for (const layout of ["collapsed", "artifacts", "launcher", "classic"] as const) controls.querySelector('button').onclick = ${layout === "artifacts" ? "authoring" : `() => { controls.innerHTML = '
Artifacts
'; controls.querySelector('[role="option"]').onclick = authoring; }`}; } setTimeout(() => { - ${layout === "classic" ? "authoring();" : layout === "collapsed" ? `controls.innerHTML = ''; controls.querySelector('button').onclick = () => setTimeout(panel, 100);` : "panel();"} + ${layout === "classic" ? "authoring();" : layout === "collapsed" || layout === "side-panel" ? `controls.innerHTML = ${JSON.stringify(layout === "side-panel" ? '' : '')}; controls.querySelector('button').onclick = () => setTimeout(panel, 100);` : "panel();"} }, 200); ` }); });