diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 1228bb99dea..8269a7f4589 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -749,7 +749,7 @@ The handshake failure code is distinct from a session-identity mismatch. A timeo An explicit recovery action is a typed liveness repair path for a source issue. It is the recovery primitive; the action can be rendered directly on the source issue or backed by a separate recovery issue when the repair needs its own work item. -A terminal native failure can retain a result accepted before checkpoint or cleanup failed. That result is historical evidence, not a live controller. New user input may start a fresh turn after the controller and execution environment have stopped and ordinary admission checks pass. Preserve the failed run, its result, and its recovery budget. Do not commit the old result, infer action outcomes, or replay the failed turn. A message saved while cleanup is pending must be reconsidered after verified cleanup and delivered once. Admission must consume its deferred receipt in the same transaction that creates the new run, including in agent chat, where later messages keep their separate turns. Completing that run must not promote the consumed receipt again. Workspace-export repair retains its separate saved-result recovery path. +A terminal native failure can retain a result accepted before checkpoint or cleanup failed. That result is historical evidence, not a live controller. New user input may start a fresh turn after the controller and execution environment have stopped and ordinary admission checks pass. Preserve the failed run, its result, and its recovery budget. Do not commit the old result, infer action outcomes, or replay the failed turn. A message saved while cleanup is pending must be reconsidered after verified cleanup and delivered once. External chat messages retain their original inbound action, wake receipt and actor. Cleanup delegates reconsideration to the live inbound owner, which revalidates the endpoint, conversation generation, principal and exact saved source before normal admission. The existing stranded-queue sweep uses the same owner after a lost callback or restart; it cannot substitute a generic assignment wake. A rejected source must not prevent other saved messages from being reconsidered. Admission must consume its deferred receipt in the same transaction that creates the new run, including in agent chat, where later messages keep their separate turns. Completing that run must not promote the consumed receipt again. Workspace-export repair retains its separate saved-result recovery path. When a legacy turn admitted by an explicit user message fails or times out, a bounded transient retry needs its own durable continuation authorization. Initial admission binds the message's exact body digest and revision. Scheduling requires that original binding and revalidates the human author, source task, assignee, and execution ownership, then records the successor's receipt in the same transaction as its run. Historical receipts without a message binding cannot authorize automatic retries and are not backfilled. The parent receipt remains intact. Dispatch checks the exact successor, retry parent, and unchanged message binding again. Deleted or edited input, changed actors or ownership, superseded work, and duplicate successors cannot renew authority. One-run Retry and Interrupt intents retain their separate admission contracts; they are not renewable user-message authority. Other retry policies cannot borrow this receipt: their scheduling fails closed. Successful explicit turns do not create a prose-only missing-comment follow-up with spent authority. @@ -1444,7 +1444,7 @@ Real gates still apply: company and task ownership, active provider ownership, b A source that already stopped and was released remains fenced until its export settles; restart reconciliation scans these released sources too. Both provider reuse and destruction check the physical allocation for a shutdown fence or file-repair pin, including historical lease rows. -An operator Stop waits for provider termination. Remote sandbox providers may return a stopped/deleted receipt after their control-plane operation completes. Paperclip binds that receipt to the company, run, and exact lease; successful file cleanup, a terminal run row, or an in-sandbox shutdown event is not sufficient. Legacy conversational runs receive their cancellation acknowledgement after all remote leases have confirmed termination. Stop alone never creates a continuation. A user message queued during remote cleanup is reconsidered when the provider confirms termination; it still passes normal admission and adopts pending comment IDs in order. Once stopped, the next explicit wake uses the same queue. A compatible saved ACP session can resume, and an unavailable or incompatible session can start fresh with the full task context. Run credentials and scratch paths remain scoped to the new run. A subtree pause requires Resume; a message does not bypass it. +An operator Stop waits for provider termination. Remote sandbox providers may return a stopped/deleted receipt after their control-plane operation completes. Paperclip binds that receipt to the company, run, and exact lease; successful file cleanup, a terminal run row, or an in-sandbox shutdown event is not sufficient. Legacy conversational runs and native runs with an audited run-only Stop receive their shared cancellation acknowledgement after all remote leases have confirmed termination. A native interrupt acknowledgement alone does not establish containment. Workspace preservation remains a separate obligation and its recovery hold survives this acknowledgement. Stop alone never creates a continuation. A user message queued during remote cleanup is reconsidered when the provider confirms termination; it still passes normal admission and adopts pending comment IDs in order. Once stopped, the next explicit wake uses the same queue. A compatible saved ACP session can resume, and an unavailable or incompatible session can start fresh with the full task context. Run credentials and scratch paths remain scoped to the new run. A subtree pause requires Resume; a message does not bypass it. Cancelled runs and runs with a recorded pending stop lose run-scoped API write authority, while reads remain available for diagnostics. This applies to ordinary tasks as well as conversations. Task and interaction mutations recheck that authority in the write transaction, so a cancelled run cannot overwrite a recovery disposition with a late Done or resolve an interaction after revocation. A task handoff that intentionally stops its own run can commit only with a server receipt tied to that exact request. Direct sandbox CLI adapters such as Grok register cancellation before preparation and keep ownership until host-owned termination is verified, including a sandbox acquired before adapter registration. A failed stop does not acknowledge cancellation or abandon an outstanding remote command. Interrupted workspace restore failures remain recorded for recovery; a stop receipt proves termination, not successful file restoration. @@ -2226,3 +2226,33 @@ use the same activity projection. Board-owned repairs stay actionable until a verified native retry actually starts. Once an explicit board retry is running, its live activity takes precedence over the prior owner and exhausted budget. Resolved and cancelled actions remain resolved. + + +### Task directories after a fresh retry + +A fresh repository-free retry receives its own private attempt directory. Once +that attempt succeeds, later ordinary input reuses the task's currently bound +directory, including an attempt directory, and preserves its saved files. The +binding is re-read before preparation. Only the exact company's task namespace +is eligible; closed or changed bindings, foreign paths, symlinks and missing +retained directories fail validation. A later explicitly fresh retry still +receives a separate clean sibling and does not copy or replay the old turn. + +### Stopped execution workspace recovery instructions + +A stopped run can retain a historical workspace repair instruction after machine +recovery has saved the files. Its execution projection uses the workspace +recovery state bound to that exact run for the next instruction, as the task +blocker does. Recovery in progress, a failed repair, and recovered files must +remain distinct. This read-only instruction does not release the replay hold, +change the stopped run, or authorize another provider turn. A recorded explicit +successor still takes precedence over the old recovery instruction. + + +A terminal native run can leave an older exact-session retry action behind when +workspace recovery takes ownership. The existing execution disposition worker +retires that action only after the exact run and coordinator are terminal, no +controller lease, result, successor or retry remains, and a separate source-bound +workspace hold records that uncertain actions cannot replay. The worker keeps +that hold and all saved work. It does not change the task or start a provider. +The same bounded reconciliation pass handles historical actions after restart. diff --git a/maps/agent-access-map/agents.md b/maps/agent-access-map/agents.md index f091a126d4e..dd89f5abef0 100644 --- a/maps/agent-access-map/agents.md +++ b/maps/agent-access-map/agents.md @@ -123,7 +123,7 @@ API: `present` — representative operation IDs: receives `canManage: false`. Saving never creates a run or wakeup. - `PUT /api/agents/{id}/instructions-bundle/file`. - `PATCH /api/agents/{id}` with an explicit `runner: "paperclip"` upgrade. -- `POST /api/agents/{id}/qualify-runner` to qualify the saved effective sandbox before upgrading. +- `POST /api/agents/{id}/qualify-runner` to qualify the saved effective sandbox before upgrading or testing an existing native runner. - `PATCH /api/agents/{id}` with `runtimeSettings: { harness, model?, aiConnection?, expected: { model, aiConnection } }` for a narrow roster edit. Each request handles one agent; the UI batches at most four writes and collects per-agent outcomes. Agent access: `available` — Native read/update/history/restore instruction tools; REST / @@ -149,7 +149,11 @@ authorization as saving the upgrade. The server resolves the saved computer; `expectedEnvironmentId` rejects stale state and cannot select another computer. Qualification checks company binding, returns only its receipt, and audits within the agent's company. General instance environment probes still require instance -admin. No probe runs merely on page load. Operator browser evidence does not +admin. An existing native agent's clean configuration Test uses the same +saved-computer qualification before its runtime test. A current proof is reused; +a missing or stale proof requires a real successful probe and verified cleanup. +Draft settings and target changes do not qualify another computer. No probe runs +merely on page load. Operator browser evidence does not qualify the agent permission paths for this operation. Evidence: `source-only` — [implementation](../../server/src/routes/agents.ts#L1395) at diff --git a/maps/agent-access-map/workspaces.md b/maps/agent-access-map/workspaces.md index 5b501668c9f..2dc6ba5ad8b 100644 --- a/maps/agent-access-map/workspaces.md +++ b/maps/agent-access-map/workspaces.md @@ -49,6 +49,12 @@ those APIs with their admitted run token. These paths do not require a project. Policy: Company/project/workspace visibility and runtime authority; active run, trust preset and task ownership where required. +Ordinary input may reuse the exact currently bound private task directory after +an explicitly fresh retry. Preparation rechecks company, source task, current +binding and canonical path; missing directories, symlinks and foreign roots are +rejected. This preserves files without granting new access or replaying the +previous turn. An explicitly fresh retry still creates a clean sibling. + Task workspace writes require `issue:mutate`, an active owned task/run for agents, and current source visibility. Shared physical roots also require source-project workspace write authority. Accepting a new pending choice advances its binding diff --git a/maps/feature-map/agents.md b/maps/feature-map/agents.md index c6e9c336570..21dee0c054b 100644 --- a/maps/feature-map/agents.md +++ b/maps/feature-map/agents.md @@ -185,6 +185,16 @@ the original agent, without showing success on the new page or redirecting back. Focused coverage: [save receipt](../../ui/src/pages/AgentRunnerUpgrade.test.tsx) and [configuration form](../../ui/src/components/AgentConfigForm.render.test.tsx). +For an existing native agent on a Daytona snapshot or immutable sandbox image, +use **Test** in its saved Harness or Runtime configuration after a managed +computer update. This action qualifies the saved computer under the existing +agent configuration permission before testing the runtime. A current proof must +remain unchanged. A missing proof requires a real successful probe and verified +cleanup. Failed qualification must stop the runtime test. Unsaved configuration +or a different computer must not qualify a draft target. A company owner must not +need instance-admin permission, and the generic environment probe must remain +restricted. Existing runs retain their recorded allocation and snapshot binding. + Storybook: [Runner upgrade](../../ui/storybook/stories/agents/codex-runner/upgrade.stories.tsx) mounts the production page and shell with mutable HTTP fixtures for both harnesses, pending, failure/retry, incompatible settings, repair, native/unsupported states, diff --git a/maps/feature-map/recovery.md b/maps/feature-map/recovery.md index 03e16003337..fcf47da1e27 100644 --- a/maps/feature-map/recovery.md +++ b/maps/feature-map/recovery.md @@ -150,7 +150,7 @@ Use these checks alongside the entry-point recipes below and the operation recor | Capability ID | Action and persisted result | Denial, race and retry checks | | --- | --- | --- | -| `recovery/diagnosis` | Open the source task and exact run; compare cause, source action, original owner, recovery owner, attempt bounds and next action after reload. | Test an unrelated/private task and a stale run notice. Read-side retirement of a stale action must not create a new attempt, participant, or permission to replay. | +| `recovery/diagnosis` | Open the source task and exact run; compare cause, source action, original owner, recovery owner, attempt bounds and next action after reload. | Test an unrelated/private task and a stale run notice. Read-side retirement of a stale action must not create a new attempt, participant, or permission to replay. After workspace recovery, compare task and run instructions; recovered files must not leave instructions to repair a retained sandbox. The stopped run keeps its replay hold until continuation admission. An obsolete exact-session notice retires only after terminal coordinator proof and a separate exact-run workspace hold; no new execution or file cleanup follows. | | `recovery/bounded-continuation` | Use a deterministic recoverable interruption and inspect the original run/session, attempt receipts, saved result and eventual useful output. | Repeat restart/receipt delivery, race Stop or a new owner, and retain unknown external outcomes. Accepted completion/finalization recovery must not start a new provider turn or repeat its side effect. | | `recovery/source-action` | Inspect active and retained actions, follow the repair owner, and resolve one eligible source-bound action with its exact ID. | Retry the settled resolution and reject a stale/foreign action. Recovery ownership alone cannot mutate the source task; inspect source status, audit and wake independently. | | `recovery/exhaustion` | Exhaust or age a bounded fixture; use the currently offered retry once, then inspect its durable acknowledgement and actual continuation. | Reassign, pause, stop, close, add an unresolved dependency, or impose a budget hold before retry. The old notice must not reopen terminal work or run the wrong owner. | diff --git a/maps/feature-map/workspaces.md b/maps/feature-map/workspaces.md index 0286edafbf9..e82349ce27c 100644 --- a/maps/feature-map/workspaces.md +++ b/maps/feature-map/workspaces.md @@ -57,6 +57,15 @@ Automated: [workspace card](../../ui/src/components/IssueWorkspaceCard.test.tsx) Manual: Run a task that writes a small file, follow the bound workspace, and inspect the file and revision. Reload and verify links still refer to the same workspace, not another checkout or a stale run. +For a repository-free isolated task, complete an explicitly fresh retry, save a +marker, then send ordinary new input. It must retain the bound attempt directory +and marker. A later explicitly fresh retry must use a separate clean sibling. +The [production preparation regression](../../server/src/services/heartbeat/workspace-preparation.test.ts) +uses PostgreSQL and real private directories; it also rejects changed bindings. +The [directory boundary suite](../../server/src/services/isolated-task-directory.test.ts) +checks namespace, missing-directory and symlink refusal. Neither suite proves a +hosted provider turn. + For slow source preparation, configure a short PostgreSQL idle-transaction timeout in a disposable instance. Hold preparation beyond that interval and verify it retains its source lock, rejects competing preparation, and leaves diff --git a/server/src/__tests__/agent-runner-qualification-routes.test.ts b/server/src/__tests__/agent-runner-qualification-routes.test.ts index 3d50ad7d6af..10c1980d248 100644 --- a/server/src/__tests__/agent-runner-qualification-routes.test.ts +++ b/server/src/__tests__/agent-runner-qualification-routes.test.ts @@ -9,6 +9,9 @@ import { grantsForHumanRole } from "../services/company-member-roles.js"; import { agentRoutes } from "../routes/agents.js"; import { environmentRoutes } from "../routes/environments.js"; import { errorHandler } from "../middleware/error-handler.js"; +import { environmentService } from "../services/environments.js"; +import { resolveEnvironmentDriverConfigForRuntime } from "../services/environment-config.js"; +import { fingerprintEnvironmentSandboxProviderConfig, SANDBOX_PLATFORM_PROBE_METADATA_KEY } from "../services/environment-custom-image-runtime.js"; const probe = vi.hoisted(() => vi.fn()); vi.mock("../services/environment-probe.js", async original => ({ @@ -69,6 +72,53 @@ describe.skipIf(!support.supported)("agent-scoped runner qualification", () => { expect(generic.status).toBe(403); expect(probe).toHaveBeenCalledTimes(1); }); + it("lets the owner requalify an existing native runner's saved computer without changing its configuration", async () => { + await db.update(agents).set({ adapterType: "paperclip_runner", adapterConfig: { provider: "opencode", model: "example/model" } }) + .where(eq(agents.id, agentId)); + const before = await db.select().from(agents).where(eq(agents.id, agentId)); + const response = await qualify(); + expect(response.status).toBe(200); + expect(response.body).toMatchObject({ ok: true, details: { platformQualificationRecorded: true } }); + expect(probe).toHaveBeenCalledWith(db, expect.objectContaining({ id: environmentId }), + expect.objectContaining({ companyId, assertCompanyBinding: true, acquireSandboxRuntimeLease: true })); + expect(await db.select().from(agents).where(eq(agents.id, agentId))).toEqual(before); + expect(JSON.stringify(response.body)).not.toMatch(/provider-secret|private-id/); + expect((await request(app).post(`/api/environments/${environmentId}/probe?companyId=${companyId}`).send({})).status).toBe(403); + }); + + it("reuses a current native snapshot proof without invalidating it or allocating another probe", async () => { + await db.update(agents).set({ adapterType: "paperclip_runner", adapterConfig: { provider: "opencode" } }).where(eq(agents.id, agentId)); + const config = { provider: "daytona", snapshot: "managed-snapshot" }; + const fingerprint = fingerprintEnvironmentSandboxProviderConfig(config); + const saved = (await environmentService(db).getById(environmentId))!; + const runtime = await resolveEnvironmentDriverConfigForRuntime(db, companyId, saved, { applyCustomImageTemplate: true, readOnlyCustomImageTemplate: true }); + const runtimeFingerprint = fingerprintEnvironmentSandboxProviderConfig(runtime.config as typeof config); + const metadata = { [SANDBOX_PLATFORM_PROBE_METADATA_KEY]: { [companyId]: { + v: 2, companyId, baseConfigFingerprint: fingerprint, runtimeConfigFingerprint: runtimeFingerprint, + snapshot: config.snapshot, template: null, + snapshotIdentity: { provider: "daytona", requestedRef: config.snapshot, id: randomUUID(), name: config.snapshot, + organizationId: "fixture-organization", createdAt: new Date().toISOString() }, + facts: { platform: "linux", architecture: "x64", libc: "glibc", glibcVersion: "2.36" }, + } } }; + await db.update(environments).set({ metadata }).where(eq(environments.id, environmentId)); + expect((await qualify()).body).toMatchObject({ ok: true, details: { platformQualificationRecorded: true } }); + expect(probe).not.toHaveBeenCalled(); + expect((await db.select().from(environments).where(eq(environments.id, environmentId)))[0]?.metadata).toEqual(metadata); + }); + + it.each(["revoked", "changed-computer", "foreign-computer"])("keeps native saved-computer qualification closed for %s authority", async state => { + await db.update(agents).set({ adapterType: "paperclip_runner", adapterConfig: { provider: "opencode" } }).where(eq(agents.id, agentId)); + if (state === "revoked") await db.delete(principalPermissionGrants).where(eq(principalPermissionGrants.principalId, userId)); + if (state === "changed-computer") await db.update(agents).set({ defaultEnvironmentId: null }).where(eq(agents.id, agentId)); + if (state === "foreign-computer") { + const other = randomUUID(); + await db.insert(companies).values({ id: other, name: "Other", issuePrefix: `F${other.slice(0, 7)}` }); + await db.insert(builtInManagedResources).values({ companyId: other, bundleKey: "test", resourceKind: "environment", resourceKey: "computer", resourceId: environmentId, stockVersion: "test", stockHash: "test" }); + } + expect((await qualify()).status).toBe(state === "changed-computer" ? 409 : 403); + expect(probe).not.toHaveBeenCalled(); + }); + it("rejects an owner whose configure grant is absent before any provider work", async () => { await db.delete(principalPermissionGrants).where(eq(principalPermissionGrants.principalId, userId)); expect((await qualify()).status).toBe(403); diff --git a/server/src/__tests__/chat-channels.integration.test.ts b/server/src/__tests__/chat-channels.integration.test.ts index 82f108ec69e..347ec611be1 100644 --- a/server/src/__tests__/chat-channels.integration.test.ts +++ b/server/src/__tests__/chat-channels.integration.test.ts @@ -2421,6 +2421,121 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { await db.update(chatEndpoints).set({ botUsername: "gonzo[bot]", status: "active" }).where(eq(chatEndpoints.id, peer.id)); return peer; } + it.each(["callback", "restart", "race", "unknown_stop", "wrong_allocation", "paused_endpoint", "revoked_principal", "deleted_comment", "changed_generation", "foreign_scope", "changed_source", "revoked_before_valid"])( + "reconsiders the same GitHub reply after stopped native workspace cleanup (%s)", async scenario => { + const f = await subscribedGitHubFixture("github:paperclipai/paperclip:419"); + const issueId = f.conversation.issueId!; + const heartbeat = heartbeatService(db); + const sourceRunId = randomUUID(), leaseId = randomUUID(), providerLeaseId = randomUUID(); + const capacityRunId = randomUUID(), stoppedAt = new Date(Date.now() - 10_000); + const initial = (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.agentId, f.assignedAgentId)))[0]; + await db.update(agentWakeupRequests).set({ status: "completed", runId: sourceRunId, finishedAt: stoppedAt }) + .where(eq(agentWakeupRequests.id, initial.id)); + await db.update(companies).set({ defaultResponsibleUserId: "owner-user" }).where(eq(companies.id, f.companyId)); + await db.update(agents).set({ adapterType: "paperclip_runner", status: "running", + runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 } } }).where(eq(agents.id, f.assignedAgentId)); + await db.insert(heartbeatRuns).values({ id: capacityRunId, companyId: f.companyId, agentId: f.assignedAgentId, + status: "running", contextSnapshot: { issueId: randomUUID() } }); + runningProcesses.set(capacityRunId, { child: {} as never, graceSec: 0, processGroupId: null }); + onTestFinished(() => { runningProcesses.delete(capacityRunId); }); + await db.insert(heartbeatRuns).values({ id: sourceRunId, companyId: f.companyId, agentId: f.assignedAgentId, + status: "cancelled", startedAt: new Date(stoppedAt.getTime() - 1000), finishedAt: stoppedAt, + runtimeMode: "native", nativeIssueId: issueId, nativePhase: "observed", + contextSnapshot: { issueId, source: "chat:github", wakeCommentIds: [initial.payload?.commentId] }, + resultJson: { workspaceRestoreFailure: "restore_failed", workspaceRestoreRecovery: { leaseIds: [leaseId] } }, + }); + await db.insert(nativeRunFinalizations).values({ companyId: f.companyId, issueId, + runId: sourceRunId, phase: "terminal_failure" }); + await db.insert(environmentLeases).values({ id: leaseId, companyId: f.companyId, issueId, + heartbeatRunId: sourceRunId, provider: "daytona", providerLeaseId, + status: "pending_cleanup", cleanupStatus: "failed", leasePolicy: "ephemeral" }); + await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: issueId, + kind: "active_run_watchdog", status: "resolved", ownerType: "system", outcome: "blocked", + cause: "native_continuation_requires_reconciliation", fingerprint: randomUUID(), nextAction: "Retained workspace repair", + evidence: { runId: sourceRunId, automaticRecovery: { replay: "blocked" }, + workspaceRestoreFailure: "restore_failed", workspaceRestoreRecovery: { leaseIds: [leaseId] }, + workspaceRecovery: { schema: "paperclip.workspace-recovery.v1", runId: sourceRunId, + status: "recovered", stage: "saving_receipts", attempt: 1, attemptLimit: 6, retryable: true, requestIds: [] }, + systemRecovery: { ...createSystemRecovery(sourceRunId), state: "complete", phase: "reconcile_effects" }, + }, + }); + f.wakeup.mockImplementation(heartbeat.wakeup); + await deliverMessage({ callbacks: f.callbacks, endpointId: f.endpoint.id, provider: "github", thread: f.thread, + trigger: "subscribed_message", message: makeMessage({ id: "41903", text: "A genuinely new author reply after Stop", + userId: "42", userName: "octocat" }) }); + const comment = (await readIssueCommentsByIssueId(issueId)).find(row => row.body.includes("genuinely new author reply"))!; + const [receipt] = (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.agentId, f.assignedAgentId))) + .filter(row => row.id !== initial.id); + expect(receipt).toMatchObject({ status: "deferred_issue_execution", runId: null, + payload: { executionWait: { reason: "remote_cleanup" } } }); + let nextReceipt: typeof receipt | undefined; + let nextCommentId: string | undefined; + if (scenario === "revoked_before_valid") { + const thread = makeThread({ channelId: "paperclipai/paperclip", id: "github:paperclipai/paperclip:419:rc:4332012267" }).thread; + const { id: _id, ...conversation } = f.conversation; + await db.insert(chatConversations).values({ ...conversation, externalThreadId: thread.id }); + await deliverMessage({ callbacks: f.callbacks, endpointId: f.endpoint.id, provider: "github", thread, + trigger: "subscribed_message", message: makeMessage({ id: "41904", text: "A live source behind the deleted source", + userId: "42", userName: "octocat" }) }); + nextCommentId = (await readIssueCommentsByIssueId(issueId)).find(row => row.body.includes("live source behind"))!.id; + nextReceipt = (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.agentId, f.assignedAgentId))) + .find(row => row.id !== initial.id && row.id !== receipt.id); + expect(nextReceipt).toMatchObject({ status: "deferred_issue_execution", runId: null }); + await db.update(issueComments).set({ deletedAt: new Date() }).where(eq(issueComments.id, comment.id)); + } + const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, leaseId)); + if (scenario !== "unknown_stop") await db.update(environmentLeases).set({ status: "expired", cleanupStatus: "success", + releasedAt: new Date(), metadata: { remoteExecutionTermination: remoteTerminationReceipt(lease, + { providerLeaseId: scenario === "wrong_allocation" ? randomUUID() : providerLeaseId, state: "destroyed" }) } }) + .where(eq(environmentLeases.id, leaseId)); + if (scenario === "paused_endpoint") await db.update(chatEndpoints).set({ status: "paused" }).where(eq(chatEndpoints.id, f.endpoint.id)); + if (scenario === "revoked_principal") await db.update(chatIdentityLinks).set({ status: "revoked", revokedAt: new Date() }) + .where(eq(chatIdentityLinks.principalId, f.principal.id)); + if (scenario === "deleted_comment") await db.update(issueComments).set({ deletedAt: new Date() }).where(eq(issueComments.id, comment.id)); + if (scenario === "changed_generation") await db.update(chatConversations).set({ sessionGeneration: 2 }).where(eq(chatConversations.id, f.conversation.id)); + if (scenario === "changed_source") { + const [action] = await db.select().from(chatActions).where(eq(chatActions.id, receipt.id)); + await db.update(chatActions).set({ payload: { ...action.payload, commentId: initial.payload?.commentId } }) + .where(eq(chatActions.id, action.id)); + } + if (scenario === "restart") { + await f.service.shutdown(); + // Fresh application owner and scheduler must recover from saved data, + // with no in-memory request object left in the retired chat service. + createService(new FakeChatSdkRuntime(), f.providerFetch, { wakeup: heartbeatService(db).wakeup }); + } + const source = (await readHeartbeatRunsByCompanyId(f.companyId)).find(row => row.id === sourceRunId)!; + const { resumeDeferredChatWake } = await import("../services/durable-chat-wakeup.js"); + if (scenario === "foreign_scope") await resumeDeferredChatWake(db, { companyId: randomUUID(), agentId: f.assignedAgentId, + issueId, requestId: receipt.id }); + else if (scenario === "callback" || scenario === "revoked_before_valid") await heartbeat.resumeRemoteStopComments(source); + else if (scenario === "race") await Promise.all([heartbeat.resumeRemoteStopComments(source), heartbeat.resumeQueuedRuns()]); + else await heartbeatService(db).resumeQueuedRuns(); + const [settled] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, receipt.id)); + const successors = (await readHeartbeatRunsByCompanyId(f.companyId)).filter(row => row.wakeupRequestId === receipt.id); + const admitted = ["callback", "restart", "race"].includes(scenario); + expect(successors).toHaveLength(admitted ? 1 : 0); + expect(settled).toMatchObject({ id: receipt.id, idempotencyKey: receipt.idempotencyKey, + requestedByActorId: receipt.requestedByActorId, requestedAt: receipt.requestedAt, + status: admitted ? "queued" : "deferred_issue_execution" }); + if (admitted) { + expect(extractWakeCommentIds(successors[0].contextSnapshot)).toEqual([comment.id]); + expect(successors[0].contextSnapshot?.chatConversationId).toBe(f.conversation.id); + expect(successors[0].retryOfRunId).toBeNull(); + await heartbeat.resumeRemoteStopComments(source); + await heartbeat.resumeQueuedRuns(); + expect((await readHeartbeatRunsByCompanyId(f.companyId)).filter(row => row.wakeupRequestId === receipt.id)).toHaveLength(1); + } + if (nextReceipt) { + const admittedBehindRevoked = (await readHeartbeatRunsByCompanyId(f.companyId)).filter(row => row.wakeupRequestId === nextReceipt.id); + expect(admittedBehindRevoked).toHaveLength(1); + expect(extractWakeCommentIds(admittedBehindRevoked[0].contextSnapshot)).toEqual([nextCommentId]); + } + expect((await readHeartbeatRunsByCompanyId(f.companyId)).find(row => row.id === sourceRunId)) + .toMatchObject({ status: "cancelled", resultJson: { workspaceRestoreFailure: "restore_failed" } }); + }, 40_000, + ); + it("loads only the current configured instruction skills, never provider links or another company", async () => { const f = await reviewBotFixture(); const other = await seedCompany(); diff --git a/server/src/__tests__/durable-chat-wakeup.test.ts b/server/src/__tests__/durable-chat-wakeup.test.ts index a979df891c2..9ef4efe4241 100644 --- a/server/src/__tests__/durable-chat-wakeup.test.ts +++ b/server/src/__tests__/durable-chat-wakeup.test.ts @@ -321,6 +321,22 @@ describe("durable inbound chat scheduler receipts", () => { }); }); + it("reconsiders the same held inbound receipt only through its live durable owner", async () => { + const f = await fixture(); + const clearHold = await executionHold(f); + const request = f.request(); + expect(await f.wake(request)).toBeNull(); + await clearHold(); + const reconsidered = createDurableChatWakeupRequest({ ...request, resumeDeferred: true }); + const [first, second] = await Promise.all([f.wake(reconsidered), f.wake(reconsidered)]); + const receipts = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.agentId, f.agentId)); + expect(receipts).toHaveLength(1); + expect(receipts[0]).toMatchObject({ id: request.id, status: "queued", requestedAt: request.requestedAt }); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.wakeupRequestId, request.id))).toHaveLength(1); + expect(first?.id ?? second?.id).toBe(receipts[0].runId); + expect(f.authorize.mock.calls.length).toBeGreaterThan(1); + }); + async function retryFixture(deferred = false) { const f = await fixture(deferred); const applicationId = randomUUID(), diff --git a/server/src/services/agent-runner-qualification.ts b/server/src/services/agent-runner-qualification.ts index f6b73618e2e..8533ed19452 100644 --- a/server/src/services/agent-runner-qualification.ts +++ b/server/src/services/agent-runner-qualification.ts @@ -2,7 +2,7 @@ import type { Db } from "@paperclipai/db"; import type { Agent, EnvironmentProbeResult } from "@paperclipai/shared"; import { conflict, unprocessable } from "../errors.js"; import { resolveAgentRunnerEnvironmentForCompany } from "./agent-runner-selection.js"; -import { probeEnvironment } from "./environment-probe.js"; +import { probeEnvironment, readSandboxPlatformQualification } from "./environment-probe.js"; import type { PluginWorkerManager } from "./plugin-worker-manager.js"; export async function qualifyAgentRunner( @@ -11,16 +11,24 @@ export async function qualifyAgentRunner( expectedEnvironmentId: string, pluginWorkerManager?: PluginWorkerManager, ): Promise { - if (!["claude_local", "codex_local"].includes(agent.adapterType)) { - throw unprocessable("This agent does not need a runner upgrade."); + if (!["claude_local", "codex_local", "paperclip_runner"].includes(agent.adapterType)) { + throw unprocessable("This agent does not support computer qualification."); } const { environment } = await resolveAgentRunnerEnvironmentForCompany(db, agent.companyId, agent.defaultEnvironmentId); if (!environment || environment.id !== expectedEnvironmentId) { - throw conflict("The agent's computer changed. Save any environment changes, reload, and retry the upgrade."); + throw conflict(agent.adapterType === "paperclip_runner" + ? "The agent's computer changed. Save the computer selection, reload, and run its test again." + : "The agent's computer changed. Save any environment changes, reload, and retry the upgrade."); } if (environment.driver !== "sandbox") { throw unprocessable("This computer does not need sandbox qualification."); } + // A normal native-agent Test still probes its runtime. Reuse a valid saved + // platform proof so that test does not invalidate a healthy computer first. + if (agent.adapterType === "paperclip_runner" && await readSandboxPlatformQualification(db, environment, agent.companyId)) { + return { ok: true, driver: "sandbox", summary: "The agent's computer is qualified for runner selection.", + details: { platformQualificationRecorded: true } }; + } const result = await probeEnvironment(db, environment, { companyId: agent.companyId, pluginWorkerManager, @@ -33,7 +41,8 @@ export async function qualifyAgentRunner( ok: result.ok, driver: "sandbox", summary: qualified ? "The agent's computer is qualified for runner selection." - : "Could not qualify the agent's computer. Retry the upgrade or keep Legacy runner.", + : agent.adapterType === "paperclip_runner" ? "Could not verify the saved computer. Run test again." + : "Could not qualify the agent's computer. Retry the upgrade or keep Legacy runner.", details: { platformQualificationRecorded: qualified, ...(typeof result.details?.leaseId === "string" ? { leaseId: result.details.leaseId } : {}), diff --git a/server/src/services/chat-channels.ts b/server/src/services/chat-channels.ts index fc41debb711..e0af2b657e3 100644 --- a/server/src/services/chat-channels.ts +++ b/server/src/services/chat-channels.ts @@ -836,7 +836,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { const { publicationFiles } = createOutboundAttachments(db, { get storage() { return options.storage; } }); const { authorizeInboundWakeup, authorizeGitHubTaskRetryBatch, authorizeInboundWorkStart, authorizeFastResponse } = createInboundWakeAuthority(db, { runtimeCallbackEndpoint, lockCurrentPrincipalAuthorization }); const { ingestAttachments, attachmentOmissionDetail } = createInboundAttachments(db, { authorizeInboundWakeup, lifecycleRuntimeFence, sanitizeFilename, issuesSvc, isExternalActionAuthorizationChange, redactError, get storage() { return options.storage; } }); - const { stageInboundWakeup, acceptInboundWakeup, settleRejectedInboundWakeups, processInboundWakeup } = createInboundWakeups(db, { authorizeInboundWakeup, authorizeInboundWorkStart, attachmentOmissionDetail, isExternalActionAuthorizationChange, redactError, fetchImpl, get heartbeat() { return options.heartbeat; } }); + const { stageInboundWakeup, acceptInboundWakeup, settleRejectedInboundWakeups, processInboundWakeup, unregisterDeferredWakeOwner } = createInboundWakeups(db, { authorizeInboundWakeup, authorizeInboundWorkStart, attachmentOmissionDetail, isExternalActionAuthorizationChange, redactError, fetchImpl, get heartbeat() { return options.heartbeat; } }); const { settleTelegramAttachmentOnlyFailure } = createTelegramAttachmentFailure(db, { attachmentOmissionDetail, runtimeContextForRecord, endpointRecord, stageProviderEffect, scheduleProviderEffect, }); @@ -3529,6 +3529,7 @@ return { shutdown: async () => { shuttingDown = true; await Promise.allSettled(pendingFailedRetries()); + unregisterDeferredWakeOwner(); unregisterFailedRetryAuthority(); unregisterSlackTaskAuthority(); voice.unregisterAgentTools(); diff --git a/server/src/services/chat-channels/inbound/wakeups.ts b/server/src/services/chat-channels/inbound/wakeups.ts index a839c9b09b9..9b9f9738310 100644 --- a/server/src/services/chat-channels/inbound/wakeups.ts +++ b/server/src/services/chat-channels/inbound/wakeups.ts @@ -5,13 +5,13 @@ import { enqueueFastResponse } from "../../fast-responses.js"; import { and, eq, sql } from "drizzle-orm"; import { retryChatControlAdmission } from "../../chat-control-admission-retry.js"; import type { Db } from "@paperclipai/db"; -import { createDurableChatWakeupRequest, assertDurableChatWakeupReceipt } from "../../durable-chat-wakeup.js"; +import { createDurableChatWakeupRequest, assertDurableChatWakeupReceipt, registerDeferredChatWakeOwner } from "../../durable-chat-wakeup.js"; import { agentWakeupRequests, chatActions, chatConversations, chatDeliveries, chatEndpoints } from "@paperclipai/db"; import { forbidden } from "../../../errors.js"; import { logger } from "../../../middleware/logger.js"; import { queueIssueAssignmentWakeup } from "../../issue-assignment-wakeup.js"; import { isGitHubAutomaticDelivery, acknowledgeGitHubWakeup } from "../github/wake-authority.js"; -import type { createInboundWakeAuthority } from "./wakeup-authorization.js"; +import { inboundWakeScope, type createInboundWakeAuthority } from "./wakeup-authorization.js"; import type { createInboundAttachments } from "../inbound-attachments.js"; import type { IssueAssignmentWakeupDeps } from "../../issue-assignment-wakeup.js"; type InboundAttachmentResult = Awaited["ingestAttachments"]>>; @@ -425,5 +425,46 @@ export function createInboundWakeups(db: Db, options: WakeupOptions) { return denied; } } - return { stageInboundWakeup, acceptInboundWakeup, settleRejectedInboundWakeups, processInboundWakeup }; + const unregisterDeferredWakeOwner = registerDeferredChatWakeOwner(db, async (scope) => { + const action = await db.select().from(chatActions).where(and( + eq(chatActions.id, scope.requestId), eq(chatActions.companyId, scope.companyId), + eq(chatActions.kind, "inbound_wakeup"), eq(chatActions.status, "processed"), + )).then(rows => rows[0] ?? null); + if (!action || action.payload.agentId !== scope.agentId || action.payload.issueId !== scope.issueId) return; + const [receipt] = await db.select().from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.id, action.id), eq(agentWakeupRequests.companyId, scope.companyId), + eq(agentWakeupRequests.agentId, scope.agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"), + )); + if (!receipt || receipt.runId) return; + const authorize = async (tx: Db) => { + const [source] = await tx.select().from(chatActions).where(and( + eq(chatActions.id, action.id), eq(chatActions.companyId, scope.companyId), + eq(chatActions.kind, "inbound_wakeup"), eq(chatActions.status, "processed"), + )); + if (!source || inboundWakeScope(source) !== inboundWakeScope(action)) + throw forbidden("The saved chat source changed", { code: "chat_action_authorization_changed" }); + const current = await authorizeInboundWorkStart(tx, source); + if (current.delivery.state !== "processed") throw new Error("chat_inbound_wakeup_acceptance_not_committed"); + return current; + }; + // The same authority used at ingress checks the live endpoint, principal, + // immutable action source, conversation generation and original comment. + const context = await db.transaction(tx => authorize(tx as unknown as Db)); + const request = createDurableChatWakeupRequest({ + id: action.id, companyId: scope.companyId, agentId: scope.agentId, issueId: scope.issueId, + commentId: String(action.payload.commentId), + requestedByActorType: action.payload.requestedByActorType as "user" | "system", + requestedByActorId: String(action.payload.requestedByActorId), requestedAt: action.createdAt, + resumeDeferred: true, authorize: async tx => { await authorize(tx); }, + }); + assertDurableChatWakeupReceipt(request, receipt); + await queueIssueAssignmentWakeup({ heartbeat: options.heartbeat, issue: context.issue, + reason: "External chat message received", mutation: "chat_message_received", + contextSource: `chat:${context.endpoint.provider}`, taskKey: context.issue.identifier, + requestedByActorType: request.requestedByActorType, requestedByActorId: request.requestedByActorId, + wakeCommentId: request.commentId, chatConversationId: context.conversation.id, + attachmentOmissionReasons: action.payload.attachmentOmissionReasons as Record | undefined, + durableChatRequest: request, rethrowOnError: true }); + }); + return { stageInboundWakeup, acceptInboundWakeup, settleRejectedInboundWakeups, processInboundWakeup, unregisterDeferredWakeOwner }; } diff --git a/server/src/services/durable-chat-wakeup.ts b/server/src/services/durable-chat-wakeup.ts index b9d9f721a28..9a689254e03 100644 --- a/server/src/services/durable-chat-wakeup.ts +++ b/server/src/services/durable-chat-wakeup.ts @@ -198,6 +198,23 @@ export function unadmittedChatWakeupCondition( )`; } +export interface DeferredChatWakeScope { + companyId: string; + agentId: string; + issueId: string; + requestId: string; +} +const deferredChatWakeOwners = new WeakMap Promise>(); + +/** The live inbound owner reauthorizes the original source. Cleanup owns no chat authority. */ +export function registerDeferredChatWakeOwner(db: Db, owner: (input: DeferredChatWakeScope) => Promise) { + deferredChatWakeOwners.set(db, owner); + return () => { if (deferredChatWakeOwners.get(db) === owner) deferredChatWakeOwners.delete(db); }; +} +export async function resumeDeferredChatWake(db: Db, input: DeferredChatWakeScope) { + await deferredChatWakeOwners.get(db)?.(input); +} + /** Internal authority: this object cannot be supplied through a JSON API. */ export interface DurableChatWakeupRequest { readonly id: string; @@ -210,6 +227,8 @@ export interface DurableChatWakeupRequest { readonly requestedAt: Date; readonly idempotencyKey: string; readonly failedRunRetry?: { readonly failedRunId: string }; + /** Only the current inbound owner may reconsider its already saved, unconsumed receipt. */ + readonly resumeDeferred?: true; readonly authorize: (tx: Db) => Promise; readonly onAdmitted?: (tx: Db, receipt: { runId: string | null }) => Promise; } diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index b3c53737bef..58a27c668a8 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -1875,7 +1875,7 @@ function createSandboxEnvironmentDriver( ? qualifiedSandboxSnapshotId(configuredEnvironment, input.companyId, parsed.config, selectedTemplate.value) : null; if (input.adapterType === "paperclip_runner" && !pinnedSnapshotId && hasSandboxSnapshotQualification(configuredEnvironment, input.companyId)) { - throw new Error("sandbox_snapshot_qualification_required: The snapshot qualification is missing or stale. Test this environment again before running Paperclip Runner."); + throw new Error("sandbox_snapshot_qualification_required: The snapshot qualification is missing or stale. Open the agent's Harness / Runtime page and select Run test before running Paperclip Runner."); } if (input.configuredEnvironment) parsed.config = { ...parsed.config, reuseLease: storedParsed.config.reuseLease, archiveOnRelease: storedParsed.config.archiveOnRelease }; if (pinnedSnapshotId) parsed.config = { ...parsed.config, snapshot: pinnedSnapshotId }; diff --git a/server/src/services/execution-projection-recovery.test.ts b/server/src/services/execution-projection-recovery.test.ts new file mode 100644 index 00000000000..a05c868d864 --- /dev/null +++ b/server/src/services/execution-projection-recovery.test.ts @@ -0,0 +1,39 @@ +import { randomUUID } from "node:crypto"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { agents, companies, createDb, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; +import { executionProjectionsForRuns } from "./execution-projection.js"; + +const support = await getEmbeddedPostgresTestSupport(); +describe.skipIf(!support.supported)("persisted stopped-workspace projection", () => { + let database: Awaited>; + let db: ReturnType; + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("stopped-workspace-projection"); + db = createDb(database.connectionString); + }, 60_000); + afterAll(async () => { await db?.$client.end(); await database?.cleanup(); }); + + it("reads recovered files on a cancelled run through the real batched query", async () => { + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Stopped workspace", issuePrefix: "SWP" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Reviewer", adapterType: "paperclip_runner" }); + await db.insert(issues).values({ id: issueId, companyId, title: "Review current changes", assigneeAgentId: agentId }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: issueId, + runtimeMode: "native", status: "cancelled", finishedAt: new Date(), + contextSnapshot: { issueId }, resultJson: { workspaceRestoreFailure: "restore_failed" } }); + await db.insert(nativeRunFinalizations).values({ companyId, issueId, runId, phase: "observed" }); + await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: issueId, kind: "active_run_watchdog", + cause: "native_continuation_requires_reconciliation", fingerprint: `native-workspace:${runId}`, + ownerType: "system", status: "resolved", outcome: "blocked", resolvedAt: new Date(), + nextAction: "The original sandbox is retained for workspace repair.", + evidence: { runId, continuationDelivery: "invalidated", + automaticRecovery: { replay: "blocked", actionOutcome: "unknown", policy: "preserve_without_replay_v1" }, + workspaceRecovery: { schema: "paperclip.workspace-recovery.v1", runId, status: "recovered", + stage: "saving_receipts", attempt: 1, attemptLimit: 6, retryable: true, requestIds: [] }, + } }); + const projections = await executionProjectionsForRuns(db, companyId, [runId]); + expect(projections.get(runId)).toMatchObject({ phase: "recovery_needed", label: "Stopped", + nextAction: "Saved work recovered. Send a message to continue.", permittedActions: ["inspect_run"] }); + }); +}); diff --git a/server/src/services/execution-projection.test.ts b/server/src/services/execution-projection.test.ts index df55aa800cc..64ed679e73e 100644 --- a/server/src/services/execution-projection.test.ts +++ b/server/src/services/execution-projection.test.ts @@ -37,6 +37,39 @@ const project = ( ) => projectExecution(r, c, pending, undefined, now); describe("execution truth projection", () => { + it.each(["queued", "recovering", "blocked", "recovered"] as const)("shows the exact saved workspace recovery state instead of stale repair instructions: %s", status => { + const sourceRunId = "ec036e06-90a3-4b69-81db-a8eab7dbfbac"; + const source = run({ id: sourceRunId, status: "failed", errorCode: "native_continuation_requires_reconciliation" }); + const action = { + status: "resolved" as const, + cause: "native_continuation_requires_reconciliation", + nextAction: "The original sandbox is retained for workspace repair.", + evidence: { + runId: sourceRunId, + automaticRecovery: { policy: "preserve_without_replay_v1", actionOutcome: "unknown", replay: "blocked" }, + continuationDelivery: "invalidated", + workspaceRecovery: { + schema: "paperclip.workspace-recovery.v1", runId: sourceRunId, status, + stage: "saving_receipts", attempt: 1, requestIds: [], + ...(status === "blocked" ? { message: "The saved source cannot be read safely." } : {}), + }, + }, + }; + const projected = projectExecution(source, coordinator({ phase: "terminal_failure" }), [], action, now); + expect(projected).toMatchObject({ + phase: "recovery_needed", label: "Stopped", cause: action.cause, + nextAction: status === "recovered" ? "Saved work recovered. Send a message to continue." + : status === "blocked" ? "The saved source cannot be read safely." : "Recovering saved work…", + permittedActions: ["inspect_run"], successorRunId: null, + }); + expect(source.status).toBe("failed"); + expect(action.evidence.automaticRecovery.replay).toBe("blocked"); + expect(projectExecution(source, undefined, [], { ...action, + evidence: { ...action.evidence, workspaceRecovery: { ...action.evidence.workspaceRecovery, + runId: "5f227d75-b314-4511-bf23-9c615277e69b" } }, + }, now).nextAction).toBe(action.nextAction); + }); + it("keeps cleanup quarantine visible without a coordinator or recovery row", () => { const stopped = run({ id: "stopped", status: "failed", errorCode: "native_session_cleanup_quarantined" }); expect(projectExecution(stopped, undefined, [], undefined, now)).toMatchObject({ diff --git a/server/src/services/execution-projection.ts b/server/src/services/execution-projection.ts index 1d857e393d9..b778c244f43 100644 --- a/server/src/services/execution-projection.ts +++ b/server/src/services/execution-projection.ts @@ -14,6 +14,7 @@ import type { ExecutionProjection } from "@paperclipai/shared"; import { EXECUTION_CONTROL_DEADLINE_MS } from "./execution-control-deadline.js"; import { executionFailureRetryCount } from "./execution-recovery-attempt.js"; import { retryIdempotentDatabaseOperation } from "../database-retry.js"; +import { workspaceRecoveryDisplay } from "./workspace-recovery-receipt.js"; const text = (v: unknown) => (typeof v === "string" ? v : null); const executionRunColumns = { id: heartbeatRuns.id, @@ -245,7 +246,8 @@ export function projectExecution( Object.assign(projection, { phase, label }); if (recoveryAction?.status === "resolved" && recoveryAction.evidence?.automaticRecovery) { projection.cause = recoveryAction.cause; - projection.nextAction = recoveryAction.nextAction; + projection.nextAction = workspaceRecoveryDisplay(recoveryAction.evidence, run.id)?.message + ?? recoveryAction.nextAction; const continuation = recoveryAction.evidence.explicitUserContinuation as { previousRunId?: unknown; runId?: unknown } | undefined; const explicitSuccessor = text(continuation?.runId); diff --git a/server/src/services/execution-recovery-resolution.ts b/server/src/services/execution-recovery-resolution.ts index e56141565b9..59b2b12cf5a 100644 --- a/server/src/services/execution-recovery-resolution.ts +++ b/server/src/services/execution-recovery-resolution.ts @@ -7,7 +7,7 @@ import { claimedAdapterType, conversationRecoveryActionPredicate, getConversatio import { persistActivity } from "./activity-log.js"; import { appendHeartbeatRunEvent } from "./heartbeat-run-events.js"; import { logger } from "../middleware/logger.js"; -import { and, eq, inArray, isNull, not, or, sql } from "drizzle-orm"; +import { and, eq, exists, inArray, isNull, not, or, sql } from "drizzle-orm"; import { chatActions, environmentLeases, @@ -28,6 +28,7 @@ import { isSupersededConversationRun } from "./agent-conversations.js"; import { hasRequiredWorkspaceRecovery } from "./workspace-restore-recovery-state.js"; import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js"; import { readSystemRecovery } from "./system-recovery-state.js"; +import { alias } from "drizzle-orm/pg-core"; /** An operator records observed outcomes; this is not permission to blindly retry. */ export async function validateExecutionReconciliation(input: { @@ -344,6 +345,24 @@ export async function deliverReconciledExecutions( } } +const supersedingWorkspaceHold = alias(issueRecoveryActions, "superseding_workspace_hold"); + +function supersedingNativeWorkspaceHold(db: Db, input: { + companyId: string | typeof heartbeatRuns.companyId; + issueId: string | typeof issueRecoveryActions.sourceIssueId; + runId: string | typeof heartbeatRuns.id; +}) { + return db.select({ id: supersedingWorkspaceHold.id }).from(supersedingWorkspaceHold).where(and( + eq(supersedingWorkspaceHold.companyId, input.companyId), + eq(supersedingWorkspaceHold.sourceIssueId, input.issueId), + eq(supersedingWorkspaceHold.status, "resolved"), + eq(supersedingWorkspaceHold.cause, "native_continuation_requires_reconciliation"), + sql`${supersedingWorkspaceHold.evidence}->>'runId' = ${input.runId}::text`, + sql`${supersedingWorkspaceHold.evidence}->'automaticRecovery'->>'replay' = 'blocked'`, + sql`${supersedingWorkspaceHold.evidence}->'workspaceRestoreRecovery'->>'schema' = 'paperclip.workspace-restore-recovery.v1'`, + )); +} + export function unresolvedExecutionDispositions(db: Db, limit: number) { return db .select({ action: issueRecoveryActions }) @@ -372,9 +391,14 @@ export function unresolvedExecutionDispositions(db: Db, limit: number) { sql`not (${issueRecoveryActions.evidence} ? 'systemRecovery')`, inArray(issueRecoveryActions.status, ["active", "escalated"]), eq(issueRecoveryActions.kind, "active_run_watchdog"), - inArray(issueRecoveryActions.cause, [ - ...EXECUTION_RECONCILIATION_CAUSES, - ]), + or(inArray(issueRecoveryActions.cause, [...EXECUTION_RECONCILIATION_CAUSES]), and( + eq(heartbeatRuns.runtimeMode, "native"), eq(heartbeatRuns.nativePhase, "terminal_failure"), + eq(nativeRunFinalizations.phase, "terminal_failure"), isNull(nativeRunFinalizations.nextAttemptAt), + sql`${issueRecoveryActions.wakePolicy}->>'kind' = 'resume_native_run'`, + sql`${issueRecoveryActions.wakePolicy}->>'runId' = ${heartbeatRuns.id}::text`, + exists(supersedingNativeWorkspaceHold(db, { companyId: heartbeatRuns.companyId, + issueId: issueRecoveryActions.sourceIssueId, runId: heartbeatRuns.id })), + )), inArray(heartbeatRuns.status, [ "failed", "timed_out", @@ -502,9 +526,6 @@ export async function settleUnrecoverableExecutions( !run || !action || action.evidence.runId !== runId || - !EXECUTION_RECONCILIATION_CAUSES.includes( - action.cause as (typeof EXECUTION_RECONCILIATION_CAUSES)[number], - ) || !["active", "escalated"].includes(action.status) || (run.nativeIssueId ?? run.contextSnapshot?.issueId) !== task.id || !["failed", "timed_out", "interrupted", "cancelled"].includes( @@ -512,6 +533,29 @@ export async function settleUnrecoverableExecutions( ) ) return; + if (!EXECUTION_RECONCILIATION_CAUSES.includes(action.cause as (typeof EXECUTION_RECONCILIATION_CAUSES)[number])) { + if (run.runtimeMode !== "native" || run.nativePhase !== "terminal_failure" + || coordinator?.phase !== "terminal_failure" || coordinator.leaseOwner || coordinator.resultId + || coordinator.nextAttemptAt || coordinator.failureDetail?.successorRunId + || action.wakePolicy?.kind !== "resume_native_run" || action.wakePolicy.runId !== run.id) return; + const [hold] = await supersedingNativeWorkspaceHold(tx as unknown as Db, { + companyId: run.companyId, issueId: task.id, runId: run.id, + }).limit(1).for("update"); + if (!hold) return; + await notifyExecutionRunWork(tx); + const note = "Exact session recovery stopped. The recorded workspace recovery holds this execution without replaying its actions."; + await tx.update(issueRecoveryActions).set({ status: "cancelled", outcome: "cancelled", + resolvedAt: now, updatedAt: now, wakePolicy: null, monitorPolicy: null, + nextAction: note, resolutionNote: note, + evidence: { ...action.evidence, supersedingRecoveryActionId: hold.id }, + }).where(eq(issueRecoveryActions.id, action.id)); + await persistActivity(tx as unknown as Db, { companyId: run.companyId, actorType: "system", + actorId: "execution-recovery", action: "issue.execution_recovery_settled", entityType: "issue", + entityId: task.id, runId: run.id, + details: { recoveryActionId: action.id, outcome: "cancelled", supersedingRecoveryActionId: hold.id }, + }); + return; + } // Adoption transfers maintenance to the retained-workspace owner. // This compatibility resolver must not close its hold or relabel it // as a user repair merely because old return-owner metadata is absent. diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index 2ac1a3f0960..7d337f06cfb 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -47,6 +47,7 @@ import { adapterExecutionControls } from "../adapter-execution-control.js"; import { executionFailureRetryCount } from "../execution-recovery-attempt.js"; import { slackMentionAllowsRunStart, + resumeDeferredChatWake, assertDurableChatWakeupReceipt, assertDurableChatWakeupRequest, withDurableChatAdmission, @@ -1857,6 +1858,13 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende )); if (!latest || !latest.finishedAt || !isHeartbeatRunTerminalStatus(latest.status)) continue; if (latest.runtimeMode !== "legacy" && latest.runtimeMode !== "native") continue; + if (wake.idempotencyKey?.startsWith("chat-inbound:")) { + await resumeDeferredChatWake(db, { companyId: wake.companyId, agentId: wake.agentId, + issueId: String(wake.payload?.issueId), requestId: wake.id }).catch(() => { + logger.warn({ requestId: wake.id }, "saved chat source could not resume after recovery"); + }); + continue; + } const releaseSource = `${latest.companyId}:${String(wake.payload?.issueId)}:${latest.id}:${latest.sourceVersion}`; if (latest.runtimeMode === "native" && releasedNativeSources.has(releaseSource)) continue; if (latest.runtimeMode === "legacy") { @@ -2373,10 +2381,13 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende receipt.payload?.museBindingGeneration !== museRequest.bindingGeneration || receipt.payload?.museRequestDigest !== museRequest.requestDigest)) { throw conflict("Request ID reused with changed input."); } - if (receipt && durableRequest?.failedRunRetry && opts.taskRetryRequestId && - receipt.status === "deferred_issue_execution" && !receipt.runId && - receipt.payload?.taskRetryRequestId === opts.taskRetryRequestId) { - // A bound ordinary Retry may reconsider its same deferred chat receipt. + if (receipt && durableRequest && receipt.status === "deferred_issue_execution" && !receipt.runId && + ((durableRequest.failedRunRetry && opts.taskRetryRequestId && + receipt.payload?.taskRetryRequestId === opts.taskRetryRequestId) || + (durableRequest.resumeDeferred && !durableRequest.failedRunRetry && + receipt.payload?.issueId === durableRequest.issueId && + queuedCommentIdsFromWakePayload(receipt.payload).includes(durableRequest.commentId)))) { + // The live inbound owner or a bound Retry may reconsider the same saved receipt. // The issue lock, live chat authority and execution holds below still // decide whether it can dispatch; this never creates another wake. executionWaitRequestId = receipt.id; @@ -2822,8 +2833,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende // system wake. Admission validates the saved user comment or board click. (opts.queuedCommentInterruptId ?? opts.queuedCommentRequestId) === executionWaitRequestId ? undefined : eq(agentWakeupRequests.requestedByActorType, - durableRequest?.failedRunRetry && opts.taskRetryRequestId - ? opts.requestedByActorType ?? "" : "user"), + durableRequest ? opts.requestedByActorType ?? "" : "user"), opts.queuedCommentInterruptId === executionWaitRequestId ? sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt'->>'actorId' = ${opts.requestedByActorId ?? ""}` : opts.queuedCommentRequestId === executionWaitRequestId ? undefined @@ -4187,7 +4197,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende }; // A deferred durable chat wake already owns this ID and source. // Promote that receipt in the same transaction as its successor. - const wakeupRequest = durableRequest?.failedRunRetry && executionWaitRequestId === durableRequest.id + const wakeupRequest = durableRequest && executionWaitRequestId === durableRequest.id ? await tx.update(agentWakeupRequests).set({ payload, status: "queued", error: null, finishedAt: null, updatedAt: new Date(), }).where(and(eq(agentWakeupRequests.id, durableRequest.id), diff --git a/server/src/services/heartbeat/run-control.test.ts b/server/src/services/heartbeat/run-control.test.ts index d0d08896538..ab0a9fbb5a8 100644 --- a/server/src/services/heartbeat/run-control.test.ts +++ b/server/src/services/heartbeat/run-control.test.ts @@ -429,6 +429,38 @@ describe.skipIf(!runControlPostgresSupport.supported)("heartbeat run-control dat .toMatchObject({ status: "done", phase: "done", completedAt: expect.any(Date) }); }); + it.each(["confirmed", "missing", "wrong_allocation", "unacknowledged", "still_running", "wrong_intent"] as const)("publishes native Stop acknowledgement only after exact remote containment: %s", async proof => { + const { company, agent } = await fixture(); + const [issue] = await db.insert(issues).values({ companyId: company.id, title: "Save work after Stop", assigneeAgentId: agent.id }).returning(); + const runId = randomUUID(), leaseId = randomUUID(), providerLeaseId = randomUUID(); + const [run] = await db.insert(heartbeatRuns).values({ id: runId, companyId: company.id, agentId: agent.id, + nativeIssueId: issue.id, runtimeMode: "native", status: proof === "still_running" ? "running" : "cancelled", finishedAt: proof === "still_running" ? null : new Date(), + resultJson: { executionCancellation: { state: "requested", requestedAt: new Date().toISOString() }, + cancelledByActorType: "user", cancelledByUserId: "operator", nativeCancellation: { + schema: "paperclip.native-cancellation.v1", runId: proof === "wrong_intent" ? randomUUID() : runId, companyId: company.id, issueId: issue.id, + scope: "run", reasonCode: "cancellation_run_only", dispatchState: proof === "unacknowledged" ? "pending" : "acknowledged", dispatched: true, + intentAuditId: randomUUID(), acknowledgementAuditId: randomUUID(), + }, workspaceRestoreFailure: "restore_failed", workspaceRestoreRecovery: { + schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [leaseId], + } }, + }).returning(); + const receipt = { schema: "paperclip.remote-termination.v1", companyId: company.id, runId, leaseId, + provider: "daytona", providerLeaseId: proof === "wrong_allocation" ? randomUUID() : providerLeaseId, + state: "stopped", confirmedAt: new Date().toISOString() }; + await db.insert(environmentLeases).values({ id: leaseId, companyId: company.id, issueId: issue.id, heartbeatRunId: runId, + provider: "daytona", providerLeaseId, status: proof === "missing" ? "active" : "released", leasePolicy: "retain_on_failure", + releasedAt: proof === "missing" ? null : new Date(), cleanupStatus: proof === "missing" ? null : "success", + metadata: proof === "missing" ? {} : { remoteExecutionTermination: receipt } }); + const control = createHeartbeatRunControl(db, callbacks(db)); + await control.acknowledgeRemoteStop(run.id, company.id); + await control.acknowledgeRemoteStop(run.id, company.id); + const [stopped] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)); + expect(stopped.resultJson?.executionCancellation).toMatchObject({ state: proof === "confirmed" ? "acknowledged" : "requested" }); + if (proof === "confirmed") expect(stopped.resultJson?.executionCancellation).toMatchObject({ proof: "provider_termination_receipt" }); + expect(stopped.resultJson?.workspaceRestoreFailure).toBe("restore_failed"); + expect(stopped.resultJson?.workspaceRestoreRecovery).toMatchObject({ leaseIds: [leaseId] }); + }); + it("keeps pre-Stop input pending until a later user message authorizes one continuation", async () => { const { company, agent } = await fixture(); const [issue] = await db.insert(issues).values({ companyId: company.id, title: "Retain my direction after Stop", diff --git a/server/src/services/heartbeat/run-control.ts b/server/src/services/heartbeat/run-control.ts index 8c87325ee4b..2ce3cf3c363 100644 --- a/server/src/services/heartbeat/run-control.ts +++ b/server/src/services/heartbeat/run-control.ts @@ -1,3 +1,4 @@ +import { resumeDeferredChatWake } from "../durable-chat-wakeup.js"; import { notifyEnvironmentCleanup } from "../environment-cleanup-notifications.js"; import { readNativeStopExecutionEvidence } from "./native-stop-evidence-read.js"; import { readTaskRetryIntent, reconcileTaskChatRetryReceipts } from "../task-retry.js"; @@ -493,21 +494,27 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont completeTerminatedRemoteNativeSessionCleanup({ companyId, runId, remoteCleanupScope }); } } - const stopped = state?.runtimeMode === "legacy" && state.status === "cancelled" ? await getRun(runId) : null; - if (stopped?.runtimeMode === "legacy" && stopped.status === "cancelled" && + const stopped = state?.status === "cancelled" ? await getRun(runId) : null; + // Native interrupt acknowledgement is not physical containment. Publish the + // shared Stop receipt only after every exact remote lease confirms teardown. + // Keep workspace preservation separate: a stopped provider can still owe files. + if (stopped?.status === "cancelled" && parseObject(stopped.resultJson?.executionCancellation).state === "requested" && - await runUsedConversationAdapter(db, stopped) && + (stopped.runtimeMode === "native" ? isAcknowledgedNativeStop(stopped) + : stopped.runtimeMode === "legacy" && await runUsedConversationAdapter(db, stopped)) && await remoteExecutionHasStopped(db, companyId, runId)) { await db.update(heartbeatRuns).set({ resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify({ executionCancellation: { ...parseObject(stopped.resultJson?.executionCancellation), state: "acknowledged", acknowledgedAt: new Date().toISOString(), proof: "provider_termination_receipt" }, - conversationContinuation: CONVERSATION_CONTINUATION_POLICY, + ...(stopped.runtimeMode === "legacy" ? { conversationContinuation: CONVERSATION_CONTINUATION_POLICY } : {}), })}::jsonb`, updatedAt: new Date(), }).where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, companyId), - eq(heartbeatRuns.status, "cancelled"))); + eq(heartbeatRuns.status, "cancelled"), + sql`${heartbeatRuns.resultJson}->'executionCancellation'->>'state' = 'requested'`, + stopped.runtimeMode === "native" ? sql`${heartbeatRuns.resultJson}->'nativeCancellation' = ${JSON.stringify(stopped.resultJson?.nativeCancellation)}::jsonb` : undefined)); } } @@ -544,12 +551,19 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont const pending = await db.select().from(agentWakeupRequests).where(and( eq(agentWakeupRequests.companyId, run.companyId), eq(agentWakeupRequests.agentId, run.agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"), - eq(agentWakeupRequests.requestedByActorType, "user"), + inArray(agentWakeupRequests.requestedByActorType, ["user", "system"]), requestId ? eq(agentWakeupRequests.id, requestId) : undefined, sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId}`, )).orderBy(asc(agentWakeupRequests.requestedAt)).limit(50); for (const wake of pending) { - if (wake.idempotencyKey?.startsWith("chat-inbound:")) continue; + if (wake.idempotencyKey?.startsWith("chat-inbound:")) { + await resumeDeferredChatWake(db, { companyId: run.companyId, agentId: run.agentId, + issueId, requestId: wake.id }).catch(() => { + logger.warn({ requestId: wake.id }, "saved chat source could not resume after cleanup"); + }); + continue; + } + if (wake.requestedByActorType !== "user") continue; if (readTaskRetryIntent(wake.payload)) { await resumeTaskRetry(wake.companyId, wake.id); continue; } let payload = parseObject(wake.payload); if (payload.queuedCommentInterrupt) { diff --git a/server/src/services/heartbeat/workspace-preparation.test.ts b/server/src/services/heartbeat/workspace-preparation.test.ts index ccbcd9066c5..eb0f0e2abc3 100644 --- a/server/src/services/heartbeat/workspace-preparation.test.ts +++ b/server/src/services/heartbeat/workspace-preparation.test.ts @@ -320,6 +320,69 @@ describe.skipIf(!support.supported)("heartbeat workspace preparation boundary", expect((await readIssue(issue.id)).executionWorkspaceId).toBe(prepared.persistedExecutionWorkspace!.id); }); + it("reuses a successful fresh task attempt for the next ordinary user turn", async () => { + const { input, issue } = await fixture(); + input.policy.useIsolatedTaskDirectory = true; + input.policy.effectiveExecutionWorkspaceMode = "isolated_workspace"; + input.policy.requestedExecutionWorkspaceMode = "isolated_workspace"; + input.task.freshWorkspace = { mode: "fresh_isolated_workspace", retryIntentId: input.run.id, + sourceRunId: input.run.id, retainedSourceIds: [] }; + const original = await prepareHeartbeatWorkspace(db, input); + const workspace = original.persistedExecutionWorkspace!; + await writeFile(join(workspace.cwd!, "saved-review.txt"), "completed review history\n"); + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }) + .where(eq(heartbeatRuns.id, input.run.id)); + await reuse(input, workspace); + input.task.freshWorkspace = null; + const next = await prepareHeartbeatWorkspace(db, input); + expect(next.persistedExecutionWorkspace!.id).toBe(workspace.id); + expect(next.executionWorkspace.cwd).toBe(workspace.cwd); + expect(await readFile(join(next.executionWorkspace.cwd, "saved-review.txt"), "utf8")) + .toBe("completed review history\n"); + expect((await readIssue(issue.id)).executionWorkspaceId).toBe(workspace.id); + await reuse(input, workspace); + input.task.freshWorkspace = { mode: "fresh_isolated_workspace", retryIntentId: input.run.id, + sourceRunId: input.run.id, retainedSourceIds: [] }; + input.reuse.existingExecutionWorkspace = null; + input.reuse.reusableExistingExecutionWorkspace = null; + input.reuse.requestedShouldReuseExisting = false; + input.reuse.workspaceReuseRequest = resolveExecutionWorkspaceReuseRequestForIssue({ + issueExecutionWorkspaceId: null, issueExecutionWorkspacePreference: "create_new", existingExecutionWorkspaceStatus: null, + }); + const replacement = await prepareHeartbeatWorkspace(db, input); + expect(replacement.executionWorkspace.cwd).not.toBe(workspace.cwd); + await expect(access(join(replacement.executionWorkspace.cwd, "saved-review.txt"))).rejects.toThrow(); + expect(await readFile(join(workspace.cwd!, "saved-review.txt"), "utf8")).toBe("completed review history\n"); + }); + + it.each(["task_binding", "workspace_row", "foreign_issue", "foreign_company", "closed_workspace"])( + "rejects a retained task attempt after its %s changes", async change => { + const { input, issue, source } = await fixture(); + input.policy.useIsolatedTaskDirectory = true; + input.policy.effectiveExecutionWorkspaceMode = "isolated_workspace"; + input.policy.requestedExecutionWorkspaceMode = "isolated_workspace"; + input.task.freshWorkspace = { mode: "fresh_isolated_workspace", retryIntentId: input.run.id, + sourceRunId: input.run.id, retainedSourceIds: [] }; + const original = await prepareHeartbeatWorkspace(db, input); + const workspace = original.persistedExecutionWorkspace!; + await writeFile(join(workspace.cwd!, "saved-review.txt"), "keep saved work\n"); + await reuse(input, workspace); + input.task.freshWorkspace = null; + if (change === "task_binding") await db.update(issues).set({ executionWorkspaceId: null }).where(eq(issues.id, issue.id)); + if (change === "workspace_row") await db.update(executionWorkspaces).set({ cwd: source }).where(eq(executionWorkspaces.id, workspace.id)); + if (change === "foreign_issue") await db.update(executionWorkspaces).set({ sourceIssueId: null }).where(eq(executionWorkspaces.id, workspace.id)); + if (change === "foreign_company") { + const [other] = await db.insert(companies).values({ name: "Other workspace company", issuePrefix: "OTH" }).returning(); + await db.update(executionWorkspaces).set({ companyId: other.id }).where(eq(executionWorkspaces.id, workspace.id)); + } + if (change === "closed_workspace") await db.update(executionWorkspaces).set({ status: "archived" }).where(eq(executionWorkspaces.id, workspace.id)); + await expect(prepareHeartbeatWorkspace(db, input)).rejects.toMatchObject({ + resultJson: { workspaceValidation: { reason: "isolated_task_directory_binding_mismatch", issueId: issue.id } }, + }); + expect(await readFile(join(workspace.cwd!, "saved-review.txt"), "utf8")).toBe("keep saved work\n"); + }, + ); + async function reuse(input: HeartbeatWorkspacePreparationInput, workspace: NonNullable>["persistedExecutionWorkspace"]>) { input.reuse.requestedShouldReuseExisting = true; input.reuse.existingExecutionWorkspace = workspace; diff --git a/server/src/services/heartbeat/workspace-preparation.ts b/server/src/services/heartbeat/workspace-preparation.ts index a23665a96b6..f56bde29125 100644 --- a/server/src/services/heartbeat/workspace-preparation.ts +++ b/server/src/services/heartbeat/workspace-preparation.ts @@ -277,18 +277,33 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace workspaceHints: [], warnings: [], baseCwdFallback: false, materializationFailures: [], additionalWorkspaces: [], referencedProjectFailures: [] }; } if (useIsolatedTaskDirectory && issueRef && !pluginTaskWorkspace) { - const cwd = await materializeIsolatedTaskDirectory({ - companyId: agent.companyId, - issueId: issueRef.id, - ...(input.task.freshWorkspace?.mode === "fresh_isolated_workspace" ? { attemptId: run.id } : {}), - }); - if (reusableExistingExecutionWorkspace && ( - reusableExistingExecutionWorkspace.companyId !== agent.companyId || - reusableExistingExecutionWorkspace.sourceIssueId !== issueRef.id || - reusableExistingExecutionWorkspace.mode !== "isolated_workspace" || - reusableExistingExecutionWorkspace.strategyType !== "project_primary" || - reusableExistingExecutionWorkspace.cwd !== cwd - )) { + // A completed fresh attempt becomes the current task directory. New + // input reuses that binding; only another fresh attempt changes roots. + let cwd: string; + try { + if (reusableExistingExecutionWorkspace) { + const [currentIssue] = await db.select({ workspaceId: issues.executionWorkspaceId }).from(issues) + .where(and(eq(issues.id, issueRef.id), eq(issues.companyId, agent.companyId))); + const stored = await executionWorkspacesSvc.getStoredById(agent.companyId, reusableExistingExecutionWorkspace.id); + if (fresh || currentIssue?.workspaceId !== reusableExistingExecutionWorkspace.id || + reusableExistingExecutionWorkspace.companyId !== agent.companyId || + reusableExistingExecutionWorkspace.sourceIssueId !== issueRef.id || + reusableExistingExecutionWorkspace.mode !== "isolated_workspace" || + reusableExistingExecutionWorkspace.strategyType !== "project_primary" || !stored || + stored.companyId !== agent.companyId || stored.sourceIssueId !== issueRef.id || + stored.mode !== "isolated_workspace" || stored.strategyType !== "project_primary" || + stored.status !== "active" || stored.cwd !== reusableExistingExecutionWorkspace.cwd || !stored.cwd) { + throw new Error("The saved task directory binding changed"); + } + } + cwd = await materializeIsolatedTaskDirectory({ + companyId: agent.companyId, + issueId: issueRef.id, + ...(fresh ? { attemptId: run.id } : {}), + ...(reusableExistingExecutionWorkspace ? { reuseCwd: reusableExistingExecutionWorkspace.cwd! } : {}), + }); + } catch (error) { + if (!reusableExistingExecutionWorkspace) throw error; throw new WorkspaceValidationFailure("The existing execution workspace is not this task's isolated directory.", { workspaceValidation: { reason: "isolated_task_directory_binding_mismatch", issueId: issueRef.id }, }); diff --git a/server/src/services/isolated-task-directory.test.ts b/server/src/services/isolated-task-directory.test.ts index bd3d350ff9f..b2d4125f87b 100644 --- a/server/src/services/isolated-task-directory.test.ts +++ b/server/src/services/isolated-task-directory.test.ts @@ -73,6 +73,37 @@ describe("isolated task directories", () => { .rejects.toThrow("not a private directory"); }); + it("reuses only the exact private task directory, including a prior fresh attempt", async () => { + const identity = await setup(); + for (const attemptId of [undefined, "previous-attempt"]) { + const cwd = await materializeIsolatedTaskDirectory({ ...identity, attemptId }); + await writeFile(path.join(cwd, "saved.txt"), "preserved"); + expect(await materializeIsolatedTaskDirectory({ ...identity, reuseCwd: cwd })).toBe(cwd); + expect(await readFile(path.join(cwd, "saved.txt"), "utf8")).toBe("preserved"); + } + }); + + it.each(["other-company", "other-issue", "outside", "traversal", "missing", "symlink"])( + "does not admit or recreate a reused task directory with %s binding", async change => { + const identity = await setup(); + const cwd = await materializeIsolatedTaskDirectory({ ...identity, attemptId: "prior-attempt" }); + await writeFile(path.join(cwd, "saved.txt"), "preserved"); + let reuseCwd = cwd; + if (change === "other-company") reuseCwd = cwd.replace(identity.companyId, "other-company"); + if (change === "other-issue") reuseCwd = cwd.replace(identity.issueId, "other-issue"); + if (change === "outside") reuseCwd = path.dirname(path.dirname(cwd)); + if (change === "traversal") reuseCwd = `${cwd}/../prior-attempt`; + if (change === "missing") reuseCwd = path.join(path.dirname(cwd), "missing-attempt"); + if (change === "symlink") { + reuseCwd = path.join(path.dirname(cwd), "linked-attempt"); + await symlink(cwd, reuseCwd); + } + await expect(materializeIsolatedTaskDirectory({ ...identity, reuseCwd })).rejects.toThrow(); + if (change === "missing") await expect(readFile(path.join(reuseCwd, "saved.txt"))).rejects.toThrow(); + expect(await readFile(path.join(cwd, "saved.txt"), "utf8")).toBe("preserved"); + }, + ); + it("round-trips sandbox output into the task directory and stages it on the next turn", async () => { const identity = await setup(); const cwd = await materializeIsolatedTaskDirectory(identity); diff --git a/server/src/services/isolated-task-directory.ts b/server/src/services/isolated-task-directory.ts index 9b0298cfe2b..26238976e08 100644 --- a/server/src/services/isolated-task-directory.ts +++ b/server/src/services/isolated-task-directory.ts @@ -26,20 +26,34 @@ export async function materializeIsolatedTaskDirectory(input: { companyId: string; issueId: string; attemptId?: string; + /** Validate retained files in place; never recreate a missing directory. */ + reuseCwd?: string; }): Promise { // Files belong to the task, so reassignment preserves the same workspace. - const identities = [input.companyId, input.issueId, ...(input.attemptId ? [input.attemptId] : [])]; + let identities = [input.companyId, input.issueId, ...(input.attemptId ? [input.attemptId] : [])]; if (identities.some((value) => !/^[a-zA-Z0-9_-]+$/.test(value))) { throw new Error("Invalid isolated task workspace identity"); } const root = resolvePaperclipInstanceRoot(); - await mkdir(root, { recursive: true }); + if (!input.reuseCwd) await mkdir(root, { recursive: true }); let cwd = await realpath(root); + let namespace = input.attemptId ? "isolated-workspace-attempts" : "isolated-workspaces"; + if (input.reuseCwd) { + const segments = path.relative(cwd, input.reuseCwd).split(path.sep); + if (input.attemptId || path.resolve(input.reuseCwd) !== input.reuseCwd || + segments[1] !== input.companyId || segments[2] !== input.issueId || + !((segments[0] === "isolated-workspaces" && segments.length === 3) || + (segments[0] === "isolated-workspace-attempts" && segments.length === 4 && /^[a-zA-Z0-9_-]+$/.test(segments[3])))) { + throw new Error("Isolated task workspace path is not this task's directory"); + } + namespace = segments[0]; + identities = segments.slice(1); + } // A retry is a sibling namespace, never a descendant of the retained root. // An old copyback operation must not be able to overwrite its replacement. - for (const segment of [input.attemptId ? "isolated-workspace-attempts" : "isolated-workspaces", ...identities]) { + for (const segment of [namespace, ...identities]) { cwd = path.join(cwd, segment); - await mkdir(cwd, { mode: 0o700 }).catch((error: NodeJS.ErrnoException) => { + if (!input.reuseCwd) await mkdir(cwd, { mode: 0o700 }).catch((error: NodeJS.ErrnoException) => { if (error.code !== "EEXIST") throw error; }); const stat = await lstat(cwd); diff --git a/server/src/services/native-runtime/native-failed-workspace-recovery.test.ts b/server/src/services/native-runtime/native-failed-workspace-recovery.test.ts index 22570efdb0f..3d5f155bd14 100644 --- a/server/src/services/native-runtime/native-failed-workspace-recovery.test.ts +++ b/server/src/services/native-runtime/native-failed-workspace-recovery.test.ts @@ -4,7 +4,7 @@ import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { activityLog, agents, companies, createDb, environmentLeases, environments, externalAgentHolds, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; import { getExecutionBlocker } from "../execution-blocker.js"; -import { markExecutionReconciliation, validateExecutionReconciliation } from "../execution-recovery-resolution.js"; +import { markExecutionReconciliation, settleUnrecoverableExecutions, validateExecutionReconciliation } from "../execution-recovery-resolution.js"; import { prepareSandboxStopAndRetain, settleStopOnlyCleanup } from "../sandbox-stop-and-retain.js"; import { preserveCancelledNativeWorkspaces, preserveNativeFailedWorkspace } from "./native-failed-workspace-recovery.js"; import type { NativeRestartRecoveryClaim } from "./native-restart-recovery.js"; @@ -51,6 +51,44 @@ const support = externalUrl ? { supported: true } : await getEmbeddedPostgresTes const readLease = async (id: string) => (await db.select().from(environmentLeases).where(eq(environmentLeases.id, id)))[0]!; const actions = async (issueId: string) => db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId)); + it.each(["settled", "historical", "live_owner", "retry", "nonterminal", "wrong_wake", "wrong_hold_run", "no_hold"])("retires only an exact superseded native resume obligation: %s", async scenario => { + const f = await seed(), actionId = randomUUID(); + await db.insert(issueRecoveryActions).values({ id: actionId, companyId: f.companyId, sourceIssueId: f.issueId, + kind: "active_run_watchdog", status: "active", ownerType: "agent", ownerAgentId: f.agentId, + returnOwnerAgentId: f.agentId, cause: "native_session_interrupted", fingerprint: randomUUID(), + evidence: { runId: f.runId }, nextAction: "Resume the exact persisted session.", + wakePolicy: { kind: "resume_native_run", runId: f.runId }, attemptCount: 1, maxAttempts: 3 }); + await preserveNativeFailedWorkspace(db, f.run); + const hold = (await actions(f.issueId)).find(action => action.id !== actionId)!; + expect(hold.evidence.automaticRecovery).toMatchObject({ replay: "blocked", actionOutcome: "unknown" }); + if (scenario === "live_owner") await db.update(nativeRunFinalizations).set({ leaseOwner: "live-controller" }).where(eq(nativeRunFinalizations.runId, f.runId)); + if (scenario === "retry") await db.update(nativeRunFinalizations).set({ nextAttemptAt: new Date(Date.now() + 60_000) }).where(eq(nativeRunFinalizations.runId, f.runId)); + if (scenario === "nonterminal") await db.update(nativeRunFinalizations).set({ phase: "observed" }).where(eq(nativeRunFinalizations.runId, f.runId)); + if (scenario === "wrong_wake") await db.update(issueRecoveryActions).set({ wakePolicy: { kind: "resume_native_run", runId: randomUUID() } }).where(eq(issueRecoveryActions.id, actionId)); + if (scenario === "wrong_hold_run") await db.update(issueRecoveryActions).set({ evidence: { ...hold.evidence, runId: randomUUID() } }).where(eq(issueRecoveryActions.id, hold.id)); + if (scenario === "no_hold") await db.delete(issueRecoveryActions).where(eq(issueRecoveryActions.id, hold.id)); + const beforeRun = await readRun(f.runId), beforeLease = await readLease(f.lease.id); + const beforeBlocker = await getExecutionBlocker(db, f.companyId, f.issueId); + const [beforeTask] = await db.select().from(issues).where(eq(issues.id, f.issueId)); + await settleUnrecoverableExecutions(db); + if (scenario === "historical") await settleUnrecoverableExecutions(db); + const retired = (await actions(f.issueId)).find(action => action.id === actionId)!; + const shouldRetire = scenario === "settled" || scenario === "historical"; + expect(retired.status).toBe(shouldRetire ? "cancelled" : "active"); + if (shouldRetire) { + expect(retired).toMatchObject({ outcome: "cancelled", wakePolicy: null, + evidence: { runId: f.runId, supersedingRecoveryActionId: hold.id } }); + expect((await actions(f.issueId)).find(action => action.id === hold.id)).toEqual(hold); + expect(await db.select().from(activityLog).where(and(eq(activityLog.entityId, f.issueId), + eq(activityLog.action, "issue.execution_recovery_settled")))).toHaveLength(1); + } + expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toEqual(beforeBlocker); + expect(await readRun(f.runId)).toEqual(beforeRun); + expect(await readLease(f.lease.id)).toEqual(beforeLease); + expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0]).toEqual(beforeTask); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(1); + }); + it.each(["acknowledged", "warm", "pending", "foreign", "owned"])("pins historical cancelled work only after an exact Stop acknowledgement: %s", async kind => { const f = await seed(); if (kind === "warm") await db.update(environmentLeases).set({ leasePolicy: "reuse_by_environment" }).where(eq(environmentLeases.id, f.lease.id)); diff --git a/ui/src/components/AgentConfigForm.render.test.tsx b/ui/src/components/AgentConfigForm.render.test.tsx index 97465cbb26b..7b8037b275b 100644 --- a/ui/src/components/AgentConfigForm.render.test.tsx +++ b/ui/src/components/AgentConfigForm.render.test.tsx @@ -1640,6 +1640,86 @@ describe("AgentConfigForm environment selector", () => { expect(result.container.textContent).not.toContain("Cannot read properties of undefined"); }); + it("qualifies an existing native runner's saved computer before its normal Test", async () => { + const computer = makeEnvironment({ id: "saved-computer", driver: "sandbox", status: "active", + config: { provider: "daytona", snapshot: "managed-snapshot" } }); + const order: string[] = []; + mockAgentsApi.qualifyRunner.mockImplementation(async () => { + order.push("qualification"); + return { ok: true, driver: "sandbox", details: { platformQualificationRecorded: true } }; + }); + mockAgentsApi.testEnvironment.mockImplementation(async () => { + order.push("runtime"); + return { adapterType: "paperclip_runner", status: "pass", checks: [], testedAt: new Date(0).toISOString() }; + }); + const result = await renderForm([computer], { adapterType: "paperclip_runner", defaultEnvironmentId: computer.id, + adapterConfig: { provider: "opencode", model: "example/model" } }, { showAdapterTestEnvironmentButton: true }); + roots.push(result.root); + await runTest(result.container); + expect(mockAgentsApi.qualifyRunner).toHaveBeenCalledWith("agent-1", computer.id, "company-1"); + expect(order).toEqual(["qualification", "runtime"]); + expect(result.onSave).not.toHaveBeenCalled(); + }); + + it.each(["failed", "missing-proof"])("does not run the native runtime test after %s saved-computer qualification", async state => { + const computer = makeEnvironment({ id: "saved-computer", driver: "sandbox", status: "active", + config: { provider: "daytona", snapshot: "managed-snapshot" } }); + mockAgentsApi.qualifyRunner.mockResolvedValue({ ok: state !== "failed", driver: "sandbox", summary: "Computer test failed.", + details: { platformQualificationRecorded: false } }); + const result = await renderForm([computer], { adapterType: "paperclip_runner", defaultEnvironmentId: computer.id, + adapterConfig: { provider: "opencode" } }, { showAdapterTestEnvironmentButton: true }); + roots.push(result.root); + await runTest(result.container); + expect(mockAgentsApi.qualifyRunner).toHaveBeenCalledTimes(1); + expect(mockAgentsApi.testEnvironment).not.toHaveBeenCalled(); + expect(result.container.textContent).toContain("Computer test failed."); + }); + + it.each(["openai_dot", "muse", "pool"])("does not turn the %s readiness preview into a computer qualification", async provider => { + const computer = makeEnvironment({ id: "saved-computer", driver: "sandbox", status: "active", + config: { provider: "daytona", snapshot: "managed-snapshot" } }); + mockAgentsApi.qualifyRunner.mockResolvedValue({ ok: true, driver: "sandbox", details: { platformQualificationRecorded: true } }); + const result = await renderForm([computer], { adapterType: "paperclip_runner", defaultEnvironmentId: computer.id, + adapterConfig: { provider: provider === "pool" ? "opencode" : provider }, + ...(provider === "pool" ? { runtimeConfig: { aiConnection: { mode: "router", connectionId: "a8d3a9a3-9866-49d8-93d6-c497ae090d05" } } } : {}) }, + { showAdapterTestEnvironmentButton: true, compactTestFeedback: true }); + roots.push(result.root); + await clickByText(result.container, provider === "pool" ? "Run test" : "Check setup"); + expect(mockAgentsApi.testEnvironment).toHaveBeenCalledTimes(1); + expect(mockAgentsApi.qualifyRunner).not.toHaveBeenCalled(); + }); + + it("does not run a native test after settings change during computer qualification", async () => { + const computer = makeEnvironment({ id: "saved-computer", driver: "sandbox", status: "active", + config: { provider: "daytona", snapshot: "managed-snapshot" } }); + let finish: (() => void) | undefined; + mockAgentsApi.qualifyRunner.mockImplementation(() => new Promise(resolve => { + finish = () => resolve({ ok: true, driver: "sandbox", details: { platformQualificationRecorded: true } }); + })); + const result = await renderForm([computer], { adapterType: "paperclip_runner", defaultEnvironmentId: computer.id, + adapterConfig: { provider: "opencode" } }, { showAdapterTestEnvironmentButton: true }); + roots.push(result.root); + await runTest(result.container); + expect(mockAgentsApi.qualifyRunner).toHaveBeenCalledTimes(1); + await act(async () => setInputValue(result.container.querySelector('[placeholder="Agent name"]')!, "Changed while testing")); + await act(async () => finish!()); + await flushReact(); + expect(mockAgentsApi.testEnvironment).not.toHaveBeenCalled(); + expect(result.container.textContent).toContain("Agent settings changed during the test."); + }); + + it("does not qualify a draft computer through the native runtime Test", async () => { + const computers = ["saved-computer", "draft-computer"].map(id => makeEnvironment({ id, driver: "sandbox", status: "active", + config: { provider: "daytona", snapshot: "managed-snapshot" } })); + const result = await renderForm(computers, { adapterType: "paperclip_runner", defaultEnvironmentId: computers[0]!.id, + adapterConfig: { provider: "opencode" } }, { showAdapterTestEnvironmentButton: true }); + roots.push(result.root); + await selectEnvironment(result.container, computers[1]!.id); + await runTest(result.container); + expect(mockAgentsApi.qualifyRunner).not.toHaveBeenCalled(); + expect(result.onSave).not.toHaveBeenCalled(); + }); + it("omits undefined adapter config entries when testing a create form with the default model", async () => { const result = await renderCreateForm([ makeEnvironment({ id: "local-1", name: "Local", driver: "local" }), diff --git a/ui/src/components/AgentConfigForm.tsx b/ui/src/components/AgentConfigForm.tsx index 55d1ff4aa94..1d7357b6f5d 100644 --- a/ui/src/components/AgentConfigForm.tsx +++ b/ui/src/components/AgentConfigForm.tsx @@ -912,11 +912,11 @@ export function AgentConfigForm(props: AgentConfigFormProps) { && Boolean(runnerAvailability && !runnerAvailability.disabled && !runnerAvailability.overriddenBuiltin) && Boolean(nativeRunnerAvailability && !nativeRunnerAvailability.disabled && !nativeRunnerAvailability.overriddenBuiltin); const upgradeWarnings = harnessType === "claude_local" ? normalizeClaudeRunnerOptions(effectiveConfig).warnings : []; - const upgradeTargetRef = useRef(""); - const upgradeTarget = JSON.stringify([!isCreate ? props.agent.id : null, selectedCompanyId, adapterType, + const runnerSettingsTargetRef = useRef(""); + const runnerSettingsTarget = JSON.stringify([!isCreate ? props.agent.id : null, selectedCompanyId, adapterType, rawCurrentDefaultEnvironmentId, upgradeEnvironmentId, upgradeEnvironment?.config, effectiveConfig, overlay]); - upgradeTargetRef.current = upgradeTarget; - useEffect(() => () => { upgradeTargetRef.current = ""; }, []); + runnerSettingsTargetRef.current = runnerSettingsTarget; + useEffect(() => () => { runnerSettingsTargetRef.current = ""; }, []); const showRunnerUpgrade = !isCreate && ["codex_local", "claude_local"].includes(props.agent.adapterType) && adapterType === props.agent.adapterType @@ -931,7 +931,7 @@ export function AgentConfigForm(props: AgentConfigFormProps) { const nextOverlay = flushedEnv ? { ...overlay, adapterConfig: { ...overlay.adapterConfig, env: flushedEnv } } : overlay; - const pendingTarget = upgradeTargetRef.current; + const pendingTarget = runnerSettingsTargetRef.current; if (runnerAvailability?.defaultRunner !== "paperclip") { if (!canQualifyRunnerUpgrade || !upgradeEnvironmentId || !selectedCompanyId) { throw new Error("Could not determine which environment to verify. Retry the upgrade."); @@ -940,7 +940,7 @@ export function AgentConfigForm(props: AgentConfigFormProps) { adapterType: harnessType, readAdapters: () => adaptersApi.list({ companyId: selectedCompanyId, environmentId: upgradeEnvironmentId }), probeEnvironment: () => agentsApi.qualifyRunner(agentId, upgradeEnvironmentId, selectedCompanyId), - isCurrent: () => upgradeTargetRef.current === pendingTarget, + isCurrent: () => runnerSettingsTargetRef.current === pendingTarget, }); if (!qualified) throw new Error("Agent settings changed during verification. Retry the upgrade with the current settings."); queryClient.setQueryData(queryKeys.adapters.availability(selectedCompanyId, rawCurrentDefaultEnvironmentId || null), qualified.adapters); @@ -1208,6 +1208,17 @@ export function AgentConfigForm(props: AgentConfigFormProps) { const aiConnection = isCreate ? undefined : aiRuntimeConnectionBindingSchema.safeParse( (overlay.runtime.runtimeConfig as Record | undefined)?.aiConnection ?? props.agent.runtimeConfig.aiConnection, ).data; + if (!isCreate && props.agent.adapterType === "paperclip_runner" && adapterType === "paperclip_runner" + && !isExternalRunner && aiConnection?.mode !== "router" && !isDirty && environmentId && canQualifyRunnerEnvironment(environmentList.find(environment => environment.id === environmentId))) { + const target = runnerSettingsTargetRef.current; + // Qualify only the saved computer. The strict agent-scoped endpoint + // accepts no draft provider config, credentials or replacement target. + const qualification = await agentsApi.qualifyRunner(props.agent.id, environmentId, selectedCompanyId); + if (runnerSettingsTargetRef.current !== target) throw new Error("Agent settings changed during the test. Run test again."); + if (qualification.ok !== true || qualification.details?.platformQualificationRecorded !== true) { + throw new Error(qualification.summary || "Could not verify the saved computer. Run test again."); + } + } if (props.compactTestFeedback) { const providerAdapter = adapterType === "paperclip_runner" ? adapterConfig.provider === "codex" ? "codex_local" diff --git a/ui/src/components/IssueRunLedger.test.tsx b/ui/src/components/IssueRunLedger.test.tsx index 117f27ff3b1..f94900ce410 100644 --- a/ui/src/components/IssueRunLedger.test.tsx +++ b/ui/src/components/IssueRunLedger.test.tsx @@ -420,6 +420,23 @@ describe("IssueRunLedger", () => { expect(container.textContent).toContain("all 2 terminal (1 done, 1 cancelled)"); }); + it("shows the current execution instruction and clears stale instructions after a recorded successor", () => { + const execution = { + phase: "recovery_needed" as const, label: "Stopped", cause: "native_continuation_requires_reconciliation", + lastConfirmedActivityAt: null, retryAt: null, attempt: 1, maxAttempts: 3, + recoveryOwner: null, nextAction: "Saved work recovered. Send a message to continue.", + permittedActions: ["inspect_run"] as const, predecessorRunId: null, successorRunId: null, + }; + const stopped = createRun({ status: "failed", nextAction: "Repair the retained sandbox.", + execution: { ...execution, permittedActions: ["inspect_run"] } }); + renderLedger({ runs: [stopped] }); + expect(container.textContent).toContain(execution.nextAction); + expect(container.textContent).not.toContain("Repair the retained sandbox."); + renderLedger({ runs: [{ ...stopped, execution: { ...stopped.execution!, phase: "completed", + label: "Continued in another run", nextAction: null, successorRunId: "successor" } }] }); + expect(container.textContent).not.toContain("Next action:"); + }); + it("uses wrapping-friendly markup for long next action text", () => { renderLedger({ runs: [ diff --git a/ui/src/components/IssueRunLedger.tsx b/ui/src/components/IssueRunLedger.tsx index 6cd7cdee99e..1bee4217d05 100644 --- a/ui/src/components/IssueRunLedger.tsx +++ b/ui/src/components/IssueRunLedger.tsx @@ -842,6 +842,7 @@ export function IssueRunLedgerContent({ ); } const run = item.run; + const nextAction = run.execution ? run.execution.nextAction : run.nextAction; const liveness = livenessCopyForRun(run); const stopReason = stopReasonLabel(run); const duration = formatDuration(run.startedAt, run.finishedAt); @@ -1001,13 +1002,13 @@ export function IssueRunLedgerContent({ /> ) : null} - {run.nextAction ? ( + {nextAction ? (
Next action:{" "} - {run.nextAction} + {nextAction}
) : null} diff --git a/ui/storybook/stories/agents/codex-runner/upgrade.stories.tsx b/ui/storybook/stories/agents/codex-runner/upgrade.stories.tsx index 8f9b103a3f9..d1fe14c18a7 100644 --- a/ui/storybook/stories/agents/codex-runner/upgrade.stories.tsx +++ b/ui/storybook/stories/agents/codex-runner/upgrade.stories.tsx @@ -16,17 +16,21 @@ type Scenario = { upgraded?: boolean; running?: boolean; delay?: number; + nativeComputerTest?: boolean; }; -function installUpgradeFixture({ harness = "codex_local", failOnce = false, incompatible = false, unavailable = false, upgraded = false, running = false, delay = 0 }: Scenario = {}) { +function installUpgradeFixture({ harness = "codex_local", failOnce = false, incompatible = false, unavailable = false, upgraded = false, running = false, delay = 0, nativeComputerTest = false }: Scenario = {}) { resetOnboardingFixtureState(); setOnboardingFixtureState({ environments: "local", authSignal: "present", localLoginStatus: "ready" }); const profile = harness === "claude_local" ? { provider: "acpx", acpxAgent: "claude" } : { provider: "codex" }; + const savedComputer = { id: "environment-native-recovery", name: "Paperclip Computer", driver: "sandbox", status: "active", + config: { provider: "daytona", snapshot: "managed-snapshot" }, metadata: { managedByPaperclip: true } }; + let qualified = false; let agent = { ...storybookAgents[0]!, urlKey: "migration-agent", name: harness === "claude_local" ? "ClaudeWriter" : "CodexCoder", - status: running ? "running" : "idle", lifecycleState: "ready", defaultEnvironmentId: null, - adapterType: upgraded ? "paperclip_runner" : harness, + status: running ? "running" : "idle", lifecycleState: "ready", defaultEnvironmentId: nativeComputerTest ? savedComputer.id : null, + adapterType: upgraded || nativeComputerTest ? "paperclip_runner" : harness, adapterConfig: { model: harness === "claude_local" ? "claude-sonnet-4-6" : "gpt-5.4", modelReasoningEffort: "medium", timeoutSec: 180, - ...(upgraded ? profile : {}), ...(incompatible ? { chrome: true } : {}), env: { MIGRATION_MARKER: { type: "plain", value: "keep-this-setting" } } }, + ...(nativeComputerTest ? { provider: "opencode" } : upgraded ? profile : {}), ...(incompatible ? { chrome: true } : {}), env: { MIGRATION_MARKER: { type: "plain", value: "keep-this-setting" } } }, runtimeConfig: { heartbeat: { enabled: false, wakeOnDemand: true } }, chainOfCommand: [], access: { canAssignTasks: true, taskAssignSource: "explicit_grant", membership: null, grants: [] }, }; @@ -34,6 +38,16 @@ function installUpgradeFixture({ harness = "codex_local", failOnce = false, inco const fixtureFetch: typeof fetch = async (input, init) => { const url = new URL(typeof input === "string" ? input : input instanceof URL ? input.href : input.url, window.location.origin); const method = init?.method ?? "GET"; + if (nativeComputerTest && url.pathname === `/api/companies/${agent.companyId}/environments`) return Response.json([savedComputer]); + if (nativeComputerTest && url.pathname === `/api/agents/${agent.id}/qualify-runner`) { + if (JSON.parse(String(init?.body ?? "{}")).expectedEnvironmentId !== savedComputer.id) return Response.json({ error: "Computer changed." }, { status: 409 }); + qualified = true; + return Response.json({ ok: true, driver: "sandbox", details: { platformQualificationRecorded: true } }); + } + if (nativeComputerTest && url.pathname.endsWith("/adapters/paperclip_runner/test-environment")) { + return Response.json({ adapterType: "paperclip_runner", status: qualified ? "pass" : "fail", testedAt: new Date().toISOString(), + checks: [{ code: "native_runtime_test", level: qualified ? "info" : "error", message: qualified ? "Native runtime ready." : "Saved computer qualification is stale." }] }); + } if (url.pathname === "/api/adapters") { const response = await previous(input, init); return Response.json((await response.json()).map((item: { type: string }) => ["codex_local", "claude_local"].includes(item.type) @@ -140,3 +154,18 @@ export const UnavailableEnvironment: Story = { parameters: { runnerUpgrade: { un export const AgentRunning: Story = { parameters: { runnerUpgrade: { running: true } } }; export const Mobile: Story = { globals: { viewport: { value: "mobile1", isRotated: false } } }; export const Light: Story = { globals: { theme: "light" } }; + +export const NativeComputerRecovery: Story = { + name: "Native saved computer Test", + parameters: { runnerUpgrade: { nativeComputerTest: true }, paperclip: { + notes: "Production native agent page. The normal Run test action qualifies its saved computer before runtime testing. Both responses are simulated; real provider qualification is separate.", + } }, + play: async ({ canvasElement }) => { + const page = within(canvasElement); + await userEvent.click(await page.findByRole("button", { name: /^Run test$/ })); + await expect(await page.findByRole("heading", { name: "Connection successful" })).toBeVisible(); + await userEvent.click(page.getByText("Test details", { exact: true })); + await expect(await page.findByText("Native runtime ready.")).toBeVisible(); + await expect(page.queryByRole("button", { name: "Upgrade runner" })).not.toBeInTheDocument(); + }, +};