From 8a2bbf09bc022dda5c44b7a343da14c7ec29cd1d Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 15 Oct 2026 08:31:52 -0500 Subject: [PATCH] fix(grok): settle false file holds through inspected recovery Preserve stopped-run fences and no-replay while allowing verified new user work. Honor disabled workspace access at the dispatch-failure retention owner. Co-Authored-By: Paperclip --- doc/grok-bot-agent.md | 11 ++ ...026-10-13-grok-bot-production-readiness.md | 19 ++++ server/src/__tests__/grok-bot-routine.test.ts | 107 +++++++++++++++++- server/src/services/execution-blocker.ts | 8 +- server/src/services/grok-bot/routine.ts | 5 +- .../services/grok-bot/workspace-inspection.ts | 87 ++++++++++++++ .../heartbeat/runtime-execution.test.ts | 16 +-- .../services/heartbeat/runtime-execution.ts | 3 +- .../native-failed-workspace-recovery.test.ts | 10 +- 9 files changed, 250 insertions(+), 16 deletions(-) create mode 100644 server/src/services/grok-bot/workspace-inspection.ts diff --git a/doc/grok-bot-agent.md b/doc/grok-bot-agent.md index 527763a041a..54a9cf49f2a 100644 --- a/doc/grok-bot-agent.md +++ b/doc/grok-bot-agent.md @@ -82,3 +82,14 @@ to inspect it. Paperclip shows reported activity, not a complete remote transcri or computer recording. The admitted Grok execution contract has no access to the Runner's workspace files. Its managed Runner still keeps durable recovery state, but stopping that session does not require workspace file repair. Inspect the Bot in Grok, clear the connection's recovery hold in the agent's Runtime settings, and verify the routine again before assigning new work. This records inspection without asserting global Grok termination. + +Historical stopped Grok runs can have an incorrect workspace-file hold from an +earlier release. Runtime shows Recovery needed for these runs even when the +routine is healthy. After inspecting Grok, use Clear recovery hold and verify +the connection again. Paperclip retires the false file obligation only when the +original admission denied workspace access, the assignment is fenced, the +coordinator has settled, and the exact managed Runner leases have stop receipts. +The inspection preserves the old result, action uncertainty, and no-replay hold. +Send a new task message to request fresh work. Inspection does not certify global +Grok termination or delete any previously retained allocation. Such allocations +still use the existing environment cleanup controls. diff --git a/doc/plans/2026-10-13-grok-bot-production-readiness.md b/doc/plans/2026-10-13-grok-bot-production-readiness.md index 7cf02c3c803..b80579ba894 100644 --- a/doc/plans/2026-10-13-grok-bot-production-readiness.md +++ b/doc/plans/2026-10-13-grok-bot-production-readiness.md @@ -261,3 +261,22 @@ assignment authority. Disconnect completes while storage is pending, and the late upload returns 403 with no attachment or publication receipt. Person write checks remain in place. The full repository command was interrupted when this new review finding arrived; its partial output is not a passing test result. + +## October 15 — Stop review follow-up + +Review found that hiding the historical file-repair card alone left the stored +file obligation blocking fresh task input. Runtime inspection now finds these +holds even after routine recovery. It retires only the false file fields after +exact run, assignment, coordinator and managed-lease cleanup checks, archives +the previous evidence, and preserves no-replay. A real authenticated readiness +exchange and new user input are still required. No global provider-stop claim is +created. Dot historical recovery controls remain available. + +The later dispatch-failure owner also now honors the validated no-workspace +contract before asking for sandbox retention. Both external providers and the +writable native path pass the runtime boundary regression. The review follow-up +passes 128 focused protocol, recovery, cancellation and runtime checks. An initial +negative assertion saw the earlier readiness gate instead of the revoked-session +gate; the corrected test checks both before and after re-verification. Server +typecheck passes after narrowing the native workspace union. Final current-head +CI, review, deployed historical recovery and fresh Stop are still release gates. diff --git a/server/src/__tests__/grok-bot-routine.test.ts b/server/src/__tests__/grok-bot-routine.test.ts index e54a3f286f2..454e8909653 100644 --- a/server/src/__tests__/grok-bot-routine.test.ts +++ b/server/src/__tests__/grok-bot-routine.test.ts @@ -8,7 +8,7 @@ import { createHash, randomBytes, randomUUID } from "node:crypto"; import { and, eq, sql } from "drizzle-orm"; import { beforeAll, afterAll, afterEach, describe, expect, it, vi } from "vitest"; import { agents, authUsers, companies, companyMemberships, createDb, dotAgentBindings, grokBotDispatches, - grokBotRoutines, mcpOauthGrants, activityLog, issues, heartbeatRuns, dotRunnerAssignments, dotRunnerOperations, dotMailboxItems, nativeRunFinalizations } from "@paperclipai/db"; + grokBotRoutines, mcpOauthGrants, activityLog, issues, heartbeatRuns, dotRunnerAssignments, dotRunnerOperations, dotMailboxItems, nativeRunFinalizations, environmentLeases, environments, issueRecoveryActions, issueComments } from "@paperclipai/db"; import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { accessService } from "../services/access.js"; import { instanceSettingsService } from "../services/instance-settings.js"; @@ -20,6 +20,10 @@ import * as secretModule from "../services/secrets.js"; import * as externalAuthority from "../services/external-bot-authority.js"; import { type GrokRoutineWake } from "../services/grok-bot/webhook.js"; +import { remoteTerminationReceipt } from "../services/remote-execution-termination.js"; +import { admitExplicitNativeContinuation } from "../services/explicit-native-continuation.js"; +import { hasRequiredWorkspaceRecovery } from "../services/workspace-restore-recovery-state.js"; + let temporary: Awaited>; let db: ReturnType; beforeAll(async () => { @@ -479,3 +483,104 @@ describe("durable Grok routine qualification", () => { }); }); + +// Exercise the public recovery service and the actual native admission boundary, +// including a stale file obligation persisted before this fix. +describe("Grok inspection of historical workspace-less Stop", () => { + async function historicalHold() { + const f = await fixture(); + const { run, task, work } = await admittedWork(f, "accepted"); + const finishedAt = new Date(Date.now() - 1000); + const fileHold = { workspaceRestoreFailure: "restore_failed", workspaceRestoreRecovery: { + schema: "paperclip.workspace-restore-recovery.v1", runId: run.id, leaseIds: [] as string[], + } }; + const [environment] = await db.insert(environments).values({ name: `Grok ${run.id}`, driver: "sandbox" }).returning(); + const [lease] = await db.insert(environmentLeases).values({ companyId: f.company.id, issueId: task.id, + heartbeatRunId: run.id, environmentId: environment!.id, provider: "daytona", providerLeaseId: randomUUID(), + status: "released", releasedAt: new Date(), cleanupStatus: "success" }).returning(); + await db.update(environmentLeases).set({ metadata: { remoteExecutionTermination: + remoteTerminationReceipt(lease!, { providerLeaseId: lease!.providerLeaseId, state: "destroyed" }) } }).where(eq(environmentLeases.id, lease!.id)); + fileHold.workspaceRestoreRecovery.leaseIds.push(lease!.id); + await db.update(heartbeatRuns).set({ status: "cancelled", driverKind: "grok_bot_mcp", finishedAt, + nativeIssueId: task.id, issueId: task.id, contextSnapshot: { issueId: task.id }, + runnerProfileJson: { nativeExecutionInput: { schema: "paperclip.native-execution-input.v6", + binding: { runId: run.id, companyId: f.company.id, agentId: f.agent.id, issueId: task.id }, + provider: { kind: "grok_bot" }, workspace: { access: "none", cwd: null } } }, + resultJson: { ...fileHold, cancellation: { source: "user", reason: "Paperclip work stopped" } }, + }).where(eq(heartbeatRuns.id, run.id)); + await db.update(issues).set({ status: "blocked", executionRunId: null, checkoutRunId: null }).where(eq(issues.id, task.id)); + await db.update(nativeRunFinalizations).set({ phase: "terminal_failure", leaseOwner: null, + leaseExpiresAt: null, recoveryState: "blocked" }).where(eq(nativeRunFinalizations.runId, run.id)); + await db.update(dotRunnerAssignments).set({ status: "fenced" }).where(eq(dotRunnerAssignments.id, work.id)); + const [action] = await db.insert(issueRecoveryActions).values({ companyId: f.company.id, sourceIssueId: task.id, + kind: "active_run_watchdog", cause: "native_continuation_requires_reconciliation", fingerprint: run.id, + status: "resolved", outcome: "blocked", nextAction: "Recover files.", + evidence: { ...fileHold, runId: run.id, automaticRecovery: { replay: "blocked", actionOutcome: "unknown" } }, + }).returning(); + const [comment] = await db.insert(issueComments).values({ companyId: f.company.id, issueId: task.id, + authorType: "user", authorUserId: f.principal.grant.userId, body: "Start a fresh task turn after inspection." }).returning(); + const continueWork = (dryRun = true) => db.transaction(async tx => { + await tx.select().from(issues).where(eq(issues.id, task.id)).for("update"); + return admitExplicitNativeContinuation({ db: tx as unknown as typeof db, companyId: f.company.id, + issueId: task.id, agentId: f.agent.id, actorType: "user", actorId: f.principal.grant.userId, + reason: "issue_commented", commentId: comment!.id, successorRunId: randomUUID(), dryRun }); + }); + return { ...f, run, task, work, lease: lease!, action: action!, continueWork }; + } + + it("offers inspection for an old task hold and admits only new explicit input after inspection and Runner cleanup", async () => { + const f = await historicalHold(); + f.delivery.mockClear(); + expect(await f.service.recoveryNeeded(f.company.id, f.agent.id)).toBe(true); + expect(await f.continueWork()).toBeNull(); + expect(await f.service.recover(f.company.id, f.agent.id, f.principal.grant.userId)).toMatchObject({ + requiresVerification: true, remoteTerminationConfirmed: false, + }); + expect(await f.service.state(f.company.id, f.agent.id, f.pairing.bindingId)).toMatchObject({ verified: false, recoveryNeeded: false }); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)); + expect(hasRequiredWorkspaceRecovery(run!.resultJson)).toBe(false); + expect(run).toMatchObject({ status: "cancelled", resultJson: { externalWorkspaceInspection: { + inspectedBy: f.principal.grant.userId, remoteTerminationConfirmed: false, + previousWorkspaceRestoreFailure: "restore_failed", runnerLeaseIds: [f.lease.id], + } } }); + const [action] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, f.action.id)); + expect(action).toMatchObject({ status: "resolved", outcome: "blocked", evidence: { + automaticRecovery: { replay: "blocked", actionOutcome: "unknown" }, externalWorkspaceInspection: { workspaceAccess: "none" }, + } }); + expect(action!.evidence).not.toHaveProperty("workspaceRestoreRecovery"); + expect(f.delivery).not.toHaveBeenCalled(); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.company.id))).toHaveLength(1); + await expect(dotRunnerBroker(db).operation(f.principal, f.work.id, randomUUID(), "progress", { text: "late effect" })).rejects.toThrow("wake verification"); + // Connection recovery still requires a real authenticated challenge exchange. + const row = await verification(f); await f.service.send(f.company.id, f.agent.id, row.id); + await f.service.acknowledge(f.principal, acknowledgement(row)); + await expect(dotRunnerBroker(db).operation(f.principal, f.work.id, randomUUID(), "progress", { text: "late effect" })).rejects.toThrow("revoked"); + expect(await f.continueWork(false)).toMatchObject({ previousRunId: f.run.id }); + const [settled] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, f.action.id)); + expect(settled!.evidence.automaticRecovery).toMatchObject({ replay: "explicit_user_continuation", actionOutcome: "unknown" }); + expect((await db.select().from(dotRunnerAssignments).where(eq(dotRunnerAssignments.id, f.work.id)))[0]!.status).toBe("fenced"); + await f.service.recover(f.company.id, f.agent.id, f.principal.grant.userId); + expect(await db.select().from(activityLog).where(and(eq(activityLog.companyId, f.company.id), eq(activityLog.action, "grok.workspace_obligation_inspected")))).toHaveLength(1); + }); + + it.each(["cleanup_pending", "receipt_mismatch", "controller_owned", "assignment_active", "running", "operator_revoked", "foreign_company"])("does not retire an old obligation when %s", async condition => { + const f = await historicalHold(); + if (condition === "cleanup_pending") await db.update(environmentLeases).set({ cleanupStatus: "failed" }).where(eq(environmentLeases.id, f.lease.id)); + if (condition === "receipt_mismatch") await db.update(environmentLeases).set({ metadata: { remoteExecutionTermination: { schema: "paperclip.remote-termination.v1", runId: randomUUID() } } }).where(eq(environmentLeases.id, f.lease.id)); + if (condition === "controller_owned") await db.update(nativeRunFinalizations).set({ leaseOwner: "other-owner" }).where(eq(nativeRunFinalizations.runId, f.run.id)); + if (condition === "assignment_active") await db.update(dotRunnerAssignments).set({ status: "accepted" }).where(eq(dotRunnerAssignments.id, f.work.id)); + if (condition === "running") await db.update(heartbeatRuns).set({ status: "running" }).where(eq(heartbeatRuns.id, f.run.id)); + if (condition === "operator_revoked") await accessService(db).setPrincipalPermission(f.company.id, "user", f.principal.grant.userId, "agents:create", false, f.principal.grant.userId); + if (condition === "foreign_company") { + const foreign = await fixture(); + await f.service.recover(foreign.company.id, foreign.agent.id, foreign.principal.grant.userId); + } else if (condition === "running") { + // Not an eligible historical source. Its active assignment is fenced in + // this deliberately malformed fixture, but no file state may be removed. + await f.service.recover(f.company.id, f.agent.id, f.principal.grant.userId); + } else await expect(f.service.recover(f.company.id, f.agent.id, f.principal.grant.userId)).rejects.toThrow(); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)); + expect(hasRequiredWorkspaceRecovery(run!.resultJson)).toBe(true); + expect(await f.continueWork()).toBeNull(); + }); +}); diff --git a/server/src/services/execution-blocker.ts b/server/src/services/execution-blocker.ts index fbaae6e65e5..b451fd41e30 100644 --- a/server/src/services/execution-blocker.ts +++ b/server/src/services/execution-blocker.ts @@ -65,7 +65,7 @@ export async function getExecutionBlocker(db: Db, companyId: string, issueId: st const [run] = runId ? await db.select().from(heartbeatRuns).where(and( eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, runId), )).limit(1) : []; - const withoutWorkspace = Boolean(run && nativeRunHasNoWorkspaceAccess(run)); + const withoutWorkspace = Boolean(run?.driverKind === "grok_bot_mcp" && nativeRunHasNoWorkspaceAccess(run)); const workspaceRepairRequired = Boolean(restoreHold && !withoutWorkspace); const queue = await db.select({ payload: agentWakeupRequests.payload }).from(agentWakeupRequests).where(and( eq(agentWakeupRequests.companyId, companyId), eq(agentWakeupRequests.status, "deferred_issue_execution"), @@ -120,12 +120,16 @@ export async function getExecutionBlocker(db: Db, companyId: string, issueId: st nextAction += " Inspect the run before sending a new message to request continuation."; } - if (withoutWorkspace && (restoreHold || run?.status === "cancelled")) { + if (withoutWorkspace && (restoreHold || run?.status === "cancelled") && !run?.resultJson?.externalWorkspaceInspection) { nextAction = run?.driverKind === "grok_bot_mcp" ? "Inspect this Bot in Grok Bot, then clear the recovery hold and verify the connection in the agent's Runtime settings before assigning more work." : "Inspect the stopped work in the external agent and its connection before requesting continuation."; } + if (run?.resultJson?.externalWorkspaceInspection && !restoreHold) { + nextAction = "Verify the Grok connection, then send a new message to continue. The stopped work will not be replayed."; + } + return { recoveryActionId: action.id, runId, diff --git a/server/src/services/grok-bot/routine.ts b/server/src/services/grok-bot/routine.ts index 791a94357fb..5a06c82c55d 100644 --- a/server/src/services/grok-bot/routine.ts +++ b/server/src/services/grok-bot/routine.ts @@ -10,6 +10,8 @@ import { scheduleAgentLifecycle } from "../agent-lifecycle.js"; import { logActivity } from "../activity-log.js"; import { deliverGrokRoutineWake, validateGrokRoutineCredential, type GrokRoutineDelivery, type GrokRoutineWake } from "./webhook.js"; +import { grokWorkspaceInspectionCandidates, inspectGrokWorkspaceObligations } from "./workspace-inspection.js"; + const digest = (value: string) => createHash("sha256").update(value).digest("hex"); const fail = (message: string) => new McpOAuthError("access_denied", message, 409); export const grokWakeAcknowledgementSchema = z.object({ correlationId: z.uuid(), bindingId: z.uuid(), @@ -44,11 +46,12 @@ export function grokRoutineService(db: Db, deliver: Delivery = deliverGrokRoutin const [uncertain] = await db.select({ id: dispatches.id }).from(dispatches).innerJoin(routines, eq(routines.id, dispatches.routineId)) .where(and(eq(dispatches.companyId, companyId), eq(routines.agentId, agentId), or(eq(dispatches.status, "unknown"), and(eq(dispatches.status, "sending"), lte(dispatches.updatedAt, new Date(Date.now() - 30_000))), and(eq(dispatches.status, "accepted"), lte(dispatches.expiresAt, new Date()))))).limit(1); - return !!held || !!uncertain; + return !!held || !!uncertain || (await grokWorkspaceInspectionCandidates(db, companyId, agentId)).length > 0; } async function recover(companyId: string, agentId: string, operatorId: string) { await authorizeExternalBotOperator(db, companyId, operatorId); + await inspectGrokWorkspaceObligations(db, companyId, agentId, operatorId); await db.transaction(async tx => { // All routine writers serialize on the binding before dispatch/routine rows. // Include retired bindings so recovery cannot bypass an old activity hold. diff --git a/server/src/services/grok-bot/workspace-inspection.ts b/server/src/services/grok-bot/workspace-inspection.ts new file mode 100644 index 00000000000..f1db9a4a851 --- /dev/null +++ b/server/src/services/grok-bot/workspace-inspection.ts @@ -0,0 +1,87 @@ +import { and, eq, inArray, isNotNull, or, sql } from "drizzle-orm"; +import { dotRunnerAssignments, environmentLeases, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, type Db } from "@paperclipai/db"; +import { authorizeExternalBotOperator } from "../external-bot-authority.js"; +import { logActivity } from "../activity-log.js"; +import { McpOAuthError } from "../public-mcp/oauth.js"; +import { hasRemoteTerminationReceipt } from "../remote-execution-termination.js"; +import { hasRequiredWorkspaceRecovery, nativeRunHasNoWorkspaceAccess } from "../workspace-restore-recovery-state.js"; + +/** Old Runner synchronization descriptors could incorrectly require a file + * export for a Bot explicitly admitted without workspace access. Keep the old + * execution fenced and its no-replay hold; inspection only retires that false + * file obligation. Native Runner cleanup never certifies global Grok exit. */ +export async function grokWorkspaceInspectionCandidates(db: Db, companyId: string, agentId: string) { + const runs = await db.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.agentId, agentId), + eq(heartbeatRuns.driverKind, "grok_bot_mcp"), isNotNull(heartbeatRuns.finishedAt), + inArray(heartbeatRuns.status, ["cancelled", "failed", "timed_out", "interrupted"]), + sql`${heartbeatRuns.resultJson}->'workspaceRestoreRecovery'->>'schema' = 'paperclip.workspace-restore-recovery.v1'`, + )); + return runs.filter(run => nativeRunHasNoWorkspaceAccess(run) && hasRequiredWorkspaceRecovery(run.resultJson)); +} + +export async function inspectGrokWorkspaceObligations(db: Db, companyId: string, agentId: string, operatorId: string) { + for (const selected of await grokWorkspaceInspectionCandidates(db, companyId, agentId)) { + // Independent task transactions keep the native task/coordinator/run lock + // order out of the routine writer's binding/routine lock order. A partial + // inspection grants no authority and never schedules work. + await db.transaction(async tx => { + const database = tx as unknown as Db; + await authorizeExternalBotOperator(database, companyId, operatorId); + const issueId = selected.nativeIssueId ?? selected.issueId; + if (!issueId) return; + const [task] = await tx.select().from(issues).where(and( + eq(issues.companyId, companyId), eq(issues.id, issueId), + )).for("update"); + const [coordinator] = await tx.select().from(nativeRunFinalizations).where(and( + eq(nativeRunFinalizations.companyId, companyId), eq(nativeRunFinalizations.runId, selected.id), + eq(nativeRunFinalizations.issueId, issueId), + )).for("update"); + const [run] = await tx.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.id, selected.id), + )).for("update"); + if (!run || !hasRequiredWorkspaceRecovery(run.resultJson)) return; + const fail = () => new McpOAuthError("access_denied", "The stopped task still needs verified Runner cleanup before inspection can clear its file obligation.", 409); + if (!task || !nativeRunHasNoWorkspaceAccess(run) || run.driverKind !== "grok_bot_mcp" + || !run.finishedAt || !["cancelled", "failed", "timed_out", "interrupted"].includes(run.status) + || !coordinator || coordinator.phase !== "terminal_failure" || coordinator.leaseOwner + || coordinator.failureDetail?.successorRunId) throw fail(); + const [active] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, companyId), inArray(heartbeatRuns.status, ["running", "queued", "scheduled_retry"]), + or(eq(heartbeatRuns.nativeIssueId, issueId), eq(heartbeatRuns.issueId, issueId)), + )).limit(1); + const work = await tx.select().from(dotRunnerAssignments).where(and( + eq(dotRunnerAssignments.companyId, companyId), eq(dotRunnerAssignments.agentId, agentId), + eq(dotRunnerAssignments.runId, run.id), + )); + const leases = await tx.select().from(environmentLeases).where(and( + eq(environmentLeases.companyId, companyId), eq(environmentLeases.heartbeatRunId, run.id), + )); + if (active || !work.length || work.some(row => row.status !== "fenced") || !leases.length + || !leases.every(lease => lease.issueId === issueId && hasRemoteTerminationReceipt(lease))) throw fail(); + const actions = await tx.select().from(issueRecoveryActions).where(and( + eq(issueRecoveryActions.companyId, companyId), eq(issueRecoveryActions.sourceIssueId, issueId), + or(sql`${issueRecoveryActions.evidence}->>'runId' = ${run.id}`, + sql`${issueRecoveryActions.evidence}->>'sourceRunId' = ${run.id}`), + )).for("update"); + const inspection = { schema: "paperclip.grok-workspace-inspection.v1", runId: run.id, + inspectedBy: operatorId, inspectedAt: new Date().toISOString(), workspaceAccess: "none", + remoteTerminationConfirmed: false, runnerLeaseIds: leases.map(lease => lease.id), + previousWorkspaceRestoreFailure: run.resultJson?.workspaceRestoreFailure, + previousWorkspaceRestoreRecovery: run.resultJson?.workspaceRestoreRecovery }; + const retireFileObligation = (value: Record) => { + const { workspaceRestoreFailure: _failure, workspaceRestoreRecovery: _recovery, ...rest } = value; + return { ...rest, externalWorkspaceInspection: inspection }; + }; + await tx.update(heartbeatRuns).set({ resultJson: retireFileObligation(run.resultJson!) }).where(eq(heartbeatRuns.id, run.id)); + for (const action of actions) { + await tx.update(issueRecoveryActions).set({ evidence: retireFileObligation(action.evidence), updatedAt: new Date(), + nextAction: "Inspect Grok and verify its connection before sending a new task message. The old work will not be replayed.", + }).where(eq(issueRecoveryActions.id, action.id)); + } + await logActivity(database, { companyId, actorType: "user", actorId: operatorId, + action: "grok.workspace_obligation_inspected", entityType: "issue", entityId: issueId, + agentId, runId: run.id, details: inspection }); + }); + } +} diff --git a/server/src/services/heartbeat/runtime-execution.test.ts b/server/src/services/heartbeat/runtime-execution.test.ts index 20f2ff1ec4d..551c20b12c7 100644 --- a/server/src/services/heartbeat/runtime-execution.test.ts +++ b/server/src/services/heartbeat/runtime-execution.test.ts @@ -56,9 +56,9 @@ describe.skipIf(!support.supported)("heartbeat runtime execution boundary", () = vi.mocked(executePaperclipNativeSession).mockResolvedValue({ exitCode: 0, signal: null, timedOut: false }); }); - async function fixture(native = false) { + async function fixture(native = false, provider: "codex" | "grok_bot" | "openai_dot" = "codex") { const [company] = await db.insert(companies).values({ name: "Runtime execution", issuePrefix: "EXE" }).returning(); - const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Executing agent", status: "running", adapterType: native ? "paperclip_runner" : "process", adapterConfig: { provider: "codex" } }).returning(); + const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Executing agent", status: "running", adapterType: native ? "paperclip_runner" : "process", adapterConfig: provider === "codex" ? { provider } : { provider, lifecycleMode: "per_turn", allowUnmeteredProvider: true, [provider === "grok_bot" ? "externalBindingId" : "dotBindingId"]: randomUUID() } }).returning(); const [issue] = await db.insert(issues).values({ companyId: company.id, title: "Complete this task", assigneeAgentId: agent.id }).returning(); const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, issueId: issue.id, scopeKind: "issue", status: "running", runtimeMode: native ? "native" : "legacy" }).returning(); const state = createHeartbeatRunState(db); @@ -68,14 +68,15 @@ describe.skipIf(!support.supported)("heartbeat runtime execution boundary", () = const nativeExecution = native ? buildNativeExecutionInput({ companyId: company.id, agentId: agent.id, runId: run.id, issue, taskPrompt: issue.title, workspace: { id: run.id, cwd: home, repoUrl: null, repoRef: null, branchName: null }, - normalizedSessionId: null, provider: "codex", runtimeContext: nativeRuntimeContextFixture(), + normalizedSessionId: null, provider, runtimeContext: nativeRuntimeContextFixture(), + dotBinding: { bindingId: randomUUID(), bindingGeneration: 1, companyId: company.id, agentId: agent.id, acceptByUnixMs: Date.now() + 60_000, expiresAtUnixMs: Date.now() + 120_000 }, completionContract: { id: randomUUID(), sha256: `sha256:${"a".repeat(64)}`, schemaVersion: "paperclip.run-result.v1", contract: { revision: "1", objective: issue.title, criteria: [{ id: "output", requirement: issue.title }] } }, }) : null; const input: HeartbeatRuntimeExecutionInput = { run, agent, options: {}, task: { issueRef: issue, issueContext: null, context: {}, issueId: issue.id, taskKey: issue.id, executionContinuation: null }, runtime: { nativeExecution, nativeRunnerInstanceId: native ? randomUUID() : null, getNativeFreshSessionHandoff: undefined, - nativeRuntimeResolution: resolveHeartbeatNativeRuntimeMode({ persisted: run, enabled: native, runtimeConfig: {}, adapterConfig: agent.adapterConfig, agent, issue, target: null, workspaceId: null }), + nativeRuntimeResolution: resolveHeartbeatNativeRuntimeMode({ persisted: run, enabled: native, grokEnabled: true, dotEnabled: true, runtimeConfig: {}, adapterConfig: agent.adapterConfig, agent, issue, target: null, workspaceId: null }), adapter, runtimeForAdapter: { taskKey: issue.id, sessionId: null, sessionParams: null, sessionDisplayId: null }, getFreshSessionHandoff: undefined, runOptions: {}, }, workspace: { persistedExecutionWorkspace: null, executionWorkspace: { baseCwd: home, cwd: home, source: "agent_home", projectId: null, workspaceId: null, repoUrl: null, repoRef: null, strategy: "project_primary", branchName: null, worktreePath: null, warnings: [], created: false, branchCreatedByRuntime: false }, @@ -234,8 +235,8 @@ describe.skipIf(!support.supported)("heartbeat runtime execution boundary", () = }]); }); - it("retains an unexported native source before provider-error teardown", async () => { - const { input, run } = await fixture(true); + it.each(["codex", "grok_bot", "openai_dot"] as const)("retains user files only when %s can access the workspace after a dispatch failure", async provider => { + const { input, run } = await fixture(true, provider); input.workspace.nativeWorkspaceSync = { mode: "host_current", reference: { schema: "paperclip.native-workspace-sync/v3", state: "prepared", @@ -246,7 +247,8 @@ describe.skipIf(!support.supported)("heartbeat runtime execution boundary", () = const error = new Error("runner_remote_process_identity_unavailable"); vi.mocked(executePaperclipNativeSession).mockRejectedValueOnce(error); input.instructions.releaseInstructionCopy = vi.fn(async () => { - expect(input.effects.onProviderResourceDisposition).toHaveBeenCalledWith("stop_and_retain"); + if (provider === "codex") expect(input.effects.onProviderResourceDisposition).toHaveBeenCalledWith("stop_and_retain"); + else expect(input.effects.onProviderResourceDisposition).not.toHaveBeenCalledWith("stop_and_retain"); }); await expect(executeHeartbeatRuntime(db, input)).rejects.toBe(error); expect(input.workspace.nativeWorkspaceSync.restoreWorkspace).not.toHaveBeenCalled(); diff --git a/server/src/services/heartbeat/runtime-execution.ts b/server/src/services/heartbeat/runtime-execution.ts index fe601c505b0..20f307d510d 100644 --- a/server/src/services/heartbeat/runtime-execution.ts +++ b/server/src/services/heartbeat/runtime-execution.ts @@ -994,7 +994,8 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe input.effects.onNativeOwnershipHeld(); throw adapterErr; } - if (nativeRuntimeResolution.kind === "native" && nativeWorkspaceSync?.reference.state === "prepared") { + if (nativeRuntimeResolution.kind === "native" && nativeWorkspaceSync?.reference.state === "prepared" + && !(nativeExecution && "access" in nativeExecution.workspace && nativeExecution.workspace.access === "none")) { input.effects.onProviderResourceDisposition("stop_and_retain"); } await db diff --git a/server/src/services/native-runtime/native-failed-workspace-recovery.test.ts b/server/src/services/native-runtime/native-failed-workspace-recovery.test.ts index 9e433abb149..aaf7d5d88e5 100644 --- a/server/src/services/native-runtime/native-failed-workspace-recovery.test.ts +++ b/server/src/services/native-runtime/native-failed-workspace-recovery.test.ts @@ -109,7 +109,7 @@ const support = externalUrl ? { supported: true } : await getEmbeddedPostgresTes expect(await readLease(f.lease.id)).toMatchObject({ leasePolicy: "retain_on_failure" }); }); - it.each(["grok_bot", "openai_dot"] as const)("keeps historical %s inspection holds without offering file repair for a workspace-less run", async provider => { + it.each(["grok_bot", "openai_dot"] as const)("keeps historical %s inspection holds with an available recovery path", async provider => { const f = await seed(); // Reproduce a hold persisted by an older cancellation path, then attach its admitted contract. await preserveNativeFailedWorkspace(db, f.run); @@ -120,9 +120,11 @@ const support = externalUrl ? { supported: true } : await getEmbeddedPostgresTes await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, f.runId)); const blocker = await getExecutionBlocker(db, f.companyId, f.issueId); expect(blocker).toMatchObject({ runId: f.runId, canRetry: false, canContinue: false, - workspaceRepairRequired: false, canRecordWorkspaceRepair: false }); - expect(blocker?.nextAction).not.toContain("files"); - expect(blocker?.nextAction).toContain(provider === "grok_bot" ? "Runtime" : "Inspect"); + workspaceRepairRequired: provider !== "grok_bot", canRecordWorkspaceRepair: provider !== "grok_bot" }); + if (provider === "grok_bot") { + expect(blocker?.nextAction).not.toContain("files"); + expect(blocker?.nextAction).toContain("Runtime"); + } expect(await actions(f.issueId)).toHaveLength(1); });