diff --git a/doc/heartbeat-internals.md b/doc/heartbeat-internals.md index 174f167cd51..c848a781377 100644 --- a/doc/heartbeat-internals.md +++ b/doc/heartbeat-internals.md @@ -339,6 +339,23 @@ Forwarding callbacks preserve construction order for budget, retry, recovery, and queue services. Existing public helpers remain re-exported by `heartbeat.ts`. Keep cancellation policy changes separate from this extraction and execution. +An operator Stop during preparation can settle before any adapter starts. The +preparation owner records `executionCancellation.state = acknowledged` only after +it unwinds and the retained dispatch, process, session, event and lease evidence confirms +that provider work never started. A terminal run status alone is not this proof. +The locked run must have no `budgetDispatch.handedOffAt` marker. The dispatch gate +writes that marker before adapter entry, so a missing invocation event or an old +owner's local flag cannot prove that dispatch never occurred. +The task page shows a confirmed operator Stop as Stopped and accepts a new message +through normal admission. Stop does not automatically restart the task. Recovery +holds, approvals, budget limits and uncertain termination keep their own gates. +The receipt's `preparation_settled` proof identifies that evidence path. Both page +projections read the same bounded evidence and require that proof; an acknowledged +receipt from a different path does not clear the preparation failure notice. +Only a user's operator Stop gets this acknowledgement and clears an obsolete +automatic-recovery marker. A system cancellation keeps its existing recovery +delivery metadata, so saved comments and continuation intents still reach admission. + Scheduling is in `server/src/services/heartbeat/scheduling.ts`. It owns timer ticks, issue-monitor claims and dispatch, exhausted-monitor recovery, and session-goal recovery. `createHeartbeatScheduling` binds explicit dependencies diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index 0a662626ef2..62379bdc484 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -10536,7 +10536,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { expect(events[0]).toMatchObject({ eventType: "lifecycle", stream: "system", - level: "warn", + level: "info", message: "run interrupted by board comment", payload: expect.objectContaining({ issueId, diff --git a/server/src/__tests__/heartbeat-stop-queued-task.test.ts b/server/src/__tests__/heartbeat-stop-queued-task.test.ts index 56fcc97aa87..1de25a8cdff 100644 --- a/server/src/__tests__/heartbeat-stop-queued-task.test.ts +++ b/server/src/__tests__/heartbeat-stop-queued-task.test.ts @@ -1,4 +1,6 @@ import { randomUUID } from "node:crypto"; +import express from "express"; +import request from "supertest"; import { eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { @@ -12,6 +14,9 @@ import { createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, + heartbeatRunEvents, + environmentLeases, + issueRecoveryActions, issues, } from "@paperclipai/db"; import { @@ -23,10 +28,29 @@ import { drainHeartbeatRunsToQuiescence } from "./helpers/drain-heartbeat-runs.j import { truncateTablesWithDeadlockRetry } from "./helpers/truncate-with-deadlock-retry.js"; import { heartbeatService } from "../services/heartbeat.ts"; import { returnStoppedWaitingTaskToTodo } from "../services/heartbeat/stopped-waiting-task.ts"; +import { executionProjectionsForRuns } from "../services/execution-projection.ts"; +import { taskInterruptionPresentation } from "../services/task-interruption-presentation.ts"; +import { agentRoutes } from "../routes/agents.ts"; +import { issueRoutes } from "../routes/issues.ts"; +import { errorHandler } from "../middleware/index.ts"; +import { issueService } from "../services/issues.ts"; import type { AdapterExecutionContext, AdapterExecutionResult } from "../adapters/index.ts"; import { HttpError } from "../errors.ts"; +import { settleCancelledLegacyPreparation } from "../services/legacy-execution-recovery.ts"; +import { settleUnrecoverableExecutions } from "../services/execution-recovery-resolution.ts"; const gates = vi.hoisted(() => new Map void) | null>()); +const preparation = vi.hoisted(() => ({ + beforeSession: undefined as (() => Promise) | undefined, + onExecution: undefined as ((context: AdapterExecutionContext) => Promise) | undefined, +})); +vi.mock("../services/heartbeat/session-preparation.js", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, prepareHeartbeatSession: async (...args: Parameters) => { + await preparation.beforeSession?.(); + return actual.prepareHeartbeatSession(...args); + } }; +}); const mockAdapterExecute = vi.hoisted(() => vi.fn(async (context: AdapterExecutionContext): Promise => { const issueId = (context.context as Record | undefined)?.issueId; @@ -34,6 +58,7 @@ const mockAdapterExecute = vi.hoisted(() => if (typeof issueId === "string" && gates.get(issueId) === null) { await new Promise((resolve) => gates.set(issueId, resolve)); } + await preparation.onExecution?.(context); return { exitCode: 0, signal: null, timedOut: false, errorMessage: null, summary: "ok", provider: "test", model: "test" }; }), ); @@ -63,6 +88,8 @@ describeEmbeddedPostgres("Stop on a task whose run is still waiting in line", () }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); afterEach(async () => { + preparation.beforeSession = undefined; + preparation.onExecution = undefined; for (const release of gates.values()) release?.(); gates.clear(); await drainHeartbeatRunsToQuiescence(db, heartbeat); @@ -104,6 +131,167 @@ describeEmbeddedPostgres("Stop on a task whose run is still waiting in line", () return { issue, run: run! }; } + it("confirms Stop during preparation and continues once from a later message", async () => { + const { companyId, agentId } = await seed(); + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + req.actor = { type: "board", source: "session", userId: "responsible-user", companyIds: [companyId], + memberships: [{ companyId, membershipRole: "admin", status: "active" }], isInstanceAdmin: false }; + next(); + }); + app.use("/api", agentRoutes(db, {})); + const unusedStorage = async () => { throw new Error("Stop and comment routes must not access object storage"); }; + app.use("/api", issueRoutes(db, { provider: "local_disk", putFile: unusedStorage, + getObject: unusedStorage, headObject: unusedStorage, deleteObject: unusedStorage })); + app.use(errorHandler); + let entered = false; + let release!: () => void; + const released = new Promise(resolve => { release = resolve; }); + preparation.beforeSession = async () => { entered = true; await released; }; + try { + const { issue, run } = await assignAndWake(companyId, agentId, "Accidental submission"); + await vi.waitFor(() => expect(entered).toBe(true), { timeout: 10_000 }); + const stop = await request(app).post(`/api/heartbeat-runs/${run.id}/cancel`).send({}); + expect(stop.status).toBe(200); + expect(stop.body.status).toBe("cancelled"); + // Terminal status alone must not confirm a Stop while preparation owns it. + expect(stop.body.resultJson.executionCancellation).toMatchObject({ state: "requested" }); + release(); + await drainHeartbeatRunsToQuiescence(db, heartbeat); + const settled = await request(app).get(`/api/heartbeat-runs/${run.id}`); + expect(settled.status).toBe(200); + expect(settled.body.resultJson.executionCancellation).toMatchObject({ state: "acknowledged", proof: "preparation_settled", acknowledgedAt: expect.any(String) }); + expect(executedIssueIds()).not.toContain(issue.id); + expect((await executionProjectionsForRuns(db, companyId, [run.id])).get(run.id)).toMatchObject({ + phase: "completed", label: "Stopped", cause: null, + nextAction: "Send a new message to continue from saved work.", + }); + expect((await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issue.id))) + .filter(action => action.status === "active" || action.status === "escalated")).toHaveLength(0); + const [task] = await db.select().from(issues).where(eq(issues.id, issue.id)); + expect(await taskInterruptionPresentation(db, task)).toMatchObject({ state: "resolved", canRetry: false }); + const events = await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, run.id)); + expect(events.find(event => event.message === "run cancelled")?.level).toBe("info"); + await heartbeat.reconcileStrandedAssignedIssues(); + await drainHeartbeatRunsToQuiescence(db, heartbeat); + expect(executedIssueIds()).not.toContain(issue.id); + + preparation.beforeSession = undefined; + // Simulate the agent's final reply and task completion. A raw adapter exit + // without a task disposition correctly triggers a separate follow-up. + preparation.onExecution = async context => { + await context.onEvent?.({ eventType: "item.completed", stream: "stdout", level: "info", payload: { + prpEvent: { sourceEventId: `fixture:${context.runId}:final`, payload: { kind: "agentMessage", channel: "final", text: "ok" } }, + } }); + await issueService(db).update(issue.id, { status: "done" }); + }; + const message = await request(app).post(`/api/issues/${issue.id}/comments`).send({ body: "Continue with the corrected request." }); + expect(message.status).toBe(201); + // Comment dispatch is asynchronous; wait for its wake to register before + // draining the process-wide executor set or tearing down its task rows. + await vi.waitFor(() => expect(executedIssueIds()).toContain(issue.id), { timeout: 10_000 }); + await drainHeartbeatRunsToQuiescence(db, heartbeat); + expect(executedIssueIds().filter(id => id === issue.id)).toHaveLength(1); + expect((await heartbeat.getRun(run.id))?.status).toBe("cancelled"); + } finally { release(); } + }, 60_000); + + it.each([false, true])("confirms Stop only after lease release, with prior automatic disposition: %s", async (disposeFirst) => { + const { companyId, agentId } = await seed(); + const [issue] = await db.insert(issues).values({ companyId, assigneeAgentId: agentId, title: "Retained preparation", status: "in_progress" }).returning(); + const requestedAt = new Date().toISOString(); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, issueId: issue.id, + status: "cancelled", runtimeMode: "legacy", startedAt: new Date(), finishedAt: new Date(), errorCode: "cancelled", + contextSnapshot: { issueId: issue.id }, resultJson: { + cancellation: { source: "operator", expected: true, initiator: { type: "user", id: "responsible-user" }, reason: "Stop", recordedAt: requestedAt }, + executionCancellation: { state: "requested", requestedAt }, + } }).returning(); + const [lease] = await db.insert(environmentLeases).values({ companyId, issueId: issue.id, heartbeatRunId: run.id, + provider: "local", status: "active" }).returning(); + expect(await settleCancelledLegacyPreparation(db, { run, providerDispatchStarted: false })).toBeNull(); + expect((await heartbeat.getRun(run.id))?.resultJson?.executionCancellation).toEqual({ state: "requested", requestedAt }); + expect((await executionProjectionsForRuns(db, companyId, [run.id])).get(run.id)?.phase).toBe("failed"); + expect(await taskInterruptionPresentation(db, issue)).toMatchObject({ state: "failed", canRetry: true }); + + if (disposeFirst) { + await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: issue.id, + kind: "active_run_watchdog", status: "active", ownerType: "system", returnOwnerAgentId: agentId, + cause: "legacy_execution_requires_reconciliation", fingerprint: `legacy-execution:${run.id}`, + nextAction: "Preserve this stopped execution until its outcome is known.", evidence: { runId: run.id } }); + await settleUnrecoverableExecutions(db); + const [hold] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issue.id)); + expect(hold).toMatchObject({ status: "resolved", outcome: "blocked", + evidence: { automaticRecovery: { replay: "blocked" }, legacyCancellationBlock: { runId: run.id } } }); + expect((await executionProjectionsForRuns(db, companyId, [run.id])).get(run.id)?.phase).toBe("recovery_needed"); + } + + await db.update(environmentLeases).set({ status: "released", releasedAt: new Date(), cleanupStatus: "success" }).where(eq(environmentLeases.id, lease.id)); + const settled = await settleCancelledLegacyPreparation(db, { run, providerDispatchStarted: false }); + expect(settled?.resultJson?.executionCancellation).toMatchObject({ state: "acknowledged", proof: "preparation_settled", requestedAt }); + expect((await executionProjectionsForRuns(db, companyId, [run.id])).get(run.id)).toMatchObject({ phase: "completed", label: "Stopped" }); + const [settledIssue] = await db.select().from(issues).where(eq(issues.id, issue.id)); + expect(settledIssue.status).toBe("in_progress"); + expect(await taskInterruptionPresentation(db, settledIssue)).toMatchObject({ state: "resolved", canRetry: false }); + expect(executedIssueIds()).not.toContain(issue.id); + }); + + it.each(["2026-01-01T00:00:00.000Z", null])("retains the Stop hold after dispatch handoff without an invocation event: %s", async (handedOffAt) => { + const { companyId, agentId } = await seed(); + const [issue] = await db.insert(issues).values({ companyId, assigneeAgentId: agentId, + title: "Dispatch handoff before adapter entry", status: "in_progress" }).returning(); + const requestedAt = new Date().toISOString(); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, issueId: issue.id, + status: "cancelled", runtimeMode: "legacy", startedAt: new Date(), finishedAt: new Date(), errorCode: "cancelled", + contextSnapshot: { issueId: issue.id }, runnerProfileJson: { adapterDispatch: { adapterType: "codex_local" } }, + resultJson: { + cancellation: { source: "operator", expected: true, initiator: { type: "user", id: "responsible-user" }, + reason: "Stop", recordedAt: requestedAt }, + executionCancellation: { state: "requested", requestedAt }, + } }).returning(); + const [hold] = await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: issue.id, + kind: "active_run_watchdog", status: "active", ownerType: "system", returnOwnerAgentId: agentId, + cause: "legacy_execution_requires_reconciliation", fingerprint: `legacy-execution:${run.id}`, + nextAction: "Preserve this stopped execution until its outcome is known.", evidence: { runId: run.id } }).returning(); + // The old owner has a stale snapshot. The replacement has passed the gate, + // but has not recorded adapter.invoke or a process identity yet. + await db.update(heartbeatRuns).set({ usageJson: { budgetDispatch: { handedOffAt } }, executionStage: "dispatching" }) + .where(eq(heartbeatRuns.id, run.id)); + expect(await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, run.id))).toEqual([]); + expect(await settleCancelledLegacyPreparation(db, { run, providerDispatchStarted: false })).toBeNull(); + const stopped = await heartbeat.getRun(run.id); + expect(stopped?.resultJson).toEqual(run.resultJson); + const [retained] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, hold.id)); + expect(retained).toMatchObject({ status: "active", outcome: null, evidence: hold.evidence }); + expect((await executionProjectionsForRuns(db, companyId, [run.id])).get(run.id)?.phase).not.toBe("completed"); + expect(await taskInterruptionPresentation(db, issue)).toMatchObject({ state: "failed" }); + expect(executedIssueIds()).not.toContain(issue.id); + }); + + it("preserves a system cancellation's recovery delivery when preparation settles", async () => { + const { companyId, agentId } = await seed(); + const [issue] = await db.insert(issues).values({ companyId, assigneeAgentId: agentId, + title: "Stale continuation", status: "in_progress" }).returning(); + const requestedAt = new Date().toISOString(); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, issueId: issue.id, + status: "cancelled", runtimeMode: "legacy", startedAt: new Date(), finishedAt: new Date(), + contextSnapshot: { issueId: issue.id }, resultJson: { + cancellation: { source: "control_plane", expected: true, initiator: { type: "system" }, + reason: "continuation_task_ownership_changed", recordedAt: requestedAt }, + executionCancellation: { state: "requested", requestedAt }, + } }).returning(); + const [hold] = await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: issue.id, + kind: "active_run_watchdog", status: "resolved", outcome: "blocked", ownerType: "system", returnOwnerAgentId: agentId, + cause: "legacy_execution_requires_reconciliation", fingerprint: `legacy-execution:${run.id}`, + nextAction: "Continue saved input through recovery.", evidence: { runId: run.id, + automaticRecovery: { policy: "preserve_without_replay_v1", replay: "blocked", continuationDelivery: "pending" } } }).returning(); + const result = await settleCancelledLegacyPreparation(db, { run, providerDispatchStarted: false }); + expect(result?.resultJson?.executionCancellation).toEqual({ state: "requested", requestedAt }); + const [settled] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, hold.id)); + expect(settled.evidence.automaticRecovery).toEqual({ policy: "preserve_without_replay_v1", + replay: "allowed", continuationDelivery: "pending" }); + }); + it.each(["todo", "in_progress"] as const)("takes a %s task out of line in one Stop and still promotes the agent's next task", async (waitingStatus) => { const { companyId, agentId } = await seed(); const busy = randomUUID(); diff --git a/server/src/__tests__/task-interruption-query.test.ts b/server/src/__tests__/task-interruption-query.test.ts index cf28f663f43..843b78e92a1 100644 --- a/server/src/__tests__/task-interruption-query.test.ts +++ b/server/src/__tests__/task-interruption-query.test.ts @@ -375,12 +375,12 @@ describe.skipIf(!support.supported)("task interruption latest run query", () => ? { state: "resolved", canRetry: false } : { state: "failed", canRetry: true }); const query = await captureInterruptionQuery(db, taskInterruptionPresentations, companyId, [issue]); const [projected] = await db.$client.unsafe(query.sql, query.params as never[]); - expect(projected.jsonb_build_object.nativeCancellation).toEqual({ ...nativeCancellation, + expect(projected.result_json.nativeCancellation).toEqual({ ...nativeCancellation, intentAuditId: variant === "numeric_audit" ? null : nativeCancellation.intentAuditId }); - expect(projected.jsonb_build_object.cancelledByUserId).toBe(variant === "numeric_user" ? null : "board"); + expect(projected.result_json.cancelledByUserId).toBe(variant === "numeric_user" ? null : "board"); expect(projected).toMatchObject({ company_id: companyId, native_issue_id: issue.id, runtime_mode: "native" }); - expect(projected.jsonb_build_object).not.toHaveProperty("unrelatedPayload"); - expect(JSON.stringify(projected.jsonb_build_object).length).toBeLessThan(4000); + expect(projected.result_json).not.toHaveProperty("unrelatedPayload"); + expect(JSON.stringify(projected.result_json).length).toBeLessThan(4000); }); it("redacts registered run secrets from both task detail and list provider explanations", async () => { diff --git a/server/src/services/execution-projection.test.ts b/server/src/services/execution-projection.test.ts index 5b144c51fcf..32cba2bc59b 100644 --- a/server/src/services/execution-projection.test.ts +++ b/server/src/services/execution-projection.test.ts @@ -39,6 +39,27 @@ const project = ( ) => projectExecution(r, c, pending, undefined, now); describe("execution truth projection", () => { + it.each(["requested", "acknowledged"])("shows a %s operator Stop without calling it a failed attempt", state => { + const stopped = run({ status: "cancelled", runtimeMode: "legacy", finishedAt: now, errorCode: "cancelled", + resultJson: { cancellation: { source: "operator", expected: true, initiator: { type: "user", id: "board" }, + reason: "Cancelled by a board operator", recordedAt: now.toISOString() }, executionCancellation: { state, proof: "preparation_settled" } } }); + if (state === "acknowledged") { + expect(project(stopped, undefined)).toMatchObject({ phase: "completed", label: "Stopped", cause: null, + nextAction: "Send a new message to continue from saved work." }); + } else { + expect(project(stopped, undefined).phase).not.toBe("completed"); + } + // Unrelated provider failures and recovery holds remain visible. + expect(project(run({ ...stopped, resultJson: {} }), undefined).phase).toBe("failed"); + expect(projectExecution(stopped, undefined, [], { cause: "workspace_restore_failed", nextAction: "Save files" }, now).phase) + .toBe("recovery_needed"); + }); + it.each([undefined, "unknown", "provider_api_stop_receipt"])("retains failure presentation for an unqualified legacy Stop proof: %s", proof => { + const stopped = run({ status: "cancelled", runtimeMode: "legacy", finishedAt: now, + resultJson: { cancellation: { source: "operator", expected: true, initiator: { type: "user" }, + reason: "Stop", recordedAt: now.toISOString() }, executionCancellation: { state: "acknowledged", proof } } }); + expect(project(stopped, undefined).phase).toBe("failed"); + }); it.each(["contained", "ordinary", "wrong-source", "continued"] as const)("projects controller containment without inventing queued work: %s", state => { const stopped = run({ id: "old", status: "failed" }); const action = { status: "resolved" as const, cause: "native_restart_recovery_blocked", nextAction: "Inspect the original owner", diff --git a/server/src/services/execution-projection.ts b/server/src/services/execution-projection.ts index 0954d4dcf44..3b990bbf3c4 100644 --- a/server/src/services/execution-projection.ts +++ b/server/src/services/execution-projection.ts @@ -23,6 +23,7 @@ import { hasPendingRunCleanup } from "./execution-blocker.js"; import { isAgentAwaitingSetup } from "../modules/agent-lifecycle/index.js"; import { NATIVE_RESPONSE_WAKE_WAITING_REASON } from "./native-runtime/status-arbiter.js"; import { agentQueueLinesForRuns } from "./heartbeat/queued-run-line.js"; +import { isAcknowledgedLegacyUserStop, runStopPresentationJson } from "./run-cancellation.js"; const MAX_PREDECESSOR_CLEANUP_CHECKS = 16; const ANOTHER_ROUTINE_TASK = "another task in this routine"; const text = (v: unknown) => (typeof v === "string" ? v : null); @@ -51,7 +52,7 @@ const executionRunColumns = { 'failureRetriesBeforeAiConnectionWait', ${heartbeatRuns.contextSnapshot}->'failureRetriesBeforeAiConnectionWait', 'failureRetriesBeforeWorkspaceWait', ${heartbeatRuns.contextSnapshot}->'failureRetriesBeforeWorkspaceWait', 'executionRetryAccounting', ${heartbeatRuns.contextSnapshot}->'executionRetryAccounting')`, - resultJson: sql>`jsonb_build_object( + resultJson: sql>`${runStopPresentationJson(heartbeatRuns.resultJson)} || jsonb_build_object( 'finalizationPhase', ${heartbeatRuns.resultJson}->'finalizationPhase', 'finalizationReasonCode', ${heartbeatRuns.resultJson}->'finalizationReasonCode')`, }; @@ -538,6 +539,11 @@ export function projectExecution( } return set("completed", "Completed"); } + if (isAcknowledgedLegacyUserStop(run)) { + projection.cause = null; + projection.nextAction = "Send a new message to continue from saved work."; + return set("completed", "Stopped"); + } if (["failed", "cancelled", "timed_out", "interrupted"].includes(run.status)) return set("failed", run.status === "cancelled" ? "Cancelled" : "Failed"); if (run.status === "running") { diff --git a/server/src/services/fresh-continuation-fallback.test.ts b/server/src/services/fresh-continuation-fallback.test.ts index 33a1dab5320..6deeed02562 100644 --- a/server/src/services/fresh-continuation-fallback.test.ts +++ b/server/src/services/fresh-continuation-fallback.test.ts @@ -3,7 +3,7 @@ import { and, eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { activityLog, agentWakeupRequests, agents, approvals, authUsers, companies, createDb, environmentLeases, environments, executionWorkspaces, heartbeatRuns, issueApprovals, issueRecoveryActions, issueThreadInteractions, - heartbeatRunEvents, issueComments, issues, nativeRunFinalizations, toolInvocations, + heartbeatRunEvents, issueComments, issues, nativeRunFinalizations, toolInvocations, companyMemberships, } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; import { deliverReconciledExecutions, settleUnrecoverableExecutions } from "./execution-recovery-resolution.js"; @@ -65,6 +65,8 @@ suite("fresh-continuation fallback", () => { afterEach(async () => { resetBacklogContinuationPacingForTests(db); await heartbeatService(db).drainActiveRunExecutions(); + await db.delete(companyMemberships).where(and(eq(companyMemberships.companyId, companyId), + eq(companyMemberships.principalId, userId), eq(companyMemberships.principalType, "user"))); // The drain covers every background run, wake and resume dispatch, so // nothing still holds these tables. A deadlock here is a leak to fix. await db.execute(sql`truncate table issue_recovery_actions, native_run_finalizations, tool_invocations, @@ -388,6 +390,8 @@ suite("fresh-continuation fallback", () => { }, 60_000); it("never folds a Retry for the admitted continuation's own failure into that ended continuation", async () => { + await db.insert(companyMemberships).values({ companyId, principalId: userId, principalType: "user", + status: "active", membershipRole: "member" }); const f = await unfinishedRun({ runtime: "legacy", errorCode: "controller_lease_lost", error: "Legacy controller lease lost" }); await db.update(heartbeatRuns).set({ processPid: 4242, processHostHash: "replaced-server", processStartedAt: new Date(Date.now() - 600_000) }) .where(eq(heartbeatRuns.id, f.runId)); diff --git a/server/src/services/heartbeat/run-control.ts b/server/src/services/heartbeat/run-control.ts index 3b26f4f01cc..3584d55ff43 100644 --- a/server/src/services/heartbeat/run-control.ts +++ b/server/src/services/heartbeat/run-control.ts @@ -1780,7 +1780,7 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont await appendRunEvent(cancelled, { eventType: "lifecycle", stream: "system", - level: "warn", + level: readRunCancellation(cancelled.resultJson)?.expected ? "info" : "warn", message: options.eventMessage ?? "run cancelled", payload: { ...options.eventPayload, cancellation: readRunCancellation(cancelled.resultJson) }, }).catch((eventErr) => { diff --git a/server/src/services/legacy-execution-recovery.ts b/server/src/services/legacy-execution-recovery.ts index 8026aed31fb..fb96f70a23b 100644 --- a/server/src/services/legacy-execution-recovery.ts +++ b/server/src/services/legacy-execution-recovery.ts @@ -481,8 +481,11 @@ export async function settleCancelledLegacyPreparation( return db.transaction(async tx => { const [task] = await tx.select().from(issues) .where(and(eq(issues.id, issueId), eq(issues.companyId, input.run.companyId))).for("update"); + // The dispatch gate writes this marker under the same run lock before + // calling the adapter. A prior owner's local flag cannot disprove handoff. const [run] = await tx.select().from(heartbeatRuns) - .where(and(eq(heartbeatRuns.id, input.run.id), eq(heartbeatRuns.companyId, input.run.companyId))).for("update"); + .where(and(eq(heartbeatRuns.id, input.run.id), eq(heartbeatRuns.companyId, input.run.companyId), + sql`${heartbeatRuns.usageJson}#>'{budgetDispatch,handedOffAt}' is null`)).for("update"); const recovery = run?.resultJson?.executionRecovery as Record | undefined; // Only a requested Stop qualifies, and only while it is still classified as // an unknown outcome: existing wait or bootstrap receipts already decide it. @@ -504,8 +507,16 @@ export async function settleCancelledLegacyPreparation( ? !lease.releasedAt || lease.status === "pending_cleanup" || lease.cleanupStatus === "failed" : !hasRemoteTerminationReceipt(lease))) return null; const now = new Date(); + const cancellation = readRunCancellation(run.resultJson); + const operatorStop = cancellation?.source === "operator" && cancellation.initiator.type === "user"; const [settled] = await tx.update(heartbeatRuns).set({ resultJson: { ...run.resultJson, executionRecovery: { kind: "bootstrap", providerWorkStarted: false, preparationSettledAt: now.toISOString() }, + // The preparation owner has unwound and every execution reference agrees. + // Confirm the user's Stop; system cancellations keep their delivery path. + ...(operatorStop ? { executionCancellation: { + ...(run.resultJson?.executionCancellation as Record | undefined), + state: "acknowledged", proof: "preparation_settled", acknowledgedAt: now.toISOString(), + } } : {}), }, updatedAt: now }).where(eq(heartbeatRuns.id, run.id)).returning(); // The automatic disposition may already have closed the incident with // replay blocked, and moved the task to blocked. Correct both verdicts. @@ -531,9 +542,12 @@ export async function settleCancelledLegacyPreparation( const resolved = holds.length ? await tx.update(issueRecoveryActions).set({ status: "resolved", outcome: "false_positive", resolutionNote: "The run was stopped before its adapter started; there are no provider actions to reconcile.", resolvedAt: now, updatedAt: now, - evidence: sql`case when ${issueRecoveryActions.evidence} ? 'automaticRecovery' - then jsonb_set(${issueRecoveryActions.evidence}, '{automaticRecovery,replay}', '"allowed"'::jsonb, false) - else ${issueRecoveryActions.evidence} end`, + // A verified operator Stop no longer needs a recovery notice. System + // cancellations still use this metadata to deliver pending saved input. + evidence: operatorStop ? sql`${issueRecoveryActions.evidence} - 'automaticRecovery'` + : sql`case when ${issueRecoveryActions.evidence} ? 'automaticRecovery' + then jsonb_set(${issueRecoveryActions.evidence}, '{automaticRecovery,replay}', '"allowed"'::jsonb, false) + else ${issueRecoveryActions.evidence} end`, }).where(inArray(issueRecoveryActions.id, holds.map(hold => hold.id))).returning({ id: issueRecoveryActions.id }) : []; if (resolved.length) { await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: run.companyId, issueId: task.id }); diff --git a/server/src/services/run-cancellation.ts b/server/src/services/run-cancellation.ts index 9ebe8e4032b..00ae73b242b 100644 --- a/server/src/services/run-cancellation.ts +++ b/server/src/services/run-cancellation.ts @@ -1,4 +1,5 @@ import type { heartbeatRuns } from "@paperclipai/db"; +import { sql, type SQLWrapper } from "drizzle-orm"; type Run = Pick; const SOURCES = ["operator", "queued_message", "shutdown", "provider", "transport", "control_plane", "controller_lease", "unknown"] as const; @@ -145,6 +146,29 @@ export function recordedUserStop(run: Pick) { ["operator", "queued_message"].includes(cancellation.source) ? cancellation : null; } +/** Presentation of a settled operator Stop; this does not grant replay authority. */ +export function isAcknowledgedLegacyUserStop(run: Pick & Partial>): boolean { + const cancellation = recordedUserStop(run); + const receipt = run.resultJson?.executionCancellation as Record | undefined; + return run.status === "cancelled" && run.runtimeMode === "legacy" && Boolean(run.finishedAt) + && cancellation?.source === "operator" && cancellation.expected + && receipt?.state === "acknowledged" && receipt.proof === "preparation_settled"; +} + +/** The same bounded Stop evidence for both task-page projections. */ +export function runStopPresentationJson(result: SQLWrapper) { + return sql>`jsonb_build_object( + 'cancellation', jsonb_build_object( + 'source', left(${result}->'cancellation'->>'source', 32), + 'expected', ${result}->'cancellation'->'expected', + 'initiator', jsonb_build_object('type', left(${result}->'cancellation'->'initiator'->>'type', 32)), + 'reason', left(${result}->'cancellation'->>'reason', 512), + 'recordedAt', left(${result}->'cancellation'->>'recordedAt', 64)), + 'executionCancellation', jsonb_build_object( + 'state', left(${result}->'executionCancellation'->>'state', 32), + 'proof', left(${result}->'executionCancellation'->>'proof', 64)))`; +} + /** When a user's Stop or interrupt took effect for message ordering. A native * run finishes only after its provider stop and workspace save settle, which * can take many seconds. A message written after the Stop request is a new diff --git a/server/src/services/task-interruption-presentation.test.ts b/server/src/services/task-interruption-presentation.test.ts index 4a8bd737803..579cc7c9ae4 100644 --- a/server/src/services/task-interruption-presentation.test.ts +++ b/server/src/services/task-interruption-presentation.test.ts @@ -308,6 +308,21 @@ describe("intentional native Stop presentation", () => { }); }); +describe("intentional legacy Stop presentation", () => { + it("clears the failed-attempt notice only after the requested Stop is acknowledged", () => { + const stopped = { ...run, runtimeMode: "legacy" as const, status: "cancelled", finishedAt: new Date(), resultJson: { + cancellation: { source: "operator", expected: true, initiator: { type: "user" }, reason: "Stop", recordedAt: new Date().toISOString() }, + executionCancellation: { state: "acknowledged", proof: "preparation_settled" }, + } }; + const input = { run: stopped, agent, paused: false, approval: false }; + expect(projectTaskInterruption(issue, input)).toMatchObject({ state: "resolved", canRetry: false }); + expect(projectTaskInterruption(issue, { ...input, run: { ...stopped, resultJson: { ...stopped.resultJson, + executionCancellation: { state: "requested" } } } })).toMatchObject({ state: "failed", canRetry: true }); + expect(projectTaskInterruption(issue, { ...input, paused: true })).toMatchObject({ userAction: "resume" }); + expect(projectTaskInterruption(issue, { ...input, approval: true })).toMatchObject({ userAction: "approval" }); + }); +}); + it("preserves a provider explanation on a failed task without changing retry authority", () => { const providerFailure = { schema: "paperclip.provider-failure.v1", provider: "anthropic", phase: "startup", category: "rate_limit", message: "API error: 429 This request would exceed your account's rate limit.", diff --git a/server/src/services/task-interruption-presentation.ts b/server/src/services/task-interruption-presentation.ts index 87b18917f83..052174fadae 100644 --- a/server/src/services/task-interruption-presentation.ts +++ b/server/src/services/task-interruption-presentation.ts @@ -22,7 +22,7 @@ import { logger } from "../middleware/logger.js"; import { canonicalExecutionUuid } from "./execution-identity.js"; import { chatActionDeadline } from "./chat-action-work.js"; import { NATIVE_CLEANUP_SETTLING_MESSAGE, NATIVE_SESSION_CLEANUP_QUARANTINED } from "./native-runtime/completed-turn-cleanup.js"; -import { isQueuedDependencyWait } from "./run-cancellation.js"; +import { isAcknowledgedLegacyUserStop, isQueuedDependencyWait, runStopPresentationJson } from "./run-cancellation.js"; import { heldRetryOffersRetry } from "./task-retry-eligibility.js"; import { exhaustedNativeFinalizationCondition, savedResultRetryPendingCondition } from "./native-runtime/exhausted-finalization-retry.js"; import { AI_CONNECTION_BUSY_RETRY_REASON, AI_CONNECTION_POOL_WAIT_RETRY_REASON, RUNNER_REVISION_WAIT_RETRY_REASON, @@ -278,6 +278,7 @@ function projectTaskInterruptionState(issue: ProjectionIssue, input: ProjectionS // The task's existing dependency UI owns this wait. Keep actual recovery and // Retry preparation failures authoritative; this does not change admission. if (!action && !retryWait?.preparationStatus && isQueuedDependencyWait(run)) return base; + if (!wait && !action && isAcknowledgedLegacyUserStop(run)) return base; // Intentional Stop is not an unsuccessful attempt. This is presentation only; // physical containment and pending cleanup keep their execution-side gates. if (!wait && run.runtimeMode === "native" && run.companyId === issue.companyId && run.nativeIssueId === issue.id @@ -510,7 +511,7 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i setupFailureRunId: sql`case when ${heartbeatRuns.resultJson}->'setupCommandFailure' is null and ${heartbeatRuns.status} = 'scheduled_retry' and ${heartbeatRuns.retryOfRunId} is not null then ${heartbeatRuns.retryOfRunId} else ${heartbeatRuns.id} end`, - resultJson: sql>`jsonb_build_object('configurationIncomplete', ${heartbeatRuns.resultJson}->'configurationIncomplete', + resultJson: sql>`${runStopPresentationJson(heartbeatRuns.resultJson)} || jsonb_build_object('configurationIncomplete', ${heartbeatRuns.resultJson}->'configurationIncomplete', 'stopReason', ${boundedJsonString(sql`${heartbeatRuns.resultJson}->'stopReason'`, 80)}, 'timeoutSource', ${boundedJsonString(sql`${heartbeatRuns.resultJson}->'timeoutSource'`, 80)}, 'timeoutFired', coalesce(${heartbeatRuns.resultJson}->'timeoutFired' = 'true'::jsonb, false), @@ -535,7 +536,7 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i 'setupCommandFailure', coalesce(${heartbeatRuns.resultJson}->'setupCommandFailure', case when ${heartbeatRuns.status} = 'scheduled_retry' then (select predecessor.result_json->'setupCommandFailure' from ${heartbeatRuns} predecessor where predecessor.company_id = ${heartbeatRuns.companyId} and predecessor.id = ${heartbeatRuns.retryOfRunId} - and predecessor.error_code = 'setup_failed') end))`, + and predecessor.error_code = 'setup_failed') end))`.as("result_json"), }).from(requested).innerJoinLateral(latestRuns, sql`true`).innerJoin(heartbeatRuns, and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, latestRuns.id))) .leftJoin(agents, and(eq(agents.companyId, heartbeatRuns.companyId), eq(agents.id, heartbeatRuns.agentId))) .orderBy(requestedIssueId),