From 098f7d370bf05dffb2327b6250dbf301d5146fc6 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sun, 8 Nov 2026 22:37:52 -0600 Subject: [PATCH 1/9] fix(workspaces): release an unchanged remote writer without a snapshot and wake its waiters (#20171) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - A remote (sandbox) task workspace has one writer run at a time. The writer holds the workspace until its files are saved as a snapshot > - A run that a user comment interrupts right after its claim did no work. No provider turn started and no command ran in the sandbox > - Paperclip still kept its sandbox, then took a full snapshot of the unchanged workspace. On a 2 GB repository this took 3 minutes > - The next runs failed setup with `workspace_busy` ("ended, but its workspace save has not settled"). The retry then waited for the one-minute settling re-check after the save. The user's message waited more than 4 minutes > - This pull request releases such a writer at once, at the saved version it started from, when durable evidence proves that it changed nothing. It also wakes parked `workspace_busy` waiters when a hold settles > - The benefit is that an interrupted run no longer delays the next message by a full workspace save ## Linked Issues or Issue Description **What happened?** A remote-workspace run was claimed (`native.coordinator.claim`). One second later a board comment interrupted it (`operator_interrupted`). It made no provider turn and no tool calls. Cancelled-workspace preservation still retained its sandbox, and workspace recovery took a full `remote_workspace_snapshot` of the unchanged workspace (3 minutes for a 2 GB repository). Two follow-up runs were cancelled with `workspace_busy`. Their retry was due during the save, and it parked for the one-minute settling re-check after the save finished. **Expected behavior** A writer that provably changed nothing releases the workspace at once and keeps its existing `snapshotId`. The next run is admitted without `workspace_busy`. When a save does run, waiters start as soon as it commits. **Steps to reproduce** 1. Run a task on a remote sandbox workspace that already has a saved version. 2. Interrupt the run with a board comment right after the claim, before the provider turn. 3. See the full snapshot operation on the interrupted run and `workspace_busy` on the follow-up runs. **Paperclip version or commit** Staging served revision b43cccbb0. Reproduced on `master` by the new regression tests. ## What Changed - `prepareRemoteWorkspace` records `remoteWorkspaceUnchangedBaseline` (epoch, writer run, saved version) when preparation started from a clean saved version and added nothing to it: no recovered attachments, no repository outside the saved version and no workspace migration. - New `proveRemoteWorkspaceUnchanged` (`remote-workspace-unchanged-proof.ts`). It uses only durable evidence. The writer run must have ended without a result, and its owner fence must be clear. It must have no process fields, no execution, provider or tool events (the shared `executionStartedEventPredicate`), no native result, no runner checkpoint rows (the controller writes one before every runner command), and no provider turn in a checkpoint that names this run. It must have no workspace operation other than a restore or a `workspace_finalize`/`workspace_recovery` receipt, so provisioning and saves count as writes. Every task repository must be in the saved version. Any missing or ambiguous evidence refuses. - New `releaseUnchangedRemoteWorkspaceWriter`. It sets the authority to clean with no writer, keeps `snapshotId`, marks the run reference `saved`, logs `workspace.snapshot_skipped_unchanged` and sends the same notifications as a final save. It never waits on a writer run that another transaction holds (`skip locked`). - The release applies in three places: the next run's `reserveRemoteWorkspace`, cancelled/failed-workspace preservation (`preserveNativeFailedWorkspace`), and `saveRemoteWorkspace` with the final boundary. The final save returns the same version on a repeated call, so a recovery that raced the release finishes without a capture. - The final-save commit and the unchanged release call `wakeWorkspaceBusyWaitersInTx`. A parked waiter whose holder was that writer is due at once, not at the one-minute settling re-check. - `doc/execution-semantics.md` describes the release and the wake. ## Verification - `pnpm --filter @paperclipai/server exec vitest run src/services/remote-workspace-unchanged-release.test.ts` (13 tests, embedded Postgres, real archive capture): - Staging shape (claim, then interrupt with no provider work): the next reservation is admitted. No snapshot operation is recorded. The run reference is `saved` at the old `snapshotId`. Red on `master`: "The workspace still has an unfinished writer or save operation." - A later final save, behind the `workspace_finalize` receipt that production finalizers publish first, returns the same version with no capture, a repeat returns it again, and a parked waiter becomes due now. Red on `master`: a new snapshot is captured. - Negatives that still do a full save: `turn.started`, `provider.event`, `tool.execution.started`, `native.process_start_requested`, a provisioning command that wrote into the workspace, a recorded process, a live owner fence, and a writer that is still running. - Upgrade safety: no baseline (a writer prepared by the previous release), a baseline without its saved version, and a baseline with another schema each keep the full save. - A full final save wakes the parked waiter. Red on `master`. - `native-cancelled-workspace-preservation.test.ts`: a new case shows that a cancelled writer with a baseline and no work is released, with no retained lease and no recovery action. Red on `master` (the restore-recovery hold was recorded). The other 44 cases pass unchanged. - I also ran the related remote-workspace, workspace-recovery, native preservation and run-dispatch suites locally. ## Risks - A false "unchanged" verdict would drop unsaved files. The proof requires a positive baseline from preparation and refuses on any write record. Only `.paperclip-runtime` (excluded from snapshots) is written by runner staging before a process starts. A deploy that lands mid-run keeps the full save, because the proof reads only durable rows. - Previous-release state: a writer prepared without the baseline (or with an unknown or partial one) keeps today's full save. An older server ignores the new metadata key. - Waiters now start sooner after a final save. Arrival order among waiters is unchanged (the park check still admits the earliest first). - Other ways a hold can settle (for example a stop receipt that turns an unused reservation free) still rely on the one-minute re-check. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, used in Claude Code with tool use (shell, file edits and a read-only browser for staging evidence). ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- doc/execution-semantics.md | 19 +- ...e-cancelled-workspace-preservation.test.ts | 25 ++ .../native-failed-workspace-recovery.ts | 7 + .../remote-workspace-unchanged-proof.ts | 120 ++++++++++ ...remote-workspace-unchanged-release.test.ts | 223 ++++++++++++++++++ server/src/services/remote-workspaces.ts | 101 +++++++- 6 files changed, 488 insertions(+), 7 deletions(-) create mode 100644 server/src/services/remote-workspace-unchanged-proof.ts create mode 100644 server/src/services/remote-workspace-unchanged-release.test.ts diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index ea215ff7344..0f5c7d37023 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -569,7 +569,24 @@ writer, measured from the time the writer ended. After the cutoff, preparation fails with the unfinished-writer error and leaves the held files unchanged. This wait is also one parked retry, for example after a reassignment stops the writer. It is re-checked every minute while the ended writer's hold settles, is -admitted once the hold releases, and stops parking at the same cutoff. +admitted once the hold releases, and stops parking at the same cutoff. The +writer's final save commit and an unchanged release (below) make these waiters +due immediately instead of at the next re-check. + +A writer that ended (failed, cancelled, interrupted or timed out) without +changing its files releases the hold at once, with no snapshot. Its workspace +keeps the saved version it started from. Preparation records that starting +point only when it began from a clean saved version and added nothing to it: +no recovered attachments, no repository outside the saved version and no +workspace migration. The release requires durable evidence that nothing ran +after that point: no execution or provider event, process, runner checkpoint, +native result or provider turn in a checkpoint for that run; no workspace +operation other than a restore or a finalize or recovery receipt; and no task +repository outside the saved version. The writer's owner fence must also be clear. Any missing or ambiguous evidence, +including a writer prepared by an earlier release without that starting point, +keeps the full save. The next run's reservation, cancelled-workspace +preservation and a later final save each apply this release. A repeated save +returns the same version. A writer stopped mid-run, for example by reassignment, keeps its unsaved files in its stopped sandbox. When the run used a reusable (warm) sandbox, the stop diff --git a/server/src/services/native-runtime/native-cancelled-workspace-preservation.test.ts b/server/src/services/native-runtime/native-cancelled-workspace-preservation.test.ts index 23b1847db48..8b72d5e589a 100644 --- a/server/src/services/native-runtime/native-cancelled-workspace-preservation.test.ts +++ b/server/src/services/native-runtime/native-cancelled-workspace-preservation.test.ts @@ -460,6 +460,31 @@ import { readRemoteWorkspaceAuthority } from "@paperclipai/shared"; } finally { await worker.stop(); } }); + it("hands the workspace on at its saved version when the stopped writer never started provider work", async () => { + const f = await seed(); + const [workspace] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, f.workspaceId)); + await db.update(executionWorkspaces).set({ metadata: { ...workspace.metadata, remoteWorkspaceUnchangedBaseline: { + schema: "paperclip.remote-workspace-unchanged-baseline.v1", epoch: 2, runId: f.runId, snapshotId: f.reference.snapshotId, + } } }).where(eq(executionWorkspaces.id, f.workspaceId)); + const worker = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => { throw error; } }); + try { + await registerCancelledWorkspacePreservationWork(db, worker, () => true, async () => true).ready; + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + expect(run.resultJson).not.toHaveProperty("workspaceRestoreRecovery"); + expect(run.runnerProfileJson?.remoteWorkspace).toMatchObject({ state: "saved", snapshotId: f.reference.snapshotId }); + const [released] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, f.workspaceId)); + const authority = readRemoteWorkspaceAuthority(released.metadata)!; + expect(authority).toMatchObject({ epoch: 2, snapshotId: f.reference.snapshotId, writerRunId: null, dirty: false }); + expect(await readRemoteWorkspaceWriterHold(db, { companyId: f.companyId, authority })).toEqual({ kind: "free" }); + expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.leaseId)))[0]) + .toMatchObject({ status: "active", leasePolicy: "ephemeral" }); + expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.companyId, f.companyId))).toHaveLength(0); + // A later pass finds nothing left to keep. + await preserveCancelledNativeWorkspaces(db); + expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.companyId, f.companyId))).toHaveLength(0); + } finally { await worker.stop(); } + }); + it("rolls preservation back with its outer transaction", async () => { const f = await seed(); await expect(db.transaction(async tx => { diff --git a/server/src/services/native-runtime/native-failed-workspace-recovery.ts b/server/src/services/native-runtime/native-failed-workspace-recovery.ts index 19b87205a18..7d2de469019 100644 --- a/server/src/services/native-runtime/native-failed-workspace-recovery.ts +++ b/server/src/services/native-runtime/native-failed-workspace-recovery.ts @@ -18,6 +18,7 @@ import { preserveRemoteWorkspacePreparationFailure } from "./native-workspace-fi import { pinFailedObservedWorkspace } from "./native-observed-workspace-preservation.js"; import { hasConfirmedSandboxStopAndRetain } from "../sandbox-stop-and-retain.js"; import { preparedRemoteWorkspaceSource } from "../remote-workspace-lifecycle.js"; +import { releaseUnchangedRemoteWorkspaceWriterById } from "../remote-workspaces.js"; const reusableLeasePolicies = ["reuse_by_environment", "reuse_by_execution_workspace"]; @@ -137,6 +138,12 @@ export async function preserveNativeFailedWorkspace(db: Db, selected: Pick; +type Repository = { id: string; relativePath: string; repoUrl: string; ref: string | null; pinnedCommit?: string }; + +const object = (value: unknown): Record => + value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; + +/** Missing or malformed fields (an older or partial record) never prove anything. */ +export function readUnchangedBaseline(metadata: Record | null | undefined): UnchangedBaseline | null { + const value = object(metadata?.[UNCHANGED_BASELINE_KEY]); + if (value.schema !== UNCHANGED_BASELINE_SCHEMA || !Number.isSafeInteger(value.epoch) || (value.epoch as number) < 1 + || typeof value.runId !== "string" || !value.runId || typeof value.snapshotId !== "string" || !value.snapshotId + || (value.releasedAt !== undefined && typeof value.releasedAt !== "string")) return null; + return value as UnchangedBaseline; +} + +/** Operations that cannot change the saved files: a restore writes exactly the + * saved version, and finalize and recovery rows are receipts that wrap a save + * (whose own snapshot operation is still counted). */ +const FILE_NEUTRAL_OPERATION_PHASES = ["remote_workspace_restore", "workspace_finalize", "workspace_recovery"]; +/** Terminal states of a run that ended without a result. */ +const ENDED_WITHOUT_RESULT = ["failed", "cancelled", "interrupted", "timed_out"]; + +export type UnchangedProofRefusal = + | "baseline_missing" | "authority_mismatch" | "writer_not_ended" | "writer_owner_live" | "process_recorded" + | "reference_mismatch" | "provider_session_started" | "execution_recorded" | "result_recorded" + | "runner_commanded" | "workspace_operation_recorded" | "repository_unsaved"; + +/** + * Whether the ended writer of `authority` provably left its files exactly as + * the saved version it started from. The caller holds the workspace row lock. + * Any missing, ambiguous or older-release evidence refuses; a refusal keeps + * today's full save. + */ +export async function proveRemoteWorkspaceUnchanged(tx: Pick, input: { + workspace: Workspace; authority: RemoteWorkspaceAuthority; + /** The caller owns the writer's finalization, so its own fence is expected. */ + callerOwnsWriter?: boolean; + isSavedRepository: (repository: Repository) => boolean; +}): Promise<{ proven: true; baseline: UnchangedBaseline } | { proven: false; reason: UnchangedProofRefusal }> { + const { workspace, authority } = input; + const runId = authority.writerRunId; + const baseline = readUnchangedBaseline(workspace.metadata); + if (!baseline || baseline.releasedAt || !runId || baseline.runId !== runId) return { proven: false, reason: "baseline_missing" }; + if (!authority.dirty || !authority.location || authority.epoch !== baseline.epoch || authority.snapshotId !== baseline.snapshotId) + return { proven: false, reason: "authority_mismatch" }; + + const [run] = await tx.select({ status: heartbeatRuns.status, processPid: heartbeatRuns.processPid, + processGroupId: heartbeatRuns.processGroupId, processStartedAt: heartbeatRuns.processStartedAt, + sessionIdAfter: heartbeatRuns.sessionIdAfter, runnerProfileJson: heartbeatRuns.runnerProfileJson, + ownerFenceHeld: runOwnerFenceHeldSql() }).from(heartbeatRuns) + .where(and(eq(heartbeatRuns.companyId, workspace.companyId), eq(heartbeatRuns.id, runId))); + if (!run || !ENDED_WITHOUT_RESULT.includes(run.status)) return { proven: false, reason: "writer_not_ended" }; + if (!input.callerOwnsWriter && run.ownerFenceHeld) return { proven: false, reason: "writer_owner_live" }; + if (run.processPid !== null || run.processGroupId !== null || run.processStartedAt !== null || run.sessionIdAfter !== null) + return { proven: false, reason: "process_recorded" }; + const profile = object(run.runnerProfileJson); + const reference = remoteWorkspaceRunReferenceSchema.safeParse(profile.remoteWorkspace); + if (profile.nativeWorkspaceSync != null || !reference.success || reference.data.state !== "prepared" + || reference.data.workspaceId !== workspace.id || reference.data.epoch !== authority.epoch + || reference.data.snapshotId !== authority.snapshotId || reference.data.leaseId !== authority.location.leaseId + || reference.data.remoteCwd !== authority.location.remoteCwd) return { proven: false, reason: "reference_mismatch" }; + // A rebound admission checkpoint may name this run; it proves nothing ran + // only while it holds no turn, result or terminal state of its own. + const checkpoint = object(profile.sessionCheckpoint); + if (object(checkpoint.identity).runId === runId && (checkpoint.activeTurnId != null || checkpoint.semanticResult != null + || checkpoint.terminal != null || (checkpoint.terminalTurns != null + && (!Array.isArray(checkpoint.terminalTurns) || checkpoint.terminalTurns.length > 0)))) + return { proven: false, reason: "provider_session_started" }; + + const [event] = await tx.select({ id: heartbeatRunEvents.id }).from(heartbeatRunEvents).where(and( + eq(heartbeatRunEvents.companyId, workspace.companyId), eq(heartbeatRunEvents.runId, runId), + executionStartedEventPredicate())).limit(1); + if (event) return { proven: false, reason: "execution_recorded" }; + const [result] = await tx.select({ id: nativeRunResults.id }).from(nativeRunResults).where(and( + eq(nativeRunResults.companyId, workspace.companyId), eq(nativeRunResults.runId, runId))).limit(1); + if (result) return { proven: false, reason: "result_recorded" }; + // The controller writes this checkpoint before every runner command. + const [commanded] = await tx.select({ runId: nativeRunnerCheckpoints.runId }).from(nativeRunnerCheckpoints).where(and( + eq(nativeRunnerCheckpoints.companyId, workspace.companyId), eq(nativeRunnerCheckpoints.runId, runId))).limit(1); + if (commanded) return { proven: false, reason: "runner_commanded" }; + // Provisioning and saves record their operation before they touch files. + const [operation] = await tx.select({ id: workspaceOperations.id }).from(workspaceOperations).where(and( + eq(workspaceOperations.companyId, workspace.companyId), eq(workspaceOperations.heartbeatRunId, runId), + sql`(${workspaceOperations.executionWorkspaceId} is null or ${workspaceOperations.executionWorkspaceId} = ${workspace.id})`, + sql`${workspaceOperations.phase} not in (${sql.join(FILE_NEUTRAL_OPERATION_PHASES.map(phase => sql`${phase}`), sql`, `)})`)).limit(1); + if (operation) return { proven: false, reason: "workspace_operation_recorded" }; + // A requested repository is cloned before its save is recorded. Only + // repositories already in the saved version are proven untouched. + const repositories = await tx.select().from(executionWorkspaceRepositories).where(and( + eq(executionWorkspaceRepositories.companyId, workspace.companyId), + eq(executionWorkspaceRepositories.executionWorkspaceId, workspace.id))); + if (repositories.some(row => !row.pinnedCommit || !input.isSavedRepository({ id: row.id, relativePath: row.relativePath, + repoUrl: row.repoUrl, ref: row.requestedRef === "HEAD" ? null : row.requestedRef, pinnedCommit: row.pinnedCommit }))) + return { proven: false, reason: "repository_unsaved" }; + return { proven: true, baseline }; +} diff --git a/server/src/services/remote-workspace-unchanged-release.test.ts b/server/src/services/remote-workspace-unchanged-release.test.ts new file mode 100644 index 00000000000..d571f2ad533 --- /dev/null +++ b/server/src/services/remote-workspace-unchanged-release.test.ts @@ -0,0 +1,223 @@ +import { execFile } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { cp, mkdir, mkdtemp, realpath, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import { and, eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { + agents, closeRegisteredClients, companies, createDb, environmentLeases, environments, executionWorkspaces, + heartbeatRunEvents, heartbeatRuns, issues, workspaceOperations, +} from "@paperclipai/db"; +import type { AdapterSandboxExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; +import type { EnvironmentLease } from "@paperclipai/shared"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; +import { createStorageService } from "../storage/service.js"; +import { createLocalDiskStorageProvider } from "../storage/local-disk-provider.js"; +import { + prepareRemoteWorkspace, readRemoteWorkspaceAuthority, RemoteWorkspaceWriterActiveError, reserveRemoteWorkspace, saveRemoteWorkspace, +} from "./remote-workspaces.js"; +import { UNCHANGED_BASELINE_KEY } from "./remote-workspace-unchanged-proof.js"; + +const exec = promisify(execFile); +const support = await getEmbeddedPostgresTestSupport(); + +describe.skipIf(!support.supported)("remote workspace writer that ended without changing its files", () => { + let database: Awaited>; + let db: ReturnType; + let root: string; + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("workspace-unchanged-release-"); + db = createDb(database.connectionString); + root = await realpath(await mkdtemp(path.join(os.tmpdir(), "workspace-unchanged-release-"))); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "63".repeat(32)); + vi.stubEnv("DATABASE_URL", database.connectionString); + }, 60_000); + afterAll(async () => { + if (database) await closeRegisteredClients(database.connectionString); + await database?.cleanup(); + if (root) await rm(root, { recursive: true, force: true }); + vi.unstubAllEnvs(); + }); + + /** A saved workspace, then a follow-up writer B prepared on the same sandbox (staging PAP-3367 shape). */ + async function preparedFollowUp(options: { runtimeProvisionCommand?: string } = {}) { + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Unchanged release", issuePrefix: `U${companyId.slice(0, 6)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Codie", adapterType: "paperclip_runner" }); + await db.insert(issues).values({ id: issueId, companyId, title: "Daily repair", status: "in_progress", assigneeAgentId: agentId }); + const [environment] = await db.insert(environments).values({ name: issueId, driver: "sandbox" }).returning(); + const storage = { ...createStorageService(createLocalDiskStorageProvider(path.join(root, issueId, "objects"))), provider: "s3" as const }; + const remoteCwd = path.join(root, issueId, "remote"); + await mkdir(remoteCwd, { recursive: true }); + const providerLeaseId = randomUUID(); + const transfer: NonNullable["syncIn"]> = async operations => { + for (const operation of operations) for (const file of operation.files) { + await mkdir(path.dirname(file.targetPath), { recursive: true }); + await cp(file.sourcePath, file.targetPath, { recursive: file.kind === "directory" }); + } + return { operations: operations.map(operation => ({ operationId: operation.operationId, + filesTransferred: operation.files.length, bytesTransferred: 0 })) }; + }; + let captures = 0; + const target = (leaseId: string): AdapterSandboxExecutionTarget => ({ + kind: "remote", transport: "sandbox", leaseId, remoteCwd, timeoutMs: 30_000, + runner: { syncIn: transfer, syncOut: async operations => { captures += 1; return transfer(operations); }, + execute: async input => { + const result = await exec(input.command, input.args ?? [], { cwd: input.cwd ?? remoteCwd, + env: { ...process.env, ...input.env }, timeout: input.timeoutMs, maxBuffer: 8 * 1024 * 1024 }); + return { exitCode: 0, stdout: result.stdout, stderr: result.stderr, signal: null, timedOut: false, + pid: process.pid, startedAt: new Date().toISOString() }; + } }, + }); + const writer = async (runId: string, existingWorkspaceId: string | null, runtimeProvisionCommand?: string) => { + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, issueId, runtimeMode: "legacy", + status: "running", contextSnapshot: { issueId } }); + const [lease] = await db.insert(environmentLeases).values({ companyId, issueId, heartbeatRunId: runId, + environmentId: environment.id, provider: "daytona", providerLeaseId, status: "active" }).returning(); + const reserved = await reserveRemoteWorkspace({ db, storage, companyId, issueId, runId, agentId, + environmentId: environment.id, projectId: null, projectWorkspaceId: null, existingWorkspaceId, source: { kind: "empty" } }); + const prepared = await prepareRemoteWorkspace({ db, storage, workspace: reserved, runId, + lease: lease as EnvironmentLease, target: target(lease.id), runtimeProvisionCommand }); + return { lease: prepared.lease, workspaceId: reserved.id, remoteRoot: prepared.remoteRoot }; + }; + // Run A does real work and saves it. + const runA = randomUUID(); + const a = await writer(runA, null); + await writeFile(path.join(a.remoteRoot, "report.md"), "earlier work\n"); + const [workspaceA] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, a.workspaceId)); + const savedA = await saveRemoteWorkspace({ db, storage, workspace: workspaceA, runId: runA, lease: a.lease, + target: target(a.lease.id), assertOwned: async () => undefined }); + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, runA)); + // Run B is the follow-up: claimed, then interrupted before any provider work. + const runB = randomUUID(); + const b = await writer(runB, a.workspaceId, options.runtimeProvisionCommand); + const end = async (patch: Partial = {}) => { + await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date(), errorCode: "operator_interrupted", ...patch }) + .where(eq(heartbeatRuns.id, runB)); + }; + const workspace = async () => (await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, a.workspaceId)))[0]!; + const saveB = async () => saveRemoteWorkspace({ db, storage, workspace: await workspace(), runId: runB, lease: b.lease, + target: target(b.lease.id), assertOwned: async () => undefined }); + const snapshotOperations = () => db.select().from(workspaceOperations).where(and( + eq(workspaceOperations.heartbeatRunId, runB), eq(workspaceOperations.phase, "remote_workspace_snapshot"))); + const reserveNext = async () => { + const runC = randomUUID(); + await db.insert(heartbeatRuns).values({ id: runC, companyId, agentId, issueId, runtimeMode: "legacy", + status: "running", contextSnapshot: { issueId } }); + return { runC, reserved: await reserveRemoteWorkspace({ db, storage, companyId, issueId, runId: runC, agentId, + environmentId: environment.id, projectId: null, projectWorkspaceId: null, existingWorkspaceId: a.workspaceId, source: { kind: "empty" } }) }; + }; + const parkWaiter = async () => { + const waiterId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: waiterId, companyId, agentId, issueId, runtimeMode: "legacy", + status: "scheduled_retry", scheduledRetryReason: "workspace_busy", scheduledRetryAt: new Date(Date.now() + 5 * 60_000), + contextSnapshot: { issueId, workspaceBusyWait: { key: `execution:${a.workspaceId}`, holderRunId: runB, + waitingSince: new Date().toISOString(), issueExecutionWorkspaceId: a.workspaceId, issueProjectWorkspaceId: null } } }); + return async () => (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, waiterId)))[0]!.scheduledRetryAt!; + }; + const event = (eventType: string) => db.insert(heartbeatRunEvents).values({ companyId, runId: runB, agentId, + seq: 1, stream: "system", level: "info", eventType }); + return { companyId, runA, runB, savedA, workspace, end, saveB, snapshotOperations, reserveNext, parkWaiter, event, + captures: () => captures }; + } + + async function expectFullSave(f: Awaited>) { + await expect(f.reserveNext()).rejects.toBeInstanceOf(RemoteWorkspaceWriterActiveError); + const before = f.captures(); + const saved = await f.saveB(); + expect(saved).not.toHaveProperty("unchanged"); + expect(saved.snapshotId).not.toBe(f.savedA.snapshotId); + expect(f.captures()).toBeGreaterThan(before); + expect(await f.snapshotOperations()).toHaveLength(1); + } + + it("admits the next run at once, without a save, when the interrupted writer never started provider work", async () => { + const f = await preparedFollowUp(); + await f.end(); + const { runC, reserved } = await f.reserveNext(); + const authority = readRemoteWorkspaceAuthority(reserved.metadata); + expect(authority).toMatchObject({ writerRunId: runC, snapshotId: f.savedA.snapshotId, dirty: false, epoch: 3 }); + expect(await f.snapshotOperations()).toEqual([]); + const [runB] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runB)); + expect(runB.runnerProfileJson?.remoteWorkspace).toMatchObject({ state: "saved", snapshotId: f.savedA.snapshotId, epoch: 2 }); + }); + + it("settles a later save of that writer without capturing, and wakes runs parked behind it", async () => { + const f = await preparedFollowUp(); + await f.end(); + const dueAt = await f.parkWaiter(); + // Production finalizers publish their wrapper receipt before the save. + await db.insert(workspaceOperations).values({ companyId: f.companyId, executionWorkspaceId: (await f.workspace()).id, + heartbeatRunId: f.runB, phase: "workspace_finalize", status: "running", metadata: { owningService: "native_workspace_finalizer" } }); + const before = f.captures(); + const saved = await f.saveB(); + expect(saved).toMatchObject({ snapshotId: f.savedA.snapshotId, unchanged: true }); + expect(f.captures()).toBe(before); + expect((await dueAt()).getTime()).toBeLessThanOrEqual(Date.now()); + // A repeated save (a recovery retry) returns the same version. + expect(await f.saveB()).toMatchObject({ snapshotId: f.savedA.snapshotId, unchanged: true }); + expect(await f.snapshotOperations()).toEqual([]); + const { runC } = await f.reserveNext(); + expect(readRemoteWorkspaceAuthority((await f.workspace()).metadata)?.writerRunId).toBe(runC); + }); + + it.each(["turn.started", "provider.event", "tool.execution.started", "native.process_start_requested"])( + "keeps the full save after a %s event", async eventType => { + const f = await preparedFollowUp(); + await f.event(eventType); + await f.end(); + await expectFullSave(f); + }); + + it("keeps the full save when setup wrote into the workspace", async () => { + const f = await preparedFollowUp({ runtimeProvisionCommand: "echo provisioned > provisioned.txt" }); + await f.end(); + await expectFullSave(f); + }); + + it("keeps the full save while the writer's owner may still act, and while it is still running", async () => { + const f = await preparedFollowUp(); + await f.end({ controllerLeaseExpiresAt: new Date(Date.now() + 60_000) }); + await expect(f.reserveNext()).rejects.toBeInstanceOf(RemoteWorkspaceWriterActiveError); + await db.update(heartbeatRuns).set({ status: "running", controllerLeaseExpiresAt: null, finishedAt: null }) + .where(eq(heartbeatRuns.id, f.runB)); + const saved = await f.saveB(); + expect(saved).not.toHaveProperty("unchanged"); + expect(await f.snapshotOperations()).toHaveLength(1); + }); + + it("keeps the full save when a process was recorded", async () => { + const f = await preparedFollowUp(); + await f.end({ processPid: 4242, processStartedAt: new Date() }); + await expectFullSave(f); + }); + + it("wakes parked runs when a full final save settles", async () => { + const f = await preparedFollowUp(); + await f.event("turn.started"); + await f.end(); + const dueAt = await f.parkWaiter(); + await f.saveB(); + expect((await dueAt()).getTime()).toBeLessThanOrEqual(Date.now()); + }); + + it.each([ + { label: "a writer prepared by the previous release (no baseline)", baseline: undefined }, + { label: "a baseline missing its saved version", baseline: "drop_snapshot" }, + { label: "a baseline from another schema", baseline: "other_schema" }, + ] as const)("keeps the full save for $label", async ({ baseline }) => { + const f = await preparedFollowUp(); + const current = await f.workspace(); + const recorded = current.metadata?.[UNCHANGED_BASELINE_KEY] as Record | undefined; + expect(recorded).toMatchObject({ runId: f.runB, epoch: 2, snapshotId: f.savedA.snapshotId }); + const { [UNCHANGED_BASELINE_KEY]: _dropped, ...previousRelease } = current.metadata ?? {}; + const next = baseline === undefined ? previousRelease + : baseline === "drop_snapshot" ? { ...previousRelease, [UNCHANGED_BASELINE_KEY]: { ...recorded, snapshotId: undefined } } + : { ...previousRelease, [UNCHANGED_BASELINE_KEY]: { ...recorded, schema: "paperclip.remote-workspace-unchanged-baseline.v0" } }; + await db.update(executionWorkspaces).set({ metadata: next }).where(eq(executionWorkspaces.id, current.id)); + await f.end(); + await expectFullSave(f); + }); +}); diff --git a/server/src/services/remote-workspaces.ts b/server/src/services/remote-workspaces.ts index e45cbb99b55..f61475bce95 100644 --- a/server/src/services/remote-workspaces.ts +++ b/server/src/services/remote-workspaces.ts @@ -42,6 +42,9 @@ import { attachPreservedRemoteWorkspaceSnapshots, retainRemoteRecoveryAttachment import { notifyRemoteWorkspaceSnapshots } from "./remote-workspace-snapshot-retention.js"; import { notifyTerminalWorkspaceWork } from "./terminal-workspace-work.js"; import { preserveRemoteWorkspacePreparationFailure } from "./native-runtime/native-workspace-finalization-state.js"; +import { proveRemoteWorkspaceUnchanged, readUnchangedBaseline, UNCHANGED_BASELINE_KEY, UNCHANGED_BASELINE_SCHEMA, + type UnchangedBaseline } from "./remote-workspace-unchanged-proof.js"; +import { wakeWorkspaceBusyWaitersInTx } from "../modules/run-dispatch/index.js"; export { readRemoteWorkspaceAuthority, type RemoteWorkspaceAuthority, type RemoteWorkspaceSource } from "@paperclipai/shared"; export { remoteWorkspaceRepositoryPath } from "./remote-workspace-transport.js"; @@ -206,9 +209,9 @@ export async function reserveRemoteWorkspace(input: { selectedSharedSource = selected !== null; } const [selectedRow] = selected ? await tx.select().from(executionWorkspaces).where(and(eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, selected))).for("update") : []; - const existing = selectedRow ?? null; + let existing = selectedRow ?? null; if (selected && !existing && !input.fresh) throw new Error("remote_workspace_missing"); - const authority = readRemoteWorkspaceAuthority(existing?.metadata); + let authority = readRemoteWorkspaceAuthority(existing?.metadata); if (existing && !authority) throw conflict("Import this existing workspace before enabling remote storage.", { code: "remote_workspace_migration_required" }); if (existing && authority) { if (["archived", "cleanup_failed"].includes(existing.status)) throw conflict("Workspace ownership changed."); @@ -236,6 +239,12 @@ export async function reserveRemoteWorkspace(input: { await settleUnverifiedProvision(tx as unknown as Db, operation, input.runId); provenProvisions.push(operation.id); } + // An ended writer that provably changed nothing hands the workspace on + // at its saved version, so this run is admitted instead of waiting. + if (authority.dirty && authority.writerRunId && authority.writerRunId !== input.runId) { + const released = await releaseUnchangedRemoteWorkspaceWriter(tx, { workspace: existing, authority, wakeWaiters: false }); + if (released) ({ workspace: existing, authority } = released); + } const hold = await readRemoteWorkspaceWriterHold(tx as unknown as Db, { companyId: input.companyId, authority, runId: input.runId }); if (hold.kind === "held") throw new RemoteWorkspaceWriterActiveError(existing.id, hold.writer); if (hold.kind === "unattributed") throw conflict("The workspace still has an unfinished writer or save operation.", { code: "remote_workspace_writer_pending" }); @@ -551,6 +560,13 @@ export async function prepareRemoteWorkspace(input: { || current.location.remoteCwd !== remoteRoot)) throw new Error("remote_workspace_authority_changed"); const authority: RemoteWorkspaceAuthority = { ...current, dirty: true, location: { environmentId: input.lease.environmentId!, leaseId: input.lease.id, providerLeaseId: input.lease.providerLeaseId!, remoteCwd: remoteRoot } }; + // Files published here equal the saved version only when this preparation + // started from a clean saved version and added nothing to it. Later writes + // (provisioning, runner commands, saves) leave their own durable records. + const unchangedBaseline: UnchangedBaseline | undefined = initial.snapshotId && !initial.dirty && migration === null + && current.snapshotId === initial.snapshotId && recoveredFiles.length === 0 + && ((input.source ?? initial.source).repositories ?? []).every(repository => isSavedWorkspaceRepository(input.workspace, initial, repository)) + ? { schema: UNCHANGED_BASELINE_SCHEMA, epoch: authority.epoch, runId: input.runId, snapshotId: initial.snapshotId } : undefined; const [lease] = await tx.select().from(environmentLeases).where(and(eq(environmentLeases.id, input.lease.id), eq(environmentLeases.companyId, workspace.companyId))).for("update"); if (!lease || lease.status !== "active" || lease.environmentId !== input.lease.environmentId @@ -559,7 +575,7 @@ export async function prepareRemoteWorkspace(input: { const [savedLease] = await tx.update(environmentLeases).set({ metadata: { ...lease.metadata, remoteCwd: remoteRoot, remoteWorkspaceId: workspace.id }, updatedAt: new Date() }) .where(eq(environmentLeases.id, lease.id)).returning(); const [savedWorkspace] = await tx.update(executionWorkspaces).set({ cwd: remoteRoot, providerRef: remoteRoot, - metadata: { ...workspace.metadata, remoteWorkspace: authority, + metadata: { ...workspace.metadata, remoteWorkspace: authority, [UNCHANGED_BASELINE_KEY]: unchangedBaseline, // A live copy names the run it is shown to; the next preparation retires // it once that run has succeeded. Retired records stay so a sandbox // restored from an older save is cleaned up too. @@ -700,6 +716,57 @@ async function settleSaveInvocation(input: { db: Db; workspace: Workspace; runId }); } +/** + * Hand a workspace on at its saved version when its writer ended without + * changing it (see proveRemoteWorkspaceUnchanged). The caller holds the + * workspace row lock. Returns null, changing nothing, when the proof fails or + * the writer run is locked by another transaction; its full save still applies. + */ +export async function releaseUnchangedRemoteWorkspaceWriter(tx: Transaction, input: { + workspace: typeof executionWorkspaces.$inferSelect; authority: RemoteWorkspaceAuthority; + callerOwnsWriter?: boolean; wakeWaiters?: boolean; +}): Promise<{ workspace: typeof executionWorkspaces.$inferSelect; authority: RemoteWorkspaceAuthority } | null> { + const { workspace, authority } = input; + const runId = authority.writerRunId; + if (!runId || !authority.dirty || !authority.snapshotId || !readUnchangedBaseline(workspace.metadata)) return null; + // Never wait on a writer run another transaction holds; that owner settles it. + const [writer] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, workspace.companyId), eq(heartbeatRuns.id, runId))).for("update", { skipLocked: true }); + if (!writer) return null; + const proof = await proveRemoteWorkspaceUnchanged(tx as unknown as Db, { workspace, authority, + callerOwnsWriter: input.callerOwnsWriter, isSavedRepository: repository => isSavedWorkspaceRepository(workspace, authority, repository) }); + if (!proof.proven) { + logger.info({ runId, workspaceId: workspace.id, reason: proof.reason }, "Remote workspace keeps its full save"); + return null; + } + const at = new Date(); + const released: RemoteWorkspaceAuthority = { ...authority, dirty: false, writerRunId: null }; + const [saved] = await tx.update(executionWorkspaces).set({ metadata: { ...workspace.metadata, remoteWorkspace: released, + [UNCHANGED_BASELINE_KEY]: { ...proof.baseline, releasedAt: at.toISOString() } }, updatedAt: at }) + .where(eq(executionWorkspaces.id, workspace.id)).returning(); + await persistRunReference(tx, workspace.companyId, runId, reference(workspace.id, released, "saved")); + await notifyRemoteWorkspaceSnapshots(tx, `workspace:${workspace.id}`); + await notifyTerminalWorkspaceWork(tx, `workspace:${workspace.id}`); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionReplacement); + if (input.wakeWaiters !== false) await wakeWorkspaceBusyWaitersInTx(tx as unknown as Db, { companyId: workspace.companyId, + runId, contextSnapshot: null, scheduledRetryReason: null, now: at }); + await persistActivity(tx as unknown as Db, { companyId: workspace.companyId, actorType: "system", actorId: "remote-workspaces", + action: "workspace.snapshot_skipped_unchanged", entityType: "execution_workspace", entityId: workspace.id, runId, + details: { snapshotId: authority.snapshotId, epoch: authority.epoch } }); + return { workspace: saved!, authority: released }; +} + +/** Locks the workspace and releases `runId` as its unchanged ended writer, if provable. */ +export async function releaseUnchangedRemoteWorkspaceWriterById(tx: Transaction, input: { + companyId: string; workspaceId: string; runId: string; +}) { + const [workspace] = await tx.select().from(executionWorkspaces).where(and(eq(executionWorkspaces.companyId, input.companyId), + eq(executionWorkspaces.id, input.workspaceId))).for("update"); + const authority = readRemoteWorkspaceAuthority(workspace?.metadata); + if (!workspace || !authority || authority.writerRunId !== input.runId) return null; + return releaseUnchangedRemoteWorkspaceWriter(tx, { workspace, authority }); +} + export async function saveRemoteWorkspace(input: { db: Db; workspace: Workspace; runId: string; lease: EnvironmentLease; target: AdapterExecutionTarget | null; assertOwned: () => Promise; storage?: StorageService; boundary?: string; @@ -711,11 +778,28 @@ export async function saveRemoteWorkspace(input: { if (!expected?.location || expected.location.providerLeaseId !== input.lease.providerLeaseId || expected.location.leaseId !== input.lease.id) throw new Error("remote_workspace_save_binding_mismatch"); await input.assertOwned(); const invocationId = randomUUID(); - const operation = await input.db.transaction(async tx => { + const reserved = await input.db.transaction(async tx => { const issueId = await lockWorkspaceRunParents(tx, input.workspace.companyId, input.runId); const workspace = await currentWorkspace(tx, input.workspace.companyId, input.workspace.id); const current = readRemoteWorkspaceAuthority(workspace.metadata); if (!current || current.epoch !== expected.epoch || (current.writerRunId !== input.runId && current.dirty)) throw new Error("remote_workspace_authority_changed"); + // A writer that ended without changing its files keeps its saved version. + // No capture runs; a repeated call returns the same version. + if (boundary === "final") { + const baseline = readUnchangedBaseline(workspace.metadata); + const replayed = baseline?.releasedAt && baseline.runId === input.runId && baseline.epoch === current.epoch + && baseline.snapshotId === current.snapshotId && !current.dirty && current.writerRunId === null; + const released = replayed ? { authority: current } : current.writerRunId === input.runId + ? await releaseUnchangedRemoteWorkspaceWriter(tx, { workspace, authority: current, callerOwnsWriter: true }) : null; + if (released?.authority.snapshotId) { + const [snapshot] = await tx.select().from(executionWorkspaceSnapshots).where(and( + eq(executionWorkspaceSnapshots.companyId, workspace.companyId), eq(executionWorkspaceSnapshots.executionWorkspaceId, workspace.id), + eq(executionWorkspaceSnapshots.id, released.authority.snapshotId))); + if (!snapshot?.sha256) throw new Error("remote_workspace_snapshot_missing"); + return { kind: "unchanged" as const, saved: { snapshotId: snapshot.id, generation: snapshot.generation, sha256: snapshot.sha256, + savedAt: snapshot.updatedAt.toISOString(), unchanged: true as const } }; + } + } const [previous] = await tx.select().from(workspaceOperations).where(and(eq(workspaceOperations.companyId, workspace.companyId), eq(workspaceOperations.executionWorkspaceId, workspace.id), eq(workspaceOperations.heartbeatRunId, input.runId), eq(workspaceOperations.phase, "remote_workspace_snapshot"), sql`${workspaceOperations.metadata}->>'boundary' = ${boundary}`, @@ -729,7 +813,7 @@ export async function saveRemoteWorkspace(input: { const [registered] = await tx.update(workspaceOperations).set({ ...reopened, metadata: { ...previous.metadata, saveWork: { schema: SAVE_WORK_SCHEMA, activeIds: [...(work?.activeIds ?? []), invocationId], unknown: work?.unknown ?? true, quiescedAt: null } }, updatedAt: new Date() }).where(eq(workspaceOperations.id, previous.id)).returning(); - return registered!; + return { kind: "registered" as const, operation: registered! }; } // The workspace lock serializes generation reservations. Failed/deleted // snapshots still own their generation, and an operation can be committed @@ -753,8 +837,10 @@ export async function saveRemoteWorkspace(input: { saveWork: { schema: SAVE_WORK_SCHEMA, activeIds: [invocationId], unknown: false, quiescedAt: null }, exportCleanup: { ...current.location, state: "pending" } }, }).returning(); - return created!; + return { kind: "registered" as const, operation: created! }; }); + if (reserved.kind === "unchanged") return reserved.saved; + const { operation } = reserved; let captured: Awaited> | null = null; let publicationScratch: string | undefined; let failure: { code: string; cause?: string } | undefined; @@ -809,6 +895,9 @@ export async function saveRemoteWorkspace(input: { if (boundary === "final") { await notifyTerminalWorkspaceWork(tx, `workspace:${workspace.id}`); await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionReplacement); + // The hold ends with this commit; runs parked behind it start now. + await wakeWorkspaceBusyWaitersInTx(tx as unknown as Db, { companyId: workspace.companyId, runId: input.runId, + contextSnapshot: null, scheduledRetryReason: null, now: at }); } const receipt = object(workspace.metadata?.remoteWorkspaceSnapshotRepositories); const priorRepositories = receipt.snapshotId === current.snapshotId && Array.isArray(receipt.repositories) From 32b3648758e9ac3832efa0acb194a6e4fdbbe3ba Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Sun, 8 Nov 2026 20:42:12 -0800 Subject: [PATCH 2/9] fix(storage): back off and reduce parallel parts when S3 asks to slow down (#20165) A workspace snapshot upload sends up to eight file-backed parts at once. The SDK never retries a part because its body is a stream, so a part that received 503 SlowDown had only the uploader's three fixed-delay attempts (0.5 s and 1 s). Under a request-rate limit the save failed and the run failed in setup. A rate-limited part now retries with exponential full-jitter backoff (up to eight attempts, at most 20 s per wait), and the upload halves the parts it admits, once per round of throttled requests. Other part failures keep the existing three attempts. Tests cover a real-client SlowDown recovery, the single halving, the attempt bound, cancellation during a wait, and the backoff schedule through its ceiling. Co-Authored-By: Claude Opus 5.5 --- .../storage-s3-multipart-backoff.test.ts | 54 ++++++ .../storage-s3-multipart-throttle.test.ts | 164 ++++++++++++++++++ server/src/storage/s3-multipart.ts | 44 ++++- 3 files changed, 255 insertions(+), 7 deletions(-) create mode 100644 server/src/__tests__/storage-s3-multipart-backoff.test.ts create mode 100644 server/src/__tests__/storage-s3-multipart-throttle.test.ts diff --git a/server/src/__tests__/storage-s3-multipart-backoff.test.ts b/server/src/__tests__/storage-s3-multipart-backoff.test.ts new file mode 100644 index 00000000000..7fccdc437d1 --- /dev/null +++ b/server/src/__tests__/storage-s3-multipart-backoff.test.ts @@ -0,0 +1,54 @@ +import { CreateMultipartUploadCommand, UploadPartCommand, S3Client } from "@aws-sdk/client-s3"; +import type { Readable } from "node:stream"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it, vi } from "vitest"; +import { THROTTLED_PART_ATTEMPTS, putS3MultipartFile } from "../storage/s3-multipart.js"; + +// Record each requested wait and resolve it at once, so the schedule is +// observable without waiting for it. +const waits = vi.hoisted(() => [] as number[]); +vi.mock("node:timers/promises", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + setTimeout: (ms: number, value?: unknown, options?: { signal?: AbortSignal }) => { + waits.push(ms); + return actual.setTimeout(0, value, options); + }, + }; +}); + +const slowDown = () => Object.assign(new Error("Please reduce your request rate."), { name: "SlowDown", $metadata: { httpStatusCode: 503 } }); + +describe("file-backed S3 multipart backoff schedule", () => { + it("doubles the jittered wait for each throttled attempt up to the 20 s ceiling", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "pc-s3-backoff-")); + const file = path.join(dir, "source.bin"), size = 3 * 8 * 1024 * 1024; + await fs.writeFile(file, Buffer.alloc(size, 7), { mode: 0o600 }); + const client = new S3Client({ region: "us-east-1" }); + let partOneAttempts = 0; + const random = vi.spyOn(Math, "random").mockReturnValue(0.5); + const send = vi.spyOn(client, "send").mockImplementation(async (command: any) => { + if (command instanceof CreateMultipartUploadCommand) return { UploadId: "upload" }; + if (command instanceof UploadPartCommand) { + (command.input.Body as Readable).resume(); + if (command.input.PartNumber === 1 && ++partOneAttempts < THROTTLED_PART_ATTEMPTS) throw slowDown(); + return { ETag: `part-${command.input.PartNumber}` }; + } + return {}; + }); + waits.length = 0; + try { + await expect(putS3MultipartFile(client, "bucket", "key", { objectKey: "key", contentType: "application/octet-stream", + contentLength: size, body: Buffer.alloc(0), sourceFile: file })).resolves.toEqual({ integrity: "server_verified_md5" }); + expect(partOneAttempts).toBe(THROTTLED_PART_ATTEMPTS); + // Half of each ceiling: 1, 2, 4, 8, 16, then 20, 20 seconds. + expect(waits).toEqual([500, 1_000, 2_000, 4_000, 8_000, 10_000, 10_000]); + } finally { + random.mockRestore(); send.mockRestore(); client.destroy(); + await fs.rm(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/server/src/__tests__/storage-s3-multipart-throttle.test.ts b/server/src/__tests__/storage-s3-multipart-throttle.test.ts new file mode 100644 index 00000000000..0572359cd79 --- /dev/null +++ b/server/src/__tests__/storage-s3-multipart-throttle.test.ts @@ -0,0 +1,164 @@ +import { AbortMultipartUploadCommand, CreateMultipartUploadCommand, UploadPartCommand, S3Client } from "@aws-sdk/client-s3"; +import { Readable } from "node:stream"; +import { createHash } from "node:crypto"; +import { createServer } from "node:http"; +import type { AddressInfo } from "node:net"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it, vi } from "vitest"; +import { FILE_PART_CONCURRENCY, THROTTLED_PART_ATTEMPTS, putS3MultipartFile } from "../storage/s3-multipart.js"; +import { createS3StorageProvider } from "../storage/s3-provider.js"; +import { createStorageService } from "../storage/service.js"; + +const partBytes = 8 * 1024 * 1024; + +async function sourceFile(parts: number) { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "pc-s3-throttle-")); + const file = path.join(dir, "source.bin"); + const handle = await fs.open(file, "wx", 0o600); + try { + for (let index = 0; index < parts; index++) await handle.write(Buffer.alloc(partBytes, index + 1)); + } finally { await handle.close(); } + return { dir, file, size: parts * partBytes }; +} + +const md5 = (bytes: Buffer) => createHash("md5").update(bytes).digest("base64"); + +async function drain(body: unknown) { + const chunks: Buffer[] = []; + for await (const chunk of body as Readable) chunks.push(Buffer.from(chunk)); + return Buffer.concat(chunks); +} + +const slowDown = () => Object.assign(new Error("Please reduce your request rate."), { name: "SlowDown", $metadata: { httpStatusCode: 503 } }); + +describe("file-backed S3 multipart uploads under a request-rate limit", () => { + it("backs off and completes when the real S3 client receives SlowDown for one part", async () => { + const source = await sourceFile(3); + const attempts = new Map(); + let completed = ""; + const server = createServer((request, response) => { + const url = new URL(request.url!, "http://localhost"); + const chunks: Buffer[] = []; + request.on("data", chunk => chunks.push(chunk)); + request.on("end", () => { + if (url.searchParams.has("uploads")) { + response.end("upload"); + } else if (request.method === "PUT") { + const number = Number(url.searchParams.get("partNumber")); + attempts.set(number, (attempts.get(number) ?? 0) + 1); + if (number === 2 && attempts.get(number)! <= 3) { + response.writeHead(503, { "Content-Type": "application/xml" }) + .end("SlowDownPlease reduce your request rate.throttled"); + } else { + expect(md5(Buffer.concat(chunks))).toBe(request.headers["content-md5"]); + response.writeHead(200, { ETag: `"part-${number}"` }).end(); + } + } else if (request.method === "POST") { + completed = Buffer.concat(chunks).toString(); + response.end("complete"); + } else { + response.writeHead(500).end(); + } + }); + }); + await new Promise((resolve, reject) => { server.once("error", reject); server.listen(0, "127.0.0.1", resolve); }); + vi.stubEnv("AWS_ACCESS_KEY_ID", "fixture"); vi.stubEnv("AWS_SECRET_ACCESS_KEY", "fixture"); + const random = vi.spyOn(Math, "random").mockReturnValue(0); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const service = createStorageService(createS3StorageProvider({ bucket: "bucket", region: "us-east-1", forcePathStyle: true, + endpoint: `http://127.0.0.1:${(server.address() as AddressInfo).port}` })); + try { + const result = await service.putFile({ companyId: "company", objectKey: "company/workspace-snapshots/throttled.pcws", namespace: "workspace-snapshots", + originalFilename: null, contentType: "application/octet-stream", body: Readable.from([Buffer.alloc(0)]), byteSize: source.size, + sha256: createHash("sha256").update(await fs.readFile(source.file)).digest("hex"), sourceFile: source.file }); + expect(result.integrity).toBe("server_verified_md5"); + expect(attempts.get(2)).toBe(4); + expect([...completed.matchAll(/(\d+)<\/PartNumber>/g)].map(match => Number(match[1]))).toEqual([1, 2, 3]); + } finally { + random.mockRestore(); warn.mockRestore(); vi.unstubAllEnvs(); + server.closeAllConnections(); + await new Promise(resolve => server.close(() => resolve())); + await fs.rm(source.dir, { recursive: true, force: true }); + } + }); + + it("halves new parts in flight once when a parallel burst of parts is throttled", async () => { + const source = await sourceFile(16); + const client = new S3Client({ region: "us-east-1" }); + const attempts = new Map(); + let laterActive = 0, laterPeak = 0; + const random = vi.spyOn(Math, "random").mockReturnValue(0); + const send = vi.spyOn(client, "send").mockImplementation(async (command: any) => { + if (command instanceof CreateMultipartUploadCommand) return { UploadId: "upload" }; + if (command instanceof UploadPartCommand) { + const number = command.input.PartNumber!; + attempts.set(number, (attempts.get(number) ?? 0) + 1); + await drain(command.input.Body); + if (number <= FILE_PART_CONCURRENCY && attempts.get(number) === 1) throw slowDown(); + if (number > FILE_PART_CONCURRENCY) { + laterActive++; laterPeak = Math.max(laterPeak, laterActive); + await new Promise(resolve => setTimeout(resolve, 20)); + laterActive--; + } + return { ETag: `part-${number}` }; + } + if (command instanceof AbortMultipartUploadCommand) throw new Error("must not abort a recovered upload"); + return {}; + }); + try { + await expect(putS3MultipartFile(client, "bucket", "key", { objectKey: "key", contentType: "application/octet-stream", + contentLength: source.size, body: Buffer.alloc(0), sourceFile: source.file })).resolves.toEqual({ integrity: "server_verified_md5" }); + expect([...attempts.keys()].sort((a, b) => a - b)).toEqual([...Array(16).keys()].map(n => n + 1)); + // Eight simultaneous throttles are one round: four slots remain, not one. + expect(laterPeak).toBe(FILE_PART_CONCURRENCY / 2); + } finally { random.mockRestore(); send.mockRestore(); client.destroy(); await fs.rm(source.dir, { recursive: true, force: true }); } + }); + + it("stops after a bounded number of throttled attempts and aborts once", async () => { + const source = await sourceFile(3); + const client = new S3Client({ region: "us-east-1" }); + const random = vi.spyOn(Math, "random").mockReturnValue(0); + const send = vi.spyOn(client, "send").mockImplementation(async (command: any) => { + if (command instanceof CreateMultipartUploadCommand) return { UploadId: "upload" }; + if (command instanceof UploadPartCommand) { + await drain(command.input.Body); + if (command.input.PartNumber === 1) throw slowDown(); + return { ETag: `part-${command.input.PartNumber}` }; + } + if (command instanceof AbortMultipartUploadCommand) return {}; + throw new Error("must not complete a failed upload"); + }); + try { + await expect(putS3MultipartFile(client, "bucket", "key", { objectKey: "key", contentType: "application/octet-stream", + contentLength: source.size, body: Buffer.alloc(0), sourceFile: source.file })).rejects.toMatchObject({ name: "SlowDown" }); + const partOne = send.mock.calls.filter(([command]) => command instanceof UploadPartCommand && command.input.PartNumber === 1); + expect(partOne).toHaveLength(THROTTLED_PART_ATTEMPTS); + expect(send.mock.calls.filter(([command]) => command instanceof AbortMultipartUploadCommand)).toHaveLength(1); + } finally { random.mockRestore(); send.mockRestore(); client.destroy(); await fs.rm(source.dir, { recursive: true, force: true }); } + }); + + it("cancels promptly while a throttled part is backing off", async () => { + const source = await sourceFile(3); + const client = new S3Client({ region: "us-east-1" }); + const cancellation = new AbortController(); + const random = vi.spyOn(Math, "random").mockReturnValue(0.99); + const send = vi.spyOn(client, "send").mockImplementation(async (command: any) => { + if (command instanceof CreateMultipartUploadCommand) return { UploadId: "upload" }; + if (command instanceof UploadPartCommand) { + await drain(command.input.Body); + if (command.input.PartNumber === 1) { setTimeout(() => cancellation.abort(new Error("run cancelled")), 10); throw slowDown(); } + return { ETag: `part-${command.input.PartNumber}` }; + } + return {}; + }); + const started = Date.now(); + try { + await expect(putS3MultipartFile(client, "bucket", "key", { objectKey: "key", contentType: "application/octet-stream", + contentLength: source.size, body: Buffer.alloc(0), sourceFile: source.file, signal: cancellation.signal })).rejects.toThrow("run cancelled"); + expect(Date.now() - started).toBeLessThan(900); + expect(send.mock.calls.filter(([command]) => command instanceof UploadPartCommand && command.input.PartNumber === 1)).toHaveLength(1); + } finally { random.mockRestore(); send.mockRestore(); client.destroy(); await fs.rm(source.dir, { recursive: true, force: true }); } + }); +}); diff --git a/server/src/storage/s3-multipart.ts b/server/src/storage/s3-multipart.ts index 0c187233eae..408443c9c1f 100644 --- a/server/src/storage/s3-multipart.ts +++ b/server/src/storage/s3-multipart.ts @@ -129,8 +129,23 @@ export async function putS3Multipart(client: S3Client, bucket: string, key: stri export const FILE_PART_CONCURRENCY = 8; const FILE_PART_ATTEMPTS = 3; +/** A provider that asks for a lower request rate gets more, longer-spaced attempts. */ +export const THROTTLED_PART_ATTEMPTS = 8; +const THROTTLE_BACKOFF_BASE_MS = 1_000; +const THROTTLE_BACKOFF_MAX_MS = 20_000; const FILE_READ_BYTES = 1024 * 1024; +/** + * S3 answers a request-rate limit with 503 SlowDown; compatible gateways also + * use 429. The SDK never retries a part because its body is a stream, so the + * upload must slow down itself. + */ +function isRequestRateLimited(error: unknown) { + if (!error || typeof error !== "object") return false; + const { name, $metadata } = error as { name?: unknown; $metadata?: { httpStatusCode?: unknown } }; + return name === "SlowDown" || $metadata?.httpStatusCode === 503 || $metadata?.httpStatusCode === 429; +} + /** Base64 MD5 of an inclusive byte range, as the Content-MD5 header expects. */ export async function fileRangeMd5(file: string, start: number, end: number, signal?: AbortSignal) { const digest = createHash("md5"); @@ -146,7 +161,9 @@ export async function assertSourceFileSize(file: string, contentLength: number) /** A file source is read per part instead of through one stream. Parts upload * in parallel without the shared memory window, each carries Content-MD5 so * the backend rejects bytes it did not receive intact, and a failed part is - * resent from disk instead of failing the whole upload. */ + * resent from disk instead of failing the whole upload. When the backend + * limits the request rate, the part backs off with jitter and the upload + * halves its parts in flight, once per round of throttled requests. */ export async function putS3MultipartFile( client: S3Client, bucket: string, key: string, input: PutObjectInput & { sourceFile: string }, ): Promise { @@ -167,12 +184,17 @@ export async function putS3MultipartFile( if (!uploadId) throw new Error("S3 did not return a multipart upload id"); const id = uploadId; const parts: { ETag: string; PartNumber: number }[] = new Array(count); + // Workers at or above this slot stop taking new parts. A reduction counts + // only for requests sent after the previous one, so a burst of parallel + // throttled parts halves the rate once rather than once per part. + let concurrency = FILE_PART_CONCURRENCY, reductions = 0; const sendPart = async (index: number) => { const start = index * partSize, length = Math.min(partSize, input.contentLength - start), end = start + length - 1; const contentMd5 = await fileRangeMd5(file, start, end, controller.signal); - for (let attempt = 1; ; attempt++) { + for (let failures = 0, throttles = 0; ;) { controller.signal.throwIfAborted(); const body = createReadStream(file, { start, end, highWaterMark: 64 * 1024 }); + const sentAfter = reductions; try { const result = await withS3FailureDiagnostics("upload_part", () => client.send(new UploadPartCommand({ Bucket: bucket, Key: key, UploadId: id, PartNumber: index + 1, Body: body, ContentLength: length, ContentMD5: contentMd5, @@ -182,21 +204,29 @@ export async function putS3MultipartFile( return; } catch (error) { // A part read from disk can be resent; the stream path cannot. - if (controller.signal.aborted || attempt >= FILE_PART_ATTEMPTS) throw error; - await delay(attempt * 500, undefined, { signal: controller.signal }); + if (controller.signal.aborted) throw error; + if (isRequestRateLimited(error)) { + if (++throttles >= THROTTLED_PART_ATTEMPTS) throw error; + if (sentAfter === reductions) { concurrency = Math.max(1, Math.floor(concurrency / 2)); reductions++; } + const ceiling = Math.min(THROTTLE_BACKOFF_MAX_MS, THROTTLE_BACKOFF_BASE_MS * 2 ** (throttles - 1)); + await delay(Math.floor(Math.random() * ceiling), undefined, { signal: controller.signal }); + continue; + } + if (++failures >= FILE_PART_ATTEMPTS) throw error; + await delay(failures * 500, undefined, { signal: controller.signal }); } finally { body.destroy(); } } }; let next = 0; - const worker = async () => { - while (next < count) { + const worker = async (slot: number) => { + while (next < count && slot < concurrency) { const index = next++; try { await sendPart(index); } catch (error) { if (!controller.signal.aborted) controller.abort(error); throw error; } } }; // Wait for every sibling to settle before the multipart upload is aborted. - const settled = await Promise.allSettled(Array.from({ length: Math.min(FILE_PART_CONCURRENCY, count) }, worker)); + const settled = await Promise.allSettled(Array.from({ length: Math.min(FILE_PART_CONCURRENCY, count) }, (_, slot) => worker(slot))); const rejected = settled.find((entry): entry is PromiseRejectedResult => entry.status === "rejected"); if (rejected) throw controller.signal.aborted && controller.signal.reason !== undefined ? controller.signal.reason : rejected.reason; controller.signal.throwIfAborted(); From b54317d3fb12531b990e615b262ca05ac2e2b686 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sun, 8 Nov 2026 22:44:18 -0600 Subject: [PATCH 3/9] fix(native): give a preparing run's first turn the user's queued messages instead of interrupting it (#20168) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - A Paperclip Runner (native) run prepares its workspace and environment before it starts its first provider turn. This can take about a minute. > - A user's message that arrives during that time waits as a deferred wake behind the run. The Interrupt control then cancelled the run before any provider turn. > - The cancel threw away the whole preparation. A new run had to prepare again, so the user waited longer for the first token. > - This pull request gives the run's first turn the queued messages. The run seals its first-turn input immediately before `turn.start`, so the messages are part of the prompt that the turn starts with. > - The benefit is that the user sees the message handled in the same run, with no cancel and no second preparation. ## Linked Issues or Issue Description **What happened?** A Paperclip Runner run took about 54 s to prepare (environment startup, workspace, runtime selection). The user sent a second message 32 s into that time and clicked Interrupt 21 s later. Interrupt cancelled the run as `operator_interrupted` ("Interrupted to send queued messages") before any provider turn. A new run then prepared again from the start. The runtime-selection transaction that froze the run's input completed about 5 s before the click, and the coordinator claim came 4 s after it, so the message could not be given to the first turn at any point in the old code. **Expected behavior** When a user message arrives for a task whose current run has not started its first provider turn, the run does not stop. The message goes into that run's first turn. A run that is in a turn is interrupted as before. **Steps to reproduce** 1. Assign a task to a Paperclip Runner agent with a sandbox environment, so preparation takes a long time. 2. While the run prepares, post a second comment on the task. 3. Click Interrupt on the queued message before the run starts its provider turn. 4. See that the run is cancelled and a new run prepares again. **Paperclip version or commit** Staging revision b43cccbb0. ## What Changed - `server/src/services/native-runtime/first-turn-queued-input.ts` (new): `claimFirstTurnQueuedInput` seals a run's first-turn input in one transaction. It locks the task, the deferred wakes, and the run in the same order as the queue routes. It takes the responsible user's plain queued comments in queue order (not interaction answers, interrupt intents, independent continuations, conversations, or other users' messages). It renders them with source-trust sanitizing and secret redaction, then writes a `firstTurnQueuedInput` receipt (`paperclip.first-turn-queued-input.v1`) into the run context. A second call for the same run returns the same messages. `runAcceptsFirstTurnQueuedInput` is true for a running native (or preparing native) run that has no receipt and no accepted provider turn. - `packages/paperclip-runner/src/native-session-runtime.ts`: new `getFirstTurnQueuedInput` option. The runner calls it immediately before the first `session.startTurn` and appends the result to `task.prompt` (or to `events` for a resumed-session continuation envelope). It is not called for the disposition-only recovery turn. - `native-session-executor.ts`: passes the claim to the runner. A failed seal commits nothing, so the executor logs it and starts the turn without the messages. The messages stay queued. - `queued-comment-delivery.ts`: a message in a run's receipt counts as delivered only after the provider accepted that run's turn (`turn.accepted` or the provider's `turn.started`), and only if the message was not edited after the seal. Release promotion, Send, and later first turns use this rule. `explicit-native-continuation.ts` uses the same rule for stopped native runs. - `routes/issues.ts` Interrupt: when the target run has not sealed its first turn and has no accepted turn, the route does not record interrupt intent and does not cancel the run. The response has `executionWait.reason = "first_turn_pending"`. Other runs are interrupted as before. - Queue views (`routes/issues.ts` and the wake-queue adapter) hide messages that the active run took into its first turn. Queue edits keep those ids on the wake, so a run that fails before acceptance still leaves them queued. - `run-state.ts`: the issue-control run projection includes the receipt, so the queue poll can hide taken messages. - `doc/execution-semantics.md`: new section "Queued messages before a native run's first turn". ## Verification - Red before the fix, green after: - `pnpm --filter @paperclipai/server exec vitest run src/__tests__/issue-queued-comments-routes.test.ts -t "first turn|already started"`: Interrupt on a preparing run and on a claimed native run before `turn.start` does not stop the run, and the first turn gets both messages in order. Before the fix, both cases cancelled the run. - `pnpm --filter @paperclipai/paperclip-runner exec vitest run src/native-session-runtime.test.ts -t "first-turn queued input"`: the claim runs immediately before `turn.start`, and the first turn's prompt ends with the queued messages. Before the fix, the claim was never called. - `pnpm --filter @paperclipai/server exec vitest run src/services/native-runtime/first-turn-queued-input.test.ts`: a run that fails before `turn.start` leaves the message queued. The next run delivers it, and after that no later run or copied retry context takes it again. Before the delivery rule, the "exactly once" and "accepted then failed" cases failed. - Safety cases that pass before and after: Interrupt still cancels a run whose provider accepted a turn (including a run saved by the previous release with no receipt) and a run whose first-turn input is already sealed. The existing "explicit interrupt fallback" cases for preparing and native runs without a responsible user still pass. - Speed: the seal is one short transaction before `turn.start`. In the embedded-Postgres test it took 5.5 ms. It removes a full second preparation (54 s on staging) from the message-to-first-token time. - Related suites: `src/modules/wake-queue`, `explicit-native-continuation`, `issue-queued-comment-queue`, `issue-queued-comments-routes`, `queued-comment-consumption`, `heartbeat/run-state` (888 tests), `native-session-executor.test.ts` (1306 tests), and the full `native-session-runtime.test.ts` (238 tests) pass. `pnpm --filter @paperclipai/server typecheck` passes. - Not done: no live run against a real provider or sandbox. Staging was read only. ## Risks - Previous-release state: a deferred wake saved by the previous release has the same shape and is taken normally. A run saved by the previous release has no receipt. If it is resumed before its first turn, it seals on the new release. If its provider already accepted a turn, Interrupt stops it as before. A run saved by the previous release while it was in `turn.start` but before acceptance counts as unsealed, so in that short window an Interrupt does not stop it and the message is delivered at the turn boundary. - Rollback: if the server rolls back after a run sealed and its turn was accepted, the old release does not read the receipt and can deliver those messages a second time. - A first turn that does not use the fresh-turn path (restart continuation, disposition-only recovery) does not call the claim. Its messages are delivered at the turn boundary as before. - A message that arrives after the seal waits for the turn boundary. Interrupt stops the run as before. - No schema migration. The receipt is a key in the run context. ## Model Used - Claude Opus 5.5 (Anthropic), model ID `claude-opus-5-5`, used through Claude Code with tool use (shell, file edits, test execution, read-only browser). ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- doc/execution-semantics.md | 26 ++ .../src/native-session-runtime.test.ts | 50 ++++ .../src/native-session-runtime.ts | 14 + .../issue-queued-comments-routes.test.ts | 102 +++++++ .../adapters/queued-comment-postgres.ts | 23 +- server/src/routes/issues.ts | 32 ++- .../services/explicit-native-continuation.ts | 5 +- server/src/services/heartbeat/run-state.ts | 5 +- .../first-turn-queued-input.test.ts | 189 ++++++++++++ .../native-runtime/first-turn-queued-input.ts | 269 ++++++++++++++++++ .../native-runtime/native-session-executor.ts | 7 + .../src/services/queued-comment-delivery.ts | 40 ++- 12 files changed, 749 insertions(+), 13 deletions(-) create mode 100644 server/src/services/native-runtime/first-turn-queued-input.test.ts create mode 100644 server/src/services/native-runtime/first-turn-queued-input.ts diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 0f5c7d37023..762d41bf647 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -3627,6 +3627,32 @@ after cancellation. The old task's Stop receipt cannot settle cleanup of a new replacement task. A lost reply with no known resource identity remains unresolved until provider inspection or operator reconciliation establishes its outcome. +### Queued messages before a native run's first turn + +A user message that arrives while a Paperclip Runner run is still preparing, +or after its coordinator claim but before `turn.start`, is not a reason to stop +that run. Immediately before the first `turn.start`, the runner seals the +turn's input: in one transaction under the task, queue, and run locks it takes +the responsible user's plain queued comments for the task, in their queued +order, adds them to that turn's prompt, and records a +`firstTurnQueuedInput` receipt (`paperclip.first-turn-queued-input.v1`) in the +run context. Interaction answers, interrupt intents, independent +continuations, conversations, and other users' messages keep their own +delivery. Taken messages leave the visible queue. + +The receipt does not consume the messages. They stay on their deferred wake +until the provider accepts that run's turn (`turn.accepted` or the provider's +`turn.started`); only then do release promotion, Send, and later first turns +treat them as delivered, whatever the run's later outcome. A run that ends +before acceptance leaves them queued for the next run, and a restart before +acceptance repeats the same sealed input. An edit after the seal is unread. + +**Interrupt** on a run that has neither sealed its first turn nor had a turn +accepted records no intent and does not stop the run; the response's +`executionWait.reason` is `first_turn_pending`. Once the input is sealed or a +turn was accepted, Interrupt stops the run as described below. A run saved by +a release without the seal counts as started once its provider accepted a turn. + ### Delivering queued messages after a legacy run stops The legacy queued-message Interrupt action accepts a null `targetRunId` when diff --git a/packages/paperclip-runner/src/native-session-runtime.test.ts b/packages/paperclip-runner/src/native-session-runtime.test.ts index 9d40f756c99..fd5d7939cab 100644 --- a/packages/paperclip-runner/src/native-session-runtime.test.ts +++ b/packages/paperclip-runner/src/native-session-runtime.test.ts @@ -10352,3 +10352,53 @@ it("fails preconfigured recovery without replacing a missing provider session", expect(openSession).not.toHaveBeenCalled(); expect(openReplacementSession).not.toHaveBeenCalled(); }); + +describe("first-turn queued input", () => { + async function startFirstTurn(executionInput: NativeExecutionInputV1 | NativeExecutionInputV5, queued: string | null) { + const order: string[] = []; + const sent: string[] = []; + const session: NativeSession = { + identity: () => identity, + async capabilities() { return { resume: false, typedEvents: true, steering: false, interruption: true, structuredResult: true }; }, + async *events() {}, + async startTurn(turn) { + order.push("turn.start"); + sent.push(turn.message.text); + throw new Error("stop after turn.start"); + }, + async result() { return null; }, + async snapshot() { + return { backendKind: "mock", sessionId: "driver-first-turn", identity, providerSessionId: "provider-first-turn", + cursor: null, activeTurnId: null, pendingRuntimeRequests: [], lineage: [] }; + }, + close: vi.fn(async () => undefined), + }; + const getFirstTurnQueuedInput = vi.fn(async () => { order.push("claim"); return queued; }); + await expect(executeNativeSession({ + input: executionInput, + getFirstTurnQueuedInput, + backend: { async descriptor() { return { kind: "mock", name: "first-turn", version: "1", capabilities: await session.capabilities(), + runtimeContextCapabilities: { instructions: "native" as const, skills: "native" as const, mcp: "native" as const } }; }, async openSession() { return session; } }, + controlPlane: { async openRun() {}, async checkpointSession() {}, async appendEvent() { return { cursor: 1, highestContiguousSourceSeq: 1, disposition: "committed" as const }; }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {} }, + runnerInstanceId: "runner-first-turn", controlPlaneInstanceId: "control-first-turn", + })).rejects.toThrow("stop after turn.start"); + return { order, sent, getFirstTurnQueuedInput }; + } + + it("adds messages queued during preparation to the first turn, sealed just before turn.start", async () => { + const queued = "New messages from the user arrived while this run was starting.\n\n[Message 1 of 2]\nfirst\n\n[Message 2 of 2]\nsecond"; + const { order, sent, getFirstTurnQueuedInput } = await startFirstTurn(preparedInput(), queued); + expect(getFirstTurnQueuedInput).toHaveBeenCalledOnce(); + expect(order).toEqual(["claim", "turn.start"]); + const envelope = JSON.parse(sent[0]!) as { task: { prompt: string }; completionContract: { criteria: Array<{ source?: unknown }> } }; + expect(envelope.task.prompt).toBe(`${preparedInput().task.prompt}\n\n${queued}`); + // Appending input keeps the prompt-bound completion sources. + expect(envelope.completionContract.criteria[0]!.source).toMatchObject({ location: "task.prompt" }); + }); + + it("starts the first turn unchanged when nothing is queued", async () => { + const { sent } = await startFirstTurn(input, null); + expect((JSON.parse(sent[0]!) as { task: { prompt: string } }).task.prompt).toBe(input.task.prompt); + }); +}); diff --git a/packages/paperclip-runner/src/native-session-runtime.ts b/packages/paperclip-runner/src/native-session-runtime.ts index 2e8bcb37074..6246376bee9 100644 --- a/packages/paperclip-runner/src/native-session-runtime.ts +++ b/packages/paperclip-runner/src/native-session-runtime.ts @@ -305,6 +305,12 @@ export interface ExecuteNativeSessionOptions { resumeInterruptedTurn?: boolean; /** Read bounded task history only after recovery requires a fresh conversation. */ getFreshSessionHandoff?: () => Promise; + /** + * Seals this run's first-turn input immediately before `turn.start` and + * returns user messages that arrived while the run was preparing. Repeated + * calls for the same run return the same messages. + */ + getFirstTurnQueuedInput?: () => Promise; /** * Steered messages the provider acknowledged but the model never read, read * when a restart continuation starts. The provider lost them with the @@ -3048,6 +3054,14 @@ export async function executeNativeSession( if ("workspace" in modelEnvelope && modelEnvelope.workspace && recoveredSnapshot.workingDirectory) { modelEnvelope.workspace.cwd = recoveredSnapshot.workingDirectory; } + if (!(dispositionOnlyRecovery && !effectFreeInitialAcpxTurn) && options.getFirstTurnQueuedInput) { + const queuedInput = await options.getFirstTurnQueuedInput(); + if (queuedInput && "task" in modelEnvelope) { + modelEnvelope.task.prompt = `${modelEnvelope.task.prompt}\n\n${queuedInput}`; + } else if (queuedInput && "events" in modelEnvelope) { + modelEnvelope.events = `${modelEnvelope.events}\n\n${queuedInput}`; + } + } await session.startTurn({ message: { role: "user", text: JSON.stringify(modelEnvelope) }, ...(recovered && modelEnvelope.schema === "paperclip.native-continuation.v1" diff --git a/server/src/__tests__/issue-queued-comments-routes.test.ts b/server/src/__tests__/issue-queued-comments-routes.test.ts index c1a7e3e2401..1aa33edb2c1 100644 --- a/server/src/__tests__/issue-queued-comments-routes.test.ts +++ b/server/src/__tests__/issue-queued-comments-routes.test.ts @@ -36,6 +36,7 @@ import { hasQueuedInteractionResponse } from "../services/queued-interaction-res import { remoteTerminationReceipt } from "../services/remote-execution-termination.js"; import { initializeRunIdentity, reconcileSteeredIdentity } from "../services/run-identity.js"; import { appendHeartbeatRunEvent } from "../services/heartbeat-run-events.js"; +import { claimFirstTurnQueuedInput } from "../services/native-runtime/first-turn-queued-input.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, @@ -901,6 +902,107 @@ describeEmbeddedPostgres("issue queued-comment routes", () => { }, ); + async function seedFirstTurnQueue(runtime: "preparing" | "native") { + const seeded = await seedQueue(); + // A user's follow-ups, as the comment route saves them behind that user's run. + const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, seeded.wakeId)); + await db.update(agentWakeupRequests).set({ payload: { ...wake!.payload, + _paperclipWakeContext: { ...(wake!.payload?._paperclipWakeContext as Record), + issueId: seeded.issueId, wakeReason: "issue_commented" } } }).where(eq(agentWakeupRequests.id, seeded.wakeId)); + await db.update(heartbeatRuns).set(runtime === "preparing" + ? { responsibleUserId: "queue-owner", runtimeMode: "legacy", runtimeModeResolvedAt: null, executionStage: "preparing", + runnerProfileJson: { adapterDispatch: { adapterType: "paperclip_runner" } } } + : { responsibleUserId: "queue-owner", nativeIssueId: seeded.issueId }).where(eq(heartbeatRuns.id, seeded.runId)); + if (runtime === "native") { + await db.insert(nativeRunFinalizations).values({ companyId: seeded.companyId, + issueId: seeded.issueId, runId: seeded.runId, phase: "observed" }); + } + // Hold any successor in the ordinary queue without launching a provider. + await db.update(agents).set({ runtimeConfig: { heartbeat: { maxConcurrentRuns: 1 } } }) + .where(eq(agents.id, seeded.agentId)); + await db.insert(heartbeatRuns).values({ companyId: seeded.companyId, agentId: seeded.agentId, + status: "running", contextSnapshot: { issueId: randomUUID() } }); + return seeded; + } + + it.each(["preparing", "native"] as const)( + "keeps a %s run that has not started its first turn and gives it the queued messages", + async (runtime) => { + const seeded = await seedFirstTurnQueue(runtime); + const client = app(seeded.companyId); + const queue = await request(client).get(`/api/issues/${seeded.issueId}/queued-comments`).expect(200); + const response = await request(client).post(`/api/issues/${seeded.issueId}/queued-comments/interrupt`).send({ + queueId: seeded.wakeId, targetRunId: seeded.runId, revision: queue.body.revision, + }); + expect(response.status, JSON.stringify(response.body)).toBe(200); + expect(response.body.executionWait).toMatchObject({ reason: "first_turn_pending" }); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, seeded.runId)); + expect(run!.status).toBe("running"); + expect(run!.errorCode).toBeNull(); + const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, seeded.wakeId)); + expect(wake!.status).toBe("deferred_issue_execution"); + expect(wake!.payload?.queuedCommentInterrupt).toBeUndefined(); + + // Preparation finishes and the runner seals its first turn's input. + if (runtime === "preparing") { + await db.update(heartbeatRuns).set({ runtimeMode: "native", runtimeModeResolvedAt: new Date(), + nativeIssueId: seeded.issueId }).where(eq(heartbeatRuns.id, seeded.runId)); + } + const firstTurn = await claimFirstTurnQueuedInput(db, { companyId: seeded.companyId, runId: seeded.runId, + issueId: seeded.issueId, agentId: seeded.agentId }); + expect(firstTurn).toContain("First queued message"); + expect(firstTurn!.indexOf("First queued message")).toBeLessThan(firstTurn!.indexOf("Second queued message")); + // The taken messages leave the visible queue; nothing else can claim them. + const after = await request(client).get(`/api/issues/${seeded.issueId}/queued-comments`).expect(200); + expect(after.body.entries).toEqual([]); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, seeded.companyId))) + .filter(row => row.status === "queued")).toEqual([]); + }, + ); + + it("still interrupts a preparing run when its queue holds another user's message", async () => { + const seeded = await seedFirstTurnQueue("native"); + // Ordinary comments from two users merge into one wake that keeps the + // first author as its requester; the first turn would take only that user's. + await db.update(issueComments).set({ authorUserId: "other-operator" }).where(eq(issueComments.id, seeded.commentIds[1]!)); + const client = app(seeded.companyId); + const queue = await request(client).get(`/api/issues/${seeded.issueId}/queued-comments`).expect(200); + const response = await request(client).post(`/api/issues/${seeded.issueId}/queued-comments/interrupt`).send({ + queueId: seeded.wakeId, targetRunId: seeded.runId, revision: queue.body.revision, + }); + expect(response.status, JSON.stringify(response.body)).toBe(200); + expect(response.body.executionWait?.reason).not.toBe("first_turn_pending"); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, seeded.runId)); + expect(run!.status).toBe("cancelled"); + expect(run!.errorCode).toBe("operator_interrupted"); + }); + + it.each(["accepted turn", "sealed first turn"] as const)( + "still interrupts a run whose %s already started", + async (state) => { + const seeded = await seedFirstTurnQueue("native"); + if (state === "accepted turn") { + // A run saved by the previous release carries no seal; its accepted turn decides. + await appendHeartbeatRunEvent(db, { companyId: seeded.companyId, runId: seeded.runId, agentId: seeded.agentId, + eventType: "turn.accepted", stream: "system", level: "info", message: "turn accepted" }); + } else { + await db.update(heartbeatRuns).set({ contextSnapshot: { issueId: seeded.issueId, firstTurnQueuedInput: { + schema: "paperclip.first-turn-queued-input.v1", runId: seeded.runId, wakeupRequestIds: [], commentIds: [], + claimedAt: new Date().toISOString() } } }).where(eq(heartbeatRuns.id, seeded.runId)); + } + const client = app(seeded.companyId); + const queue = await request(client).get(`/api/issues/${seeded.issueId}/queued-comments`).expect(200); + expect(queue.body.entries).toHaveLength(2); + const response = await request(client).post(`/api/issues/${seeded.issueId}/queued-comments/interrupt`).send({ + queueId: seeded.wakeId, targetRunId: seeded.runId, revision: queue.body.revision, + }); + expect(response.status, JSON.stringify(response.body)).toBe(200); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, seeded.runId)); + expect(run!.status).toBe("cancelled"); + expect(run!.errorCode).toBe("operator_interrupted"); + }, + ); + it("does not accept interruption authority from an agent wake payload", async () => { const seeded = await seedQueue(); await db.update(agents).set({ adapterType: "claude_local", diff --git a/server/src/modules/wake-queue/adapters/queued-comment-postgres.ts b/server/src/modules/wake-queue/adapters/queued-comment-postgres.ts index 4ab29038938..8b4d6ebbd0c 100644 --- a/server/src/modules/wake-queue/adapters/queued-comment-postgres.ts +++ b/server/src/modules/wake-queue/adapters/queued-comment-postgres.ts @@ -21,6 +21,7 @@ import { } from "../../../services/issue-queued-comment-queue.js"; import { logActivity as persistActivityLogRow, type ActivityPublication } from "../../../services/activity-log.js"; import { decideQueuedCommentWakeLookup } from "../domain/policy.js"; +import { firstTurnClaimedCommentIds } from "../../../services/native-runtime/first-turn-queued-input.js"; import { parseObject, readNonEmptyString } from "../domain/values.js"; import { QueuedCommentMutationError } from "../application/queued-comment-use-cases.js"; import type { @@ -56,6 +57,10 @@ export type QueuedCommentQueuePostgresAdapterDeps = { }; function buildTransaction(tx: Db, companyId: string, issueId: string, deps: QueuedCommentQueuePostgresAdapterDeps, wakeAgentId: string): QueuedCommentQueueTransaction { + // Messages the active run took into its first turn leave the visible queue + // but stay on the wake until that turn is accepted, so a run that fails + // before then still leaves them queued. Queue edits keep them. + let heldByFirstTurn: string[] = []; return { async updateCommentBody({ issueId, commentId, body, updatedAt }) { await notifyStatusCardIssueChange(tx, companyId); @@ -74,9 +79,10 @@ function buildTransaction(tx: Db, companyId: string, issueId: string, deps: Queu }, async updateWakeQueuedCommentIds({ wakeId, payload, ids, updatedAt }) { + const kept = [...heldByFirstTurn.filter((id) => !ids.includes(id)), ...ids]; const row = await tx .update(agentWakeupRequests) - .set({ payload: withQueuedCommentIdsInWakePayload(payload, ids), updatedAt }) + .set({ payload: withQueuedCommentIdsInWakePayload(payload, kept), updatedAt }) .where(and(eq(agentWakeupRequests.id, wakeId), eq(agentWakeupRequests.companyId, companyId))) .returning() .then((rows) => rows[0]); @@ -105,6 +111,16 @@ function buildTransaction(tx: Db, companyId: string, issueId: string, deps: Queu }, async cancelWake({ wakeId, reason, now }) { + if (heldByFirstTurn.length > 0) { + const [held] = await tx.select({ payload: agentWakeupRequests.payload }).from(agentWakeupRequests) + .where(and(eq(agentWakeupRequests.id, wakeId), eq(agentWakeupRequests.companyId, companyId))); + if (held) { + await tx.update(agentWakeupRequests) + .set({ payload: withQueuedCommentIdsInWakePayload(held.payload, heldByFirstTurn), updatedAt: now }) + .where(and(eq(agentWakeupRequests.id, wakeId), eq(agentWakeupRequests.companyId, companyId))); + return; + } + } await notifyStrandedAgentStatus(tx, { companyId, agentId: wakeAgentId }); await notifyChatInboundWakeNotices(tx, { companyId, issueId }); await tx @@ -139,7 +155,10 @@ function buildTransaction(tx: Db, companyId: string, issueId: string, deps: Queu }, async buildQueueSnapshot({ issue, actor, wake, state, queueRun, activeRun }): Promise { - const commentIds = queuedCommentIdsFromWakePayload(wake?.payload ?? null); + const wakeCommentIds = queuedCommentIdsFromWakePayload(wake?.payload ?? null); + const claimed = state === "deferred" ? firstTurnClaimedCommentIds(activeRun) : new Set(); + if (wake) heldByFirstTurn = wakeCommentIds.filter((id) => claimed.has(id)); + const commentIds = wakeCommentIds.filter((id) => !claimed.has(id)); const rows = commentIds.length > 0 ? await tx diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.ts index d87ca4fb3a7..7c191590925 100644 --- a/server/src/routes/issues.ts +++ b/server/src/routes/issues.ts @@ -44,6 +44,7 @@ import { confirmNativeControllerStopped, NATIVE_CONTROLLER_HOST_UNVERIFIED } fro import { confirmOwnerGone, OWNER_UNPROVABLE_CAUSE } from "../services/recovery/owner-gone-attestation.js"; import { OWNABLE_KINDS } from "../services/recovery/ownership-registry.js"; import { retryNativeWorkspaceExport } from "../services/native-runtime/native-workspace-export-retry.js"; +import { deferredWakeJoinsFirstTurn, firstTurnClaimedCommentIds, firstTurnTakesAllQueuedComments, runAcceptsFirstTurnQueuedInput } from "../services/native-runtime/first-turn-queued-input.js"; import { reconstructNativeWorkspace, unreconciledNativeExportPredicate } from "../services/native-runtime/native-workspace-reconstruction.js"; import { reconstructWorkspaceSchema } from "@paperclipai/shared"; import { queuedInteractionId, readQueuedInteractionResponse, hasQueuedInteractionResponse, type QueuedInteractionResponse } from "../services/queued-interaction-response.js"; @@ -7637,12 +7638,14 @@ export function issueRoutes( ? queueState.response : await readQueuedInteractionResponse(input.executor as Db, input.issue.companyId, input.issue.id, wake.payload) : null; - const comments = await queueCommentsForWake( + // Messages the active run took into its first turn are no longer queued. + const claimedByFirstTurn = queueState?.state === "deferred" ? firstTurnClaimedCommentIds(input.activeRun) : new Set(); + const comments = (await queueCommentsForWake( input.executor, input.issue.id, wake, response, - ); + )).filter((comment) => !claimedByFirstTurn.has(comment.id)); const assignedAgent = input.issue.assigneeAgentId ? await input.executor .select({ adapterType: agents.adapterType }) @@ -17266,7 +17269,7 @@ export function issueRoutes( } } const actor = getActorInfo(req); - const runToInterrupt = await db.transaction(async (tx) => { + const interruptPlan = await db.transaction(async (tx): Promise<{ firstTurnRunId: string } | { runToInterrupt: string | null }> => { const locked = await lockQueuedCommentState({ tx, issue, actor, queueId: req.body.queueId, targetRunId: req.body.targetRunId ?? undefined, allowStoppedTarget: true, @@ -17283,6 +17286,16 @@ export function issueRoutes( if (locked.activeRun?.status === "running" && locked.activeRun.id !== req.body.targetRunId) { throw conflict("The queued message targets a stale run", { code: "queued_comment_stale_target" }); } + // A run that has not started its first provider turn takes these + // messages into that turn. Stopping it would only discard its setup. + if (locked.activeRun && !issue.conversationAgentId && issue.executionRunId === locked.activeRun.id && + deferredWakeJoinsFirstTurn(locked.wake, locked.activeRun) && + await firstTurnTakesAllQueuedComments(tx, { companyId: issue.companyId, issueId: issue.id, + responsibleUserId: locked.activeRun.responsibleUserId, + commentIds: queuedCommentIdsFromWakePayload(locked.wake.payload) }) && + await runAcceptsFirstTurnQueuedInput(tx, locked.activeRun)) { + return { firstTurnRunId: locked.activeRun.id }; + } // The click is durable fresh user intent, including when the message // predates a failed run's stop. Keep its content and original attribution. await notifyDeferredExecutionWork(tx, { companyId: issue.companyId, requestId: locked.wake.id }); @@ -17294,8 +17307,10 @@ export function issueRoutes( } }, updatedAt: new Date(), }).where(eq(agentWakeupRequests.id, locked.wake.id)); - return locked.activeRun?.status === "running" ? locked.activeRun.id : null; + return { runToInterrupt: locked.activeRun?.status === "running" ? locked.activeRun.id : null }; }); + const firstTurnRunId = "firstTurnRunId" in interruptPlan ? interruptPlan.firstTurnRunId : null; + const runToInterrupt = "runToInterrupt" in interruptPlan ? interruptPlan.runToInterrupt : null; // Never hold the issue lock while joining the adapter. Queue edits and // discards stay authoritative until the dispatcher claims the successor. const options = operatorInterruptCancelOptions({ issueId: issue.id, actor }); @@ -17304,18 +17319,23 @@ export function issueRoutes( suppressImmediateRecovery: true, resultJson: { ...options.resultJson, queuedCommentInterruptQueueId: req.body.queueId }, }); - await heartbeat.resumeQueuedCommentInterrupt(issue.companyId, req.body.queueId, { retryCleanup: true }); + if (!firstTurnRunId) await heartbeat.resumeQueuedCommentInterrupt(issue.companyId, req.body.queueId, { retryCleanup: true }); await logActivity(db, { companyId: issue.companyId, actorType: actor.actorType, actorId: actor.actorId, agentId: actor.agentId, runId: actor.runId, agentApiKeyId: actor.agentApiKeyId, action: "issue.queued_comments_interrupted", entityType: "issue", entityId: issue.id, - details: { queueId: req.body.queueId, targetRunId: req.body.targetRunId }, + details: { queueId: req.body.queueId, targetRunId: req.body.targetRunId, + ...(firstTurnRunId ? { deliveredWithFirstTurnOfRunId: firstTurnRunId } : {}) }, }); const currentIssue = await svc.getById(issue.id); const queue = await buildQueuedCommentQueue({ executor: db, issue: currentIssue ?? issue, activeRun: await resolveActiveIssueRun(currentIssue ?? issue), actor, }); + if (firstTurnRunId && queue.state === "deferred" && queue.entries.length > 0) { + queue.executionWait = { reason: "first_turn_pending", + message: "The agent is still starting. These messages will be included in its first turn." }; + } res.json(await runRedactions.redactForIssue(issue.companyId, issue.id, queue)); }, ); diff --git a/server/src/services/explicit-native-continuation.ts b/server/src/services/explicit-native-continuation.ts index c42208d44fe..7ea79bd92b0 100644 --- a/server/src/services/explicit-native-continuation.ts +++ b/server/src/services/explicit-native-continuation.ts @@ -34,6 +34,7 @@ import { activeIssueInteractionCondition } from "./issue-question-context.js"; import { historicalAdapterType, isConversationAdapter } from "./conversation-continuation.js"; import { queuedCommentIdsFromWakePayload } from "./issue-queued-comment-queue.js"; +import { firstTurnDeliveredCommentIds } from "./queued-comment-delivery.js"; type Run = Pick valid.has(id)); + // An accepted first turn that took a queued message consumed it too. + const firstTurn = await firstTurnDeliveredCommentIds(db, { companyId, issueId, agentId, commentIds: [...valid] }); + return commentIds.filter(id => valid.has(id) && !firstTurn.has(id)); } /** Called under the issue lock, in the transaction that creates the new turn. diff --git a/server/src/services/heartbeat/run-state.ts b/server/src/services/heartbeat/run-state.ts index 4584ea7fd8a..3150bff2e5e 100644 --- a/server/src/services/heartbeat/run-state.ts +++ b/server/src/services/heartbeat/run-state.ts @@ -372,7 +372,10 @@ const heartbeatRunIssueControlColumns = { runtimeModeResolvedAt: heartbeatRuns.runtimeModeResolvedAt, startedAt: heartbeatRuns.startedAt, createdAt: heartbeatRuns.createdAt, contextSnapshot: sql | null>`case when jsonb_typeof(${heartbeatRuns.contextSnapshot}) = 'object' - then jsonb_build_object('issueId', ${heartbeatRuns.contextSnapshot}->'issueId') else null end`, + then jsonb_build_object('issueId', ${heartbeatRuns.contextSnapshot}->'issueId') + || case when ${heartbeatRuns.contextSnapshot} ? 'firstTurnQueuedInput' + then jsonb_build_object('firstTurnQueuedInput', ${heartbeatRuns.contextSnapshot}->'firstTurnQueuedInput') else '{}'::jsonb end + else null end`, runnerProfileJson: sql | null>`case when jsonb_typeof(${heartbeatRuns.runnerProfileJson}) = 'object' then jsonb_build_object('adapterDispatch', jsonb_build_object('adapterType', ${heartbeatRuns.runnerProfileJson}->'adapterDispatch'->'adapterType')) else null end`, diff --git a/server/src/services/native-runtime/first-turn-queued-input.test.ts b/server/src/services/native-runtime/first-turn-queued-input.test.ts new file mode 100644 index 00000000000..b86b869b85c --- /dev/null +++ b/server/src/services/native-runtime/first-turn-queued-input.test.ts @@ -0,0 +1,189 @@ +import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { + agents, agentWakeupRequests, assets, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issueAttachments, + issueComments, issues, +} from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; +import { appendHeartbeatRunEvent } from "../heartbeat-run-events.js"; +import { undeliveredLegacyUserCommentIds } from "../explicit-native-continuation.js"; +import { deliveredQueuedCommentIds } from "../queued-comment-delivery.js"; +import { + claimFirstTurnQueuedInput, firstTurnClaimedCommentIds, firstTurnTakesAllQueuedComments, runAcceptsFirstTurnQueuedInput, +} from "./first-turn-queued-input.js"; + +const support = await getEmbeddedPostgresTestSupport(); +const USER = "first-turn-user"; + +describe.skipIf(!support.supported)("first-turn queued input", () => { + let database: Awaited>; + let db: ReturnType; + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("paperclip-first-turn-queued-input-"); + db = createDb(database.connectionString); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + afterAll(async () => { await db?.$client.end(); await database?.cleanup(); }); + + async function seed() { + const [company] = await db.insert(companies).values({ name: "First turn", issuePrefix: `F${randomUUID().slice(0, 4)}` }).returning(); + const [agent] = await db.insert(agents).values({ companyId: company!.id, name: "Codie", adapterType: "paperclip_runner" }).returning(); + const issueId = randomUUID(); + const runId = await startRun(company!.id, agent!.id, issueId); + await db.insert(issues).values({ id: issueId, companyId: company!.id, title: "Fix the reports", status: "in_progress", + assigneeAgentId: agent!.id, executionRunId: runId }); + return { companyId: company!.id, agentId: agent!.id, issueId, runId }; + } + + async function startRun(companyId: string, agentId: string, issueId: string) { + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "running", runtimeMode: "native", + runtimeModeResolvedAt: new Date(), nativeIssueId: issueId, responsibleUserId: USER, startedAt: new Date(), + contextSnapshot: { issueId } }).returning(); + return run!.id; + } + + /** A user's comment saved behind the active run, in the shape the previous release writes. */ + async function queueMessage(f: Awaited>, body: string) { + const [comment] = await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId, + authorType: "user", authorUserId: USER, body }).returning(); + const [existing] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.status, "deferred_issue_execution")) + .then(rows => rows.filter(row => row.companyId === f.companyId)); + const ids = [...((existing?.payload?._paperclipWakeContext as { wakeCommentIds?: string[] } | undefined)?.wakeCommentIds ?? []), comment!.id]; + const payload = { issueId: f.issueId, commentId: comment!.id, _paperclipWakeContext: { issueId: f.issueId, taskId: f.issueId, + wakeReason: "issue_commented", source: "issue.comment", wakeCommentId: comment!.id, wakeCommentIds: ids } }; + if (existing) await db.update(agentWakeupRequests).set({ payload }).where(eq(agentWakeupRequests.id, existing.id)); + else await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agentId, source: "automation", + triggerDetail: "system", reason: "issue_execution_deferred", status: "deferred_issue_execution", + requestedByActorType: "user", requestedByActorId: USER, payload }); + return comment!.id; + } + + const binding = (f: { companyId: string; issueId: string; agentId: string }, runId: string) => + ({ companyId: f.companyId, issueId: f.issueId, agentId: f.agentId, runId }); + const accept = (f: { companyId: string; agentId: string }, runId: string) => appendHeartbeatRunEvent(db, { + companyId: f.companyId, runId, agentId: f.agentId, eventType: "turn.accepted", stream: "system", level: "info", message: "turn accepted" }); + const readRun = async (id: string) => (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)))[0]!; + + it("gives a message that arrived during preparation to the first turn, without a stop", async () => { + const f = await seed(); + const message = await queueMessage(f, "oh i mean each of the ones that appeared within the last 24 hours"); + expect(await runAcceptsFirstTurnQueuedInput(db, await readRun(f.runId))).toBe(true); + const startedAt = performance.now(); + const firstTurn = await claimFirstTurnQueuedInput(db, binding(f, f.runId)); + const sealMs = performance.now() - startedAt; + console.info(`first-turn queued input seal took ${sealMs.toFixed(1)}ms before turn.start`); + expect(firstTurn).toContain("oh i mean each of the ones that appeared within the last 24 hours"); + expect(firstTurn).toContain(`comment ${message}`); + const run = await readRun(f.runId); + expect(run.status).toBe("running"); + expect([...firstTurnClaimedCommentIds(run)]).toEqual([message]); + // Sealed: a later message waits for the turn boundary and Interrupt applies. + expect(await runAcceptsFirstTurnQueuedInput(db, run)).toBe(false); + const later = await queueMessage(f, "one more thing"); + expect(await claimFirstTurnQueuedInput(db, binding(f, f.runId))).not.toContain("one more thing"); + expect(firstTurnClaimedCommentIds(await readRun(f.runId)).has(later)).toBe(false); + expect(sealMs).toBeLessThan(1_000); + }); + + it("delivers two messages from preparation in their queued order", async () => { + const f = await seed(); + await queueMessage(f, "FIRST follow-up"); + await queueMessage(f, "SECOND follow-up"); + const firstTurn = await claimFirstTurnQueuedInput(db, binding(f, f.runId)); + expect(firstTurn).toMatch(/Message 1 of 2[^\n]*\nFIRST follow-up[\s\S]*Message 2 of 2[^\n]*\nSECOND follow-up/); + }); + + it("keeps the message queued when the run fails before turn.start, then delivers it exactly once", async () => { + const f = await seed(); + const message = await queueMessage(f, "please include the staging run"); + expect(await claimFirstTurnQueuedInput(db, binding(f, f.runId))).toContain("please include the staging run"); + // A restart before the provider accepted the turn repeats the same input. + expect(await claimFirstTurnQueuedInput(db, binding(f, f.runId))).toContain("please include the staging run"); + await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.runId)); + // Not consumed: release promotion and Send both still deliver it. + const ids = { companyId: f.companyId, issueId: f.issueId, agentId: f.agentId, commentIds: [message] }; + expect([...await deliveredQueuedCommentIds(db, ids)]).toEqual([]); + expect(await undeliveredLegacyUserCommentIds(db, f.companyId, f.issueId, f.agentId, [message])).toEqual([message]); + const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId)); + expect(wake!.status).toBe("deferred_issue_execution"); + + // The next run takes it into its own first turn; the provider accepts. + const nextRunId = await startRun(f.companyId, f.agentId, f.issueId); + await db.update(issues).set({ executionRunId: nextRunId }).where(eq(issues.id, f.issueId)); + expect(await claimFirstTurnQueuedInput(db, binding(f, nextRunId))).toContain("please include the staging run"); + await accept(f, nextRunId); + expect([...await deliveredQueuedCommentIds(db, ids)]).toEqual([message]); + expect(await undeliveredLegacyUserCommentIds(db, f.companyId, f.issueId, f.agentId, [message])).toEqual([]); + + // Exactly once: a later run, and a retry that copied the context, take nothing. + await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, nextRunId)); + const thirdRunId = await startRun(f.companyId, f.agentId, f.issueId); + await db.update(heartbeatRuns).set({ contextSnapshot: (await readRun(nextRunId)).contextSnapshot }) + .where(eq(heartbeatRuns.id, thirdRunId)); + await db.update(issues).set({ executionRunId: thirdRunId }).where(eq(issues.id, f.issueId)); + expect(await claimFirstTurnQueuedInput(db, binding(f, thirdRunId))).toBeNull(); + }); + + it("counts a message as delivered once the provider accepted the turn, even if the run later fails", async () => { + const f = await seed(); + const message = await queueMessage(f, "ship it"); + await claimFirstTurnQueuedInput(db, binding(f, f.runId)); + await accept(f, f.runId); + await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.runId)); + expect([...await deliveredQueuedCommentIds(db, { companyId: f.companyId, issueId: f.issueId, agentId: f.agentId, + commentIds: [message] })]).toEqual([message]); + }); + + it("treats a run saved by the previous release as started once its provider accepted a turn", async () => { + const f = await seed(); + await queueMessage(f, "mid-turn message"); + await accept(f, f.runId); + expect(await runAcceptsFirstTurnQueuedInput(db, await readRun(f.runId))).toBe(false); + }); + + it("leaves another user's message and a run without a responsible user to their own delivery", async () => { + const f = await seed(); + await queueMessage(f, "mine"); + await db.update(agentWakeupRequests).set({ requestedByActorId: "someone-else" }).where(eq(agentWakeupRequests.companyId, f.companyId)); + expect(await claimFirstTurnQueuedInput(db, binding(f, f.runId))).toBeNull(); + }); + it("seals on a one-connection pool without waiting for a second connection", async () => { + const f = await seed(); + await queueMessage(f, "secret-free follow-up"); + const single = createDb(database.connectionString, { maxConnections: 1 }); + try { + const sealed = await Promise.race([ + claimFirstTurnQueuedInput(single, binding(f, f.runId)), + new Promise<"stuck">((resolve) => setTimeout(() => resolve("stuck"), 5_000)), + ]); + expect(sealed).toContain("secret-free follow-up"); + } finally { + await single.$client.end({ timeout: 1 }); + } + }); + + it("leaves a message with files on its own delivery path so the files reach the agent", async () => { + const f = await seed(); + await queueMessage(f, "text only"); + const withFile = await queueMessage(f, "see the attached screenshot"); + const [asset] = await db.insert(assets).values({ companyId: f.companyId, provider: "local_disk", + objectKey: `${f.companyId}/screenshot`, originalFilename: "screenshot.png", contentType: "image/png", + byteSize: 1, sha256: "0".repeat(64) }).returning(); + await db.insert(issueAttachments).values({ companyId: f.companyId, issueId: f.issueId, assetId: asset!.id, + issueCommentId: withFile }); + const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId)); + const queuedIds = (wake!.payload?._paperclipWakeContext as { wakeCommentIds: string[] }).wakeCommentIds; + // Interrupt keeps its meaning for this queue: the first turn cannot take all of it. + expect(await firstTurnTakesAllQueuedComments(db, { companyId: f.companyId, issueId: f.issueId, responsibleUserId: USER, + commentIds: queuedIds })).toBe(false); + const firstTurn = await claimFirstTurnQueuedInput(db, binding(f, f.runId)); + expect(firstTurn).toContain("text only"); + expect(firstTurn).not.toContain("see the attached screenshot"); + const run = await readRun(f.runId); + expect(firstTurnClaimedCommentIds(run).has(withFile)).toBe(false); + await accept(f, f.runId); + // Still owed after the turn: release promotion delivers it with its file. + expect([...await deliveredQueuedCommentIds(db, { companyId: f.companyId, issueId: f.issueId, agentId: f.agentId, + commentIds: [withFile] })]).toEqual([]); + }); +}); diff --git a/server/src/services/native-runtime/first-turn-queued-input.ts b/server/src/services/native-runtime/first-turn-queued-input.ts new file mode 100644 index 00000000000..216b173c276 --- /dev/null +++ b/server/src/services/native-runtime/first-turn-queued-input.ts @@ -0,0 +1,269 @@ +import { and, asc, eq, inArray, isNull, sql } from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; +import { agentWakeupRequests, heartbeatRunEvents, heartbeatRuns, issueAttachments, issueComments, issues } from "@paperclipai/db"; +import type { SourceTrustMetadata } from "@paperclipai/shared"; + +import { queuedCommentIdsFromWakePayload } from "../issue-queued-comment-queue.js"; +import { deliveredQueuedCommentIds } from "../queued-comment-delivery.js"; +import { createRunSecretRedactionRegistry } from "../run-secret-redaction.js"; +import { sanitizeQuarantinedCommentForHigherTrust } from "../source-trust.js"; +import { PROVIDER_TURN_ACCEPTED_EVENT_TYPES } from "./provider-attempt-outcome.js"; + +/** + * Queued input for a run's first provider turn. + * + * A user's message that arrives while a run is still preparing waits as a + * deferred wake behind that run. Interrupting the run for it throws away the + * whole preparation. Instead, the run takes the waiting messages into the + * turn it is about to start: the runner calls `claimFirstTurnQueuedInput` + * immediately before `turn.start`, and the claim and the first-turn input are + * sealed in one transaction on the run row. + * + * The claim does not remove the messages from their deferred wake. It records + * a receipt in the run's context. A message counts as delivered only after the + * provider accepted that run's turn (`deliveredQueuedCommentIds`). If the run + * ends before that, the receipt proves nothing and the deferred wake still + * holds the message, so the next run delivers it exactly once. A restart + * before the turn was accepted repeats the same claim from the receipt. + */ +export const FIRST_TURN_QUEUED_INPUT_KEY = "firstTurnQueuedInput"; +export const FIRST_TURN_QUEUED_INPUT_SCHEMA = "paperclip.first-turn-queued-input.v1"; + +const DEFERRED_WAKE_STATUS = "deferred_issue_execution"; +const DEFERRED_WAKE_CONTEXT_KEY = "_paperclipWakeContext"; +const COMMENT_WAKE_REASONS = new Set(["issue_commented", "issue_reopened_via_comment", "issue_comment_mentioned"]); +const MAX_COMMENT_CHARS = 16_000; +const MAX_TOTAL_CHARS = 48_000; +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +export type FirstTurnQueuedInputReceipt = { + schema: typeof FIRST_TURN_QUEUED_INPUT_SCHEMA; + runId: string; + wakeupRequestIds: string[]; + commentIds: string[]; + claimedAt: string; +}; + +function record(value: unknown): Record { + return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; +} + +function nonEmptyString(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 ? value : null; +} + +function stringIds(value: unknown): string[] { + return Array.isArray(value) ? [...new Set(value.filter((id): id is string => typeof id === "string" && UUID.test(id)))] : []; +} + +/** + * The receipt this exact run sealed. A retry copies its predecessor's context, + * so a receipt naming another run is inert. + */ +export function readFirstTurnQueuedInputReceipt(run: { id: string; contextSnapshot?: unknown }): FirstTurnQueuedInputReceipt | null { + const receipt = record(record(run.contextSnapshot)[FIRST_TURN_QUEUED_INPUT_KEY]); + if (receipt.schema !== FIRST_TURN_QUEUED_INPUT_SCHEMA || receipt.runId !== run.id) return null; + const claimedAt = nonEmptyString(receipt.claimedAt); + if (!claimedAt) return null; + return { + schema: FIRST_TURN_QUEUED_INPUT_SCHEMA, + runId: run.id, + wakeupRequestIds: stringIds(receipt.wakeupRequestIds), + commentIds: stringIds(receipt.commentIds), + claimedAt, + }; +} + +/** Message ids a live run already took into its first turn; queue views hide them. */ +export function firstTurnClaimedCommentIds(run: { id: string; status?: string | null; contextSnapshot?: unknown } | null | undefined): Set { + if (!run || (run.status !== undefined && run.status !== "running")) return new Set(); + return new Set(readFirstTurnQueuedInputReceipt(run)?.commentIds ?? []); +} + +/** + * Whether a deferred wake is plain user input that the run's first turn may + * take. Interaction answers, interrupt intents, independent continuations and + * chat-conversation wakes keep their own delivery contracts. Only the run's + * responsible user's own messages qualify, so taking them cannot change the + * run's execution authority. + */ +export function deferredWakeJoinsFirstTurn( + wake: { agentId: string; status: string; requestedByActorType: string | null; requestedByActorId: string | null; reason: string | null; payload: unknown }, + run: { agentId: string; responsibleUserId: string | null }, +): boolean { + const payload = record(wake.payload); + const context = record(payload[DEFERRED_WAKE_CONTEXT_KEY]); + const reason = nonEmptyString(context.wakeReason) ?? nonEmptyString(wake.reason); + return wake.status === DEFERRED_WAKE_STATUS && + wake.agentId === run.agentId && + wake.requestedByActorType === "user" && + Boolean(run.responsibleUserId) && wake.requestedByActorId === run.responsibleUserId && + reason !== null && COMMENT_WAKE_REASONS.has(reason) && + !payload.interactionId && !context.interactionId && !context.interactionStatus && + !payload.queuedCommentInterrupt && context.resumeIntent !== true && + !nonEmptyString(context.chatConversationId) && !nonEmptyString(payload.externalChatProvider) && + queuedCommentIdsFromWakePayload(payload).length > 0; +} + +type QueuedMessage = { id: string; body: string; createdAt: Date }; + +export function renderFirstTurnQueuedInput(messages: readonly QueuedMessage[]): string | null { + if (messages.length === 0) return null; + let remaining = MAX_TOTAL_CHARS; + const sections = messages.map((message, index) => { + const limit = Math.max(0, Math.min(MAX_COMMENT_CHARS, remaining)); + const truncated = message.body.length > limit; + const body = truncated ? message.body.slice(0, limit) : message.body; + remaining -= body.length; + return [ + `[Message ${index + 1} of ${messages.length} · comment ${message.id} · ${message.createdAt.toISOString()}]`, + body, + ...(truncated ? [`(Truncated here. Read comment ${message.id} on the task for the rest.)`] : []), + ].join("\n"); + }); + return [ + "New messages from the user arrived while this run was starting. They were not in the request above. Handle each of them in this turn, oldest first, together with that request:", + ...sections, + ].join("\n\n"); +} + +/** + * The one rule for which queued comment a first turn may take: a live, plain + * comment written directly by the run's responsible user, with no files. The + * prompt carries text only; a message with files keeps its own delivery, which + * stages those files for the turn that reads it. + */ +function firstTurnEligibleComment(input: { companyId: string; issueId: string; responsibleUserId: string }) { + return and( + eq(issueComments.companyId, input.companyId), eq(issueComments.issueId, input.issueId), + eq(issueComments.authorUserId, input.responsibleUserId), + isNull(issueComments.authorAgentId), isNull(issueComments.createdByRunId), isNull(issueComments.deletedAt), + sql`not exists (select 1 from ${issueAttachments} where ${issueAttachments.companyId} = ${input.companyId} + and ${issueAttachments.issueCommentId} = ${issueComments.id})`); +} + +async function readQueuedMessages( + tx: Db, + input: { companyId: string; issueId: string; agentId: string; responsibleUserId: string; commentIds: string[]; excludeDelivered: boolean }, +): Promise { + if (input.commentIds.length === 0) return []; + const rows = await tx.select({ + id: issueComments.id, body: issueComments.body, createdAt: issueComments.createdAt, + sourceTrust: issueComments.sourceTrust, + }).from(issueComments).where(and(inArray(issueComments.id, input.commentIds), firstTurnEligibleComment(input))); + const delivered = input.excludeDelivered + ? await deliveredQueuedCommentIds(tx, { companyId: input.companyId, issueId: input.issueId, agentId: input.agentId, + commentIds: rows.map(row => row.id) }) + : new Set(); + const byId = new Map(rows.map(row => [row.id, row])); + return input.commentIds.flatMap((id) => { + const row = byId.get(id); + if (!row || delivered.has(id)) return []; + const safe = sanitizeQuarantinedCommentForHigherTrust({ ...row, sourceTrust: row.sourceTrust as SourceTrustMetadata | null }); + return [{ id, body: safe.body, createdAt: row.createdAt }]; + }); +} + +/** + * Seals the first provider turn's input. Returns the queued messages to add to + * that turn, or null. Called once per turn start, immediately before + * `turn.start`; repeated calls for the same run return the same messages. + */ +export async function claimFirstTurnQueuedInput( + db: Db, + binding: { companyId: string; runId: string; issueId: string; agentId: string }, +): Promise { + return db.transaction(async (tx) => { + // Redact on this transaction's own connection: a second pool connection + // while the seal holds its locks can wait forever on a one-connection pool. + const render = async (messages: QueuedMessage[]) => messages.length === 0 ? null : renderFirstTurnQueuedInput( + await createRunSecretRedactionRegistry(tx as unknown as Db).redactForIssue(binding.companyId, binding.issueId, messages)); + // Lock in the queue routes' order (task, queue, run) so a concurrent + // Interrupt and this seal serialize instead of deadlocking. + const [issue] = await tx.select({ conversationAgentId: issues.conversationAgentId, executionRunId: issues.executionRunId }) + .from(issues).where(and(eq(issues.id, binding.issueId), eq(issues.companyId, binding.companyId))).for("update").limit(1); + const wakes = !issue ? [] : await tx.select().from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.companyId, binding.companyId), eq(agentWakeupRequests.agentId, binding.agentId), + eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS), + sql`${agentWakeupRequests.payload} ->> 'issueId' = ${binding.issueId}`, + )).orderBy(asc(agentWakeupRequests.requestedAt), asc(agentWakeupRequests.id)).for("update"); + const [run] = await tx.select({ + id: heartbeatRuns.id, agentId: heartbeatRuns.agentId, status: heartbeatRuns.status, + runtimeMode: heartbeatRuns.runtimeMode, responsibleUserId: heartbeatRuns.responsibleUserId, + contextSnapshot: heartbeatRuns.contextSnapshot, + }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.id, binding.runId), eq(heartbeatRuns.companyId, binding.companyId), + eq(heartbeatRuns.agentId, binding.agentId))).for("update").limit(1); + if (!run || run.status !== "running" || run.runtimeMode !== "native" || !run.responsibleUserId) return null; + const receipt = readFirstTurnQueuedInputReceipt(run); + // A restart before the provider accepted the turn repeats the sealed input. + if (receipt) { + return render(await readQueuedMessages(tx as unknown as Db, { ...binding, responsibleUserId: run.responsibleUserId, + commentIds: receipt.commentIds, excludeDelivered: false })); + } + // Only the run that holds the task takes its queue. Conversations keep + // their own turn-boundary contract. + const joining = !issue || issue.conversationAgentId || issue.executionRunId !== run.id + ? [] : wakes.filter(wake => deferredWakeJoinsFirstTurn(wake, run)); + const commentIds = [...new Set(joining.flatMap(wake => queuedCommentIdsFromWakePayload(wake.payload)))]; + const taken = await readQueuedMessages(tx as unknown as Db, { ...binding, responsibleUserId: run.responsibleUserId, + commentIds, excludeDelivered: true }); + // Render before sealing: once the receipt commits, nothing can fail + // between it and the turn input that the receipt describes. + const text = await render(taken); + // Seal even when nothing joined: later messages wait for the turn boundary. + const sealed: FirstTurnQueuedInputReceipt = { + schema: FIRST_TURN_QUEUED_INPUT_SCHEMA, + runId: run.id, + wakeupRequestIds: joining.filter(wake => queuedCommentIdsFromWakePayload(wake.payload) + .some(id => taken.some(message => message.id === id))).map(wake => wake.id), + commentIds: taken.map(message => message.id), + claimedAt: new Date().toISOString(), + }; + await tx.update(heartbeatRuns).set({ + contextSnapshot: sql`coalesce(${heartbeatRuns.contextSnapshot}, '{}'::jsonb) || ${JSON.stringify({ [FIRST_TURN_QUEUED_INPUT_KEY]: sealed })}::jsonb`, + updatedAt: new Date(), + }).where(eq(heartbeatRuns.id, run.id)); + return text; + }); +} + +/** + * True while a running native run can still take queued input into its first + * turn: it has not sealed that input and no provider turn was accepted. A run + * saved by a release without the seal counts as started once its provider + * accepted a turn, so a mid-turn Interrupt keeps its old meaning. + */ +export async function runAcceptsFirstTurnQueuedInput( + tx: Pick, + run: { id: string; companyId: string; status: string; runtimeMode: string | null; runtimeModeResolvedAt?: Date | null; + runnerProfileJson?: unknown; contextSnapshot?: unknown }, +): Promise { + if (run.status !== "running") return false; + const preparingNative = run.runtimeMode === "legacy" && !run.runtimeModeResolvedAt && + record(record(run.runnerProfileJson).adapterDispatch).adapterType === "paperclip_runner"; + if (run.runtimeMode !== "native" && !preparingNative) return false; + if (readFirstTurnQueuedInputReceipt(run)) return false; + const [accepted] = await tx.select({ id: heartbeatRunEvents.id }).from(heartbeatRunEvents).where(and( + eq(heartbeatRunEvents.companyId, run.companyId), eq(heartbeatRunEvents.runId, run.id), + inArray(heartbeatRunEvents.eventType, [...PROVIDER_TURN_ACCEPTED_EVENT_TYPES]))).limit(1); + return !accepted; +} + +/** + * True when the first turn would take every pending message in this queue, + * under the same rule as the claim. Another user's message, an agent's, or one + * with files keeps its own delivery, so Interrupt must still apply. + */ +export async function firstTurnTakesAllQueuedComments( + tx: Pick, + input: { companyId: string; issueId: string; responsibleUserId: string | null; commentIds: string[] }, +): Promise { + if (input.commentIds.length === 0 || !input.responsibleUserId) return false; + const rows = await tx.select({ + id: issueComments.id, + eligible: sql`coalesce((${firstTurnEligibleComment({ ...input, responsibleUserId: input.responsibleUserId })}), false)`, + }).from(issueComments).where(and(eq(issueComments.companyId, input.companyId), eq(issueComments.issueId, input.issueId), + inArray(issueComments.id, input.commentIds), isNull(issueComments.deletedAt))); + return rows.length > 0 && rows.every(row => row.eligible === true); +} diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index c2d683a6cbe..a7b8873c3a0 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -253,6 +253,7 @@ import { NATIVE_CURRENT_WAKE_COMMENTS_UNREAD, resolveCurrentWakeCommentsBinding, } from "./current-wake-comments.js"; +import { claimFirstTurnQueuedInput } from "./first-turn-queued-input.js"; import { renderNativeRunnerStagedAttachmentPrompt, stageNativeRunnerWakeAttachments, @@ -11062,6 +11063,12 @@ async function executePaperclipNativeSessionWithinScope( executeNativeSession({ resumeInterruptedTurn: input.restartRecovery?.kind === "resume_dead_runner", getFreshSessionHandoff: input.getFreshSessionHandoff, + // A failed seal commits nothing: the messages stay queued for the + // turn boundary instead of failing this run before its turn. + getFirstTurnQueuedInput: () => claimFirstTurnQueuedInput(input.db, input.execution.binding).catch((error) => { + logger.warn({ err: error, runId: input.execution.binding.runId }, "first-turn queued input was not sealed"); + return null; + }), getUnreadSteeredMessages: () => readUnreadSteeredMessages(input.db, { companyId: input.execution.binding.companyId, runId: input.execution.binding.runId, diff --git a/server/src/services/queued-comment-delivery.ts b/server/src/services/queued-comment-delivery.ts index 557a8b0b283..6ef16eab4fd 100644 --- a/server/src/services/queued-comment-delivery.ts +++ b/server/src/services/queued-comment-delivery.ts @@ -1,5 +1,5 @@ import { and, eq, inArray, sql } from "drizzle-orm"; -import { heartbeatRuns, issueComments, type Db } from "@paperclipai/db"; +import { heartbeatRunEvents, heartbeatRuns, issueComments, type Db } from "@paperclipai/db"; /** * Per-comment consumption (work-intent outbox, invariant 4). A succeeded run @@ -11,6 +11,11 @@ import { heartbeatRuns, issueComments, type Db } from "@paperclipai/db"; * counts: its context can be refreshed after the fact, and an edit made after * it started is still unread. Every path that replays a saved message reads * this before delivering it. + * + * A run that took the message into its first provider turn + * (`firstTurnQueuedInput`, sealed by that run before `turn.start`) consumed + * it once the provider accepted that turn, whatever the run's later outcome. + * Before acceptance the receipt proves nothing and the message stays queued. */ export async function deliveredQueuedCommentIds(tx: Pick, input: { companyId: string; issueId: string; agentId: string; commentIds: string[]; @@ -20,7 +25,7 @@ export async function deliveredQueuedCommentIds(tx: Pick, input: { const rows = await tx.select({ id: issueComments.id }).from(issueComments).where(and( eq(issueComments.companyId, companyId), eq(issueComments.issueId, issueId), inArray(issueComments.id, input.commentIds), - sql`exists (select 1 from ${heartbeatRuns} delivered_run + sql`(exists (select 1 from ${heartbeatRuns} delivered_run where delivered_run.company_id = ${companyId} and delivered_run.agent_id = ${input.agentId} and delivered_run.status = 'succeeded' and (delivered_run.native_issue_id = ${issueId} or delivered_run.context_snapshot->>'issueId' = ${issueId}) @@ -30,7 +35,36 @@ export async function deliveredQueuedCommentIds(tx: Pick, input: { or (delivered_run.retry_of_run_id is not null and jsonb_path_exists(delivered_run.context_snapshot, '$.executionContinuation.messages[*] ? (@.id == $id)', jsonb_build_object('id', ${issueComments.id}::text)) or jsonb_path_exists(delivered_run.context_snapshot, '$.paperclipWake.messages[*] ? (@.id == $id)', - jsonb_build_object('id', ${issueComments.id}::text)))))`, + jsonb_build_object('id', ${issueComments.id}::text))))) + or ${firstTurnDeliveredSql(companyId, issueId, input.agentId)})`, )); return new Set(rows.map(row => row.id)); } + +/** + * The comment row was taken into an accepted first provider turn of a run of + * this agent on this task, and not edited after that run sealed its input. + */ +function firstTurnDeliveredSql(companyId: string, issueId: string, agentId: string) { + return sql`exists (select 1 from ${heartbeatRuns} first_turn_run + where first_turn_run.company_id = ${companyId} and first_turn_run.agent_id = ${agentId} + and (first_turn_run.native_issue_id = ${issueId} or first_turn_run.context_snapshot->>'issueId' = ${issueId}) + and first_turn_run.context_snapshot->'firstTurnQueuedInput'->>'schema' = 'paperclip.first-turn-queued-input.v1' + and first_turn_run.context_snapshot->'firstTurnQueuedInput'->>'runId' = first_turn_run.id::text + and first_turn_run.context_snapshot->'firstTurnQueuedInput'->'commentIds' ? ${issueComments.id}::text + and (first_turn_run.context_snapshot->'firstTurnQueuedInput'->>'claimedAt')::timestamptz >= date_trunc('milliseconds', ${issueComments.updatedAt}) + and exists (select 1 from ${heartbeatRunEvents} accepted + where accepted.company_id = ${companyId} and accepted.run_id = first_turn_run.id + and accepted.event_type in ('turn.accepted', 'turn.started')))`; +} + +/** Ids that an accepted first provider turn already received (see `deliveredQueuedCommentIds`). */ +export async function firstTurnDeliveredCommentIds(tx: Pick, input: { + companyId: string; issueId: string; agentId: string; commentIds: string[]; +}): Promise> { + if (input.commentIds.length === 0) return new Set(); + const rows = await tx.select({ id: issueComments.id }).from(issueComments).where(and( + eq(issueComments.companyId, input.companyId), eq(issueComments.issueId, input.issueId), + inArray(issueComments.id, input.commentIds), firstTurnDeliveredSql(input.companyId, input.issueId, input.agentId))); + return new Set(rows.map(row => row.id)); +} From 61fa13c6d8881e837699974a98e27cbbab986f7a Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sun, 8 Nov 2026 22:52:48 -0600 Subject: [PATCH 4/9] test(chaos): add S1, a board Stop across a restart then a continue comment (#20161) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The restart chaos harness drives real user actions across control-plane restarts: comments, Retry, approvals and budget resumes > - The board's Stop button on a running task was not exercised. The cancel route stamps the stop as operator-initiated so that recovery stands down instead of re-waking the agent > - A restart right after a Stop is where recovery could wrongly restart work the operator stopped, or lose the operator's later "continue" direction > - This pull request adds S1: stop the running run, fire a fault right after, then comment to continue > - The benefit is a graded check that a Stop holds across a restart and that a later continue comment finishes the task ## Linked Issues or Issue Description Refs #20071 **What is missing?** The campaign's scenario list includes mid-run stops. C1 covers a mid-run comment (a steer), but no workload clicks Stop. Without one, a regression where recovery restarts operator-stopped work, or drops the continue comment, would go unseen. **Expected behavior** - After the board stops a run and a restart follows, the stopped run stays cancelled. - Nothing runs until the operator comments. - The continue comment then finishes the task. ## What Changed - S1 workload ("Stop while running, then continue"): a brief task. At its follow-up time, while a run is running, the observer posts `POST /api/heartbeat-runs/:id/cancel` (the board's Stop) and records `stop`. - New `stop` trigger, for S1 only: the planned fault fires 0–2 s after the stop. - The continue comment is posted only after the stop has settled: at least 20 s after the stop, with no run running or queued and no fault in progress. It asks the agent to continue and add a `CONTINUE-` line. - The oracle requires: - status done; - the brief with the chaos and continue markers; - `stop_took_effect`: the stopped run ended `cancelled`; - `nothing_ran_after_stop`: no run was created between the stop and the continue comment. - S1 is not a default workload. There is a README section. ## Verification - `npx vitest run` in `tests/restart-chaos`: 205 passed. New tests cover: - the S1 grade (pass, a run restarted after the stop, the stop not taking effect, no stop); - the trigger-to-workload pairing; - an observer test that clicks Stop once and posts the continue comment only after the 20 s settle. - `node scripts/typecheck-test-projects.mjs tests/restart-chaos`: clean. - A live S1 wave runs after the current chaos wave. Its result will be posted here before merge. ## Risks - If the product moves the stopped run to a status other than `cancelled`, `stop_took_effect` fails. That check is intentional. - The continue comment waits for the stop to settle, so a run stuck `queued` delays it. R5 and the wave deadline still bound the wait. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, with tool use and code execution in Claude Code. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- tests/restart-chaos/README.md | 15 +++++ tests/restart-chaos/config.ts | 2 +- tests/restart-chaos/governance.test.ts | 2 + tests/restart-chaos/grader.test.ts | 11 ++++ tests/restart-chaos/observer.test.ts | 78 +++++++++++++++++++++++++- tests/restart-chaos/observer.ts | 53 ++++++++++++++++- tests/restart-chaos/scheduler.ts | 4 +- tests/restart-chaos/types.ts | 8 ++- tests/restart-chaos/workloads.ts | 25 +++++++++ 9 files changed, 189 insertions(+), 9 deletions(-) diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index a70b2d4a65f..ddc2881ae27 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -102,6 +102,21 @@ Neither G1 nor H1 is in the default workloads. The board's actions and what the observer saw are in the timeline (`governance_*`) and in each task's grade (`governance`). +## Stop while running, then continue (S1, `--triggers stop`) + +```sh +node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env local \ + --adapters codex,claude --workloads S1 --faults K1,K2,K4 --triggers stop --concurrency 4 --tasks 6 --seed 23 +``` + +S1 is the board's Stop button. While the task's run is running, the harness cancels it through +`POST /api/heartbeat-runs/:id/cancel`. The `stop` trigger fires the fault 0–2 s later. At least +20 s after the stop, once no run is running or queued and no fault is in progress, the harness +comments that it stopped the run on purpose and asks the agent to continue with an extra marker +line. The oracle checks that the stopped run ended `cancelled`, that nothing started between the +stop and the continue comment (recovery must not restart work the operator stopped), and that the +finished brief has both markers. S1 is not a default workload. + ## Code change in a real repository (R1, `--workspace-repo`) ```sh diff --git a/tests/restart-chaos/config.ts b/tests/restart-chaos/config.ts index 5c24f708ae4..8852fd00693 100644 --- a/tests/restart-chaos/config.ts +++ b/tests/restart-chaos/config.ts @@ -140,7 +140,7 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC env, adapters: list(values.adapters, ADAPTER_IDS, "adapters", ["codex", "claude"]), // The governance scenarios (G1, H1) are opt-in: H1 needs a wave of its own. - workloads: list(values.workloads, workloadChoices, "workloads", workloadChoices.filter(id => id !== "G1" && id !== "H1" && id !== "R1")), + workloads: list(values.workloads, workloadChoices, "workloads", workloadChoices.filter(id => id !== "G1" && id !== "H1" && id !== "R1" && id !== "S1")), // A crash-point wave schedules no fault; K1 is the restart its self-kill gets. faults: list(values.faults, FAULT_KINDS, "faults", stormMinutes ? STORM_DEFAULT_FAULTS : crashPoint ? ["K1"] : ["K1", "K2", "K4"]), triggers: list(values.triggers, TRIGGER_IDS, "triggers", ["time"]), diff --git a/tests/restart-chaos/governance.test.ts b/tests/restart-chaos/governance.test.ts index bc6ef406610..0ce5a2d189b 100644 --- a/tests/restart-chaos/governance.test.ts +++ b/tests/restart-chaos/governance.test.ts @@ -260,6 +260,8 @@ describe("governance wave options", () => { expect(command.options.workloads).not.toContain("G1"); expect(command.options.workloads).not.toContain("H1"); expect(triggerFitsWorkload("approval", "G1")).toBe(true); + expect(triggerFitsWorkload("stop", "S1")).toBe(true); + expect(triggerFitsWorkload("stop", "C1")).toBe(false); expect(triggerFitsWorkload("approval", "A1")).toBe(false); expect(triggerFitsWorkload("paused", "H1")).toBe(true); expect(() => planTasks({ seed: 1, tasks: 2, workloads: ["A1"], adapters: ["codex"], faults: ["K1"], triggers: ["approval"] })) diff --git a/tests/restart-chaos/grader.test.ts b/tests/restart-chaos/grader.test.ts index 1f133a67178..79f0043516b 100644 --- a/tests/restart-chaos/grader.test.ts +++ b/tests/restart-chaos/grader.test.ts @@ -238,6 +238,17 @@ describe("T-oracles", () => { expect(stallWakeChecks({ ...stalled, childStalls: [] } as never)).toEqual([]); }); + it("grades S1: stopped run cancelled, nothing ran before the continue comment, brief has both markers", () => { + const base: OracleArtifacts = { issueStatus: "done", documents: { brief: "CHAOS-abcd1234\nCONTINUE-abcd1234\n- one\n- two\n- three\n" }, + agentComments: [], files: {}, commands: {}, workspacePath: null, + stop: { runId: "r1", runStatus: "cancelled", runsStartedBeforeContinue: [], continuePosted: true } }; + expect(workloadDefinitions.S1.grade("abcd1234", base).verdict).toBe("pass"); + const restarted = workloadDefinitions.S1.grade("abcd1234", { ...base, stop: { ...base.stop!, runsStartedBeforeContinue: ["r2"] } }); + expect(restarted.checks.find(check => check.name === "nothing_ran_after_stop")).toMatchObject({ ok: false, detail: "continue=true runs=r2" }); + expect(workloadDefinitions.S1.grade("abcd1234", { ...base, stop: { ...base.stop!, runStatus: "succeeded" } }).verdict).toBe("fail"); + expect(workloadDefinitions.S1.grade("abcd1234", { ...base, stop: undefined }).verdict).toBe("fail"); + }); + it("grades B1 by running the tests and the CLI", () => { const base: OracleArtifacts = { issueStatus: "done", documents: {}, agentComments: [], workspacePath: "/tmp/ws", files: { "NOTES-abcd1234.md": "B1-abcd1234\n" }, diff --git a/tests/restart-chaos/observer.test.ts b/tests/restart-chaos/observer.test.ts index 1ea8d6b37da..4e6510fe111 100644 --- a/tests/restart-chaos/observer.test.ts +++ b/tests/restart-chaos/observer.test.ts @@ -4,7 +4,7 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import type { ChaosApi } from "./api.js"; -import { CHILD_WAKE_GRACE_MS, REST_CONFIRM_MS, STALL_GIVE_UP_MS, TaskObserver, childrenPending, listWorkspaceFiles } from "./observer.js"; +import { CHILD_WAKE_GRACE_MS, REST_CONFIRM_MS, STALL_GIVE_UP_MS, STOP_CONTINUE_AFTER_MS, TaskObserver, childrenPending, listWorkspaceFiles } from "./observer.js"; import { gradeTask } from "./grader.js"; import { createFixtureRepository, type ChaosLab } from "./seed.js"; import { Timeline } from "./timeline.js"; @@ -138,6 +138,82 @@ describe("TaskObserver", () => { expect(childRetries).toHaveLength(1); }); + it("S1: stops the running run as the board would, then posts the continue comment only after it settles", async () => { + let runStatus = "running"; + const base = fakeApi({ "/runs": () => [{ id: "r1", status: runStatus, createdAt: new Date(0).toISOString(), finishedAt: null }] }, "in_progress"); + const posted: string[] = []; + const api = { ...base.api, get: base.api.get.bind(base.api), async post(route: string) { + posted.push(route); + if (route === "/api/heartbeat-runs/r1/cancel") { runStatus = "cancelled"; return { id: "r1", status: "cancelled" }; } + return route.endsWith("/issues") ? { id: "i1", identifier: "CHA-1" } : { id: "comment-1" }; + } } as unknown as ChaosApi; + const observer = new TaskObserver({ ...plan, workload: "S1", followUpAfterMs: 1_000 }, api, lab, new Timeline(null), ready); + await observer.launch(); + const start = observer.record.createdAt!; + await observer.poll(start + 2_000); + expect(posted).toContain("/api/heartbeat-runs/r1/cancel"); + expect(observer.record.stop).toMatchObject({ runId: "r1", httpStatus: 200, error: null }); + await observer.poll(start + 2_000 + STOP_CONTINUE_AFTER_MS - 1); + expect(observer.record.followUp).toBeNull(); + await observer.poll(start + 2_000 + STOP_CONTINUE_AFTER_MS + 1); + expect(observer.record.followUp).toMatchObject({ commentId: "comment-1", duringRun: false }); + expect(posted.filter(route => route.endsWith("/cancel"))).toHaveLength(1); + }); + + it("S1: does not count the continue comment's own run as a run after the stop, even when it starts before the comment returns", async () => { + let runs: Array> = [{ id: "r1", status: "running", createdAt: new Date(0).toISOString(), finishedAt: null }]; + const base = fakeApi({ "/runs": () => runs, "/comments": () => [], "/documents/": () => null, "/workspace-operations": () => [] }, "in_progress"); + let clock = 0; + const api = { ...base.api, get: base.api.get.bind(base.api), async post(route: string) { + if (route === "/api/heartbeat-runs/r1/cancel") { runs = [{ ...runs[0]!, status: "cancelled" }]; return { id: "r1" }; } + if (route.endsWith("/comments")) { + runs = [...runs, { id: "r2", status: "running", createdAt: new Date(clock - 10).toISOString(), finishedAt: null, wakeCommentIds: ["continue-1"] }]; + return { id: "continue-1" }; + } + return { id: "i1", identifier: "CHA-1" }; + } } as unknown as ChaosApi; + const observer = new TaskObserver({ ...plan, workload: "S1", followUpAfterMs: 1_000 }, api, lab, new Timeline(null), ready); + await observer.launch(); + const start = observer.record.createdAt!; + await observer.poll(start + 2_000); + clock = start + 2_000 + STOP_CONTINUE_AFTER_MS + 1; + await observer.poll(clock); + await observer.poll(clock + 3_000); + const artifacts = await observer.collectArtifacts(); + expect(artifacts.stop).toMatchObject({ runId: "r1", runStatus: "cancelled", runsStartedBeforeContinue: [], continuePosted: true }); + }); + + it("S1: counts a run recovery started between the poll's reads and the continue request", async () => { + let runs: Array> = [{ id: "r1", status: "running", createdAt: new Date(0).toISOString(), finishedAt: null }]; + let stopped = false; + const base = fakeApi({ "/runs": () => runs, "/comments": () => [], "/documents/": () => null, "/workspace-operations": () => [] }, "in_progress"); + let sentAt = 0; + const api = { ...base.api, async get(route: string) { + const value = await base.api.get(route); + if (stopped && route.endsWith("/runs") && runs.length === 1) { + runs = [...runs, { id: "r2", status: "queued", createdAt: new Date(sentAt - 5).toISOString(), finishedAt: null }]; + } + return value; + }, async post(route: string) { + if (route === "/api/heartbeat-runs/r1/cancel") { stopped = true; runs = [{ ...runs[0]!, status: "cancelled" }]; return { id: "r1" }; } + if (route.endsWith("/comments")) return { id: "continue-1" }; + return { id: "i1", identifier: "CHA-1" }; + } } as unknown as ChaosApi; + const observer = new TaskObserver({ ...plan, workload: "S1", followUpAfterMs: 1_000 }, api, lab, new Timeline(null), ready); + await observer.launch(); + const start = observer.record.createdAt!; + await observer.poll(start + 2_000); + const at = start + 2_000 + STOP_CONTINUE_AFTER_MS + 1; + sentAt = at + 100; + runs = [runs[0]!]; + observer.record.followUp = { at, sentAt, postedAt: sentAt + 50, commentId: "continue-1", error: null, duringRun: false }; + observer.record.samples.push({ at: sentAt + 200, reachable: true, serverReady: true, issueStatus: "in_progress", interruption: null, + activeRecoveryAction: false, recoveryOwner: null, executionBlocker: false, scheduledRetry: false, + runs: [runs[0], { id: "r2", status: "queued", createdAt: new Date(sentAt - 5).toISOString(), finishedAt: null }] } as never); + const artifacts = await observer.collectArtifacts(); + expect(artifacts.stop?.runsStartedBeforeContinue).toEqual(["r2"]); + }); + it("closes an open task as unfinished at interrupt, so the wave grades it as a failure", async () => { const { api } = fakeApi({}, "in_progress"); const timeline = new Timeline(null); diff --git a/tests/restart-chaos/observer.ts b/tests/restart-chaos/observer.ts index efd5dd47e70..251a2b9bf90 100644 --- a/tests/restart-chaos/observer.ts +++ b/tests/restart-chaos/observer.ts @@ -50,6 +50,7 @@ const RESULT_TRIGGER_EVENTS = new Set(["run.result.proposed", "run.result.accept export const REST_CONFIRM_MS = 12_000; export const CHILD_WAKE_GRACE_MS = 60_000; export const CHILD_STALL_CHECK_MS = 10_000; +export const STOP_CONTINUE_AFTER_MS = 20_000; export function childrenPending(children: ReadonlyArray<{ status: string | null }> | null): boolean { return !children || children.some(child => !["done", "cancelled", "blocked"].includes(child.status ?? "")); @@ -344,6 +345,11 @@ export class TaskObserver { case "tool": return this.record.benchFirstSeenAt ?? null; case "approval": return this.record.governance?.approvalSeenAt ?? null; case "paused": return this.record.governance?.budgetPausedAt ?? null; + case "stop": { + const stop = this.record.stop; + if (!stop) return null; + return stop.error ? "missed" : stop.at; + } case "result": { // B8: the run that received the follow-up has accepted its result but // has not settled yet. A follow-up posted at rest cannot cross it. @@ -415,6 +421,10 @@ export class TaskObserver { this.restCandidateSince = null; this.childrenSettledAt = null; } + if (await this.maybeStop(sample, now)) { + this.restCandidateSince = null; + return; + } if (await this.maybeFollowUp(sample, now)) { // The comment response is not the agent's answer: require fresh samples. this.restCandidateSince = null; @@ -490,18 +500,42 @@ export class TaskObserver { } } + private async maybeStop(sample: TaskSample, now: number): Promise { + if (!this.definition.stopBeforeFollowUp || this.record.stop || this.plan.followUpAfterMs === null || !sample.reachable) return false; + if (now < this.record.createdAt! + this.plan.followUpAfterMs) return false; + const running = (sample.runs ?? []).find(run => run.status === "running"); + if (!running) return false; + const stop: NonNullable = { at: now, runId: running.id, httpStatus: null, error: null }; + try { + await this.api.post(`/api/heartbeat-runs/${running.id}/cancel`, { cancellationRequestId: randomUUID() }); + stop.httpStatus = 200; + } catch (error) { + stop.httpStatus = error instanceof ChaosApiError ? error.status : null; + stop.error = error instanceof Error ? error.message : String(error); + } + this.record.stop = stop; + this.timeline.add("task_stop_clicked", { taskId: this.plan.taskId, ...stop }); + return true; + } + private async maybeFollowUp(sample: TaskSample, now: number): Promise { const definition = this.definition; if (!definition.followUp || this.record.followUp || this.plan.followUpAfterMs === null || !sample.reachable) return false; if (now < this.record.createdAt! + this.plan.followUpAfterMs) return false; + if (definition.stopBeforeFollowUp) { + const stop = this.record.stop; + if (!stop || now - stop.at < STOP_CONTINUE_AFTER_MS || this.supervisor.faultInProgress) return false; + if ((sample.runs ?? []).some(run => run.status === "running" || run.status === "queued")) return false; + } // C1 targets a live run (B8); wait for one unless the task already rests. const running = (sample.runs ?? []).some(run => run.status === "running"); - if (!running && !isResting(sample)) return false; + if (!running && !definition.stopBeforeFollowUp && !isResting(sample)) return false; + const sentAt = Math.max(now, Date.now()); try { const comment = await this.api.post(`/api/issues/${this.record.issueId}/comments`, { body: definition.followUp(this.plan.nonce) }); - this.record.followUp = { at: now, postedAt: Math.max(now, Date.now()), commentId: comment?.id ?? null, error: null, duringRun: running }; + this.record.followUp = { at: now, sentAt, postedAt: Math.max(now, Date.now()), commentId: comment?.id ?? null, error: null, duringRun: running }; } catch (error) { - this.record.followUp = { at: now, commentId: null, error: error instanceof Error ? error.message : String(error), duringRun: running }; + this.record.followUp = { at: now, sentAt, commentId: null, error: error instanceof Error ? error.message : String(error), duringRun: running }; } this.timeline.add("task_follow_up", { taskId: this.plan.taskId, ...this.record.followUp }); return true; @@ -567,6 +601,18 @@ export class TaskObserver { return this.retryBlockedChildren(blocked, now); } + private stopArtifacts(stop: NonNullable): NonNullable { + const runs = new Map[number]>(); + for (const sample of this.record.samples) for (const run of sample.runs ?? []) runs.set(run.id, run); + const continueAt = this.record.followUp?.sentAt ?? this.record.followUp?.at ?? null; + const continueCommentId = this.record.followUp?.commentId ?? null; + const startedBeforeContinue = [...runs.values()].filter(run => run.id !== stop.runId && run.createdAt + && !(continueCommentId && (run.wakeCommentIds ?? []).includes(continueCommentId)) + && Date.parse(run.createdAt) > stop.at && (continueAt === null || Date.parse(run.createdAt) < continueAt)).map(run => run.id); + return { runId: stop.runId, runStatus: runs.get(stop.runId)?.status ?? null, runsStartedBeforeContinue: startedBeforeContinue, + continuePosted: Boolean(this.record.followUp?.commentId) }; + } + private async retryBlockedChildren(childIds: readonly string[], now: number): Promise { let clicked = false; for (const childIssueId of childIds) { @@ -831,6 +877,7 @@ export class TaskObserver { agentComments: (comments ?? []).filter(comment => comment.authorAgentId).map(comment => String(comment.body ?? "")), files, commands, workspacePath, ...(this.plan.workload === "B5" ? { benchProgressObserved: this.record.benchProgressSeen?.length ?? 0 } : {}), + ...(this.record.stop ? { stop: this.stopArtifacts(this.record.stop) } : {}), ...(definition.collect.childDocuments ? { children: await this.collectChildren(issueId), childStalls: this.record.childStalls ?? [], stallWakes: await this.collectStallWakes(issueId) } : {}), ...(this.plan.workload === "G1" ? { approvals: await this.collectApprovals(issueId) } : {}), diff --git a/tests/restart-chaos/scheduler.ts b/tests/restart-chaos/scheduler.ts index 63b5131408a..696c4799ead 100644 --- a/tests/restart-chaos/scheduler.ts +++ b/tests/restart-chaos/scheduler.ts @@ -44,6 +44,7 @@ export const TRIGGER_DELAY_RANGE_MS: Record, readonly result: [0, 3_000], approval: [0, 3_000], paused: [0, 3_000], + stop: [0, 2_000], }; export function triggerFitsWorkload(trigger: TriggerId, workload: WorkloadId): boolean { @@ -51,11 +52,12 @@ export function triggerFitsWorkload(trigger: TriggerId, workload: WorkloadId): b if (trigger === "tool") return workload === "B5"; if (trigger === "approval") return workload === "G1"; if (trigger === "paused") return workload === "H1"; + if (trigger === "stop") return workload === "S1"; return true; } /** The workload a trigger needs, for the error when none is selected. */ -const TRIGGER_WORKLOAD: Partial> = { comment: "C1", result: "C1", tool: "B5", approval: "G1", paused: "H1" }; +const TRIGGER_WORKLOAD: Partial> = { comment: "C1", result: "C1", tool: "B5", approval: "G1", paused: "H1", stop: "S1" }; /** Faults a `result` task may fire inside a storm: a fast restart that lands on the boundary. */ const STORM_RESULT_FAULTS: readonly FaultKind[] = ["K1", "K2"]; diff --git a/tests/restart-chaos/types.ts b/tests/restart-chaos/types.ts index fb02f8a617f..93676713716 100644 --- a/tests/restart-chaos/types.ts +++ b/tests/restart-chaos/types.ts @@ -7,13 +7,13 @@ export type FaultKind = (typeof FAULT_KINDS)[number]; export type TaskFaultKind = FaultKind | "K13"; /** G1: an approval-gated action across a restart. H1: a budget hard stop crossed mid-run with a restart (its own wave). */ -export const WORKLOAD_IDS = ["A1", "B1", "B5", "C1", "M1", "G1", "H1", "R1"] as const; +export const WORKLOAD_IDS = ["A1", "B1", "B5", "C1", "M1", "G1", "H1", "R1", "S1"] as const; export type WorkloadId = (typeof WORKLOAD_IDS)[number]; export const ADAPTER_IDS = ["codex", "claude", "opencode"] as const; export type AdapterId = (typeof ADAPTER_IDS)[number]; -export const TRIGGER_IDS = ["time", "comment", "retry", "queued", "tool", "result", "approval", "paused"] as const; +export const TRIGGER_IDS = ["time", "comment", "retry", "queued", "tool", "result", "approval", "paused", "stop"] as const; export type TriggerId = (typeof TRIGGER_IDS)[number]; export const ENVIRONMENT_KINDS = ["local", "daytona"] as const; @@ -221,6 +221,7 @@ export interface OracleArtifacts { workspacePath: string | null; /** B5: distinct `.bench/progress` values the harness itself saw while the task was live. */ benchProgressObserved?: number; + stop?: { runId: string; runStatus: string | null; runsStartedBeforeContinue: string[]; continuePosted: boolean }; children?: Array<{ id: string; title?: string | null; status: string | null; assigneeAgentId: string | null; documents: Record }>; childStalls?: Array<{ childIssueId: string; recoveryActionId: string | null }>; stallWakes?: Array<{ stalledIssueId: string | null; recoveryActionId: string | null }>; @@ -281,7 +282,8 @@ export interface TaskRecord { finalStatus: string | null; deadlineAt: number | null; timedOut: boolean; - followUp: { at: number; postedAt?: number; commentId: string | null; error: string | null; duringRun?: boolean } | null; + followUp: { at: number; sentAt?: number; postedAt?: number; commentId: string | null; error: string | null; duringRun?: boolean } | null; + stop?: { at: number; runId: string; httpStatus: number | null; error: string | null }; faultIds: string[]; sweep: RestSweep | null; /** Workspace files present just before the first fault that hit this task (R2). */ diff --git a/tests/restart-chaos/workloads.ts b/tests/restart-chaos/workloads.ts index 46693c100d6..3bd8ebc73da 100644 --- a/tests/restart-chaos/workloads.ts +++ b/tests/restart-chaos/workloads.ts @@ -16,6 +16,7 @@ export interface WorkloadDefinition { title(nonce: string): string; description(nonce: string, agents?: WorkloadAgents): string; followUp?(nonce: string): string; + stopBeforeFollowUp?: boolean; /** Files and commands the observer must collect for the T-oracle. */ collect: { documents: string[]; files(nonce: string): string[]; commands: Array<{ name: string; argv: string[] | ((nonce: string) => string[]) }>; childDocuments?: string[] }; @@ -35,6 +36,7 @@ const partMarker = (nonce: string, part: number) => `PART${part}-${nonce}`; const briefMarker = (nonce: string) => `CHAOS-${nonce}`; const followUpMarker = (nonce: string) => `FOLLOWUP-${nonce}`; +const continueMarker = (nonce: string) => `CONTINUE-${nonce}`; const notesFile = (nonce: string) => `NOTES-${nonce}.md`; const benchFile = (nonce: string) => `bench-result-${nonce}.txt`; const benchRecord = (nonce: string) => `.bench/run-${nonce}.txt`; @@ -251,6 +253,29 @@ export const workloadDefinitions: Record = { return result([statusCheck(artifacts), ...briefChecks(artifacts.documents.brief, [briefMarker(nonce), followUpMarker(nonce)])]); }, }, + S1: { + id: "S1", + label: "Stop while running, then continue", + usesRepository: false, + fireWindowMs: [20_000, 60_000], + followUpWindowMs: [8_000, 25_000], + budgetMs: 12 * 60_000, + title: nonce => `Chaos S1 brief stopped and continued ${nonce}`, + description: briefDescription, + stopBeforeFollowUp: true, + followUp: nonce => `I stopped your previous run on purpose. Please continue the task now: finish the \`brief\` document and also include the line \`${continueMarker(nonce)}\` on its own line before you mark the task done.`, + collect: { documents: ["brief"], files: () => [], commands: [] }, + grade(nonce, artifacts) { + const stop = artifacts.stop; + return result([ + statusCheck(artifacts), + ...briefChecks(artifacts.documents.brief, [briefMarker(nonce), continueMarker(nonce)]), + { name: "stop_took_effect", ok: stop?.runStatus === "cancelled", detail: `run=${stop?.runId ?? "none"} status=${stop?.runStatus ?? "none"}` }, + { name: "nothing_ran_after_stop", ok: Boolean(stop) && stop!.continuePosted && stop!.runsStartedBeforeContinue.length === 0, + detail: stop ? `continue=${stop.continuePosted} runs=${stop.runsStartedBeforeContinue.join(",") || "none"}` : "not stopped" }, + ]); + }, + }, B1: { id: "B1", label: "Python CLI with tests", From 5a51a7685d75a83846722a5d99962f8bf0a96902 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sun, 8 Nov 2026 22:56:28 -0600 Subject: [PATCH 5/9] fix(budgets): raise_budget_and_resume continues the work its budget stopped, once (#20144) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - A budget hard stop cancels the scope's running work; the board resolves it with Raise budget and resume > - In the restart-chaos H1 scenario, the cancelled tasks did not continue after resume and each needed a manual Retry > - Resume is the board's explicit decision to continue, so the work it stopped should continue once, as if the board clicked Retry > - This pull request files that Retry automatically, keyed by the cancelled run, once the run's stop has settled > - The benefit is that a budget raise restores the work without manual Retries, without repeating completed work, and without charging the run's attempt budget ## Linked Issues or Issue Description Refs #20143 (bug B; this PR builds on it), #20113 (bug A), #20101 (superseded: split into #20143 and this PR) **What happened?** Restart-chaos waves w-20261108T195944-s42 and w-20261108T210118-s9802 (H1). After `raise_budget_and_resume`, CHA-3's budget-cancelled run (enforcement stop) and CHA-1's run (agent lifecycle pause from the budget hold) stayed cancelled until someone clicked Retry. **Expected behavior** Resuming continues each task its budget stopped, exactly once, with no attempt charged, an activity entry, and no repeat of work that already continued. A run a person stopped stays stopped. **Steps to reproduce** Set a budget below spend while a task runs; let the stop cancel the run; resolve with Raise budget and resume. Before this change, no new run is queued. ## What Changed - New table `budget_resume_claims` (migration 0451). One row per cancelled run, outside the run's result JSON, so a Stop that settles during the resume cannot erase it. - `markBudgetCancelledWorkForResume` (called by `resolveIncident` before the incident closes): claims every task run its budget stopped since the incident opened. That covers runs budget enforcement fenced, including stops still in flight, and runs the agent lifecycle pause stopped for this budget (`pauseStop`, carried from the lifecycle operation's `pauseCause` so a resume before the Stop keeps the link). Runs a person stopped are excluded. Two indexed passes led by `(company_id, status)`. - `deliverBudgetResumeContinuations`: once a claimed run's stop settles, files the resolving board member's Retry (`requestTaskRetry`, idempotency key `budget-resume:`, with the member's session source and the chat Retry callback). The Retry path checks, under the task lock, that the run is still the task's latest and the task is open, so continued or completed work is never repeated. Claim settlement and its activity entry commit together; retryable refusals and errors retry. - Delivery runs in the always-on budget-enforcement worker after each pass's Stop delivery. Its schedule includes deliverable claims and the earliest stale-claim expiry. The execution-continuation worker is a backstop. - `agent.paused` / `agent.resumed` logging is in #20113. `budget.cancelled_work_resume_requested` (incident) and `budget.cancelled_work_continued` / `_not_continued` (task) are recorded here. ## Verification - New `budget-resume-continuation.test.ts` (Postgres, real `resolveIncident` → claims → `requestTaskRetry` → real `resumeTaskRetry` into a queued run): - each task continues once, including a blocked task; repeat delivery or resolve saves nothing new; - a stop still in flight at resume, whose terminal write replaces the result JSON, is still continued; - a 30-task scope is fully continued; - a lifecycle-pause stop caused by the budget is continued, and one caused by a manual pause is not; - continued work, done tasks, a run that finished before its stop, and person-stopped runs (including ones enforcement also fenced) are not continued. - `lifecycle.test.ts`: the pause records its cause on the operation and on stopped runs, including when the budget is resumed before the pause's Stop. `budget-enforcement-worker.test.ts`: delivery after Stop delivery, and a wake at claim expiry. - 287 tests across the affected suites passed. Server typecheck (`typecheck-layers`, low-memory): no errors. - Not run: the H1 chaos rerun (Phase 3 runs it after merge). ## Risks - Resume now continues budget-stopped tasks automatically. A board that wants a task stopped can Stop it again; person-stopped runs are never continued. - A new table and one index (migration 0451). > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, via Claude Code with tool use (shell, file edits, test execution). Extended reasoning enabled. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- doc/execution-semantics.md | 2 + .../migrations/0451_budget_resume_claims.sql | 18 + ...{0446_snapshot.json => 0451_snapshot.json} | 310 ++++++++++++++- packages/db/src/migrations/meta/_journal.json | 7 + .../db/src/schema/budget_resume_claims.ts | 32 ++ packages/db/src/schema/index.ts | 1 + packages/shared/src/types/agent-lifecycle.ts | 3 + .../budget-enforcement-worker.test.ts | 48 +++ server/src/app.ts | 4 + server/src/index.ts | 10 + .../__tests__/lifecycle.test.ts | 44 ++- .../agent-lifecycle/adapters/postgres.ts | 5 + server/src/routes/costs.ts | 3 +- server/src/services/agent-lifecycle-driver.ts | 12 +- server/src/services/agent-lifecycle.ts | 2 + .../src/services/budget-enforcement-worker.ts | 39 +- .../budget-resume-continuation.test.ts | 353 ++++++++++++++++++ .../services/budget-resume-continuation.ts | 352 +++++++++++++++++ server/src/services/budgets.ts | 28 +- server/src/services/execution-control-work.ts | 3 +- server/src/services/heartbeat/queue.test.ts | 47 ++- server/src/services/heartbeat/queue.ts | 35 +- server/src/services/heartbeat/run-control.ts | 4 + 23 files changed, 1344 insertions(+), 18 deletions(-) create mode 100644 packages/db/src/migrations/0451_budget_resume_claims.sql rename packages/db/src/migrations/meta/{0446_snapshot.json => 0451_snapshot.json} (99%) create mode 100644 packages/db/src/schema/budget_resume_claims.ts create mode 100644 server/src/services/budget-resume-continuation.test.ts create mode 100644 server/src/services/budget-resume-continuation.ts diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 762d41bf647..6541f4088b1 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -514,6 +514,8 @@ Another run's unfinished accounting or cleanup never refuses new work (see `doc/ A user's message refused by a pause (a paused agent, a paused company, or a budget pause that is not a reached hard stop) is saved as a deferred intent that names the refusal and its release trigger; it is never written as a terminal skip. The pause's own stop keeps it. Lifting the pause is the operator's explicit authorization to send what the pause refused, as Send is after a Stop: when the pause stopped the task's run, the lifted pause issues the task's next explicit wake, which adopts the saved message once. A task that never ran keeps the message in the task, and stranded-work recovery starts it. +A reached budget hard stop cancels the scope's running work. When a board member resolves the incident with Raise budget and resume, Paperclip claims every task run that the stop cancelled since the incident opened (runs the budget enforcement fenced, and runs the agent lifecycle pause stopped for this budget: each pause records why it started, and the runs it stops carry that cause), with that member and their session source, in one write before the incident closes, as rows in `budget_resume_claims`, outside the run's result JSON so a Stop settling at the same moment cannot erase them. This includes runs whose stop is still being delivered. Resolving again keeps existing claims. The execution-continuation worker delivers each claim once its run's stop has settled: it files that member's Retry for the task. The Retry clears the task's execution hold and is accepted only while the cancelled run is still the task's latest run and the task is open and assigned, checked under the task lock, so completed or continued work is never repeated. Its idempotency key names the cancelled run, so every pass, restart, or repeated resume returns the same single request. The claim and its activity entry settle in one commit. The Retry starts a new run through normal admission and charges no attempt to the cancelled run. Paperclip records `budget.cancelled_work_resume_requested` on the incident and `budget.cancelled_work_continued` (or `budget.cancelled_work_not_continued`, with the reason) on each task. A run a person stopped is never continued this way, even if budget enforcement also fenced it. A pause's stop is intentional, not a failure: a native run that the agent lifecycle pause or a budget pause cancelled keeps its task's status and never opens a reconciliation incident; resuming continues it. A retryable Retry refusal leaves the claim pending for the next pass. + The same holds for accepted work beyond a first assignment. A temporary refusal never terminally cancels or escalates accepted work: an unstarted queued run (a promoted comment, a Retry, or a recovery) stays queued, and stranded-work recovery leaves the task as it is instead of moving it to `blocked`. These waits are passive, so settling a run's accounting wakes queued-run dispatch, the saved-receipt worker, and stranded-work recovery for the company. The held work then starts through normal admission, once. When a native run reaches a terminal status through recovery, or releases its environment, each message saved behind it is hinted by its own receipt, so delivery does not wait out a company-wide rescan interval. New comments received during an execution hold retain their individual deferred receipts and ordered comment ids. Release cannot drain those receipts while replay remains blocked; the next eligible wake can adopt them. Authorized external-chat requests also remain deferred with their exact durable receipt. They must use normal promotion and current authorization; a generic wake cannot adopt only their comment ids and discard their actor or session contract. A wait does not authorize replay, reset an incident retry budget, or bypass an interaction's delivery rules. diff --git a/packages/db/src/migrations/0451_budget_resume_claims.sql b/packages/db/src/migrations/0451_budget_resume_claims.sql new file mode 100644 index 00000000000..0c55c9107c9 --- /dev/null +++ b/packages/db/src/migrations/0451_budget_resume_claims.sql @@ -0,0 +1,18 @@ +CREATE TABLE "budget_resume_claims" ( + "run_id" uuid PRIMARY KEY NOT NULL, + "company_id" uuid NOT NULL, + "incident_id" uuid NOT NULL, + "actor_user_id" text NOT NULL, + "actor_source" text, + "state" text DEFAULT 'pending' NOT NULL, + "claimed_at" timestamp with time zone, + "settled_at" timestamp with time zone, + "reason" text, + "request_id" uuid, + "created_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +ALTER TABLE "budget_resume_claims" ADD CONSTRAINT "budget_resume_claims_run_id_heartbeat_runs_id_fk" FOREIGN KEY ("run_id") REFERENCES "public"."heartbeat_runs"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "budget_resume_claims" ADD CONSTRAINT "budget_resume_claims_company_id_companies_id_fk" FOREIGN KEY ("company_id") REFERENCES "public"."companies"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "budget_resume_claims" ADD CONSTRAINT "budget_resume_claims_incident_id_budget_incidents_id_fk" FOREIGN KEY ("incident_id") REFERENCES "public"."budget_incidents"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +CREATE INDEX "budget_resume_claims_pending_idx" ON "budget_resume_claims" USING btree ("company_id","run_id") WHERE "budget_resume_claims"."state" = 'pending'; \ No newline at end of file diff --git a/packages/db/src/migrations/meta/0446_snapshot.json b/packages/db/src/migrations/meta/0451_snapshot.json similarity index 99% rename from packages/db/src/migrations/meta/0446_snapshot.json rename to packages/db/src/migrations/meta/0451_snapshot.json index f7c4f4c2756..371419bc95d 100644 --- a/packages/db/src/migrations/meta/0446_snapshot.json +++ b/packages/db/src/migrations/meta/0451_snapshot.json @@ -1,6 +1,6 @@ { - "id": "782688f1-7843-4a93-8ea3-2cec362d269e", - "prevId": "f0d8430b-7abf-4232-86a9-11a268a0d20e", + "id": "b5699366-77c2-48c8-b7dc-658a75de7658", + "prevId": "b8f43f1a-3230-4cda-991d-0e7b6323114b", "version": "7", "dialect": "postgresql", "tables": { @@ -4102,6 +4102,28 @@ "method": "btree", "with": {} }, + "agent_wakeup_requests_issue_blocker_stalled_uq": { + "name": "agent_wakeup_requests_issue_blocker_stalled_uq", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"agent_wakeup_requests\".\"idempotency_key\" LIKE 'issue_blocker_stalled:%' AND \"agent_wakeup_requests\".\"status\" IN ('queued', 'deferred_issue_execution', 'claimed', 'coalesced', 'completed')", + "concurrently": true, + "method": "btree", + "with": {} + }, "agent_wakeup_requests_company_payload_issue_idx": { "name": "agent_wakeup_requests_company_payload_issue_idx", "columns": [ @@ -4252,6 +4274,18 @@ "primaryKey": false, "notNull": false }, + "harness_verified_digest": { + "name": "harness_verified_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "harness_verified_at": { + "name": "harness_verified_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, "reports_to": { "name": "reports_to", "type": "uuid", @@ -7848,6 +7882,150 @@ }, "isRLSEnabled": false }, + "public.budget_resume_claims": { + "name": "budget_resume_claims", + "schema": "", + "columns": { + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "incident_id": { + "name": "incident_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "actor_user_id": { + "name": "actor_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_source": { + "name": "actor_source", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "claimed_at": { + "name": "claimed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "settled_at": { + "name": "settled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "budget_resume_claims_pending_idx": { + "name": "budget_resume_claims_pending_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"budget_resume_claims\".\"state\" = 'pending'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "budget_resume_claims_run_id_heartbeat_runs_id_fk": { + "name": "budget_resume_claims_run_id_heartbeat_runs_id_fk", + "tableFrom": "budget_resume_claims", + "tableTo": "heartbeat_runs", + "columnsFrom": [ + "run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "budget_resume_claims_company_id_companies_id_fk": { + "name": "budget_resume_claims_company_id_companies_id_fk", + "tableFrom": "budget_resume_claims", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "budget_resume_claims_incident_id_budget_incidents_id_fk": { + "name": "budget_resume_claims_incident_id_budget_incidents_id_fk", + "tableFrom": "budget_resume_claims", + "tableTo": "budget_incidents", + "columnsFrom": [ + "incident_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.built_in_managed_resources": { "name": "built_in_managed_resources", "schema": "", @@ -21965,6 +22143,112 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.controller_instances": { + "name": "controller_instances", + "schema": "", + "columns": { + "boot_id": { + "name": "boot_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "epoch": { + "name": "epoch", + "type": "bigserial", + "primaryKey": false, + "notNull": true + }, + "process_space": { + "name": "process_space", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pid": { + "name": "pid", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "renewed_at": { + "name": "renewed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "lease_expires_at": { + "name": "lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "fenced_at": { + "name": "fenced_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "fenced_by_epoch": { + "name": "fenced_by_epoch", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "stopped_at": { + "name": "stopped_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "controller_instances_epoch_idx": { + "name": "controller_instances_epoch_idx", + "columns": [ + { + "expression": "epoch", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "controller_instances_lease_idx": { + "name": "controller_instances_lease_idx", + "columns": [ + { + "expression": "lease_expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.cost_events": { "name": "cost_events", "schema": "", @@ -34856,6 +35140,12 @@ "primaryKey": false, "notNull": false }, + "controller_process": { + "name": "controller_process", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, "execution_stage": { "name": "execution_stage", "type": "text", @@ -35125,6 +35415,22 @@ "method": "btree", "with": {} }, + "heartbeat_runs_legacy_controller_lease_idx": { + "name": "heartbeat_runs_legacy_controller_lease_idx", + "columns": [ + { + "expression": "controller_lease_expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"heartbeat_runs\".\"runtime_mode\" = 'legacy' AND \"heartbeat_runs\".\"status\" = 'running' AND \"heartbeat_runs\".\"controller_boot_id\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, "heartbeat_runs_cost_accounting_pending_idx": { "name": "heartbeat_runs_cost_accounting_pending_idx", "columns": [ diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index fbb1f107ccc..cbd48bace5c 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -3144,6 +3144,13 @@ "when": 1794170960477, "tag": "0450_petite_frightful_four", "breakpoints": true + }, + { + "idx": 451, + "version": "7", + "when": 1794185816322, + "tag": "0451_budget_resume_claims", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/db/src/schema/budget_resume_claims.ts b/packages/db/src/schema/budget_resume_claims.ts new file mode 100644 index 00000000000..1e07d46cb3b --- /dev/null +++ b/packages/db/src/schema/budget_resume_claims.ts @@ -0,0 +1,32 @@ +import { sql } from "drizzle-orm"; +import { index, pgTable, text, timestamp, uuid } from "drizzle-orm/pg-core"; +import { budgetIncidents } from "./budget_incidents.js"; +import { companies } from "./companies.js"; +import { heartbeatRuns } from "./heartbeat_runs.js"; + +/** + * A run a budget stop cancelled, owed one continuation because a board member + * resolved the incident with Raise budget and resume. Kept outside the run's + * result JSON, which terminal writers replace, so a Stop that settles at the + * same moment cannot erase it. + */ +export const budgetResumeClaims = pgTable( + "budget_resume_claims", + { + runId: uuid("run_id").primaryKey().references(() => heartbeatRuns.id, { onDelete: "cascade" }), + companyId: uuid("company_id").notNull().references(() => companies.id, { onDelete: "cascade" }), + incidentId: uuid("incident_id").notNull().references(() => budgetIncidents.id, { onDelete: "cascade" }), + actorUserId: text("actor_user_id").notNull(), + actorSource: text("actor_source"), + state: text("state").notNull().default("pending"), + claimedAt: timestamp("claimed_at", { withTimezone: true }), + settledAt: timestamp("settled_at", { withTimezone: true }), + reason: text("reason"), + requestId: uuid("request_id"), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + }, + (table) => ({ + pendingIdx: index("budget_resume_claims_pending_idx").on(table.companyId, table.runId) + .where(sql`${table.state} = 'pending'`), + }), +); diff --git a/packages/db/src/schema/index.ts b/packages/db/src/schema/index.ts index 1feb03de978..33ebbd85998 100644 --- a/packages/db/src/schema/index.ts +++ b/packages/db/src/schema/index.ts @@ -22,6 +22,7 @@ export { invites } from "./invites.js"; export { joinRequests } from "./join_requests.js"; export { budgetPolicies } from "./budget_policies.js"; export { budgetIncidents } from "./budget_incidents.js"; +export { budgetResumeClaims } from "./budget_resume_claims.js"; export { agentConfigRevisions } from "./agent_config_revisions.js"; export { agentApiKeys } from "./agent_api_keys.js"; export { agentRuntimeState } from "./agent_runtime_state.js"; diff --git a/packages/shared/src/types/agent-lifecycle.ts b/packages/shared/src/types/agent-lifecycle.ts index fca9e2df1de..3ff9d17a2b7 100644 --- a/packages/shared/src/types/agent-lifecycle.ts +++ b/packages/shared/src/types/agent-lifecycle.ts @@ -25,6 +25,9 @@ export interface AgentLifecycleOperation { verificationFailures?: number; attempts: number; responsibleUserId?: string | null; + /** Why a pause started: its holds and the company's pause reason at that moment. Runs the + * pause stops record it, so a budget resume continues only work its own budget stopped. */ + pauseCause?: { holds: string[]; companyPauseReason: string | null }; } export interface AgentLifecycleRequest { diff --git a/server/src/__tests__/budget-enforcement-worker.test.ts b/server/src/__tests__/budget-enforcement-worker.test.ts index 73a21c18e55..8cde8c84d41 100644 --- a/server/src/__tests__/budget-enforcement-worker.test.ts +++ b/server/src/__tests__/budget-enforcement-worker.test.ts @@ -17,6 +17,10 @@ vi.mock("../services/budgets.js", () => ({ flushBudgetEvaluationWorkPage: work.evaluate, })); vi.mock("../services/budget-evaluation-work.js", () => ({ hasPendingBudgetEvaluation: work.evaluationPending })); +const resumeWork = vi.hoisted(() => ({ expiry: vi.fn(async (): Promise => null), + pendingForRun: vi.fn(async (_db: unknown, runId: string) => runId !== "run-unclaimed") })); +vi.mock("../services/budget-resume-continuation.js", () => ({ hasDeliverableBudgetResume: vi.fn(async () => false), + hasPendingBudgetResumeForRun: resumeWork.pendingForRun, nextBudgetResumeClaimExpiryAt: resumeWork.expiry })); vi.mock("../services/run-cost-accounting.js", () => ({ readCostAccountingWork: work.accounting })); const coordinators: ReturnType[] = []; @@ -49,6 +53,50 @@ function setup() { } describe("budget accounting delivery", () => { + // H1 review: budget-resume claims are delivered by the always-on budget + // worker, after the pass's Stop delivery, independent of scheduled agent work. + it("delivers budget-resume continuations after Stop delivery in each pass", async () => { + const s = setup(); + const order: string[] = []; + work.deliver.mockImplementation(async () => { order.push("stops"); }); + const deliverResumeContinuations = vi.fn(async () => { order.push("continuations"); }); + await registerBudgetEnforcementWork(s.db, s.coordinator, { deliverResumeContinuations }).ready; + expect(order).toEqual(["stops", "continuations"]); + }); + + it("wakes resume delivery when a run's Stop settles", async () => { + const s = setup(); + const deliverResumeContinuations = vi.fn(async () => undefined); + const worker = registerBudgetEnforcementWork(s.db, s.coordinator, { deliverResumeContinuations }); + await worker.ready; + expect(deliverResumeContinuations).toHaveBeenCalledTimes(1); + // Another tenant's cancellation without a resume claim wakes nothing. + publishLiveEvent({ companyId: "company-2", type: "heartbeat.run.status", payload: { runId: "run-unclaimed", status: "cancelled" } } as never); + await vi.advanceTimersByTimeAsync(1_500); + expect(deliverResumeContinuations).toHaveBeenCalledTimes(1); + publishLiveEvent({ companyId: "company-1", type: "heartbeat.run.status", payload: { runId: "run-1", status: "cancelled" } } as never); + await vi.advanceTimersByTimeAsync(1_500); + expect(deliverResumeContinuations).toHaveBeenCalledTimes(2); + // Stopping the worker removes the listener: later Stops wake nothing. + await worker.stop(); + publishLiveEvent({ companyId: "company-1", type: "heartbeat.run.status", payload: { runId: "run-2", status: "cancelled" } } as never); + await vi.advanceTimersByTimeAsync(1_500); + expect(deliverResumeContinuations).toHaveBeenCalledTimes(2); + }); + + it("wakes when a failed dispatcher's resume claim goes stale", async () => { + const s = setup(); + const expiry = Date.now() + 60_000; + resumeWork.expiry.mockResolvedValue(expiry); + const deliverResumeContinuations = vi.fn(async () => undefined); + await registerBudgetEnforcementWork(s.db, s.coordinator, { deliverResumeContinuations }).ready; + expect(deliverResumeContinuations).toHaveBeenCalledTimes(1); + expect(s.coordinator.nextWakeAt()).toBe(expiry); + resumeWork.expiry.mockResolvedValue(null); + await vi.advanceTimersByTimeAsync(60_000); + expect(deliverResumeContinuations).toHaveBeenCalledTimes(2); + }); + it("continues cancellation delivery and retries a failed startup reconciliation", async () => { const s = setup(); work.reconcile.mockResolvedValueOnce({ nextCursor: null, hadFailures: true }); diff --git a/server/src/app.ts b/server/src/app.ts index dba13b431fa..9bc926dd912 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -214,6 +214,7 @@ import { createPluginToolDispatcher } from "./services/plugin-tool-dispatcher.js import { createToolGatewayService } from "./services/tool-gateway.js"; import { toolAccessService } from "./services/tool-access.js"; import { chatChannelService } from "./services/chat-channels.js"; +import type { BudgetResumeDelivery } from "./services/budget-resume-continuation.js"; import { deliverNativeQuestionResponse } from "./services/native-runtime/native-question-bridge.js"; import { nativeChatProgressWork } from "./services/chat-native-progress-work.js"; import { chatInboundWakeNoticeWork } from "./services/chat-inbound-wake-notice-work.js"; @@ -448,6 +449,8 @@ export async function createApp( databaseBackupService?: InstanceDatabaseBackupService; prepareIdleDatabaseBackup?: () => Promise; idleDeadlineOwners?: IdleDeadlineOwners; + /** Filled with the chat service's Retry callback for background budget-resume delivery. */ + budgetResumeDelivery?: BudgetResumeDelivery; databaseBackupHealth?: InspectDatabaseBackupHealthOptions; deploymentMode: DeploymentMode; deploymentExposure: DeploymentExposure; @@ -590,6 +593,7 @@ export async function createApp( deliverNativeQuestionResponse(db, interaction), storage: opts.storageService, }); + if (opts.budgetResumeDelivery) opts.budgetResumeDelivery.prepareChatRetry = chatChannels.prepareFailedChatRunRetry; // Voice capabilities are verified by the signed callback route, not by // board/agent bearer authentication. Mount before actorMiddleware so its // per-session Bearer cannot be mistaken for an agent API key. diff --git a/server/src/index.ts b/server/src/index.ts index 7bfaf346c27..70af6d29a30 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -92,6 +92,7 @@ import { embeddedPostgresOwnerPort } from "./embedded-postgres-owner.js"; import { deliverExecutionStatuses } from "./services/execution-status-delivery.js"; import { deliverReconciledExecutions, settleUnrecoverableExecutions } from "./services/execution-recovery-resolution.js"; import { deliverPendingPauseResumes } from "./services/issue-tree-resume.js"; +import { deliverBudgetResumeContinuations, type BudgetResumeDelivery } from "./services/budget-resume-continuation.js"; import { invalidateNativeReplacementDiscovery, reconcileSafeNativeReplacements } from "./services/native-runtime/native-safe-replacement.js"; import { reconcileAbandonedExecutionControl } from "./services/execution-control-reconciliation.js"; import { acknowledgeNativeStartupCleanup } from "./services/native-runtime/native-startup-cleanup.js"; @@ -948,8 +949,11 @@ async function startServerWithDatabaseTeardown( logger.warn({ scope: cloudQualification.scope, blockerCodes: cloudQualification.blockers.map((blocker) => blocker.code) }, "Cloud configuration and package qualification has blockers"); } const idleDeadlineOwners: IdleDeadlineOwners = {}; + // createApp fills the chat Retry callback; the continuation worker reads it. + const budgetResumeDelivery: BudgetResumeDelivery = {}; const app = await createApp(db as any, { idleDeadlineOwners, + budgetResumeDelivery, prepareIdleDatabaseBackup: idleDatabaseBackupEnabled ? async () => (await runServerDatabaseBackup("idle")) !== null : undefined, @@ -1396,13 +1400,19 @@ async function startServerWithDatabaseTeardown( { queue: DELIVERY_QUEUES.executionContinuation, run: async () => { await deliverReconciledExecutions(db, heartbeat.wakeup); await deliverPendingPauseResumes(db, heartbeat.wakeup); + await deliverBudgetResumeContinuations(db, { ...budgetResumeDelivery, + resumeTaskRetry: (companyId, requestId) => heartbeat.resumeTaskRetry(companyId, requestId) }); }, next: executionSchedule.continuation }, ] : []), ], () => !heartbeatSchedulerStopped); const budgetEnforcementWorker = registerBudgetEnforcementWork(db, deliveryWork, { cancelWorkForScope: accountingHeartbeat.cancelBudgetScopeWork, + deliverResumeContinuations: () => deliverBudgetResumeContinuations(db, { ...budgetResumeDelivery, + resumeTaskRetry: (companyId, requestId) => accountingHeartbeat.resumeTaskRetry(companyId, requestId) }), }); + // Its stop also removes the run-status listener that wakes resume delivery. + server.on("close", () => { void budgetEnforcementWorker.stop(); }); const accountingWorker = registerCostAccountingWork(db, deliveryWork, accountingHeartbeat.reconcileCostAccounting); server.on("close", accountingWorker.stop); skillSourceSync.start(deliveryWork); diff --git a/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts b/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts index ea0b6ba6da7..2502fae80cf 100644 --- a/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts +++ b/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts @@ -371,8 +371,8 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu expect(await db.select().from(principalPermissionGrants).where(eq(principalPermissionGrants.companyId, companyId))).toHaveLength(0); }); - // H1 bug B: a pause's stop is told apart from termination by the cause the - // pause records on each run it stops. + // H1 review: a budget resume continues only work its own budget stopped. The + // pause records the holds and the company's pause reason on each run it stops. it("records what a pause stopped each active run for", async () => { const agent = await hire(); await db.update(companies).set({ status: "paused", pauseReason: "budget" }).where(eq(companies.id, companyId)); @@ -384,6 +384,46 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu expect(stopped.resultJson).toMatchObject({ pauseStop: { holds: ["company_paused"], companyPauseReason: "budget" } }); }); + // H1 review: a resume that removed the holds before the pause's Stop ran must + // not erase why the pause stopped the run (the Stop itself stays owed). + it("records the pause's original cause on runs it stops after a resume removed the holds", async () => { + const agent = await hire(); + await db.update(agents).set({ lifecycleState: "pausing", lifecycleHolds: [], + lifecycleOperation: { id: randomUUID(), hostComplete: false, completedPluginIds: [], attempts: 0, + pauseCause: { holds: ["budget"], companyPauseReason: null } } }).where(eq(agents.id, agent.id)); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: agent.id, status: "queued" }).returning(); + await createLifecycleDriver(db, {} as never).runHost(await current(agent.id) as never); + const [stopped] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)); + expect(stopped.status).toBe("cancelled"); + expect(stopped.resultJson).toMatchObject({ pauseStop: { holds: ["budget"], companyPauseReason: null } }); + }); + + it("keeps a budget pause's cause when the budget is resumed before the pause's Stop runs", async () => { + const agent = await hire(); + await db.update(agents).set({ lifecycleState: "ready", status: "idle" }).where(eq(agents.id, agent.id)); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: agent.id, status: "queued" }).returning(); + const { applyAgentBudgetHoldInTransaction } = await import("../../../services/agent-lifecycle.js"); + // The budget hold pauses the agent, then the raise removes it before the + // lifecycle worker has run the pause's Stop. + await db.transaction(tx => applyAgentBudgetHoldInTransaction(tx as never, companyId, agent.id, true)); + await db.transaction(tx => applyAgentBudgetHoldInTransaction(tx as never, companyId, agent.id, false)); + const paused = await current(agent.id); + expect(paused).toMatchObject({ lifecycleState: "pausing", lifecycleHolds: [] }); + await createLifecycleDriver(db, {} as never).runHost(paused as never); + const [stopped] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)); + expect(stopped.resultJson).toMatchObject({ pauseStop: { holds: ["budget"] } }); + }); + + it("records why a budget pause started on its lifecycle operation", async () => { + const agent = await hire(); + await db.update(agents).set({ lifecycleState: "ready", status: "idle" }).where(eq(agents.id, agent.id)); + await db.update(companies).set({ status: "paused", pauseReason: "budget" }).where(eq(companies.id, companyId)); + const { applyAgentBudgetHoldInTransaction } = await import("../../../services/agent-lifecycle.js"); + await db.transaction(tx => applyAgentBudgetHoldInTransaction(tx as never, companyId, agent.id, true)); + expect((await current(agent.id)).lifecycleOperation).toMatchObject({ + pauseCause: { holds: ["budget"], companyPauseReason: "budget" } }); + }); + it("runs the saved harness test and keeps a failed configuration out of ready", async () => { const driver = createLifecycleDriver(db, {} as never); const work = worker(driver.runHost); diff --git a/server/src/modules/agent-lifecycle/adapters/postgres.ts b/server/src/modules/agent-lifecycle/adapters/postgres.ts index 12da4ae5c7d..9c113b14569 100644 --- a/server/src/modules/agent-lifecycle/adapters/postgres.ts +++ b/server/src/modules/agent-lifecycle/adapters/postgres.ts @@ -147,6 +147,11 @@ async function changeLockedAgent(tx: Db, effects: LifecycleEffects, agent: Lifec : agent.lifecycleOperation?.resumeState, }; if (next === "ready") operation.resumeState = "ready"; + if (next === "pausing") { + const [company] = await tx.select({ pauseReason: companies.pauseReason }).from(companies) + .where(eq(companies.id, agent.companyId)); + operation.pauseCause = { holds, companyPauseReason: company?.pauseReason ?? null }; + } const status = compatibilityStatus(next, agent.status); if(!canConfigureAgentConnection(agent)&&canConfigureAgentConnection({lifecycleState:next,status}))await effects.beforeConnectionEligibilityRestore?.(tx,agent.companyId,agent.id); if (status !== agent.status) await effects.beforeStatusChange(tx, agent.companyId, agent.id, agent.status, status); diff --git a/server/src/routes/costs.ts b/server/src/routes/costs.ts index d599e008a90..ca55e577587 100644 --- a/server/src/routes/costs.ts +++ b/server/src/routes/costs.ts @@ -371,7 +371,8 @@ export function costRoutes( const companyId = req.params.companyId as string; const incidentId = req.params.incidentId as string; assertCompanyAccess(req, companyId); - const incident = await budgets.resolveIncident(companyId, incidentId, req.body, req.actor.userId ?? "board"); + const incident = await budgets.resolveIncident(companyId, incidentId, req.body, req.actor.userId ?? "board", + req.actor.source ?? null); res.json(incident); }, ); diff --git a/server/src/services/agent-lifecycle-driver.ts b/server/src/services/agent-lifecycle-driver.ts index f3147acd956..00cd438faf6 100644 --- a/server/src/services/agent-lifecycle-driver.ts +++ b/server/src/services/agent-lifecycle-driver.ts @@ -63,12 +63,16 @@ export function createLifecycleDriver(db: Db, manager: PluginWorkerManager): Lif await db.select().from(agents).where(eq(agents.companyId, agent.companyId)))]; const reason = agent.lifecycleHolds?.includes("company_archived") ? "Cancelled because the company was archived" : "Agent lifecycle stop requested"; - // A pause records why it stopped each run (pauseStop). It marks the - // stop as a pause, not termination or cleanup, which share this reason. + // Record what this pause stopped the runs for, so a budget resume + // continues only the work its own budget stopped, never a manual pause. if (agent.lifecycleState === "pausing") { + // The cause recorded when the pause started survives a resume that + // already removed the holds before this Stop ran. + const cause = agent.lifecycleOperation?.pauseCause; const pauseStop = sql`jsonb_build_object('pauseStop', jsonb_build_object( - 'holds', ${JSON.stringify(agent.lifecycleHolds ?? [])}::jsonb, - 'companyPauseReason', (select ${companies.pauseReason} from ${companies} where ${companies.id} = ${agent.companyId})))`; + 'holds', ${JSON.stringify(cause?.holds ?? agent.lifecycleHolds ?? [])}::jsonb, + 'companyPauseReason', ${cause ? sql`${cause.companyPauseReason}::text` + : sql`(select ${companies.pauseReason} from ${companies} where ${companies.id} = ${agent.companyId})`}))`; await db.update(heartbeatRuns).set({ resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${pauseStop}`, }).where(and(eq(heartbeatRuns.companyId, agent.companyId), eq(heartbeatRuns.agentId, agent.id), diff --git a/server/src/services/agent-lifecycle.ts b/server/src/services/agent-lifecycle.ts index a324b34b72f..f24dc30810a 100644 --- a/server/src/services/agent-lifecycle.ts +++ b/server/src/services/agent-lifecycle.ts @@ -1,4 +1,5 @@ import { notifyMuseContactVerification } from "../modules/external-agents/index.js"; +import { notifyBudgetResumeAgentReady } from "./budget-resume-continuation.js"; import { currentControllerProcessStamp } from "./controller-process-identity.js"; import { notifyStrandedAgentStatus } from "./live-run-recovery-notifications.js"; import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; @@ -68,6 +69,7 @@ export function createAgentLifecycleEffects(_hooks: BudgetServiceHooks = {}): li await notifyNativeResponsePresentation(tx, { companyId, agentId }); await notifyNativeReviewResponse(tx, { companyId, agentId }); await notifyChatInboundWakeNotices(tx, { companyId }); + await notifyBudgetResumeAgentReady(tx, agentId); } }, reportFailure(context, error) { diff --git a/server/src/services/budget-enforcement-worker.ts b/server/src/services/budget-enforcement-worker.ts index e5edb73609b..bd1414f6678 100644 --- a/server/src/services/budget-enforcement-worker.ts +++ b/server/src/services/budget-enforcement-worker.ts @@ -1,4 +1,5 @@ import type { Db } from "@paperclipai/db"; +import { logger } from "../middleware/logger.js"; import { withBudgetSpan } from "./budget-tracing.js"; import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; import { DELIVERY_QUEUES, subscribeDeliveryWork } from "./delivery-work-notifications.js"; @@ -11,6 +12,7 @@ import { type BudgetServiceHooks, } from "./budgets.js"; import { hasPendingBudgetEvaluation } from "./budget-evaluation-work.js"; +import { hasDeliverableBudgetResume, hasPendingBudgetResumeForRun, nextBudgetResumeClaimExpiryAt } from "./budget-resume-continuation.js"; import { readCostAccountingWork } from "./run-cost-accounting.js"; import { subscribeAllCompanyLiveEvents } from "./live-events.js"; @@ -32,7 +34,8 @@ export function registerBudgetEnforcementWork(db: Db, coordinator: Coordinator, let reconciling = false; let reconciliationFailed = false; let nextDeadline: number | null = null; - return coordinator.register(DELIVERY_QUEUES.budgetEnforcement, { + let resumeClaimDeadline: number | null = null; + const worker = coordinator.register(DELIVERY_QUEUES.budgetEnforcement, { retryMs: 1_000, coalesceMs: 1_000, runDuringDrain: true, @@ -50,16 +53,44 @@ export function registerBudgetEnforcementWork(db: Db, coordinator: Coordinator, evaluationCursor = evaluation.nextCursor; if (signal.aborted) return; await withBudgetSpan("budget.enforcement.deliver", () => deliverBudgetEnforcement(db, hooks)); + if (signal.aborted) return; + // Claims a resume saved, including runs whose Stop this pass just landed. + // Registered with the always-on budget worker, not scheduled agent work. + await hooks.deliverResumeContinuations?.().catch(error => + logger.warn({ err: error }, "Budget resume continuations remain pending")); }, async hasPending() { - const [pending, next, evaluation] = await Promise.all([ + const [pending, next, evaluation, resume, claimExpiry] = await Promise.all([ hasPendingBudgetEnforcement(db), nextBudgetReconciliationAt(db), hasPendingBudgetEvaluation(db), + // A deliverable resume claim (for example after a retryable refusal or + // a database error) keeps this worker on its retry schedule. + hooks.deliverResumeContinuations ? hasDeliverableBudgetResume(db) : Promise.resolve(false), + // A claim held by a dispatcher that failed becomes deliverable when it + // goes stale; wake for that moment without another budget signal. + hooks.deliverResumeContinuations ? nextBudgetResumeClaimExpiryAt(db) : Promise.resolve(null), ]); nextDeadline = next?.getTime() ?? null; - return pending || evaluation || reconciling || startup || evaluationCursor !== null; + resumeClaimDeadline = claimExpiry ?? null; + return pending || evaluation || resume || reconciling || startup || evaluationCursor !== null; }, - nextRunAt: () => nextDeadline, + nextRunAt: () => nextDeadline === null ? resumeClaimDeadline + : resumeClaimDeadline === null ? nextDeadline : Math.min(nextDeadline, resumeClaimDeadline), }); + if (!hooks.deliverResumeContinuations) return worker; + // A Stop that settles later (a pause's lifecycle Stop, a delivered budget + // Stop) makes a pending resume claim deliverable. The terminal status event + // is published after its persisted transition; wake for it, so delivery + // never waits for unrelated budget work. + // Only a run that carries a pending resume claim wakes the worker: one + // primary-key lookup, so other cancellations never start an instance pass. + const unsubscribe = subscribeAllCompanyLiveEvents(event => { + if (event.type !== "heartbeat.run.status" || String(event.payload.status) !== "cancelled") return; + const runId = event.payload.runId; + if (typeof runId !== "string") return; + void hasPendingBudgetResumeForRun(db, runId).then(pending => { if (pending) worker.wake(); }) + .catch(error => logger.warn({ err: error, runId }, "Budget resume wake check failed")); + }); + return { ...worker, stop: async () => { unsubscribe(); await worker.stop(); } }; } /** Accounting repairs admitted work and never dispatches another paid turn. diff --git a/server/src/services/budget-resume-continuation.test.ts b/server/src/services/budget-resume-continuation.test.ts new file mode 100644 index 00000000000..94d758a8185 --- /dev/null +++ b/server/src/services/budget-resume-continuation.test.ts @@ -0,0 +1,353 @@ +import { randomUUID } from "node:crypto"; +import { and, eq, like } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { + agents, agentWakeupRequests, budgetIncidents, budgetPolicies, budgetResumeClaims, companies, createDb, heartbeatRuns, issues, +} from "@paperclipai/db"; +import { budgetService } from "./budgets.js"; +import { heartbeatService } from "./heartbeat.js"; +import { HttpError } from "../errors.js"; +import { + deliverBudgetResumeContinuations, hasDeliverableBudgetResume, hasPendingBudgetResume, markBudgetCancelledWorkForResume, +} from "./budget-resume-continuation.js"; +import { + EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, +} from "../__tests__/helpers/embedded-postgres.js"; + +const mockLogActivity = vi.hoisted(() => vi.fn()); +vi.mock("./activity-log.js", async (importOriginal) => ({ + ...await importOriginal(), + logActivity: mockLogActivity, +})); + +const support = await getEmbeddedPostgresTestSupport(); + +// H1 (restart chaos): a budget hard stop cancelled a run mid-task. After the +// board raised the budget and resumed, the task stayed stopped until someone +// clicked Retry. Resuming must continue that work, exactly once. +describe.skipIf(!support.supported)("resuming after a budget hard stop", () => { + let db!: ReturnType; + let tempDb: Awaited> | null = null; + + beforeAll(async () => { + await import("./agent-lifecycle.js"); + tempDb = await startEmbeddedPostgresTestDatabase("paperclip-budget-resume-continuation-"); + db = createDb(tempDb.connectionString); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + afterAll(async () => { + await db?.$client.end(); + await tempDb?.cleanup(); + }); + + async function seedScope(label: string, scopeType: "company" | "agent" = "company") { + const [company] = await db.insert(companies).values({ name: `Budget ${label}`, + issuePrefix: `B${randomUUID().slice(0, 4).toUpperCase()}` }).returning(); + const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Worker" }).returning(); + const scopeId = scopeType === "agent" ? agent.id : company.id; + const [policy] = await db.insert(budgetPolicies).values({ companyId: company.id, scopeType, + scopeId, windowKind: "calendar_month_utc", amount: 1, hardStopEnabled: true, isActive: true }).returning(); + const [incident] = await db.insert(budgetIncidents).values({ companyId: company.id, policyId: policy.id, + scopeType, scopeId, metric: "billed_cents", windowKind: "calendar_month_utc", + windowStart: new Date(Date.now() - 86_400_000), windowEnd: new Date(Date.now() + 86_400_000), + thresholdType: "hard", amountLimit: 1, amountObserved: 0, createdAt: new Date(Date.now() - 60_000) }).returning(); + return { company, agent, policy, incident }; + } + + async function seedTask(scope: Awaited>, options: { + budgetStop?: boolean; lifecycleStop?: "budget" | "agentBudget" | "manual"; issueStatus?: string; runStatus?: string; laterRun?: boolean; + } = {}) { + const { company, agent, policy } = scope; + const [issue] = await db.insert(issues).values({ companyId: company.id, title: "Long task", + status: options.issueStatus ?? "in_progress", assigneeAgentId: agent.id }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, + issueId: issue.id, status: options.runStatus ?? "cancelled", runtimeMode: "legacy", + contextSnapshot: { issueId: issue.id }, createdAt: new Date(Date.now() - 50_000), + resultJson: options.lifecycleStop ? { + // H1: the agent's budget hold paused it, and the lifecycle stop cancelled the run. + cancellation: { reason: "Agent lifecycle stop requested", source: "control_plane", expected: true, + initiator: { type: "system" } }, + pauseStop: options.lifecycleStop === "budget" ? { holds: ["company_paused"], companyPauseReason: "budget" } + : options.lifecycleStop === "agentBudget" ? { holds: ["budget"], companyPauseReason: null } + : { holds: ["manual"], companyPauseReason: null }, + startupCancellation: { requestedAt: new Date(Date.now() - 40_000).toISOString(), errorCode: "cancelled" }, + } : options.budgetStop === false ? { stopReason: "cancelled" } : { startupCancellation: { + requestedAt: new Date(Date.now() - 40_000).toISOString(), errorCode: "cancelled", + budgetEnforcement: { companyId: company.id, scopeType: "company", scopeId: company.id, + policyId: policy.id, version: 1 } } }, + }).returning(); + if (options.laterRun) await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, + issueId: issue.id, status: "succeeded", runtimeMode: "legacy", contextSnapshot: { issueId: issue.id }, + createdAt: new Date(Date.now() - 20_000) }); + return { issue, run }; + } + + const resume = (scope: Awaited>) => budgetService(db).resolveIncident( + scope.company.id, scope.incident.id, { action: "raise_budget_and_resume", amount: 500 }, "local-board", + "local_implicit"); + const continuations = (companyId: string) => db.select().from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.companyId, companyId), like(agentWakeupRequests.idempotencyKey, "task-retry:budget-resume:%"))); + const claimOf = async (runId: string) => (await db.select().from(budgetResumeClaims) + .where(eq(budgetResumeClaims.runId, runId)))[0] as Record | undefined; + + it("continues each task the stop cancelled exactly once, through one continuation wake", async () => { + const scope = await seedScope("continue"); + const working = await seedTask(scope); + const blocked = await seedTask(scope, { issueStatus: "blocked" }); + await resume(scope); + expect(await claimOf(working.run.id)).toMatchObject({ state: "pending", incidentId: scope.incident.id, + actorUserId: "local-board", actorSource: "local_implicit" }); + expect(await hasPendingBudgetResume(db)).toBe(true); + await deliverBudgetResumeContinuations(db); + const saved = await continuations(scope.company.id); + expect(saved.map(wake => wake.payload?.issueId).sort()).toEqual([working.issue.id, blocked.issue.id].sort()); + expect(saved.every(wake => wake.reason === "task_retry" && wake.requestedByActorId === "local-board")).toBe(true); + expect(await claimOf(working.run.id)).toMatchObject({ state: "delivered", requestId: expect.any(String) }); + expect(await hasPendingBudgetResume(db)).toBe(false); + expect(mockLogActivity).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ + action: "budget.cancelled_work_continued", entityId: working.issue.id, actorId: "local-board" }), expect.anything()); + // Delivering or resolving again files nothing new. + await deliverBudgetResumeContinuations(db); + await markBudgetCancelledWorkForResume(db, { companyId: scope.company.id, scopeType: "company", + scopeId: scope.company.id, incidentId: scope.incident.id, incidentOpenedAt: scope.incident.createdAt, + actorUserId: "local-board" }); + await deliverBudgetResumeContinuations(db); + expect(await continuations(scope.company.id)).toHaveLength(2); + }); + + it("continues a run whose stop was still being delivered when the board resumed", async () => { + const scope = await seedScope("in-flight"); + const task = await seedTask(scope, { runStatus: "running" }); + await resume(scope); + await deliverBudgetResumeContinuations(db); + expect(await continuations(scope.company.id)).toEqual([]); + expect(await claimOf(task.run.id)).toMatchObject({ state: "pending" }); + // The committed Stop lands after the resume. Its terminal write replaces + // the run's result JSON with a copy built before the claim was saved; the + // claim lives outside that JSON, so it survives. + await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date(), + resultJson: { ...(task.run.resultJson ?? {}), stopReason: "cancelled" } }).where(eq(heartbeatRuns.id, task.run.id)); + await deliverBudgetResumeContinuations(db); + expect(await continuations(scope.company.id)).toEqual([expect.objectContaining({ + payload: expect.objectContaining({ issueId: task.issue.id }) })]); + }); + + it("dispatches the continuation through the real Retry path into a queued run", async () => { + const scope = await seedScope("dispatch"); + const task = await seedTask(scope, { issueStatus: "blocked" }); + // Keep the new run queued: one run slot, already taken. + await db.update(agents).set({ runtimeConfig: { heartbeat: { maxConcurrentRuns: 1 } } }).where(eq(agents.id, scope.agent.id)); + await db.insert(heartbeatRuns).values({ companyId: scope.company.id, agentId: scope.agent.id, status: "running" }); + await resume(scope); + const heartbeat = heartbeatService(db); + await deliverBudgetResumeContinuations(db, { + resumeTaskRetry: (companyId, requestId) => heartbeat.resumeTaskRetry(companyId, requestId) }); + const queued = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, scope.company.id), + eq(heartbeatRuns.status, "queued"))); + expect(queued, JSON.stringify(await db.select({ status: agentWakeupRequests.status, error: agentWakeupRequests.error, + reason: agentWakeupRequests.reason, payload: agentWakeupRequests.payload }).from(agentWakeupRequests) + .where(eq(agentWakeupRequests.companyId, scope.company.id)))).toHaveLength(1); + expect(queued[0]).toMatchObject({ agentId: scope.agent.id }); + expect(queued[0].contextSnapshot).toMatchObject({ issueId: task.issue.id }); + }); + + it("continues every task of a large scope", async () => { + const scope = await seedScope("large"); + for (let index = 0; index < 30; index += 1) await seedTask(scope); + await resume(scope); + await deliverBudgetResumeContinuations(db); + expect(await continuations(scope.company.id)).toHaveLength(30); + }); + + it.each([ + ["work that already continued", { laterRun: true }, "work_continued_or_task_changed"], + ["a task that is now done", { issueStatus: "done" }, "work_continued_or_task_changed"], + ] as const)("does not continue %s", async (label, options, reason) => { + const scope = await seedScope(label.replace(/\W+/g, "-")); + const task = await seedTask(scope, options); + await resume(scope); + await deliverBudgetResumeContinuations(db); + expect(await continuations(scope.company.id)).toEqual([]); + expect(await claimOf(task.run.id)).toMatchObject({ state: "abandoned", reason }); + }); + + it("abandons a claim whose run finished before its stop landed", async () => { + const scope = await seedScope("finished-first"); + const task = await seedTask(scope, { runStatus: "running" }); + await resume(scope); + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, task.run.id)); + await deliverBudgetResumeContinuations(db); + expect(await continuations(scope.company.id)).toEqual([]); + expect(await claimOf(task.run.id)).toMatchObject({ state: "abandoned", reason: "run_not_cancelled" }); + }); + + it("continues a run the agent's budget pause stopped through the lifecycle stop", async () => { + const scope = await seedScope("lifecycle-stop"); + const task = await seedTask(scope, { lifecycleStop: "budget", issueStatus: "in_progress" }); + await resume(scope); + expect(await claimOf(task.run.id)).toMatchObject({ state: "pending" }); + await deliverBudgetResumeContinuations(db); + expect(await continuations(scope.company.id)).toEqual([expect.objectContaining({ + payload: expect.objectContaining({ issueId: task.issue.id }) })]); + }); + + it("continues a run an agent-scope budget pause stopped through the lifecycle stop", async () => { + const scope = await seedScope("agent-lifecycle-stop", "agent"); + const task = await seedTask(scope, { lifecycleStop: "agentBudget" }); + await resume(scope); + await deliverBudgetResumeContinuations(db); + expect(await continuations(scope.company.id)).toEqual([expect.objectContaining({ + payload: expect.objectContaining({ issueId: task.issue.id }) })]); + }); + + it("continues a run whose budget pause Stop lands after the resume", async () => { + const scope = await seedScope("late-pause-stop", "agent"); + // The budget pause started (its cause is on the lifecycle operation), but + // the lifecycle worker has not stopped the run yet when the board resumes. + await db.update(agents).set({ lifecycleState: "pausing", lifecycleHolds: [], + lifecycleOperation: { id: randomUUID(), hostComplete: false, completedPluginIds: [], attempts: 0, + pauseCause: { holds: ["budget"], companyPauseReason: null } } }).where(eq(agents.id, scope.agent.id)); + const task = await seedTask(scope, { budgetStop: false, runStatus: "running" }); + await resume(scope); + expect(await claimOf(task.run.id)).toMatchObject({ state: "pending" }); + await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date(), resultJson: { + cancellation: { reason: "Agent lifecycle stop requested", source: "control_plane", expected: true, initiator: { type: "system" } }, + pauseStop: { holds: ["budget"], companyPauseReason: null } } }).where(eq(heartbeatRuns.id, task.run.id)); + await deliverBudgetResumeContinuations(db); + expect(await continuations(scope.company.id)).toHaveLength(1); + }); + + it("does not continue a run a person stopped after the resume claimed it", async () => { + const scope = await seedScope("later-person-stop"); + const task = await seedTask(scope, { runStatus: "running" }); + await resume(scope); + expect(await claimOf(task.run.id)).toMatchObject({ state: "pending" }); + await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date(), resultJson: { + ...(task.run.resultJson ?? {}), + cancellation: { reason: "Stopped by board", source: "operator", expected: true, initiator: { type: "user", id: "board-user" } } }, + }).where(eq(heartbeatRuns.id, task.run.id)); + await deliverBudgetResumeContinuations(db); + expect(await continuations(scope.company.id)).toEqual([]); + expect(await claimOf(task.run.id)).toMatchObject({ state: "abandoned", reason: "stopped_by_person" }); + }); + + it("releases the claim on a retryable Retry refusal, and settles it on a permanent one", async () => { + const scope = await seedScope("retry-refusals"); + const task = await seedTask(scope); + await resume(scope); + const busy = vi.fn(async () => { throw new HttpError(409, "busy", { code: "task_retry_busy", retryable: true }); }); + await deliverBudgetResumeContinuations(db, { requestRetry: busy as never }); + expect(busy).toHaveBeenCalledOnce(); + expect(await claimOf(task.run.id)).toMatchObject({ state: "pending", claimedAt: null }); + const refused = vi.fn(async () => { throw new HttpError(409, "no", { code: "chat_failed_run_retry_not_authorized" }); }); + await deliverBudgetResumeContinuations(db, { requestRetry: refused as never }); + expect(await claimOf(task.run.id)).toMatchObject({ state: "abandoned", reason: "chat_failed_run_retry_not_authorized" }); + }); + + async function admissionStopped(label: string, cancellationReason: string) { + const scope = await seedScope(label); + const task = await seedTask(scope, { budgetStop: false }); + const reason = "Company cannot start work because its budget hard-stop is still exceeded."; + await db.update(heartbeatRuns).set({ resultJson: { stopReason: "cancelled", + cancellation: { reason: cancellationReason, source: "control_plane", expected: true, initiator: { type: "system" } }, + startupCancellation: { requestedAt: new Date(Date.now() - 30_000).toISOString(), errorCode: "cancelled" }, + budgetAdmissionStop: { scopeType: "company", scopeId: scope.company.id, reason } } }).where(eq(heartbeatRuns.id, task.run.id)); + return { scope, task, reason }; + } + + it("continues a queued run that admission cancelled for this budget", async () => { + const { scope, task } = await admissionStopped("admission-stop", + "Company cannot start work because its budget hard-stop is still exceeded."); + await resume(scope); + await deliverBudgetResumeContinuations(db); + expect(await continuations(scope.company.id)).toEqual([expect.objectContaining({ + payload: expect.objectContaining({ issueId: task.issue.id }) })]); + }); + + it("never claims an admission-stamped run that another reason cancelled", async () => { + // Admission saves its stamp before it cancels. A crash between them leaves + // a queued, stamped run that may later be cancelled for something else. + const { scope, task } = await admissionStopped("admission-other", "Cancelled because the agent no longer exists"); + await resume(scope); + expect(await claimOf(task.run.id)).toBeUndefined(); + }); + + it("claims a queued run admission stamped, then follows how it was cancelled", async () => { + // Admission saved its stamp, and the resume ran before its cancel landed. + const reason = "Company cannot start work because its budget hard-stop is still exceeded."; + for (const [label, cancelledFor, expected] of [ + ["admission-pending", reason, "delivered"], + ["admission-pending-other", "Cancelled because the agent no longer exists", "abandoned"], + ] as const) { + const scope = await seedScope(label); + const task = await seedTask(scope, { budgetStop: false, runStatus: "queued" }); + await db.update(heartbeatRuns).set({ resultJson: { + budgetAdmissionStop: { scopeType: "company", scopeId: scope.company.id, reason } } }).where(eq(heartbeatRuns.id, task.run.id)); + await resume(scope); + expect(await claimOf(task.run.id)).toMatchObject({ state: "pending" }); + await db.update(heartbeatRuns).set({ status: "cancelled", resultJson: { + budgetAdmissionStop: { scopeType: "company", scopeId: scope.company.id, reason }, + cancellation: { reason: cancelledFor, source: "control_plane", expected: true, initiator: { type: "system" } } } }) + .where(eq(heartbeatRuns.id, task.run.id)); + await deliverBudgetResumeContinuations(db); + expect(await claimOf(task.run.id)).toMatchObject({ state: expected, + ...(expected === "abandoned" ? { reason: "cancelled_for_another_reason" } : {}) }); + } + }); + + it("releases a stale claim while its agent is paused, so no pass is scheduled for it", async () => { + const scope = await seedScope("stale-paused"); + const task = await seedTask(scope); + await resume(scope); + await db.update(budgetResumeClaims).set({ claimedAt: new Date(Date.now() - 120_000) }) + .where(eq(budgetResumeClaims.runId, task.run.id)); + await db.update(agents).set({ status: "paused" }).where(eq(agents.id, scope.agent.id)); + await deliverBudgetResumeContinuations(db); + expect(await claimOf(task.run.id)).toMatchObject({ state: "pending", claimedAt: null }); + await db.update(agents).set({ status: "idle" }).where(eq(agents.id, scope.agent.id)); + await deliverBudgetResumeContinuations(db); + expect(await claimOf(task.run.id)).toMatchObject({ state: "delivered" }); + }); + + it("waits for a paused agent to resume before filing its Retry", async () => { + // A connected chat refuses a Retry while the resume is still lifting the + // agent's hold; that refusal must not abandon the continuation. + const scope = await seedScope("agent-resuming"); + const task = await seedTask(scope); + await db.update(agents).set({ status: "paused" }).where(eq(agents.id, scope.agent.id)); + await resume(scope); + await db.update(agents).set({ status: "paused" }).where(eq(agents.id, scope.agent.id)); + const requestRetry = vi.fn(async () => ({ accepted: true, requestId: null })); + await deliverBudgetResumeContinuations(db, { requestRetry: requestRetry as never }); + expect(requestRetry).not.toHaveBeenCalled(); + expect(await hasDeliverableBudgetResume(db)).toBe(false); + expect(await claimOf(task.run.id)).toMatchObject({ state: "pending", claimedAt: null }); + await db.update(agents).set({ status: "idle" }).where(eq(agents.id, scope.agent.id)); + expect(await hasDeliverableBudgetResume(db)).toBe(true); + await deliverBudgetResumeContinuations(db, { requestRetry: requestRetry as never }); + expect(requestRetry).toHaveBeenCalledOnce(); + }); + + it("never claims a run a manual agent pause stopped", async () => { + const scope = await seedScope("manual-pause"); + const task = await seedTask(scope, { lifecycleStop: "manual" }); + await resume(scope); + expect(await claimOf(task.run.id)).toBeUndefined(); + }); + + it("never claims a run a person stopped while budget enforcement also fenced it", async () => { + const scope = await seedScope("person-and-budget"); + const task = await seedTask(scope); + await db.update(heartbeatRuns).set({ resultJson: { ...(task.run.resultJson ?? {}), + cancellation: { reason: "Stopped by board", source: "operator", expected: true, initiator: { type: "user", id: "board-user" } }, + cancelledByActorType: "user" } }).where(eq(heartbeatRuns.id, task.run.id)); + await resume(scope); + expect(await claimOf(task.run.id)).toBeUndefined(); + }); + + it("never claims a run a person stopped", async () => { + const scope = await seedScope("person-stop"); + const task = await seedTask(scope, { budgetStop: false }); + await resume(scope); + expect(await claimOf(task.run.id)).toBeUndefined(); + }); +}); diff --git a/server/src/services/budget-resume-continuation.ts b/server/src/services/budget-resume-continuation.ts new file mode 100644 index 00000000000..94e43d206cf --- /dev/null +++ b/server/src/services/budget-resume-continuation.ts @@ -0,0 +1,352 @@ +import { and, asc, eq, gt, inArray, isNull, lte, notInArray, or, sql } from "drizzle-orm"; +import { agents, budgetResumeClaims, heartbeatRuns, type Db } from "@paperclipai/db"; +import { HttpError } from "../errors.js"; +import { logger } from "../middleware/logger.js"; +import { logActivity, publishActivity, type ActivityPublication } from "./activity-log.js"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; +import { budgetCancellationScopePredicate } from "./heartbeat/budget-cancellation.js"; +import { AGENT_LIFECYCLE_STOP_REASON } from "./governance-pause-stop.js"; + +/** + * Resuming after a budget hard stop continues the work that the stop + * cancelled, exactly once. + * + * When a board member resolves the incident with Raise budget and resume, + * `markBudgetCancelledWorkForResume` saves a pending `budget_resume_claims` row + * (the board member and their session source) for every task run this + * scope's stop cancelled, before the incident closes. That includes runs whose + * stop has not been delivered yet. Claims live outside the run's result JSON, + * which terminal writers replace, so a Stop settling at the same moment cannot + * erase one. The budget-enforcement worker (and the execution-continuation + * worker as a backstop) calls `deliverBudgetResumeContinuations`: once a run's + * stop has settled (`cancelled`), it files that board member's Retry. + * - The Retry path clears the stopped task's execution hold, and accepts only + * while the cancelled run is still the task's latest run and the task is + * open and assigned, checked under the task lock. Work that already + * continued or completed is never repeated. + * - Its idempotency key names the cancelled run, so every pass, restart or + * repeated resume returns the same single request. + * - The claim and its activity entry settle in one commit. + * - A Retry starts a new run through normal admission; nothing charges the + * cancelled run's provider attempt or failure budget. + */ + +/** A claim older than this with no settled outcome belongs to an interrupted dispatcher. */ +export const BUDGET_RESUME_CLAIM_STALE_MS = 60_000; +const PAGE_SIZE = 25; +const ACTIVE_RUN_STATUSES = ["queued", "running", "scheduled_retry"] as const; +/** Agent statuses that cannot take a Retry yet but may later: a resume still + * lifting the agent's hold, or an approval. Their claims wait, unclaimed, and + * the agent's status change wakes delivery (`notifyBudgetResumeAgentReady`). */ +const WAITING_AGENT_STATUSES = ["paused", "pending_approval"] as const; + +type BudgetResumeClaim = { + incidentId: string; actorUserId: string; actorSource: string | null; claimedAt: Date | null; +}; + +type PrepareChatRetry = NonNullable[0]["prepareChatRetry"]>; + +export type BudgetResumeDelivery = { + /** The chat service's Retry callback, which connected conversations need. */ + prepareChatRetry?: PrepareChatRetry; + /** Optional latency nudge; the deferred-receipt worker also dispatches the Retry. */ + resumeTaskRetry?: (companyId: string, requestId: string) => Promise; + /** The Retry path; defaults to requestTaskRetry. Tests substitute refusals. */ + requestRetry?: typeof import("./task-retry.js").requestTaskRetry; +}; + +const runIssueId = sql`coalesce(${heartbeatRuns.issueId}::text, ${heartbeatRuns.nativeIssueId}::text, + ${heartbeatRuns.contextSnapshot}->>'issueId')`; + +/** + * Stamp every task run this scope's budget stop cancelled since the incident + * opened: runs the budget enforcement fenced, and runs the agent lifecycle + * pause stopped for this budget (the run's `pauseStop` names the agent's + * budget hold, or the company's budget pause). Runs that already carry a claim + * keep it, so resolving again is a no-op. Arms the continuation worker in the + * same commit. Two indexed passes: cancelled runs, and runs whose stop is + * still being delivered. + */ +export async function markBudgetCancelledWorkForResume(db: Db, input: { + companyId: string; scopeType: "company" | "agent" | "project"; scopeId: string; + incidentId: string; incidentOpenedAt: Date; actorUserId: string; actorSource?: string | null; +}): Promise { + return db.transaction(async tx => { + // Register the work signals before the write they announce: the budget + // worker delivers claims; the execution-continuation worker is a backstop. + await notifyDeliveryWork(tx, DELIVERY_QUEUES.budgetEnforcement, input.companyId); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionContinuation); + const openedAt = input.incidentOpenedAt.toISOString(); + // The pause records the holds it stopped the run for (pauseStop). Only a + // stop this budget caused qualifies: the agent's own budget hold, or the + // company's budget pause. A manual pause never does. The holds come from + // the lifecycle operation's pauseCause, saved when the pause started, so a + // resume that removed the holds before the Stop ran still leaves them here. + const pauseStop = sql`${heartbeatRuns.resultJson}->'pauseStop'`; + const lifecycleStop = input.scopeType === "project" ? sql`false` : and( + sql`${heartbeatRuns.resultJson}->'cancellation'->>'reason' = ${AGENT_LIFECYCLE_STOP_REASON}`, + sql`${heartbeatRuns.resultJson}->'cancellation'->'initiator'->>'type' = 'system'`, + input.scopeType === "agent" + ? and(eq(heartbeatRuns.agentId, input.scopeId), sql`(${pauseStop}->'holds') ? 'budget'`) + : sql`(${pauseStop}->'holds') ? 'company_paused' and ${pauseStop}->>'companyPauseReason' = 'budget'`)!; + // A run still active on an agent whose pause this budget caused: the + // pause's Stop may land after the incident closes. The lifecycle operation + // recorded the cause when the pause started (pauseCause). + const pauseCause = sql`a.lifecycle_operation->'pauseCause'`; + const pendingPauseStop = input.scopeType === "project" ? sql`false` : sql`exists (select 1 from agents a + where a.id = ${heartbeatRuns.agentId} and a.company_id = ${heartbeatRuns.companyId} + and a.lifecycle_state = 'pausing' + and ${input.scopeType === "agent" + ? sql`a.id = ${input.scopeId} and (${pauseCause}->'holds') ? 'budget'` + : sql`(${pauseCause}->'holds') ? 'company_paused' and ${pauseCause}->>'companyPauseReason' = 'budget'`})`; + // A queued run that admission cancelled for this budget's board-owned block. + // Admission saves the stamp before it cancels; the recorded cancellation + // must carry the stamp's reason, so a stamped run that later ran, or was + // cancelled for another reason, never matches. + const admissionStop = and( + sql`${heartbeatRuns.resultJson}->'budgetAdmissionStop'->>'scopeType' = ${input.scopeType}`, + sql`${heartbeatRuns.resultJson}->'budgetAdmissionStop'->>'scopeId' = ${input.scopeId}`, + sql`${heartbeatRuns.resultJson}->'cancellation'->>'reason' = ${heartbeatRuns.resultJson}->'budgetAdmissionStop'->>'reason'`)!; + const stoppedByThisScope = or(budgetCancellationScopePredicate(input), lifecycleStop, admissionStop)!; + // A queued run admission stamped for this budget but has not cancelled + // yet. Delivery waits for its Stop, and abandons the claim if the run was + // cancelled for another reason. + const pendingAdmissionStop = and( + sql`${heartbeatRuns.resultJson}->'budgetAdmissionStop'->>'scopeType' = ${input.scopeType}`, + sql`${heartbeatRuns.resultJson}->'budgetAdmissionStop'->>'scopeId' = ${input.scopeId}`)!; + // A person's Stop stays stopped, even if budget enforcement also fenced + // the run while that Stop was being delivered. + const stoppedByPerson = sql`(coalesce(${heartbeatRuns.resultJson}->'cancellation'->'initiator'->>'type', '') = 'user' + or coalesce(${heartbeatRuns.resultJson}->'cancellation'->>'source', '') = 'operator' + or coalesce(${heartbeatRuns.resultJson}->>'cancelledByActorType', '') in ('user', 'board'))`; + const claimable = and( + eq(heartbeatRuns.companyId, input.companyId), + sql`not ${stoppedByPerson}`, + sql`${runIssueId} is not null`, + ); + const stoppedSinceIncident = and(stoppedByThisScope, + sql`coalesce((${heartbeatRuns.resultJson}->'startupCancellation'->>'requestedAt')::timestamptz, + ${heartbeatRuns.updatedAt}) >= ${openedAt}::timestamptz`)!; + // Both passes lead with (company_id, status), which the company status + // indexes serve. A run that finished any other way is owed nothing. An + // existing claim is kept, so resolving again saves nothing new. + const claim = (status: ReturnType) => tx.insert(budgetResumeClaims).select( + tx.select({ runId: heartbeatRuns.id, companyId: heartbeatRuns.companyId, + incidentId: sql`${input.incidentId}::uuid`.as("incident_id"), + actorUserId: sql`${input.actorUserId}`.as("actor_user_id"), + actorSource: sql`${input.actorSource ?? null}::text`.as("actor_source"), + state: sql`'pending'`.as("state"), + claimedAt: sql`null::timestamptz`.as("claimed_at"), + settledAt: sql`null::timestamptz`.as("settled_at"), + reason: sql`null::text`.as("reason"), + requestId: sql`null::uuid`.as("request_id"), + createdAt: sql`now()`.as("created_at"), + }).from(heartbeatRuns).where(and(claimable, status)), + ).onConflictDoNothing().returning({ id: budgetResumeClaims.runId }); + const finished = await claim(and(eq(heartbeatRuns.status, "cancelled"), stoppedSinceIncident) as ReturnType); + // Active runs are being stopped now: fenced by this budget's enforcement, + // or on an agent whose budget pause has not delivered its Stop yet. + const active = await claim(and(inArray(heartbeatRuns.status, [...ACTIVE_RUN_STATUSES]), + or(stoppedSinceIncident, pendingPauseStop, pendingAdmissionStop)) as ReturnType); + return [...finished, ...active].map(row => row.id); + }); +} + +/** Compare-and-set the run's claim. Only the writer that wins dispatches. */ +async function setClaim(db: Pick, input: { + runId: string; expectedClaimedAt: Date | null; + next: Partial>; +}) { + const [row] = await db.update(budgetResumeClaims).set(input.next).where(and( + eq(budgetResumeClaims.runId, input.runId), eq(budgetResumeClaims.state, "pending"), + input.expectedClaimedAt === null ? isNull(budgetResumeClaims.claimedAt) + : sql`date_trunc('milliseconds', ${budgetResumeClaims.claimedAt}) = ${input.expectedClaimedAt.toISOString()}::timestamptz`, + )).returning({ id: budgetResumeClaims.runId }); + return Boolean(row); +} + +/** True while a resumed budget stop still owes a continuation. */ +export async function hasPendingBudgetResume(db: Db) { + const [row] = await db.select({ id: budgetResumeClaims.runId }).from(budgetResumeClaims) + .where(eq(budgetResumeClaims.state, "pending")).limit(1); + return Boolean(row); +} + +/** True when this run carries a pending resume claim (a primary-key lookup). */ +export async function hasPendingBudgetResumeForRun(db: Db, runId: string) { + const [row] = await db.select({ id: budgetResumeClaims.runId }).from(budgetResumeClaims) + .where(and(eq(budgetResumeClaims.runId, runId), eq(budgetResumeClaims.state, "pending"))).limit(1); + return Boolean(row); +} + +/** True while a claim can be delivered now: its stop settled and no live + * dispatcher holds it. A claim waiting for its Stop re-arms with that Stop. */ +export async function hasDeliverableBudgetResume(db: Db, now = new Date()) { + const [row] = await db.select({ id: budgetResumeClaims.runId }).from(budgetResumeClaims) + .innerJoin(heartbeatRuns, eq(heartbeatRuns.id, budgetResumeClaims.runId)) + .innerJoin(agents, eq(agents.id, heartbeatRuns.agentId)) + .where(and(eq(budgetResumeClaims.state, "pending"), + notInArray(heartbeatRuns.status, [...ACTIVE_RUN_STATUSES]), + notInArray(agents.status, [...WAITING_AGENT_STATUSES]), + or(isNull(budgetResumeClaims.claimedAt), + lte(budgetResumeClaims.claimedAt, new Date(now.getTime() - BUDGET_RESUME_CLAIM_STALE_MS))))).limit(1); + return Boolean(row); +} + +/** When the earliest claimed, undelivered claim goes stale and becomes + * deliverable again, so a worker can wake for it without another signal. */ +export async function nextBudgetResumeClaimExpiryAt(db: Db): Promise { + const [row] = await db.select({ at: sql`min(${budgetResumeClaims.claimedAt})` }).from(budgetResumeClaims) + .where(and(eq(budgetResumeClaims.state, "pending"), sql`${budgetResumeClaims.claimedAt} is not null`)); + const claimedAt = row?.at ? new Date(row.at).getTime() : null; + return claimedAt === null ? null : claimedAt + BUDGET_RESUME_CLAIM_STALE_MS; +} + +type PendingRow = { + id: string; companyId: string; agentId: string; agentStatus: string; status: string; createdAt: Date; + issueId: string | null; stoppedByPerson: boolean; otherCancellation: boolean; claim: BudgetResumeClaim | null; +}; + +/** + * Deliver every pending claim whose stop has settled. Pages through all + * pending claims; a claim that waits (stop not settled, or a live + * dispatcher's fresh claim) is skipped without a write. + */ +export async function deliverBudgetResumeContinuations(db: Db, delivery: BudgetResumeDelivery = {}, now = new Date()) { + let after: string | null = null; + for (;;) { + const page = await db.select({ + id: heartbeatRuns.id, companyId: heartbeatRuns.companyId, agentId: heartbeatRuns.agentId, + agentStatus: agents.status, status: heartbeatRuns.status, createdAt: heartbeatRuns.createdAt, issueId: runIssueId, + incidentId: budgetResumeClaims.incidentId, actorUserId: budgetResumeClaims.actorUserId, + actorSource: budgetResumeClaims.actorSource, claimedAt: budgetResumeClaims.claimedAt, + stoppedByPerson: sql`(coalesce(${heartbeatRuns.resultJson}->'cancellation'->'initiator'->>'type', '') = 'user' + or coalesce(${heartbeatRuns.resultJson}->'cancellation'->>'source', '') = 'operator' + or coalesce(${heartbeatRuns.resultJson}->>'cancelledByActorType', '') in ('user', 'board'))`, + // Claimed only for its admission stamp, but cancelled for another reason. + otherCancellation: sql`(${heartbeatRuns.resultJson} ? 'budgetAdmissionStop' + and coalesce(${heartbeatRuns.resultJson}->'cancellation'->>'reason', '') + <> coalesce(${heartbeatRuns.resultJson}->'budgetAdmissionStop'->>'reason', '') + and not coalesce(${heartbeatRuns.resultJson}->'startupCancellation' ? 'budgetEnforcement', false) + and not ${heartbeatRuns.resultJson} ? 'pauseStop')`, + }).from(budgetResumeClaims).innerJoin(heartbeatRuns, eq(heartbeatRuns.id, budgetResumeClaims.runId)) + .innerJoin(agents, eq(agents.id, heartbeatRuns.agentId)) + .where(and(eq(budgetResumeClaims.state, "pending"), after ? gt(budgetResumeClaims.runId, after) : undefined)) + .orderBy(asc(budgetResumeClaims.runId)).limit(PAGE_SIZE); + if (!page.length) return; + after = page[page.length - 1]!.id; + const rows: PendingRow[] = page.map(({ incidentId, actorUserId, actorSource, claimedAt, ...run }) => + ({ ...run, claim: { incidentId, actorUserId, actorSource, claimedAt } })); + for (const run of rows) { + // One task's failure must not stop the others. Its claim goes stale and + // a later pass retries it. + try { await deliverOne(db, delivery, now, run); } + catch (error) { logger.warn({ err: error, runId: run.id }, "Budget resume continuation remains pending"); } + } + if (rows.length < PAGE_SIZE) return; + } +} + +/** + * Settle a claim and record why, in one commit. A crash before it leaves the + * claim pending; the Retry's idempotency key makes the next pass return the + * same request, so the record is never lost or doubled. + */ +async function settle(db: Db, run: PendingRow, expectedClaimedAt: Date | null, outcome: { + state: "delivered" | "abandoned"; reason?: string; requestId?: string | null; +}) { + const claim = run.claim!; + const publications: ActivityPublication[] = []; + const settled = await db.transaction(async tx => { + if (!await setClaim(tx, { runId: run.id, expectedClaimedAt, + next: { state: outcome.state, settledAt: new Date(), + ...(outcome.reason ? { reason: outcome.reason } : {}), + ...(outcome.requestId !== undefined ? { requestId: outcome.requestId } : {}) } })) return false; + await logActivity(tx as unknown as Db, { companyId: run.companyId, actorType: "user", actorId: claim.actorUserId, + action: outcome.state === "delivered" ? "budget.cancelled_work_continued" : "budget.cancelled_work_not_continued", + entityType: "issue", entityId: run.issueId!, + details: { budgetIncidentId: claim.incidentId, interruptedRunId: run.id, + ...(outcome.requestId ? { retryRequestId: outcome.requestId } : {}), + ...(outcome.reason ? { reason: outcome.reason } : {}) } }, publications); + return true; + }); + if (settled) for (const publication of publications) publishActivity(publication); + return settled; +} + +async function deliverOne(db: Db, delivery: BudgetResumeDelivery, now: Date, run: PendingRow) { + const claim = run.claim; + if (!claim || !run.issueId) return; + // The stop is still being delivered; revisit once it settles. + if ((ACTIVE_RUN_STATUSES as readonly string[]).includes(run.status)) return; + const claimedAt = claim.claimedAt ?? null; + // A fresh claim belongs to a live dispatcher. + if (claimedAt && now.getTime() - claimedAt.getTime() < BUDGET_RESUME_CLAIM_STALE_MS) return; + // The run finished before its stop landed: nothing was cancelled. + if (run.status !== "cancelled") { await settle(db, run, claimedAt, { state: "abandoned", reason: "run_not_cancelled" }); return; } + // A person stopped the run after the resume claimed it: their Stop wins. + if (run.stoppedByPerson) { await settle(db, run, claimedAt, { state: "abandoned", reason: "stopped_by_person" }); return; } + if (run.otherCancellation) { await settle(db, run, claimedAt, { state: "abandoned", reason: "cancelled_for_another_reason" }); return; } + // The resume is still lifting the agent's hold (a connected chat refuses a + // Retry for a paused agent). Wait without claiming; the agent's status + // change wakes delivery. + // A stale claim is released first, so its expiry no longer schedules passes. + if ((WAITING_AGENT_STATUSES as readonly string[]).includes(run.agentStatus)) { + if (claimedAt) await setClaim(db, { runId: run.id, expectedClaimedAt: claimedAt, next: { claimedAt: null } }); + return; + } + const nextClaim = new Date(now.getTime()); + if (!await setClaim(db, { runId: run.id, expectedClaimedAt: claimedAt, next: { claimedAt: nextClaim } })) return; + // The board member's own Retry: the path that clears a stopped task's + // execution hold, and that accepts only while this run is still the task's + // latest run and the task is open and assigned (checked under the task lock). + const requestTaskRetry = delivery.requestRetry ?? (await import("./task-retry.js")).requestTaskRetry; + let receipt: Awaited>; + try { + receipt = await requestTaskRetry({ db, companyId: run.companyId, issueId: run.issueId, + observedRunId: run.id, idempotencyKey: `budget-resume:${run.id}`, actorId: claim.actorUserId, + ...(claim.actorSource ? { actorSource: claim.actorSource } : {}), + ...(delivery.prepareChatRetry ? { prepareChatRetry: delivery.prepareChatRetry } : {}) }); + } catch (error) { + // A refusal the Retry path names is permanent for this claim; record it. + // Anything else (a database error) leaves the claim to go stale and retry. + // A refusal the Retry path marks retryable (brief chat-authorization + // contention) releases the claim for the next pass. + if (error instanceof HttpError && (error.details as { retryable?: unknown } | undefined)?.retryable === true) { + await setClaim(db, { runId: run.id, expectedClaimedAt: nextClaim, next: { claimedAt: null } }); + return; + } + if (error instanceof HttpError && error.status >= 400 && error.status < 500) { + const code = (error.details as { code?: unknown } | undefined)?.code; + await settle(db, run, nextClaim, { state: "abandoned", + reason: typeof code === "string" ? code : `retry_refused_${error.status}` }); + return; + } + throw error; + } + if (!receipt.accepted) { + await settle(db, run, nextClaim, { state: "abandoned", reason: "work_continued_or_task_changed" }); + return; + } + if (!await settle(db, run, nextClaim, { state: "delivered", requestId: receipt.requestId ?? null })) return; + if (receipt.requestId && !receipt.chatActionId && delivery.resumeTaskRetry) + await delivery.resumeTaskRetry(run.companyId, receipt.requestId).catch(error => + logger.warn({ err: error, runId: run.id }, "Budget resume Retry remains queued")); +} + +/** Wake budget delivery when an agent becomes able to take work and a + * resume claim waits on it. Runs inside the agent's status-change transaction. */ +export async function notifyBudgetResumeAgentReady(tx: Db, agentId: string) { + const [pending] = await tx.select({ id: budgetResumeClaims.runId }).from(budgetResumeClaims) + .innerJoin(heartbeatRuns, eq(heartbeatRuns.id, budgetResumeClaims.runId)) + .where(and(eq(heartbeatRuns.agentId, agentId), eq(budgetResumeClaims.state, "pending"))).limit(1); + if (pending) await notifyDeliveryWork(tx, DELIVERY_QUEUES.budgetEnforcement); +} + +/** Wake the continuation worker after budget Stops land. */ +export async function notifyBudgetStopsSettled(db: Db, runIds: string[]) { + if (!runIds.length) return; + const [pending] = await db.select({ id: budgetResumeClaims.runId }).from(budgetResumeClaims).where(and( + inArray(budgetResumeClaims.runId, runIds), eq(budgetResumeClaims.state, "pending"))).limit(1); + if (pending) await notifyDeliveryWork(db, DELIVERY_QUEUES.executionContinuation); +} diff --git a/server/src/services/budgets.ts b/server/src/services/budgets.ts index 51160723bad..c81daddbb10 100644 --- a/server/src/services/budgets.ts +++ b/server/src/services/budgets.ts @@ -1,3 +1,4 @@ +import { markBudgetCancelledWorkForResume } from "./budget-resume-continuation.js"; import { notifyMuseContactVerification } from "../modules/external-agents/index.js"; import { promoteDeferredWorkIntents } from "./work-intent-outbox.js"; import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; @@ -70,6 +71,8 @@ export type BudgetEnforcementScope = { export type BudgetServiceHooks = { cancelWorkForScope?: (scope: BudgetEnforcementScope) => Promise; + /** Deliver budget-resume continuations; the enforcement worker calls it after Stop delivery. */ + deliverResumeContinuations?: () => Promise; }; function currentUtcMonthWindow(now = new Date()) { @@ -1007,6 +1010,8 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica incidentId: string, input: BudgetIncidentResolutionInput, actorUserId: string, + /** The resolving actor's session source, kept for the continuation Retry. */ + actorSource?: string | null, ): Promise => { const incident = await db .select() @@ -1066,6 +1071,24 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica } await resumeScopeFromBudget(policy); + // Claim the work this stop cancelled for one continuation each, before + // the incident closes: a crash before here leaves the incident open, + // and resolving it again keeps existing claims. The continuation + // worker delivers each claim once its stop has settled. + const claimedRunIds = await markBudgetCancelledWorkForResume(db, { + companyId, scopeType: policy.scopeType as BudgetScopeType, scopeId: policy.scopeId, + incidentId: incident.id, incidentOpenedAt: incident.createdAt, actorUserId, actorSource, + }); + if (claimedRunIds.length > 0) await recordActivity({ + companyId, + actorType: "user", + actorId: actorUserId, + action: "budget.cancelled_work_resume_requested", + entityType: "budget_incident", + entityId: incident.id, + details: { scopeType: policy.scopeType, scopeId: policy.scopeId, runCount: claimedRunIds.length, + sampleRunIds: claimedRunIds.slice(0, 20) }, + }); await db .update(budgetIncidents) .set({ @@ -1355,11 +1378,12 @@ export function budgetService(db: Db, _hooks: BudgetServiceHooks = {}) { getInvocationBlock: (companyId: string, agentId: string | null, context?: AdmissionContext) => withBudgetSpan("budget.admission.inspect", () => withAccountingReadSnapshot(db, companyId, tx => budgetServiceInTransaction(tx).getInvocationBlock(companyId, agentId, context))), - resolveIncident: async (companyId: string, incidentId: string, input: BudgetIncidentResolutionInput, actorUserId: string) => { + resolveIncident: async (companyId: string, incidentId: string, input: BudgetIncidentResolutionInput, actorUserId: string, + actorSource?: string | null) => { const [scope] = await db.select({ scopeType: budgetIncidents.scopeType, scopeId: budgetIncidents.scopeId }).from(budgetIncidents) .where(and(eq(budgetIncidents.id, incidentId), eq(budgetIncidents.companyId, companyId))); if (!scope) throw notFound("Budget incident not found"); - return mutate(companyId, service => service.resolveIncident(companyId, incidentId, input, actorUserId), scope); + return mutate(companyId, service => service.resolveIncident(companyId, incidentId, input, actorUserId, actorSource), scope); }, }; } diff --git a/server/src/services/execution-control-work.ts b/server/src/services/execution-control-work.ts index 5a0e24f24e0..b0c7ca75ce7 100644 --- a/server/src/services/execution-control-work.ts +++ b/server/src/services/execution-control-work.ts @@ -8,6 +8,7 @@ import { obsoleteExecutionConversationHold, unresolvedExecutionDispositions } fr import { hasPendingNativeReplacements } from "./native-runtime/native-safe-replacement.js"; import { nextAbandonedNativeOwnerReleaseAt } from "./native-runtime/native-failed-run-owner-release.js"; import { hasPendingPauseResume } from "./issue-tree-resume.js"; +import { hasPendingBudgetResume } from "./budget-resume-continuation.js"; export function registerExecutionControlWork( coordinator: ReturnType, @@ -94,7 +95,7 @@ export function executionControlSchedule(db: Db) { eq(issueRecoveryActions.status, "resolved"), sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`, )).limit(1); - return rows.length || await hasPendingPauseResume(db) ? retry() : null; + return rows.length || await hasPendingPauseResume(db) || await hasPendingBudgetResume(db) ? retry() : null; }, async status() { const rows = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns) diff --git a/server/src/services/heartbeat/queue.test.ts b/server/src/services/heartbeat/queue.test.ts index 3f288527682..b3f0d0e97b7 100644 --- a/server/src/services/heartbeat/queue.test.ts +++ b/server/src/services/heartbeat/queue.test.ts @@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto"; import { contextCompactionService } from "../context-compaction.js"; import { claimNativeSessionResumptions } from "../native-runtime/native-finalization-reconciler.js"; import type { Db } from "@paperclipai/db"; -import { authUsers, agents, agentWakeupRequests, caseHandoffs, cases, chatCompletionDeliveries, companies, companyMemberships, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, environmentLeases, heartbeatRuns, issueComments, issueThreadInteractions, issues, nativeRunFinalizations } from "@paperclipai/db"; +import { authUsers, agents, agentWakeupRequests, budgetIncidents, budgetPolicies, caseHandoffs, cases, chatCompletionDeliveries, companies, companyMemberships, costEvents, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, environmentLeases, heartbeatRuns, issueComments, issueThreadInteractions, issues, nativeRunFinalizations } from "@paperclipai/db"; import { eq, inArray, sql } from "drizzle-orm"; import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js"; @@ -810,6 +810,51 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => { await expectSavedMessages([...firstMessages, ...secondMessages]); }); + // H1 bug C review: a queued run admission cancels for a board-owned budget + // block records which budget stopped it, so Raise budget and resume can + // continue it. + async function admissionBudgetBlock(options: { openIncident: boolean }) { + const fixtureRows = await fixture("queued"); + const { company, agent } = fixtureRows; + const [policy] = await db.insert(budgetPolicies).values({ companyId: company.id, scopeType: "company", scopeId: company.id, + windowKind: "lifetime", amount: 1, hardStopEnabled: true, isActive: true }).returning(); + await db.insert(costEvents).values({ companyId: company.id, agentId: agent.id, provider: "test", model: "test", + costCents: 5, occurredAt: new Date() }); + if (options.openIncident) await db.insert(budgetIncidents).values({ companyId: company.id, policyId: policy.id, + scopeType: "company", scopeId: company.id, metric: "billed_cents", windowKind: "lifetime", + windowStart: new Date(0), windowEnd: new Date(Date.now() + 86_400_000), thresholdType: "hard", + amountLimit: 1, amountObserved: 5 }); + const deps = callbacks(db); + deps.getAgent.mockResolvedValue(agent); + deps.budgets.getInvocationBlock.mockResolvedValue({ scopeType: "company", scopeId: company.id, + scopeName: company.name, reason: "Company cannot start work because its budget hard-stop is still exceeded." }); + deps.cancelRunInternal.mockImplementation(async (runId: string) => { + await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date() }).where(eq(heartbeatRuns.id, runId)); + }); + return { ...fixtureRows, deps }; + } + + it("leaves a queued run queued when a resume closed its budget incident first", async () => { + // Raise budget and resume closed the incident between the admission + // check and the cancel: the re-read finds no block, so the task runs. + const { run, deps } = await admissionBudgetBlock({ openIncident: false }); + deps.budgets.getInvocationBlock.mockResolvedValueOnce({ scopeType: "company", scopeId: run.companyId, + scopeName: "Budget", reason: "Company cannot start work because its budget hard-stop is still exceeded." }) + .mockResolvedValueOnce(null); + expect(await createHeartbeatQueue(db, deps).claimQueuedRun({ ...run, responsibleUserId: "queue-test-user" })).toBeNull(); + expect(deps.cancelRunInternal).not.toHaveBeenCalled(); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0].status).toBe("queued"); + }); + + it("records the budget that cancelled a queued run at admission", async () => { + const { company, agent, run, deps } = await admissionBudgetBlock({ openIncident: true }); + void agent; + expect(await createHeartbeatQueue(db, deps).claimQueuedRun({ ...run, responsibleUserId: "queue-test-user" })).toBeNull(); + expect(deps.cancelRunInternal).toHaveBeenCalledWith(run.id, expect.stringContaining("budget"), expect.anything()); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0].resultJson) + .toMatchObject({ budgetAdmissionStop: { scopeType: "company", scopeId: company.id } }); + }); + it("rechecks a handoff's assignment version under the issue claim lock", async () => { const { company, agent, issue, wake, run } = await fixture("queued"); const [record] = await db.insert(cases).values({ companyId: company.id, caseNumber: 1, identifier: "QM-CASE-1", diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index c17ae70b663..dfeba70e0a1 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -103,6 +103,7 @@ import { cases, agentWakeupRequests, activityLog, + budgetIncidents, chatConversations, chatEndpoints, companies, @@ -923,7 +924,39 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende // message, Retry, or recovery it carries; keep it queued. Settling // accounting re-notifies run dispatch. A reached hard stop is the board's. if (!run.startedAt && !(await isBoardOwnedBudgetBlock(db, run.companyId, budgetBlock))) return null; - await cancelRunInternal(run.id, budgetBlock.reason); + // Record which budget stops this queued run, so Raise budget and resume + // continues it (budget-resume-continuation.ts). Saved before the cancel + // so a crash between them cannot lose it; the resume matches it only + // against a recorded cancellation with this same reason. + const budgetAdmissionStop = { scopeType: budgetBlock.scopeType, scopeId: budgetBlock.scopeId, + reason: budgetBlock.reason.slice(0, 512) }; + const stamped = await db.transaction(async tx => { + // The same commit touches the scope's open incident. A Raise budget + // and resume whose repeatable-read snapshot predates this commit then + // conflicts on that row and retries, so its resume sees the stamp + // (a still-queued stamped run is claimed and waits for this cancel). + const open = await tx.update(budgetIncidents).set({ updatedAt: new Date() }).where(and( + eq(budgetIncidents.companyId, run.companyId), eq(budgetIncidents.scopeType, budgetBlock.scopeType), + eq(budgetIncidents.scopeId, budgetBlock.scopeId), eq(budgetIncidents.status, "open"))) + .returning({ id: budgetIncidents.id }); + if (!open.length) return false; + await tx.update(heartbeatRuns).set({ resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) + || jsonb_build_object('budgetAdmissionStop', jsonb_build_object('scopeType', ${budgetAdmissionStop.scopeType}::text, + 'scopeId', ${budgetAdmissionStop.scopeId}::text, 'reason', ${budgetAdmissionStop.reason}::text))` }) + .where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, run.status))); + return true; + }); + if (!stamped) { + // No open incident: a resume may have just closed it and raised the + // budget. Read the block again; resumed work stays queued. + const current = await budgets.getInvocationBlock(run.companyId, run.agentId, { + issueId: readNonEmptyString(context.issueId), projectId: readNonEmptyString(context.projectId) }); + if (!current) return null; + if (!run.startedAt && !(await isBoardOwnedBudgetBlock(db, run.companyId, current))) return null; + await cancelRunInternal(run.id, current.reason); + return null; + } + await cancelRunInternal(run.id, budgetBlock.reason, { resultJson: { budgetAdmissionStop } }); return null; } diff --git a/server/src/services/heartbeat/run-control.ts b/server/src/services/heartbeat/run-control.ts index fe5bb4940a6..6cae0c38ab3 100644 --- a/server/src/services/heartbeat/run-control.ts +++ b/server/src/services/heartbeat/run-control.ts @@ -73,6 +73,7 @@ import { } from "../adapter-execution-control.js"; import { AdapterStopTimeoutError } from "../adapter-stop-timeout.js"; import { budgetCancellationScopePredicate, pendingBudgetRunCancellationPredicate } from "./budget-cancellation.js"; +import { notifyBudgetStopsSettled } from "../budget-resume-continuation.js"; import { withBudgetSpan } from "../budget-tracing.js"; import { captureException } from "../../sentry.js"; import { randomUUID } from "node:crypto"; @@ -1842,6 +1843,9 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont } } })); + // A resumed budget owes these runs a continuation once their Stop lands. + await notifyBudgetStopsSettled(db, runIds).catch(error => + logger.warn({ err: error, companyId: scope.companyId }, "Budget resume continuation notification deferred")); // Drain the batch before rejecting; one failed provider must not prevent // the remaining runs from receiving their already-committed Stop. if (failures.length) throw new AggregateError(failures, "Budget cancellation delivery remains pending"); From c5c587f16231b22dc60a3252a94e7fdef7a53437 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:14:47 -0600 Subject: [PATCH 6/9] fix(issues): start a deferred children-completed wake the release drain missed (#20180) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - A parent that delegates waits for its children, and is woken when they finish (`issue_children_completed`) > - Restart chaos t06 ended with both children done and the parent blocked forever > - The children finished while the parent's run was still going, so the wake was deferred as a follow-up. That run then ended with the agent setting the parent blocked, and the deferred wake was never started > - The durable re-drivers for deferred wakes cover only `issue_blockers_resolved`, so nothing else picks up a missed `issue_children_completed` > - This pull request makes both re-drivers cover children-completed wakes too > - The benefit is that a parent always hears that its children finished, even when the release drain misses the deferred wake ## Linked Issues or Issue Description Refs #19831 (durable ownership plan). Restart chaos wave `w-20261109T041248-s9501`, task t06 (claude, M1, K2), Daytona per-turn, master fa7950121d. **Problem:** parent CHA-10 (all times UTC): - 04:29:57: run 9cf49f52 started. - 04:32:35 and 04:32:43: its children CHA-13 and CHA-14 finished. - 04:32:43: `issue_children_completed` merged into the running turn. - 04:32:53: that wake was deferred as a follow-up (`issue_execution_deferred`, `deferred_issue_execution`, no run). - 04:33:13: the turn ended with the agent setting the parent `blocked`, a decision made before it saw the children's results. The deferred wake stayed deferred for the rest of the wave, including three later restarts. The parent sat blocked with no Retry. The deferred-receipt worker (`deferred-receipt-work.ts`) and the stranded-queue resume (`queue.ts`) select deferred dependency wakes with `issueBlockersResolvedWakePayloadSql`, which matches only `issue_blockers_resolved`. ## What Changed - `server/src/services/issue-dependency-wakeups.ts`: `dependencyResolvedWakePayloadSql` and `isDependencyResolvedWakePayload` replace the blockers-only predicates. They match both `issue_blockers_resolved` and `issue_children_completed` in `_paperclipWakeContext.wakeReason`. - The deferred-receipt worker and the stranded-queue resume use them, so a deferred children-completed wake is started once the parent has no live run, the same way a blockers-resolved one is. - No code comments added. ## Verification Red/green: - `deferred-execution-receipt-work.test.ts` "starts a deferred children-completed wake left behind when its parent's run blocked the parent (restart chaos t06)". It sets up a blocked parent whose last run succeeded, a done child, and the deferred `issue_children_completed` receipt as t06 left it. - On master: the receipt stays `deferred_issue_execution` (red). - With this PR: the receipt completes with exactly one `issue_children_completed` run. - `deferred-execution-receipt-work` and `heartbeat-dependency-scheduling` pass (60 tests). `tsc --noEmit -p server` is clean. ## Risks - The workers now also select deferred children-completed wakes. Admission is unchanged: they still pass the queue's own gates, and they start only when the issue has no live run, the same as blockers-resolved wakes. - No schema change. ## Model Used Anthropic Claude Opus 5.5 (`claude-opus-5-5`), via Claude Code, with tool use (shell, file editing). ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above (#20069, #20108) - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes (no user docs affected) - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green (pending) - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups (pending) - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 --- .../deferred-execution-receipt-work.test.ts | 28 +++++++++++++++++++ .../heartbeat/deferred-receipt-work.ts | 4 +-- server/src/services/heartbeat/queue.ts | 8 +++--- .../src/services/issue-dependency-wakeups.ts | 13 ++++++--- 4 files changed, 43 insertions(+), 10 deletions(-) diff --git a/server/src/__tests__/deferred-execution-receipt-work.test.ts b/server/src/__tests__/deferred-execution-receipt-work.test.ts index 414ac9d5abf..76fe58ed5dd 100644 --- a/server/src/__tests__/deferred-execution-receipt-work.test.ts +++ b/server/src/__tests__/deferred-execution-receipt-work.test.ts @@ -185,6 +185,34 @@ describe("deferred receipt durable dispatch", () => { return { ...f, receipt, blocker }; } + it("starts a deferred children-completed wake left behind when its parent's run blocked the parent (restart chaos t06)", async () => { + const f = await fixture(); + await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: f.issue.id, status: "succeeded", + errorCode: null, contextSnapshot: { issueId: f.issue.id, wakeReason: "issue_assigned" } }).where(eq(heartbeatRuns.id, f.source.id)); + await db.insert(nativeRunFinalizations).values({ companyId: f.companyId, issueId: f.issue.id, runId: f.source.id, phase: "committed" }); + await db.update(issues).set({ status: "blocked", executionRunId: null, checkoutRunId: null }).where(eq(issues.id, f.issue.id)); + const [child] = await db.insert(issues).values({ companyId: f.companyId, title: "Part 1", parentId: f.issue.id, + status: "done", completedAt: new Date() }).returning(); + const [receipt] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id, + source: "automation", reason: "issue_execution_deferred", status: "deferred_issue_execution", + requestedByActorType: "system", idempotencyKey: `issue_children_completed:${f.issue.id}`, + payload: { issueId: f.issue.id, childIssueIds: [child.id], _paperclipWakeContext: { issueId: f.issue.id, + taskId: f.issue.id, wakeReason: "issue_children_completed", source: "native_status_decision", childIssueIds: [child.id] } }, + }).returning(); + await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready; + await heartbeat.registerRunDispatchWork(coordinator, () => !drain).ready; + await waitForReal(async () => { + await Promise.allSettled([...passes]); await heartbeat.drainActiveRunExecutions(); + const [saved] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, receipt.id)); + expect(saved.status, JSON.stringify({ errors })).toBe("completed"); + expect(saved.runId).not.toBeNull(); + }, { timeout: 15_000 }); + await finish(f.issue.id); + const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId)); + expect(runs.filter(run => run.contextSnapshot?.wakeReason === "issue_children_completed")).toHaveLength(1); + await quiet(); + }); + it.each(["restart", "cleanup"] as const)("recovers a dependency wake after native audit %s into one continuation", async event => { const f = await strandedDependency(); const { receipt } = f; diff --git a/server/src/services/heartbeat/deferred-receipt-work.ts b/server/src/services/heartbeat/deferred-receipt-work.ts index 915d72b7b07..5a3e3bddd64 100644 --- a/server/src/services/heartbeat/deferred-receipt-work.ts +++ b/server/src/services/heartbeat/deferred-receipt-work.ts @@ -5,7 +5,7 @@ import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator import { DELIVERY_QUEUES } from "../delivery-work-notifications.js"; import type { DeferredExecutionScope, DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js"; import { registerScopedDeadlineWork } from "../scoped-deadline-work.js"; -import { issueBlockersResolvedWakePayloadSql } from "../issue-dependency-wakeups.js"; +import { dependencyResolvedWakePayloadSql } from "../issue-dependency-wakeups.js"; type Entry = { id: string; companyId: string; dueAt: number; immediate: boolean }; type NativeIntentOptions = DeferredExecutionScope & { @@ -75,7 +75,7 @@ export function registerDeferredReceiptWork( sql`jsonb_typeof(${agentWakeupRequests.payload}#>'{_paperclipWakeContext,wakeCommentIds}') = 'array' and ${agentWakeupRequests.payload}#>'{_paperclipWakeContext,wakeCommentIds}' <> '[]'::jsonb`, sql`${agentWakeupRequests.payload}->>'mutation' = 'interaction'`, - issueBlockersResolvedWakePayloadSql, + dependencyResolvedWakePayloadSql, sql`exists (select 1 from heartbeat_runs r where r.company_id = ${agentWakeupRequests.companyId} and r.runtime_mode = 'legacy' and r.status = 'cancelled' and r.result_json->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text)`, diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index dfeba70e0a1..be86fd31205 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -153,8 +153,8 @@ import { import { logActivity, publishActivity, type ActivityPublication } from "../activity-log.js"; import { ISSUE_BLOCKERS_RESOLVED_WAKE_REASON, - isIssueBlockersResolvedWakePayload, - issueBlockersResolvedWakePayloadSql, + isDependencyResolvedWakePayload, + dependencyResolvedWakePayloadSql, } from "../issue-dependency-wakeups.js"; import { resumeIfPauseAlreadyReleased } from "../issue-tree-resume.js"; import { @@ -1901,7 +1901,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende sql`jsonb_typeof(${agentWakeupRequests.payload} #> '{_paperclipWakeContext,wakeCommentIds}') = 'array'`, sql`${agentWakeupRequests.payload} #> '{_paperclipWakeContext,wakeCommentIds}' <> '[]'::jsonb`, ), sql`${agentWakeupRequests.payload}->>'mutation' = 'interaction'`, - issueBlockersResolvedWakePayloadSql), + dependencyResolvedWakePayloadSql), sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is null`, sql`NOT EXISTS (SELECT 1 FROM heartbeat_runs live_issue_run WHERE live_issue_run.company_id = ${issues.companyId} @@ -1912,7 +1912,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende .orderBy(asc(agentWakeupRequests.updatedAt)).limit(50); const releasedNativeSources = new Set(); for (const { wake } of strandedQueues) { - const dependencyWake = isIssueBlockersResolvedWakePayload(wake.payload); + const dependencyWake = isDependencyResolvedWakePayload(wake.payload); if (!dependencyWake && !queuedCommentIdsFromWakePayload(wake.payload).length && !await readQueuedInteractionResponse(db, wake.companyId, String(wake.payload?.issueId), wake.payload)) continue; const [latest] = await db.select({ diff --git a/server/src/services/issue-dependency-wakeups.ts b/server/src/services/issue-dependency-wakeups.ts index 4d891d583a8..48591daaedc 100644 --- a/server/src/services/issue-dependency-wakeups.ts +++ b/server/src/services/issue-dependency-wakeups.ts @@ -5,13 +5,18 @@ import { agentWakeupRequests } from "@paperclipai/db"; export const ISSUE_BLOCKERS_RESOLVED_WAKE_REASON = "issue_blockers_resolved"; -export const issueBlockersResolvedWakePayloadSql = - sql`${agentWakeupRequests.payload}#>>'{_paperclipWakeContext,wakeReason}' = ${ISSUE_BLOCKERS_RESOLVED_WAKE_REASON}`; +export const ISSUE_CHILDREN_COMPLETED_WAKE_REASON = "issue_children_completed"; -export function isIssueBlockersResolvedWakePayload(payload: Record | null | undefined): boolean { +const DEPENDENCY_RESOLVED_WAKE_REASONS = [ISSUE_BLOCKERS_RESOLVED_WAKE_REASON, ISSUE_CHILDREN_COMPLETED_WAKE_REASON]; + +export const dependencyResolvedWakePayloadSql = + sql`${agentWakeupRequests.payload}#>>'{_paperclipWakeContext,wakeReason}' in (${sql.join( + DEPENDENCY_RESOLVED_WAKE_REASONS.map(reason => sql`${reason}`), sql`, `)})`; + +export function isDependencyResolvedWakePayload(payload: Record | null | undefined): boolean { const context = payload?._paperclipWakeContext; return context !== null && typeof context === "object" && !Array.isArray(context) - && (context as Record).wakeReason === ISSUE_BLOCKERS_RESOLVED_WAKE_REASON; + && DEPENDENCY_RESOLVED_WAKE_REASONS.includes(String((context as Record).wakeReason)); } /** From 95bf90bd2c71d38a9c188274fc01effcf1380236 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:17:36 -0600 Subject: [PATCH 7/9] fix(chat): post a conversation turn's answer while an approval holds the task (#20179) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - People talk to agents in chat. The agent's final message becomes the chat reply. > - A chat turn can end while the task stays held. For example, the agent resubmits a hire approval and the approval is pending. > - The finalizer keeps the task in review for the approval (`governed_response_waiting`). That decision is not `conversation_turn_finished`. > - Run completion then treats the yielded turn as a control-plane wait. It records `presentationDecision: none` and posts no comment. > - The person never sees the agent's answer. The chat shows nothing after the request. > - This pull request counts a committed conversation turn as answered unless its own pending question or confirmation card presents it. > - The benefit is that the person always gets the reply, whatever state the task keeps. ## Linked Issues or Issue Description Refs #19971 (post the full final reply). That fix covers the reply text. This fix covers replies that were never posted. **What happened** In a chat, the person asked the CEO agent to change a proposed hire to a cheaper model and send it back for approval. The agent resubmitted the approval correctly. It called `paperclip_finish`, and Paperclip accepted the report with "task is still waiting for approval". The run succeeded and the task moved to `in_review`. The run event `run.presentation.resolved` recorded `commentAction: none` (reason `yielded_control_plane_wait` on master). The agent's final message never reached the chat. **Expected behavior** A chat turn's accepted final answer is always posted to the conversation. An approval, an execution stage or a dependency blocker can hold the task. It must not hide the reply. Only a question or confirmation card that the turn itself left pending presents the turn instead of its prose. **Steps to reproduce** 1. Open an Agent Chat with an agent. Link a hire approval in `revision_requested` to the chat task. 2. Ask the agent to change the hire and resubmit it. 3. The agent resubmits the approval and ends the turn with a yielded result and a response wake. 4. See that the chat shows no reply. The run is `succeeded` and its presentation decision is `none`. ## What Changed - `server/src/services/native-runtime/native-governance.ts`: move the pending-card query of `pendingNativeGovernance()` into `pendingNativeInteraction()`, with no change in behavior. Add `conversationTurnAnswered()`. It returns false only when a pending thread interaction (the same predicate as status arbitration) gates the turn. It checks cards on their own, so a pending execution stage does not hide a card. - `server/src/services/heartbeat/run-completion.ts`: a successful conversation run whose status decision committed now passes `conversationTurnFinished: true` to the presentation resolver, unless `conversationTurnAnswered()` is false. The existing `conversation_turn_finished` path is unchanged. - `server/src/services/heartbeat/run-completion.test.ts`: add a regression table with a real database. A pending approval, a revision-requested approval, a pending execution stage and a dependency blocker now post the answer. A finished turn still posts. The turn's own question card still owns the response, also when an execution stage is pending. - `doc/execution-semantics.md`: state the rule. Other finish outcomes I checked: - `done` or `needs_review` with a gate (`governed_gate_pending`): the result is not yielded, so the reply was already posted. - `blocked` (`paperclip_block`, waiting on a person or another task): not yielded, so the reply was already posted. - Yielded with a dependency blocker (`durable_dependency_blocker_bound`), or with `record_finalization_error` (`prior_status_preserved_no_live_path`, `monitor_wait_authority_lost`): the reply was also dropped. This fix covers them, because the rule does not depend on the reason code. ## Verification - `cd server && npx vitest run src/services/heartbeat/run-completion.test.ts`: 118 passed. Before the fix, 4 new cases failed (both approval states, execution stage, dependency blocker). The finished-turn and question-card cases passed both before and after. - `cd server && npx vitest run src/__tests__/heartbeat-run-summary.test.ts src/services/heartbeat/run-comment-selection.test.ts src/__tests__/slack-conversation-lifecycle.test.ts src/services/native-runtime/agent-chat-response-recovery.test.ts src/__tests__/chat-run-milestone-governed-work.test.ts src/__tests__/agent-conversations.test.ts`: all passed. - `tsc --noEmit -p server`: no errors in the changed files. Unrelated errors in this worktree come from an unbuilt local Runner package. - An end-to-end chat eval run on the case branch is in progress. I will add its result here. ## Risks - Low risk. The change only affects successful Agent Chat runs with a committed status decision. Ordinary tasks and external chat (Slack, GitHub) keep their own paths. External chat returns before this check. - A turn that yields while an approval is pending now posts its prose. The approval stays pending and the task status does not change. - One extra small read (the task row and pending gates) runs after a conversation turn that is not `conversation_turn_finished`. It runs after the provider turn ends, so it does not add time before the first token. - Previous-release state: no persisted shape changes. The fix reads `finalizationPhase` and `finalizationReasonCode`, which the previous release already writes. A run in flight at the deploy completes on the new code and posts its reply. A run that completed before the deploy keeps its recorded decision; nothing re-presents it. - Known gap, unchanged: background presentation recovery for conversations still requires `conversation_turn_finished` with a `done` result. If the server restarts between the status commit and the live presentation, a held yielded turn is not re-posted. That is a separate follow-up. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - Anthropic Claude Opus 5.5 (`claude-opus-5-5`), with tool use (shell, file edits, test runs) in Claude Code. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- doc/execution-semantics.md | 7 +++ .../services/heartbeat/run-completion.test.ts | 63 ++++++++++++++++++- .../src/services/heartbeat/run-completion.ts | 12 +++- .../native-runtime/native-governance.ts | 60 +++++++++++++----- 4 files changed, 123 insertions(+), 19 deletions(-) diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 6541f4088b1..2478035988a 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -2241,6 +2241,13 @@ apply. Stream closure without a turn terminal is not proof of success. Event replay uses the existing source receipts and never repeats provider work merely to recover recorded output. +In an Agent Chat conversation, a successful turn whose status decision committed +posts its final answer to the conversation even when the task stays held: a +pending or revision-requested approval, a pending execution stage, or a +dependency blocker. A held task does not present the turn. Only a pending +thread interaction (a question or confirmation card) presents the turn in place +of its prose. + A delegated-task completion callback can finish native execution before its accepted final reply is saved. The completion outbox must retain that callback's queued target and attempt while the existing presentation selector finds pending diff --git a/server/src/services/heartbeat/run-completion.test.ts b/server/src/services/heartbeat/run-completion.test.ts index 6ff985ed53d..42c34facc34 100644 --- a/server/src/services/heartbeat/run-completion.test.ts +++ b/server/src/services/heartbeat/run-completion.test.ts @@ -16,7 +16,7 @@ import { createHash, randomUUID } from "node:crypto"; import { appendHeartbeatRunEvent } from "../heartbeat-run-events.js"; import { createNativeRunLogCoverageVerifier, formatNativeRunLogEvent } from "../native-runtime/native-run-log-export.js"; import type { Db } from "@paperclipai/db"; -import { activityLog, agents, agentWakeupRequests, budgetReservations, caseHandoffs, cases, companies, completionContracts, costEvents, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, heartbeatRunEvents, issueComments, issueQuestionResponseDeliveries, issueRecoveryActions, issues, nativeRunFinalizations } from "@paperclipai/db"; +import { activityLog, agents, agentWakeupRequests, approvals, issueApprovals, budgetReservations, caseHandoffs, cases, companies, completionContracts, costEvents, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, heartbeatRunEvents, issueComments, issueQuestionResponseDeliveries, issueRecoveryActions, issues, nativeRunFinalizations } from "@paperclipai/db"; import { and, eq, sql } from "drizzle-orm"; import { afterAll, beforeAll, beforeEach, describe, expect, it, onTestFinished, vi } from "vitest"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; @@ -469,6 +469,67 @@ describe.skipIf(!support.supported && !process.env.PAPERCLIP_TEST_DATABASE_URL)( expect(f.deps.finalizeAgentStatus).toHaveBeenCalledWith(f.agent.id, "succeeded", null, expect.any(Object)); }); + // A chat turn can finish while its task keeps waiting: a resubmitted hire + // approval, an execution stage or a dependency blocker holds the status. + // The accepted final answer must still reach the conversation. Only a card + // the run itself left pending in the thread presents the turn instead. + it.each([ + { hold: "pending approval", reasonCode: "governed_response_waiting", replies: true }, + { hold: "approval sent back for revision", reasonCode: "governed_response_waiting", replies: true }, + { hold: "execution stage", reasonCode: "governed_response_waiting", replies: true }, + { hold: "dependency blocker", reasonCode: "durable_dependency_blocker_bound", replies: true }, + { hold: "finished turn", reasonCode: "conversation_turn_finished", replies: true }, + { hold: "own question card", reasonCode: "governed_response_waiting", replies: false }, + { hold: "execution stage and own question card", reasonCode: "governed_response_waiting", replies: false }, + ] as const)("posts a waiting conversation turn's accepted answer: $hold", async ({ hold, reasonCode, replies }) => { + const f = await fixture({ runtimeMode: "native" }, `CNV${randomUUID().slice(0, 6)}`); + const [issue] = await db.insert(issues).values({ companyId: f.run.companyId, title: "Chat with CEO", + status: "in_progress", assigneeAgentId: f.agent.id, responsibleUserId: "local-board", + conversationAgentId: f.agent.id, conversationUserId: "local-board", conversationState: "active", + ...(hold.startsWith("execution stage") ? { executionState: { status: "pending" } } : {}), + }).returning(); + await db.update(heartbeatRuns).set({ nativeIssueId: issue.id, responsibleUserId: "local-board", + contextSnapshot: { issueId: issue.id, conversationSessionGeneration: 0 } }).where(eq(heartbeatRuns.id, f.run.id)); + if (hold.includes("approval")) { + const [approval] = await db.insert(approvals).values({ companyId: f.run.companyId, type: "hire_agent", + status: hold === "pending approval" ? "pending" : "revision_requested", payload: { name: "Researcher" } }).returning(); + await db.insert(issueApprovals).values({ companyId: f.run.companyId, issueId: issue.id, approvalId: approval.id }); + } + if (hold.endsWith("own question card")) { + await issueThreadInteractionService(db).create(issue, { kind: "ask_user_questions", sourceRunId: f.run.id, + continuationPolicy: "wake_assignee", payload: { version: 1, questions: [{ id: "model", prompt: "Which model?", + selectionMode: "single", required: true, options: [{ id: "flash", label: "Flash" }] }] } }, + { agentId: f.agent.id, runId: f.run.id }); + } + const answer = "Resubmitted the hire with the cheaper model. [Review the pending approval](/CNV/approvals/a1)."; + f.input.issueId = issue.id; + f.input.issueRef = issue; + f.input.issueContext = await createHeartbeatRunPreparation(db).getIssueExecutionContext(f.run.companyId, issue.id); + // The finalizer's saved shape, unchanged from the previous release. + f.input.adapterResult.resultJson = { + nativeResult: { schema: "paperclip.run_result.v1", reportedWorkDisposition: "yielded", + summary: "Waiting for the approval decision.", + continuation: { kind: "response_wake", summary: "Wait for the decision.", idempotencyKey: "hire-wait" } }, + finalizationPhase: "committed", finalizationReasonCode: reasonCode, + }; + f.deps.findLatestCompletedFinalAgentMessage.mockResolvedValue({ seq: 1, text: answer, sourceEventId: "final-event", channel: "final", + reasonCode: "latest_non_empty_completed_final_agent_message" }); + f.deps.issuesSvc.addComment.mockImplementation(async (issueId, body) => { + const [comment] = await db.insert(issueComments).values({ companyId: f.run.companyId, issueId, + authorAgentId: f.agent.id, authorType: "agent", createdByRunId: f.run.id, body }).returning(); + return { ...comment, authorType: "agent" as const }; + }); + + await f.completion.completeRun(f.input); + + const posted = (await db.select().from(issueComments).where(eq(issueComments.issueId, issue.id))) + .filter(comment => comment.createdByRunId === f.run.id); + expect(posted.map(comment => comment.body)).toEqual(replies ? [answer] : []); + expect((await read(f.run.id)).resultJson).toMatchObject({ presentationDecision: replies + ? { chosenSource: "final_agent_message", commentAction: "create", commentId: posted[0]?.id } + : { chosenSource: "none", commentAction: "none" } }); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + it("settles the accepted native exact rate-card price without replacing it with zero", async () => { const f = await fixture({ runtimeMode: "native" }); const runnerInstanceId = randomUUID(); diff --git a/server/src/services/heartbeat/run-completion.ts b/server/src/services/heartbeat/run-completion.ts index 2a24bbe3006..425e681f025 100644 --- a/server/src/services/heartbeat/run-completion.ts +++ b/server/src/services/heartbeat/run-completion.ts @@ -45,6 +45,7 @@ import { isUnresolvedWorkspaceBaseRefError, readUnresolvedWorkspaceBaseRefDiagno import { runnerGoalService } from "../runner-goals.js"; import { resolveChatRunPresentationAuthorizationReason } from "../chat-run-publications.js"; import { hasChatRunOwnedProviderInteraction } from "../chat-interaction-arbitration.js"; +import { conversationTurnAnswered } from "../native-runtime/native-governance.js"; import { readAcceptedTaskContinuationSummary } from "../heartbeat-run-summary.js"; import { SANDBOX_PROVIDER_PLUGIN_NOT_READY_REASON } from "../recovery/stranded-notice.js"; import { MAX_TURN_CONTINUATION_RETRY_REASON, isResolvedInteractionContinuationWakeContext } from "../../modules/run-dispatch/index.js"; @@ -845,8 +846,15 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC resultJson: persistedResultJson, interactionOwnsResponse: Boolean(issueId && readAcceptedTaskContinuationSummary(persistedResultJson) && await hasChatRunOwnedProviderInteraction(db, { companyId: livenessRun.companyId, issueId, runId: livenessRun.id })), - conversationTurnFinished: isConversation(issueContext) && livenessRun.status === "succeeded" - && persistedResultJson?.finalizationReasonCode === "conversation_turn_finished", + conversationTurnFinished: Boolean(issueId) && isConversation(issueContext) && livenessRun.status === "succeeded" + && (persistedResultJson?.finalizationReasonCode === "conversation_turn_finished" + // A turn left waiting on an approval, an execution stage or a + // blocker still answered the person. Only its own question card + // presents it instead. + || (persistedResultJson?.finalizationPhase === "committed" + && await measureRunPhase("response.resolve", () => conversationTurnAnswered({ + db, companyId: livenessRun.companyId, issueId: issueId!, runId: livenessRun.id, + })))), existingComment: existingRunComment, finalAgentMessage, preferFinalResponseOverExistingComment: diff --git a/server/src/services/native-runtime/native-governance.ts b/server/src/services/native-runtime/native-governance.ts index dd8f2a27835..84b7c5c5b80 100644 --- a/server/src/services/native-runtime/native-governance.ts +++ b/server/src/services/native-runtime/native-governance.ts @@ -24,6 +24,32 @@ export async function lockNativeGovernanceSources(input: { db: Db; companyId: st eq(approvals.companyId, input.companyId))).for("share", { noWait: true }); } +/** The pending thread interaction (question or confirmation card) that gates this run, if any. */ +async function pendingNativeInteraction(input: { + db: Db; + companyId: string; + issueId: string; + runId: string; + ignoredInteractionIds?: string[]; +}): Promise<{ id: string } | null> { + const [issue] = await input.db.select({ conversationAgentId: issues.conversationAgentId, conversationUserId: issues.conversationUserId }) + .from(issues).where(and(eq(issues.id, input.issueId), eq(issues.companyId, input.companyId))); + return input.db + .select({ id: issueThreadInteractions.id }) + .from(issueThreadInteractions) + .where( + and( + eq(issueThreadInteractions.companyId, input.companyId), + eq(issueThreadInteractions.issueId, input.issueId), + eq(issueThreadInteractions.status, "pending"), + ...(input.ignoredInteractionIds?.length ? [notInArray(issueThreadInteractions.id, input.ignoredInteractionIds)] : []), + activeIssueInteractionCondition({ runId: input.runId, conversationMode: isConversation(issue) }), + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null); +} + export async function pendingNativeGovernance(input: { db: Db; companyId: string; @@ -37,23 +63,8 @@ export async function pendingNativeGovernance(input: { if (record(input.executionState).status === "pending") { return { kind: "execution_stage", id: input.runId }; } - const [issue] = await input.db.select({ conversationAgentId: issues.conversationAgentId, conversationUserId: issues.conversationUserId }) - .from(issues).where(and(eq(issues.id, input.issueId), eq(issues.companyId, input.companyId))); const [pendingInteraction, pendingApproval] = await Promise.all([ - input.db - .select({ id: issueThreadInteractions.id }) - .from(issueThreadInteractions) - .where( - and( - eq(issueThreadInteractions.companyId, input.companyId), - eq(issueThreadInteractions.issueId, input.issueId), - eq(issueThreadInteractions.status, "pending"), - ...(input.ignoredInteractionIds?.length ? [notInArray(issueThreadInteractions.id, input.ignoredInteractionIds)] : []), - activeIssueInteractionCondition({ runId: input.runId, conversationMode: isConversation(issue) }), - ), - ) - .limit(1) - .then((rows) => rows[0] ?? null), + pendingNativeInteraction(input), input.db .select({ id: approvals.id }) .from(issueApprovals) @@ -75,3 +86,20 @@ export async function pendingNativeGovernance(input: { return null; } + +/** + * A committed conversation turn has answered the person, whatever status its + * task keeps. An approval or an execution stage can hold the task, but neither + * presents that answer in the conversation. Only a pending thread interaction + * (a question or confirmation card) is the turn's visible response. + */ +export async function conversationTurnAnswered(input: { + db: Db; + companyId: string; + issueId: string; + runId: string; +}): Promise { + // Check cards on their own: an execution stage outranks a card as the + // status gate, but it does not present the turn. + return !(await pendingNativeInteraction(input)); +} From 0ddc9f1ef153ceb31da64e7033fcdee705710d51 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:36:40 -0600 Subject: [PATCH 8/9] test(chaos): count an M1 subtask stall only when its recovery needs the board (#20178) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The M1 chaos oracle checks that a parent gets exactly one `issue_blocker_stalled` wake per stalled subtask (#20147) > - The product wakes a parent only for a stall that needs the board. The harness recorded a stall for any blocked subtask, whatever its latest recovery action > - In the Daytona M1 wave, a subtask blocked twice on agent- and system-owned recovery that resolved by itself. The harness counted two stalls and failed the parent for getting no wake, a false failure > - This pull request records a stall only when the subtask's latest recovery action needs the board, matching the product's definition > - The benefit is that `parent_woken_once_per_stall` fails only for a wake the product owed ## Linked Issues or Issue Description Refs #20147, #20098 **What happened?** In wave `w-20261109T041248-s9501`, t10's subtask 195b5fd7 was blocked twice. Action 56d036e0 was agent-owned (`native_session_interrupted`) and was cancelled. Action 53eb91b0 was system-owned (`native_continuation_requires_reconciliation`) and resolved with outcome cancelled. Neither needed the board, so the product correctly sent no stall wake. The harness still failed `parent_woken_once_per_stall` with 2 stalls and 0 wakes. **Expected behavior** A stall counts when the subtask's latest recovery action is: - open (active or escalated) and board-owned, or escalated; or - board-owned and resolved with outcome blocked, with the subtask still blocked. That is the definition the product's blocker-stall wake uses. ## What Changed - `observeChildStalls` reads the parent's `stalledDependencies` from `GET /api/issues/:id`. That is the product's own stalled-dependency view (`listStalledDependencies`, built with the same `readStalls` the blocker-stall wake uses). - Each `relation: "child"` entry is recorded as a stall (`stalledIssueId`, `recoveryAction.id`). The harness no longer infers stalls from `/recovery-actions`. That endpoint lists only open actions and selected holds, and can omit a board action resolved with outcome blocked. - Clicking the board Retry on a blocked subtask is unchanged: it still happens whenever the product offers Retry, and it still counts as a needed Retry. - Tests: - a stall the product lists is recorded; - for t10's two non-board recovery states, the product lists nothing, so no stall is recorded, and the subtask's Retry is still clicked. ## Verification - `npx vitest run` in `tests/restart-chaos`: 212 passed. - `node scripts/typecheck-test-projects.mjs tests/restart-chaos`: clean. ## Risks - The harness checks `stalledDependencies` every 10 s. A stall that appears and clears inside one interval is not recorded, so its wake goes unchecked. It is never reported as a false failure. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, with tool use and code execution in Claude Code. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- tests/restart-chaos/observer.test.ts | 29 ++++++++++++++++++++++++++-- tests/restart-chaos/observer.ts | 12 +++++++----- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/tests/restart-chaos/observer.test.ts b/tests/restart-chaos/observer.test.ts index 4e6510fe111..2e0cba8b33f 100644 --- a/tests/restart-chaos/observer.test.ts +++ b/tests/restart-chaos/observer.test.ts @@ -111,10 +111,14 @@ describe("TaskObserver", () => { it("clicks the board Retry on a blocked subtask, as a user would, and keeps watching the parent", async () => { let childStatus = "blocked"; const base = fakeApi({ "parentIds=": () => [{ id: "c1", parentId: "i1", status: "done" }, { id: "c2", parentId: "i1", status: childStatus }], - "/documents/": () => null, "/recovery-actions": () => ({ active: { id: "a1" }, actions: [] }), "/api/issues/c2": () => ({ id: "c2", status: childStatus, + "/documents/": () => null, "/recovery-actions": () => ({ active: { id: "a1", status: "active", ownerType: "board" }, actions: [] }), "/api/issues/c2": () => ({ id: "c2", status: childStatus, interruption: { canRetry: childStatus === "blocked", observedRunId: "child-run" } }) }, "blocked"); const childRetries: string[] = []; - const api = { ...base.api, get: base.api.get.bind(base.api), + const api = { ...base.api, async get(route: string) { + if (route === "/api/issues/i1") return { id: "i1", status: "blocked", stalledDependencies: childStatus === "blocked" + ? [{ stalledIssueId: "c2", relation: "child", recoveryAction: { id: "a1", status: "active", ownerType: "board" } }] : [] }; + return base.api.get(route); + }, async post(route: string, body: unknown) { if (route === "/api/issues/c2/retry") { childRetries.push(route); childStatus = "in_progress"; return { accepted: true }; } return (base.api as unknown as { post(route: string, body: unknown): Promise }).post(route, body); @@ -214,6 +218,27 @@ describe("TaskObserver", () => { expect(artifacts.stop?.runsStartedBeforeContinue).toEqual(["r2"]); }); + it.each([ + ["agent-owned interrupted session", { id: "a-agent", status: "cancelled", ownerType: "agent" }], + ["system reconciliation resolved as cancelled", { id: "a-system", status: "resolved", ownerType: "system", outcome: "cancelled" }], + ])("records no stall for a blocked subtask the product does not list as stalled (%s), but still clicks its Retry", async (_name, action) => { + let childStatus = "blocked"; + const base = fakeApi({ "parentIds=": () => [{ id: "c1", parentId: "i1", status: "done" }, { id: "c2", parentId: "i1", status: childStatus }], + "/documents/": () => null, "/recovery-actions": () => ({ active: null, actions: [action] }), + "/api/issues/i1": route => route === "/api/issues/i1" ? { id: "i1", status: "blocked", stalledDependencies: [] } : null, + "/api/issues/c2": () => ({ id: "c2", status: childStatus, interruption: { canRetry: childStatus === "blocked", observedRunId: "child-run" } }) }, "blocked"); + const childRetries: string[] = []; + const api = { ...base.api, get: base.api.get.bind(base.api), async post(route: string, body: unknown) { + if (route === "/api/issues/c2/retry") { childRetries.push(route); childStatus = "in_progress"; return { accepted: true }; } + return (base.api as unknown as { post(route: string, body: unknown): Promise }).post(route, body); + } } as unknown as ChaosApi; + const observer = new TaskObserver({ ...plan, workload: "M1", followUpAfterMs: null }, api, lab, new Timeline(null), ready); + await observer.launch(); + await observer.poll(observer.record.createdAt! + 1_000); + expect(observer.record.childStalls ?? []).toEqual([]); + expect(childRetries).toEqual(["/api/issues/c2/retry"]); + }); + it("closes an open task as unfinished at interrupt, so the wave grades it as a failure", async () => { const { api } = fakeApi({}, "in_progress"); const timeline = new Timeline(null); diff --git a/tests/restart-chaos/observer.ts b/tests/restart-chaos/observer.ts index 251a2b9bf90..54155f1f253 100644 --- a/tests/restart-chaos/observer.ts +++ b/tests/restart-chaos/observer.ts @@ -587,11 +587,13 @@ export class TaskObserver { const rows = await this.api.get(`/api/companies/${this.lab.companyId}/issues?view=index&parentIds=${encodeURIComponent(this.record.issueId)}&limit=20`) .catch(() => null); const blocked = (Array.isArray(rows) ? rows : []).filter(row => row.parentId === this.record.issueId && row.status === "blocked").map(row => String(row.id)); - for (const childIssueId of blocked) { - const recovery = await this.api.get(`/api/issues/${childIssueId}/recovery-actions`).catch(() => null); - const actions = Array.isArray(recovery?.actions) ? recovery.actions as Json[] : []; - const latest = (recovery?.active as Json | null | undefined) ?? actions[0] ?? null; - const recoveryActionId = typeof latest?.id === "string" ? latest.id : null; + const parent = await this.api.get(`/api/issues/${this.record.issueId}`).catch(() => null); + const stalled = Array.isArray(parent?.stalledDependencies) ? parent.stalledDependencies as Json[] : []; + for (const dependency of stalled) { + if (dependency.relation !== "child" || typeof dependency.stalledIssueId !== "string") continue; + const action = (dependency.recoveryAction ?? null) as Json | null; + const childIssueId = dependency.stalledIssueId; + const recoveryActionId = typeof action?.id === "string" ? action.id : null; const stalls = this.record.childStalls ??= []; if (!stalls.some(stall => stall.childIssueId === childIssueId && stall.recoveryActionId === recoveryActionId)) { stalls.push({ childIssueId, recoveryActionId }); From 2d934130850bd396647537f308aed154f7804521 Mon Sep 17 00:00:00 2001 From: Nicky Leach Date: Sun, 8 Nov 2026 21:38:57 -0800 Subject: [PATCH 9/9] perf(runner): prepare the next Claude provider generation at park (#20145) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip runs agents through Paperclip Runner, and a host can park an idle Runner between runs and resume it for the next run on the same task. > - For Claude (ACPX), a follow-up run on a parked Runner pays about 1.5–2.3 s at `run.attach` before the provider can start the turn. The attachment closes the Claude sidecar and spawns a new one: initialize, `session.open` with `session/load`, `run.attach`, then a dormant verify. > - The respawn is required. The tool-bridge MCP endpoint lasts for the whole session and can't tell which turn a late callback came from, so a new turn must not run in a process that ran an earlier one. > - Nothing requires the respawn to happen at the *start* of the next run. A turn that ends 2 s later is a better experience than one that starts 2 s later. > - This pull request does the respawn while parking. The next run's `run.attach` then only rebinds the run id on a fresh, never-used generation. > - Follow-up Claude runs on a parked Runner reach the provider about 1.5–2 s sooner, and the late-callback guarantees are unchanged. ## Linked Issues or Issue Description No public issue exists. The issue is described inline. ### What happened? On a hosted deployment with parked Runners, Claude follow-up runs reached the provider about 1.8 s later than Codex follow-ups. `runner.session.resume` medians were 5.2 s for Claude and 3.4 s for Codex. The extra time is the ACPX respawn inside `run.attach`. Codex keeps its process in place. ### Expected behavior The respawn happens while the Runner parks, so the next run starts on a prepared generation. ### Steps to reproduce 1. Run a Claude (ACPX) task on a host that parks Runners between runs. 2. Let the run finish and the Runner park. 3. Send a follow-up. `run.attach` closes the provider and starts, initializes, opens and attaches a new sidecar before `turn.start`. The Claude CLI itself starts at the first prompt on either path. ### Paperclip version or commit `master` at e194e193. ### Deployment mode Hosted, with parked Runner tasks. ## What Changed - **`durable/runner.rs`:** - New `CommandExecutor::prepare_for_park(deadline)` hook, a no-op by default. The runner passes a deadline 20 s after the park request; the host waits 30 s for the reply. - The Park host-control arm calls it after `host_park_ready()`, re-proves readiness, and only then replies `parked`. - A failure records a diagnostic and replies `conflict`, the existing park-failure path. - **ACPX, park side** (`acpx_park_preparation.rs`, `acpx_provider_generation.rs`): - For Claude, with the session open, nothing pending and nothing queued: reap the old generation, using the same suspend-and-reap sequence a second turn uses. - Start the replacement with the current run attached, and prove it dormant. - Roll the tool receipt epochs and the turn-identity ledger over. - Record a digest of the session configuration it was opened with, ignoring the MCP binding id and the aggregate digest. - Every sidecar request during preparation is capped at the park deadline. Running out of time fails like any other preparation failure. - **ACPX, attach side:** - A prepared, never-used generation whose digest matches the next run is rebound: the sidecar gets `run.attach` with the new run id, the reducer and tool bridges are reset, and the snapshot is checked for the new run id. - Anything else takes today's cold path: a different tool set, skill, prompt or launch; a buffered callback; or any rebind error. - **ACPX, turn start:** the provider restarts when its generation has run a turn, instead of when the run has settled turns. `has_settled_turns` is removed. - **ACPX, polling:** polling skips a prepared generation until it is rebound or starts a turn, so nothing it emits is attributed to the parked run. - **Sidecar** (`acpx-runtime-sidecar.ts`): rejects `run.attach` once a turn has started in that process. - **Fake sidecar:** reports live predecessors at process start, and adds the faults `reject-open`, `late-tool-after-suspend` and `pre-turn-tool`. ## Verification - `cargo test -p paperclip-runner-core`: 1029 passed (debug). - New `tests/acpx_park_preparation.rs` (10 tests). These prove: - park reaps the old pid before spawning the new one, and acks only after the dormant verify; - a late callback from the old generation after park is rejected, and so is a pre-turn call from the new generation; - a compatible attach keeps the pid, and the sidecar and snapshot carry the new run id; - changed instructions, tools, skills or runtime context take the cold path; - a used generation is never rebound; - a prepare failure produces a park conflict and a recoverable state; - Retire works after a prepared park; - a replacement whose `session.open` stalls fails at the park deadline, leaves the session suspended, and the next run attaches and completes a turn. - `cargo fmt --check` and `cargo check --workspace --locked`: clean. - `vitest` `acpx-runtime-sidecar.test.ts` and `acpx-sidecar-input.test.ts`: 126 passed. The new re-attach guard test fails if the guard is removed. - `vitest` `plugin-runner-release-resume.integration.test.ts` and `plugin-runner-eviction.integration.test.ts` with the branch's Runner: 48 passed. Both now expect the provider process to be replaced at park instead of at attach. - Fake-sidecar timing (release build, 3 runs): `run.attach` took 230–260 ms cold and 40–80 ms after a prepared park. - Real Claude (claude-agent-acp 0.73.0, release build on a hosted test VM, 3 runs each way plus one with a 180 s cgroup freeze between park and attach): | | before | this PR | |---|---|---| | `run.attach` | 1691 ms mean | 212 ms mean | | `run.attach` start → follow-up `turn.started` | 2565 ms mean | 1065 ms mean | | `turn.start` → first provider output | 2261 ms mean | 2232 ms mean | | park preparation added to park | — | about 1450 ms | Session recall, an authorized tool call after the rebind, and the absence of any tool call or event from the old run all passed in every run, including after the freeze. The prepared generation holds only the sidecar: the Claude CLI starts at the follow-up prompt on both paths, so a parked Runner also holds one node process instead of the whole Claude tree. - Hit rate: on a hosted test deployment, all 15 consecutive Claude runs across two tasks carried identical content-addressed instruction and skill bundle paths, so the prepared digest would have matched. - Not run: the hosted park, claim and attach loop with real Claude, a native MCP binding or skills with real Claude, `cargo test --release`, smoke tests. - Pre-existing flake: `unconfirmed_suspension_state_becomes_recoverable_only_after_cleanup_and_suspend` fails intermittently on clean `master` when the `hosted acpx durable::` subset runs in parallel. It passed in both full runs. ## Risks - **Park takes about 1.5 s longer** before it replies with real Claude. Preparation is capped at 20 s; the host's park wait is 30 s. - **Configuration changes:** a change between park and the next run (registered asset paths, tools, prompt) takes the cold path. That's safe, but saves nothing. - **Stale environment:** a prepared process was opened with the previous run's environment and disk state. If credential or configuration files change on disk while the Runner is parked, the prepared process won't see them, whereas the cold path would. - **Failed prepare:** a park that fails after the reap leaves no live provider. The next turn or `run.attach` starts Claude cold, as today. - **Scope:** only ACPX Claude prepares. Other providers park exactly as before. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`) in Claude Code, with tool use (shell, git, GitHub CLI) and subagents. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- .../runner-core/src/acpx_event_scope.rs | 4 - .../runner-core/src/acpx_park_preparation.rs | 114 ++++ .../runner-core/src/acpx_provider_backend.rs | 46 +- .../src/acpx_provider_generation.rs | 203 ++++++ .../runner-core/src/acpx_provider_session.rs | 27 +- .../runner-core/src/acpx_provider_state.rs | 4 - .../runner-core/src/acpx_sidecar_transport.rs | 18 +- .../runner-core/src/bin/fake-acpx-sidecar.rs | 56 +- .../crates/runner-core/src/durable/runner.rs | 149 +++++ .../src/durable/tests/hosted_retention.rs | 9 + .../src/native_provider_backend.rs | 7 + .../tests/acpx_park_preparation.rs | 594 ++++++++++++++++++ .../runner-core/tests/acpx_park_readiness.rs | 10 + .../runner-core/tests/acpx_provider_turns.rs | 145 ++++- .../src/cli/acpx-runtime-sidecar.test.ts | 44 ++ .../src/cli/acpx-runtime-sidecar.ts | 3 + ...plugin-runner-eviction.integration.test.ts | 4 +- ...-runner-release-resume.integration.test.ts | 2 +- 18 files changed, 1402 insertions(+), 37 deletions(-) create mode 100644 packages/paperclip-runner/runner/crates/runner-core/src/acpx_park_preparation.rs create mode 100644 packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_generation.rs create mode 100644 packages/paperclip-runner/runner/crates/runner-core/tests/acpx_park_preparation.rs diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_event_scope.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_event_scope.rs index 70a8adb2fad..414715f168d 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_event_scope.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_event_scope.rs @@ -38,10 +38,6 @@ impl AcpxEventScope { self.active_turn_id.as_deref() } - pub(crate) fn has_settled_turns(&self) -> bool { - !self.settled_turn_ids.is_empty() - } - pub fn bind_turn(&mut self, turn_id: impl Into) -> Result<(), LocalRunnerError> { let turn_id = turn_id.into(); validate_scope_id(&turn_id, "turn", DURABLE_STABLE_ID_CHARS)?; diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_park_preparation.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_park_preparation.rs new file mode 100644 index 00000000000..1b6f91c5bfa --- /dev/null +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_park_preparation.rs @@ -0,0 +1,114 @@ +use std::time::Instant; + +use super::{ + await_provider_lifetime_fence, AcpxCommandExecutor, AcpxProviderDescriptor, AcpxProviderSession, +}; +use crate::durable::DurableRunnerError; +use crate::provider_bridge::AuthorizedToolSet; + +impl AcpxCommandExecutor { + pub(super) fn prepare_session_for_park( + &mut self, + deadline: Instant, + ) -> Result { + self.ensure_persistence_healthy()?; + let eligible = self.candidate_admission().is_none() + && self.state.as_ref().is_some_and(|state| { + state.descriptor.agent == "claude" + && state.lifecycle == "session_open" + && state.descriptor.run_id == self.context.run_id + && state.identity.is_some() + && state.active_turn_id.is_none() + && !state.provider_exit_unconfirmed + && state.pending_events.is_empty() + && state.pending_runtime_requests.is_empty() + }) + && self + .session + .as_ref() + .is_some_and(AcpxProviderSession::can_prepare_next_generation); + if !eligible { + return Ok(false); + } + let prepared = self + .session + .as_mut() + .expect("checked ACPX session") + .prepare_next_generation(deadline); + match prepared { + Ok(()) => Ok(true), + Err(error) => { + self.recover_failed_park_preparation()?; + Err(DurableRunnerError::invalid(format!( + "ACPX park preparation failed: {error}" + ))) + } + } + } + + fn recover_failed_park_preparation(&mut self) -> Result<(), DurableRunnerError> { + self.session = None; + self.context.provider_turn_id = None; + let candidates = { + let state = self + .state + .as_mut() + .expect("ACPX state remains available after park preparation"); + state.lifecycle = "prepared".to_owned(); + state.provider_exit_unconfirmed = true; + state + .identity + .as_ref() + .expect("park preparation requires a provider identity") + .provider_lifetime_fence_candidates + }; + self.save_state()?; + let _provider_lifetime_fence = await_provider_lifetime_fence(candidates)?; + let state = self + .state + .as_mut() + .expect("ACPX state remains available after park preparation"); + state.provider_exit_unconfirmed = false; + state.lifecycle = "suspended".to_owned(); + if let Err(error) = self.save_state() { + let state = self + .state + .as_mut() + .expect("ACPX state remains available after save failure"); + state.provider_exit_unconfirmed = true; + state.lifecycle = "prepared".to_owned(); + return Err(error); + } + Ok(()) + } + + pub(super) fn rebind_prepared_session( + &mut self, + descriptor: &AcpxProviderDescriptor, + tool_set: &AuthorizedToolSet, + next_candidate: bool, + ) -> bool { + if next_candidate + || self.candidate_admission().is_some() + || !self + .session + .as_ref() + .is_some_and(AcpxProviderSession::awaits_run_rebind) + { + return false; + } + let Some(identity) = self.state.as_ref().and_then(|state| state.identity.clone()) else { + return false; + }; + let Ok(config) = descriptor.session_config( + tool_set.clone(), + Some(identity), + self.launch_profile.as_ref(), + ) else { + return false; + }; + self.session + .as_mut() + .is_some_and(|session| session.rebind_run(&config).unwrap_or(false)) + } +} diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs index e16c4f3e7c2..9063b2da0e1 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs @@ -39,6 +39,9 @@ use crate::provider_events::{ }; use crate::qualified_launch::{verify_executable_launch_artifact, verify_launch_artifact}; +#[path = "acpx_park_preparation.rs"] +mod park_preparation; + fn bounded_turn_timing(response: &Value, elapsed: Duration) -> Value { let mut receipt = serde_json::Map::new(); for key in [ @@ -1703,16 +1706,18 @@ impl AcpxCommandExecutor { "run.attach requires the same settled ACPX provider profile and session", )); } - if let Some(session) = self.session.as_mut() { - session - .shutdown("run authority rotation") - .map_err(|error| { - DurableRunnerError::invalid(format!( - "failed to checkpoint ACPX before attaching a new run: {error}" - )) - })?; + if !self.rebind_prepared_session(&descriptor, &tool_set, next_candidate.is_some()) { + if let Some(session) = self.session.as_mut() { + session + .shutdown("run authority rotation") + .map_err(|error| { + DurableRunnerError::invalid(format!( + "failed to checkpoint ACPX before attaching a new run: {error}" + )) + })?; + } + self.session = None; } - self.session = None; if let Some(admission) = next_candidate { self.launch_profile .as_mut() @@ -1982,7 +1987,7 @@ impl AcpxCommandExecutor { if self .session .as_ref() - .is_some_and(|session| session.state().has_settled_turns()) + .is_some_and(AcpxProviderSession::generation_has_run_turn) { self.retain_buffered_provider_events()?; } @@ -1992,7 +1997,10 @@ impl AcpxCommandExecutor { let (provider_process_will_be_replaced, previous_process_id) = self .session .as_ref() - .map(|session| (session.state().has_settled_turns(), session.process_id())) + .map(|session| match session.prepared_from_process_id() { + Some(process_id) => (true, process_id), + None => (session.generation_has_run_turn(), session.process_id()), + }) .expect("ACPX session exists after availability check"); { let state = self @@ -2600,7 +2608,10 @@ impl AcpxCommandExecutor { .state .as_ref() .is_some_and(|state| !state.pending_events.is_empty() || state.lifecycle == "closed") - || self.session.is_none() + || self + .session + .as_ref() + .is_none_or(AcpxProviderSession::awaits_run_rebind) { return Ok(()); } @@ -2608,6 +2619,13 @@ impl AcpxCommandExecutor { } fn retain_buffered_provider_events(&mut self) -> Result<(), DurableRunnerError> { + if self + .session + .as_ref() + .is_some_and(AcpxProviderSession::awaits_run_rebind) + { + return Ok(()); + } if let Some(session) = self.session.as_mut() { session.settle_park_observation().map_err(|error| { DurableRunnerError::invalid(format!("ACPX observation settlement failed: {error}")) @@ -2887,6 +2905,10 @@ impl CommandExecutor for AcpxCommandExecutor { })) } + fn prepare_for_park(&mut self, deadline: Instant) -> Result { + self.prepare_session_for_park(deadline) + } + fn execute(&mut self, command: &Command) -> Result { self.restore()?; if self diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_generation.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_generation.rs new file mode 100644 index 00000000000..cd826f6df07 --- /dev/null +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_generation.rs @@ -0,0 +1,203 @@ +use std::time::Instant; + +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; + +use super::{ + reserved_terminal_tool_bridge, sidecar_run_tool_operations, AcpxProviderSession, + AcpxProviderSessionConfig, +}; +use crate::acpx_provider_state::AcpxProviderState; +use crate::generated_acpx_sidecar_contract::{ + GeneratedAcpxSidecarCommand, GeneratedAcpxSidecarEventType, +}; +use crate::local_runner::LocalRunnerError; +use crate::provider_bridge::ProviderToolBridge; + +fn generation_digest(config: &AcpxProviderSessionConfig) -> String { + let mut runtime_context = config.runtime_context.clone(); + let mcp_bound = runtime_context + .pointer("/mcp/bindingId") + .is_some_and(|binding| !binding.is_null()); + if let Some(object) = runtime_context.as_object_mut() { + object.remove("mcp"); + object.remove("aggregateDigest"); + } + let shape = json!({ + "candidateAdmission": config.candidate_admission, + "command": config.transport.command, + "args": config.transport.args, + "agent": config.agent, + "model": config.model, + "catalogRevision": config.catalog_revision, + "runtimeDirectory": config.runtime_directory, + "normalizedSessionId": config.normalized_session_id, + "workingDirectory": config.working_directory, + "permissionMode": config.permission_mode, + "mode": config.mode, + "piThinkingLevel": config.pi_thinking_level, + "claudeEffort": config.claude_effort, + "grokEffort": config.grok_effort, + "grokDisableWebSearch": config.grok_disable_web_search, + "permissionModePinned": config.permission_mode_pinned, + "providerPolicy": config.provider_policy, + "systemInstructions": config.system_instructions, + "runtimeContext": runtime_context, + "mcpBound": mcp_bound, + "toolSet": config.tool_set, + }); + format!("sha256:{:x}", Sha256::digest(shape.to_string().as_bytes())) +} + +impl AcpxProviderSession { + pub fn generation_has_run_turn(&self) -> bool { + self.generation_has_run_turn + } + + pub fn awaits_run_rebind(&self) -> bool { + self.prepared_generation.is_some() + } + + pub fn prepared_from_process_id(&self) -> Option { + self.prepared_generation + .as_ref() + .map(|(_, process_id)| *process_id) + } + + pub(super) fn park_tail_is_quiet(&self) -> bool { + if self.prepared_generation.is_none() { + return self.transport.buffered_events().len() == 0; + } + self.transport.buffered_events().all(|event| { + event.turn_id.is_none() + && matches!( + event.event_type, + GeneratedAcpxSidecarEventType::RuntimeProcess + | GeneratedAcpxSidecarEventType::RuntimeDiagnostic + ) + }) + } + + pub fn can_prepare_next_generation(&self) -> bool { + !self.closed + && self.failure.is_none() + && !self.transport_terminated + && !self.runtime_retired + && self.generation_has_run_turn + && self.completed_turn_observed + && self.prepared_generation.is_none() + && self.config.candidate_admission.is_none() + && self.state.active_turn_id().is_none() + && !self.state.has_pending_requests() + && !self.state.has_pending_tools() + && self.transport.buffered_events().len() == 0 + } + + pub fn prepare_next_generation(&mut self, deadline: Instant) -> Result<(), LocalRunnerError> { + if !self.can_prepare_next_generation() { + return Err(LocalRunnerError::invalid( + "ACPX provider is not at a reusable generation boundary", + )); + } + let previous_process_id = self.process_id(); + let mut tool_bridge = self.tool_bridge.clone(); + let mut reserved_tool_bridge = self.reserved_tool_bridge.clone(); + tool_bridge + .prepare_turn_after_provider_restart() + .and_then(|()| reserved_tool_bridge.prepare_turn_after_provider_restart()) + .map_err(|error| { + LocalRunnerError::invalid(format!( + "ACPX tool receipt rotation before park failed: {error}" + )) + })?; + if let Err(error) = self.restart_idle_provider(Some(deadline)) { + return Err(self.fail_closed(error)); + } + if self.state.settled_turn_identity_capacity_reached() { + if let Err(error) = self + .state + .rotate_settled_turn_identities_after_provider_restart() + { + return Err(self.fail_closed(error)); + } + } + self.tool_bridge = tool_bridge; + self.reserved_tool_bridge = reserved_tool_bridge; + self.completed_turn_observed = false; + if let Err(error) = self.verify_dormant_session() { + return Err(self.fail_closed(error)); + } + self.transport.bound_requests_by(None); + self.prepared_generation = Some((generation_digest(&self.config), previous_process_id)); + Ok(()) + } + + pub fn rebind_run( + &mut self, + next: &AcpxProviderSessionConfig, + ) -> Result { + let Some((prepared, _)) = self.prepared_generation.as_ref() else { + return Ok(false); + }; + if self.closed + || self.failure.is_some() + || self.transport_terminated + || self.runtime_retired + || self.generation_has_run_turn + || self.state.active_turn_id().is_some() + || self.state.has_pending_requests() + || next.run_id == self.config.run_id + || next.expected_identity.as_ref() != Some(&self.identity) + || &generation_digest(next) != prepared + || next.validate().is_err() + { + return Ok(false); + } + let state = AcpxProviderState::new(&next.run_id)?; + let mut tool_bridge = ProviderToolBridge::default(); + tool_bridge + .prepare(next.tool_set.clone()) + .map_err(|error| { + LocalRunnerError::invalid(format!("ACPX authorized tools are invalid: {error}")) + })?; + let reserved_tool_bridge = reserved_terminal_tool_bridge()?; + let previous_run_id = self.config.run_id.clone(); + let attached = match self.transport.request( + GeneratedAcpxSidecarCommand::RunAttach, + json!({ + "runId": next.run_id, + "catalogRevision": next.catalog_revision, + "tools": sidecar_run_tool_operations(&next.tool_set), + }), + ) { + Ok(attached) => attached, + Err(error) => return Err(self.fail_closed(error)), + }; + if attached.get("runId").and_then(Value::as_str) != Some(next.run_id.as_str()) + || attached.get("catalogRevision").and_then(Value::as_u64) + != Some(next.catalog_revision) + { + return Err(self.fail_closed(LocalRunnerError::invalid( + "ACPX sidecar did not confirm the rebound run", + ))); + } + if self.transport.buffered_events().any(|event| { + event.turn_id.is_some() || event.run_id.as_deref() == Some(previous_run_id.as_str()) + }) { + return Err(self.fail_closed(LocalRunnerError::invalid( + "ACPX prepared provider emitted work bound to its previous run", + ))); + } + self.state = state; + self.tool_bridge = tool_bridge; + self.reserved_tool_bridge = reserved_tool_bridge; + self.config = next.clone(); + self.catalog_revision = next.catalog_revision; + self.working_directory = next.working_directory.clone(); + self.prepared_generation = None; + if let Err(error) = self.verify_dormant_session() { + return Err(self.fail_closed(error)); + } + Ok(true) + } +} diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_session.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_session.rs index bb2045e82ba..a81d3db3742 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_session.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_session.rs @@ -22,6 +22,9 @@ use crate::provider_bridge::{ use crate::question_response::validate_question_response; use crate::stable_identity::{is_stable_id, DURABLE_STABLE_ID_CHARS, SHORT_STABLE_ID_CHARS}; +#[path = "acpx_provider_generation.rs"] +mod generation; + const MAX_ID_CHARS: usize = 240; const MAX_MODEL_CHARS: usize = 240; pub(crate) const MAX_SYSTEM_INSTRUCTIONS_BYTES: usize = 1024 * 1024; @@ -353,6 +356,8 @@ pub struct AcpxProviderSession { rejected_start_resumable: bool, runtime_retired: bool, completed_turn_observed: bool, + generation_has_run_turn: bool, + prepared_generation: Option<(String, u32)>, } pub(crate) fn goal_is_dormant(projection: &Value) -> bool { @@ -411,6 +416,8 @@ impl AcpxProviderSession { rejected_start_resumable: false, runtime_retired: false, completed_turn_observed: false, + generation_has_run_turn: false, + prepared_generation: None, }) } @@ -506,10 +513,10 @@ impl AcpxProviderSession { || self.failure.is_some() || self.transport_terminated || self.runtime_retired - || !self.completed_turn_observed + || !(self.completed_turn_observed || self.prepared_generation.is_some()) || self.state.active_turn_id().is_some() || self.state.has_pending_requests() - || self.transport.buffered_events().len() != 0 + || !self.park_tail_is_quiet() { return Ok(false); } @@ -529,8 +536,7 @@ impl AcpxProviderSession { return Ok(false); }; // Polling still owns every frame observed during these reads. - Ok(self.validate_dormant_snapshot(&snapshot).is_ok() - && self.transport.buffered_events().len() == 0) + Ok(self.validate_dormant_snapshot(&snapshot).is_ok() && self.park_tail_is_quiet()) } /// Read the same live sidecar before exposing its in-memory pending ledger. @@ -654,9 +660,9 @@ impl AcpxProviderSession { // provider turn originated a late HTTP callback. Reap the old sidecar // and provider before releasing its call-ID tombstones, then resume // the same verified persistent session in a fresh process generation. - let provider_restarted = self.state.has_settled_turns(); + let provider_restarted = self.generation_has_run_turn; if provider_restarted { - if let Err(error) = self.restart_idle_provider() { + if let Err(error) = self.restart_idle_provider(None) { return Err(self.fail_closed(error)); } } @@ -693,6 +699,8 @@ impl AcpxProviderSession { "ACPX reserved tool receipt rotation failed: {error}" )))); } + self.generation_has_run_turn = true; + self.prepared_generation = None; let response = match self.transport.request_turn_start( json!({"turnId":turn_id,"message":message,"performance":performance}), ) { @@ -1307,7 +1315,8 @@ impl AcpxProviderSession { Ok(()) } - fn restart_idle_provider(&mut self) -> Result<(), LocalRunnerError> { + fn restart_idle_provider(&mut self, deadline: Option) -> Result<(), LocalRunnerError> { + self.transport.bound_requests_by(deadline); let suspended = self.transport.request( GeneratedAcpxSidecarCommand::SessionSuspend, json!({"reason":"ACPX turn receipt epoch rotation"}), @@ -1322,6 +1331,7 @@ impl AcpxProviderSession { &restart_config.transport, &restart_config.agent, )?; + replacement.bound_requests_by(deadline); let (replacement_identity, _, turn_controls) = match bootstrap(&mut replacement, &restart_config) { Ok(value) => value, @@ -1340,6 +1350,7 @@ impl AcpxProviderSession { self.transport = replacement; self.turn_controls = turn_controls; self.transport_terminated = false; + self.generation_has_run_turn = false; Ok(()) } @@ -2244,6 +2255,8 @@ mod mode_tests { rejected_start_resumable: false, runtime_retired: false, completed_turn_observed: false, + generation_has_run_turn: false, + prepared_generation: None, } } diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_state.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_state.rs index c55718e3453..58f537c07a8 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_state.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_state.rs @@ -155,10 +155,6 @@ impl AcpxProviderState { self.scope.active_turn_id() } - pub(crate) fn has_settled_turns(&self) -> bool { - self.scope.has_settled_turns() - } - pub(crate) fn validate_new_turn_identity(&self, turn_id: &str) -> Result<(), LocalRunnerError> { self.scope.validate_new_turn_identity(turn_id) } diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs index be77e3be7ce..631a61b3c9b 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs @@ -85,6 +85,7 @@ pub struct AcpxSidecarTransport { process: SupervisedProcess, request_timeout: Duration, session_open_timeout: Duration, + request_deadline: Option, next_request_id: u64, last_event_sequence: u64, buffered_events: VecDeque, @@ -389,6 +390,7 @@ impl AcpxSidecarTransport { process, request_timeout: config.request_timeout, session_open_timeout: config.request_timeout, + request_deadline: None, next_request_id: 1, last_event_sequence: 0, buffered_events: VecDeque::new(), @@ -407,6 +409,10 @@ impl AcpxSidecarTransport { } } + pub(crate) fn bound_requests_by(&mut self, deadline: Option) { + self.request_deadline = deadline; + } + pub fn process_id(&self) -> u32 { self.process.id() } @@ -508,13 +514,13 @@ impl AcpxSidecarTransport { command: GeneratedAcpxSidecarCommand, params: Value, ) -> Result { + let timeout_at = Instant::now() + self.command_timeout(command); + let deadline = self + .request_deadline + .map_or(timeout_at, |bound| bound.min(timeout_at)); let request_id = self.send_request(command, params)?; - self.receive_response( - command, - request_id, - Instant::now() + self.command_timeout(command), - )? - .ok_or_else(|| self.request_timeout_error(command)) + self.receive_response(command, request_id, deadline)? + .ok_or_else(|| self.request_timeout_error(command)) } /// A park observation may expire without revoking a healthy provider. diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-acpx-sidecar.rs b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-acpx-sidecar.rs index 49b2d151dc9..76a6a309f09 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-acpx-sidecar.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-acpx-sidecar.rs @@ -74,11 +74,27 @@ fn run() -> Result<(), Box> { .windows(2) .find(|pair| pair[0] == "--journal") .ok_or("source journal is missing")?; + let live_predecessors = std::fs::read_to_string(&path[1]) + .unwrap_or_default() + .lines() + .filter_map(|line| serde_json::from_str::(line).ok()) + .filter_map(|line| line["processStart"].as_u64()) + .filter(|pid| { + std::process::Command::new("kill") + .args(["-0", &pid.to_string()]) + .stderr(std::process::Stdio::null()) + .status() + .is_ok_and(|status| status.success()) + }) + .collect::>(); let mut file = std::fs::OpenOptions::new() .create(true) .append(true) .open(&path[1])?; - write_json(&mut file, &json!({"processStart": std::process::id()}))?; + write_json( + &mut file, + &json!({"processStart": std::process::id(), "livePredecessors": live_predecessors}), + )?; if args.iter().any(|arg| arg == "--spawn-owned-child") { let child = std::process::Command::new("/bin/sleep") .arg("120") @@ -224,7 +240,45 @@ fn run() -> Result<(), Box> { response = json!({"protocolVersion":GENERATED_ACPX_SIDECAR_PROTOCOL_VERSION,"id":id,"ok":false, "error":{"code":"temporarily_unavailable","message":"Optional read is temporarily unavailable.","retryable":true}}); } + if command == "session.open" && fault == "slow-open" { + std::thread::sleep(std::time::Duration::from_secs(10)); + } + if command == "session.open" && fault == "reject-open" { + response = json!({"protocolVersion":GENERATED_ACPX_SIDECAR_PROTOCOL_VERSION,"id":id,"ok":false, + "error":{"code":"session_open_failed","message":"The replacement provider could not open.","retryable":false}}); + } write_json(&mut stdout, &response)?; + if command == "session.open" && args.iter().any(|arg| arg == "--emit-process") { + write_json( + &mut stdout, + &json!({"protocolVersion":GENERATED_ACPX_SIDECAR_PROTOCOL_VERSION, + "sequence":next_sequence,"eventType":"runtime.process","runId":null,"turnId":null, + "payload":{"role":"sidecar","pid":std::process::id(),"processGroupId":null,"startedAt":"2026-01-01T00:00:00.000Z"}}), + )?; + next_sequence += 1; + } + if command == "session.suspend" && fault == "late-tool-after-suspend" { + write_turn_event( + &mut stdout, + next_sequence, + "runtime.tool_called", + &source_run, + "late-turn", + json!({"callId":"late-call","operationId":"issues.delete","input":{"source":"late-old-generation"}}), + )?; + next_sequence += 1; + } + if command == "run.attach" && fault == "pre-turn-tool" { + write_turn_event( + &mut stdout, + next_sequence, + "runtime.tool_called", + &source_run, + "pre-turn", + json!({"callId":"pre-turn-call","operationId":"issues.read","input":{"id":"issue-1"}}), + )?; + next_sequence += 1; + } if command == "turn.start" { let turn = request["params"]["turnId"].as_str().unwrap(); if args.iter().any(|arg| arg == "--wait-completion-feedback") { diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs b/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs index fb0040aec08..5a45df27369 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs @@ -47,6 +47,7 @@ enum CommandLifecycle { } const TERMINAL_RESULT_ACK_TIMEOUT: Duration = Duration::from_secs(2); +const HOST_PARK_PREPARATION_BUDGET: Duration = Duration::from_secs(20); const CUMULATIVE_ACK_PERSIST_INTERVAL: usize = 16; fn reconnect_delay(base: Duration, max_delay: Duration, attempt: &mut u32) -> Duration { @@ -318,6 +319,10 @@ pub trait CommandExecutor { Ok(false) } + fn prepare_for_park(&mut self, _deadline: Instant) -> Result { + Ok(false) + } + fn execute(&mut self, command: &Command) -> Result; fn observe_recovery_state(&mut self, _state: &DurableState) -> Result<(), DurableRunnerError> { @@ -1451,6 +1456,20 @@ fn process_host_control( ) && executor.host_park_ready()? => { + let ready = + match executor.prepare_for_park(Instant::now() + HOST_PARK_PREPARATION_BUDGET) { + Ok(false) => true, + Ok(true) => executor.host_park_ready()?, + Err(error) => { + state.record_diagnostic(format!("host park preparation failed: {error}")); + store.save(state)?; + false + } + }; + if !ready { + control.channel.reply(&request, "conflict")?; + return Ok(HostAction::None); + } control.park_id = Some(id.clone()); control.channel.reply(&request, "parked")?; Ok(HostAction::Park) @@ -3216,6 +3235,136 @@ mod tests { fs::remove_dir_all(directory).unwrap(); } + #[test] + #[cfg(unix)] + fn hosted_park_reproves_a_prepared_provider_before_replying() { + use std::io::{BufRead, BufReader, Write}; + use std::os::unix::net::UnixStream; + struct PreparingExecutor { + calls: Vec<&'static str>, + prepared: Option, + ready_after_prepare: bool, + prepared_once: bool, + } + impl CommandExecutor for PreparingExecutor { + fn execute(&mut self, _: &Command) -> Result { + panic!("private control cannot execute PRP commands"); + } + fn host_park_ready(&mut self) -> Result { + self.calls.push("proof"); + Ok(!self.prepared_once || self.ready_after_prepare) + } + fn prepare_for_park(&mut self, _deadline: Instant) -> Result { + self.calls.push("prepare"); + self.prepared_once = true; + self.prepared + .ok_or_else(|| DurableRunnerError::invalid("replacement provider failed")) + } + fn shutdown(&mut self) -> Result<(), DurableRunnerError> { + self.calls.push("shutdown"); + Ok(()) + } + } + for (prepared, ready_after_prepare, status, calls) in [ + ( + Some(true), + true, + "parked", + vec!["proof", "prepare", "proof"], + ), + ( + Some(true), + false, + "conflict", + vec!["proof", "prepare", "proof"], + ), + (None, true, "conflict", vec!["proof", "prepare"]), + (Some(false), false, "parked", vec!["proof", "prepare"]), + ] { + let directory = std::env::temp_dir() + .join(format!("paperclip-host-prepare-{}", uuid::Uuid::new_v4())); + let config = config(directory.clone()); + let store = DurableStateStore::new(&config.state_dir).unwrap(); + let mut state = DurableState::new(&config); + let (socket, mut host) = UnixStream::pair().unwrap(); + let mut reader = BufReader::new(host.try_clone().unwrap()); + let mut control = HostedRunnerControl::new(HostControl::new(socket).unwrap()); + let mut executor = PreparingExecutor { + calls: Vec::new(), + prepared, + ready_after_prepare, + prepared_once: false, + }; + let identity = json!({ + "runnerInstanceId":state.runner_instance_id,"environmentLeaseId":state.environment_lease_id, + "runId":state.run_id,"normalizedSessionId":state.normalized_session_id,"turnId":state.turn_id,"itemId":state.item_id, + }); + let mut exchange = |request: Value, + state: &mut DurableState, + control: &mut HostedRunnerControl, + executor: &mut PreparingExecutor, + parked: bool| { + writeln!(host, "{request}").unwrap(); + let action = process_host_control( + control, + state, + &store, + executor, + parked, + Some(Duration::from_secs(1)), + ) + .unwrap(); + let mut line = String::new(); + reader.read_line(&mut line).unwrap(); + (action, serde_json::from_str::(&line).unwrap()) + }; + let (action, reply) = exchange( + json!({"op":"park","id":"park-1","identity":identity, + "lastControllerCommandSeq":0,"ackedSourceSeq":0}), + &mut state, + &mut control, + &mut executor, + false, + ); + assert_eq!(reply["status"], status); + assert_eq!(executor.calls, calls); + assert_eq!(matches!(action, HostAction::Park), status == "parked"); + assert_eq!(control.park_id.is_some(), status == "parked"); + assert_eq!( + state + .diagnostics + .iter() + .any(|message| message.contains("host park preparation failed")), + prepared.is_none() + ); + if prepared == Some(true) && status == "parked" { + let (action, _) = exchange( + json!({"op":"claim","id":"claim-1","parkId":"park-1","bootstrapTicket":"fresh-private-ticket"}), + &mut state, + &mut control, + &mut executor, + true, + ); + assert!(matches!(action, HostAction::Claim(_))); + let (action, reply) = exchange( + json!({"op":"retire","id":"retire-claim","parkId":"park-1","claimId":"claim-1", + "identity":identity,"lastControllerCommandSeq":0,"ackedSourceSeq":0}), + &mut state, + &mut control, + &mut executor, + false, + ); + assert_eq!(reply["status"], "retired"); + assert!(matches!(action, HostAction::Retire)); + assert_eq!( + executor.calls, + ["proof", "prepare", "proof", "proof", "shutdown"] + ); + } + fs::remove_dir_all(directory).unwrap(); + } + } + #[test] #[cfg(unix)] fn hosted_park_blocks_without_provider_polls_and_retires_on_eof_or_lifetime() { diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/durable/tests/hosted_retention.rs b/packages/paperclip-runner/runner/crates/runner-core/src/durable/tests/hosted_retention.rs index 6f25a381c8c..5a55c968543 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/durable/tests/hosted_retention.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/durable/tests/hosted_retention.rs @@ -10,6 +10,7 @@ use std::os::unix::net::UnixStream; struct RetentionExecutor { polls: Arc, shutdowns: Arc, + prepares: Arc, } impl CommandExecutor for RetentionExecutor { fn execute(&mut self, _: &Command) -> Result { @@ -18,6 +19,10 @@ impl CommandExecutor for RetentionExecutor { fn host_park_ready(&mut self) -> Result { Ok(true) } + fn prepare_for_park(&mut self, _deadline: std::time::Instant) -> Result { + self.prepares.fetch_add(1, Ordering::SeqCst); + Ok(true) + } fn poll_events(&mut self) -> Result, DurableRunnerError> { self.polls.fetch_add(1, Ordering::SeqCst); Ok(vec![]) @@ -174,9 +179,11 @@ fn exercise_hosted_loop(blocked_work: Option) { }); let polls = Arc::new(AtomicUsize::new(0)); let shutdowns = Arc::new(AtomicUsize::new(0)); + let prepares = Arc::new(AtomicUsize::new(0)); let executor = RetentionExecutor { polls: polls.clone(), shutdowns: shutdowns.clone(), + prepares: prepares.clone(), }; let (socket, mut host) = UnixStream::pair().unwrap(); host.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); @@ -201,6 +208,7 @@ fn exercise_hosted_loop(blocked_work: Option) { "parked" ); let parked_polls = polls.load(Ordering::SeqCst); + assert_eq!(prepares.load(Ordering::SeqCst), 1); // Outlast the ordinary reconnect grace without any polling or shutdown. thread::sleep(Duration::from_millis(80)); assert_eq!(polls.load(Ordering::SeqCst), parked_polls); @@ -245,6 +253,7 @@ fn exercise_hosted_loop(blocked_work: Option) { finish_tx.send(()).unwrap(); server.join().unwrap(); assert_eq!(shutdowns.load(Ordering::SeqCst), 1); + assert_eq!(prepares.load(Ordering::SeqCst), 1); assert_eq!( DurableStateStore::new(&directory) .unwrap() diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/native_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/native_provider_backend.rs index 7db36c3ea39..f1f593634d9 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/native_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/native_provider_backend.rs @@ -337,6 +337,13 @@ impl CommandExecutor for NativeProviderCommandExecutor { } } + fn prepare_for_park(&mut self, deadline: Instant) -> Result { + match self.selected.as_mut() { + Some(SelectedExecutor::Acpx(executor)) => executor.prepare_for_park(deadline), + _ => Ok(false), + } + } + fn observe_recovery_state( &mut self, state: &crate::durable::DurableState, diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_park_preparation.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_park_preparation.rs new file mode 100644 index 00000000000..0f7fcf96fc7 --- /dev/null +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_park_preparation.rs @@ -0,0 +1,594 @@ +#![cfg(unix)] + +use std::fs; +use std::os::unix::fs::PermissionsExt; +use std::path::PathBuf; +use std::time::{Duration, Instant}; + +use paperclip_runner_core::durable::{ + AcpxLaunchProfile, Command, CommandExecution, CommandExecutor, DurableRunnerConfig, + QualifiedLaunchArtifact, +}; +use paperclip_runner_core::native_provider_backend::NativeProviderCommandExecutor; +use paperclip_runner_core::provider_bridge::{authorized_tool_catalog_digest, AuthorizedTool}; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; + +struct Fixture(PathBuf); +impl Drop for Fixture { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +impl Fixture { + fn fault(&self, fault: &str) { + fs::write(self.0.join("fault"), fault).unwrap(); + } + + fn journal(&self) -> Vec { + fs::read_to_string(self.0.join("sidecar.jsonl")) + .unwrap() + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect() + } + + fn process_starts(&self) -> Vec { + self.journal() + .into_iter() + .filter(|line| line.get("processStart").is_some()) + .collect() + } + + fn state(&self) -> Value { + serde_json::from_slice(&fs::read(self.0.join("acpx-provider-state.json")).unwrap()).unwrap() + } +} + +struct Harness { + fixture: Fixture, + config: DurableRunnerConfig, + executor: NativeProviderCommandExecutor, + descriptor: Value, + opened: Value, + sequence: u64, +} + +impl Harness { + fn new() -> Self { + let catalog: Value = + serde_json::from_str(include_str!("../../../../acpx-profiles.json")).unwrap(); + let profile = &catalog["profiles"]["claude"]; + let digest = profile["commandDigest"].as_str().unwrap(); + let fixture = + Fixture(std::env::temp_dir().join(format!("acpx-park-prep-{}", uuid::Uuid::new_v4()))); + fs::create_dir_all(&fixture.0).unwrap(); + fs::set_permissions(&fixture.0, fs::Permissions::from_mode(0o700)).unwrap(); + let native = fixture.0.join("native-sidecar"); + fs::copy(env!("CARGO_BIN_EXE_fake-acpx-sidecar"), &native).unwrap(); + fs::set_permissions(&native, fs::Permissions::from_mode(0o700)).unwrap(); + let sidecar = fixture.0.join("sidecar"); + fs::write( + &sidecar, + format!( + "#!/bin/sh\nexec '{}' --mode source-reconcile --emit-process --journal '{}' --source-fault '{}' \"$@\"\n", + native.display(), + fixture.0.join("sidecar.jsonl").display(), + fixture.0.join("fault").display() + ), + ) + .unwrap(); + fs::set_permissions(&sidecar, fs::Permissions::from_mode(0o700)).unwrap(); + let args = vec!["--profile-digest".to_owned(), digest.to_owned()]; + let config = DurableRunnerConfig { + connect_url: "ws://127.0.0.1/runner".to_owned(), + ca_bundle_path: None, + state_dir: fixture.0.clone(), + runner_instance_id: "runner-1".to_owned(), + environment_lease_id: "lease-1".to_owned(), + run_id: "run-1".to_owned(), + normalized_session_id: "session-1".to_owned(), + turn_id: "turn-1".to_owned(), + item_id: "item-1".to_owned(), + runner_version: "0.0.0".to_owned(), + runner_digest: format!("sha256:{}", "a".repeat(64)), + acpx_launch_profile: Some(AcpxLaunchProfile { + candidate_admission: None, + authority_digest: format!("sha256:{}", "d".repeat(64)), + command: sidecar.clone(), + args: args.clone(), + artifacts: vec![QualifiedLaunchArtifact { + path: sidecar.clone(), + sha256: format!("sha256:{:x}", Sha256::digest(fs::read(&sidecar).unwrap())), + }], + }), + opencode_launch_profile: None, + cursor_cloud_launch_profile: None, + max_outbox_bytes: 1024 * 1024, + p0_reserve_bytes: 64 * 1024, + max_frame_bytes: 1024 * 1024, + reconnect_delay: Duration::from_millis(1), + reconnect_grace: None, + max_runtime: Duration::from_secs(30), + }; + let descriptor = json!({ + "kind": "acpx", "provider": "acpx", "driver": "acpx_runtime", + "providerVersion": "0.13.1", "agent": "claude", "model": "explicit-model", + "acpxVersion": "0.13.1", "agentServerPackage": profile["agentServerPackage"], + "agentServerVersion": profile["agentServerVersion"], "agentRuntimePackage": profile["agentRuntimePackage"], + "agentRuntimeVersion": profile["agentRuntimeVersion"], "commandDigest": digest, + "sidecarCommand": sidecar, "sidecarArgs": args, "runtimeDirectory": fixture.0, + "normalizedSessionId": "session-1", "runId": "run-1", "cwd": fixture.0, + "instructions": "Complete the supplied work.", + "permissionMode": "approve-all", "permissionModePinned": true, + "runtimeContext": { + "aggregateDigest": "a".repeat(64), + "skills": [{"key": "skill-1", "bundle": {"digest": "d".repeat(64), "rootPath": "/skills/one"}}], + "mcp": {"assignmentSetId": "assignment-1", "digest": "e".repeat(64), "bindingId": "binding-run-1"}, + }, + }); + let mut executor = NativeProviderCommandExecutor::with_runner_config(&fixture.0, &config); + executor + .execute(&command(1, "run.prepare", json!({"provider": descriptor}))) + .unwrap(); + let opened = executor + .execute(&command(2, "session.open", json!({}))) + .unwrap() + .result; + let mut harness = Self { + fixture, + config, + executor, + descriptor, + opened, + sequence: 2, + }; + harness.drain(); + harness + } + + fn execute(&mut self, kind: &str, payload: Value) -> CommandExecution { + self.sequence += 1; + self.executor + .execute(&command(self.sequence, kind, payload)) + .unwrap() + } + + fn drain(&mut self) -> Vec { + let mut seen = Vec::new(); + loop { + let events = self.executor.poll_events().unwrap(); + if events.is_empty() { + return seen; + } + seen.extend( + events + .iter() + .map(|event| json!({"type":event.event_type,"payload":event.payload})), + ); + self.executor.acknowledge_events(events.len()).unwrap(); + } + } + + fn complete_turn(&mut self, turn: &str) -> Vec { + let started = self.execute( + "turn.start", + json!({"turnId": turn, "text": "Complete the work."}), + ); + let deadline = Instant::now() + Duration::from_secs(5); + let mut seen = started + .events + .iter() + .map(|(kind, _, payload)| json!({"type":kind,"payload":payload})) + .collect::>(); + while !seen + .iter() + .any(|event: &Value| event["type"] == "turn.completed") + { + assert!(Instant::now() < deadline, "no completed provider turn"); + seen.extend(self.drain()); + } + seen.extend(self.drain()); + seen + } + + fn attach_payload(&self, run: &str, mutate: impl FnOnce(&mut Value)) -> Value { + let mut descriptor = self.descriptor.clone(); + descriptor["runId"] = json!(run); + descriptor["runtimeContext"]["mcp"]["bindingId"] = json!(format!("binding-{run}")); + descriptor["runtimeContext"]["aggregateDigest"] = json!(format!("{run}-aggregate")); + let mut payload = json!({ + "provider": descriptor, + "paperclipNextAuthority": {"identity": { + "runId": run, "normalizedSessionId": self.config.normalized_session_id, + }}, + }); + mutate(&mut payload); + payload + } + + fn attach( + &mut self, + run: &str, + mutate: impl FnOnce(&mut Value), + ) -> (CommandExecution, Duration) { + let payload = self.attach_payload(run, mutate); + let started = Instant::now(); + let attached = self.execute("run.attach", payload); + let elapsed = started.elapsed(); + self.config.run_id = run.to_owned(); + self.executor.rotate_authority(&self.config); + (attached, elapsed) + } + + fn quiesce(&mut self) { + let ready = self.execute("session.snapshot", json!({"quiesceForWarmAttach": true})); + assert_eq!(self.drain(), Vec::::new()); + assert_eq!(ready.result["warmAttachReady"], true); + } +} + +fn command(sequence: u64, kind: &str, payload: Value) -> Command { + Command { + schema: "paperclip.prp.command.v1".to_owned(), + command_id: format!("command-{sequence}"), + controller_seq: sequence, + command_type: kind.to_owned(), + issued_at: "2026-09-21T00:00:00.000Z".to_owned(), + deadline_at: None, + precondition: None, + payload, + } +} + +fn alive(pid: &Value) -> bool { + std::process::Command::new("kill") + .args(["-0", &pid.to_string()]) + .stderr(std::process::Stdio::null()) + .status() + .unwrap() + .success() +} + +fn commands_after(journal: &[Value], start: usize) -> Vec { + journal[start..] + .iter() + .filter_map(|line| line["command"].as_str().map(str::to_owned)) + .collect() +} + +fn last_process_start(journal: &[Value]) -> usize { + journal + .iter() + .rposition(|line| line.get("processStart").is_some()) + .unwrap() +} + +fn prepared_park(harness: &mut Harness) -> Value { + harness.complete_turn("first"); + assert!(harness.executor.host_park_ready().unwrap()); + assert!(harness.executor.prepare_for_park(park_deadline()).unwrap()); + harness.fixture.process_starts().last().unwrap()["processStart"].clone() +} + +#[test] +fn park_reaps_the_old_generation_before_spawning_and_proves_the_new_one_dormant() { + let mut harness = Harness::new(); + let old_pid = harness.opened["processId"].clone(); + harness.complete_turn("first"); + let before = harness.fixture.journal().len(); + assert!(harness.executor.host_park_ready().unwrap()); + assert!(harness.executor.prepare_for_park(park_deadline()).unwrap()); + let journal = harness.fixture.journal(); + let spawned = last_process_start(&journal); + assert!(spawned > before); + assert_eq!( + commands_after(&journal[..spawned], before), + ["session.goal.get", "session.snapshot", "session.suspend"] + ); + assert_eq!(journal[spawned]["livePredecessors"], json!([])); + assert_ne!(journal[spawned]["processStart"], old_pid); + assert_eq!( + commands_after(&journal, spawned), + [ + "initialize", + "session.open", + "run.attach", + "session.goal.get", + "session.snapshot" + ] + ); + assert_eq!(journal[spawned + 3]["params"]["runId"], "run-1"); + assert!(!alive(&old_pid)); + assert!(alive(&journal[spawned]["processStart"])); + assert!( + harness.executor.host_park_ready().unwrap(), + "a prepared dormant generation is park-ready" + ); + assert_eq!(harness.drain(), Vec::::new()); + assert_eq!(harness.fixture.state()["lifecycle"], "session_open"); + assert!( + !harness.executor.prepare_for_park(park_deadline()).unwrap(), + "a prepared generation is not prepared twice" + ); + assert_eq!(harness.fixture.process_starts().len(), 2); + harness.executor.shutdown().unwrap(); + assert!(!alive(&journal[spawned]["processStart"])); +} + +#[test] +fn compatible_attach_keeps_the_prepared_provider_and_rebinds_its_run() { + let mut cold = Harness::new(); + cold.complete_turn("first"); + cold.quiesce(); + let (_, cold_attach) = cold.attach("run-2", |_| {}); + cold.executor.shutdown().unwrap(); + + let mut harness = Harness::new(); + let prepared_pid = prepared_park(&mut harness); + let before = harness.fixture.journal().len(); + let (attached, warm_attach) = harness.attach("run-2", |_| {}); + eprintln!("run.attach cold={cold_attach:?} prepared={warm_attach:?}"); + assert_eq!(attached.result["status"], "resumed"); + assert_eq!(attached.result["processId"], prepared_pid); + assert_eq!( + attached.result["providerSessionId"], + harness.opened["providerSessionId"] + ); + let journal = harness.fixture.journal(); + assert_eq!(harness.fixture.process_starts().len(), 2); + assert_eq!( + commands_after(&journal, before), + [ + "run.attach", + "session.goal.get", + "session.snapshot", + "session.goal.get" + ] + ); + assert_eq!(journal[before]["params"]["runId"], "run-2"); + let state = harness.fixture.state(); + assert_eq!(state["descriptor"]["runId"], "run-2"); + assert_eq!( + state["descriptor"]["runtimeContext"]["mcp"]["bindingId"], + "binding-run-2" + ); + let events = harness.complete_turn("second"); + assert!(events + .iter() + .any(|event| event["type"] == "harness.diagnostic" + && event["payload"]["details"]["pid"] == prepared_pid)); + assert_eq!( + harness.fixture.process_starts().len(), + 2, + "the first turn of the rebound run reuses the prepared provider" + ); + let snapshot = harness.execute("session.snapshot", json!({})); + assert_eq!(snapshot.result["warmAttachReady"], true); + assert!(harness.executor.host_park_ready().unwrap()); + harness.executor.shutdown().unwrap(); +} + +#[test] +fn changed_tools_or_skills_take_the_cold_path() { + let operations = vec![AuthorizedTool { + operation_id: "issues.read".to_owned(), + version: 1, + description: "Read an issue.".to_owned(), + input_schema: json!({"type":"object"}), + response_schema: json!({"type":"object"}), + }]; + let tools = json!({ + "schema": "paperclip.runner.authorized-tools.v1", "schemaVersion": 1, + "catalogDigest": authorized_tool_catalog_digest(&operations).unwrap(), + "operations": operations, + }); + let changes: [(&str, Box); 2] = [ + ( + "tools", + Box::new(move |payload| payload["authorizedTools"] = tools.clone()), + ), + ( + "skills", + Box::new(|payload| { + payload["provider"]["runtimeContext"]["skills"][0]["bundle"]["rootPath"] = + json!("/skills/two") + }), + ), + ]; + for (name, change) in changes { + let mut harness = Harness::new(); + let prepared_pid = prepared_park(&mut harness); + let (attached, _) = harness.attach("run-2", |payload| change(payload)); + assert_eq!(attached.result["status"], "resumed", "{name}"); + assert_ne!(attached.result["processId"], prepared_pid, "{name}"); + assert!(!alive(&prepared_pid), "{name}"); + assert_eq!(harness.fixture.process_starts().len(), 3, "{name}"); + let journal = harness.fixture.journal(); + let spawned = last_process_start(&journal); + assert_eq!(journal[spawned]["livePredecessors"], json!([]), "{name}"); + assert_eq!(journal[spawned + 3]["params"]["runId"], "run-2", "{name}"); + harness.complete_turn("second"); + harness.executor.shutdown().unwrap(); + } +} + +#[test] +fn a_generation_that_ran_a_turn_is_never_rebound() { + let mut harness = Harness::new(); + let prepared_pid = prepared_park(&mut harness); + harness.attach("run-2", |_| {}); + harness.complete_turn("second"); + harness.quiesce(); + let (attached, _) = harness.attach("run-3", |_| {}); + assert_ne!(attached.result["processId"], prepared_pid); + assert!(!alive(&prepared_pid)); + assert_eq!(harness.fixture.process_starts().len(), 3); + harness.executor.shutdown().unwrap(); +} + +#[test] +fn late_and_pre_turn_callbacks_never_reach_the_next_run() { + let mut harness = Harness::new(); + harness.complete_turn("first"); + harness.fixture.fault("late-tool-after-suspend"); + assert!(harness.executor.prepare_for_park(park_deadline()).unwrap()); + harness.fixture.fault(""); + let (attached, _) = harness.attach("run-2", |_| {}); + let events = harness.complete_turn("second"); + assert!(!format!("{events:?}").contains("late-call")); + assert!(!format!("{events:?}").contains("late-old-generation")); + let reused_pid = attached.result["processId"].clone(); + assert_eq!(harness.fixture.process_starts().len(), 2); + harness.executor.shutdown().unwrap(); + assert!(!alive(&reused_pid)); + + let mut harness = Harness::new(); + harness.complete_turn("first"); + harness.fixture.fault("pre-turn-tool"); + assert!(harness.executor.prepare_for_park(park_deadline()).unwrap()); + harness.fixture.fault(""); + let prepared_pid = harness.fixture.process_starts().last().unwrap()["processStart"].clone(); + assert!( + !harness.executor.host_park_ready().unwrap(), + "a turn-bound callback before any turn blocks the park" + ); + let (attached, _) = harness.attach("run-2", |_| {}); + assert_ne!(attached.result["processId"], prepared_pid); + assert!(!alive(&prepared_pid)); + let events = harness.complete_turn("second"); + assert!(!format!("{events:?}").contains("pre-turn-call")); + harness.executor.shutdown().unwrap(); +} + +#[test] +fn failed_preparation_conflicts_and_leaves_a_cold_recoverable_session() { + let mut harness = Harness::new(); + let old_pid = harness.opened["processId"].clone(); + harness.complete_turn("first"); + harness.fixture.fault("reject-open"); + let error = harness + .executor + .prepare_for_park(park_deadline()) + .unwrap_err(); + assert!( + error.to_string().contains("park preparation failed"), + "{error}" + ); + harness.fixture.fault(""); + assert!(!alive(&old_pid)); + let failed_pid = harness.fixture.process_starts().last().unwrap()["processStart"].clone(); + assert!(!alive(&failed_pid)); + let state = harness.fixture.state(); + assert_eq!(state["lifecycle"], "suspended"); + assert_eq!(state["providerExitUnconfirmed"], false); + assert!(state["identity"].is_object()); + assert!(!harness.executor.host_park_ready().unwrap()); + assert!(!harness.executor.prepare_for_park(park_deadline()).unwrap()); + assert_eq!(harness.drain(), Vec::::new()); + let (attached, _) = harness.attach("run-2", |_| {}); + assert_eq!(attached.result["status"], "resumed"); + assert_eq!( + attached.result["providerSessionId"], + harness.opened["providerSessionId"] + ); + assert_eq!(harness.fixture.process_starts().len(), 3); + harness.complete_turn("second"); + harness.executor.shutdown().unwrap(); +} + +#[test] +fn a_slow_replacement_open_fails_at_the_park_deadline_and_leaves_the_runner_responsive() { + let mut harness = Harness::new(); + let old_pid = harness.opened["processId"].clone(); + harness.complete_turn("first"); + harness.fixture.fault("slow-open"); + let started = Instant::now(); + let error = harness + .executor + .prepare_for_park(Instant::now() + Duration::from_secs(1)) + .unwrap_err(); + let elapsed = started.elapsed(); + harness.fixture.fault(""); + assert!( + error.to_string().contains("timed out at session.open"), + "{error}" + ); + assert!( + elapsed < Duration::from_secs(5), + "park preparation took {elapsed:?}" + ); + assert!(!alive(&old_pid)); + let slow_pid = harness.fixture.process_starts().last().unwrap()["processStart"].clone(); + assert!(!alive(&slow_pid)); + let state = harness.fixture.state(); + assert_eq!(state["lifecycle"], "suspended"); + assert_eq!(state["providerExitUnconfirmed"], false); + assert!(!harness.executor.host_park_ready().unwrap()); + let (attached, _) = harness.attach("run-2", |_| {}); + assert_eq!(attached.result["status"], "resumed"); + harness.complete_turn("second"); + harness.executor.shutdown().unwrap(); +} + +#[test] +fn failed_preparation_recovers_the_same_run_on_its_next_turn() { + let mut harness = Harness::new(); + harness.complete_turn("first"); + harness.fixture.fault("reject-open"); + assert!(harness.executor.prepare_for_park(park_deadline()).is_err()); + harness.fixture.fault(""); + let events = harness.complete_turn("second"); + assert!(events + .iter() + .any(|event| event["type"] == "session.resumed")); + assert_eq!(harness.fixture.process_starts().len(), 3); + harness.executor.shutdown().unwrap(); +} + +#[test] +fn prepared_park_retires_through_ordinary_shutdown() { + let mut harness = Harness::new(); + let prepared_pid = prepared_park(&mut harness); + assert!(harness.executor.host_park_ready().unwrap()); + harness.executor.shutdown().unwrap(); + assert!(!alive(&prepared_pid)); + let journal = harness.fixture.journal(); + assert_eq!( + journal.last().unwrap()["command"], + "session.close", + "retirement closes the prepared provider" + ); + assert_eq!(harness.fixture.state()["descriptor"]["runId"], "run-1"); +} + +#[test] +fn a_conflicted_park_keeps_the_prepared_provider_for_the_same_run() { + let mut harness = Harness::new(); + let old_pid = harness.opened["processId"].clone(); + let prepared_pid = prepared_park(&mut harness); + let events = harness.complete_turn("second"); + let reconciled = events + .iter() + .find(|event| event["type"] == "session.reconciled") + .expect("the turn reports the replaced provider process"); + assert_eq!(reconciled["payload"]["previousProcessId"], old_pid); + assert_eq!(reconciled["payload"]["processId"], prepared_pid); + assert!(events + .iter() + .any(|event| event["type"] == "harness.diagnostic" + && event["payload"]["details"]["pid"] == prepared_pid)); + assert_eq!(harness.fixture.process_starts().len(), 2); + let events = harness.complete_turn("third"); + assert!(events + .iter() + .any(|event| event["type"] == "session.reconciled" + && event["payload"]["previousProcessId"] == prepared_pid)); + assert_eq!(harness.fixture.process_starts().len(), 3); + harness.executor.shutdown().unwrap(); +} + +fn park_deadline() -> Instant { + Instant::now() + Duration::from_secs(20) +} diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_park_readiness.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_park_readiness.rs index 9303af23960..4b56a5abc4c 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_park_readiness.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_park_readiness.rs @@ -266,6 +266,13 @@ fn hosted_park_acpx_requires_completion_in_this_lifetime_and_exact_authority() { .unwrap(); settle(&mut executor); assert!(executor.host_park_ready().unwrap()); + assert!(executor + .prepare_for_park(Instant::now() + Duration::from_secs(20)) + .unwrap()); + assert!( + executor.host_park_ready().unwrap(), + "a prepared dormant generation inherits its park proof" + ); config.run_id = "other-run".to_owned(); executor.rotate_authority(&config); assert!(!executor.host_park_ready().unwrap()); @@ -467,5 +474,8 @@ fn hosted_park_acpx_does_not_enable_an_unqualified_park_provider() { .unwrap(); settle(&mut executor); assert!(!executor.host_park_ready().unwrap()); + assert!(!executor + .prepare_for_park(Instant::now() + Duration::from_secs(20)) + .unwrap()); executor.shutdown().unwrap(); } diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_turns.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_turns.rs index e4485a9f7bb..1294ea1adf5 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_turns.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_turns.rs @@ -1,5 +1,5 @@ use std::path::PathBuf; -use std::time::Duration; +use std::time::{Duration, Instant}; use paperclip_runner_core::acpx_provider_session::{ AcpxPermissionMode, AcpxProviderSession, AcpxProviderSessionConfig, @@ -791,3 +791,146 @@ fn forced_cancellation_retirement_preserves_terminal_polling_but_rejects_live_re session.shutdown("test complete").unwrap(); } } + +fn reusable_config(directory: &std::path::Path) -> AcpxProviderSessionConfig { + let mut config = config("source-reconcile"); + config.transport.args = vec![ + "--mode".to_owned(), + "source-reconcile".to_owned(), + "--journal".to_owned(), + directory.join("journal.jsonl").display().to_string(), + "--source-fault".to_owned(), + directory.join("fault").display().to_string(), + ]; + config.runtime_context = json!({ + "aggregateDigest": "run-1-aggregate", + "skills": [{"key": "skill-1", "bundle": {"rootPath": "/skills/one"}}], + "mcp": {"bindingId": "binding-run-1"}, + }); + config +} + +fn settle_turn(session: &mut AcpxProviderSession, turn_id: &str) -> Vec { + session + .start_turn(turn_id, "Please continue", &std::env::temp_dir()) + .unwrap(); + let mut seen = Vec::new(); + loop { + let events = session.poll_event(Duration::from_secs(1)).unwrap().unwrap(); + let settled = events.iter().any(|event| { + matches!(event, AcpxProviderStateEvent::TurnTerminal { turn_id: settled, .. } if settled == turn_id) + }); + seen.extend(events); + if settled { + return seen; + } + } +} + +fn process_starts(directory: &std::path::Path) -> usize { + std::fs::read_to_string(directory.join("journal.jsonl")) + .unwrap() + .lines() + .filter(|line| line.contains("processStart")) + .count() +} + +#[test] +fn prepared_generation_rebinds_only_an_identical_unused_configuration() { + let directory = std::env::temp_dir().join(format!("acpx-generation-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&directory).unwrap(); + let config = reusable_config(&directory); + let mut session = AcpxProviderSession::start(&config).unwrap(); + assert!(!session.can_prepare_next_generation()); + assert!(session + .prepare_next_generation(Instant::now() + Duration::from_secs(20)) + .is_err()); + settle_turn(&mut session, "turn-1"); + let first_pid = session.process_id(); + session + .prepare_next_generation(Instant::now() + Duration::from_secs(20)) + .unwrap(); + let prepared_pid = session.process_id(); + assert_ne!(prepared_pid, first_pid); + assert!(session.awaits_run_rebind() && !session.generation_has_run_turn()); + let mut next = config.clone(); + next.run_id = "run-2".to_owned(); + next.expected_identity = Some(session.identity().clone()); + next.runtime_context["mcp"]["bindingId"] = json!("binding-run-2"); + next.runtime_context["aggregateDigest"] = json!("run-2-aggregate"); + let changes: [fn(&mut AcpxProviderSessionConfig); 7] = [ + |config| config.system_instructions.push_str(" Changed."), + |config| config.tool_set.operations.clear(), + |config| config.runtime_context["skills"][0]["bundle"]["rootPath"] = json!("/skills/two"), + |config| config.runtime_context["mcp"]["bindingId"] = serde_json::Value::Null, + |config| config.permission_mode = AcpxPermissionMode::DenyAll, + |config| config.run_id = "run-1".to_owned(), + |config| config.expected_identity = None, + ]; + for change in changes { + let mut changed = next.clone(); + change(&mut changed); + assert!(!session.rebind_run(&changed).unwrap()); + assert_eq!(session.process_id(), prepared_pid); + assert!(session.awaits_run_rebind()); + } + assert!(session.rebind_run(&next).unwrap()); + assert_eq!(session.process_id(), prepared_pid); + assert_eq!(session.state().run_id(), "run-2"); + assert!(!session.awaits_run_rebind()); + let attached_runs = std::fs::read_to_string(directory.join("journal.jsonl")) + .unwrap() + .lines() + .map(|line| serde_json::from_str::(line).unwrap()) + .filter(|line| line["command"] == "run.attach") + .map(|line| line["params"]["runId"].clone()) + .collect::>(); + assert_eq!( + attached_runs, + [json!("run-1"), json!("run-1"), json!("run-2")] + ); + settle_turn(&mut session, "turn-1"); + assert_eq!(session.process_id(), prepared_pid); + assert!(session.generation_has_run_turn()); + let mut later = next.clone(); + later.run_id = "run-3".to_owned(); + assert!( + !session.rebind_run(&later).unwrap(), + "a used generation is never rebound" + ); + session + .prepare_next_generation(Instant::now() + Duration::from_secs(20)) + .unwrap(); + settle_turn(&mut session, "turn-2"); + assert!( + !session.rebind_run(&later).unwrap(), + "a prepared generation that ran a turn is never rebound" + ); + assert_eq!(process_starts(&directory), 3); + session.shutdown("test complete").unwrap(); + std::fs::remove_dir_all(directory).unwrap(); +} + +#[test] +fn prepared_generation_never_admits_the_reaped_generation_late_callback() { + let directory = std::env::temp_dir().join(format!("acpx-generation-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&directory).unwrap(); + let mut session = AcpxProviderSession::start(&reusable_config(&directory)).unwrap(); + settle_turn(&mut session, "turn-1"); + std::fs::write(directory.join("fault"), "late-tool-after-suspend").unwrap(); + session + .prepare_next_generation(Instant::now() + Duration::from_secs(20)) + .unwrap(); + std::fs::write(directory.join("fault"), "").unwrap(); + let events = settle_turn(&mut session, "turn-2"); + assert!(!events + .iter() + .any(|event| matches!(event, AcpxProviderStateEvent::ToolCall { .. }))); + assert_eq!( + process_starts(&directory), + 2, + "the prepared turn does not restart again" + ); + session.shutdown("test complete").unwrap(); + std::fs::remove_dir_all(directory).unwrap(); +} diff --git a/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.test.ts b/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.test.ts index db33af5ffd6..784b98e8c26 100644 --- a/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.test.ts +++ b/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.test.ts @@ -1404,6 +1404,50 @@ describe("qualified ACPX runtime sidecar", () => { expect(attachedRunId).toBe("run-1"); }); + it("rebinds a prepared sidecar to another run only before its first turn", async () => { + const root = mkdtempSync(join(tmpdir(), "acpx-run-rebind-")); + const loader = join(root, "provider-loader.mjs"), host = join(root, "rebind-host.mjs"); + const baseHost = new URL("../test-support/acpx-blocked-output-host.mjs", import.meta.url).href; + writeFileSync(host, `import { AcpxRuntimeHost as BaseHost } from ${JSON.stringify(baseHost)}; + export class AcpxRuntimeHost extends BaseHost { + static async open(options) { const host = await BaseHost.open(options); Object.setPrototypeOf(host, this.prototype); return host; } + startTurn({ requestId }) { return { requestId, promptStarted: Promise.resolve(), + result: Promise.resolve({ status: "completed" }), events: (async function* () {})() }; } + }`); + writeFileSync(loader, `import { registerHooks } from "node:module"; + import { pathToFileURL } from "node:url"; + registerHooks({ resolve(specifier, context, next) { + if (context.parentURL?.includes("/acpx-runtime-sidecar.") && specifier.endsWith("/runtime-host.js")) + return { url: pathToFileURL(${JSON.stringify(host)}).href, shortCircuit: true }; + return next(specifier, context); + } });`); + const sidecar = new SidecarProcess({loader, environment:{PAPERCLIP_TEST_OUTPUT_ROOT:root}}); + children.add(sidecar); + const send = (id:number, command:string, params:Record) => sidecar.write({protocolVersion:ACPX_SIDECAR_PROTOCOL_VERSION,id,command,params}); + try { + sidecar.write(initializeRequest(1,"codex")); + expect(await sidecar.next(frame=>frame.id === 1)).toMatchObject({ok:true}); + send(2,"session.open",{agent:"codex",model:"gpt-5.6-sol",runtimeDirectory:root,workingDirectory:root, + normalizedSessionId:"rebind-session",permissionMode:"deny-all",systemInstructions:"Complete the local fixture turn.",tools:[]}); + expect(await sidecar.next(frame=>frame.id === 2)).toMatchObject({ok:true}); + send(3,"run.attach",{runId:"parked-run",catalogRevision:1,tools:[]}); + expect(await sidecar.next(frame=>frame.id === 3)).toMatchObject({ok:true,result:{runId:"parked-run"}}); + send(4,"run.attach",{runId:"next-run",catalogRevision:1,tools:[]}); + expect(await sidecar.next(frame=>frame.id === 4)).toMatchObject({ok:true,result:{runId:"next-run"}}); + send(5,"session.snapshot",{}); + expect(await sidecar.next(frame=>frame.id === 5)).toMatchObject({ok:true,result:{runId:"next-run",turnId:null}}); + send(6,"turn.start",{turnId:"next-turn",message:"Complete the fixture turn."}); + expect(await sidecar.next(frame=>frame.id === 6)).toMatchObject({ok:true}); + expect(await sidecar.next(frame=>frame.eventType === "runtime.turn_terminal")) + .toMatchObject({runId:"next-run",turnId:"next-turn",payload:{status:"completed"}}); + send(7,"run.attach",{runId:"later-run",catalogRevision:1,tools:[]}); + expect(await sidecar.next(frame=>frame.id === 7)).toMatchObject({ok:false, + error:{message:"cannot attach another run after this sidecar started a turn"}}); + send(8,"session.snapshot",{}); + expect(await sidecar.next(frame=>frame.id === 8)).toMatchObject({ok:true,result:{runId:"next-run",turnId:null}}); + } finally {await sidecar.close();children.delete(sidecar);rmSync(root,{recursive:true,force:true});} + }, 15_000); + it("recovers after malformed input and reports its qualified Codex profile", async () => { const sidecar = startSidecar(); sidecar.write({ diff --git a/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.ts b/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.ts index a95d6a4f7a6..941ae867f17 100644 --- a/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.ts +++ b/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.ts @@ -176,6 +176,7 @@ let openParams: AcpxSidecarOpenParams | null = null; let claudeCompactionCount = 0; let runId: string | null = null; let turnId: string | null = null; +let turnStartedInProcess = false; /** * A restored session runs in a new provider process whose MCP request ids * restart. The controller keys semantic calls by run and call id, so prefix @@ -402,6 +403,7 @@ async function dispatch( if (request.command === "run.attach") { requireHost(); if (turnId) throw new Error("cannot attach a run during an active turn"); + if (turnStartedInProcess) throw new Error("cannot attach another run after this sidecar started a turn"); const attachedTools = parseTools(request.params.tools); if ( canonicalJson(attachedTools) !== canonicalJson(openParams?.tools ?? []) @@ -424,6 +426,7 @@ async function dispatch( const currentTurnId = boundedIdentity(request.params.turnId, "turnId"); const activeAgent = openParams!.agent; turnId = currentTurnId; + turnStartedInProcess = true; turnControls.begin(currentTurnId); let runtimeTurn: AcpxRuntimeTurn; const extensions = bindAcpxExtensionTurn({ diff --git a/server/src/services/native-runtime/plugin-runner-eviction.integration.test.ts b/server/src/services/native-runtime/plugin-runner-eviction.integration.test.ts index c10017c71e9..837787940cc 100644 --- a/server/src/services/native-runtime/plugin-runner-eviction.integration.test.ts +++ b/server/src/services/native-runtime/plugin-runner-eviction.integration.test.ts @@ -296,7 +296,9 @@ describeEmbeddedPostgres("plugin Runner eviction through the Core execution entr expect((await state()).lifecycle).toBe("suspended"); } else { const providerPids: number[] = harness === "codex" ? [Number(await readFile(providerPidFile, "utf8"))] - : (await readFile(calls, "utf8")).trim().split(/\r?\n/).flatMap(line => { const entry = JSON.parse(line); return [entry.processStart, entry.childPid].filter(Number.isInteger); }); + : (await readFile(calls, "utf8")).trim().split(/\r?\n/).flatMap(line => { const entry = JSON.parse(line); return [entry.processStart, entry.childPid].filter(Number.isInteger); }) + .filter(pid => { try { process.kill(pid, 0); return true; } catch (error) { if ((error as NodeJS.ErrnoException).code === "ESRCH") return false; throw error; } }); + expect(providerPids.length).toBeGreaterThan(0); const nativePid = handles.get(taskA)!.child.pid!; for (const pid of providerPids) { const [parent, group] = execFileSync("ps", ["-p", String(pid), "-o", "ppid=", "-o", "pgid="], { encoding: "utf8" }).trim().split(/\s+/).map(Number); diff --git a/server/src/services/native-runtime/plugin-runner-release-resume.integration.test.ts b/server/src/services/native-runtime/plugin-runner-release-resume.integration.test.ts index 4da3feb7710..fed25e693ab 100644 --- a/server/src/services/native-runtime/plugin-runner-release-resume.integration.test.ts +++ b/server/src/services/native-runtime/plugin-runner-release-resume.integration.test.ts @@ -298,7 +298,7 @@ describeEmbeddedPostgres("plugin Runner release/resume through the Core executio expect(methods.filter(method => method === "turn/start")).toHaveLength(adopted ? 3 : 2); } else { const journal = methods.map(line => JSON.parse(line)); - expect(journal.filter(entry => entry.processStart)).toHaveLength(adopted ? 3 : 2); + expect(journal.filter(entry => entry.processStart)).toHaveLength(adopted ? 4 : 3); expect(journal.filter(entry => entry.command === "turn.start")).toHaveLength(adopted ? 3 : 2); } } catch (error) {