diff --git a/packages/paperclip-runner/src/contracts/completion-result.test.ts b/packages/paperclip-runner/src/contracts/completion-result.test.ts index 9431e18643d..191754955c6 100644 --- a/packages/paperclip-runner/src/contracts/completion-result.test.ts +++ b/packages/paperclip-runner/src/contracts/completion-result.test.ts @@ -2,6 +2,10 @@ import { normalizeLegacyPrpStructuredRunResult } from "../protocol/result-normal import { Ajv2020 } from "ajv/dist/2020.js"; import { describe, expect, it } from "vitest"; import { + PRP_BLOCKER_ALREADY_REPORTED_MESSAGE, + PRP_ENVIRONMENT_BLOCKER_ALREADY_REPORTED_MESSAGE, + PRP_SEMANTIC_RESULT_CONFLICT_MESSAGE, + prpSemanticResultConflictMessage, PRP_BLOCK_RESULT_OUTPUT_SCHEMA, PRP_BLOCK_RESULT_PROVIDER_INPUT_SCHEMA, PRP_BLOCK_TOOL_DESCRIPTION, @@ -291,3 +295,15 @@ describe("provider-neutral completion result schema", () => { expect(validate(result)).toBe(false); }); }); + +describe("semantic result conflict feedback", () => { + it("promises an automatic continuation only for a system environment or tool blocker", () => { + const blocked = (kind: string, reasonCode: string) => ({ reportedWorkDisposition: "blocked", blocker: { owner: { kind }, reasonCode } }); + expect(prpSemanticResultConflictMessage(blocked("system", "environment_unavailable"))).toBe(PRP_ENVIRONMENT_BLOCKER_ALREADY_REPORTED_MESSAGE); + expect(prpSemanticResultConflictMessage(blocked("system", "tool_unavailable"))).toBe(PRP_ENVIRONMENT_BLOCKER_ALREADY_REPORTED_MESSAGE); + expect(prpSemanticResultConflictMessage(blocked("user", "credential_missing"))).toBe(PRP_BLOCKER_ALREADY_REPORTED_MESSAGE); + expect(prpSemanticResultConflictMessage(blocked("system", "credential_missing"))).toBe(PRP_BLOCKER_ALREADY_REPORTED_MESSAGE); + expect(prpSemanticResultConflictMessage({ reportedWorkDisposition: "done" })).toBe(PRP_SEMANTIC_RESULT_CONFLICT_MESSAGE); + expect(PRP_BLOCKER_ALREADY_REPORTED_MESSAGE).not.toContain("automatically"); + }); +}); diff --git a/packages/paperclip-runner/src/contracts/completion-result.ts b/packages/paperclip-runner/src/contracts/completion-result.ts index ac59f6c4f8e..42750c88507 100644 --- a/packages/paperclip-runner/src/contracts/completion-result.ts +++ b/packages/paperclip-runner/src/contracts/completion-result.ts @@ -6,6 +6,24 @@ export const PRP_COMPLETION_TOOL_DESCRIPTION = "Report completed work (done), work requiring review (needs_review), or an explicit wait (yielded with response_wake for a response, monitor after set_task_monitor confirms a schedule on this task, or omit continuation to use this run's authorized published-review wait or saved repository-preparation continuation). Use the current completion contract and supporting evidence. If rejected, correct the report and retry. After acceptance, read the returned outcome; do not claim completion while gated. Explain any required approval with its supplied link and action, then write the final response and end the turn without further tool calls."; export const PRP_BLOCK_TOOL_DESCRIPTION = "Report work that cannot continue because of a concrete blocker. Routine repository preparation is not a blocker: use paperclip_finish with yielded and omit continuation to resume through the saved workspace continuation. Identify the blocker, its owner, and the action needed to unblock it; use the current completion contract and supporting evidence. If rejected, correct the report and retry. After acceptance, read the returned outcome, explain the blocker and any required action in the final response, and end the turn without further tool calls."; +export const PRP_SEMANTIC_RESULT_CONFLICT_MESSAGE = "A different semantic result was already committed"; +export const PRP_BLOCKER_ALREADY_REPORTED_MESSAGE = + "This turn already reported a blocker, and a turn's first accepted report is final, so this report was not recorded. Do not report again in this turn. " + + "In your final response, list the steps that are complete and say whether the blocker still applies, then end the turn."; +export const PRP_ENVIRONMENT_BLOCKER_ALREADY_REPORTED_MESSAGE = PRP_BLOCKER_ALREADY_REPORTED_MESSAGE + + " Paperclip continues this environment or tool blocker automatically in a fresh session that checks the saved work."; + +const PRP_ENVIRONMENT_BLOCKER_REASON_CODES: ReadonlySet = new Set(["environment_unavailable", "tool_unavailable"]); + +/** Feedback for a later report in a turn that already committed a different one. */ +export function prpSemanticResultConflictMessage(committed: { + reportedWorkDisposition: string; + blocker?: { owner?: { kind?: string } | null; reasonCode?: string } | null; +} | null | undefined): string { + if (committed?.reportedWorkDisposition !== "blocked") return PRP_SEMANTIC_RESULT_CONFLICT_MESSAGE; + return committed.blocker?.owner?.kind === "system" && PRP_ENVIRONMENT_BLOCKER_REASON_CODES.has(committed.blocker.reasonCode ?? "") + ? PRP_ENVIRONMENT_BLOCKER_ALREADY_REPORTED_MESSAGE : PRP_BLOCKER_ALREADY_REPORTED_MESSAGE; +} export const PRP_SEMANTIC_TOOL_NAMES = [ PRP_COMPLETION_TOOL_NAME, PRP_BLOCK_TOOL_NAME, diff --git a/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.ts b/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.ts index 73751878397..da8f07e34bf 100644 --- a/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.ts +++ b/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.ts @@ -23,6 +23,7 @@ import type { import { PRP_BLOCK_TOOL_NAME, PRP_COMPLETION_TOOL_NAME, + prpSemanticResultConflictMessage, } from "../../contracts/completion-result.js"; import { HarnessCapabilityUnavailableError, @@ -1199,7 +1200,7 @@ class CodexAcpxSession implements HarnessSession { this.#semanticFingerprint !== null && this.#semanticFingerprint !== fingerprint ) { - throw new Error("A different semantic result is already committed"); + throw new Error(prpSemanticResultConflictMessage(this.#semanticResult)); } const claimsLaterTurn = this.#semanticFingerprint === fingerprint && diff --git a/packages/paperclip-runner/src/drivers/codex/codex-session-server-requests.ts b/packages/paperclip-runner/src/drivers/codex/codex-session-server-requests.ts index 95d3c329305..ccd7e223a7f 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-session-server-requests.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-session-server-requests.ts @@ -6,6 +6,7 @@ import { CODEX_BLOCK_TOOL_NAME, } from "../../contracts/codex.js"; import { validatePrpStructuredRunResult } from "../../protocol/replay-contract.js"; +import { prpSemanticResultConflictMessage } from "../../contracts/completion-result.js"; import type { CodexRpcServerRequest } from "./app-server-transport.js"; import { boundedCodexValue, @@ -235,9 +236,7 @@ async function handleServerRequestBody( } const admission = admitResult(state, validation.result, callId, turnId); if (admission === "conflict") { - return rejectedToolCall( - "A different semantic result was already committed.", - ); + return rejectedToolCall(prpSemanticResultConflictMessage(state.result)); } return { success: true, diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts index dddee26c6fa..f783077e503 100644 --- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts +++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts @@ -30,6 +30,7 @@ import { type NativeRuntimeContextSnapshot, } from "../../contracts/runtime-context.js"; import { createCodexTaskEnvelope } from "../../contracts/codex.js"; +import { PRP_BLOCKER_ALREADY_REPORTED_MESSAGE } from "../../contracts/completion-result.js"; import { CONFIGURED_ENVIRONMENT_KEYS, configuredEnvironmentProjection } from "../../configured-environment.js"; import { localIntegrityBoundaryGolden } from "../../../test-support/local-integrity-boundary-golden.js"; import type { PrpEvent } from "../../protocol/replay-contract.js"; @@ -2867,6 +2868,27 @@ describe("OpenCodeServerDriver", () => { await session.close({ reason: "test" }); }); + it("tells a provider that a turn's accepted blocker is final when it later tries to finish", async () => { + await chmod(fixture, 0o755); + const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-block-finish-")); + const workspace = await mkdtemp(join(tmpdir(), "paperclip-opencode-workspace-")); + roots.push(root, workspace); + const driver = new OpenCodeServerDriver({ model: "openrouter/deepseek/deepseek-v4-flash-0731", runtimeDirectory: root, command: fixture, environment: { PATH: process.env.PATH, OPENROUTER_API_KEY: "fixture-key" } }); + const session = await driver.openSession({ runId: "block-then-finish", normalizedSessionId: "block-then-finish", workingDirectory: workspace }); + await session.startTurn({ message: { role: "user", text: "block-result finish-after-block" } }); + const events = await collectTurnEvents(session.events()); + const results = events.filter((event) => event.eventType === "run.result.proposed"); + expect(results).toHaveLength(1); + expect(results[0].payload).toMatchObject({ reportedWorkDisposition: "blocked" }); + const files = await readdir(root, { recursive: true }); + const evidence = files.find((name) => name.endsWith("fake-finish-after-block.json")); + expect(evidence).toBeDefined(); + expect(JSON.parse(await readFile(join(root, evidence!), "utf8"))).toMatchObject({ result: { isError: true, + content: [{ text: expect.stringContaining(PRP_BLOCKER_ALREADY_REPORTED_MESSAGE) }] } }); + expect(JSON.stringify(JSON.parse(await readFile(join(root, evidence!), "utf8")))).not.toContain("continues this environment or tool blocker"); + await session.close({ reason: "test" }); + }); + it.each(["monitor", "response_wake"] as const)("resolves an omitted %s through the real OpenCode MCP boundary only from current server state", async kind => { for (const eligible of [true, false]) { await chmod(fixture, 0o755); diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts index 50ce8fbddbf..48452ee6dd9 100644 --- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts +++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts @@ -44,6 +44,7 @@ import { import { PRP_BLOCK_TOOL_NAME, PRP_COMPLETION_TOOL_NAME, + prpSemanticResultConflictMessage, } from "../../contracts/completion-result.js"; import type { HarnessDriver, @@ -1325,7 +1326,7 @@ class OpenCodeHarnessSession implements HarnessSession { } const fingerprint = canonicalJson(validation.result); if (this.#resultFingerprint && this.#resultFingerprint !== fingerprint) - throw new Error("A different semantic result was already committed"); + throw new Error(prpSemanticResultConflictMessage(this.#result)); // Wait for the bound controller before committing or resolving the // provider call. A rejection remains repairable in this same turn. const feedback = this.#completionFeedback @@ -1337,7 +1338,7 @@ class OpenCodeHarnessSession implements HarnessSession { if (typeof feedback !== "string" || !feedback.trim()) throw new Error("Completion feedback omitted its response text"); if (this.#resultFingerprint && this.#resultFingerprint !== fingerprint) - throw new Error("A different semantic result was already committed"); + throw new Error(prpSemanticResultConflictMessage(this.#result)); if (!this.#resultFingerprint) { this.#result = structuredClone(validation.result); this.#resultFingerprint = fingerprint; diff --git a/packages/paperclip-runner/test/fixtures/fake-opencode-server.mjs b/packages/paperclip-runner/test/fixtures/fake-opencode-server.mjs index 100ac75fca1..97ec466b340 100644 --- a/packages/paperclip-runner/test/fixtures/fake-opencode-server.mjs +++ b/packages/paperclip-runner/test/fixtures/fake-opencode-server.mjs @@ -311,6 +311,18 @@ async function callTerminalTool(promptBody, signal) { arguments: result, }, signal); const first = await call(); + if (blocked && String(prompt.message ?? "").includes("finish-after-block")) { + const done = structuredClone(result); + done.reportedWorkDisposition = "done"; + done.summary = "Fake provider recovered and completed the task."; + done.completionClaim.objectiveSatisfied = true; + done.completionClaim.criteria = done.completionClaim.criteria.map(criterion => ({ ...criterion, status: "satisfied" })); + done.completionClaim.remainingWork = []; + done.attentionRequests = []; + delete done.blocker; + const rejected = await mcpRequest("tools/call", { name: "paperclip_finish", arguments: done }); + await writeFile(join(process.env.XDG_DATA_HOME, "fake-finish-after-block.json"), JSON.stringify(rejected)); + } if (String(prompt.message ?? prompt.task?.prompt ?? "").includes("completion-feedback")) { const outcomes = [first]; if (first.result?.isError && String(prompt.message ?? prompt.task?.prompt ?? "").includes("current-wake-completion-feedback")) { diff --git a/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts b/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts index 9cad1909412..fdfcd6d300c 100644 --- a/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts +++ b/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts @@ -901,6 +901,34 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => { }, ); + it("cancels an environment-blocker fresh-session retry whose task is reassigned before adapter dispatch", async () => { + const { companyId, agentId } = await seedCompanyAndAgent(); + const replacementAgentId = randomUUID(); + await db.insert(agents).values({ id: replacementAgentId, companyId, name: "ReplacementCoder", role: "engineer", status: "active", + adapterType: "codex_local", adapterConfig: {}, runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 } }, permissions: {} }); + const issueId = randomUUID(); + await db.insert(issues).values({ id: issueId, companyId, title: "Environment blocker retry reassigned at final dispatch", + status: "in_progress", priority: "medium", assigneeAgentId: agentId }); + const { runId } = await seedQueuedRun({ companyId, agentId, issueId, wakeReason: "native_environment_blocker", + invocationSource: "automation", scheduledRetryReason: "native_environment_blocker", + contextExtras: { retryReason: "native_environment_blocker", forceFreshSession: true } }); + let guarded = false; + beforeContinuationDispatchCheck = async ({ runId: guardedRunId }) => { + if (guardedRunId !== runId) return; + guarded = true; + await db.update(issues).set({ assigneeAgentId: replacementAgentId, updatedAt: new Date() }).where(eq(issues.id, issueId)); + }; + + await heartbeat.resumeQueuedRuns(); + await heartbeat.drainActiveRunExecutions(); + + const [run] = await db.select({ status: heartbeatRuns.status, errorCode: heartbeatRuns.errorCode }) + .from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + expect(guarded).toBe(true); + expect(run).toMatchObject({ status: "cancelled", errorCode: "issue_assignee_changed" }); + expect(countExecuteCallsForRun(runId)).toBe(0); + }); + it("rejects ownership changes immediately before the final continuation handoff", async () => { const { companyId, agentId } = await seedCompanyAndAgent(); const issueId = randomUUID(); diff --git a/server/src/modules/run-dispatch/adapters/postgres.test.ts b/server/src/modules/run-dispatch/adapters/postgres.test.ts index e4b5ca41e92..2c7a17fafe2 100644 --- a/server/src/modules/run-dispatch/adapters/postgres.test.ts +++ b/server/src/modules/run-dispatch/adapters/postgres.test.ts @@ -358,7 +358,7 @@ describeEmbeddedPostgres("run-dispatch postgres adapter", () => { expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, competingId)))[0]?.status).toBe("running"); }); - it.each(["native_safe_replacement", "native_provider_overloaded", "interrupted_preparation"])("does not dispatch %s when the task becomes blocked after scheduling", async retryReason => { + it.each(["native_safe_replacement", "native_provider_overloaded", "native_environment_blocker", "interrupted_preparation"])("does not dispatch %s when the task becomes blocked after scheduling", async retryReason => { const { companyId, agentId } = await seedCompanyAndAgent(); const issueId = randomUUID(); await seedIssue({ companyId, issueId, assigneeAgentId: agentId, status: "in_progress" }); diff --git a/server/src/modules/run-dispatch/adapters/postgres.ts b/server/src/modules/run-dispatch/adapters/postgres.ts index e526d078ded..e68204939ec 100644 --- a/server/src/modules/run-dispatch/adapters/postgres.ts +++ b/server/src/modules/run-dispatch/adapters/postgres.ts @@ -180,7 +180,7 @@ function classifyRetryReasonKind(retryReason: string | null): RetryReasonKind { if (retryReason === MAX_TURN_CONTINUATION_RETRY_REASON) return "max_turn_continuation"; if (retryReason === ISSUE_DISPOSITION_REPAIR_RETRY_REASON) return "disposition_repair"; if ((retryReason === "ai_connection_busy" || retryReason === "ai_connection_pool_wait")) return "ai_connection_wait"; - if (["native_safe_replacement", "native_provider_overloaded", "interrupted_preparation"].includes(retryReason ?? "")) return "native_safe_replacement"; + if (["native_safe_replacement", "native_provider_overloaded", "native_environment_blocker", "interrupted_preparation"].includes(retryReason ?? "")) return "native_safe_replacement"; return "other"; } diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 3648d2c5770..8a692261e4b 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -3135,7 +3135,7 @@ export function heartbeatService( } const enforceContinuationOwnership = Boolean(issueId && (isFailedChatRunRetry || isResolvedInteractionContinuationWakeContext(context) || typeof context.workspaceReconstructionActionId === "string" || - ["native_safe_replacement", "native_provider_overloaded"].includes(run.scheduledRetryReason ?? ""))); + ["native_safe_replacement", "native_provider_overloaded", "native_environment_blocker"].includes(run.scheduledRetryReason ?? ""))); if (issueId && enforceContinuationOwnership) { await options.beforeResolvedInteractionContinuationDispatchCheck?.({ runId: run.id, issueId }); await options.afterResolvedInteractionContinuationDispatchCheck?.({ runId: run.id, issueId }); diff --git a/server/src/services/native-runtime/native-environment-blocker.integration.test.ts b/server/src/services/native-runtime/native-environment-blocker.integration.test.ts new file mode 100644 index 00000000000..2562e5e8e0c --- /dev/null +++ b/server/src/services/native-runtime/native-environment-blocker.integration.test.ts @@ -0,0 +1,125 @@ +import { randomUUID } from "node:crypto"; +import { and, eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { agents, companies, completionContracts, createDb, heartbeatRuns, issues, issueTreeHoldMembers, issueTreeHolds, nativeRunFinalizations, statusDecisionEffects, statusDecisions } from "@paperclipai/db"; +import type { ControlPlanePort, NativeExecutionInputV1, PrpStructuredRunResult, PrpTerminalState } from "@paperclipai/paperclip-runner"; +import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; +import { CONTROL_PLANE_CONFORMANCE_RESULT } from "../../vendor/paperclip-runner/testing.js"; + +const provider = vi.hoisted(() => ({ execute: vi.fn() })); +vi.mock("../../vendor/paperclip-runner/index.js", async (importOriginal) => ({ + ...(await importOriginal()), executeNativeSession: provider.execute, +})); +import { executePaperclipNativeSession } from "./native-session-executor.js"; +import { buildNativeCompletionContract } from "./completion-contracts.js"; +import { prepareNativeHeartbeatRun } from "./prepare-native-run.js"; +import { finalizeNativeRun } from "./native-run-finalizer.js"; +import { NATIVE_ENVIRONMENT_BLOCKER_CODE, NATIVE_ENVIRONMENT_BLOCKER_EXHAUSTED_ACTION, NATIVE_ENVIRONMENT_BLOCKER_EXHAUSTED_CODE, + NATIVE_ENVIRONMENT_BLOCKER_HINT, NATIVE_ENVIRONMENT_BLOCKER_MAX_RETRIES } from "./native-environment-blocker.js"; +import { taskInterruptionPresentation } from "../task-interruption-presentation.js"; + +type Blocker = NonNullable; + +const environmentBlocker: Blocker = { + owner: { kind: "system", name: "execution-environment" }, scope: "task_wide", reasonCode: "environment_unavailable", + unblockAction: "Resume/retry the run so the tool-result channel is healthy; then finish the remaining steps.", +}; + +describe("native run that reports a system environment blocker", () => { + let temporary: Awaited>; + let db: ReturnType; + const companyId = randomUUID(), agentId = randomUUID(); + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("native-environment-blocker-"); + db = createDb(temporary.connectionString); + await db.insert(companies).values({ id: companyId, name: "Environment blocker regression", issuePrefix: "ENV" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Environment worker", status: "active", adapterType: "paperclip_runner", adapterConfig: { provider: "opencode", model: "test-model" } }); + }); + afterAll(async () => { await temporary?.cleanup(); }); + + async function reportBlocked(blocker: Blocker, contextSnapshot: Record = {}, beforeFinalize?: (issueId: string) => Promise) { + const issueId = randomUUID(), runId = randomUUID(); + await db.insert(issues).values({ id: issueId, companyId, title: "Ledger task", status: "in_progress", assigneeAgentId: agentId, responsibleUserId: "env-owner" }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, responsibleUserId: "env-owner", contextSnapshot: { issueId, ...contextSnapshot } }); + const [issue] = await db.select().from(issues).where(eq(issues.id, issueId)); + const [run] = await db.update(heartbeatRuns).set({ status: "running", runtimeMode: "native", nativeIssueId: issueId }).where(eq(heartbeatRuns.id, runId)).returning(); + await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId)); + const native = await prepareNativeHeartbeatRun({ db, run, issue, environmentLeaseId: randomUUID() }); + const [contract] = await db.select().from(completionContracts).where(eq(completionContracts.issueId, issueId)); + const contractInput = buildNativeCompletionContract(issue, { revision: contract.revision }); + await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId, phase: "observed" }); + const execution: NativeExecutionInputV1 = { + schema: "paperclip.native-execution-input.v1", binding: { companyId, runId, issueId, agentId, executionWorkspaceId: runId }, + provider: { kind: "codex", model: "test-model" }, + task: { identifier: issueId, title: issue.title, description: null, prompt: "Continue the task using its history.", workMode: "standard" }, + workspace: { cwd: "/tmp", repoUrl: null, repoRef: null, branchName: null }, + session: { normalizedSessionId: native.normalizedSessionId, driverKind: "codex_app_server", protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } }, + completionContract: { id: contract.id, sha256: contract.canonicalSha256, schemaVersion: "paperclip.completion-contract.v1", contract: contractInput }, + interactionResponses: [], credentialBindings: [], + }; + await db.update(heartbeatRuns).set({ runnerProfileJson: { nativeExecutionInput: execution } }).where(eq(heartbeatRuns.id, runId)); + const terminal: PrpTerminalState = { schema: "paperclip.prp.terminal.v1", turnTerminalState: "completed", runTerminalState: "succeeded", reportedWorkDisposition: "blocked" }; + const result: PrpStructuredRunResult = { ...CONTROL_PLANE_CONFORMANCE_RESULT, reportedWorkDisposition: "blocked", + summary: "Tool results looked corrupted, so the run stopped.", + completionClaim: { contractRevision: contractInput.revision, objectiveSatisfied: false, + criteria: contractInput.criteria.map(criterion => ({ criterionId: criterion.id, status: "unknown" as const, evidenceRefs: [] })), + remainingWork: [{ description: "Finish the ledger", blocksCompletion: true }] }, + evidence: [], verification: [], attentionRequests: [], blocker }; + provider.execute.mockReset().mockImplementation(async (options: { controlPlane: ControlPlanePort }) => { + await options.controlPlane.openRun({ identity: { companyId, issueId, runId, agentId, sessionId: native.normalizedSessionId }, backendKind: "mock", sourceInstanceId: native.runnerInstanceId }); + await options.controlPlane.completeRun({ result, terminal, turnId: native.turnId }); + return { result, terminal, turnId: native.turnId, normalizedSessionId: native.normalizedSessionId, providerSessionId: "fixture-session", driverKind: "codex_app_server", driverVersion: "fixture", nativeEventCount: 0, highestContiguousSourceSeq: 0, usage: null }; + }); + const unexpected = async () => { throw new Error("Unexpected provider call"); }; + await executePaperclipNativeSession({ db, execution, runnerInstanceId: native.runnerInstanceId, backend: { descriptor: unexpected, openSession: unexpected } }); + await beforeFinalize?.(issueId); + for (let replay = 0; replay < 2; replay++) await finalizeNativeRun({ db, runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true }); + const [decision] = await db.select().from(statusDecisions).where(eq(statusDecisions.runId, runId)); + const retries = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.retryOfRunId, runId))); + const [settled] = await db.select().from(issues).where(eq(issues.id, issueId)); + return { runId, issueId, decision: decision!, retries, issue: settled! }; + } + + it("continues the task automatically in a fresh session instead of resting it blocked", async () => { + const { runId, decision, retries, issue } = await reportBlocked(environmentBlocker); + expect(decision).toMatchObject({ reasonCode: NATIVE_ENVIRONMENT_BLOCKER_CODE }); + expect(retries).toHaveLength(1); + expect(retries[0]).toMatchObject({ status: "scheduled_retry", scheduledRetryReason: NATIVE_ENVIRONMENT_BLOCKER_CODE, scheduledRetryAttempt: 1, + contextSnapshot: { forceFreshSession: true, livenessContinuationSourceRunId: runId, + livenessContinuationReason: NATIVE_ENVIRONMENT_BLOCKER_CODE, livenessContinuationInstruction: NATIVE_ENVIRONMENT_BLOCKER_HINT } }); + expect(await db.select().from(statusDecisionEffects).where(eq(statusDecisionEffects.decisionId, decision.id))) + .toEqual(expect.arrayContaining([expect.objectContaining({ effectKind: "schedule_retry", targetType: "heartbeat_run", targetId: retries[0]!.id })])); + expect(issue).toMatchObject({ status: "in_progress", unblockDescriptor: null }); + expect(await taskInterruptionPresentation(db, issue)).toMatchObject({ state: "retry_scheduled", canRetry: false, userAction: null }); + await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, retries[0]!.id)); + }); + + it("hands the task to the board once the automatic retries are spent", async () => { + const { decision, retries, issue } = await reportBlocked(environmentBlocker, { + executionRetryAccounting: { version: 1, failureRetries: NATIVE_ENVIRONMENT_BLOCKER_MAX_RETRIES, maxTurnContinuations: 0 }, + }); + expect(decision).toMatchObject({ reasonCode: NATIVE_ENVIRONMENT_BLOCKER_EXHAUSTED_CODE }); + expect(retries).toHaveLength(0); + expect(issue).toMatchObject({ status: "blocked", + unblockDescriptor: { owner: "board", action: NATIVE_ENVIRONMENT_BLOCKER_EXHAUSTED_ACTION } }); + expect(await taskInterruptionPresentation(db, issue)).toMatchObject({ canRetry: false }); + }); + + it("does not continue a paused task automatically", async () => { + const { decision, retries } = await reportBlocked(environmentBlocker, {}, async issueId => { + const holdId = randomUUID(); + await db.insert(issueTreeHolds).values({ id: holdId, companyId, rootIssueId: issueId, mode: "pause", status: "active" }); + await db.insert(issueTreeHoldMembers).values({ companyId, holdId, issueId, depth: 0, issueTitle: "Ledger task", issueStatus: "in_progress" }); + }); + expect(decision.reasonCode).not.toBe(NATIVE_ENVIRONMENT_BLOCKER_CODE); + expect(retries).toHaveLength(0); + }); + + it("keeps a blocker that only its owner can resolve", async () => { + const { decision, retries, issue } = await reportBlocked({ ...environmentBlocker, owner: { kind: "user", name: "board" }, + reasonCode: "credential_missing", unblockAction: "Add the deploy key." }); + expect(decision.reasonCode).toBe("task_wide_blocker_bound"); + expect(retries).toHaveLength(0); + expect(issue.status).toBe("blocked"); + }); +}); diff --git a/server/src/services/native-runtime/native-environment-blocker.ts b/server/src/services/native-runtime/native-environment-blocker.ts new file mode 100644 index 00000000000..75f85c577c0 --- /dev/null +++ b/server/src/services/native-runtime/native-environment-blocker.ts @@ -0,0 +1,22 @@ +export const NATIVE_ENVIRONMENT_BLOCKER_CODE = "native_environment_blocker"; +export const NATIVE_ENVIRONMENT_BLOCKER_EXHAUSTED_CODE = "native_environment_blocker_exhausted"; +export const NATIVE_ENVIRONMENT_BLOCKER_MAX_RETRIES = 2; +export const NATIVE_ENVIRONMENT_BLOCKER_RETRY_DELAY_MS = 30_000; +export const NATIVE_ENVIRONMENT_BLOCKER_MESSAGE = + "The agent reported its execution environment unavailable. Paperclip continues the task in a fresh session."; +export const NATIVE_ENVIRONMENT_BLOCKER_HINT = "The previous run stopped because it reported its execution environment or tools unavailable. " + + "Paperclip started this fresh session to continue. Earlier attempts may already have completed some steps: " + + "check the workspace files, this task's documents and its comments before repeating a step, and never post a comment or save a document that already exists. " + + "Then finish the remaining work and report the real outcome. If a tool result looks wrong, retry that step once before reporting a blocker."; +export const NATIVE_ENVIRONMENT_BLOCKER_EXHAUSTED_ACTION = + "The agent still reported its execution environment unavailable after Paperclip continued the task in fresh sessions. The saved work is kept. Comment on the task when the agent should continue."; + +const ENVIRONMENT_BLOCKER_REASON_CODES: ReadonlySet = new Set(["environment_unavailable", "tool_unavailable"]); + +export function isNativeEnvironmentBlocker(result: { + reportedWorkDisposition?: unknown; + blocker?: { owner?: { kind?: unknown } | null; reasonCode?: unknown } | null; +}) { + return result.reportedWorkDisposition === "blocked" && result.blocker?.owner?.kind === "system" + && typeof result.blocker.reasonCode === "string" && ENVIRONMENT_BLOCKER_REASON_CODES.has(result.blocker.reasonCode); +} diff --git a/server/src/services/native-runtime/native-provider-capacity-retry.ts b/server/src/services/native-runtime/native-provider-capacity-retry.ts index 79952d5869b..224818a8d87 100644 --- a/server/src/services/native-runtime/native-provider-capacity-retry.ts +++ b/server/src/services/native-runtime/native-provider-capacity-retry.ts @@ -7,6 +7,8 @@ import { accountingForScheduledRetry, executionFailureRetryCount } from "../exec import { NATIVE_PROVIDER_CAPACITY_MAX_RETRIES, NATIVE_PROVIDER_OVERLOADED_CODE, NATIVE_PROVIDER_OVERLOADED_MESSAGE } from "./native-provider-failure.js"; import { NATIVE_TOOL_RECEIPT_LIMIT_CODE, NATIVE_TOOL_RECEIPT_LIMIT_CONTINUATION_DELAY_MS, NATIVE_TOOL_RECEIPT_LIMIT_HINT, NATIVE_TOOL_RECEIPT_LIMIT_MAX_CONTINUATIONS, NATIVE_TOOL_RECEIPT_LIMIT_MESSAGE, nativeRunHasStopRequest } from "./native-tool-receipt-limit.js"; +import { NATIVE_ENVIRONMENT_BLOCKER_CODE, NATIVE_ENVIRONMENT_BLOCKER_HINT, NATIVE_ENVIRONMENT_BLOCKER_MAX_RETRIES, NATIVE_ENVIRONMENT_BLOCKER_MESSAGE, + NATIVE_ENVIRONMENT_BLOCKER_RETRY_DELAY_MS } from "./native-environment-blocker.js"; import { NativeStatusRaceError } from "./status-decision-committer.js"; type NativeRetryTarget = { @@ -57,6 +59,23 @@ export async function materializeNativeToolReceiptLimitContinuation(input: Nativ }); } +export async function materializeNativeEnvironmentBlockerRetry(input: NativeRetryTarget) { + return materializeNativeFreshSessionRetry(input, { + reason: NATIVE_ENVIRONMENT_BLOCKER_CODE, + maxRetries: NATIVE_ENVIRONMENT_BLOCKER_MAX_RETRIES, + delayMs: attempt => NATIVE_ENVIRONMENT_BLOCKER_RETRY_DELAY_MS * 2 ** (attempt - 1), + message: NATIVE_ENVIRONMENT_BLOCKER_MESSAGE, + idempotencyPrefix: "native-environment-blocker", + exhaustedError: "native_environment_blocker_retry_exhausted", + refuseStopped: true, + context: run => ({ + livenessContinuationSourceRunId: run.id, + livenessContinuationReason: NATIVE_ENVIRONMENT_BLOCKER_CODE, + livenessContinuationInstruction: NATIVE_ENVIRONMENT_BLOCKER_HINT, + }), + }); +} + async function materializeNativeFreshSessionRetry(input: NativeRetryTarget, lane: NativeRetryLane) { const [run] = await input.tx.select().from(heartbeatRuns).where(and( eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.runId), diff --git a/server/src/services/native-runtime/native-run-finalizer.ts b/server/src/services/native-runtime/native-run-finalizer.ts index eb5dd3977eb..74ba5f15276 100644 --- a/server/src/services/native-runtime/native-run-finalizer.ts +++ b/server/src/services/native-runtime/native-run-finalizer.ts @@ -33,6 +33,7 @@ import { executionFailureRetryCount } from "../execution-recovery-attempt.js"; import { isFailedTurnContinuationRun } from "./native-failed-turn-continuation.js"; import { readPersistedNativeFailureEvidence } from "./native-provider-failure-evidence.js"; import { NATIVE_TOOL_RECEIPT_LIMIT_CODE, cancelStoppedRunContinuation, nativeRunHasStopRequest, readPersistedNativeToolReceiptLimitStop } from "./native-tool-receipt-limit.js"; +import { NATIVE_ENVIRONMENT_BLOCKER_CODE, NATIVE_ENVIRONMENT_BLOCKER_EXHAUSTED_CODE, isNativeEnvironmentBlocker } from "./native-environment-blocker.js"; import { explainFailedRun, explainNativeTurnFailure, NATIVE_RUN_FAILED_UNEXPLAINED } from "./native-run-failure-explanation.js"; import { dismissAutomaticCompletionReviews } from "./automatic-completion-reviews.js"; import { findSupersededExternalReviewRequests } from "./superseded-external-review-requests.js"; @@ -1700,7 +1701,8 @@ export async function finalizeNativeRun(input: { issueId: authoritativeIssue.id, runId: run.id, }), - (terminalState === "failed" || (assessment.reportedDisposition === "yielded" && + (terminalState === "failed" || (terminalState === "succeeded" && isNativeEnvironmentBlocker(result)) || + (assessment.reportedDisposition === "yielded" && assessment.continuation?.kind === "response_wake" && assessment.hasBlockingRemainingWork)) ? issueTreeControlService(input.db).getActivePauseHoldGate( @@ -1769,6 +1771,8 @@ export async function finalizeNativeRun(input: { providerFailureSuperseded: providerFailure?.errorCode === "native_provider_overloaded" && !ownsProviderFailureDecision, toolReceiptLimitStopped: terminalState !== "succeeded" && ownsProviderFailureDecision && !reviewContext && !stopRequested && await readPersistedNativeToolReceiptLimitStop(input.db, run, resultRow.turnId), + environmentBlocker: terminalState === "succeeded" && ownsProviderFailureDecision && !reviewContext && !stopRequested + && isNativeEnvironmentBlocker(result), failureRetryCount: executionFailureRetryCount(run), // Only the task's current owner may turn its own failed continuation // into a blocker; the committer rechecks that under the issue lock. @@ -1932,6 +1936,8 @@ export async function finalizeNativeRun(input: { requireProviderFailureOwner: decision.reasonCode?.startsWith("native_provider_overloaded") || decision.reasonCode === "native_run_failed_continuation_exhausted" || decision.reasonCode === NATIVE_TOOL_RECEIPT_LIMIT_CODE + || decision.reasonCode === NATIVE_ENVIRONMENT_BLOCKER_CODE + || decision.reasonCode === NATIVE_ENVIRONMENT_BLOCKER_EXHAUSTED_CODE ? { agentId: run.agentId, reviewContext } : undefined, requireBoardResponseWaitSource: decision.reasonCode === "board_response_waiting" || repairBoardResponseWait diff --git a/server/src/services/native-runtime/status-arbiter.test.ts b/server/src/services/native-runtime/status-arbiter.test.ts index 2a90adef678..6d0ad84f2ef 100644 --- a/server/src/services/native-runtime/status-arbiter.test.ts +++ b/server/src/services/native-runtime/status-arbiter.test.ts @@ -757,4 +757,15 @@ describe("native status authority", () => { ]); }); + it("continues a system environment blocker in a fresh session, bounded, and never for a paused task or chat", () => { + const blocked = assessment({ reportedDisposition: "blocked", objectiveSatisfied: false, allCriteriaSatisfied: false, + blocker: { unblockAction: "Retry the run.", boardOwned: false, ownerKind: "system", scope: "task_wide" } }); + expect(arbitrate({ assessment: blocked, environmentBlocker: true })).toMatchObject({ statusAction: "preserve", + reasonCode: "native_environment_blocker", effects: [{ kind: "schedule_retry", cause: "native_environment_blocker", agentId: "agent" }] }); + expect(arbitrate({ assessment: blocked, environmentBlocker: true, failureRetryCount: 2 })).toMatchObject({ toStatus: "blocked", + reasonCode: "native_environment_blocker_exhausted", unblockDescriptor: { owner: "board" } }); + for (const held of [{ hasActivePauseHold: true }, { isConversation: true }, { environmentBlocker: false }]) { + expect(arbitrate({ assessment: blocked, environmentBlocker: true, ...held })).toMatchObject({ reasonCode: "task_wide_blocker_bound" }); + } + }); }); diff --git a/server/src/services/native-runtime/status-arbiter.ts b/server/src/services/native-runtime/status-arbiter.ts index 594f3011fef..2e3896847ef 100644 --- a/server/src/services/native-runtime/status-arbiter.ts +++ b/server/src/services/native-runtime/status-arbiter.ts @@ -2,6 +2,8 @@ import { COMPLETION_REVIEW_STAGE_REASON, COMPLETION_REVIEW_STAGE_REJECTED_REASON import type { NativeEvidenceAssessment } from "./evidence-classifier.js"; import { NATIVE_MODEL_REJECTION_MESSAGE, NATIVE_PROVIDER_CAPACITY_MAX_RETRIES, NATIVE_PROVIDER_OVERLOADED_CODE, NATIVE_PROVIDER_OVERLOADED_MESSAGE } from "./native-provider-failure.js"; import { NATIVE_TOOL_RECEIPT_LIMIT_CODE, NATIVE_TOOL_RECEIPT_LIMIT_MAX_CONTINUATIONS } from "./native-tool-receipt-limit.js"; +import { NATIVE_ENVIRONMENT_BLOCKER_CODE, NATIVE_ENVIRONMENT_BLOCKER_EXHAUSTED_ACTION, NATIVE_ENVIRONMENT_BLOCKER_EXHAUSTED_CODE, + NATIVE_ENVIRONMENT_BLOCKER_MAX_RETRIES } from "./native-environment-blocker.js"; export const NATIVE_STATUS_ARBITER_POLICY_VERSION = "phase6-v14"; @@ -258,6 +260,8 @@ export function arbitrateNativeStatus(input: { providerOverloaded?: boolean; providerFailureSuperseded?: boolean; toolReceiptLimitStopped?: boolean; + /** The current owner's blocked report names a system-owned environment or tool outage. */ + environmentBlocker?: boolean; failureRetryCount?: number; /** * This run was itself the automatic continuation of an earlier failed turn. @@ -624,6 +628,31 @@ export function arbitrateNativeStatus(input: { })), }; } + if ( + input.assessment.reportedDisposition === "blocked" && + input.assessment.blocker && + input.environmentBlocker && + !input.isConversation && + !input.hasActivePauseHold + ) { + if ((input.failureRetryCount ?? 0) < NATIVE_ENVIRONMENT_BLOCKER_MAX_RETRIES) { + return { + policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, + statusAction: "preserve", toStatus: input.priorIssueStatus, + reasonCode: NATIVE_ENVIRONMENT_BLOCKER_CODE, unblockDescriptor: null, + effects: [{ kind: "schedule_retry", cause: NATIVE_ENVIRONMENT_BLOCKER_CODE, + summary: "Continue the task in a fresh session.", agentId: input.agentId }], + }; + } + return { + policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, + statusAction: "blocked", toStatus: "blocked", + reasonCode: NATIVE_ENVIRONMENT_BLOCKER_EXHAUSTED_CODE, + blockerOwnerKind: input.assessment.blocker.ownerKind, + unblockDescriptor: { owner: "board", action: NATIVE_ENVIRONMENT_BLOCKER_EXHAUSTED_ACTION }, + effects: [{ kind: "bind_blocker", owner: "board", action: NATIVE_ENVIRONMENT_BLOCKER_EXHAUSTED_ACTION }], + }; + } if ( input.assessment.reportedDisposition === "blocked" && input.assessment.blocker diff --git a/server/src/services/native-runtime/status-decision-committer.ts b/server/src/services/native-runtime/status-decision-committer.ts index d64421cecc5..9588635c781 100644 --- a/server/src/services/native-runtime/status-decision-committer.ts +++ b/server/src/services/native-runtime/status-decision-committer.ts @@ -87,7 +87,8 @@ import { type ActivityPublication, } from "../activity-log.js"; import { emitAgentTaskRun } from "../agent-task-run-telemetry.js"; -import { materializeNativeProviderCapacityRetry, materializeNativeToolReceiptLimitContinuation } from "./native-provider-capacity-retry.js"; +import { materializeNativeEnvironmentBlockerRetry, materializeNativeProviderCapacityRetry, materializeNativeToolReceiptLimitContinuation } from "./native-provider-capacity-retry.js"; +import { NATIVE_ENVIRONMENT_BLOCKER_CODE, NATIVE_ENVIRONMENT_BLOCKER_EXHAUSTED_CODE } from "./native-environment-blocker.js"; import { NATIVE_PROVIDER_OVERLOADED_CODE } from "./native-provider-failure.js"; import { NATIVE_TOOL_RECEIPT_LIMIT_CODE } from "./native-tool-receipt-limit.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; @@ -909,9 +910,11 @@ async function materializeDecisionEffect(input: { } if (effect.kind === "schedule_retry") { failAt("continuation_materialization", input.failpoint); - if (effect.cause === NATIVE_PROVIDER_OVERLOADED_CODE || effect.cause === NATIVE_TOOL_RECEIPT_LIMIT_CODE) { - const materialize = effect.cause === NATIVE_PROVIDER_OVERLOADED_CODE - ? materializeNativeProviderCapacityRetry : materializeNativeToolReceiptLimitContinuation; + if (effect.cause === NATIVE_PROVIDER_OVERLOADED_CODE || effect.cause === NATIVE_TOOL_RECEIPT_LIMIT_CODE + || effect.cause === NATIVE_ENVIRONMENT_BLOCKER_CODE) { + const materialize = effect.cause === NATIVE_PROVIDER_OVERLOADED_CODE ? materializeNativeProviderCapacityRetry + : effect.cause === NATIVE_ENVIRONMENT_BLOCKER_CODE ? materializeNativeEnvironmentBlockerRetry + : materializeNativeToolReceiptLimitContinuation; const retryId = await materialize({ tx: input.tx as unknown as Db, companyId: input.companyId, issueId: input.issue.id, runId: input.runId, agentId: effect.agentId, @@ -1716,7 +1719,8 @@ export async function commitNativeStatusDecision(input: { throw new Error("native_model_rejection_owner_required"); } if ((reasonCode.startsWith("native_provider_overloaded") || reasonCode === "native_run_failed_continuation_exhausted" - || reasonCode === NATIVE_TOOL_RECEIPT_LIMIT_CODE) && !input.requireProviderFailureOwner) { + || reasonCode === NATIVE_TOOL_RECEIPT_LIMIT_CODE || reasonCode === NATIVE_ENVIRONMENT_BLOCKER_CODE + || reasonCode === NATIVE_ENVIRONMENT_BLOCKER_EXHAUSTED_CODE) && !input.requireProviderFailureOwner) { throw new Error("native_provider_failure_owner_required"); } const publications: ActivityPublication[] = [];