diff --git a/packages/paperclip-runner/src/contracts/completion-result.test.ts b/packages/paperclip-runner/src/contracts/completion-result.test.ts index e01308fa806..0f3e14eafb3 100644 --- a/packages/paperclip-runner/src/contracts/completion-result.test.ts +++ b/packages/paperclip-runner/src/contracts/completion-result.test.ts @@ -324,6 +324,9 @@ describe("semantic result conflict feedback", () => { expect(prpSemanticResultConflictMessage(blocked("system", "tool_unavailable"))).toBe(PRP_ENVIRONMENT_BLOCKER_ALREADY_REPORTED_MESSAGE); expect(prpSemanticResultConflictMessage(blocked("user", "credential_missing"))).toBe(PRP_BLOCKER_ALREADY_REPORTED_MESSAGE); expect(prpSemanticResultConflictMessage(blocked("system", "credential_missing"))).toBe(PRP_BLOCKER_ALREADY_REPORTED_MESSAGE); + expect(prpSemanticResultConflictMessage(blocked("system", "tool_output_failure"))).toBe(PRP_ENVIRONMENT_BLOCKER_ALREADY_REPORTED_MESSAGE); + expect(prpSemanticResultConflictMessage(blocked("agent", "tool_unavailable"))).toBe(PRP_ENVIRONMENT_BLOCKER_ALREADY_REPORTED_MESSAGE); + expect(prpSemanticResultConflictMessage(blocked("agent", "work_incomplete"))).toBe(PRP_BLOCKER_ALREADY_REPORTED_MESSAGE); expect(PRP_BLOCKER_ALREADY_REPORTED_MESSAGE).not.toContain("automatically"); }); diff --git a/packages/paperclip-runner/src/contracts/completion-result.ts b/packages/paperclip-runner/src/contracts/completion-result.ts index 6ec8aef35aa..4ea01718a94 100644 --- a/packages/paperclip-runner/src/contracts/completion-result.ts +++ b/packages/paperclip-runner/src/contracts/completion-result.ts @@ -17,7 +17,19 @@ export const PRP_BLOCKER_ALREADY_REPORTED_MESSAGE = export const PRP_ENVIRONMENT_BLOCKER_ALREADY_REPORTED_MESSAGE = PRP_BLOCKER_ALREADY_REPORTED_MESSAGE + " Paperclip continues this environment or tool blocker automatically in a fresh session that checks the saved work."; -const PRP_ENVIRONMENT_BLOCKER_REASON_CODES: ReadonlySet = new Set(["environment_unavailable", "tool_unavailable"]); +export const PRP_ENVIRONMENT_BLOCKER_REASON_CODES: ReadonlySet = new Set(["environment_unavailable", "tool_unavailable"]); +export const PRP_PERSON_BLOCKER_REASON_CODES: ReadonlySet = new Set(["permission_denied", "credential_missing", "policy_restricted", "budget_exhausted"]); + +export function isPrpAutomaticallyContinuedBlocker(result: { + reportedWorkDisposition?: unknown; + blocker?: { owner?: { kind?: unknown } | null; reasonCode?: unknown } | null; +} | null | undefined): boolean { + if (result?.reportedWorkDisposition !== "blocked") return false; + const ownerKind = result.blocker?.owner?.kind; + const reasonCode = typeof result.blocker?.reasonCode === "string" ? result.blocker.reasonCode : ""; + return (ownerKind === "system" && !PRP_PERSON_BLOCKER_REASON_CODES.has(reasonCode)) + || (ownerKind === "agent" && PRP_ENVIRONMENT_BLOCKER_REASON_CODES.has(reasonCode)); +} /** Feedback for a later report in a turn that already committed a different one. */ export function prpSemanticResultConflictMessage(committed: { @@ -25,7 +37,7 @@ export function prpSemanticResultConflictMessage(committed: { blocker?: { owner?: { kind?: string } | null; reasonCode?: string } | null; } | null | undefined): string { if (committed?.reportedWorkDisposition !== "blocked") return PRP_RESULT_ALREADY_ACCEPTED_MESSAGE; - return committed.blocker?.owner?.kind === "system" && PRP_ENVIRONMENT_BLOCKER_REASON_CODES.has(committed.blocker.reasonCode ?? "") + return isPrpAutomaticallyContinuedBlocker(committed) ? PRP_ENVIRONMENT_BLOCKER_ALREADY_REPORTED_MESSAGE : PRP_BLOCKER_ALREADY_REPORTED_MESSAGE; } export const PRP_SEMANTIC_TOOL_NAMES = [ diff --git a/packages/shared/src/types/execution-projection.ts b/packages/shared/src/types/execution-projection.ts index 7a4b8567cb1..58b80ab6fc3 100644 --- a/packages/shared/src/types/execution-projection.ts +++ b/packages/shared/src/types/execution-projection.ts @@ -169,7 +169,7 @@ export interface TaskInterruptionPresentation { /** Board confirms that a previous server, which can no longer report, stopped its workspace copyback. */ | "confirm_owner_stop" /** Board confirms that a run whose server restarted before it recorded the agent's process has stopped. */ - | "confirm_run_stopped" | null; + | "confirm_run_stopped" | "unblock" | null; /** Only the task's own project id is exposed. Other source projects stay unnamed. */ projectAccess?: { agentId: string; projectId: string | null; ready: boolean }; sourceRunIds: string[]; diff --git a/server/src/__tests__/heartbeat-blocked-question-answer.test.ts b/server/src/__tests__/heartbeat-blocked-question-answer.test.ts index 995787b76d7..3aba0ef7572 100644 --- a/server/src/__tests__/heartbeat-blocked-question-answer.test.ts +++ b/server/src/__tests__/heartbeat-blocked-question-answer.test.ts @@ -83,7 +83,7 @@ describeEmbeddedPostgres("an answer to the agent's own question continues the ta } async function agentReportsBlocker(input: { companyId: string; agentId: string; issueId: string; - blocker?: Record }): Promise { + blocker?: Record; arbiter?: Partial[0]> }): Promise { const runId = randomUUID(), contractId = randomUUID(), resultId = randomUUID(), assessmentId = randomUUID(); await db.insert(completionContracts).values({ id: contractId, companyId: input.companyId, issueId: input.issueId, revision: 1, schemaVersion: "paperclip.completion-contract.v1", policyVersion: "test", risk: "standard", @@ -113,6 +113,7 @@ describeEmbeddedPostgres("an answer to the agent's own question continues the ta evidence: [], verification: [], attentionRequests: [], artifacts: [] } }), terminalState: "succeeded", workspaceFinalizeStatus: "succeeded", agentId: input.agentId, priorIssueStatus: snapshot!.status as "in_review", + ...input.arbiter, }) : { policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, statusAction: "blocked" as const, toStatus: "blocked" as const, reasonCode: "current_track_blocker_waiting", unblockDescriptor: { owner: "board" as const, action: "Answer the pending codename question." }, @@ -121,6 +122,7 @@ describeEmbeddedPostgres("an answer to the agent's own question continues the ta db, companyId: input.companyId, issueId: input.issueId, runId, assessmentId, priorStatus: snapshot!.status, priorStatusVersion: Number(snapshot!.statusVersion), priorDecisionId: snapshot!.lastStatusDecisionId, decision, + ...(input.arbiter ? { requireProviderFailureOwner: { agentId: input.agentId, reviewContext: null } } : {}), }); await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, runId)); } @@ -177,6 +179,35 @@ describeEmbeddedPostgres("an answer to the agent's own question continues the ta expect({ status: run?.status, errorCode: run?.errorCode ?? null }).toEqual({ status: "succeeded", errorCode: null }); }, 60_000); + it.each([["agent", { kind: "agent", name: "Writer" }], ["system", { kind: "system", name: "Paperclip" }]])( + "delivers the answer once Paperclip hands the agent's or the system's blocker to a person (%s)", async (_label, owner) => { + mockAdapterExecute.mockClear(); + const { agentId, issueId, question, companyId } = await seed(); + await agentReportsBlocker({ companyId, agentId, issueId, arbiter: { agentBlockerRecoverable: true, failureRetryCount: 1 }, blocker: { owner, scope: "task_wide", + reasonCode: "credential_missing", unblockAction: "Resolve the blocker." } }); + const [blocked] = await db.select().from(issues).where(eq(issues.id, issueId)); + expect(blocked).toMatchObject({ status: "blocked", unblockDescriptor: { owner: "board" } }); + const [decision] = await db.select().from(statusDecisions).where(eq(statusDecisions.id, blocked!.lastStatusDecisionId!)); + expect(decision).toMatchObject({ reasonCode: "native_agent_blocker_needs_person", decisionJson: { awaitingInteractionIds: [question.id] } }); + await answer(question); + const run = await deliverAnswer({ agentId, issueId, questionId: question.id }); + expect({ status: run?.status, errorCode: run?.errorCode ?? null }).toEqual({ status: "succeeded", errorCode: null }); + }, 60_000); + + it("delivers the answer after the automatic environment retries are spent", async () => { + mockAdapterExecute.mockClear(); + const { agentId, issueId, question, companyId } = await seed(); + await agentReportsBlocker({ companyId, agentId, issueId, arbiter: { environmentBlocker: true, failureRetryCount: 2 }, + blocker: { owner: { kind: "system", name: "Paperclip" }, scope: "task_wide", reasonCode: "tool_output_failure", unblockAction: "Resolve the blocker." } }); + const [blocked] = await db.select().from(issues).where(eq(issues.id, issueId)); + expect(blocked).toMatchObject({ status: "blocked", unblockDescriptor: { owner: "board" } }); + const [decision] = await db.select().from(statusDecisions).where(eq(statusDecisions.id, blocked!.lastStatusDecisionId!)); + expect(decision).toMatchObject({ reasonCode: "native_environment_blocker_exhausted", decisionJson: { awaitingInteractionIds: [question.id] } }); + await answer(question); + const run = await deliverAnswer({ agentId, issueId, questionId: question.id }); + expect({ status: run?.status, errorCode: run?.errorCode ?? null }).toEqual({ status: "succeeded", errorCode: null }); + }, 60_000); + it.each([["agent", { kind: "agent", name: "Writer" }], ["system", { kind: "system", name: "Paperclip" }]])( "keeps cancelling the answer for a blocker the agent or the system owns (%s)", async (_label, owner) => { mockAdapterExecute.mockClear(); diff --git a/server/src/modules/run-dispatch/adapters/postgres.test.ts b/server/src/modules/run-dispatch/adapters/postgres.test.ts index 2c7a17fafe2..81c52459ff4 100644 --- a/server/src/modules/run-dispatch/adapters/postgres.test.ts +++ b/server/src/modules/run-dispatch/adapters/postgres.test.ts @@ -358,7 +358,8 @@ describeEmbeddedPostgres("run-dispatch postgres adapter", () => { expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, competingId)))[0]?.status).toBe("running"); }); - it.each(["native_safe_replacement", "native_provider_overloaded", "native_environment_blocker", "interrupted_preparation"])("does not dispatch %s when the task becomes blocked after scheduling", async retryReason => { + it.each(["native_safe_replacement", "native_provider_overloaded", "native_environment_blocker", "native_agent_blocker_continuation", + "interrupted_preparation"])("does not dispatch %s when the task becomes blocked after scheduling", async retryReason => { const { companyId, agentId } = await seedCompanyAndAgent(); const issueId = randomUUID(); await seedIssue({ companyId, issueId, assigneeAgentId: agentId, status: "in_progress" }); diff --git a/server/src/modules/run-dispatch/adapters/postgres.ts b/server/src/modules/run-dispatch/adapters/postgres.ts index e2e24bd6c9f..592a8d1517d 100644 --- a/server/src/modules/run-dispatch/adapters/postgres.ts +++ b/server/src/modules/run-dispatch/adapters/postgres.ts @@ -181,7 +181,8 @@ function classifyRetryReasonKind(retryReason: string | null): RetryReasonKind { if (retryReason === MAX_TURN_CONTINUATION_RETRY_REASON) return "max_turn_continuation"; if (retryReason === ISSUE_DISPOSITION_REPAIR_RETRY_REASON) return "disposition_repair"; if ((retryReason === "ai_connection_busy" || retryReason === "ai_connection_pool_wait")) return "ai_connection_wait"; - if (["native_safe_replacement", "native_provider_overloaded", "native_environment_blocker", "interrupted_preparation"].includes(retryReason ?? "")) return "native_safe_replacement"; + if (["native_safe_replacement", "native_provider_overloaded", "native_environment_blocker", "native_agent_blocker_continuation", + "interrupted_preparation"].includes(retryReason ?? "")) return "native_safe_replacement"; return "other"; } diff --git a/server/src/services/blocked-question-answer.ts b/server/src/services/blocked-question-answer.ts index da298f9260d..ea970fc50ab 100644 --- a/server/src/services/blocked-question-answer.ts +++ b/server/src/services/blocked-question-answer.ts @@ -2,8 +2,11 @@ import { and, eq, inArray, isNotNull, isNull, sql } from "drizzle-orm"; import { heartbeatRuns, issueThreadInteractions, issues, statusDecisions, type Db } from "@paperclipai/db"; import { queuedInteractionId } from "./queued-interaction-response.js"; import { ordinaryQuestionCondition } from "./issue-question-context.js"; +import { NATIVE_AGENT_BLOCKER_HANDOFF_CODE } from "./native-runtime/native-agent-blocker.js"; +import { NATIVE_ENVIRONMENT_BLOCKER_EXHAUSTED_CODE } from "./native-runtime/native-environment-blocker.js"; -const AGENT_REPORTED_BLOCKER_REASON_CODES = ["current_track_blocker_waiting", "task_wide_blocker_bound"]; +const BOARD_BOUND_BLOCKER_REASON_CODES = [NATIVE_AGENT_BLOCKER_HANDOFF_CODE, NATIVE_ENVIRONMENT_BLOCKER_EXHAUSTED_CODE]; +const AGENT_REPORTED_BLOCKER_REASON_CODES = ["current_track_blocker_waiting", "task_wide_blocker_bound", ...BOARD_BOUND_BLOCKER_REASON_CODES]; const PERSON_RESOLVED_BLOCKER_OWNER_KINDS = ["user", "board", "external", "unspecified"]; export type AgentBlockingQuestionState = { lastStatusDecisionId: string; statusVersion: number }; @@ -12,7 +15,7 @@ export async function questionsAwaitedByAgentBlock(db: Pick, input companyId: string; issueId: string; agentId: string | null | undefined; reasonCode: string | null; toStatus: string; unblockOwner: unknown; blockerOwnerKind?: string; }): Promise { - const personResolves = input.blockerOwnerKind === undefined + const personResolves = input.blockerOwnerKind === undefined || BOARD_BOUND_BLOCKER_REASON_CODES.includes(input.reasonCode ?? "") ? input.unblockOwner === "board" : PERSON_RESOLVED_BLOCKER_OWNER_KINDS.includes(input.blockerOwnerKind); if (!input.agentId || input.toStatus !== "blocked" || !personResolves diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 09e63fc3c90..200faf00f62 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -3130,7 +3130,7 @@ export function heartbeatService( } const enforceContinuationOwnership = Boolean(issueId && (isFailedChatRunRetry || isResolvedInteractionContinuationWakeContext(context) || typeof context.workspaceReconstructionActionId === "string" || - ["native_safe_replacement", "native_provider_overloaded", "native_environment_blocker"].includes(run.scheduledRetryReason ?? ""))); + ["native_safe_replacement", "native_provider_overloaded", "native_environment_blocker", "native_agent_blocker_continuation"].includes(run.scheduledRetryReason ?? ""))); if (issueId && enforceContinuationOwnership) { await options.beforeResolvedInteractionContinuationDispatchCheck?.({ runId: run.id, issueId }); await options.afterResolvedInteractionContinuationDispatchCheck?.({ runId: run.id, issueId }); diff --git a/server/src/services/native-runtime/native-agent-blocker.integration.test.ts b/server/src/services/native-runtime/native-agent-blocker.integration.test.ts new file mode 100644 index 00000000000..c9d99e1ad51 --- /dev/null +++ b/server/src/services/native-runtime/native-agent-blocker.integration.test.ts @@ -0,0 +1,169 @@ +import { randomUUID } from "node:crypto"; +import { and, eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { agents, agentWakeupRequests, companies, companyMemberships, completionContracts, createDb, heartbeatRuns, issues, + nativeRunFinalizations, statusDecisions } from "@paperclipai/db"; +import type { ControlPlanePort, NativeExecutionInputV1, PrpStructuredRunResult, PrpTerminalState } from "@paperclipai/paperclip-runner"; +import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; +import { CONTROL_PLANE_CONFORMANCE_RESULT } from "../../vendor/paperclip-runner/testing.js"; + +const provider = vi.hoisted(() => ({ execute: vi.fn() })); +vi.mock("../../vendor/paperclip-runner/index.js", async (importOriginal) => ({ + ...(await importOriginal()), executeNativeSession: provider.execute, +})); +import { executePaperclipNativeSession } from "./native-session-executor.js"; +import { buildNativeCompletionContract } from "./completion-contracts.js"; +import { prepareNativeHeartbeatRun } from "./prepare-native-run.js"; +import { finalizeNativeRun } from "./native-run-finalizer.js"; +import { NATIVE_ENVIRONMENT_BLOCKER_CODE, NATIVE_ENVIRONMENT_BLOCKER_HINT, NATIVE_ENVIRONMENT_BLOCKER_MESSAGE } from "./native-environment-blocker.js"; +import { NATIVE_AGENT_BLOCKER_CONTINUATION_CODE, NATIVE_AGENT_BLOCKER_HANDOFF_CODE, NATIVE_AGENT_BLOCKER_HINT } from "./native-agent-blocker.js"; +import { taskInterruptionPresentation } from "../task-interruption-presentation.js"; +import { requestTaskRetry } from "../task-retry.js"; +import { heartbeatService } from "../heartbeat.js"; + +type Blocker = NonNullable; + +const toolOutputBlocker: Blocker = { + owner: { kind: "system", name: "execution environment" }, scope: "task_wide", reasonCode: "tool_output_failure", + unblockAction: "Resolve the persistent tool-output failure ('cryptic' returned for every tool call) and/or provide the current contents of the ledger file.", +}; +const selfBlocker: Blocker = { + owner: { kind: "agent", name: "ledger worker" }, scope: "task_wide", reasonCode: "work_incomplete", + unblockAction: "Re-read the ledger file and save the ledger document.", +}; + +describe("native run that reports a task-wide blocker it owns", () => { + let temporary: Awaited>; + let db: ReturnType; + const companyId = randomUUID(), agentId = randomUUID(), boardUserId = "ledger-board-user"; + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("native-agent-blocker-"); + db = createDb(temporary.connectionString); + await db.insert(companies).values({ id: companyId, name: "Agent blocker regression", issuePrefix: "AGB" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Ledger worker", status: "active", adapterType: "paperclip_runner", + adapterConfig: { provider: "opencode", model: "test-model" } }); + await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: boardUserId, status: "active", membershipRole: "member" }); + }); + afterAll(async () => { await temporary?.cleanup(); }); + + async function createTask() { + const issueId = randomUUID(); + await db.insert(issues).values({ id: issueId, companyId, title: "Ledger task", status: "in_progress", workMode: "standard", + assigneeAgentId: agentId, createdByUserId: boardUserId, responsibleUserId: boardUserId }); + return issueId; + } + + async function runTurn(issueId: string, report: { blocker: Blocker } | { done: true }, runId: string = randomUUID()) { + const [existing] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + if (!existing) await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, responsibleUserId: boardUserId, contextSnapshot: { issueId } }); + const [issue] = await db.select().from(issues).where(eq(issues.id, issueId)); + const [run] = await db.update(heartbeatRuns).set({ status: "running", runtimeMode: "native", nativeIssueId: issueId, startedAt: new Date() }) + .where(eq(heartbeatRuns.id, runId)).returning(); + await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId)); + const native = await prepareNativeHeartbeatRun({ db, run, issue, environmentLeaseId: randomUUID() }); + const [contract] = await db.select().from(completionContracts).where(eq(completionContracts.issueId, issueId)); + const contractInput = buildNativeCompletionContract(issue, { revision: contract.revision }); + await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId, phase: "observed" }); + const execution: NativeExecutionInputV1 = { + schema: "paperclip.native-execution-input.v1", binding: { companyId, runId, issueId, agentId, executionWorkspaceId: runId }, + provider: { kind: "codex", model: "test-model" }, + task: { identifier: issueId, title: issue.title, description: null, prompt: "Continue the task using its history.", workMode: "standard" }, + workspace: { cwd: "/tmp", repoUrl: null, repoRef: null, branchName: null }, + session: { normalizedSessionId: native.normalizedSessionId, driverKind: "codex_app_server", protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } }, + completionContract: { id: contract.id, sha256: contract.canonicalSha256, schemaVersion: "paperclip.completion-contract.v1", contract: contractInput }, + interactionResponses: [], credentialBindings: [], + }; + await db.update(heartbeatRuns).set({ runnerProfileJson: { nativeExecutionInput: execution } }).where(eq(heartbeatRuns.id, runId)); + const disposition = "blocker" in report ? "blocked" as const : "done" as const; + const terminal: PrpTerminalState = { schema: "paperclip.prp.terminal.v1", turnTerminalState: "completed", runTerminalState: "succeeded", reportedWorkDisposition: disposition }; + const result: PrpStructuredRunResult = "blocker" in report + ? { ...CONTROL_PLANE_CONFORMANCE_RESULT, reportedWorkDisposition: "blocked", + summary: "All rounds ran, but every tool call returned 'cryptic', so the ledger could not be read.", + completionClaim: { contractRevision: contractInput.revision, objectiveSatisfied: false, + criteria: contractInput.criteria.map(criterion => ({ criterionId: criterion.id, status: "not_satisfied" as const, evidenceRefs: [] })), + remainingWork: [{ description: "Save the ledger document", blocksCompletion: true }] }, + evidence: [], verification: [], attentionRequests: [], blocker: report.blocker } + : { ...CONTROL_PLANE_CONFORMANCE_RESULT, reportedWorkDisposition: "done", summary: "Saved the ledger and posted the comment.", + completionClaim: { contractRevision: contractInput.revision, objectiveSatisfied: true, + criteria: contractInput.criteria.map(criterion => ({ criterionId: criterion.id, status: "satisfied" as const, evidenceRefs: [] })), + remainingWork: [] }, + evidence: [], verification: [], attentionRequests: [] }; + provider.execute.mockReset().mockImplementation(async (options: { controlPlane: ControlPlanePort }) => { + await options.controlPlane.openRun({ identity: { companyId, issueId, runId, agentId, sessionId: native.normalizedSessionId }, backendKind: "mock", sourceInstanceId: native.runnerInstanceId }); + await options.controlPlane.completeRun({ result, terminal, turnId: native.turnId }); + return { result, terminal, turnId: native.turnId, normalizedSessionId: native.normalizedSessionId, providerSessionId: "fixture-session", driverKind: "codex_app_server", driverVersion: "fixture", nativeEventCount: 0, highestContiguousSourceSeq: 0, usage: null }; + }); + const unexpected = async () => { throw new Error("Unexpected provider call"); }; + await executePaperclipNativeSession({ db, execution, runnerInstanceId: native.runnerInstanceId, backend: { descriptor: unexpected, openSession: unexpected } }); + for (let replay = 0; replay < 2; replay++) await finalizeNativeRun({ db, runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true }); + const [decision] = await db.select().from(statusDecisions).where(eq(statusDecisions.runId, runId)); + const retries = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.retryOfRunId, runId))); + const [settled] = await db.select().from(issues).where(eq(issues.id, issueId)); + return { runId, decision: decision!, retries, issue: settled! }; + } + + const ownerWakes = async (issueId: string) => (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId))) + .filter(wake => wake.payload?.issueId === issueId && wake.requestedByActorId === "native-status-committer"); + + it("continues a tool-output blocker in a fresh session, and the continuation finishes the task", async () => { + const issueId = await createTask(); + const first = await runTurn(issueId, { blocker: toolOutputBlocker }); + expect(first.decision.reasonCode).toBe(NATIVE_ENVIRONMENT_BLOCKER_CODE); + expect(first.issue).toMatchObject({ status: "in_progress", unblockDescriptor: null }); + expect(first.retries).toHaveLength(1); + expect(first.retries[0]).toMatchObject({ status: "scheduled_retry", scheduledRetryReason: NATIVE_ENVIRONMENT_BLOCKER_CODE, + error: NATIVE_ENVIRONMENT_BLOCKER_MESSAGE, + contextSnapshot: { forceFreshSession: true, livenessContinuationInstruction: NATIVE_ENVIRONMENT_BLOCKER_HINT } }); + expect(NATIVE_ENVIRONMENT_BLOCKER_MESSAGE).not.toContain("unavailable"); + expect(NATIVE_ENVIRONMENT_BLOCKER_HINT).toContain("Check whether that blocker still applies"); + expect(await ownerWakes(issueId)).toEqual([]); + const second = await runTurn(issueId, { done: true }, first.retries[0]!.id); + expect(second.issue.status).toBe("done"); + }, 120_000); + + it("continues a blocker the agent owns once, then hands a repeated one to a person instead of retrying again", async () => { + const issueId = await createTask(); + const first = await runTurn(issueId, { blocker: selfBlocker }); + expect(first.decision.reasonCode).toBe(NATIVE_AGENT_BLOCKER_CONTINUATION_CODE); + expect(first.issue).toMatchObject({ status: "in_progress", unblockDescriptor: null }); + expect(first.retries).toHaveLength(1); + expect(first.retries[0]).toMatchObject({ status: "scheduled_retry", scheduledRetryReason: NATIVE_AGENT_BLOCKER_CONTINUATION_CODE, + scheduledRetryAttempt: 1, contextSnapshot: { forceFreshSession: true, livenessContinuationInstruction: NATIVE_AGENT_BLOCKER_HINT } }); + expect(await taskInterruptionPresentation(db, first.issue)).toMatchObject({ canRetry: false }); + const second = await runTurn(issueId, { blocker: selfBlocker }, first.retries[0]!.id); + expect(second.decision.reasonCode).toBe(NATIVE_AGENT_BLOCKER_HANDOFF_CODE); + expect(second.retries).toHaveLength(0); + expect(second.issue).toMatchObject({ status: "blocked", unblockDescriptor: { owner: "board", action: selfBlocker.unblockAction } }); + expect(await ownerWakes(issueId)).toEqual([]); + }, 120_000); + + it("hands a system blocker that needs a credential to a person without automatic retries", async () => { + const issueId = await createTask(); + const blocked = await runTurn(issueId, { blocker: { ...toolOutputBlocker, reasonCode: "credential_missing", + unblockAction: "Connect the deploy account." } }); + expect(blocked.decision.reasonCode).toBe(NATIVE_AGENT_BLOCKER_HANDOFF_CODE); + expect(blocked.retries).toHaveLength(0); + expect(blocked.issue).toMatchObject({ status: "blocked", unblockDescriptor: { owner: "board", action: "Connect the deploy account." } }); + }, 120_000); + + it("hands a blocker that only a person can resolve to the board with a Retry that starts the task again", async () => { + const issueId = await createTask(); + const blocked = await runTurn(issueId, { blocker: { ...selfBlocker, owner: { kind: "external", name: "deploy key holder" }, + reasonCode: "credential_missing", unblockAction: "Add the deploy key to the project." } }); + expect(blocked.decision.reasonCode).toBe(NATIVE_AGENT_BLOCKER_HANDOFF_CODE); + expect(blocked.retries).toHaveLength(0); + expect(blocked.issue).toMatchObject({ status: "blocked", unblockDescriptor: { owner: "board", action: "Add the deploy key to the project." } }); + expect(await ownerWakes(issueId)).toEqual([]); + expect(await taskInterruptionPresentation(db, blocked.issue)).toMatchObject({ responsibility: "user", state: "waiting_for_user", + userAction: "unblock", canRetry: true, observedRunId: blocked.runId }); + const receipt = await requestTaskRetry({ db, companyId, issueId, actorId: boardUserId, actorSource: "session", + idempotencyKey: randomUUID(), observedRunId: blocked.runId }); + expect(receipt).toMatchObject({ accepted: true, stale: false }); + const heartbeat = heartbeatService(db); + await heartbeat.resumeTaskRetry(companyId, receipt.requestId!); + const retried = (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.agentId, agentId))) + .filter(run => run.contextSnapshot?.issueId === issueId && run.contextSnapshot?.taskRetryRequestId === receipt.requestId); + expect(retried).toHaveLength(1); + await heartbeat.drainActiveRunExecutions(); + }, 120_000); +}); diff --git a/server/src/services/native-runtime/native-agent-blocker.ts b/server/src/services/native-runtime/native-agent-blocker.ts new file mode 100644 index 00000000000..f64391bd574 --- /dev/null +++ b/server/src/services/native-runtime/native-agent-blocker.ts @@ -0,0 +1,11 @@ +export const NATIVE_AGENT_BLOCKER_CONTINUATION_CODE = "native_agent_blocker_continuation"; +export const NATIVE_AGENT_BLOCKER_HANDOFF_CODE = "native_agent_blocker_needs_person"; +export const NATIVE_AGENT_BLOCKER_MAX_CONTINUATIONS = 1; +export const NATIVE_AGENT_BLOCKER_CONTINUATION_DELAY_MS = 30_000; +export const NATIVE_AGENT_BLOCKER_MESSAGE = + "The agent reported a blocker it owns. Paperclip continues the task once in a fresh session."; +export const NATIVE_AGENT_BLOCKER_HINT = "The previous run stopped on a blocker that it said the agent itself owns. " + + "Paperclip started this fresh session once to continue. Earlier attempts may already have completed some steps: " + + "check the workspace files, this task's documents and its comments before repeating a step, and never post a comment or save a document that already exists. " + + "Resolve the blocker yourself if you can, then finish the remaining work and report the real outcome. " + + "If only a person can resolve it, report the blocker with owner kind user and the exact action that person must take."; diff --git a/server/src/services/native-runtime/native-completion-format-feedback.test.ts b/server/src/services/native-runtime/native-completion-format-feedback.test.ts index 4a43b230858..8794940811c 100644 --- a/server/src/services/native-runtime/native-completion-format-feedback.test.ts +++ b/server/src/services/native-runtime/native-completion-format-feedback.test.ts @@ -12,7 +12,7 @@ vi.mock("./automatic-completion-reviews.js", () => ({ findAutomaticCompletionRev vi.mock("../agent-invokability.js", () => ({ evaluateAgentInvokabilityFromDb: vi.fn() })); vi.mock("../issues.js", () => ({ issueService: () => ({ getDependencyReadiness: async () => ({ unresolvedBlockerCount: state.blockers }) }) })); vi.mock("./native-review-participant.js", () => ({ readNativeReviewAssignmentContext: () => null, getNativeReviewAssignment: vi.fn() })); -vi.mock("../../vendor/paperclip-runner/index.js", () => ({ normalizePrpResultSignals: () => ({ verification: [], actionableAttentionRequests: [] }) })); +vi.mock("../../vendor/paperclip-runner/index.js", async (importOriginal) => ({ ...(await importOriginal()), normalizePrpResultSignals: () => ({ verification: [], actionableAttentionRequests: [] }) })); const result = { reportedWorkDisposition: "done", summary: "EXACT_PRIVATE_MARKER", completionClaim: { objectiveSatisfied: true, criteria: [], remainingWork: [] } } as unknown as PrpStructuredRunResult; function database(options: { interaction?: boolean; approval?: boolean; review?: boolean } = {}): Db { diff --git a/server/src/services/native-runtime/native-environment-blocker.ts b/server/src/services/native-runtime/native-environment-blocker.ts index 75f85c577c0..ebd381d91e5 100644 --- a/server/src/services/native-runtime/native-environment-blocker.ts +++ b/server/src/services/native-runtime/native-environment-blocker.ts @@ -1,22 +1,16 @@ +import { isPrpAutomaticallyContinuedBlocker } from "../../vendor/paperclip-runner/index.js"; + export const NATIVE_ENVIRONMENT_BLOCKER_CODE = "native_environment_blocker"; export const NATIVE_ENVIRONMENT_BLOCKER_EXHAUSTED_CODE = "native_environment_blocker_exhausted"; export const NATIVE_ENVIRONMENT_BLOCKER_MAX_RETRIES = 2; export const NATIVE_ENVIRONMENT_BLOCKER_RETRY_DELAY_MS = 30_000; export const NATIVE_ENVIRONMENT_BLOCKER_MESSAGE = - "The agent reported its execution environment unavailable. Paperclip continues the task in a fresh session."; -export const NATIVE_ENVIRONMENT_BLOCKER_HINT = "The previous run stopped because it reported its execution environment or tools unavailable. " + "The agent reported a blocker in its execution environment, tools or platform. Paperclip continues the task in a fresh session."; +export const NATIVE_ENVIRONMENT_BLOCKER_HINT = "The previous run stopped on a blocker that it said its execution environment, tools or platform caused. " + "Paperclip started this fresh session to continue. Earlier attempts may already have completed some steps: " + "check the workspace files, this task's documents and its comments before repeating a step, and never post a comment or save a document that already exists. " - + "Then finish the remaining work and report the real outcome. If a tool result looks wrong, retry that step once before reporting a blocker."; + + "Check whether that blocker still applies, then finish the remaining work and report the real outcome. If a tool result looks wrong, retry that step once before reporting a blocker."; export const NATIVE_ENVIRONMENT_BLOCKER_EXHAUSTED_ACTION = - "The agent still reported its execution environment unavailable after Paperclip continued the task in fresh sessions. The saved work is kept. Comment on the task when the agent should continue."; + "The agent still reported a blocker in its execution environment, tools or platform after Paperclip continued the task in fresh sessions. The saved work is kept. Comment on the task when the agent should continue."; -const ENVIRONMENT_BLOCKER_REASON_CODES: ReadonlySet = new Set(["environment_unavailable", "tool_unavailable"]); - -export function isNativeEnvironmentBlocker(result: { - reportedWorkDisposition?: unknown; - blocker?: { owner?: { kind?: unknown } | null; reasonCode?: unknown } | null; -}) { - return result.reportedWorkDisposition === "blocked" && result.blocker?.owner?.kind === "system" - && typeof result.blocker.reasonCode === "string" && ENVIRONMENT_BLOCKER_REASON_CODES.has(result.blocker.reasonCode); -} +export const isNativeEnvironmentBlocker = isPrpAutomaticallyContinuedBlocker; diff --git a/server/src/services/native-runtime/native-provider-capacity-retry.ts b/server/src/services/native-runtime/native-provider-capacity-retry.ts index 224818a8d87..98d5db313e0 100644 --- a/server/src/services/native-runtime/native-provider-capacity-retry.ts +++ b/server/src/services/native-runtime/native-provider-capacity-retry.ts @@ -9,6 +9,8 @@ import { NATIVE_TOOL_RECEIPT_LIMIT_CODE, NATIVE_TOOL_RECEIPT_LIMIT_CONTINUATION_ NATIVE_TOOL_RECEIPT_LIMIT_MAX_CONTINUATIONS, NATIVE_TOOL_RECEIPT_LIMIT_MESSAGE, nativeRunHasStopRequest } from "./native-tool-receipt-limit.js"; import { NATIVE_ENVIRONMENT_BLOCKER_CODE, NATIVE_ENVIRONMENT_BLOCKER_HINT, NATIVE_ENVIRONMENT_BLOCKER_MAX_RETRIES, NATIVE_ENVIRONMENT_BLOCKER_MESSAGE, NATIVE_ENVIRONMENT_BLOCKER_RETRY_DELAY_MS } from "./native-environment-blocker.js"; +import { NATIVE_AGENT_BLOCKER_CONTINUATION_CODE, NATIVE_AGENT_BLOCKER_CONTINUATION_DELAY_MS, NATIVE_AGENT_BLOCKER_HINT, + NATIVE_AGENT_BLOCKER_MAX_CONTINUATIONS, NATIVE_AGENT_BLOCKER_MESSAGE } from "./native-agent-blocker.js"; import { NativeStatusRaceError } from "./status-decision-committer.js"; type NativeRetryTarget = { @@ -76,6 +78,23 @@ export async function materializeNativeEnvironmentBlockerRetry(input: NativeRetr }); } +export async function materializeNativeAgentBlockerContinuation(input: NativeRetryTarget) { + return materializeNativeFreshSessionRetry(input, { + reason: NATIVE_AGENT_BLOCKER_CONTINUATION_CODE, + maxRetries: NATIVE_AGENT_BLOCKER_MAX_CONTINUATIONS, + delayMs: () => NATIVE_AGENT_BLOCKER_CONTINUATION_DELAY_MS, + message: NATIVE_AGENT_BLOCKER_MESSAGE, + idempotencyPrefix: "native-agent-blocker", + exhaustedError: "native_agent_blocker_continuation_exhausted", + refuseStopped: true, + context: run => ({ + livenessContinuationSourceRunId: run.id, + livenessContinuationReason: NATIVE_AGENT_BLOCKER_CONTINUATION_CODE, + livenessContinuationInstruction: NATIVE_AGENT_BLOCKER_HINT, + }), + }); +} + async function materializeNativeFreshSessionRetry(input: NativeRetryTarget, lane: NativeRetryLane) { const [run] = await input.tx.select().from(heartbeatRuns).where(and( eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.runId), diff --git a/server/src/services/native-runtime/native-run-finalizer.ts b/server/src/services/native-runtime/native-run-finalizer.ts index 74ba5f15276..12346fca829 100644 --- a/server/src/services/native-runtime/native-run-finalizer.ts +++ b/server/src/services/native-runtime/native-run-finalizer.ts @@ -34,6 +34,7 @@ import { isFailedTurnContinuationRun } from "./native-failed-turn-continuation.j import { readPersistedNativeFailureEvidence } from "./native-provider-failure-evidence.js"; import { NATIVE_TOOL_RECEIPT_LIMIT_CODE, cancelStoppedRunContinuation, nativeRunHasStopRequest, readPersistedNativeToolReceiptLimitStop } from "./native-tool-receipt-limit.js"; import { NATIVE_ENVIRONMENT_BLOCKER_CODE, NATIVE_ENVIRONMENT_BLOCKER_EXHAUSTED_CODE, isNativeEnvironmentBlocker } from "./native-environment-blocker.js"; +import { NATIVE_AGENT_BLOCKER_CONTINUATION_CODE, NATIVE_AGENT_BLOCKER_HANDOFF_CODE } from "./native-agent-blocker.js"; import { explainFailedRun, explainNativeTurnFailure, NATIVE_RUN_FAILED_UNEXPLAINED } from "./native-run-failure-explanation.js"; import { dismissAutomaticCompletionReviews } from "./automatic-completion-reviews.js"; import { findSupersededExternalReviewRequests } from "./superseded-external-review-requests.js"; @@ -1701,7 +1702,7 @@ export async function finalizeNativeRun(input: { issueId: authoritativeIssue.id, runId: run.id, }), - (terminalState === "failed" || (terminalState === "succeeded" && isNativeEnvironmentBlocker(result)) || + (terminalState === "failed" || (terminalState === "succeeded" && result.reportedWorkDisposition === "blocked") || (assessment.reportedDisposition === "yielded" && assessment.continuation?.kind === "response_wake" && assessment.hasBlockingRemainingWork)) @@ -1773,6 +1774,8 @@ export async function finalizeNativeRun(input: { && !stopRequested && await readPersistedNativeToolReceiptLimitStop(input.db, run, resultRow.turnId), environmentBlocker: terminalState === "succeeded" && ownsProviderFailureDecision && !reviewContext && !stopRequested && isNativeEnvironmentBlocker(result), + agentBlockerRecoverable: terminalState === "succeeded" && ownsProviderFailureDecision && !reviewContext && !stopRequested + && result.reportedWorkDisposition === "blocked", failureRetryCount: executionFailureRetryCount(run), // Only the task's current owner may turn its own failed continuation // into a blocker; the committer rechecks that under the issue lock. @@ -1938,6 +1941,8 @@ export async function finalizeNativeRun(input: { || decision.reasonCode === NATIVE_TOOL_RECEIPT_LIMIT_CODE || decision.reasonCode === NATIVE_ENVIRONMENT_BLOCKER_CODE || decision.reasonCode === NATIVE_ENVIRONMENT_BLOCKER_EXHAUSTED_CODE + || decision.reasonCode === NATIVE_AGENT_BLOCKER_CONTINUATION_CODE + || decision.reasonCode === NATIVE_AGENT_BLOCKER_HANDOFF_CODE ? { agentId: run.agentId, reviewContext } : undefined, requireBoardResponseWaitSource: decision.reasonCode === "board_response_waiting" || repairBoardResponseWait diff --git a/server/src/services/native-runtime/status-arbiter.test.ts b/server/src/services/native-runtime/status-arbiter.test.ts index 6d0ad84f2ef..88d28514df1 100644 --- a/server/src/services/native-runtime/status-arbiter.test.ts +++ b/server/src/services/native-runtime/status-arbiter.test.ts @@ -768,4 +768,20 @@ describe("native status authority", () => { expect(arbitrate({ assessment: blocked, environmentBlocker: true, ...held })).toMatchObject({ reasonCode: "task_wide_blocker_bound" }); } }); + + it("continues a blocker the agent owns once, then hands it to the board, and never binds it back to the agent", () => { + const blocked = (ownerKind: "agent" | "external") => assessment({ reportedDisposition: "blocked", objectiveSatisfied: false, + allCriteriaSatisfied: false, blocker: { unblockAction: "Save the ledger.", boardOwned: false, ownerKind, scope: "task_wide" } }); + expect(arbitrate({ assessment: blocked("agent"), agentBlockerRecoverable: true })).toMatchObject({ statusAction: "preserve", + reasonCode: "native_agent_blocker_continuation", effects: [{ kind: "schedule_retry", cause: "native_agent_blocker_continuation", agentId: "agent" }] }); + const handoff = { toStatus: "blocked", reasonCode: "native_agent_blocker_needs_person", + unblockDescriptor: { owner: "board", action: "Save the ledger." }, + effects: [{ kind: "bind_blocker", owner: "board", action: "Save the ledger." }] }; + expect(arbitrate({ assessment: blocked("agent"), agentBlockerRecoverable: true, failureRetryCount: 1 })).toMatchObject(handoff); + expect(arbitrate({ assessment: blocked("agent"), agentBlockerRecoverable: true, hasActivePauseHold: true })).toMatchObject(handoff); + expect(arbitrate({ assessment: blocked("external"), agentBlockerRecoverable: true })).toMatchObject(handoff); + for (const held of [{ isConversation: true }, { agentBlockerRecoverable: false }]) { + expect(arbitrate({ assessment: blocked("agent"), agentBlockerRecoverable: true, ...held })).toMatchObject({ reasonCode: "task_wide_blocker_bound" }); + } + }); }); diff --git a/server/src/services/native-runtime/status-arbiter.ts b/server/src/services/native-runtime/status-arbiter.ts index 2e3896847ef..dd285d34c00 100644 --- a/server/src/services/native-runtime/status-arbiter.ts +++ b/server/src/services/native-runtime/status-arbiter.ts @@ -4,6 +4,8 @@ import { NATIVE_MODEL_REJECTION_MESSAGE, NATIVE_PROVIDER_CAPACITY_MAX_RETRIES, N import { NATIVE_TOOL_RECEIPT_LIMIT_CODE, NATIVE_TOOL_RECEIPT_LIMIT_MAX_CONTINUATIONS } from "./native-tool-receipt-limit.js"; import { NATIVE_ENVIRONMENT_BLOCKER_CODE, NATIVE_ENVIRONMENT_BLOCKER_EXHAUSTED_ACTION, NATIVE_ENVIRONMENT_BLOCKER_EXHAUSTED_CODE, NATIVE_ENVIRONMENT_BLOCKER_MAX_RETRIES } from "./native-environment-blocker.js"; +import { NATIVE_AGENT_BLOCKER_CONTINUATION_CODE, NATIVE_AGENT_BLOCKER_HANDOFF_CODE, + NATIVE_AGENT_BLOCKER_MAX_CONTINUATIONS } from "./native-agent-blocker.js"; export const NATIVE_STATUS_ARBITER_POLICY_VERSION = "phase6-v14"; @@ -262,6 +264,7 @@ export function arbitrateNativeStatus(input: { toolReceiptLimitStopped?: boolean; /** The current owner's blocked report names a system-owned environment or tool outage. */ environmentBlocker?: boolean; + agentBlockerRecoverable?: boolean; failureRetryCount?: number; /** * This run was itself the automatic continuation of an earlier failed turn. @@ -653,6 +656,33 @@ export function arbitrateNativeStatus(input: { effects: [{ kind: "bind_blocker", owner: "board", action: NATIVE_ENVIRONMENT_BLOCKER_EXHAUSTED_ACTION }], }; } + if ( + input.assessment.reportedDisposition === "blocked" && + input.assessment.blocker?.scope === "task_wide" && + !input.assessment.blocker.boardOwned && + input.agentBlockerRecoverable && + !input.isConversation + ) { + if (input.assessment.blocker.ownerKind === "agent" && !input.hasActivePauseHold && + (input.failureRetryCount ?? 0) < NATIVE_AGENT_BLOCKER_MAX_CONTINUATIONS) { + return { + policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, + statusAction: "preserve", toStatus: input.priorIssueStatus, + reasonCode: NATIVE_AGENT_BLOCKER_CONTINUATION_CODE, unblockDescriptor: null, + effects: [{ kind: "schedule_retry", cause: NATIVE_AGENT_BLOCKER_CONTINUATION_CODE, + summary: "Continue the task once in a fresh session.", agentId: input.agentId }], + }; + } + const action = input.assessment.blocker.unblockAction; + return { + policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, + statusAction: "blocked", toStatus: "blocked", + reasonCode: NATIVE_AGENT_BLOCKER_HANDOFF_CODE, + blockerOwnerKind: input.assessment.blocker.ownerKind, + unblockDescriptor: { owner: "board", action }, + effects: [{ kind: "bind_blocker", owner: "board", action }], + }; + } if ( input.assessment.reportedDisposition === "blocked" && input.assessment.blocker diff --git a/server/src/services/native-runtime/status-decision-committer.ts b/server/src/services/native-runtime/status-decision-committer.ts index 9588635c781..cc44ecd1659 100644 --- a/server/src/services/native-runtime/status-decision-committer.ts +++ b/server/src/services/native-runtime/status-decision-committer.ts @@ -87,8 +87,10 @@ import { type ActivityPublication, } from "../activity-log.js"; import { emitAgentTaskRun } from "../agent-task-run-telemetry.js"; -import { materializeNativeEnvironmentBlockerRetry, materializeNativeProviderCapacityRetry, materializeNativeToolReceiptLimitContinuation } from "./native-provider-capacity-retry.js"; +import { materializeNativeAgentBlockerContinuation, materializeNativeEnvironmentBlockerRetry, materializeNativeProviderCapacityRetry, + materializeNativeToolReceiptLimitContinuation } from "./native-provider-capacity-retry.js"; import { NATIVE_ENVIRONMENT_BLOCKER_CODE, NATIVE_ENVIRONMENT_BLOCKER_EXHAUSTED_CODE } from "./native-environment-blocker.js"; +import { NATIVE_AGENT_BLOCKER_CONTINUATION_CODE, NATIVE_AGENT_BLOCKER_HANDOFF_CODE } from "./native-agent-blocker.js"; import { NATIVE_PROVIDER_OVERLOADED_CODE } from "./native-provider-failure.js"; import { NATIVE_TOOL_RECEIPT_LIMIT_CODE } from "./native-tool-receipt-limit.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; @@ -911,10 +913,11 @@ async function materializeDecisionEffect(input: { if (effect.kind === "schedule_retry") { failAt("continuation_materialization", input.failpoint); if (effect.cause === NATIVE_PROVIDER_OVERLOADED_CODE || effect.cause === NATIVE_TOOL_RECEIPT_LIMIT_CODE - || effect.cause === NATIVE_ENVIRONMENT_BLOCKER_CODE) { + || effect.cause === NATIVE_ENVIRONMENT_BLOCKER_CODE || effect.cause === NATIVE_AGENT_BLOCKER_CONTINUATION_CODE) { const materialize = effect.cause === NATIVE_PROVIDER_OVERLOADED_CODE ? materializeNativeProviderCapacityRetry : effect.cause === NATIVE_ENVIRONMENT_BLOCKER_CODE ? materializeNativeEnvironmentBlockerRetry - : materializeNativeToolReceiptLimitContinuation; + : effect.cause === NATIVE_AGENT_BLOCKER_CONTINUATION_CODE ? materializeNativeAgentBlockerContinuation + : materializeNativeToolReceiptLimitContinuation; const retryId = await materialize({ tx: input.tx as unknown as Db, companyId: input.companyId, issueId: input.issue.id, runId: input.runId, agentId: effect.agentId, @@ -1720,7 +1723,8 @@ export async function commitNativeStatusDecision(input: { } if ((reasonCode.startsWith("native_provider_overloaded") || reasonCode === "native_run_failed_continuation_exhausted" || reasonCode === NATIVE_TOOL_RECEIPT_LIMIT_CODE || reasonCode === NATIVE_ENVIRONMENT_BLOCKER_CODE - || reasonCode === NATIVE_ENVIRONMENT_BLOCKER_EXHAUSTED_CODE) && !input.requireProviderFailureOwner) { + || reasonCode === NATIVE_ENVIRONMENT_BLOCKER_EXHAUSTED_CODE || reasonCode === NATIVE_AGENT_BLOCKER_CONTINUATION_CODE + || reasonCode === NATIVE_AGENT_BLOCKER_HANDOFF_CODE) && !input.requireProviderFailureOwner) { throw new Error("native_provider_failure_owner_required"); } const publications: ActivityPublication[] = []; diff --git a/server/src/services/recovery/assigned-work-invariant.test.ts b/server/src/services/recovery/assigned-work-invariant.test.ts index 5bebab6a487..bb66ff96fd0 100644 --- a/server/src/services/recovery/assigned-work-invariant.test.ts +++ b/server/src/services/recovery/assigned-work-invariant.test.ts @@ -182,6 +182,7 @@ describe("assigned-work invariant", () => { ["approval", { interruption: { responsibility: "user", state: "waiting_for_user", userAction: "approval", nextRetryAt: null, canRetry: false } }], ["credential", { interruption: { responsibility: "user", state: "waiting_for_user", userAction: "reconnect", nextRetryAt: null, canRetry: false } }], ["board_confirmation", { interruption: { responsibility: "user", state: "waiting_for_user", userAction: "confirm_owner_stop", nextRetryAt: null, canRetry: false } }], + ["assignee_declared_block", { interruption: { responsibility: "user", state: "waiting_for_user", userAction: "unblock", nextRetryAt: null, canRetry: true } }], ["user_stop", { issue: { status: "blocked", blockedTransitionAt: ago(60 * 60_000) }, blockedBy: "user" }], ["assignee_declared_block", { issue: { status: "blocked", blockedTransitionAt: ago(60 * 60_000) }, blockedBy: "assignee" }], ["assignee_declared_block", { issue: { status: "blocked", blockedTransitionAt: ago(60 * 60_000), unblockOwner: "board" }, blockedBy: "unknown" }], diff --git a/server/src/services/recovery/assigned-work-invariant.ts b/server/src/services/recovery/assigned-work-invariant.ts index 9bca61b2f82..76e81ac9244 100644 --- a/server/src/services/recovery/assigned-work-invariant.ts +++ b/server/src/services/recovery/assigned-work-invariant.ts @@ -91,6 +91,7 @@ const PERSON_ACTIONS: Readonly> = { resume: "agent_paused", confirm_owner_stop: "board_confirmation", confirm_run_stopped: "board_confirmation", + unblock: "assignee_declared_block", }; /** Active recovery-action causes that name a person's move. Every other cause diff --git a/server/src/services/task-interruption-presentation.ts b/server/src/services/task-interruption-presentation.ts index 98e7d259545..6debf54ce92 100644 --- a/server/src/services/task-interruption-presentation.ts +++ b/server/src/services/task-interruption-presentation.ts @@ -6,7 +6,8 @@ import { and, desc, eq, inArray, isNull, not, notInArray, sql, type SQL } from " import { unidentifiedLocalLaunchRefusals } from "./native-local-process-stop.js"; import { nativeSessionCleanupBlock } from "../vendor/paperclip-runner/index.js"; import { agents, agentWakeupRequests, approvals, chatActions, companies, environmentLeases, heartbeatRuns, issueApprovals, issueRecoveryActions, - issues, issueTreeHoldMembers, issueTreeHolds, nativeRunFinalizations, projects, type Db } from "@paperclipai/db"; + issues, issueTreeHoldMembers, issueTreeHolds, nativeRunFinalizations, projects, statusDecisions, type Db } from "@paperclipai/db"; +import { NATIVE_AGENT_BLOCKER_HANDOFF_CODE } from "./native-runtime/native-agent-blocker.js"; import { unionAll } from "drizzle-orm/pg-core"; import { EXECUTION_RECONCILIATION_CAUSES, HEARTBEAT_RUN_STATUSES, readProviderFailure, type TaskInterruptionPresentation } from "@paperclipai/shared"; import { captureTaskInterruptionFailure } from "../sentry.js"; @@ -115,6 +116,7 @@ type ProjectionState = { automaticDispositionAt?: number; waitingContinuation?: { queuedAt: number; personRequested: boolean }; recoveryHandedToPerson?: boolean; + agentBlockerNeedsPerson?: boolean; }; // A task read reports the change that made a task fail, not every later read of @@ -257,7 +259,8 @@ function projectTaskInterruptionState(issue: ProjectionIssue, input: ProjectionS // Retry took effect: the saved result is being applied again. if (run.status === "succeeded" && run.nativeFinalizationRetryPending) return { ...base, state: "working", recoveryStep: "finishing_recovery" }; - if (run.status === "succeeded" && !wait && !run.nativeFinalizationRetry) return base; + if (run.status === "succeeded" && !wait && !run.nativeFinalizationRetry) return issue.status === "blocked" && input.agentBlockerNeedsPerson + ? { ...base, responsibility: "user", state: "waiting_for_user", canRetry: true, userAction: "unblock" } : base; const providerFailure = run.status === "failed" || run.status === "scheduled_retry" ? readProviderFailure(run.resultJson?.providerFailure) : null; if (providerFailure) base.providerFailure = providerFailure; @@ -732,6 +735,12 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i sql`coalesce(${issueRecoveryActions.evidence}#>>'{workspaceOwnerRecovery,state}', '') <> 'resuming'`, )) : []; const ownerStopRequired = new Set(ownerStops.map(row => `${row.issueId}:${row.runId}`)); + const blockedSucceededRunIds = runs.filter(run => run.status === "succeeded" + && selected.some(issue => issue.id === run.issueId && issue.status === "blocked")).map(run => run.id); + const agentBlockerHandoffRunIds = new Set(blockedSucceededRunIds.length ? (await db.select({ runId: statusDecisions.runId }) + .from(statusDecisions).where(and(eq(statusDecisions.companyId, companyId), inArray(statusDecisions.runId, blockedSucceededRunIds), + eq(statusDecisions.reasonCode, NATIVE_AGENT_BLOCKER_HANDOFF_CODE), eq(statusDecisions.applicationState, "applied")))) + .map(row => row.runId) : []); const byIssue = (rows: T[]) => new Map(rows.map(row => [row.issueId, row])); const runMap = byIssue(runs), waitMap = new Map(); // A Retry belongs to the run it observed. New conversation work supersedes @@ -837,6 +846,7 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i automaticDispositionAt: automaticDispositionAt.get(issue.id), waitingContinuation: waitingContinuationFor(facts.run), recoveryHandedToPerson: handedToPerson.has(issue.id), + agentBlockerNeedsPerson: agentBlockerHandoffRunIds.has(facts.run?.id ?? ""), interruptWait: interruptWaitFor(issue.id), projectAccessAllowed: denied?.length === 0, projectAccessProjectId: denied?.includes(issue.projectId ?? "") ? issue.projectId : null, diff --git a/server/src/vendor/paperclip-runner/index.ts b/server/src/vendor/paperclip-runner/index.ts index c9110058ab0..94de832ec95 100644 --- a/server/src/vendor/paperclip-runner/index.ts +++ b/server/src/vendor/paperclip-runner/index.ts @@ -95,6 +95,7 @@ export const CURSOR_CLOUD_DEFAULT_TIMEOUT_MS = runner.CURSOR_CLOUD_DEFAULT_TIMEO export const RUNNER_TOOL_SCOPE_TOKEN_HEADER = runner.RUNNER_TOOL_SCOPE_TOKEN_HEADER; export const DurablePrpControlPlane = runner.DurablePrpControlPlane; export const providerFailureFromError = runner.providerFailureFromError; +export const isPrpAutomaticallyContinuedBlocker = runner.isPrpAutomaticallyContinuedBlocker; export const safeErrorCauseCode = runner.safeErrorCauseCode; export const decodeDurablePrpCheckpoint = runner.decodeDurablePrpCheckpoint; export const captureLegacyDurablePrpCheckpoint = runner.captureLegacyDurablePrpCheckpoint; diff --git a/tests/runner-e2e/source-sdk-public-inputs.json b/tests/runner-e2e/source-sdk-public-inputs.json index 00674cbe3c3..bbf3767d0fe 100644 --- a/tests/runner-e2e/source-sdk-public-inputs.json +++ b/tests/runner-e2e/source-sdk-public-inputs.json @@ -413,7 +413,7 @@ "packages/shared/src/types/email.ts": "556ff6a0dcefa01eb704d6d00d3b191de81ff4dc", "packages/shared/src/types/environment.ts": "62e62afcb3d5b7f246db1e8910a34fb0091f54db", "packages/shared/src/types/execution-continuation.ts": "5afe62afd227efa03ff847d8f73291c14707ab5b", - "packages/shared/src/types/execution-projection.ts": "7a4b8567cb1a8204f9e9f878eb594b745a0428cc", + "packages/shared/src/types/execution-projection.ts": "58b80ab6fc38609fa6254f2805cae16392f291b0", "packages/shared/src/types/external-object.ts": "7a225e0f5a6777d449019a30530c858b753eb4c1", "packages/shared/src/types/feedback.ts": "c5eb31545c7f67c04ad9afd9db45a6b4774bcb35", "packages/shared/src/types/finance.ts": "5c2208c7950020f1005aa0dd86a1680c54dd4354", diff --git a/ui/src/components/ExecutionBlockerNotice.test.tsx b/ui/src/components/ExecutionBlockerNotice.test.tsx index 2b1d388ca16..060b4af8af9 100644 --- a/ui/src/components/ExecutionBlockerNotice.test.tsx +++ b/ui/src/components/ExecutionBlockerNotice.test.tsx @@ -38,6 +38,14 @@ describe("ordinary task interruption", () => { expect(container.querySelector("a, details, input, textarea")).toBeNull(); expect(container.textContent).not.toMatch(/service attention|process|receipt|workspace/i); }); + it("asks the person to help a blocked agent and offers Retry", async () => { + await render({ ...failed, responsibility: "user", state: "waiting_for_user", userAction: "unblock" }); + expect(container.textContent).toContain("needs your help to continue"); + expect(container.textContent).not.toMatch(/blocker_|native|owner kind/i); + await act(async () => container.querySelector('[data-testid="task-execution-retry"]')!.click()); + expect(retry).toHaveBeenCalledOnce(); + }); + it("keeps the failure visible and names the gate while a saved Retry waits on the budget", async () => { await render({ ...failed, responsibility: "user", state: "waiting_for_user", userAction: "budget", canRetry: false }); const notice = container.querySelector('[data-testid="task-execution-waiting"]'); diff --git a/ui/src/components/ExecutionBlockerNotice.tsx b/ui/src/components/ExecutionBlockerNotice.tsx index 59ad7ec2372..9ee5f5b2c7f 100644 --- a/ui/src/components/ExecutionBlockerNotice.tsx +++ b/ui/src/components/ExecutionBlockerNotice.tsx @@ -118,6 +118,16 @@ export function ExecutionBlockerNotice({ presentation, canRetry, onRetry, optimi ); + if (!optimistic && presentation?.userAction === "unblock") return ( +
+
+
+
+ ); if (optimistic || !presentation || presentation.responsibility !== "system" || !["failed", "retry_scheduled"].includes(presentation.state)) return null; return ( diff --git a/ui/src/pages/Inbox.tsx b/ui/src/pages/Inbox.tsx index afba4faf967..51708895392 100644 --- a/ui/src/pages/Inbox.tsx +++ b/ui/src/pages/Inbox.tsx @@ -417,6 +417,7 @@ export function FailedRunInboxRow({ project_access: "Agent needs project access", confirm_owner_stop: "Confirm the previous server stopped", confirm_run_stopped: "Confirm the previous run stopped", + unblock: "The agent needs your help", }[interruption.userAction ?? "resume"]) : runFailureMessage(run); const retryAvailable = !interruption || (interruption.canRetry && interruption.retryAction !== "move_forward"); const showUnreadSlot = unreadState !== null;