diff --git a/doc/architecture/durable-continuation-scheduler.md b/doc/architecture/durable-continuation-scheduler.md index ecb79c4bd37..41f810dfc65 100644 --- a/doc/architecture/durable-continuation-scheduler.md +++ b/doc/architecture/durable-continuation-scheduler.md @@ -100,6 +100,23 @@ another turn. Its invariant is: an assigned open issue should have either a live execution path, a durable reason to wait, or a visible terminal/blocking disposition. +Active pending interactions remain a reason to wait under the shared +interaction eligibility rules. A newer Retry alone does not answer a question. +Dismissed or historical ordinary questions remain answerable conversation +data, without blocking current work; governed requests retain their gates. +Assigned-work reconciliation leaves active question waits quiet, and the +existing interaction-resolution notification makes them eligible for another +check. The task and inbox show an active question as the current blocker, +while older provider failures remain in run history. +After a failed attempt, the question notice preserves an available manual +Retry; automatic recovery still waits. Existing saved input prevents a +duplicate Retry, and other current user gates retain their precedence. + +A provider-limit hold before provider work is scheduling, so it does not send +a cancellation toast for every held task. Its durable retry rechecks provider +availability. When the provider reports no reset time, the fallback deadline +is a recheck time rather than a promised reset or work-start time. + ## Why DOT-2 looped DOT-2's runner repeatedly produced a successful result that reported `done`. diff --git a/packages/shared/src/types/execution-projection.ts b/packages/shared/src/types/execution-projection.ts index 58b80ab6fc3..75ae5e13121 100644 --- a/packages/shared/src/types/execution-projection.ts +++ b/packages/shared/src/types/execution-projection.ts @@ -165,7 +165,7 @@ export interface TaskInterruptionPresentation { nextRetryAt: string | null; canRetry: boolean; retryAction?: "move_forward"; - userAction: "approval" | "reconnect" | "assign_agent" | "choose_model" | "resume" | "budget" | "project_access" + userAction: "approval" | "question" | "reconnect" | "assign_agent" | "choose_model" | "resume" | "budget" | "project_access" /** Board confirms that a previous server, which can no longer report, stopped its workspace copyback. */ | "confirm_owner_stop" /** Board confirms that a run whose server restarted before it recorded the agent's process has stopped. */ diff --git a/server/src/__tests__/assigned-work-invariant.test.ts b/server/src/__tests__/assigned-work-invariant.test.ts index 52d1d9b1367..5b3abbbd3da 100644 --- a/server/src/__tests__/assigned-work-invariant.test.ts +++ b/server/src/__tests__/assigned-work-invariant.test.ts @@ -275,7 +275,7 @@ describePostgres("assigned-work invariant sweep", () => { expect(await recovery.assignedWork.report(f.companyId)).toMatchObject({ violationCount: 1, heldByRecoveryOwner: 1 }); }, 60_000); - it("Retry-started tasks the scheduler refused to revalidate get one continuation, even with a question asked before the Retry", async () => { + it("continues a refused Retry only after its pending question is actually resolved", async () => { // Nine staging tasks: a person's Retry, stopped by a deploy during setup, // certified as no provider work; the scheduler then refused with // continuation_user_authorization_missing on every sweep. One of them @@ -289,7 +289,7 @@ describePostgres("assigned-work invariant sweep", () => { createdAt: new Date(Date.now() - 4 * HOUR) }); } await db.insert(issueThreadInteractions).values({ companyId: f.companyId, issueId: withQuestion, kind: "ask_user_questions", - status: "pending", continuationPolicy: "wake_assignee", payload: {}, createdAt: new Date(Date.now() - 12 * HOUR) } as never); + status: "pending", continuationPolicy: "wake_assignee", payload: { supersedeOnUserComment: false }, createdAt: new Date(Date.now() - 12 * HOUR) } as never); const queue = wakeQueue(); // On staging the refusal also left the sweep's escalation without effect // (its stale-snapshot guard): the tasks stayed in_progress with nothing @@ -304,11 +304,23 @@ describePostgres("assigned-work invariant sweep", () => { await recovery.reconcileStrandedAssignedIssues({ companyId: f.companyId, skipRelatedRecovery: true }); expect(queue.calls).toHaveLength(0); - for (const issueId of [plain, withQuestion]) { - expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0].status).toBe("in_progress"); - expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId })) - .toMatchObject({ outcome: "repaired", verdict: { reason: "retry_offered" } }); + expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId: plain })) + .toMatchObject({ outcome: "repaired", verdict: { reason: "retry_offered" } }); + for (let pass = 0; pass < 3; pass += 1) { + expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId: withQuestion })) + .toMatchObject({ outcome: "ok", verdict: { clause: "waiting", reason: "question" } }); + expect((await recovery.assignedWork.listSuspects({ companyId: f.companyId }, null, 100)).map(row => row.id)) + .not.toContain(withQuestion); } + expect(queue.calls.map(call => call.issueId)).toEqual([plain]); + expect(await db.select().from(activityLog).where(and(eq(activityLog.entityId, withQuestion), + eq(activityLog.action, ASSIGNED_WORK_CONTINUED_ACTION)))).toHaveLength(0); + + await db.update(issueThreadInteractions).set({ status: "answered" }).where(eq(issueThreadInteractions.issueId, withQuestion)); + expect((await recovery.assignedWork.listSuspects({ companyId: f.companyId }, null, 100)).map(row => row.id)) + .toContain(withQuestion); + expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId: withQuestion })) + .toMatchObject({ outcome: "repaired", verdict: { reason: "retry_offered" } }); expect(queue.calls.map(call => call.issueId).sort()).toEqual([plain, withQuestion].sort()); }, 60_000); @@ -324,6 +336,26 @@ describePostgres("assigned-work invariant sweep", () => { expect(queue.calls).toHaveLength(1); }, 60_000); + it("continues past a dismissed ordinary question while retaining a governed question's gate", async () => { + const f = await seedCompany(); + const ordinary = await f.issue(), governed = await f.issue(); + for (const issueId of [ordinary, governed]) { + await f.run(issueId); + await db.insert(issueThreadInteractions).values({ companyId: f.companyId, issueId, kind: "ask_user_questions", + status: "pending", dismissedAt: new Date(), payload: { version: 1, questions: [], + ...(issueId === governed ? { toolAction: {} } : {}) } } as never); + } + const queue = wakeQueue(); + const sweep = assignedWorkSweep(db, { enqueueWakeup: queue.enqueueWakeup }); + const suspects = (await sweep.listSuspects({ companyId: f.companyId }, null, 100)).map(row => row.id); + expect(suspects).toContain(ordinary); + expect(suspects).not.toContain(governed); + expect(await sweep.checkIssue({ companyId: f.companyId, issueId: ordinary })).toMatchObject({ outcome: "repaired" }); + expect(await sweep.checkIssue({ companyId: f.companyId, issueId: governed })) + .toMatchObject({ outcome: "ok", verdict: { clause: "waiting", reason: "question" } }); + expect(queue.calls.map(call => call.issueId)).toEqual([ordinary]); + }, 60_000); + it("two servers sweeping the same violation at once create one wake", async () => { const f = await seedCompany(); const issueId = await f.issue(); diff --git a/server/src/__tests__/assigned-work-recheck-idle.test.ts b/server/src/__tests__/assigned-work-recheck-idle.test.ts index 90ccd0b4314..b5bbd27538a 100644 --- a/server/src/__tests__/assigned-work-recheck-idle.test.ts +++ b/server/src/__tests__/assigned-work-recheck-idle.test.ts @@ -1,8 +1,9 @@ import { setImmediate as immediate } from "node:timers/promises"; -import { and, desc, eq } from "drizzle-orm"; +import { and, desc, eq, sql } from "drizzle-orm"; import { expect, it, vi } from "vitest"; -import { activityLog, agents, heartbeatRuns, issues } from "@paperclipai/db"; +import { activityLog, agents, heartbeatRuns, issues, issueThreadInteractions } from "@paperclipai/db"; import { notifyNativeStatusReassessment } from "../services/native-status-reassessment-notifications.js"; +import { notifyChatRunMilestones } from "../services/chat-run-milestone-notifications.js"; import { ASSIGNED_WORK_CONTINUED_ACTION } from "../services/recovery/assigned-work-sweep.js"; import { describeIssueRecovery, issueRecoveryFixture } from "./issue-recovery.test-fixture.js"; @@ -54,6 +55,30 @@ describeIssueRecovery("assigned-work re-checks of tasks whose continuation is re expect(report.violations).toEqual({ retry_offered: TASKS }); } + it("stays quiet behind unanswered questions and wakes on their resolution event", async () => { + const f = await seed(); + await state.db.update(heartbeatRuns).set({ contextSnapshot: + sql`${heartbeatRuns.contextSnapshot} || '{"explicitUserContinuation":{"previousRunId":"11111111-1111-4111-8111-111111111111"}}'::jsonb` }) + .where(eq(heartbeatRuns.companyId, f.companyId)); + for (const task of f.tasks) await state.db.insert(issueThreadInteractions).values({ companyId: f.companyId, + issueId: task.id, kind: "ask_user_questions", status: "pending", continuationPolicy: "wake_assignee", + payload: { version: 1, supersedeOnUserComment: false, questions: [{ id: "scope", prompt: "Choose the scope", + selectionMode: "single", required: true, options: [{ id: "small", label: "Small" }, { id: "full", label: "Full" }] }] }, + createdAt: new Date(Date.now() - HOUR) }); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + await state.start({}); + await advance(20 * MINUTE); + expect(await attempts(f.companyId)).toBe(0); + await state.db.transaction(async tx => { + await tx.update(issueThreadInteractions).set({ status: "answered", updatedAt: new Date() }) + .where(eq(issueThreadInteractions.issueId, f.tasks[0]!.id)); + await notifyChatRunMilestones(tx, { companyId: f.companyId, issueId: f.tasks[0]!.id }); + }); + await advance(2 * MINUTE); + expect(await attempts(f.companyId, f.tasks[0]!.id)).toBeGreaterThan(0); + expect(await attempts(f.companyId)).toBe(await attempts(f.companyId, f.tasks[0]!.id)); + }, 600_000); + it("backs off while every check finds the same refused state", async () => { const f = await seed(); await boot(f); diff --git a/server/src/__tests__/heartbeat-provider-limit-wait.test.ts b/server/src/__tests__/heartbeat-provider-limit-wait.test.ts index 551bee138ef..3a778c6003d 100644 --- a/server/src/__tests__/heartbeat-provider-limit-wait.test.ts +++ b/server/src/__tests__/heartbeat-provider-limit-wait.test.ts @@ -104,7 +104,7 @@ describePostgres("heartbeat provider usage limit wait", () => { message: "API error: 429 This request would exceed your account's rate limit. Please try again later", ...overrides } }, }); - it("defers new work until the reported reset instead of dispatching a doomed run", async () => { + it("defers new work to a recheck without claiming an unreported reset", async () => { const limited = rateLimited(16); const f = await seed([limited]); const resetAt = new Date(limited.finishedAt.getTime() + 60 * 60_000); @@ -120,6 +120,8 @@ describePostgres("heartbeat provider usage limit wait", () => { }, }); expect(finished.error).toContain(resetAt.toISOString()); + expect(finished.error).toContain("recheck"); + expect(finished.error).not.toMatch(/expected to reset|start this work then/); const successors = await db.select().from(heartbeatRuns) .where(eq(heartbeatRuns.agentId, f.agentId)); const retry = successors.find(run => run.status === "scheduled_retry"); @@ -132,7 +134,7 @@ describePostgres("heartbeat provider usage limit wait", () => { expect(issue.status).not.toBe("blocked"); // The persisted projection names the wait and its reset. expect((await executionProjectionsForRuns(db, f.companyId, [retry!.id])).get(retry!.id)).toMatchObject({ - label: "Waiting for usage limit reset", phase: "retry_scheduled", + label: "Waiting for provider usage", phase: "retry_scheduled", nextAction: expect.stringContaining(retry!.scheduledRetryAt!.toISOString()), }); diff --git a/server/src/__tests__/task-interruption-query.test.ts b/server/src/__tests__/task-interruption-query.test.ts index 843b78e92a1..a069710c8ec 100644 --- a/server/src/__tests__/task-interruption-query.test.ts +++ b/server/src/__tests__/task-interruption-query.test.ts @@ -1,9 +1,9 @@ import { randomUUID } from "node:crypto"; -import { and, eq } from "drizzle-orm"; +import { and, eq, sql } from "drizzle-orm"; import { createRunSecretRedactionRegistry } from "../services/run-secret-redaction.js"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; -import { agents, agentWakeupRequests, chatActions, chatEndpoints, companies, createDb, applyPendingMigrations, environmentLeases, heartbeatRuns, issueRecoveryActions, - issueRelations, issues, +import { agents, agentWakeupRequests, approvals, chatActions, chatEndpoints, companies, createDb, applyPendingMigrations, environmentLeases, heartbeatRuns, issueRecoveryActions, + issueApprovals, issueComments, issueRelations, issues, issueThreadInteractions, toolApplications, toolConnections, nativeRunFinalizations, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db"; import { createSystemRecovery } from "../services/system-recovery-state.js"; @@ -32,6 +32,46 @@ describe.skipIf(!support.supported)("task interruption latest run query", () => const markFailuresFresh = (companyId: string) => db.update(heartbeatRuns).set({ finishedAt: new Date() }) .where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.status, "failed"))); + it("shows an unresolved question over an older provider failure, until the question resolves", async () => { + const companyId = randomUUID(), agentId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Question fixture", issuePrefix: `Q${companyId.slice(0, 7)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Question agent", adapterType: "paperclip_runner" }); + const tasks = await db.insert(issues).values([ + { companyId, title: "Question pending", status: "todo", assigneeAgentId: agentId }, + { companyId, title: "No question", status: "todo", assigneeAgentId: agentId }, + ]).returning(); + for (const task of tasks) await db.insert(heartbeatRuns).values({ companyId, agentId, nativeIssueId: task.id, + scopeKind: "issue", status: "failed", runtimeMode: "native", errorCode: "native_session_interrupted", + resultJson: { providerFailure: { schema: "paperclip.provider-failure.v1", provider: "anthropic", phase: "startup", + category: "rate_limit", httpStatus: 429, retryability: "unknown", message: "API error: 429" } } }); + const [question] = await db.insert(issueThreadInteractions).values({ companyId, issueId: tasks[0].id, + kind: "ask_user_questions", status: "pending", continuationPolicy: "wake_assignee", + payload: { version: 1, supersedeOnUserComment: false, questions: [{ id: "scope", prompt: "Choose the scope", + selectionMode: "single", required: true, options: [{ id: "small", label: "Small" }, { id: "full", label: "Full" }] }] } }).returning(); + const presentations = await taskInterruptionPresentations(db, companyId, tasks); + expect(presentations.get(tasks[0].id)).toMatchObject({ state: "waiting_for_user", userAction: "question", canRetry: true }); + expect(presentations.get(tasks[0].id)?.providerFailure).toBeUndefined(); + expect(presentations.get(tasks[1].id)).toMatchObject({ state: "failed", providerFailure: { httpStatus: 429 } }); + const [approval] = await db.insert(approvals).values({ companyId, type: "approve_ceo_strategy", status: "pending", payload: {} }).returning(); + await db.insert(issueApprovals).values({ companyId, issueId: tasks[1].id, approvalId: approval.id }); + const gated = await taskInterruptionPresentations(db, companyId, tasks); + expect(gated.get(tasks[0].id)).toMatchObject({ userAction: "question", canRetry: true }); + expect(gated.get(tasks[1].id)).toMatchObject({ userAction: "approval", canRetry: false }); + await db.update(issueThreadInteractions).set({ dismissedAt: new Date() }).where(eq(issueThreadInteractions.id, question.id)); + expect(await taskInterruptionPresentation(db, tasks[0])).toMatchObject({ state: "failed", canRetry: true }); + await db.update(issueThreadInteractions).set({ dismissedAt: null }).where(eq(issueThreadInteractions.id, question.id)); + await db.insert(issueComments).values({ companyId, issueId: tasks[0].id, authorUserId: "board-user", authorType: "user", + body: "Continue with the current scope", createdAt: new Date(question.createdAt.getTime() + 1) }); + expect(await taskInterruptionPresentation(db, tasks[0])).toMatchObject({ state: "failed", canRetry: true }); + await db.update(issueThreadInteractions).set({ dismissedAt: new Date(), payload: sql`${issueThreadInteractions.payload} || '{"toolAction":{}}'::jsonb` }) + .where(eq(issueThreadInteractions.id, question.id)); + expect(await taskInterruptionPresentation(db, tasks[0])).toMatchObject({ state: "waiting_for_user", userAction: "question" }); + await db.update(issueThreadInteractions).set({ status: "answered" }).where(eq(issueThreadInteractions.id, question.id)); + expect(await taskInterruptionPresentation(db, tasks[0])).toMatchObject({ state: "failed", providerFailure: { httpStatus: 429 } }); + const [historical] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.nativeIssueId, tasks[0].id)); + expect(historical.resultJson).toMatchObject({ providerFailure: { httpStatus: 429 } }); + }); + it("shows a pending native same-run resume instead of a competing Retry", async () => { const companyId = randomUUID(), agentId = randomUUID(), runId = randomUUID(); await db.insert(companies).values({ id: companyId, name: "Resuming task", issuePrefix: `R${companyId.slice(0, 7)}` }); diff --git a/server/src/services/execution-projection.test.ts b/server/src/services/execution-projection.test.ts index 32cba2bc59b..d9d17d2df30 100644 --- a/server/src/services/execution-projection.test.ts +++ b/server/src/services/execution-projection.test.ts @@ -191,12 +191,12 @@ describe("execution truth projection", () => { scheduledRetryAttempt: 12, contextSnapshot: { failureRetriesBeforeAiConnectionWait: 0 } }), undefined, [], undefined, now)) .toMatchObject({ label: "Waiting for AI subscription", phase: "retry_scheduled", attempt: 1, recoveryOwner: null }); }); - it("shows a provider usage limit wait with its reset time", () => { + it("shows a provider usage limit recheck without promising work will start then", () => { const scheduledRetryAt = new Date("2026-11-05T13:47:50.375Z"); expect(projectExecution(run({ runtimeMode: "native", status: "scheduled_retry", scheduledRetryReason: "ai_connection_pool_wait", scheduledRetryAt, scheduledRetryAttempt: 3, contextSnapshot: { wakeReason: "provider_limit_retry", failureRetriesBeforeAiConnectionWait: 0 } }), undefined, [], undefined, now)).toMatchObject({ - label: "Waiting for usage limit reset", phase: "retry_scheduled", attempt: 1, recoveryOwner: null, + label: "Waiting for provider usage", phase: "retry_scheduled", attempt: 1, recoveryOwner: null, nextAction: expect.stringContaining(scheduledRetryAt.toISOString()), }); }); diff --git a/server/src/services/execution-projection.ts b/server/src/services/execution-projection.ts index 3b990bbf3c4..a572e7184d0 100644 --- a/server/src/services/execution-projection.ts +++ b/server/src/services/execution-projection.ts @@ -462,9 +462,9 @@ export function projectExecution( } if (run.status === "scheduled_retry" && run.scheduledRetryReason === "ai_connection_pool_wait" && run.contextSnapshot?.wakeReason === "provider_limit_retry") { - const resetAt = run.scheduledRetryAt ? new Date(run.scheduledRetryAt).toISOString() : "its reported reset"; - projection.nextAction = `Waiting for the agent's provider usage limit to reset; the work starts at ${resetAt}. To continue sooner, restore the provider allowance or change the agent's AI connection, then start a manual run.`; - return set("retry_scheduled", "Waiting for usage limit reset"); + const recheckAt = run.scheduledRetryAt ? ` at ${new Date(run.scheduledRetryAt).toISOString()}` : " when its scheduled check is due"; + projection.nextAction = `Paperclip will recheck the agent's provider usage limit${recheckAt} before starting work. The provider may still refuse it. To continue sooner, restore the provider allowance or change the agent's AI connection, then start a manual run.`; + return set("retry_scheduled", "Waiting for provider usage"); } if (run.status === "scheduled_retry" && run.scheduledRetryReason === "agent_directory_busy") { projection.nextAction = "Waiting for another run of this agent to finish with its files; this task continues automatically."; diff --git a/server/src/services/fresh-continuation-fallback.test.ts b/server/src/services/fresh-continuation-fallback.test.ts index 9ef35c330c1..d130aea9cbd 100644 --- a/server/src/services/fresh-continuation-fallback.test.ts +++ b/server/src/services/fresh-continuation-fallback.test.ts @@ -1152,6 +1152,15 @@ suite("fresh-continuation fallback", () => { expect(await successors(f.runId)).toHaveLength(0); }, 60_000); + it.each([false, true])("respects deferred question authority in fresh continuation (governed=%s)", async governed => { + const f = await unfinishedRun({ runtime: "native", errorCode: "native_continuation_requires_reconciliation" }); + await db.insert(issueThreadInteractions).values({ companyId, issueId: f.issueId, kind: "ask_user_questions", + status: "pending", dismissedAt: new Date(), payload: { version: 1, questions: [], + ...(governed ? { toolAction: {} } : {}) } } as never); + expect(await ensureFreshContinuation(db, { companyId, runId: f.runId, trigger: "recovery_refused" })) + .toMatchObject(governed ? { kind: "person_owned", reason: "question_pending" } : { kind: "continued" }); + }, 60_000); + // Interruption with an unknown outcome is not a governance gate: the task // continues, but as a fresh run, never a replay, and with an audit trail. it("continues an interrupted governed action as a fresh run, never a replay, and keeps the audit trail", async () => { diff --git a/server/src/services/fresh-continuation-fallback.ts b/server/src/services/fresh-continuation-fallback.ts index 16886a1a560..138e819a929 100644 --- a/server/src/services/fresh-continuation-fallback.ts +++ b/server/src/services/fresh-continuation-fallback.ts @@ -25,6 +25,7 @@ import { abandonUnverifiedRemoteWorkspaceEpoch } from "./remote-workspace-epoch- import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js"; import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { readRunCancellation } from "./run-cancellation.js"; +import { activeIssueInteractionCondition } from "./issue-question-context.js"; import { readSystemRecovery } from "./system-recovery-state.js"; import { RESTART_CLASS_FAILURE_CODES, continuationCauseIsSystemOwned, isPersonOwnedFailureCode } from "./continuation-cause.js"; import { sharedContinuationAttempts } from "./recovery/continuation-ledger.js"; @@ -150,7 +151,7 @@ async function personOwnedTaskWait(tx: Db, input: { companyId: string; issue: ty if (approval) return "approval_pending"; const [question] = await tx.select({ id: issueThreadInteractions.id }).from(issueThreadInteractions).where(and( eq(issueThreadInteractions.companyId, companyId), eq(issueThreadInteractions.issueId, issue.id), - eq(issueThreadInteractions.status, "pending"))).limit(1); + eq(issueThreadInteractions.status, "pending"), activeIssueInteractionCondition())).limit(1); if (question) return "question_pending"; const [agent] = await tx.select({ status: agents.status, lifecycleState: agents.lifecycleState }).from(agents) .where(and(eq(agents.companyId, companyId), eq(agents.id, run.agentId))); diff --git a/server/src/services/governed-wait-runner-loss.test.ts b/server/src/services/governed-wait-runner-loss.test.ts index 796c46b82ad..58a49dbedad 100644 --- a/server/src/services/governed-wait-runner-loss.test.ts +++ b/server/src/services/governed-wait-runner-loss.test.ts @@ -100,7 +100,8 @@ describe("a held task with a pending decision after its remote runner was lost", const issue = (await db.select().from(issues).where(eq(issues.id, f.issueId)))[0]; expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toMatchObject({ runId: f.runId, canRetry: true, runError: "runner_remote_process_identity_unavailable", workspaceRecovery: { status: "recovered" } }); - expect(await taskInterruptionPresentation(db, issue)).toMatchObject({ state: "failed", canRetry: true, observedRunId: f.runId }); + expect(await taskInterruptionPresentation(db, issue)).toMatchObject({ state: shape === "parkedQuestion" ? "waiting_for_user" : "failed", + userAction: shape === "parkedQuestion" ? "question" : null, canRetry: true, observedRunId: f.runId }); const receipt = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issueId, actorId: "local-board", actorSource: "local_implicit", idempotencyKey: randomUUID(), observedRunId: f.runId }); expect(receipt).toMatchObject({ accepted: true }); @@ -295,7 +296,7 @@ describe("a held task with a pending decision after its remote runner was lost", .toEqual({ successors: 1, deferred: [] }); }, 60_000); - it("reads every saved decision wait on a task list in one bounded query", async () => { + it("keeps question and saved decision reads bounded on a task list", async () => { const f = await heldOnConfirmation(); const source = await currentIssue(f.issueId); const listed = await db.insert(issues).values(Array.from({ length: 40 }, (_, index) => ({ @@ -317,7 +318,8 @@ describe("a held task with a pending decision after its remote runner was lost", try { await taskInterruptionPresentations(db, f.companyId, [source, ...listed]); } finally { client.debug = previous; } - expect(statements.length).toBeLessThanOrEqual(2); + expect(statements.length).toBeLessThanOrEqual(3); + expect(statements.filter(query => /\bas held\s*\(/.test(query))).toHaveLength(1); const waits = await db.select().from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, f.companyId), eq(agentWakeupRequests.status, "deferred_issue_execution"))); const held = await pendingContinuationDecisionWaits(db, f.companyId, waits.map(wait => ({ id: wait.id, diff --git a/server/src/services/heartbeat/provider-limit-wait.ts b/server/src/services/heartbeat/provider-limit-wait.ts index 154bed8250c..1d11df95ded 100644 --- a/server/src/services/heartbeat/provider-limit-wait.ts +++ b/server/src/services/heartbeat/provider-limit-wait.ts @@ -14,7 +14,7 @@ export interface ProviderLimitWaitDependencies { }) => Promise<{ updated: boolean; run?: HeartbeatRun | null }>; setWakeupStatus: (wakeupRequestId: string | null | undefined, status: "cancelled", patch: { finishedAt: Date; error: string }) => Promise; appendRunEvent: (run: HeartbeatRun, event: { - eventType: string; stream: "system"; level: "warn"; message: string; payload: Record; + eventType: string; stream: "system"; level: "info" | "warn"; message: string; payload: Record; }) => Promise; scheduleBoundedRetryForRun: (run: HeartbeatRun, agent: Agent, opts: { now: Date; retryReason: string; wakeReason: string; maxAttempts: number; delayMs: number; @@ -55,9 +55,9 @@ export function createProviderLimitWait(db: Db, deps: ProviderLimitWaitDependenc }); if (!cooldown) return false; const resetAt = cooldown.resetAt.toISOString(); - const message = `The provider usage limit for this agent is expected to reset at ${resetAt}${ - cooldown.resetReported ? "" : " (no reset time was reported, so Paperclip waits one hour)" - }. Paperclip will start this work then.`; + const message = cooldown.resetReported + ? `This agent's provider usage limit has a reported reset at ${resetAt}. Work is held; a retry must recheck availability before starting work.` + : `This agent's provider usage limit has no reported reset time. Work is held; a retry can recheck availability after ${resetAt}; the limit may still apply then.`; const cancelled = await setRunStatusIfRunning(run.id, "cancelled", { error: message, errorCode: "provider_limit_wait", finishedAt: now, resultJson: { @@ -95,10 +95,10 @@ export function createProviderLimitWait(db: Db, deps: ProviderLimitWaitDependenc } } await appendRunEvent(deferred, { - eventType: "lifecycle", stream: "system", level: "warn", + eventType: "lifecycle", stream: "system", level: scheduled ? "info" : "warn", message: scheduled - ? `Waiting for the provider usage limit to reset at ${resetAt} before starting work` - : `The provider usage limit resets at ${resetAt}; this wake could not schedule its retry and released the task`, + ? `Waiting to recheck the provider usage limit after ${resetAt} before starting work` + : "Waiting for provider availability; this wake could not schedule its recheck and released the task", payload: { resetAt, resetReported: cooldown.resetReported, category: cooldown.category, sourceRunId: cooldown.sourceRunId, retryScheduled: scheduled }, }).catch((err: unknown) => logger.warn({ err, runId: run.id }, "failed to record the provider limit wait")); await setWakeupStatus(run.wakeupRequestId, "cancelled", { finishedAt: now, error: message }) diff --git a/server/src/services/recovery/assigned-work-invariant.ts b/server/src/services/recovery/assigned-work-invariant.ts index 76e81ac9244..6b391b9836a 100644 --- a/server/src/services/recovery/assigned-work-invariant.ts +++ b/server/src/services/recovery/assigned-work-invariant.ts @@ -83,6 +83,7 @@ export const NATIVE_STALL_MS = 5 * 60_000; /** Interruption presentation `userAction`s that are a person's move. */ const PERSON_ACTIONS: Readonly> = { approval: "approval", + question: "question", reconnect: "credential", choose_model: "credential", project_access: "credential", diff --git a/server/src/services/recovery/assigned-work-sweep.ts b/server/src/services/recovery/assigned-work-sweep.ts index 14a79515d93..52e7e4c9f7c 100644 --- a/server/src/services/recovery/assigned-work-sweep.ts +++ b/server/src/services/recovery/assigned-work-sweep.ts @@ -1,5 +1,5 @@ import { createHash } from "node:crypto"; -import { and, desc, eq, gte, inArray, isNull, or, sql, type SQL, type SQLWrapper } from "drizzle-orm"; +import { and, desc, eq, gte, inArray, isNull, or, sql, type SQL } from "drizzle-orm"; import { activityLog, agentTaskSessions, @@ -28,6 +28,7 @@ import { legacyControllerOwnerStamp } from "../legacy-controller-lease.js"; import { currentProcessProbe, proveOwnerGone, type OwnerProof, type LocalOwnerStamp } from "../owner-proof.js"; import { issueTreeControlService } from "../issue-tree-control.js"; import { taskInterruptionPresentations } from "../task-interruption-presentation.js"; +import { activeIssueInteractionCondition } from "../issue-question-context.js"; import { CHAT_CONTROL_RECOVERY_STOP_CODE, CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE } from "../chat-control-recovery-stop.js"; import { onboardingFirstTaskAwaitingUserSql } from "./onboarding-first-task-wait.js"; import { workspaceRecoverySourceEligible } from "../workspace-recovery-source.js"; @@ -204,9 +205,6 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { }; const agentIds = [...new Set(issueRows.flatMap(issue => issue.assigneeAgentId ? [issue.assigneeAgentId] : []))]; const blockedRows = issueRows.filter(issue => issue.status === "blocked"); - const notSupersededByUserContinuation = interactionNotSupersededSql(issueThreadInteractions.companyId, - sql`${issueThreadInteractions.issueId}::text`, issueThreadInteractions.createdAt); - const [activeRuns, latestRuns, wakes, chatWakes, nativePending, handoffs, goals, interactions, approvalRows, blockerRows, resolvedBlockerRows, childRows, actionMap, agentRows, companyRows, onboardingRows, blockActivity, interruptions, savedWorkRows] = await Promise.all([ @@ -262,12 +260,9 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { goalDesiredState: agentTaskSessions.goalDesiredState }) .from(agentTaskSessions).where(and(eq(agentTaskSessions.companyId, companyId), inArray(agentTaskSessions.taskKey, ids), sql`${agentTaskSessions.goalStatus} is not null`, sql`${agentTaskSessions.goalStatus} <> 'complete'`)), - // A pending question counts as the person's move until the person - // answers it another way: a Retry or message started after the question - // was asked (an explicit user continuation) supersedes it as a wait. database.select({ issueId: issueThreadInteractions.issueId, count: sql`count(*)::int` }).from(issueThreadInteractions) .where(and(eq(issueThreadInteractions.companyId, companyId), inArray(issueThreadInteractions.issueId, ids), - eq(issueThreadInteractions.status, "pending"), notSupersededByUserContinuation)) + eq(issueThreadInteractions.status, "pending"), activeIssueInteractionCondition())) .groupBy(issueThreadInteractions.issueId), database.select({ issueId: issueApprovals.issueId, count: sql`count(*)::int` }).from(issueApprovals) .innerJoin(approvals, eq(approvals.id, issueApprovals.approvalId)) @@ -552,9 +547,9 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { or coalesce(w.claimed_at, w.created_at) > ${new Date(at.getTime() - DEFERRED_WAKE_PENDING_BOUND_MS).toISOString()}::timestamptz))) and not exists (select 1 from heartbeat_runs wake_run where wake_run.id = w.run_id and ${queuedRunStalledSql(aliasedRun("wake_run"), at)})))`, - sql`not exists (select 1 from issue_thread_interactions t where t.company_id = ${issues.companyId} - and t.issue_id = ${issues.id} and t.status = 'pending' - and ${interactionNotSupersededSql(sql`t.company_id`, sql`t.issue_id::text`, sql`t.created_at`)})`, + sql`not exists (select 1 from ${issueThreadInteractions} where ${issueThreadInteractions.companyId} = ${issues.companyId} + and ${issueThreadInteractions.issueId} = ${issues.id} and ${issueThreadInteractions.status} = 'pending' + and ${activeIssueInteractionCondition()})`, // Plain waits on someone else, decided without the full read. The // predicate re-checks every task this lets through. sql`not exists (select 1 from issue_relations rel inner join issues blocker @@ -1020,16 +1015,6 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { return { checkIssue, listSuspects, report, loadFacts, flushPassLog: passLog.flush }; } -/** A pending interaction no explicit user continuation (Retry or message) - * started after it. Uses the (company, issueId, created) run index. */ -function interactionNotSupersededSql(companyId: SQLWrapper, issueIdText: SQLWrapper, createdAt: SQLWrapper) { - return sql`not exists (select 1 from heartbeat_runs retry where retry.company_id = ${companyId} - and (retry.context_snapshot->>'issueId') = ${issueIdText} - and jsonb_typeof(retry.context_snapshot->'explicitUserContinuation') = 'object' - and retry.created_at > ${createdAt})`; -} - - /** The status a block replaced, from the activity entry that blocked the task: * the normal update route records it as `_previous.status`, recovery as * `previousStatus`, the native status committer as `fromStatus`. */ diff --git a/server/src/services/task-interruption-presentation.test.ts b/server/src/services/task-interruption-presentation.test.ts index 10970b0cfcf..4e7ce6b2800 100644 --- a/server/src/services/task-interruption-presentation.test.ts +++ b/server/src/services/task-interruption-presentation.test.ts @@ -7,6 +7,39 @@ const run = { id: "run", agentId: "agent", status: "failed", scheduledRetryAt: n resultJson: { projectAccessDenied: { projectId: "22222222-2222-4222-8222-222222222222" } } }; const agent = { status: "idle", pauseReason: null }; +describe("pending question presentation", () => { + const failed = { ...run, errorCode: "native_session_interrupted", resultJson: null }; + const input = { run: failed, agent, paused: false, approval: false, question: true }; + it("names the unanswered question instead of a historical failure, preserving an available manual Retry", () => { + for (const previousRun of [failed, { ...failed, status: "succeeded" }, undefined]) { + const presentation = projectTaskInterruption(issue, { ...input, run: previousRun }); + expect(presentation).toMatchObject({ responsibility: "user", state: "waiting_for_user", userAction: "question", + canRetry: previousRun?.status === "failed" }); + expect(presentation?.providerFailure).toBeUndefined(); + } + }); + it("does not offer another Retry while saved input or a Retry is pending", () => { + const wait = { id: "saved-input", requestedAt: new Date(), sourceRunId: failed.id }; + for (const pending of [{ wait }, { retryWait: wait }]) + expect(projectTaskInterruption(issue, { ...input, ...pending })).toMatchObject({ canRetry: false }); + }); + it("shows the question while automatic continuation waits, without offering to move it forward", () => { + for (const pendingSince of [undefined, Date.now() - 16 * 60_000]) + expect(projectTaskInterruption(issue, { ...input, freshContinuation: { state: "continuing", pendingSince } })) + .toMatchObject({ state: "waiting_for_user", userAction: "question", canRetry: false }); + }); + it("preserves other current user gates", () => { + for (const [gate, userAction] of [[{ approval: true }, "approval"], [{ paused: true }, "resume"], + [{ ownerStopRequired: true }, "confirm_owner_stop"]] as const) + expect(projectTaskInterruption(issue, { ...input, ...gate })).toMatchObject({ state: "waiting_for_user", userAction, canRetry: false }); + }); + it("preserves live work and terminal task states", () => { + expect(projectTaskInterruption(issue, { ...input, run: { ...run, status: "running", errorCode: null } })) + .toMatchObject({ state: "working" }); + expect(projectTaskInterruption({ ...issue, status: "done" }, input)).toMatchObject({ state: "resolved" }); + }); +}); + describe("saved governed wait shutdown", () => { const now = Date.parse("2026-11-09T12:00:00Z"); const pending = { ...run, runtimeMode: "native", errorCode: "native_governed_wait_settlement_pending", resultJson: null }; diff --git a/server/src/services/task-interruption-presentation.ts b/server/src/services/task-interruption-presentation.ts index 7222d614954..283a9075c4f 100644 --- a/server/src/services/task-interruption-presentation.ts +++ b/server/src/services/task-interruption-presentation.ts @@ -6,7 +6,7 @@ import { and, desc, eq, inArray, isNull, not, notInArray, sql, type SQL } from " import { unidentifiedLocalLaunchRefusals } from "./native-local-process-stop.js"; import { nativeSessionCleanupBlock } from "../vendor/paperclip-runner/index.js"; import { agents, agentWakeupRequests, approvals, chatActions, companies, environmentLeases, heartbeatRuns, issueApprovals, issueRecoveryActions, - issues, issueTreeHoldMembers, issueTreeHolds, nativeRunFinalizations, projects, statusDecisions, type Db } from "@paperclipai/db"; + issues, issueThreadInteractions, issueTreeHoldMembers, issueTreeHolds, nativeRunFinalizations, projects, statusDecisions, type Db } from "@paperclipai/db"; import { NATIVE_AGENT_BLOCKER_HANDOFF_CODE } from "./native-runtime/native-agent-blocker.js"; import { unionAll } from "drizzle-orm/pg-core"; import { EXECUTION_RECONCILIATION_CAUSES, HEARTBEAT_RUN_STATUSES, readProviderFailure, type TaskInterruptionPresentation } from "@paperclipai/shared"; @@ -15,6 +15,7 @@ import { isAiAuthenticationBlocked } from "./ai-auth-failure.js"; import { readSystemRecovery } from "./system-recovery-state.js"; import { PROJECT_ACCESS_DENIED } from "./project-access-denial.js"; import { authorizationService } from "./authorization.js"; +import { activeIssueInteractionCondition } from "./issue-question-context.js"; import { taskWorkspaceProjectRequirements } from "./task-workspace-access.js"; import { isAcknowledgedNativeStop } from "./acknowledged-native-stop.js"; import { createRunSecretRedactionRegistry } from "./run-secret-redaction.js"; @@ -97,6 +98,7 @@ type ProjectionState = { company?: Pick; paused: boolean; approval: boolean; + question?: boolean; /** A board-owned workspace-owner fence on this run that no automatic proof has released. */ ownerStopRequired?: boolean; unidentifiedLaunchConfirmable?: boolean; @@ -229,6 +231,11 @@ function deniedProjectId(issue: ProjectionIssue, run?: ProjectionRun): string | * proves the old session stopped and continues in a fresh session. */ export function projectTaskInterruption(issue: ProjectionIssue, input: ProjectionState, now = Date.now()): TaskInterruptionPresentation | null { const presentation = projectTaskInterruptionState(issue, input, now); + if (input.question && !["done", "cancelled"].includes(issue.status) && !["running", "queued"].includes(input.run?.status ?? "") + && presentation?.responsibility !== "user") + return { intentId: input.wait?.id ?? null, responsibility: "user", state: "waiting_for_user", userAction: "question", + canRetry: presentation?.canRetry === true && presentation.retryAction !== "move_forward" && !input.wait && !input.retryWait, + nextRetryAt: null, sourceRunIds: input.run ? [input.run.id] : [], observedRunId: input.run?.id ?? null }; if (!presentation?.canRetry || input.run?.errorCode !== NATIVE_SESSION_CLEANUP_QUARANTINED || !input.cleanupSettling) return presentation; return { ...presentation, canRetry: false }; @@ -467,7 +474,7 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i const agentIds = selected.flatMap(issue => issue.assigneeAgentId ? [issue.assigneeAgentId] : []); const projectIds = selected.flatMap(issue => issue.projectId ? [issue.projectId] : []); const startingIssueIds = selected.filter(issue => issue.status === "todo" && issue.assigneeAgentId).map(issue => issue.id); - const [runs, waits, agentRows, projectRows, pauseRows, approvalRows, companyRows, admissionWaits] = await Promise.all([ + const [runs, waits, agentRows, projectRows, pauseRows, gateRows, companyRows, admissionWaits] = await Promise.all([ db.select({ issueId: requestedIssueId, id: heartbeatRuns.id, status: heartbeatRuns.status, createdAt: heartbeatRuns.createdAt, finishedAt: heartbeatRuns.finishedAt, agentId: heartbeatRuns.agentId, companyId: heartbeatRuns.companyId, nativeIssueId: heartbeatRuns.nativeIssueId, runtimeMode: heartbeatRuns.runtimeMode, startedAt: heartbeatRuns.startedAt, @@ -581,10 +588,15 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i eq(issueTreeHolds.companyId, issueTreeHoldMembers.companyId), eq(issueTreeHolds.id, issueTreeHoldMembers.holdId))) .where(and(eq(issueTreeHoldMembers.companyId, companyId), inArray(issueTreeHoldMembers.issueId, ids), eq(issueTreeHolds.status, "active"), eq(issueTreeHolds.mode, "pause"))), - db.select({ issueId: issueApprovals.issueId }).from(issueApprovals).innerJoin(approvals, and( - eq(approvals.companyId, issueApprovals.companyId), eq(approvals.id, issueApprovals.approvalId))) - .where(and(eq(issueApprovals.companyId, companyId), inArray(issueApprovals.issueId, ids), - inArray(approvals.status, ["pending", "revision_requested"]))), + unionAll( + db.select({ issueId: issueApprovals.issueId, kind: sql<"approval" | "question">`'approval'` }).from(issueApprovals).innerJoin(approvals, and( + eq(approvals.companyId, issueApprovals.companyId), eq(approvals.id, issueApprovals.approvalId))) + .where(and(eq(issueApprovals.companyId, companyId), inArray(issueApprovals.issueId, ids), + inArray(approvals.status, ["pending", "revision_requested"]))), + db.selectDistinct({ issueId: issueThreadInteractions.issueId, kind: sql<"approval" | "question">`'question'` }).from(issueThreadInteractions) + .where(and(eq(issueThreadInteractions.companyId, companyId), inArray(issueThreadInteractions.issueId, ids), + eq(issueThreadInteractions.kind, "ask_user_questions"), eq(issueThreadInteractions.status, "pending"), activeIssueInteractionCondition())), + ), db.select({ status: companies.status, pauseReason: companies.pauseReason }).from(companies).where(eq(companies.id, companyId)), // Only consulted for tasks without a run: the latest wake says whether admission refused it. startingIssueIds.length ? db.selectDistinctOn([wakeIssue], { issueId: wakeIssue, id: agentWakeupRequests.id, @@ -826,8 +838,10 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i }; const agentMap = new Map(agentRows.map(agent => [agent.id, agent])); const projectMap = new Map(projectRows.map(project => [project.id, project])); - const pauses = new Set(pauseRows.map(row => row.issueId)), approvalsSet = new Set(approvalRows.map(row => row.issueId)); + const pauses = new Set(pauseRows.map(row => row.issueId)); + const approvalsSet = new Set(gateRows.filter(row => row.kind === "approval").map(row => row.issueId)); const admissionWaitMap = byIssue(admissionWaits); + const questions = new Set(gateRows.filter(row => row.kind === "question").map(row => row.issueId)); const launchWaits = [...retryWaitMap].flatMap(([issueId, wait]) => { const run = runMap.get(issueId); return run && wait.sourceRunId === run.id && wait.executionWaitReason === "process_identity_missing" ? [{ issueId, run }] : []; @@ -853,7 +867,7 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i projectAccessProjectId: denied?.includes(issue.projectId ?? "") ? issue.projectId : null, workspaceSelectionAvailable: required?.available ?? false, agent: agentMap.get(issue.assigneeAgentId ?? ""), project: projectMap.get(issue.projectId ?? ""), - company: companyRows[0], paused: pauses.has(issue.id), approval: approvalsSet.has(issue.id), + company: companyRows[0], paused: pauses.has(issue.id), approval: approvalsSet.has(issue.id), question: questions.has(issue.id), ownerStopRequired: ownerStopRequired.has(`${issue.id}:${facts.run?.id}`), unidentifiedLaunchConfirmable: confirmableLaunches.has(issue.id), admissionWait: admissionWaitMap.get(issue.id), diff --git a/tests/runner-e2e/source-sdk-public-inputs.json b/tests/runner-e2e/source-sdk-public-inputs.json index d04f5b48890..0fe138d417f 100644 --- a/tests/runner-e2e/source-sdk-public-inputs.json +++ b/tests/runner-e2e/source-sdk-public-inputs.json @@ -418,7 +418,7 @@ "packages/shared/src/types/email.ts": "556ff6a0dcefa01eb704d6d00d3b191de81ff4dc", "packages/shared/src/types/environment.ts": "62e62afcb3d5b7f246db1e8910a34fb0091f54db", "packages/shared/src/types/execution-continuation.ts": "5afe62afd227efa03ff847d8f73291c14707ab5b", - "packages/shared/src/types/execution-projection.ts": "58b80ab6fc38609fa6254f2805cae16392f291b0", + "packages/shared/src/types/execution-projection.ts": "75ae5e1312160fd9cec79e5234004221ef296ee6", "packages/shared/src/types/external-object.ts": "7a225e0f5a6777d449019a30530c858b753eb4c1", "packages/shared/src/types/feedback.ts": "c5eb31545c7f67c04ad9afd9db45a6b4774bcb35", "packages/shared/src/types/finance.ts": "5c2208c7950020f1005aa0dd86a1680c54dd4354", diff --git a/ui/src/components/ExecutionBlockerNotice.test.tsx b/ui/src/components/ExecutionBlockerNotice.test.tsx index 060b4af8af9..1dcffa21447 100644 --- a/ui/src/components/ExecutionBlockerNotice.test.tsx +++ b/ui/src/components/ExecutionBlockerNotice.test.tsx @@ -46,6 +46,22 @@ describe("ordinary task interruption", () => { expect(retry).toHaveBeenCalledOnce(); }); + it.each([false, true])("names the question without repeating a historical failure (manual Retry=%s)", async canRetry => { + const presentation = { ...failed, responsibility: "user" as const, state: "waiting_for_user" as const, userAction: "question" as const, canRetry, + providerFailure: { schema: "paperclip.provider-failure.v1", provider: "anthropic", phase: "startup", + category: "rate_limit", httpStatus: 429, retryability: "unknown", message: "API error: 429" } } satisfies TaskInterruptionPresentation; + await render(presentation); + expect(container.textContent).toContain("Respond to the pending question to continue."); + expect(container.textContent).not.toMatch(/429|rate limit|failed/i); + expect(Boolean(container.querySelector("button"))).toBe(canRetry); + if (canRetry) { + await act(async () => container.querySelector("button")!.click()); + expect(retry).toHaveBeenCalledOnce(); + } + await render(presentation, { canRetry: false }); + expect(container.querySelector("button")).toBeNull(); + }); + it("keeps the failure visible and names the gate while a saved Retry waits on the budget", async () => { await render({ ...failed, responsibility: "user", state: "waiting_for_user", userAction: "budget", canRetry: false }); const notice = container.querySelector('[data-testid="task-execution-waiting"]'); diff --git a/ui/src/components/ExecutionBlockerNotice.tsx b/ui/src/components/ExecutionBlockerNotice.tsx index 9ee5f5b2c7f..840d1b8e88b 100644 --- a/ui/src/components/ExecutionBlockerNotice.tsx +++ b/ui/src/components/ExecutionBlockerNotice.tsx @@ -60,6 +60,11 @@ export function ExecutionBlockerNotice({ presentation, canRetry, onRetry, optimi && ["working", "retry_scheduled"].includes(presentation.state) ? recoveryStepText[presentation.recoveryStep] : null; const seconds = Math.max(0, Math.ceil((Date.parse(presentation?.nextRetryAt ?? "") - Date.now()) / 1000)); const retryIn = `Trying again in ${seconds > 60 ? `${Math.ceil(seconds / 60)}m` : `${seconds}s`}…`; + if (!optimistic && presentation?.state === "waiting_for_user" && presentation.userAction === "question") + return
+ Respond to the pending question to continue. + {canRetry && presentation.canRetry && } +
; if (!optimistic && !presentation?.retryWait && presentation?.providerFailure && ["failed", "retry_scheduled", "waiting_for_user"].includes(presentation.state)) { // Saved-work recovery is what the task waits for, so name it over the // generic automatic-retry line while keeping the provider explanation. diff --git a/ui/src/context/LiveUpdatesProvider.test.ts b/ui/src/context/LiveUpdatesProvider.test.ts index ca7156853e5..ccf1ebbaaf4 100644 --- a/ui/src/context/LiveUpdatesProvider.test.ts +++ b/ui/src/context/LiveUpdatesProvider.test.ts @@ -1791,6 +1791,12 @@ describe("LiveUpdatesProvider visible issue toast suppression", () => { }); describe("LiveUpdatesProvider run lifecycle toasts", () => { + it("does not notify each task held before provider work for the same usage limit", () => { + expect(__liveUpdatesTestUtils.buildRunStatusToast({ + runId: "held-run", agentId: "agent-1", status: "cancelled", errorCode: "provider_limit_wait", + error: "Waiting to recheck the provider usage limit", + }, () => "Assistant")).toBeNull(); + }); it("does not build start or success toasts for agent runs", () => { const queryClient = { getQueryCache: () => ({ findAll: () => [] }), diff --git a/ui/src/context/LiveUpdatesProvider.tsx b/ui/src/context/LiveUpdatesProvider.tsx index a9aacca3502..ba35d31e285 100644 --- a/ui/src/context/LiveUpdatesProvider.tsx +++ b/ui/src/context/LiveUpdatesProvider.tsx @@ -1362,6 +1362,7 @@ function buildRunStatusToast( // Interrupt is an intentional conversation control. Its caller gives // feedback; the terminal event must not announce a cancelled/failed run. if (errorCode === "operator_interrupted") return null; + if (status === "cancelled" && errorCode === "provider_limit_wait") return null; // Workspace contention is ordinary scheduling, not a failed user action. if (errorCode === "workspace_busy" || errorCode === "agent_directory_busy") return null; const contextSource = readString(payload.contextSource); diff --git a/ui/src/lib/provider-limit-message.test.ts b/ui/src/lib/provider-limit-message.test.ts index 57b4ab5c993..bdf0a816974 100644 --- a/ui/src/lib/provider-limit-message.test.ts +++ b/ui/src/lib/provider-limit-message.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import { formatProviderLimitMessage } from "./provider-limit-message"; +import { formatProviderLimitMessage, providerLimitMessageResetAt } from "./provider-limit-message"; afterEach(() => { vi.useRealTimers(); @@ -39,9 +39,13 @@ describe("provider limit messages", () => { `Waiting for the provider usage limit to reset at ${resetAt} before starting work`, `The provider usage limit resets at ${resetAt}; this wake could not schedule its retry and released the task`, `Waiting for the agent's provider usage limit to reset; the work starts at ${resetAt}.`, + `This agent's provider usage limit has no reported reset time. Work is held; a retry can recheck availability after ${resetAt}; the limit may still apply then.`, + `Waiting to recheck the provider usage limit after ${resetAt} before starting work`, + `Paperclip will recheck the agent's provider usage limit at ${resetAt} before starting work.`, ]) { expect(formatProviderLimitMessage(text)).toContain("in 1 hour ("); expect(formatProviderLimitMessage(text)).not.toContain(resetAt); + expect(providerLimitMessageResetAt(text)).toBe(Date.parse(resetAt)); } }); diff --git a/ui/src/lib/provider-limit-message.ts b/ui/src/lib/provider-limit-message.ts index e2880d05f0d..7f839b39122 100644 --- a/ui/src/lib/provider-limit-message.ts +++ b/ui/src/lib/provider-limit-message.ts @@ -4,7 +4,7 @@ const relativeTimeFormatter = new Intl.RelativeTimeFormat("en-US", { numeric: "a const relativeUnits = [ [86_400_000, "day"], [3_600_000, "hour"], [60_000, "minute"], [1000, "second"], ] as const; -const resetTimestampPattern = /\b(resets?|starts) at (\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,3})?Z)\b/g; +const resetTimestampPattern = /\b(resets?|starts|recheck(?: availability| the (?:agent's )?provider usage limit)?) (?:at|after) (\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,3})?Z)\b/g; export function providerLimitMessageResetAt(message: string): number | null { if (!message.includes("provider usage limit")) return null; diff --git a/ui/src/pages/Inbox.test.tsx b/ui/src/pages/Inbox.test.tsx index 4e0a4a9e8e5..e0517fa897d 100644 --- a/ui/src/pages/Inbox.test.tsx +++ b/ui/src/pages/Inbox.test.tsx @@ -2650,6 +2650,19 @@ describe("FailedRunInboxRow", () => { container.remove(); }); + it.each([false, true])("shows the question over historical errors and preserves manual Retry=%s", canRetry => { + const root = createRoot(container); + const issue = createIssue({ interruption: { intentId: null, responsibility: "user", state: "waiting_for_user", + nextRetryAt: null, canRetry, userAction: "question", sourceRunIds: ["run-1"], observedRunId: "run-1" } }); + act(() => root.render( {}} onRetry={() => {}} isRetrying={false} />)); + expect(container.textContent).toContain("Respond to the pending question"); + expect(container.textContent).not.toContain("429"); + expect(container.textContent?.includes("Retry")).toBe(canRetry); + act(() => root.unmount()); + }); + it.each(["working", "failed"] as const)("shows saved-work shutdown guidance while %s without Retry", state => { const root = createRoot(container); const message = state === "working" ? "Your work is saved. Paperclip is finishing shutdown." diff --git a/ui/src/pages/Inbox.tsx b/ui/src/pages/Inbox.tsx index 22d235f7c43..42a41eec35e 100644 --- a/ui/src/pages/Inbox.tsx +++ b/ui/src/pages/Inbox.tsx @@ -415,7 +415,7 @@ export function FailedRunInboxRow({ : interruption?.state === "retry_scheduled" ? `Trying again in ${retrySeconds}s…` : interruption?.state === "failed" ? "Couldn’t finish this attempt. Your message is saved." : interruption?.state === "waiting_for_user" ? ({ - approval: "Approval required", reconnect: "Reconnect account", assign_agent: "Assign an agent", choose_model: "Choose a supported model", + approval: "Approval required", question: "Respond to the pending question", reconnect: "Reconnect account", assign_agent: "Assign an agent", choose_model: "Choose a supported model", resume: "Work is paused", budget: "Budget limit reached", project_access: "Agent needs project access", confirm_owner_stop: "Confirm the previous server stopped",