diff --git a/packages/shared/src/constants.ts b/packages/shared/src/constants.ts index 25fbab24acc..36ef970e6f3 100644 --- a/packages/shared/src/constants.ts +++ b/packages/shared/src/constants.ts @@ -402,6 +402,23 @@ export const ISSUE_RECOVERY_ACTION_OWNER_TYPES = [ ] as const; export type IssueRecoveryActionOwnerType = (typeof ISSUE_RECOVERY_ACTION_OWNER_TYPES)[number]; +/** + * Delivery state of one human feedback comment, as projected onto that comment + * for operator surfaces. This is deliberately *not* a task status: the task + * keeps its own workflow status while a delivery attempt is in flight. + * + * - `retrying` — automatic replay is still in flight; no operator action. + * - `exhausted_needs_attention` — automatic replay gave up; a human must act. + * - `recovered_after_attention` — delivery succeeded after it had needed + * attention, so the resolution stays visible in thread history. + */ +export const ISSUE_FEEDBACK_DELIVERY_STATES = [ + "retrying", + "exhausted_needs_attention", + "recovered_after_attention", +] as const; +export type IssueFeedbackDeliveryState = (typeof ISSUE_FEEDBACK_DELIVERY_STATES)[number]; + export const ISSUE_RECOVERY_ACTION_OUTCOMES = [ "restored", "handed_back", diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index b98c0db9de9..6d8bdae6760 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -299,6 +299,7 @@ export { ONBOARDING_FIRST_TASK_ORIGIN_KIND, ISSUE_WATCHDOG_DISCOVERY_KINDS, ISSUE_SURFACE_VISIBILITIES, + ISSUE_FEEDBACK_DELIVERY_STATES, ISSUE_RECOVERY_ACTION_KINDS, ISSUE_RECOVERY_ACTION_STATUSES, ISSUE_RECOVERY_ACTION_OWNER_TYPES, @@ -492,6 +493,7 @@ export { type IssueOriginKind, type IssueWatchdogDiscoveryKind, type IssueSurfaceVisibility, + type IssueFeedbackDeliveryState, type IssueRecoveryActionKind, type IssueRecoveryActionStatus, type IssueRecoveryActionOwnerType, @@ -1008,6 +1010,8 @@ export type { IssueExecutionStagePrincipal, IssueExecutionDecision, IssueComment, + IssueCommentFeedbackDelivery, + IssueFeedbackDeliveryRetryResponse, IssueCommentDerivedAuthorSource, IssueCommentMetadata, IssueCommentMetadataSection, diff --git a/packages/shared/src/types/index.ts b/packages/shared/src/types/index.ts index 1f4971b940d..77149f2e85b 100644 --- a/packages/shared/src/types/index.ts +++ b/packages/shared/src/types/index.ts @@ -649,6 +649,8 @@ export type { IssueReviewRequest, IssueExecutionDecision, IssueComment, + IssueCommentFeedbackDelivery, + IssueFeedbackDeliveryRetryResponse, IssueCommentDerivedAuthorSource, IssueCommentMetadata, IssueCommentMetadataSection, diff --git a/packages/shared/src/types/issue.ts b/packages/shared/src/types/issue.ts index 0dc0bc030ac..485f217e246 100644 --- a/packages/shared/src/types/issue.ts +++ b/packages/shared/src/types/issue.ts @@ -14,6 +14,7 @@ import type { IssueReferenceSourceKind, IssueExecutionStageType, IssueExecutionStateStatus, + IssueFeedbackDeliveryState, IssueHarnessKind, IssueOriginKind, IssuePriority, @@ -941,6 +942,54 @@ export type IssueCommentDerivedAuthorSource = | "run_id" | "run_log_comment_post"; +/** + * Delivery state projected onto a single human feedback comment. + * + * Every field here is safe to render to any operator who can already read the + * comment: raw adapter errors, stack traces, request bodies, credentials, and + * filesystem paths are mapped to a short `safeFailureReason` code server-side + * and never passed through verbatim. + */ +export interface IssueCommentFeedbackDelivery { + state: IssueFeedbackDeliveryState; + /** The human comment this delivery obligation belongs to. */ + sourceCommentId: string; + /** Completed automatic replay generations, or `null` when unknown. */ + attempt: number | null; + /** Automatic replay budget, or `null` when unknown. */ + maxAttempts: number | null; + nextAttemptAt: string | null; + lastAttemptAt: string | null; + deliveredAt: string | null; + /** + * Run the operator may open for context. Omitted (`null`) when no run exists + * or the viewer is not authorized to read it — never rendered as a disabled + * mystery action. + */ + targetRunId: string | null; + /** True when the viewer may ask for one more explicit delivery attempt. */ + canRetry: boolean; + /** + * True when a replay is already queued for this obligation. Keeps the + * "Retrying…" affordance durable across reloads instead of living only in + * button-local component state. + */ + retryInFlight: boolean; + /** Short operator-safe reason code, never a raw adapter error. */ + safeFailureReason: string | null; + /** + * How many human comments share this one delivery obligation (a batched wake + * can cover several). Used for the thread rollup copy. + */ + batchedCommentCount: number; +} + +export interface IssueFeedbackDeliveryRetryResponse { + outcome: "queued" | "already_queued" | "not_exhausted" | "no_invokable_assignee"; + message: string; + feedbackDelivery: IssueCommentFeedbackDelivery | null; +} + export interface IssueComment { id: string; companyId: string; @@ -964,6 +1013,12 @@ export interface IssueComment { deletedByRunId?: string | null; sourceTrust?: SourceTrustMetadata | null; followUpRequested?: boolean; + /** + * Present only while this human comment's delivery is retrying, exhausted, or + * freshly recovered. Absent for the overwhelming majority of comments, which + * were delivered on the first attempt. + */ + feedbackDelivery?: IssueCommentFeedbackDelivery | null; createdAt: Date; updatedAt: Date; } diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.ts index cff29679612..f02ec664dc5 100644 --- a/server/src/routes/issues.ts +++ b/server/src/routes/issues.ts @@ -88,6 +88,7 @@ import { type IssueWakeDiagnosticWakeFailureClass, type IssueWakeDiagnosticWakeRequest, type IssueWakeDiagnosticsResponse, + type IssueFeedbackDeliveryRetryResponse, type IssueRelationIssueSummary, type IssueReviewPolicy, type IssueThreadInteractionCanonicalResolverPolicy, @@ -104,6 +105,14 @@ import { type IssueWriteDenialContext, } from "@paperclipai/shared"; import { trackAgentTaskCompleted } from "@paperclipai/shared/telemetry"; +import { + buildFeedbackDeliveryManualRetryIdempotencyKey, + FEEDBACK_DELIVERY_MAX_AUTOMATIC_RETRIES, + FEEDBACK_DELIVERY_RECOVERY_ACTION_KIND, + FEEDBACK_DELIVERY_RECOVERY_CAUSE, + FEEDBACK_DELIVERY_RETRY_WAKE_REASON, + FEEDBACK_DELIVERY_WAKE_COMMENT_IDS_KEY, +} from "../services/recovery/feedback-delivery.js"; import { getTelemetryClient } from "../telemetry.js"; import { isUniqueViolation } from "../db-errors.js"; import type { StorageService } from "../storage/types.js"; @@ -116,6 +125,7 @@ import { companyService, companySearchService, executionWorkspaceService, + feedbackDeliveryCommentStateService, goalService, heartbeatService, issueApprovalService, @@ -2756,6 +2766,7 @@ export function issueRoutes( const goalsSvc = goalService(db); const issueApprovalsSvc = issueApprovalService(db); const recoveryActionsSvc = issueRecoveryActionService(db); + const feedbackDeliveryStatesSvc = feedbackDeliveryCommentStateService(db); const executionWorkspacesSvc = executionWorkspaceServiceDirect(db); const workProductsSvc = workProductService(db); const documentsSvc = documentService(db); @@ -10763,7 +10774,159 @@ export function issueRoutes( order, limit, }); - res.json(await runRedactions.redactForIssue(issue.companyId, issue.id, comments)); + // Operator surfaces only: an agent reading its own thread has no delivery + // banner to act on, and the run reference stays gated behind the same + // board/user authority that guards run detail. + const isOperatorViewer = req.actor.type !== "agent"; + const feedbackDeliveryByComment = await feedbackDeliveryStatesSvc.listForIssue({ + companyId: issue.companyId, + issueId: issue.id, + viewerCanRetry: isOperatorViewer, + viewerCanReadRuns: isOperatorViewer, + }); + const withFeedbackDelivery = feedbackDeliveryByComment.size === 0 + ? comments + : comments.map((comment) => { + const feedbackDelivery = feedbackDeliveryByComment.get(comment.id) ?? null; + return feedbackDelivery ? { ...comment, feedbackDelivery } : comment; + }); + res.json(await runRedactions.redactForIssue(issue.companyId, issue.id, withFeedbackDelivery)); + }); + + /** + * Explicit operator "Retry delivery" for a human comment whose automatic + * replay budget is exhausted. Board-only: this is the operator escape hatch + * from the exhausted-needs-attention state, not an agent-callable API. + * + * The recovery action stays open. It is discharged by the normal + * revalidation path once the replay actually produces a disposition, which is + * what turns the banner into the recovered notice — a retry request alone must + * not clear the attention item. + */ + router.post("/issues/:id/feedback-delivery/retry", async (req, res) => { + assertBoard(req); + const id = req.params.id as string; + const issue = await getAccessibleResource(req, res, svc.getById(id), "Issue not found"); + if (!issue) return; + + const actor = getActorInfo(req); + const readStates = () => + feedbackDeliveryStatesSvc.listForIssue({ + companyId: issue.companyId, + issueId: issue.id, + viewerCanRetry: true, + viewerCanReadRuns: true, + }); + + const activeAction = await recoveryActionsSvc.getActiveForIssue(issue.companyId, issue.id); + const isFeedbackDeliveryAction = + activeAction?.kind === FEEDBACK_DELIVERY_RECOVERY_ACTION_KIND + && activeAction.cause === FEEDBACK_DELIVERY_RECOVERY_CAUSE; + if (!activeAction || !isFeedbackDeliveryAction) { + const states = await readStates(); + res.json({ + outcome: "not_exhausted", + message: "This task has no exhausted feedback delivery to retry.", + feedbackDelivery: states.values().next().value ?? null, + } satisfies IssueFeedbackDeliveryRetryResponse); + return; + } + + const evidence = (typeof activeAction.evidence === "object" && activeAction.evidence !== null + ? activeAction.evidence + : {}) as Record; + const rootWakeupRequestId = typeof evidence.rootWakeupRequestId === "string" + ? evidence.rootWakeupRequestId + : null; + const outstandingCommentIds = Array.isArray(evidence.outstandingCommentIds) + ? evidence.outstandingCommentIds.filter((entry): entry is string => typeof entry === "string") + : []; + // The original assignee owns the feedback: hand the replay back to it rather + // than to whichever manager the recovery ladder parked the action on. + const targetAgentId = activeAction.returnOwnerAgentId + ?? activeAction.previousOwnerAgentId + ?? issue.assigneeAgentId; + if (!targetAgentId || !rootWakeupRequestId) { + res.json({ + outcome: "no_invokable_assignee", + message: "This delivery has no agent to retry against. Reassign the task first.", + feedbackDelivery: (await readStates()).get(outstandingCommentIds[0] ?? "") ?? null, + } satisfies IssueFeedbackDeliveryRetryResponse); + return; + } + + const statesBefore = await readStates(); + const alreadyInFlight = outstandingCommentIds.some( + (commentId) => statesBefore.get(commentId)?.retryInFlight === true, + ); + if (alreadyInFlight) { + res.json({ + outcome: "already_queued", + message: "A delivery retry is already queued.", + feedbackDelivery: statesBefore.get(outstandingCommentIds[0] ?? "") ?? null, + } satisfies IssueFeedbackDeliveryRetryResponse); + return; + } + + const latestCommentId = outstandingCommentIds.at(-1) ?? null; + const deliveryContext = { + issueId: issue.id, + taskId: issue.id, + feedbackDeliveryRootWakeupRequestId: rootWakeupRequestId, + feedbackDeliveryRetryGeneration: FEEDBACK_DELIVERY_MAX_AUTOMATIC_RETRIES, + feedbackDeliveryOperatorRequested: true, + [FEEDBACK_DELIVERY_WAKE_COMMENT_IDS_KEY]: outstandingCommentIds, + ...(latestCommentId ? { commentId: latestCommentId, wakeCommentId: latestCommentId } : {}), + }; + await heartbeat.wakeup(targetAgentId, { + source: "on_demand", + triggerDetail: "manual", + reason: FEEDBACK_DELIVERY_RETRY_WAKE_REASON, + payload: deliveryContext, + idempotencyKey: buildFeedbackDeliveryManualRetryIdempotencyKey({ + fingerprint: activeAction.fingerprint, + attempt: (activeAction.attemptCount ?? 0) + 1, + }), + requestedByActorType: actor.actorType === "user" ? "user" : "system", + requestedByActorId: actor.actorId, + contextSnapshot: { + ...deliveryContext, + wakeReason: FEEDBACK_DELIVERY_RETRY_WAKE_REASON, + source: "issue.operator_feedback_delivery_retry", + instruction: + "An operator asked Paperclip to deliver human feedback on this task again. Handle the outstanding " + + "comment(s) now and record a valid disposition.", + }, + }); + + await logActivity(db, { + companyId: issue.companyId, + actorType: actor.actorType, + actorId: actor.actorId, + agentId: targetAgentId, + action: "issue.feedback_delivery_retry_requested", + entityType: "issue", + entityId: issue.id, + details: { + recoveryActionId: activeAction.id, + rootWakeupRequestId, + sourceCommentIds: outstandingCommentIds, + attempt: (activeAction.attemptCount ?? 0) + 1, + }, + }); + + const statesAfter = await readStates(); + const projected = latestCommentId ? statesAfter.get(latestCommentId) ?? null : null; + res.json({ + outcome: "queued", + message: "Retrying feedback delivery.", + // Deferred behind a live run still reads as in-flight to the operator, so + // fall back to a synthetic in-flight projection rather than reporting a + // state that would re-enable the button it just disabled. + feedbackDelivery: projected + ? { ...projected, retryInFlight: true, canRetry: false } + : null, + } satisfies IssueFeedbackDeliveryRetryResponse); }); router.get("/issues/:id/interactions", async (req, res) => { diff --git a/server/src/services/feedback-delivery-comment-states.ts b/server/src/services/feedback-delivery-comment-states.ts new file mode 100644 index 00000000000..32f2a82b8c7 --- /dev/null +++ b/server/src/services/feedback-delivery-comment-states.ts @@ -0,0 +1,228 @@ +import { and, eq, gte, inArray, or, sql } from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; +import { agentWakeupRequests, heartbeatRuns, issueRecoveryActions } from "@paperclipai/db"; +import type { IssueCommentFeedbackDelivery } from "@paperclipai/shared"; + +import { + buildFeedbackDeliveryFingerprint, + FEEDBACK_DELIVERY_RECOVERY_ACTION_KIND, + FEEDBACK_DELIVERY_RETRY_WAKE_REASON, + readFeedbackDeliveryWakeCommentIds, +} from "./recovery/feedback-delivery.js"; +import { + FEEDBACK_DELIVERY_RECOVERED_WINDOW_MS, + projectFeedbackDeliveryByComment, + type FeedbackDeliveryPendingReplay, +} from "./recovery/feedback-delivery-projection.js"; + +/** Run statuses that still represent a delivery attempt the operator is waiting on. */ +const PENDING_REPLAY_RUN_STATUSES = ["queued", "scheduled_retry", "running"] as const; + +/** + * Wake statuses that still represent an undelivered replay. A replay deferred + * behind a live run on the same issue is still something the operator is + * waiting on, so it reads as `retrying` just like a queued one. + */ +const PENDING_REPLAY_WAKE_STATUSES = ["queued", "deferred_issue_execution"] as const; + +function readRecord(value: unknown): Record { + return typeof value === "object" && value !== null ? (value as Record) : {}; +} + +function readNonEmptyString(value: unknown) { + return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; +} + +function readGeneration(value: unknown) { + return typeof value === "number" && Number.isFinite(value) ? Math.max(0, Math.floor(value)) : 0; +} + +/** + * The two replay lanes spell the wake payload slightly differently — the + * immediate lane writes `rootWakeupRequestId`/`commentIds`, the backstop lane + * writes the run-context spelling. Read both so neither lane goes invisible. + */ +function readReplayRootWakeupRequestId(payload: Record) { + return ( + readNonEmptyString(payload.feedbackDeliveryRootWakeupRequestId) + ?? readNonEmptyString(payload.rootWakeupRequestId) + ); +} + +function readReplayCommentIds(payload: Record) { + const merged = [...readFeedbackDeliveryWakeCommentIds(payload)]; + const legacy = payload.commentIds; + if (Array.isArray(legacy)) { + for (const entry of legacy) { + const normalized = readNonEmptyString(entry); + if (normalized && !merged.includes(normalized)) merged.push(normalized); + } + } + return merged; +} + +/** + * Reads the durable feedback-delivery signals for one issue and projects them + * onto the individual human comments they belong to. + * + * Three cheap, index-backed reads, all scoped to the issue: + * - `feedback_delivery` recovery actions (active plus recently resolved), + * - queued replay wakes (`payload->>'issueId'` index), + * - non-terminal runs carrying replay context (`contextSnapshot->>'issueId'`). + * + * The projection itself is pure and unit-tested separately. + */ +export function feedbackDeliveryCommentStateService(db: Db) { + async function loadRecoveryActions(input: { + companyId: string; + issueId: string; + resolvedSince: Date; + }) { + return db + .select({ + id: issueRecoveryActions.id, + kind: issueRecoveryActions.kind, + cause: issueRecoveryActions.cause, + status: issueRecoveryActions.status, + fingerprint: issueRecoveryActions.fingerprint, + evidence: issueRecoveryActions.evidence, + attemptCount: issueRecoveryActions.attemptCount, + maxAttempts: issueRecoveryActions.maxAttempts, + lastAttemptAt: issueRecoveryActions.lastAttemptAt, + resolvedAt: issueRecoveryActions.resolvedAt, + createdAt: issueRecoveryActions.createdAt, + }) + .from(issueRecoveryActions) + .where( + and( + eq(issueRecoveryActions.companyId, input.companyId), + eq(issueRecoveryActions.sourceIssueId, input.issueId), + eq(issueRecoveryActions.kind, FEEDBACK_DELIVERY_RECOVERY_ACTION_KIND), + or( + inArray(issueRecoveryActions.status, ["active", "escalated"]), + gte(issueRecoveryActions.resolvedAt, input.resolvedSince), + ), + ), + ); + } + + async function loadPendingReplays(input: { + companyId: string; + issueId: string; + }): Promise { + const [wakes, runs] = await Promise.all([ + db + .select({ + agentId: agentWakeupRequests.agentId, + payload: agentWakeupRequests.payload, + runId: agentWakeupRequests.runId, + requestedAt: agentWakeupRequests.requestedAt, + }) + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, input.companyId), + inArray(agentWakeupRequests.status, [...PENDING_REPLAY_WAKE_STATUSES]), + eq(agentWakeupRequests.reason, FEEDBACK_DELIVERY_RETRY_WAKE_REASON), + sql`${agentWakeupRequests.payload} ->> 'issueId' = ${input.issueId}`, + ), + ), + db + .select({ + id: heartbeatRuns.id, + agentId: heartbeatRuns.agentId, + contextSnapshot: heartbeatRuns.contextSnapshot, + wakeupRequestId: heartbeatRuns.wakeupRequestId, + scheduledRetryAt: heartbeatRuns.scheduledRetryAt, + createdAt: heartbeatRuns.createdAt, + }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, input.companyId), + inArray(heartbeatRuns.status, [...PENDING_REPLAY_RUN_STATUSES]), + sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${input.issueId}`, + sql`${heartbeatRuns.contextSnapshot} ->> 'feedbackDeliveryRootWakeupRequestId' is not null`, + ), + ), + ]); + + const replays: FeedbackDeliveryPendingReplay[] = []; + + for (const wake of wakes) { + const payload = readRecord(wake.payload); + const rootWakeupRequestId = readReplayRootWakeupRequestId(payload); + if (!rootWakeupRequestId) continue; + replays.push({ + fingerprint: buildFeedbackDeliveryFingerprint({ + companyId: input.companyId, + issueId: input.issueId, + agentId: wake.agentId, + rootWakeupRequestId, + }), + commentIds: readReplayCommentIds(payload), + generation: readGeneration(payload.feedbackDeliveryRetryGeneration), + runId: wake.runId ?? null, + nextAttemptAt: null, + requestedAt: wake.requestedAt, + }); + } + + for (const run of runs) { + const context = readRecord(run.contextSnapshot); + const rootWakeupRequestId = readReplayRootWakeupRequestId(context) ?? run.wakeupRequestId; + if (!rootWakeupRequestId) continue; + replays.push({ + fingerprint: buildFeedbackDeliveryFingerprint({ + companyId: input.companyId, + issueId: input.issueId, + agentId: run.agentId, + rootWakeupRequestId, + }), + commentIds: readReplayCommentIds(context), + generation: readGeneration(context.feedbackDeliveryRetryGeneration), + runId: run.id, + nextAttemptAt: run.scheduledRetryAt, + requestedAt: run.createdAt, + }); + } + + return replays; + } + + return { + /** + * Returns a comment-id keyed map of delivery state, or an empty map when + * this issue has no feedback-delivery history at all (the common case). + */ + async listForIssue(input: { + companyId: string; + issueId: string; + viewerCanRetry: boolean; + viewerCanReadRuns: boolean; + now?: Date; + }): Promise> { + const now = input.now ?? new Date(); + // Issued together: the first automatic replay is `retrying` *before* any + // recovery action exists, so recovery actions cannot gate the replay read. + // All three predicates are index-backed and scoped to one issue. + const [recoveryActions, pendingReplays] = await Promise.all([ + loadRecoveryActions({ + companyId: input.companyId, + issueId: input.issueId, + resolvedSince: new Date(now.getTime() - FEEDBACK_DELIVERY_RECOVERED_WINDOW_MS), + }), + loadPendingReplays({ companyId: input.companyId, issueId: input.issueId }), + ]); + if (recoveryActions.length === 0 && pendingReplays.length === 0) return new Map(); + + return projectFeedbackDeliveryByComment({ + recoveryActions, + pendingReplays, + viewerCanRetry: input.viewerCanRetry, + viewerCanReadRuns: input.viewerCanReadRuns, + now, + }); + }, + }; +} diff --git a/server/src/services/index.ts b/server/src/services/index.ts index 2ab2994c246..3ff658fd209 100644 --- a/server/src/services/index.ts +++ b/server/src/services/index.ts @@ -49,6 +49,7 @@ export { issueTreeControlService } from "./issue-tree-control.js"; export { issueApprovalService } from "./issue-approvals.js"; export { issueReferenceService } from "./issue-references.js"; export { issueRecoveryActionService } from "./issue-recovery-actions.js"; +export { feedbackDeliveryCommentStateService } from "./feedback-delivery-comment-states.js"; export { stalledReviewDecisionService, type DecideStalledReviewInput, diff --git a/server/src/services/recovery/feedback-delivery-projection.test.ts b/server/src/services/recovery/feedback-delivery-projection.test.ts new file mode 100644 index 00000000000..b8973a397f7 --- /dev/null +++ b/server/src/services/recovery/feedback-delivery-projection.test.ts @@ -0,0 +1,293 @@ +import { describe, expect, it } from "vitest"; + +import { + FEEDBACK_DELIVERY_RECOVERED_WINDOW_MS, + projectFeedbackDeliveryByComment, + toSafeFeedbackDeliveryFailureReason, + type FeedbackDeliveryPendingReplay, + type FeedbackDeliveryRecoveryActionRow, +} from "./feedback-delivery-projection.js"; +import { FEEDBACK_DELIVERY_RECOVERY_CAUSE } from "./feedback-delivery.js"; + +const NOW = new Date("2026-08-14T18:00:00.000Z"); +const FINGERPRINT = "feedback_delivery:company-1:issue-1:agent-1:root-wake-1"; + +function action( + overrides: Partial = {}, +): FeedbackDeliveryRecoveryActionRow { + return { + id: "action-1", + kind: "feedback_delivery", + cause: FEEDBACK_DELIVERY_RECOVERY_CAUSE, + status: "active", + fingerprint: FINGERPRINT, + evidence: { + outstandingCommentIds: ["comment-1"], + rootWakeupRequestId: "root-wake-1", + retryGeneration: 1, + maxAutomaticRetries: 1, + sourceRunId: "run-1", + lastFailureCode: "process_lost", + // Deliberately present: the projection must never surface it. + lastFailureReason: "Error: ECONNREFUSED /home/agent/.secret/token.json", + }, + attemptCount: 1, + maxAttempts: 1, + lastAttemptAt: new Date("2026-08-14T17:58:00.000Z"), + resolvedAt: null, + createdAt: new Date("2026-08-14T17:57:00.000Z"), + ...overrides, + }; +} + +function replay( + overrides: Partial = {}, +): FeedbackDeliveryPendingReplay { + return { + fingerprint: FINGERPRINT, + commentIds: ["comment-1"], + generation: 1, + runId: "run-2", + nextAttemptAt: new Date("2026-08-14T18:00:12.000Z"), + requestedAt: new Date("2026-08-14T17:59:30.000Z"), + ...overrides, + }; +} + +function project(input: { + recoveryActions?: FeedbackDeliveryRecoveryActionRow[]; + pendingReplays?: FeedbackDeliveryPendingReplay[]; + viewerCanRetry?: boolean; + viewerCanReadRuns?: boolean; + now?: Date; +}) { + return projectFeedbackDeliveryByComment({ + recoveryActions: input.recoveryActions ?? [], + pendingReplays: input.pendingReplays ?? [], + viewerCanRetry: input.viewerCanRetry ?? true, + viewerCanReadRuns: input.viewerCanReadRuns ?? true, + now: input.now ?? NOW, + }); +} + +describe("projectFeedbackDeliveryByComment", () => { + it("returns nothing for an issue with no delivery history", () => { + expect(project({}).size).toBe(0); + }); + + it("projects a queued replay as quiet retrying state with no operator action", () => { + const states = project({ pendingReplays: [replay()] }); + const state = states.get("comment-1"); + expect(state).toMatchObject({ + state: "retrying", + sourceCommentId: "comment-1", + attempt: 1, + maxAttempts: 1, + canRetry: false, + retryInFlight: true, + safeFailureReason: null, + nextAttemptAt: "2026-08-14T18:00:12.000Z", + targetRunId: "run-2", + }); + }); + + it("projects an active recovery action as exhausted and offers retry", () => { + const state = project({ recoveryActions: [action()] }).get("comment-1"); + expect(state).toMatchObject({ + state: "exhausted_needs_attention", + attempt: 1, + maxAttempts: 1, + canRetry: true, + retryInFlight: false, + targetRunId: "run-1", + safeFailureReason: "The agent's process stopped before it read the comment.", + batchedCommentCount: 1, + }); + }); + + it("never leaks a raw failure reason into the projection", () => { + const state = project({ + recoveryActions: [ + action({ + evidence: { + outstandingCommentIds: ["comment-1"], + lastFailureCode: "some_unmapped_adapter_code", + lastFailureReason: "Error: ECONNREFUSED /home/agent/.secret/token.json", + gateReason: "workspace /srv/secret/path is dirty", + }, + }), + ], + }).get("comment-1"); + expect(state?.safeFailureReason).toBeNull(); + expect(JSON.stringify(state)).not.toContain("ECONNREFUSED"); + expect(JSON.stringify(state)).not.toContain("/srv/secret/path"); + }); + + it("keeps the actionable banner while an explicit retry is in flight", () => { + const state = project({ + recoveryActions: [action()], + pendingReplays: [replay()], + }).get("comment-1"); + // Still exhausted: clearing it here would drop the attention item before the + // replay has actually delivered anything. + expect(state).toMatchObject({ + state: "exhausted_needs_attention", + retryInFlight: true, + canRetry: false, + }); + }); + + it("projects a recently resolved action as recovered after attention", () => { + const state = project({ + recoveryActions: [ + action({ + status: "resolved", + resolvedAt: new Date("2026-08-14T17:59:00.000Z"), + }), + ], + }).get("comment-1"); + expect(state).toMatchObject({ + state: "recovered_after_attention", + deliveredAt: "2026-08-14T17:59:00.000Z", + canRetry: false, + retryInFlight: false, + safeFailureReason: null, + }); + }); + + it("drops the recovered notice once it ages out of the window", () => { + const states = project({ + recoveryActions: [ + action({ + status: "resolved", + resolvedAt: new Date(NOW.getTime() - FEEDBACK_DELIVERY_RECOVERED_WINDOW_MS - 1000), + }), + ], + }); + expect(states.size).toBe(0); + }); + + it("lets a re-opened obligation win over its own earlier resolution", () => { + const state = project({ + recoveryActions: [ + action({ + id: "action-old", + status: "resolved", + resolvedAt: new Date("2026-08-14T17:30:00.000Z"), + }), + action({ id: "action-new", status: "active" }), + ], + }).get("comment-1"); + expect(state?.state).toBe("exhausted_needs_attention"); + }); + + it("keeps a newer comment's obligation from clearing an older one", () => { + const states = project({ + recoveryActions: [ + action({ + id: "action-old", + fingerprint: `${FINGERPRINT}:old`, + status: "active", + evidence: { outstandingCommentIds: ["comment-1"], retryGeneration: 1 }, + }), + action({ + id: "action-new", + fingerprint: `${FINGERPRINT}:new`, + status: "resolved", + resolvedAt: new Date("2026-08-14T17:59:00.000Z"), + evidence: { outstandingCommentIds: ["comment-2"], retryGeneration: 1 }, + }), + ], + }); + expect(states.get("comment-1")?.state).toBe("exhausted_needs_attention"); + expect(states.get("comment-2")?.state).toBe("recovered_after_attention"); + }); + + it("rolls one batched obligation onto every comment it covers", () => { + const states = project({ + recoveryActions: [ + action({ + evidence: { + outstandingCommentIds: ["comment-1", "comment-2", "comment-3"], + retryGeneration: 1, + }, + }), + ], + }); + expect([...states.keys()]).toEqual(["comment-1", "comment-2", "comment-3"]); + for (const state of states.values()) { + expect(state.batchedCommentCount).toBe(3); + expect(state.state).toBe("exhausted_needs_attention"); + } + }); + + it("omits the run reference for a viewer who cannot read runs", () => { + const state = project({ + recoveryActions: [action()], + pendingReplays: [replay()], + viewerCanReadRuns: false, + }).get("comment-1"); + expect(state?.targetRunId).toBeNull(); + // The recovery action stays available even without a run link. + expect(state?.state).toBe("exhausted_needs_attention"); + }); + + it("withholds retry from a viewer without operator authority", () => { + const state = project({ + recoveryActions: [action()], + viewerCanRetry: false, + }).get("comment-1"); + expect(state?.canRetry).toBe(false); + expect(state?.state).toBe("exhausted_needs_attention"); + }); + + it("ignores recovery actions of other kinds and causes", () => { + const states = project({ + recoveryActions: [ + action({ kind: "stranded_assigned_issue" }), + action({ id: "action-2", cause: "other_cause" }), + ], + }); + expect(states.size).toBe(0); + }); + + it("keeps the highest generation when several replays share a fingerprint", () => { + const state = project({ + pendingReplays: [ + replay({ generation: 0, runId: "run-old" }), + replay({ generation: 1, runId: "run-new" }), + ], + }).get("comment-1"); + expect(state).toMatchObject({ attempt: 1, targetRunId: "run-new" }); + }); + + it("ignores replays that carry no fingerprint", () => { + expect(project({ pendingReplays: [replay({ fingerprint: null })] }).size).toBe(0); + }); +}); + +describe("toSafeFeedbackDeliveryFailureReason", () => { + it("maps known codes to plain operator language", () => { + expect(toSafeFeedbackDeliveryFailureReason("agent_not_invokable")).toBe( + "The assigned agent can't be started right now.", + ); + expect(toSafeFeedbackDeliveryFailureReason(" budget_blocked ")).toBe( + "The agent is over its budget.", + ); + }); + + it("collapses anything unrecognized rather than echoing it", () => { + expect(toSafeFeedbackDeliveryFailureReason("Error: connect ECONNREFUSED 127.0.0.1:5432")).toBeNull(); + expect(toSafeFeedbackDeliveryFailureReason("")).toBeNull(); + expect(toSafeFeedbackDeliveryFailureReason(null)).toBeNull(); + expect(toSafeFeedbackDeliveryFailureReason(undefined)).toBeNull(); + }); + + it("uses no transport or harness jargon in operator copy", () => { + const jargon = ["wake", "adapter", "heartbeat", "pid", "stack"]; + for (const code of ["process_lost", "server_shutdown_interrupted", "agent_not_invokable"]) { + const copy = toSafeFeedbackDeliveryFailureReason(code)?.toLowerCase() ?? ""; + for (const term of jargon) expect(copy).not.toContain(term); + } + }); +}); diff --git a/server/src/services/recovery/feedback-delivery-projection.ts b/server/src/services/recovery/feedback-delivery-projection.ts new file mode 100644 index 00000000000..8a32568bb12 --- /dev/null +++ b/server/src/services/recovery/feedback-delivery-projection.ts @@ -0,0 +1,258 @@ +import type { IssueCommentFeedbackDelivery, IssueRecoveryActionStatus } from "@paperclipai/shared"; + +import { + FEEDBACK_DELIVERY_MAX_AUTOMATIC_RETRIES, + FEEDBACK_DELIVERY_RECOVERY_CAUSE, +} from "./feedback-delivery.js"; + +/** + * How long a recovered-after-attention notice keeps its own projected state. + * After this the comment falls back to no delivery state at all; the audit + * trail lives in activity history, not in a permanently decorated comment. + */ +export const FEEDBACK_DELIVERY_RECOVERED_WINDOW_MS = 24 * 60 * 60 * 1000; + +/** + * Operator-safe failure vocabulary. Run/gate failure codes are mapped through + * this whitelist so no adapter message, stack frame, request body, credential, + * or filesystem path can reach a comment surface. An unrecognized code is + * deliberately collapsed to `null` rather than echoed: the banner copy already + * states the operator-relevant fact ("Paperclip stopped retrying"). + */ +const SAFE_FAILURE_REASONS: Record = { + agent_not_found: "The assigned agent no longer exists.", + agent_not_invokable: "The assigned agent can't be started right now.", + budget_blocked: "The agent is over its budget.", + issue_paused: "Work on this task is paused.", + issue_dependencies_blocked: "This task is waiting on a blocker.", + issue_assignee_changed: "The assignee changed while delivering.", + issue_reassigned: "The assignee changed while delivering.", + issue_execution_lock_changed: "Another run took over this task.", + issue_review_participant_changed: "The review owner changed while delivering.", + issue_terminal_status: "The task was closed while delivering.", + process_lost: "The agent's process stopped before it read the comment.", + server_shutdown_interrupted: "Paperclip restarted before the agent read the comment.", + workspace_validation_failed: "The task's workspace couldn't be prepared.", + configuration_validation_failed: "Required agent configuration is missing.", +}; + +export function toSafeFeedbackDeliveryFailureReason( + code: string | null | undefined, +): string | null { + if (typeof code !== "string") return null; + const normalized = code.trim(); + if (!normalized) return null; + return SAFE_FAILURE_REASONS[normalized] ?? null; +} + +/** Minimal recovery-action shape this projection needs. */ +export type FeedbackDeliveryRecoveryActionRow = { + id: string; + kind: string; + cause: string; + /** Raw DB column: narrowed against the active-status list rather than cast. */ + status: string; + fingerprint: string; + evidence: unknown; + attemptCount: number; + maxAttempts: number | null; + lastAttemptAt: Date | string | null; + resolvedAt: Date | string | null; + createdAt: Date | string; +}; + +/** + * A replay this issue is currently waiting on: either a queued wake or a + * queued/scheduled run that carries feedback-delivery context. + */ +export type FeedbackDeliveryPendingReplay = { + fingerprint: string | null; + commentIds: readonly string[]; + generation: number; + runId: string | null; + nextAttemptAt: Date | string | null; + requestedAt: Date | string | null; +}; + +export type FeedbackDeliveryProjectionInput = { + recoveryActions: readonly FeedbackDeliveryRecoveryActionRow[]; + pendingReplays: readonly FeedbackDeliveryPendingReplay[]; + /** Whether this viewer may request one more explicit delivery attempt. */ + viewerCanRetry: boolean; + /** Whether this viewer may open run detail. Gates `targetRunId` entirely. */ + viewerCanReadRuns: boolean; + now: Date; + recoveredWindowMs?: number; +}; + +const ACTIVE_STATUSES: readonly string[] = [ + "active", + "escalated", +] satisfies readonly IssueRecoveryActionStatus[]; + +function toIso(value: Date | string | null | undefined): string | null { + if (!value) return null; + const date = value instanceof Date ? value : new Date(value); + return Number.isNaN(date.getTime()) ? null : date.toISOString(); +} + +function toMillis(value: Date | string | null | undefined): number | null { + if (!value) return null; + const date = value instanceof Date ? value : new Date(value); + return Number.isNaN(date.getTime()) ? null : date.getTime(); +} + +function readEvidenceCommentIds(evidence: unknown): string[] { + if (typeof evidence !== "object" || evidence === null) return []; + const raw = (evidence as Record).outstandingCommentIds; + if (!Array.isArray(raw)) return []; + const ids: string[] = []; + for (const entry of raw) { + if (typeof entry !== "string") continue; + const trimmed = entry.trim(); + if (trimmed && !ids.includes(trimmed)) ids.push(trimmed); + } + return ids; +} + +function readEvidenceString(evidence: unknown, key: string): string | null { + if (typeof evidence !== "object" || evidence === null) return null; + const raw = (evidence as Record)[key]; + return typeof raw === "string" && raw.trim().length > 0 ? raw.trim() : null; +} + +function readEvidenceNumber(evidence: unknown, key: string): number | null { + if (typeof evidence !== "object" || evidence === null) return null; + const raw = (evidence as Record)[key]; + return typeof raw === "number" && Number.isFinite(raw) ? raw : null; +} + +function isFeedbackDeliveryAction(action: FeedbackDeliveryRecoveryActionRow) { + return action.kind === "feedback_delivery" && action.cause === FEEDBACK_DELIVERY_RECOVERY_CAUSE; +} + +/** + * Projects feedback-delivery state onto the individual human comments it + * belongs to. + * + * Precedence is deliberate: + * 1. An active/escalated recovery action always wins, because the operator must + * keep seeing the actionable banner until the obligation is discharged — an + * in-flight explicit retry surfaces as `retryInFlight`, not as a state change + * that would make the banner (and its attention item) disappear. + * 2. Otherwise a queued replay renders the quiet `retrying` footer line. + * 3. Otherwise a recently resolved action renders the historical recovered + * notice, so attention disappearing is explained rather than silent. + * + * A newer feedback comment never clears an older obligation: each comment id is + * keyed independently off the evidence of the action that owns it. + */ +export function projectFeedbackDeliveryByComment( + input: FeedbackDeliveryProjectionInput, +): Map { + const byComment = new Map(); + const recoveredWindowMs = input.recoveredWindowMs ?? FEEDBACK_DELIVERY_RECOVERED_WINDOW_MS; + const nowMs = input.now.getTime(); + + const actions = input.recoveryActions.filter(isFeedbackDeliveryAction); + const activeActions = actions.filter((action) => ACTIVE_STATUSES.includes(action.status)); + const activeFingerprints = new Set(activeActions.map((action) => action.fingerprint)); + + // Recovered notices come first so an active action or queued replay for the + // same comment overwrites them, never the other way round. + const resolvedActions = actions + .filter((action) => !ACTIVE_STATUSES.includes(action.status)) + // A fingerprint that resolved and then re-opened is owned by the open one. + .filter((action) => !activeFingerprints.has(action.fingerprint)) + .filter((action) => { + const resolvedMs = toMillis(action.resolvedAt); + return resolvedMs !== null && nowMs - resolvedMs <= recoveredWindowMs; + }) + // Oldest first: the freshest resolution wins a shared comment. + .sort((a, b) => (toMillis(a.resolvedAt) ?? 0) - (toMillis(b.resolvedAt) ?? 0)); + + for (const action of resolvedActions) { + const commentIds = readEvidenceCommentIds(action.evidence); + const targetRunId = input.viewerCanReadRuns + ? readEvidenceString(action.evidence, "sourceRunId") + : null; + for (const commentId of commentIds) { + byComment.set(commentId, { + state: "recovered_after_attention", + sourceCommentId: commentId, + attempt: readEvidenceNumber(action.evidence, "retryGeneration"), + maxAttempts: action.maxAttempts ?? FEEDBACK_DELIVERY_MAX_AUTOMATIC_RETRIES, + nextAttemptAt: null, + lastAttemptAt: toIso(action.lastAttemptAt), + deliveredAt: toIso(action.resolvedAt), + targetRunId, + canRetry: false, + retryInFlight: false, + safeFailureReason: null, + batchedCommentCount: commentIds.length, + }); + } + } + + const replaysByFingerprint = new Map(); + for (const replay of input.pendingReplays) { + if (!replay.fingerprint) continue; + const existing = replaysByFingerprint.get(replay.fingerprint); + if (!existing || replay.generation > existing.generation) { + replaysByFingerprint.set(replay.fingerprint, replay); + } + } + + for (const replay of replaysByFingerprint.values()) { + if (activeFingerprints.has(replay.fingerprint as string)) continue; + for (const commentId of replay.commentIds) { + byComment.set(commentId, { + state: "retrying", + sourceCommentId: commentId, + attempt: replay.generation, + maxAttempts: FEEDBACK_DELIVERY_MAX_AUTOMATIC_RETRIES, + nextAttemptAt: toIso(replay.nextAttemptAt), + lastAttemptAt: toIso(replay.requestedAt), + deliveredAt: null, + targetRunId: input.viewerCanReadRuns ? replay.runId : null, + canRetry: false, + retryInFlight: true, + safeFailureReason: null, + batchedCommentCount: replay.commentIds.length, + }); + } + } + + for (const action of activeActions) { + const commentIds = readEvidenceCommentIds(action.evidence); + const replay = replaysByFingerprint.get(action.fingerprint) ?? null; + const retryInFlight = replay !== null; + const targetRunId = input.viewerCanReadRuns + ? replay?.runId ?? readEvidenceString(action.evidence, "sourceRunId") + : null; + const safeFailureReason = + toSafeFeedbackDeliveryFailureReason(readEvidenceString(action.evidence, "lastFailureCode")) + ?? toSafeFeedbackDeliveryFailureReason(readEvidenceString(action.evidence, "gateErrorCode")); + for (const commentId of commentIds) { + byComment.set(commentId, { + state: "exhausted_needs_attention", + sourceCommentId: commentId, + attempt: readEvidenceNumber(action.evidence, "retryGeneration"), + maxAttempts: + readEvidenceNumber(action.evidence, "maxAutomaticRetries") + ?? action.maxAttempts + ?? FEEDBACK_DELIVERY_MAX_AUTOMATIC_RETRIES, + nextAttemptAt: toIso(replay?.nextAttemptAt ?? null), + lastAttemptAt: toIso(action.lastAttemptAt ?? action.createdAt), + deliveredAt: null, + targetRunId, + canRetry: input.viewerCanRetry && !retryInFlight, + retryInFlight, + safeFailureReason, + batchedCommentCount: commentIds.length, + }); + } + } + + return byComment; +} diff --git a/server/src/services/recovery/feedback-delivery.ts b/server/src/services/recovery/feedback-delivery.ts index e62c0fdd41e..cbb75aefa46 100644 --- a/server/src/services/recovery/feedback-delivery.ts +++ b/server/src/services/recovery/feedback-delivery.ts @@ -63,6 +63,20 @@ export function buildFeedbackDeliveryRetryIdempotencyKey(input: { return `${input.fingerprint}:generation:${input.generation}`; } +/** + * Explicit operator "Retry delivery" replays get their own key namespace: the + * automatic `:generation:N` keys are already spent by the time delivery is + * exhausted, so reusing them would silently swallow the operator's request. + * Keying on the recovery action's attempt count still collapses a double-click + * into one replay while allowing a fresh attempt after a failure. + */ +export function buildFeedbackDeliveryManualRetryIdempotencyKey(input: { + fingerprint: string; + attempt: number; +}) { + return `${input.fingerprint}:manual:${input.attempt}`; +} + export function readFeedbackDeliveryWakeCommentIds( contextSnapshot: Record | null | undefined, ): string[] {