feat(server): emit recovery/stranded escalation comments as structured system notices

Migrate the run-recovery/watchdog escalation comments onto the structured
system-notice mechanism the successful-run-handoff notices already use.

- New recovery/stranded-notice.ts: per-family notice seeds (short body +
  title + tone) for the no-live-execution-path / workspace-validation /
  configuration-incomplete / execution-review-participant families, and
  buildStrandedRecoveryEscalationNotice which assembles presentation
  {kind: system_notice} plus metadata.sections (Recovery action key_value,
  Recovery owner agent_link or board-escalation key_value, Next action,
  Source run run_link, Failure code key_value).
- escalateStrandedAssignedIssue posts the structured notice instead of
  appending the plain Recovery action/owner/Next action footer; plain
  comment callers keep working via fallbackBody + cause-derived titles.
- Escalation dedupe still checks the legacy body marker for old comments
  and relies on noticeMetadataReferencesRecoveryAction for new ones.
- heartbeat.ts builders replaced by the shared seeds; inline failure
  fragments dropped from bodies (failure code now lives in metadata; raw
  error text stays withheld from the thread).
- Shared row helpers extracted to recovery/notice-format.ts.
- Tests: unit coverage for every family seed + escalation notice shape +
  metadata dedupe matcher; embedded-Postgres dedupe regressions updated to
  assert the structured shape.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
scotttongandClaude Fable 5 committed 2026-08-07 16:58:40 -07:00
1 parent e907db5926
commit eac5d3ee10
10 files changed
+506 -275

No files matched your search

@@ -116,6 +116,7 @@ import {
SUCCESSFUL_RUN_HANDOFF_EXHAUSTED_NOTICE_BODY,
SUCCESSFUL_RUN_HANDOFF_REQUIRED_NOTICE_BODY,
SUCCESSFUL_RUN_MISSING_STATE_REASON,
noticeMetadataReferencesRecoveryAction,
} from "../services/recovery/index.ts";
import {
UNMANAGED_BACKGROUND_TASK_LIVENESS_REASON,
@@ -130,6 +131,11 @@ if (!embeddedPostgresSupport.supported) {
);
}
function commentMetadataRows(comment: { metadata?: unknown } | null | undefined) {
const metadata = comment?.metadata as { sections?: Array<{ rows?: unknown[] }> } | null | undefined;
return (metadata?.sections ?? []).flatMap((section) => section.rows ?? []) as Array<Record<string, unknown>>;
}
function spawnAliveProcess() {
return spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], {
stdio: "ignore",
@@ -2489,8 +2495,11 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
});
expect(comments).toHaveLength(1);
expect(comments[0]?.body).toContain("retried continuation");
expect(comments[0]?.body).toContain(`Recovery action: \`${recoveryAction.id}\``);
expect(comments[0]?.body).toContain("Recovery owner: [CodexCoder]");
expect(comments[0]?.presentation).toMatchObject({ kind: "system_notice", tone: "danger" });
expect(noticeMetadataReferencesRecoveryAction(comments[0]?.metadata, recoveryAction.id)).toBe(true);
expect(commentMetadataRows(comments[0]).some((row) =>
row.type === "agent_link" && row.label === "Recovery owner" && row.name === "CodexCoder",
)).toBe(true);
});
it("blocks failed recovery work in place during immediate terminal-run cleanup", async () => {
@@ -2944,7 +2953,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
authorType: "system",
body: expect.stringContaining("Agent failed to resume after approval: `adapter_failed` — needs attention"),
});
expect(comments[0]?.body).toContain("Recovery action:");
expect(commentMetadataRows(comments[0]).some((row) => row.label === "Recovery action")).toBe(true);
const interaction = await db
.select({ result: issueThreadInteractions.result })
@@ -4697,7 +4706,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId));
const recoveryComment = comments.find((comment) =>
comment.body.includes("pending execution-review participant once") &&
comment.body.includes(`Recovery action: \`${recoveryAction.id}\``),
noticeMetadataReferencesRecoveryAction(comment.metadata, recoveryAction.id),
);
expect(recoveryComment).toBeTruthy();
@@ -5511,25 +5520,12 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId));
expect(comments).toHaveLength(1);
expect(comments[0]?.body).toContain("retried dispatch");
expect(comments[0]?.body).toContain("Latest retry failure details were withheld from the issue thread");
expect(comments[0]?.body).toContain(`Recovery action: \`${recoveryAction.id}\``);
expect(comments[0]?.body).toContain(`Recovery owner: [${longRecoveryOwnerName}]`);
expect(comments[0]?.presentation).toMatchObject({
kind: "system_notice",
tone: "warning",
title: `${`Recovery: retries exhausted — moved to blocked (owner: ${longRecoveryOwnerName})`.slice(0, 159)}…`,
density: "compact",
});
expect(comments[0]?.metadata).toMatchObject({
version: 1,
sections: [expect.objectContaining({
rows: expect.arrayContaining([
expect.objectContaining({ type: "key_value", label: "Recovery action", value: recoveryAction.id }),
expect.objectContaining({ type: "key_value", label: "Cause", value: "process_lost" }),
expect.objectContaining({ type: "agent_link", label: "Recovery owner", name: "R".repeat(160) }),
]),
})],
});
expect(comments[0]?.body).not.toContain("sk-test-recovery-secret");
expect(comments[0]?.presentation).toMatchObject({ kind: "system_notice", tone: "danger" });
expect(noticeMetadataReferencesRecoveryAction(comments[0]?.metadata, recoveryAction.id)).toBe(true);
expect(commentMetadataRows(comments[0]).some((row) =>
row.type === "agent_link" && row.label === "Recovery owner" && row.name === longRecoveryOwnerName,
)).toBe(true);
});
it("blocks an already stranded recovery issue without creating a recovery child", async () => {
@@ -5933,9 +5929,11 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId));
expect(comments).toHaveLength(1);
expect(comments[0]?.body).toContain("retried continuation");
expect(comments[0]?.body).toContain("Latest retry failure details were withheld from the issue thread");
expect(comments[0]?.body).toContain(`Recovery action: \`${recoveryAction.id}\``);
expect(comments[0]?.body).toContain("Recovery owner: [CodexCoder]");
expect(comments[0]?.presentation).toMatchObject({ kind: "system_notice", tone: "danger" });
expect(noticeMetadataReferencesRecoveryAction(comments[0]?.metadata, recoveryAction.id)).toBe(true);
expect(commentMetadataRows(comments[0]).some((row) =>
row.type === "agent_link" && row.label === "Recovery owner" && row.name === "CodexCoder",
)).toBe(true);
});
it("redacts error-code-only stranded recovery failures in issue copy", async () => {
@@ -5966,8 +5964,15 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId));
expect(comments).toHaveLength(1);
expect(comments[0]?.body).toContain("Latest retry failure details were withheld from the issue thread");
// The short structured body carries no failure details; the normalized
// failure code surfaces only as a metadata row.
expect(comments[0]?.body).not.toContain("adapter_exit_code");
expect(comments[0]?.body).not.toContain("- Failure: none recorded");
expect(commentMetadataRows(comments[0])).toContainEqual({
type: "key_value",
label: "Failure code",
value: "adapter_exit_code",
});
});
it("keeps retrying transient adapter_failed continuation runs before the cap", async () => {
@@ -6060,7 +6065,11 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
expect(comments).toHaveLength(1);
expect(comments[0]?.body).toContain("retried continuation");
expect(comments[0]?.body).toContain("3× attempts");
expect(comments[0]?.body).toContain("Latest cause: `adapter_failed`");
expect(commentMetadataRows(comments[0])).toContainEqual({
type: "key_value",
label: "Failure code",
value: "adapter_failed",
});
});
it("does not count mixed-cause continuation failures toward the transient cap", async () => {
@@ -6687,8 +6696,10 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
expect(comments).toHaveLength(1);
expect(comments[0]?.body).toContain("automatically retried continuation");
expect(comments[0]?.body).toContain("still has no live execution path");
expect(comments[0]?.body).toContain(`Recovery action: \`${recoveryAction.id}\``);
expect(comments[0]?.body).toContain("Recovery owner: [CodexCoder]");
expect(noticeMetadataReferencesRecoveryAction(comments[0]?.metadata, recoveryAction.id)).toBe(true);
expect(commentMetadataRows(comments[0]).some((row) =>
row.type === "agent_link" && row.label === "Recovery owner" && row.name === "CodexCoder",
)).toBe(true);
});
it("allows one productive-terminal recovery after regular continuation recovery made progress", async () => {
@@ -37,6 +37,7 @@ import {
} from "./helpers/embedded-postgres.js";
import { drainHeartbeatRunsToQuiescence } from "./helpers/drain-heartbeat-runs.js";
import { heartbeatService } from "../services/heartbeat.ts";
import { noticeMetadataReferencesRecoveryAction } from "../services/recovery/index.ts";
import { instanceSettingsService } from "../services/instance-settings.ts";
import {
WORKSPACE_WORKTREE_REQUIRES_PROJECT_CODE,
@@ -221,7 +222,7 @@ async function waitForContainmentSideEffects(input: {
const hasRecoveryActionComment = recoveryActionId
? comments.some((comment) =>
comment.issueId === input.sourceIssueId &&
comment.body.includes(`Recovery action: \`${recoveryActionId}\``))
noticeMetadataReferencesRecoveryAction(comment.metadata, recoveryActionId))
: false;
if (
source?.status === "blocked" &&
@@ -685,7 +686,10 @@ async function expectContainedWorkspaceBranchFailure(input: {
}),
});
expect(comments.filter((comment) => comment.issueId === input.sourceIssueId && comment.body.includes(`Recovery action: \`${action.id}\``))).toHaveLength(1);
expect(comments.filter((comment) =>
comment.issueId === input.sourceIssueId &&
noticeMetadataReferencesRecoveryAction(comment.metadata, action.id),
)).toHaveLength(1);
expect(comments.filter((comment) => comment.issueId === input.sameWorkspaceSiblingId)).toHaveLength(0);
expect(comments.filter((comment) => comment.issueId === input.otherWorkspaceSiblingId)).toHaveLength(0);
}
@@ -812,14 +816,10 @@ async function expectForwardBranchReconciled(input: {
]),
);
if (resolvedRecoveryActionId) {
expect(comments).toEqual(
expect.arrayContaining([
expect.objectContaining({
authorType: "system",
body: expect.stringContaining(`Recovery action: \`${resolvedRecoveryActionId}\``),
}),
]),
);
expect(comments.some((comment) =>
comment.authorType === "system" &&
noticeMetadataReferencesRecoveryAction(comment.metadata, resolvedRecoveryActionId),
)).toBe(true);
}
const activities = await input.db
@@ -27,6 +27,7 @@ import { issueRoutes } from "../routes/issues.js";
import { buildPaperclipWakePayload } from "../services/heartbeat.js";
import { issueRecoveryActionService } from "../services/issue-recovery-actions.js";
import { recoveryService } from "../services/recovery/service.js";
import { noticeMetadataReferencesRecoveryAction } from "../services/recovery/successful-run-handoff.js";
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
@@ -1143,21 +1144,14 @@ describeEmbeddedPostgres("issue recovery actions", () => {
});
const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, sourceIssue.id));
expect(comments).toHaveLength(1);
expect(comments[0]?.presentation).toMatchObject({
const escalationComments = comments.filter((comment) =>
noticeMetadataReferencesRecoveryAction(comment.metadata, actionRows[0]!.id),
);
expect(escalationComments).toHaveLength(1);
expect(escalationComments[0]?.presentation).toMatchObject({
kind: "system_notice",
tone: "warning",
title: "Recovery: workspace validation failed — moved to blocked (owner: CTO)",
density: "compact",
});
expect(comments[0]?.metadata).toMatchObject({
version: 1,
sections: [expect.objectContaining({
rows: expect.arrayContaining([
expect.objectContaining({ type: "key_value", label: "Recovery action", value: actionRows[0]?.id }),
expect.objectContaining({ type: "key_value", label: "Cause", value: "workspace_validation_failed" }),
]),
})],
tone: "danger",
title: "Workspace validation failed",
});
expect(enqueueWakeup).toHaveBeenCalledTimes(2);
expect(enqueueWakeup).toHaveBeenCalledWith(
@@ -1231,7 +1225,11 @@ describeEmbeddedPostgres("issue recovery actions", () => {
const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, sourceIssue.id));
expect(comments).toHaveLength(1);
expect(comments[0]?.body).toContain("Recovery action:");
// Dedupe for structured notices is metadata-based: the short body no longer
// carries the `Recovery action: \`id\`` marker line.
expect(comments[0]?.body).not.toContain("Recovery action:");
expect(noticeMetadataReferencesRecoveryAction(comments[0]?.metadata, actionRows[0]!.id)).toBe(true);
expect(comments[0]?.presentation).toMatchObject({ kind: "system_notice", tone: "danger" });
});
it("does not create nested recovery artifacts when issue-backed fallback work itself fails", async () => {
+16 -102
View File
@@ -224,6 +224,12 @@ import {
readContinuationAttempt,
} from "./recovery/index.js";
import { isAutomaticRecoverySuppressedByPauseHold } from "./recovery/pause-hold-guard.js";
import {
buildConfigurationIncompleteRecoveryNoticeSeed,
buildExecutionReviewParticipantRecoveryNoticeSeed,
buildImmediateExecutionPathRecoveryNoticeSeed,
buildWorkspaceValidationRecoveryNoticeSeed,
} from "./recovery/stranded-notice.js";
import {
recoveryAssigneeAdapterOverrides,
withRecoveryModelProfileHint,
@@ -3616,30 +3622,6 @@ export function summarizeHeartbeatRunListResultJson(input: {
return Object.keys(summary).length > 0 ? summary : null;
}
function summarizeRunFailureForIssueComment(
run: Pick<typeof heartbeatRuns.$inferSelect, "error" | "errorCode"> | null | undefined,
) {
if (!run) return null;
const errorCode = readNonEmptyString(run.errorCode)?.trim() ?? null;
const rawError = readNonEmptyString(run.error)?.trim() ?? null;
const apiMessageMatch = rawError?.match(/"message"\s*:\s*"([^"]+)"/);
const firstLine = rawError
?.split(/\r?\n/)
.map((line) => line.trim())
.find(Boolean) ?? null;
const summarySource = apiMessageMatch?.[1] ?? firstLine;
const summary =
summarySource && summarySource.length > 240
? `${summarySource.slice(0, 237)}...`
: summarySource;
if (errorCode && summary) return ` Latest retry failure: \`${errorCode}\` - ${summary}.`;
if (errorCode) return ` Latest retry failure: \`${errorCode}\`.`;
if (summary) return ` Latest retry failure: ${summary}.`;
return null;
}
function didAutomaticRecoveryFail(
latestRun: Pick<typeof heartbeatRuns.$inferSelect, "status" | "contextSnapshot"> | null,
expectedRetryReason:
@@ -16379,73 +16361,6 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
}
}
function buildImmediateExecutionPathRecoveryComment(input: {
status: "todo" | "in_progress";
latestRun: Pick<typeof heartbeatRuns.$inferSelect, "error" | "errorCode"> | null | undefined;
}) {
const failureSummary = summarizeRunFailureForIssueComment(input.latestRun);
if (input.status === "todo") {
return (
"Paperclip automatically retried dispatch for this assigned `todo` issue during terminal run recovery, " +
`but it still has no live execution path.${failureSummary ?? ""} ` +
"Moving it to `blocked` so it is visible for intervention."
);
}
return (
"Paperclip automatically retried continuation for this assigned `in_progress` issue during terminal run " +
`recovery, but it still has no live execution path.${failureSummary ?? ""} ` +
"Moving it to `blocked` so it is visible for intervention."
);
}
function buildWorkspaceValidationRecoveryComment(input: {
latestRun:
| Pick<typeof heartbeatRuns.$inferSelect, "error" | "errorCode" | "resultJson">
| null
| undefined;
}) {
const failureSummary = summarizeRunFailureForIssueComment(input.latestRun);
const validationReason = readNonEmptyString(
readWorkspaceValidationPayloadFromRun(input.latestRun).reason,
);
if (validationReason === "git_worktree_base_materialization_failed") {
return (
"Paperclip stopped before launching the local adapter because the project workspace checkout could not be prepared " +
`(for example the repository clone failed).${failureSummary ?? ""} ` +
"Moving it to `blocked` with a source-scoped recovery action so the repository URL, clone access, or configured local cwd can be repaired before resuming."
);
}
return (
"Paperclip stopped before launching the local adapter because the issue workspace failed validation. " +
`This prevents git-sensitive adapters from running in an unrelated fallback cwd.${failureSummary ?? ""} ` +
"Moving it to `blocked` with a source-scoped recovery action so the workspace link, cwd, or git checkout can be repaired before resuming."
);
}
function buildConfigurationIncompleteRecoveryComment(input: {
latestRun: Pick<typeof heartbeatRuns.$inferSelect, "error" | "errorCode"> | null | undefined;
}) {
const failureSummary = summarizeRunFailureForIssueComment(input.latestRun);
return (
"Paperclip stopped before dispatching the adapter because required secret/env bindings are missing. " +
`Resolving them as a runtime failure would only produce repeated opaque setup failures.${failureSummary ?? ""} ` +
"Moving it to `blocked` with a source-scoped recovery action so an operator can bind the missing secret(s) before resuming."
);
}
function buildExecutionReviewParticipantRecoveryComment(input: {
latestRun: Pick<typeof heartbeatRuns.$inferSelect, "error" | "errorCode"> | null | undefined;
}) {
const failureSummary = summarizeRunFailureForIssueComment(input.latestRun);
return (
"Paperclip retried the pending execution-review participant once, but the review stage still has no completed decision " +
`or live reviewer run.${failureSummary ?? ""} ` +
"Moving it to `blocked` with a source-scoped recovery action so the recovery owner can repair the reviewer runtime, " +
"restore the review stage, or record an intentional manual resolution."
);
}
async function releaseIssueExecutionAndPromote(
run: typeof heartbeatRuns.$inferSelect,
options: { suppressImmediateRecovery?: boolean } = {},
@@ -16579,9 +16494,9 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
kind: "blocked" as const,
issue,
previousStatus: issue.status,
comment: configurationIncomplete
? buildConfigurationIncompleteRecoveryComment({ latestRun: run })
: buildWorkspaceValidationRecoveryComment({ latestRun: run }),
notice: configurationIncomplete
? buildConfigurationIncompleteRecoveryNoticeSeed()
: buildWorkspaceValidationRecoveryNoticeSeed(),
recoveryCause: configurationIncomplete
? CONFIGURATION_INCOMPLETE_RECOVERY_CAUSE
: WORKSPACE_VALIDATION_RECOVERY_CAUSE,
@@ -16925,7 +16840,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
kind: "blocked" as const,
issue,
previousStatus: issue.status,
comment: buildExecutionReviewParticipantRecoveryComment({ latestRun: run }),
notice: buildExecutionReviewParticipantRecoveryNoticeSeed(),
recoveryCause: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_CAUSE,
recoveryOwnerAgentId: currentParticipant.agentId,
};
@@ -17046,19 +16961,18 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
if (shouldBlockImmediately) {
const workspaceValidationFailure = isWorkspaceValidationFailedRun(run);
const configurationIncompleteFailure = isConfigurationIncompleteFailedRun(run);
const comment = workspaceValidationFailure
? buildWorkspaceValidationRecoveryComment({ latestRun: run })
const notice = workspaceValidationFailure
? buildWorkspaceValidationRecoveryNoticeSeed()
: configurationIncompleteFailure
? buildConfigurationIncompleteRecoveryComment({ latestRun: run })
: buildImmediateExecutionPathRecoveryComment({
? buildConfigurationIncompleteRecoveryNoticeSeed()
: buildImmediateExecutionPathRecoveryNoticeSeed({
status: issue.status as "todo" | "in_progress",
latestRun: run,
});
return {
kind: "blocked" as const,
issue,
previousStatus: issue.status,
comment,
notice,
recoveryCause: workspaceValidationFailure
? WORKSPACE_VALIDATION_RECOVERY_CAUSE
: configurationIncompleteFailure
@@ -17168,7 +17082,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
issue: promotionResult.issue,
previousStatus: promotionResult.previousStatus as "todo" | "in_progress" | "in_review",
latestRun: run,
comment: promotionResult.comment,
notice: promotionResult.notice,
recoveryCause:
promotionResult.recoveryCause === WORKSPACE_VALIDATION_RECOVERY_CAUSE
? WORKSPACE_VALIDATION_RECOVERY_CAUSE
+14
View File
@@ -58,8 +58,22 @@ export {
findExistingFinishSuccessfulRunHandoffWake,
isSuccessfulRunHandoffValidPathSkip,
isSuccessfulRunHandoffRequiredNoticeBody,
noticeMetadataReferencesRecoveryAction,
} from "./successful-run-handoff.js";
export type {
SuccessfulRunHandoffNotice,
SuccessfulRunHandoffDecision,
} from "./successful-run-handoff.js";
export {
DEFAULT_STRANDED_RECOVERY_NOTICE_BODY,
buildConfigurationIncompleteRecoveryNoticeSeed,
buildExecutionReviewParticipantRecoveryNoticeSeed,
buildExecutionReviewParticipantUnavailableNoticeSeed,
buildImmediateExecutionPathRecoveryNoticeSeed,
buildStrandedRecoveryEscalationNotice,
buildWorkspaceValidationRecoveryNoticeSeed,
} from "./stranded-notice.js";
export type {
StrandedRecoveryEscalationNotice,
StrandedRecoveryNoticeSeed,
} from "./stranded-notice.js";
@@ -0,0 +1,56 @@
import type { IssueCommentMetadata, IssueCommentPresentation } from "@paperclipai/shared";
export type NoticeMetadataRow = IssueCommentMetadata["sections"][number]["rows"][number];
export type NoticeMetadataSection = IssueCommentMetadata["sections"][number];
export function metadataText(value: unknown, fallback = "unknown") {
const text = typeof value === "string" ? value.trim() : value == null ? "" : String(value).trim();
const resolved = text.length > 0 ? text : fallback;
return resolved.length > 2000 ? `${resolved.slice(0, 1997)}...` : resolved;
}
export function keyValueRow(label: string, value: unknown): NoticeMetadataRow {
return { type: "key_value", label, value: metadataText(value) };
}
export function issueLinkRow(
label: string,
issue: { id: string; identifier: string | null; title: string } | null | undefined,
): NoticeMetadataRow {
if (!issue) return keyValueRow(label, "unknown");
return {
type: "issue_link",
label,
issueId: issue.id,
identifier: issue.identifier,
title: issue.title,
};
}
export function runLinkRow(
label: string,
run: { id: string; status: string } | null | undefined,
): NoticeMetadataRow {
if (!run) return keyValueRow(label, "unknown");
return { type: "run_link", label, runId: run.id, title: run.status };
}
export function agentLinkRow(
label: string,
agent: { id: string; name: string | null } | null | undefined,
): NoticeMetadataRow {
if (!agent) return keyValueRow(label, "unknown");
return { type: "agent_link", label, agentId: agent.id, name: agent.name };
}
export function systemNoticePresentation(input: {
tone: IssueCommentPresentation["tone"];
title: string;
}): IssueCommentPresentation {
return {
kind: "system_notice",
tone: input.tone,
title: input.title,
detailsDefaultOpen: false,
};
}
+49 -66
View File
@@ -61,6 +61,12 @@ import {
noticeMetadataReferencesRecoveryAction,
type SuccessfulRunHandoffNotice,
} from "./successful-run-handoff.js";
import {
buildExecutionReviewParticipantRecoveryNoticeSeed,
buildExecutionReviewParticipantUnavailableNoticeSeed,
buildStrandedRecoveryEscalationNotice,
type StrandedRecoveryNoticeSeed,
} from "./stranded-notice.js";
import {
RECOVERY_ORIGIN_KINDS,
buildIssueGraphLivenessLeafKey,
@@ -312,25 +318,6 @@ function summarizeRunFailureForIssueComment(run: LatestIssueRun) {
return null;
}
function buildExecutionReviewParticipantRecoveryComment(latestRun: LatestIssueRun) {
const failureSummary = summarizeRunFailureForIssueComment(latestRun);
return (
"Paperclip retried the pending execution-review participant once, but the review stage still has no completed decision " +
`or live reviewer run.${failureSummary ?? ""} ` +
"Moving it to `blocked` with a source-scoped recovery action so the recovery owner can repair the reviewer runtime, " +
"restore the review stage, or record an intentional manual resolution."
);
}
function buildExecutionReviewParticipantUnavailableComment(latestRun: LatestIssueRun) {
const failureSummary = summarizeRunFailureForIssueComment(latestRun);
return (
"Paperclip cannot continue the pending execution-review participant because the participant is not invokable " +
`and the review stage has no completed decision or live reviewer run.${failureSummary ?? ""} ` +
"Moving it to `blocked` with a source-scoped recovery action so the recovery owner can repair the reviewer runtime, " +
"restore the review stage, or record an intentional manual resolution."
);
}
function didAutomaticRecoveryFail(
latestRun: LatestIssueRun,
@@ -3315,6 +3302,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
previousStatus: StrandedPreviousStatus;
latestRun: LatestIssueRun;
comment?: string;
notice?: StrandedRecoveryNoticeSeed | null;
recoveryCause?: StrandedRecoveryCause;
recoveryOwnerAgentId?: string | null;
successfulRunHandoffEvidence?: SuccessfulRunHandoffRecoveryEvidence | null;
@@ -3356,7 +3344,6 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
if (!updated) return null;
if (isProviderQuotaWait) return updated;
const prefix = await getCompanyIssuePrefix(input.issue.companyId);
const recoveryOwner = recoveryAction.ownerAgentId ? await getAgent(recoveryAction.ownerAgentId) : null;
const sourceAssignee = input.issue.assigneeAgentId ? await getAgent(input.issue.assigneeAgentId) : null;
let notice: SuccessfulRunHandoffNotice | null = null;
@@ -3376,19 +3363,18 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
missingDisposition: input.successfulRunHandoffEvidence.missingDisposition,
});
}
const recoveryLine = recoveryAction.ownerAgentId
? [
"",
`- Recovery action: \`${recoveryAction.id}\``,
`- Recovery owner: ${agentUiLink(recoveryOwner, prefix)}`,
"- Next action: the recovery owner should either restore a live execution path or record the manual resolution on the source issue.",
].join("\n")
: [
"",
`- Recovery action: \`${recoveryAction.id}\``,
"- Recovery owner: board escalation, because Paperclip could not find an invokable manager, creator, or executive owner with budget available.",
"- Next action: a board operator should assign an invokable recovery owner, fix the agent/runtime state, or record an intentional manual resolution.",
].join("\n");
const escalationNotice = buildStrandedRecoveryEscalationNotice({
seed: input.notice,
fallbackBody: input.comment,
recoveryCause,
recoveryActionId: recoveryAction.id,
recoveryOwner: recoveryAction.ownerAgentId && recoveryOwner
? { id: recoveryOwner.id, name: recoveryOwner.name }
: null,
sourceRun: input.latestRun
? { id: input.latestRun.id, status: input.latestRun.status, errorCode: input.latestRun.errorCode }
: null,
});
const shouldPostEscalationComment =
recoveryAction.attemptCount === 1 ||
@@ -3421,19 +3407,10 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
metadata: notice.metadata,
});
} else {
await issuesSvc.addComment(input.issue.id, `${input.comment ?? ""}${recoveryLine}`, {}, {
await issuesSvc.addComment(input.issue.id, escalationNotice.body, {}, {
authorType: "system",
presentation: compactRecoveryPresentation(
`Recovery: ${recoveryCauseTitle(recoveryCause)} — moved to blocked ` +
`(owner: ${recoveryOwner?.name ?? "board"})`,
),
metadata: recoveryNoticeMetadata({
cause: recoveryCause,
latestRun: input.latestRun,
recoveryActionId: recoveryAction.id,
previousStatus: input.previousStatus,
recoveryOwner,
}),
presentation: escalationNotice.presentation,
metadata: escalationNotice.metadata,
});
}
}
@@ -3907,7 +3884,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
issue,
previousStatus: "in_review",
latestRun: participantLatestRun,
comment: buildExecutionReviewParticipantUnavailableComment(participantLatestRun),
notice: buildExecutionReviewParticipantUnavailableNoticeSeed(),
recoveryCause: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_REASON,
recoveryOwnerAgentId: participantAgentId,
});
@@ -3974,7 +3951,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
issue,
previousStatus: "in_review",
latestRun: participantLatestRun,
comment: buildExecutionReviewParticipantUnavailableComment(participantLatestRun),
notice: buildExecutionReviewParticipantUnavailableNoticeSeed(),
recoveryCause: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_REASON,
recoveryOwnerAgentId: participantAgentId,
});
@@ -3992,7 +3969,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
issue,
previousStatus: "in_review",
latestRun: participantLatestRun,
comment: buildExecutionReviewParticipantRecoveryComment(participantLatestRun),
notice: buildExecutionReviewParticipantRecoveryNoticeSeed(),
recoveryCause: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_REASON,
recoveryOwnerAgentId: participantAgentId,
});
@@ -4069,15 +4046,18 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
}
if (didAutomaticRecoveryFail(latestRun, "assignment_recovery")) {
const failureSummary = summarizeRunFailureForIssueComment(latestRun);
const updated = await escalateStrandedAssignedIssue({
issue,
previousStatus: "todo",
latestRun,
comment:
"Paperclip automatically retried dispatch for this assigned `todo` issue after a lost wake/run, " +
`but it still has no live execution path.${failureSummary ?? ""} ` +
"Moving it to `blocked` so it is visible for intervention.",
notice: {
body:
"Paperclip automatically retried dispatch for this assigned `todo` issue after a lost wake/run, " +
"but it still has no live execution path. " +
"Moving it to `blocked` so it is visible for intervention.",
title: "No live execution path",
tone: "danger",
},
});
if (updated) {
result.escalated += 1;
@@ -4210,15 +4190,18 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
}
if (classification.kind === "non_retryable") {
const failureSummary = summarizeRunFailureForIssueComment(latestRun);
const updated = await escalateStrandedAssignedIssue({
issue,
previousStatus: "in_progress",
latestRun,
comment:
"Paperclip detected a non-retryable failure on this issue's continuation run " +
`(\`${classification.errorCode}\`). Skipping automatic retries and moving it to \`blocked\` ` +
`so it is visible for intervention.${failureSummary ?? ""}`,
notice: {
body:
"Paperclip detected a non-retryable failure on this issue's continuation run " +
`(\`${classification.errorCode}\`). Skipping automatic retries and moving it to \`blocked\` ` +
"so it is visible for intervention.",
title: "Continuation failed",
tone: "danger",
},
});
if (updated) {
result.escalated += 1;
@@ -4237,19 +4220,19 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
classification.errorCode,
);
if (consecutive >= classification.maxAttempts) {
const failureSummary = summarizeRunFailureForIssueComment(latestRun);
const attemptCopy = consecutive <= 1 ? "" : ` (${consecutive}× attempts)`;
const causeCopy = classification.errorCode
? ` Latest cause: \`${classification.errorCode}\`.`
: "";
const updated = await escalateStrandedAssignedIssue({
issue,
previousStatus: "in_progress",
latestRun,
comment:
"Paperclip automatically retried continuation for this assigned `in_progress` issue after its live " +
`execution disappeared, but it still has no live execution path${attemptCopy}.${causeCopy}${failureSummary ?? ""} ` +
"Moving it to `blocked` so it is visible for intervention.",
notice: {
body:
"Paperclip automatically retried continuation for this assigned `in_progress` issue after its live " +
`execution disappeared, but it still has no live execution path${attemptCopy}. ` +
"Moving it to `blocked` so it is visible for intervention.",
title: "No live execution path",
tone: "danger",
},
});
if (updated) {
result.escalated += 1;
@@ -0,0 +1,150 @@
import { describe, expect, it } from "vitest";
import { noticeMetadataReferencesRecoveryAction } from "./successful-run-handoff.js";
import {
DEFAULT_STRANDED_RECOVERY_NOTICE_BODY,
buildConfigurationIncompleteRecoveryNoticeSeed,
buildExecutionReviewParticipantRecoveryNoticeSeed,
buildExecutionReviewParticipantUnavailableNoticeSeed,
buildImmediateExecutionPathRecoveryNoticeSeed,
buildStrandedRecoveryEscalationNotice,
buildWorkspaceValidationRecoveryNoticeSeed,
} from "./stranded-notice.js";
function allRows(metadata: { sections: Array<{ rows: unknown[] }> }) {
return metadata.sections.flatMap((section) => section.rows) as Array<Record<string, unknown>>;
}
describe("stranded recovery notice seeds", () => {
it.each([
["todo dispatch", buildImmediateExecutionPathRecoveryNoticeSeed({ status: "todo" }), "No live execution path"],
["in_progress continuation", buildImmediateExecutionPathRecoveryNoticeSeed({ status: "in_progress" }), "No live execution path"],
["workspace validation", buildWorkspaceValidationRecoveryNoticeSeed(), "Workspace validation failed"],
["configuration incomplete", buildConfigurationIncompleteRecoveryNoticeSeed(), "Configuration incomplete"],
["review participant recovery", buildExecutionReviewParticipantRecoveryNoticeSeed(), "Review recovery stalled"],
["review participant unavailable", buildExecutionReviewParticipantUnavailableNoticeSeed(), "Review recovery stalled"],
])("%s seed has a short body plus title and tone", (_label, seed, expectedTitle) => {
expect(seed.title).toBe(expectedTitle);
expect(seed.tone).toBe("danger");
expect(seed.body.length).toBeGreaterThan(0);
// Failure details now live in metadata rows, never inline in the body.
expect(seed.body).not.toContain("Latest retry failure");
expect(seed.body).not.toContain("Recovery action:");
});
it("distinguishes todo dispatch from in_progress continuation copy", () => {
expect(buildImmediateExecutionPathRecoveryNoticeSeed({ status: "todo" }).body).toContain("retried dispatch");
expect(buildImmediateExecutionPathRecoveryNoticeSeed({ status: "in_progress" }).body).toContain(
"retried continuation",
);
});
});
describe("buildStrandedRecoveryEscalationNotice", () => {
const actionId = "6a2f8e64-6f5e-4b58-b7fd-111111111111";
const owner = { id: "9b1c2d3e-4f50-4a61-8b72-222222222222", name: "CTO" };
const sourceRun = {
id: "0c1d2e3f-4a5b-4c6d-8e7f-333333333333",
status: "failed",
errorCode: "workspace_validation_failed",
};
it("emits system_notice presentation and the required metadata rows", () => {
const seed = buildWorkspaceValidationRecoveryNoticeSeed();
const notice = buildStrandedRecoveryEscalationNotice({
seed,
recoveryActionId: actionId,
recoveryOwner: owner,
sourceRun,
});
expect(notice.body).toBe(seed.body);
expect(notice.presentation).toMatchObject({
kind: "system_notice",
tone: "danger",
title: "Workspace validation failed",
});
expect(notice.metadata.version).toBe(1);
expect(notice.metadata.sourceRunId).toBe(sourceRun.id);
const rows = allRows(notice.metadata);
expect(rows).toContainEqual({ type: "key_value", label: "Recovery action", value: actionId });
expect(rows).toContainEqual({ type: "agent_link", label: "Recovery owner", agentId: owner.id, name: owner.name });
expect(rows).toContainEqual({ type: "run_link", label: "Source run", runId: sourceRun.id, title: "failed" });
expect(rows).toContainEqual({
type: "key_value",
label: "Failure code",
value: "workspace_validation_failed",
});
expect(rows.some((row) => row.label === "Next action")).toBe(true);
});
it("records board escalation when there is no invokable recovery owner", () => {
const notice = buildStrandedRecoveryEscalationNotice({
seed: buildConfigurationIncompleteRecoveryNoticeSeed(),
recoveryActionId: actionId,
recoveryOwner: null,
sourceRun,
});
const ownerRow = allRows(notice.metadata).find((row) => row.label === "Recovery owner");
expect(ownerRow?.type).toBe("key_value");
expect(String(ownerRow?.value)).toContain("Board escalation");
});
it("derives title from the recovery cause and body from the plain-comment fallback", () => {
const notice = buildStrandedRecoveryEscalationNotice({
fallbackBody: "Legacy plain comment body.",
recoveryCause: "configuration_incomplete",
recoveryActionId: actionId,
recoveryOwner: owner,
sourceRun: null,
});
expect(notice.body).toBe("Legacy plain comment body.");
expect(notice.presentation.title).toBe("Configuration incomplete");
expect(notice.metadata.sourceRunId).toBeNull();
expect(notice.metadata.sections).toHaveLength(1);
});
it("falls back to the default body and title when no seed or comment is given", () => {
const notice = buildStrandedRecoveryEscalationNotice({
recoveryActionId: actionId,
recoveryOwner: null,
sourceRun: null,
});
expect(notice.body).toBe(DEFAULT_STRANDED_RECOVERY_NOTICE_BODY);
expect(notice.presentation.title).toBe("Automatic recovery blocked");
});
it("omits the failure code row when the source run has no error code", () => {
const notice = buildStrandedRecoveryEscalationNotice({
seed: buildImmediateExecutionPathRecoveryNoticeSeed({ status: "in_progress" }),
recoveryActionId: actionId,
recoveryOwner: owner,
sourceRun: { id: sourceRun.id, status: "cancelled", errorCode: null },
});
const rows = allRows(notice.metadata);
expect(rows).toContainEqual({ type: "run_link", label: "Source run", runId: sourceRun.id, title: "cancelled" });
expect(rows.some((row) => row.label === "Failure code")).toBe(false);
});
it("is matched by the metadata-based escalation dedupe matcher", () => {
const notice = buildStrandedRecoveryEscalationNotice({
seed: buildImmediateExecutionPathRecoveryNoticeSeed({ status: "todo" }),
recoveryActionId: actionId,
recoveryOwner: owner,
sourceRun,
});
// The escalation dedupe in recovery/service.ts no longer relies on the
// `Recovery action: \`id\`` body marker for new comments; the metadata
// matcher must recognize every notice this builder produces.
expect(notice.body).not.toContain(actionId);
expect(noticeMetadataReferencesRecoveryAction(notice.metadata, actionId)).toBe(true);
expect(
noticeMetadataReferencesRecoveryAction(notice.metadata, "1f2e3d4c-5b6a-4798-8899-444444444444"),
).toBe(false);
});
});
@@ -0,0 +1,150 @@
import type { IssueCommentMetadata, IssueCommentPresentation } from "@paperclipai/shared";
import {
agentLinkRow,
keyValueRow,
runLinkRow,
systemNoticePresentation,
type NoticeMetadataRow,
type NoticeMetadataSection,
} from "./notice-format.js";
// Short human-readable body plus the presentation header for one recovery
// family. The escalation path merges in the metadata rows only it knows
// (recovery action, owner, source run) via buildStrandedRecoveryEscalationNotice.
export type StrandedRecoveryNoticeSeed = {
body: string;
title: string;
tone: IssueCommentPresentation["tone"];
};
export type StrandedRecoveryEscalationNotice = {
body: string;
presentation: IssueCommentPresentation;
metadata: IssueCommentMetadata;
};
export const DEFAULT_STRANDED_RECOVERY_NOTICE_BODY =
"Paperclip could not restore a live execution path for this issue automatically. " +
"Moving it to `blocked` so it is visible for intervention.";
const DEFAULT_STRANDED_RECOVERY_NOTICE_TITLE = "Automatic recovery blocked";
const STRANDED_RECOVERY_NOTICE_TITLES_BY_CAUSE: Record<string, string> = {
workspace_validation_failed: "Workspace validation failed",
configuration_incomplete: "Configuration incomplete",
execution_review_participant_recovery: "Review recovery stalled",
};
export function buildImmediateExecutionPathRecoveryNoticeSeed(input: {
status: "todo" | "in_progress";
}): StrandedRecoveryNoticeSeed {
const retryDescription = input.status === "todo"
? "Paperclip automatically retried dispatch for this assigned `todo` issue during terminal run recovery"
: "Paperclip automatically retried continuation for this assigned `in_progress` issue during terminal run recovery";
return {
body:
`${retryDescription}, but it still has no live execution path. ` +
"Moving it to `blocked` so it is visible for intervention.",
title: "No live execution path",
tone: "danger",
};
}
export function buildWorkspaceValidationRecoveryNoticeSeed(): StrandedRecoveryNoticeSeed {
return {
body:
"Paperclip stopped before launching the local adapter because the issue workspace failed validation. " +
"Moving it to `blocked` so the workspace link, cwd, or git checkout can be repaired before resuming.",
title: "Workspace validation failed",
tone: "danger",
};
}
export function buildConfigurationIncompleteRecoveryNoticeSeed(): StrandedRecoveryNoticeSeed {
return {
body:
"Paperclip stopped before dispatching the adapter because required secret/env bindings are missing. " +
"Moving it to `blocked` so an operator can bind the missing secret(s) before resuming.",
title: "Configuration incomplete",
tone: "danger",
};
}
export function buildExecutionReviewParticipantRecoveryNoticeSeed(): StrandedRecoveryNoticeSeed {
return {
body:
"Paperclip retried the pending execution-review participant once, but the review stage still has no " +
"completed decision or live reviewer run. Moving it to `blocked` so the recovery owner can repair the " +
"reviewer runtime, restore the review stage, or record an intentional manual resolution.",
title: "Review recovery stalled",
tone: "danger",
};
}
export function buildExecutionReviewParticipantUnavailableNoticeSeed(): StrandedRecoveryNoticeSeed {
return {
body:
"Paperclip cannot continue the pending execution-review participant because the participant is not " +
"invokable and the review stage has no completed decision or live reviewer run. Moving it to `blocked` " +
"so the recovery owner can repair the reviewer runtime, restore the review stage, or record an " +
"intentional manual resolution.",
title: "Review recovery stalled",
tone: "danger",
};
}
// Escalation dedupe matches the `Recovery action` key_value row via
// noticeMetadataReferencesRecoveryAction, so this builder must always emit
// that row with the raw action id.
export function buildStrandedRecoveryEscalationNotice(input: {
seed?: StrandedRecoveryNoticeSeed | null;
fallbackBody?: string | null;
recoveryCause?: string | null;
recoveryActionId: string;
recoveryOwner: { id: string; name: string | null } | null | undefined;
sourceRun: { id: string; status: string; errorCode?: string | null } | null | undefined;
}): StrandedRecoveryEscalationNotice {
const fallbackBody = input.fallbackBody?.trim();
const body = input.seed?.body ?? (fallbackBody || DEFAULT_STRANDED_RECOVERY_NOTICE_BODY);
const title = input.seed?.title ??
STRANDED_RECOVERY_NOTICE_TITLES_BY_CAUSE[input.recoveryCause ?? ""] ??
DEFAULT_STRANDED_RECOVERY_NOTICE_TITLE;
const recoveryRows: NoticeMetadataRow[] = [
keyValueRow("Recovery action", input.recoveryActionId),
input.recoveryOwner
? agentLinkRow("Recovery owner", input.recoveryOwner)
: keyValueRow(
"Recovery owner",
"Board escalation - Paperclip could not find an invokable manager, creator, or executive owner with budget available",
),
keyValueRow(
"Next action",
input.recoveryOwner
? "The recovery owner should either restore a live execution path or record the manual resolution on the source issue"
: "A board operator should assign an invokable recovery owner, fix the agent/runtime state, or record an intentional manual resolution",
),
];
const runRows: NoticeMetadataRow[] = [];
if (input.sourceRun) {
runRows.push(runLinkRow("Source run", input.sourceRun));
const failureCode = input.sourceRun.errorCode?.trim();
if (failureCode) runRows.push(keyValueRow("Failure code", failureCode));
}
const sections: NoticeMetadataSection[] = [
{ title: "Recovery", rows: recoveryRows },
...(runRows.length > 0 ? [{ title: "Run evidence", rows: runRows }] : []),
];
return {
body,
presentation: systemNoticePresentation({ tone: input.seed?.tone ?? "danger", title }),
metadata: {
version: 1,
sourceRunId: input.sourceRun?.id ?? null,
sections,
},
};
}
@@ -3,6 +3,13 @@ import type { Db } from "@paperclipai/db";
import { agentWakeupRequests, agents, heartbeatRuns, issues } from "@paperclipai/db";
import type { IssueCommentMetadata, IssueCommentPresentation, RunLivenessState } from "@paperclipai/shared";
import { withRecoveryModelProfileHint } from "./model-profile-hint.js";
import {
agentLinkRow,
issueLinkRow,
keyValueRow,
runLinkRow,
systemNoticePresentation,
} from "./notice-format.js";
export const FINISH_SUCCESSFUL_RUN_HANDOFF_REASON = "finish_successful_run_handoff";
export const SUCCESSFUL_RUN_MISSING_STATE_REASON = "successful_run_missing_state";
@@ -115,58 +122,6 @@ export function isSuccessfulRunHandoffValidPathSkip(
return decision.kind === "skip" && SUCCESSFUL_RUN_HANDOFF_VALID_PATH_SKIP_REASONS.has(decision.reason);
}
function metadataText(value: unknown, fallback = "unknown") {
const text = typeof value === "string" ? value.trim() : value == null ? "" : String(value).trim();
const resolved = text.length > 0 ? text : fallback;
return resolved.length > 2000 ? `${resolved.slice(0, 1997)}...` : resolved;
}
function keyValueRow(label: string, value: unknown): IssueCommentMetadata["sections"][number]["rows"][number] {
return { type: "key_value", label, value: metadataText(value) };
}
function issueLinkRow(
label: string,
issue: NullableNoticeIssue,
): IssueCommentMetadata["sections"][number]["rows"][number] {
if (!issue) return keyValueRow(label, "unknown");
return {
type: "issue_link",
label,
issueId: issue.id,
identifier: issue.identifier,
title: issue.title,
};
}
function runLinkRow(
label: string,
run: NullableNoticeRun,
): IssueCommentMetadata["sections"][number]["rows"][number] {
if (!run) return keyValueRow(label, "unknown");
return { type: "run_link", label, runId: run.id, title: run.status };
}
function agentLinkRow(
label: string,
agent: NullableNoticeAgent,
): IssueCommentMetadata["sections"][number]["rows"][number] {
if (!agent) return keyValueRow(label, "unknown");
return { type: "agent_link", label, agentId: agent.id, name: agent.name };
}
function systemNoticePresentation(input: {
tone: IssueCommentPresentation["tone"];
title: string;
}): IssueCommentPresentation {
return {
kind: "system_notice",
tone: input.tone,
title: input.title,
detailsDefaultOpen: false,
};
}
export function isSuccessfulRunHandoffRequiredNoticeBody(body: string) {
const trimmed = body.trim();
return trimmed === SUCCESSFUL_RUN_HANDOFF_REQUIRED_NOTICE_BODY ||