mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-23 00:53:38 +02:00
test: cover recursive recovery prevention
This commit is contained in:
1 parent
22c23e1957
commit
b2e619a1a8
5 files changed
+161
-19
No files matched your search
@@ -459,6 +459,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
||||
retryReason?: "assignment_recovery" | "issue_continuation_needed" | null;
|
||||
assignToUser?: boolean;
|
||||
activePauseHold?: boolean;
|
||||
runErrorCode?: string | null;
|
||||
runError?: string | null;
|
||||
}) {
|
||||
const companyId = randomUUID();
|
||||
const agentId = randomUUID();
|
||||
@@ -500,7 +502,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
||||
runId,
|
||||
claimedAt: now,
|
||||
finishedAt: new Date("2026-03-19T00:05:00.000Z"),
|
||||
error: input.runStatus === "succeeded" ? null : "run failed before issue advanced",
|
||||
error: input.runStatus === "succeeded" ? null : (input.runError ?? "run failed before issue advanced"),
|
||||
});
|
||||
|
||||
await db.insert(heartbeatRuns).values({
|
||||
@@ -522,8 +524,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
||||
startedAt: now,
|
||||
finishedAt: new Date("2026-03-19T00:05:00.000Z"),
|
||||
updatedAt: new Date("2026-03-19T00:05:00.000Z"),
|
||||
errorCode: input.runStatus === "succeeded" ? null : "process_lost",
|
||||
error: input.runStatus === "succeeded" ? null : "run failed before issue advanced",
|
||||
errorCode: input.runStatus === "succeeded" ? null : (input.runErrorCode ?? "process_lost"),
|
||||
error: input.runStatus === "succeeded" ? null : (input.runError ?? "run failed before issue advanced"),
|
||||
});
|
||||
|
||||
await db.insert(issues).values([
|
||||
@@ -650,6 +652,20 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
||||
return recovery;
|
||||
}
|
||||
|
||||
async function sourceBlockerIssueIds(companyId: string, sourceIssueId: string) {
|
||||
return db
|
||||
.select({ blockerIssueId: issueRelations.issueId })
|
||||
.from(issueRelations)
|
||||
.where(
|
||||
and(
|
||||
eq(issueRelations.companyId, companyId),
|
||||
eq(issueRelations.relatedIssueId, sourceIssueId),
|
||||
eq(issueRelations.type, "blocks"),
|
||||
),
|
||||
)
|
||||
.then((rows) => rows.map((row) => row.blockerIssueId));
|
||||
}
|
||||
|
||||
async function seedQueuedIssueRunFixture() {
|
||||
const companyId = randomUUID();
|
||||
const agentId = randomUUID();
|
||||
@@ -936,6 +952,22 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
||||
originId: sourceIssueId,
|
||||
})
|
||||
.where(eq(issues.id, issueId));
|
||||
const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
|
||||
await db.insert(issues).values({
|
||||
id: sourceIssueId,
|
||||
companyId,
|
||||
title: "Original stranded source",
|
||||
status: "blocked",
|
||||
priority: "medium",
|
||||
issueNumber: 2,
|
||||
identifier: `${issuePrefix}-2`,
|
||||
});
|
||||
await db.insert(issueRelations).values({
|
||||
companyId,
|
||||
issueId,
|
||||
relatedIssueId: sourceIssueId,
|
||||
type: "blocks",
|
||||
});
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
const result = await heartbeat.reapOrphanedRuns();
|
||||
@@ -974,6 +1006,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
||||
expect(comments[0]?.body).toContain("stopped automatic stranded-work recovery");
|
||||
expect(comments[0]?.body).toContain("recovery issues do not create nested `stranded_issue_recovery` issues");
|
||||
expect(comments[0]?.body).toContain("Latest retry failure: `process_lost`");
|
||||
await expect(sourceBlockerIssueIds(companyId, sourceIssueId)).resolves.toEqual([issueId]);
|
||||
});
|
||||
|
||||
it("does not block paused-tree work when immediate continuation recovery is suppressed by the hold", async () => {
|
||||
@@ -1510,6 +1543,22 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
||||
originId: sourceIssueId,
|
||||
})
|
||||
.where(eq(issues.id, issueId));
|
||||
const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
|
||||
await db.insert(issues).values({
|
||||
id: sourceIssueId,
|
||||
companyId,
|
||||
title: "Original stranded source",
|
||||
status: "blocked",
|
||||
priority: "medium",
|
||||
issueNumber: 2,
|
||||
identifier: `${issuePrefix}-2`,
|
||||
});
|
||||
await db.insert(issueRelations).values({
|
||||
companyId,
|
||||
issueId,
|
||||
relatedIssueId: sourceIssueId,
|
||||
type: "blocks",
|
||||
});
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
const result = await heartbeat.reconcileStrandedAssignedIssues();
|
||||
@@ -1539,6 +1588,97 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
|
||||
expect(comments[0]?.body).toContain("stopped automatic stranded-work recovery");
|
||||
expect(comments[0]?.body).toContain("Latest retry failure: `process_lost` - run failed before issue advanced.");
|
||||
expect(comments[0]?.body).toContain("recovery issues do not create nested `stranded_issue_recovery` issues");
|
||||
await expect(sourceBlockerIssueIds(companyId, sourceIssueId)).resolves.toEqual([issueId]);
|
||||
});
|
||||
|
||||
it("keeps repeated recovery failures on the same canonical recovery issue", async () => {
|
||||
const sourceIssueId = randomUUID();
|
||||
const { companyId, agentId, issueId, runId } = await seedStrandedIssueFixture({
|
||||
status: "in_progress",
|
||||
runStatus: "failed",
|
||||
});
|
||||
const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
|
||||
await db.insert(issues).values({
|
||||
id: sourceIssueId,
|
||||
companyId,
|
||||
title: "Original stranded source",
|
||||
status: "blocked",
|
||||
priority: "medium",
|
||||
issueNumber: 2,
|
||||
identifier: `${issuePrefix}-2`,
|
||||
});
|
||||
await db
|
||||
.update(issues)
|
||||
.set({
|
||||
title: "Recover stalled issue PAP-1",
|
||||
originKind: "stranded_issue_recovery",
|
||||
originId: sourceIssueId,
|
||||
})
|
||||
.where(eq(issues.id, issueId));
|
||||
await db.insert(issueRelations).values({
|
||||
companyId,
|
||||
issueId,
|
||||
relatedIssueId: sourceIssueId,
|
||||
type: "blocks",
|
||||
});
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
const firstResult = await heartbeat.reconcileStrandedAssignedIssues();
|
||||
expect(firstResult.escalated).toBe(1);
|
||||
expect(firstResult.issueIds).toEqual([issueId]);
|
||||
|
||||
const secondRunId = randomUUID();
|
||||
await db.insert(heartbeatRuns).values({
|
||||
id: secondRunId,
|
||||
companyId,
|
||||
agentId,
|
||||
invocationSource: "assignment",
|
||||
triggerDetail: "system",
|
||||
status: "failed",
|
||||
contextSnapshot: {
|
||||
issueId,
|
||||
taskId: issueId,
|
||||
wakeReason: "issue_assigned",
|
||||
source: "stranded_issue_recovery",
|
||||
},
|
||||
startedAt: new Date("2030-03-19T00:10:00.000Z"),
|
||||
finishedAt: new Date("2030-03-19T00:15:00.000Z"),
|
||||
createdAt: new Date("2030-03-19T00:10:00.000Z"),
|
||||
updatedAt: new Date("2030-03-19T00:15:00.000Z"),
|
||||
errorCode: "adapter_failed",
|
||||
error: "adapter failed while retrying recovery issue",
|
||||
});
|
||||
await db
|
||||
.update(issues)
|
||||
.set({
|
||||
status: "in_progress",
|
||||
checkoutRunId: secondRunId,
|
||||
executionRunId: null,
|
||||
})
|
||||
.where(eq(issues.id, issueId));
|
||||
|
||||
const secondResult = await heartbeat.reconcileStrandedAssignedIssues();
|
||||
expect(secondResult.dispatchRequeued).toBe(0);
|
||||
expect(secondResult.continuationRequeued).toBe(0);
|
||||
expect(secondResult.escalated).toBe(1);
|
||||
expect(secondResult.issueIds).toEqual([issueId]);
|
||||
|
||||
const recoveryIssuesForSource = await db
|
||||
.select()
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, companyId), eq(issues.originKind, "stranded_issue_recovery"), eq(issues.originId, sourceIssueId)));
|
||||
expect(recoveryIssuesForSource.map((issue) => issue.id)).toEqual([issueId]);
|
||||
|
||||
const nestedRecoveries = await db
|
||||
.select()
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, companyId), eq(issues.originKind, "stranded_issue_recovery"), eq(issues.originId, issueId)));
|
||||
expect(nestedRecoveries).toHaveLength(0);
|
||||
await expect(sourceBlockerIssueIds(companyId, sourceIssueId)).resolves.toEqual([issueId]);
|
||||
|
||||
const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId));
|
||||
expect(comments).toHaveLength(2);
|
||||
expect(comments[1]?.body).toContain("Latest retry failure: `adapter_failed` - adapter failed while retrying recovery issue.");
|
||||
});
|
||||
|
||||
it("does not escalate paused-tree recovery when the automatic continuation retry was cancelled by the hold", async () => {
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
buildRunLivenessContinuationIdempotencyKey,
|
||||
classifyIssueGraphLiveness,
|
||||
decideRunLivenessContinuation,
|
||||
isStrandedIssueRecoveryOriginKind,
|
||||
parseIssueGraphLivenessIncidentKey,
|
||||
} from "../services/recovery/index.ts";
|
||||
|
||||
@@ -143,4 +144,11 @@ describe("recovery classifier boundary", () => {
|
||||
nextAttempt: 1,
|
||||
})).toBe("run_liveness_continuation:issue-1:run-1:plan_only:1");
|
||||
});
|
||||
|
||||
it("classifies stranded recovery origins as recovery-owned work", () => {
|
||||
expect(isStrandedIssueRecoveryOriginKind("stranded_issue_recovery")).toBe(true);
|
||||
expect(isStrandedIssueRecoveryOriginKind("harness_liveness_escalation")).toBe(false);
|
||||
expect(isStrandedIssueRecoveryOriginKind("manual")).toBe(false);
|
||||
expect(isStrandedIssueRecoveryOriginKind(null)).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -4,6 +4,7 @@ export {
|
||||
RECOVERY_REASON_KINDS,
|
||||
buildIssueGraphLivenessIncidentKey,
|
||||
buildIssueGraphLivenessLeafKey,
|
||||
isStrandedIssueRecoveryOriginKind,
|
||||
parseIssueGraphLivenessIncidentKey,
|
||||
} from "./origins.js";
|
||||
export type {
|
||||
|
||||
@@ -17,6 +17,10 @@ export type RecoveryOriginKind = typeof RECOVERY_ORIGIN_KINDS[keyof typeof RECOV
|
||||
export type RecoveryReasonKind = typeof RECOVERY_REASON_KINDS[keyof typeof RECOVERY_REASON_KINDS];
|
||||
export type RecoveryKeyPrefix = typeof RECOVERY_KEY_PREFIXES[keyof typeof RECOVERY_KEY_PREFIXES];
|
||||
|
||||
export function isStrandedIssueRecoveryOriginKind(originKind: string | null | undefined) {
|
||||
return originKind === RECOVERY_ORIGIN_KINDS.strandedIssueRecovery;
|
||||
}
|
||||
|
||||
export function buildIssueGraphLivenessIncidentKey(input: {
|
||||
companyId: string;
|
||||
issueId: string;
|
||||
|
||||
@@ -25,6 +25,7 @@ import { getRunLogStore } from "../run-log-store.js";
|
||||
import {
|
||||
RECOVERY_ORIGIN_KINDS,
|
||||
buildIssueGraphLivenessLeafKey,
|
||||
isStrandedIssueRecoveryOriginKind,
|
||||
parseIssueGraphLivenessIncidentKey,
|
||||
} from "./origins.js";
|
||||
import {
|
||||
@@ -93,21 +94,9 @@ function summarizeRunFailureForIssueComment(run: LatestIssueRun) {
|
||||
if (!run) return null;
|
||||
|
||||
const errorCode = readNonEmptyString(run.errorCode)?.trim() ?? null;
|
||||
const rawError = readNonEmptyString(run.error)?.trim() ?? null;
|
||||
const apiMessageMatch = rawError?.match(/"message"\s*:\s*"([^"]+)"/);
|
||||
const firstLine = rawError
|
||||
?.split(/\r?\n/)
|
||||
.map((line) => line.trim())
|
||||
.find(Boolean) ?? null;
|
||||
const summarySource = apiMessageMatch?.[1] ?? firstLine;
|
||||
const summary =
|
||||
summarySource && summarySource.length > 240
|
||||
? `${summarySource.slice(0, 237)}...`
|
||||
: summarySource;
|
||||
|
||||
if (errorCode && summary) return ` Latest retry failure: \`${errorCode}\` - ${summary}.`;
|
||||
if (errorCode) return ` Latest retry failure: \`${errorCode}\`.`;
|
||||
if (summary) return ` Latest retry failure: ${summary}.`;
|
||||
if (readNonEmptyString(run.error)) {
|
||||
return " Latest retry failure details were withheld from the issue thread; inspect the linked run for evidence.";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -179,7 +168,7 @@ function isAgentInvokable(agent: typeof agents.$inferSelect | null | undefined)
|
||||
}
|
||||
|
||||
function isStrandedIssueRecoveryIssue(issue: Pick<typeof issues.$inferSelect, "originKind">) {
|
||||
return issue.originKind === STRANDED_ISSUE_RECOVERY_ORIGIN_KIND;
|
||||
return isStrandedIssueRecoveryOriginKind(issue.originKind);
|
||||
}
|
||||
|
||||
function isUnsuccessfulTerminalIssueRun(latestRun: LatestIssueRun) {
|
||||
|
||||
Reference in new issue
Block a user