mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-16 11:59:26 +01:00
fix(recovery): bound the invariant's 'pending' clauses for stale recovery state
The assigned-work invariant counted a task as pending forever when its projection said system recovery was working, or when a wake sat deferred behind a run that had already ended. Both were seen in the release-swap wave: an undelivered fresh continuation and follow-ups never promoted after restart recovery ended their run were reported OK and never repaired. 'Working' now counts only within 20 minutes of the last run's end, and a deferred wake with no live run only within 5 minutes of its creation; past that the task is a violation and gets the sweep's one continuation. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
ade7e0643f
commit
b7eb506eac
4 files changed
+53
-2
No files matched your search
@@ -105,6 +105,30 @@ describePostgres("assigned-work invariant sweep", () => {
|
||||
expect(queue.calls).toHaveLength(1);
|
||||
}, 60_000);
|
||||
|
||||
it("a wake deferred behind a run that ended long ago is not a pending path; a fresh one still is", async () => {
|
||||
// The 8/14 release-swap wave: a person's follow-up stayed deferred_issue_execution
|
||||
// after restart recovery ended the run it waited behind, and the report said "queued_wake".
|
||||
const f = await seedCompany();
|
||||
const issueId = await f.issue();
|
||||
await f.run(issueId, { status: "failed", errorCode: "native_restart_recovery_blocked" });
|
||||
const deferred = (createdAt: Date) => db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agentId,
|
||||
source: "automation", triggerDetail: "system", reason: "issue_execution_deferred", status: "deferred_issue_execution",
|
||||
requestedByActorType: "user", requestedByActorId: "local-board", payload: { issueId }, createdAt, updatedAt: new Date() });
|
||||
await deferred(new Date(Date.now() - 30 * 60_000));
|
||||
const queue = wakeQueue();
|
||||
const recovery = recoveryService(db, { enqueueWakeup: queue.enqueueWakeup, scheduleRecoveryRetry: async () => null });
|
||||
expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId }))
|
||||
.toMatchObject({ outcome: "repaired", verdict: { kind: "violation", reason: "no_pending_path" } });
|
||||
|
||||
const freshIssue = await f.issue();
|
||||
await f.run(freshIssue, { status: "failed", errorCode: "native_restart_recovery_blocked" });
|
||||
await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agentId, source: "automation",
|
||||
triggerDetail: "system", reason: "issue_execution_deferred", status: "deferred_issue_execution",
|
||||
payload: { issueId: freshIssue }, createdAt: new Date(Date.now() - 60_000) });
|
||||
expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId: freshIssue }))
|
||||
.toMatchObject({ outcome: "ok", verdict: { clause: "pending_intent", intent: "queued_wake" } });
|
||||
}, 60_000);
|
||||
|
||||
it("an in_progress task whose native run failed needing reconciliation (silently skipped today) gets one continuation", async () => {
|
||||
const f = await seedCompany();
|
||||
const issueId = await f.issue();
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
SETTLING_GRACE_MS,
|
||||
SETTLING_MAX_MS,
|
||||
STALE_BLOCK_MS,
|
||||
SYSTEM_RECOVERY_WORKING_BOUND_MS,
|
||||
VIOLATION_REASONS,
|
||||
type AssignedWorkFacts,
|
||||
} from "./assigned-work-invariant.js";
|
||||
@@ -74,7 +75,8 @@ describe("assigned-work invariant", () => {
|
||||
["pending_handoff", { pendingHandoff: true }],
|
||||
["session_goal", { sessionGoal: "active" }],
|
||||
["scheduled_monitor", { issue: { monitorNextCheckAt: new Date(now.getTime() + 60_000) } }],
|
||||
["system_recovery_working", { interruption: { responsibility: "system", state: "working", userAction: null, nextRetryAt: null, canRetry: false } }],
|
||||
["system_recovery_working", { interruption: { responsibility: "system", state: "working", userAction: null, nextRetryAt: null, canRetry: false },
|
||||
latestRun: { id: "run-1", status: "failed", errorCode: "native_restart_recovery_blocked", finishedAt: ago(5 * 60_000), settlingUntil: null, operatorStopped: false } }],
|
||||
];
|
||||
for (const [intent, overrides] of cases) {
|
||||
it(`accepts ${intent}`, () => {
|
||||
@@ -82,6 +84,18 @@ describe("assigned-work invariant", () => {
|
||||
});
|
||||
}
|
||||
|
||||
it("stops believing system recovery is working once the last run ended longer ago than the bound", () => {
|
||||
// A recorded continuation that is never delivered keeps the projection at
|
||||
// "working" forever; the invariant must see it as nothing pending.
|
||||
const working = { responsibility: "system" as const, state: "working", userAction: null, nextRetryAt: null, canRetry: false };
|
||||
const run = (msAgo: number) => ({ id: "run-1", status: "failed", errorCode: "native_restart_recovery_blocked",
|
||||
finishedAt: ago(msAgo), settlingUntil: null, operatorStopped: false });
|
||||
expect(evaluateAssignedWork(facts({ interruption: working, latestRun: run(SYSTEM_RECOVERY_WORKING_BOUND_MS - 60_000) }), now))
|
||||
.toEqual({ kind: "ok", clause: "pending_intent", intent: "system_recovery_working" });
|
||||
expect(evaluateAssignedWork(facts({ interruption: working, latestRun: run(SYSTEM_RECOVERY_WORKING_BOUND_MS + 60_000) }), now))
|
||||
.toMatchObject({ kind: "violation", reason: "no_pending_path" });
|
||||
});
|
||||
|
||||
it("does not count a monitor or retry whose time has passed", () => {
|
||||
expect(evaluateAssignedWork(facts({ issue: { monitorNextCheckAt: ago(1) } }), now)).toMatchObject({ kind: "violation" });
|
||||
expect(evaluateAssignedWork(facts({ interruption: { responsibility: "system", state: "retry_scheduled", userAction: null,
|
||||
|
||||
@@ -100,6 +100,11 @@ const PROVIDER_LIMIT_CAUSES = new Set(["provider_quota", "native_provider_usage_
|
||||
export const SETTLING_GRACE_MS = 90_000;
|
||||
/** Upper bound on how long another server's lease can keep a finished run settling. */
|
||||
export const SETTLING_MAX_MS = 5 * 60_000;
|
||||
/** "System recovery working" with nothing else pending is believed for this long
|
||||
* after the task's last run ended. Every recovery wait the fallback owns ends
|
||||
* within 15 minutes (saved-work recovery that keeps advancing: its own budget);
|
||||
* past this the state is stale and the task is continued like any other. */
|
||||
export const SYSTEM_RECOVERY_WORKING_BOUND_MS = 20 * 60_000;
|
||||
/** Blocked tasks older than this are reported, never continued automatically. */
|
||||
export const STALE_BLOCK_MS = 7 * 24 * 60 * 60_000;
|
||||
/** Automatic continuations per task, shared with the server-restart lane. */
|
||||
@@ -231,7 +236,8 @@ function pendingIntent(facts: AssignedWorkFacts, now: Date): PendingIntent | nul
|
||||
if (facts.issue.monitorNextCheckAt && facts.issue.monitorNextCheckAt.getTime() > now.getTime()) return "scheduled_monitor";
|
||||
const interruption = facts.interruption;
|
||||
if (interruption?.responsibility === "system") {
|
||||
if (interruption.state === "working") return "system_recovery_working";
|
||||
if (interruption.state === "working" && !(facts.latestRun?.finishedAt
|
||||
&& now.getTime() - facts.latestRun.finishedAt.getTime() > SYSTEM_RECOVERY_WORKING_BOUND_MS)) return "system_recovery_working";
|
||||
if (interruption.state === "retry_scheduled" && interruption.nextRetryAt
|
||||
&& Date.parse(interruption.nextRetryAt) > now.getTime()) return "scheduled_retry";
|
||||
}
|
||||
|
||||
@@ -57,6 +57,8 @@ export const ASSIGNED_WORK_REPORT_LIMIT = 2_000;
|
||||
|
||||
const OPEN_STATUSES = ["todo", "in_progress", "blocked"] as const;
|
||||
const PENDING_WAKE_STATUSES = ["queued", "deferred_issue_execution", "claimed"];
|
||||
/** How long a deferred wake counts as pending when no run holds its task. */
|
||||
const DEFERRED_WAKE_PENDING_BOUND_MS = 5 * 60_000;
|
||||
const NON_TERMINAL_RUN_STATUSES = ["queued", "running", "scheduled_retry"];
|
||||
const NATIVE_PENDING_RECOVERY_STATES = ["awaiting_evidence", "awaiting_runner_reattach", "resuming_session", "bootstrap_incomplete"];
|
||||
const PROVIDER_LIMIT_CAUSES = ["provider_quota", "native_provider_usage_limit"];
|
||||
@@ -195,6 +197,11 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) {
|
||||
database.select({ issueId: sql<string>`${agentWakeupRequests.payload}->>'issueId'`, count: sql<number>`count(*)::int` })
|
||||
.from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, companyId),
|
||||
inArray(agentWakeupRequests.status, PENDING_WAKE_STATUSES),
|
||||
// A wake deferred behind a run is promoted when that run releases the
|
||||
// task. With no live run (checked first), one older than the bound has
|
||||
// nothing left that will promote it, so it is not a pending path.
|
||||
sql`(${agentWakeupRequests.status} <> 'deferred_issue_execution'
|
||||
or ${agentWakeupRequests.createdAt} > ${at.toISOString()}::timestamptz - ${`${DEFERRED_WAKE_PENDING_BOUND_MS} milliseconds`}::interval)`,
|
||||
sql`${agentWakeupRequests.payload}->>'issueId' = any(${idParam})`))
|
||||
.groupBy(sql`${agentWakeupRequests.payload}->>'issueId'`),
|
||||
database.select({ id: issues.id }).from(issues).where(and(eq(issues.companyId, companyId), inArray(issues.id, ids),
|
||||
|
||||
Reference in new issue
Block a user