mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-07 07:55:47 +01:00
## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - The heartbeat/recovery subsystem decides whether an agent run has a durable continuation path after the process stops. > - External waits need stricter semantics than local background watchers: a killed local process is not durable, while a first-class blocker/monitor/scheduled wake is. > - Without that distinction, recovery can repeatedly treat adapter-failed continuations as live work and obscure the real reason a task stopped. > - This pull request adds explicit durable external-wait liveness handling and documents the expected execution semantics. > - It also improves operator-visible recovery evidence so invalid external-wait paths explain why they were rejected. > - The benefit is clearer recovery behavior, fewer duplicate continuation recoveries, and a safer contract for monitor-backed external waits. ## Linked Issues or Issue Description - Refs #5978 - Related PRs: #4988, #7495, #8502 ## What Changed - Added durable external-wait liveness classification so local/background watchers are not accepted as durable live paths after the owning process exits. - Preserved first-class blocker/monitor/scheduled wake paths as valid external-wait continuations. - Added backend regression coverage for killed watcher failure, monitor-backed durable wait resumption, normal completion, blocker behavior, and no duplicate recovery. - Added adapter utility coverage for terminal cleanup behavior used by local process adapters. - Surfaced invalid external-wait recovery evidence in the recovery action card and run ledger. - Updated execution semantics documentation and the V1 implementation contract. ## Verification - `pnpm check:token-gates` passed. - `pnpm -r typecheck` passed. - `node scripts/run-vitest-stable.mjs --mode general --group general-server` equivalent lane passed in CI-clean env: 238 files, 2164 tests passed, 1 skipped. - `node scripts/run-vitest-stable.mjs --mode general --group general-workspaces-a` passed in fully Paperclip-env-clean env: UI 305 files / 2430 tests; CLI 43 files / 230 tests. - `node scripts/run-vitest-stable.mjs --mode general --group general-workspaces-b` passed in fully Paperclip-env-clean env: shared/db/adapters/plugin packages all green. - `node scripts/run-vitest-stable.mjs --mode serialized` passed in fully Paperclip-env-clean env: 107 serialized server suites green, including 84/84 heartbeat-process-recovery tests. - `pnpm build` passed in fully Paperclip-env-clean env. Notes: running `pnpm test:run` directly inside the Paperclip heartbeat environment exposed local harness env contamination in existing tests (`PAPERCLIP_CONFIG`, `PAPERCLIP_DB_BACKUP_DIR`, and `PAPERCLIP_WORKTREE_START_POINT`). Re-running the same lanes with inherited `PAPERCLIP_*` and port env removed produced the CI-equivalent green results above. ## Risks - Medium behavioral risk: this changes recovery classification for stopped local external-wait processes, so adapters relying on unmanaged background watchers must use blockers, monitors, scheduled wakes, or explicit durable handoff instead. - Low UI risk: recovery-card copy changes are covered by component tests and Storybook screenshot QA. - No database migration is included. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - OpenAI Codex, GPT-5-based coding agent, tool-enabled terminal/code execution. Exact context-window metadata was not exposed in the runtime. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip <noreply@paperclip.ing> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
134 lines
4.7 KiB
TypeScript
134 lines
4.7 KiB
TypeScript
export type HeartbeatRunOutcome = "succeeded" | "interrupted" | "failed" | "cancelled" | "timed_out";
|
|
|
|
export type HeartbeatRunStopReason =
|
|
| "completed"
|
|
| "interrupted"
|
|
| "timeout"
|
|
| "cancelled"
|
|
| "budget_paused"
|
|
| "paused"
|
|
| "max_turns_exhausted"
|
|
| "process_lost"
|
|
| "unmanaged_background_task_stopped"
|
|
| "adapter_failed";
|
|
|
|
export interface HeartbeatRunTimeoutPolicy {
|
|
effectiveTimeoutSec: number | null;
|
|
effectiveTimeoutMs?: number | null;
|
|
timeoutConfigured: boolean;
|
|
timeoutSource: "config" | "default" | "unknown";
|
|
}
|
|
|
|
export interface HeartbeatRunStopMetadata extends HeartbeatRunTimeoutPolicy {
|
|
stopReason: HeartbeatRunStopReason;
|
|
timeoutFired: boolean;
|
|
}
|
|
|
|
function readFiniteNumber(value: unknown): number | null {
|
|
if (typeof value === "number") {
|
|
return Number.isFinite(value) ? value : null;
|
|
}
|
|
if (typeof value === "string") {
|
|
const parsed = Number(value.trim());
|
|
return Number.isFinite(parsed) ? parsed : null;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function hasOwn(record: Record<string, unknown>, key: string) {
|
|
return Object.prototype.hasOwnProperty.call(record, key);
|
|
}
|
|
|
|
function defaultTimeoutSecForAdapter(adapterType: string) {
|
|
return adapterType === "openclaw_gateway" ? 120 : 0;
|
|
}
|
|
|
|
export function normalizeMaxTurnStopReason(value: unknown): Extract<HeartbeatRunStopReason, "max_turns_exhausted"> | null {
|
|
return value === "max_turns_exhausted" || value === "turn_limit_exhausted"
|
|
? "max_turns_exhausted"
|
|
: null;
|
|
}
|
|
|
|
export function resolveHeartbeatRunTimeoutPolicy(
|
|
adapterType: string,
|
|
adapterConfig: Record<string, unknown> | null | undefined,
|
|
): HeartbeatRunTimeoutPolicy {
|
|
const config = adapterConfig ?? {};
|
|
|
|
if (adapterType === "http") {
|
|
const hasTimeoutMs = hasOwn(config, "timeoutMs");
|
|
const rawTimeoutMs = hasTimeoutMs ? readFiniteNumber(config.timeoutMs) : 0;
|
|
const timeoutMs = Math.max(0, Math.floor(rawTimeoutMs ?? 0));
|
|
return {
|
|
effectiveTimeoutSec: timeoutMs / 1000,
|
|
effectiveTimeoutMs: timeoutMs,
|
|
timeoutConfigured: timeoutMs > 0,
|
|
timeoutSource: hasTimeoutMs ? "config" : "default",
|
|
};
|
|
}
|
|
|
|
const hasTimeoutSec = hasOwn(config, "timeoutSec");
|
|
const defaultTimeoutSec = defaultTimeoutSecForAdapter(adapterType);
|
|
const rawTimeoutSec = hasTimeoutSec ? readFiniteNumber(config.timeoutSec) : defaultTimeoutSec;
|
|
const timeoutSec = Math.max(0, Math.floor(rawTimeoutSec ?? defaultTimeoutSec));
|
|
|
|
return {
|
|
effectiveTimeoutSec: timeoutSec,
|
|
timeoutConfigured: timeoutSec > 0,
|
|
timeoutSource: hasTimeoutSec ? "config" : "default",
|
|
};
|
|
}
|
|
|
|
export function inferHeartbeatRunStopReason(input: {
|
|
outcome: HeartbeatRunOutcome;
|
|
errorCode?: string | null;
|
|
errorMessage?: string | null;
|
|
}): HeartbeatRunStopReason {
|
|
if (input.outcome === "succeeded") return "completed";
|
|
if (input.outcome === "interrupted") return "interrupted";
|
|
const maxTurnStopReason = normalizeMaxTurnStopReason(input.errorCode);
|
|
if (maxTurnStopReason) return maxTurnStopReason;
|
|
if (input.outcome === "timed_out") return "timeout";
|
|
if (input.outcome === "failed" && input.errorCode === "unmanaged_background_task_stopped") return "unmanaged_background_task_stopped";
|
|
if (input.outcome === "failed" && input.errorCode === "process_lost") return "process_lost";
|
|
if (input.outcome === "cancelled") {
|
|
const message = (input.errorMessage ?? "").toLowerCase();
|
|
if (message.includes("budget")) return "budget_paused";
|
|
if (message.includes("pause") || message.includes("paused")) return "paused";
|
|
return "cancelled";
|
|
}
|
|
return "adapter_failed";
|
|
}
|
|
|
|
export function buildHeartbeatRunStopMetadata(input: {
|
|
adapterType: string;
|
|
adapterConfig: Record<string, unknown> | null | undefined;
|
|
outcome: HeartbeatRunOutcome;
|
|
errorCode?: string | null;
|
|
errorMessage?: string | null;
|
|
}): HeartbeatRunStopMetadata {
|
|
const timeoutPolicy = resolveHeartbeatRunTimeoutPolicy(input.adapterType, input.adapterConfig);
|
|
const stopReason = inferHeartbeatRunStopReason(input);
|
|
return {
|
|
...timeoutPolicy,
|
|
stopReason,
|
|
timeoutFired: stopReason === "timeout",
|
|
};
|
|
}
|
|
|
|
export function mergeHeartbeatRunStopMetadata(
|
|
resultJson: Record<string, unknown> | null | undefined,
|
|
metadata: HeartbeatRunStopMetadata,
|
|
): Record<string, unknown> {
|
|
const existingMaxTurnStopReason = normalizeMaxTurnStopReason(resultJson?.stopReason);
|
|
return {
|
|
...(resultJson ?? {}),
|
|
stopReason: existingMaxTurnStopReason ?? metadata.stopReason,
|
|
effectiveTimeoutSec: metadata.effectiveTimeoutSec,
|
|
timeoutConfigured: metadata.timeoutConfigured,
|
|
timeoutSource: metadata.timeoutSource,
|
|
timeoutFired: metadata.timeoutFired,
|
|
...(metadata.effectiveTimeoutMs != null ? { effectiveTimeoutMs: metadata.effectiveTimeoutMs } : {}),
|
|
};
|
|
}
|