mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-17 13:01:05 +01:00
## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - A person can stop an agent run with Stop (cancel). The run must then end as cancelled > - A run that a person stops just as it starts tears down its environment lease at the same moment > - The lease-release teardown wrote `interrupted` (`lease_released_before_terminal`). It recognized only an acknowledged Stop, and it decided from a run snapshot that it read before the Stop request landed > - An interrupted run looks like an infrastructure failure, and it is a recovery candidate. That is against the person's decision > - This pull request makes a recorded person Stop win over the teardown in both orderings > - The benefit is that a person's Stop always ends the run as cancelled ## Linked Issues or Issue Description **What happened?** In the paid chat eval `steering.stop-versus-pause`, the fixture cancelled a resumed run as it reached native running. The Stop request was recorded at 04:02:11.049. The teardown wrote `interrupted` with `lease_released_before_terminal` at 04:02:11.083. The native cancellation intent was still `pending`, so the teardown did not see it as a Stop. The eval needs `cancelled`. **Steps to reproduce** 1. Start a native run. 2. Record a person Stop request on the run (the cancel route does this first). 3. Let the lease-release teardown run before the cancel finalizer writes `cancelled`. 4. The run ends `interrupted`. **Expected behavior** The run ends `cancelled`. This is true when the Stop is recorded before the teardown reads the run, and when it is recorded between that read and the write. ## What Changed - `server/src/services/acknowledged-native-stop.ts`: `hasRecordedUserStopRequest` and `recordedUserStopRequestSql` recognize a person Stop that the cancel route recorded (`cancelledByActorType: "user"`, a user id, and a requested or acknowledged execution cancellation or a startup cancellation fence). - `server/src/services/heartbeat/run-lifecycle.ts`: `terminalizeRunOnLeaseRelease` re-reads the run before it decides. A recorded person Stop now gives `cancelled`. The `interrupted` write is fenced on "no person Stop recorded", so a Stop that lands between the read and the write makes the write fail. The function then reads again and writes `cancelled`. - `server/src/__tests__/heartbeat-run-lease-release-terminalization.test.ts`: two orderings (Stop recorded before the teardown read; Stop recorded after the read, with the stale snapshot) and a control (an agent-only request still reports `interrupted`). ## Verification - `cd server && npx vitest run src/__tests__/heartbeat-run-lease-release-terminalization.test.ts` — both new ordering tests fail on master (`interrupted`, `lease_released_before_terminal`) and pass here. - `cd server && npx vitest run src/__tests__/heartbeat-run-lease-release-terminalization.test.ts src/__tests__/heartbeat-run-terminalize-before-release.test.ts src/services/heartbeat/run-lifecycle.test.ts` (67 passed). - `cd server && npx tsc --noEmit` passes. - Paid E2E: see the combined-branch run in the comments. ## Risks - Low. Only the lease-release teardown changes, and only when a person Stop is recorded on the run. The cancel path already wrote `cancelled` for the same run when it won the race. - The teardown now makes one more read of the run row. This is one indexed primary-key read per teardown. - Previous-release saved state: no persisted shape changes. Runs saved by the previous release carry the same Stop fields. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, with tool use and code execution in Claude Code. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
133 lines
9.0 KiB
TypeScript
133 lines
9.0 KiB
TypeScript
import { and, desc, eq, inArray, isNotNull, sql } from "drizzle-orm";
|
|
import { environmentLeases, heartbeatRunEvents, heartbeatRuns, type Db } from "@paperclipai/db";
|
|
import { parseObject } from "../adapters/utils.js";
|
|
import { resolveProviderApiPlacement } from "./native-runtime/provider-api-placement.js";
|
|
import { hasNativeLocalProcessStop } from "./native-local-process-stop.js";
|
|
import { hasRecoveredCleanupStopReceipt, hasRemoteTerminationReceipt } from "./remote-execution-termination.js";
|
|
import { validatePrpEvent } from "../vendor/paperclip-runner/index.js";
|
|
import { nativeSha256 } from "./native-runtime/canonical.js";
|
|
import { isConfirmedCursorCloudTerminalObservation } from "./native-runtime/cursor-cloud-runtime.js";
|
|
import { isAgentLifecycleStopRecord } from "./governance-pause-stop.js";
|
|
|
|
export function hasAcknowledgedNativeRunStopRequest(run: {
|
|
id: string; companyId: string; status: string; nativeIssueId: string | null;
|
|
resultJson: Record<string, unknown> | null;
|
|
}): boolean {
|
|
const result = run.resultJson;
|
|
const intent = result?.nativeCancellation as Record<string, unknown> | undefined;
|
|
return intent?.schema === "paperclip.native-cancellation.v1" && intent.runId === run.id &&
|
|
intent.companyId === run.companyId && intent.issueId === run.nativeIssueId &&
|
|
intent.scope === "run" && intent.reasonCode === "cancellation_run_only" &&
|
|
intent.dispatchState === "acknowledged" && typeof intent.dispatched === "boolean" &&
|
|
typeof intent.intentAuditId === "string" && Boolean(intent.intentAuditId) &&
|
|
typeof intent.acknowledgementAuditId === "string" && Boolean(intent.acknowledgementAuditId);
|
|
}
|
|
|
|
export function hasAcknowledgedNativeRunStopReceipt(run: Parameters<typeof hasAcknowledgedNativeRunStopRequest>[0]): boolean {
|
|
return hasAcknowledgedNativeRunStopRequest(run) &&
|
|
(run.resultJson?.nativeCancellation as Record<string, unknown>).dispatched === true;
|
|
}
|
|
|
|
export function hasAcknowledgedNativeStopIntent(run: Parameters<typeof hasAcknowledgedNativeRunStopReceipt>[0]): boolean {
|
|
return run.resultJson?.cancelledByActorType === "user" &&
|
|
typeof run.resultJson.cancelledByUserId === "string" && Boolean(run.resultJson.cancelledByUserId) &&
|
|
hasAcknowledgedNativeRunStopReceipt(run);
|
|
}
|
|
|
|
export function hasRecordedUserStopRequest(run: { resultJson: Record<string, unknown> | null }): boolean {
|
|
const result = run.resultJson;
|
|
if (result?.cancelledByActorType !== "user" || typeof result.cancelledByUserId !== "string" || !result.cancelledByUserId) return false;
|
|
const state = parseObject(result.executionCancellation).state;
|
|
const startup = result.startupCancellation;
|
|
return state === "requested" || state === "acknowledged" ||
|
|
(Boolean(startup) && typeof startup === "object" && !Array.isArray(startup));
|
|
}
|
|
|
|
export function recordedUserStopRequestSql() {
|
|
const result = heartbeatRuns.resultJson;
|
|
return sql`(coalesce(${result}->>'cancelledByActorType', '') = 'user'
|
|
and coalesce(${result}->>'cancelledByUserId', '') <> ''
|
|
and (coalesce(${result}->'executionCancellation'->>'state', '') in ('requested', 'acknowledged')
|
|
or jsonb_typeof(${result}->'startupCancellation') = 'object'))`;
|
|
}
|
|
|
|
export function hasAcknowledgedNativeLifecycleStopIntent(run: Parameters<typeof hasAcknowledgedNativeRunStopReceipt>[0]): boolean {
|
|
return isAgentLifecycleStopRecord(run.resultJson) && hasAcknowledgedNativeRunStopReceipt(run);
|
|
}
|
|
|
|
/** An intentional handoff retains task state and deferred work for its successor. */
|
|
export function hasAcknowledgedNativeReassignmentStopIntent(run: Parameters<typeof hasAcknowledgedNativeRunStopReceipt>[0]): boolean {
|
|
return run.resultJson?.reassignmentStopRequested === true && hasAcknowledgedNativeRunStopReceipt(run);
|
|
}
|
|
|
|
export function isAcknowledgedNativeReassignmentStop(run: Parameters<typeof hasAcknowledgedNativeRunStopReceipt>[0]): boolean {
|
|
return run.status === "cancelled" && hasAcknowledgedNativeReassignmentStopIntent(run);
|
|
}
|
|
|
|
export function isAcknowledgedNativeStop(run: Parameters<typeof hasAcknowledgedNativeStopIntent>[0]): boolean {
|
|
return run.status === "cancelled" && hasAcknowledgedNativeStopIntent(run);
|
|
}
|
|
|
|
/** API-owned execution has no Computer lease. Its provider terminal and retired
|
|
* local controller must both be recorded before publishing shared Stop proof. */
|
|
export async function acknowledgedNativeApiStopExecutionHasStopped(db: Db, run: Pick<typeof heartbeatRuns.$inferSelect,
|
|
"id" | "companyId" | "agentId" | "nativeIssueId" | "status" | "resultJson" | "processPid" | "processGroupId" |
|
|
"runtimeMode" | "driverKind" | "runnerProfileJson" | "nativeSessionId" | "runnerInstanceId">): Promise<boolean> {
|
|
if (run.runtimeMode !== "native" || run.driverKind !== "cursor_cloud_api" || !isAcknowledgedNativeStop(run) ||
|
|
run.processPid !== null || run.processGroupId !== null || !run.nativeIssueId || !run.nativeSessionId || !run.runnerInstanceId) return false;
|
|
const intent = parseObject(run.resultJson?.nativeCancellation);
|
|
const requestedAt = typeof intent.recordedAt === "string" ? Date.parse(intent.recordedAt) : NaN;
|
|
if (!Number.isFinite(requestedAt) || !Array.isArray(intent.effects) || intent.effects.length !== 1 ||
|
|
intent.effects[0] !== "release_run_resources" ||
|
|
!String(intent.intentAuditId ?? "").trim() || !String(intent.acknowledgementAuditId ?? "").trim()) return false;
|
|
const profile = parseObject(run.runnerProfileJson);
|
|
const execution = parseObject(profile.nativeExecutionInput);
|
|
const binding = parseObject(execution.binding), session = parseObject(execution.session);
|
|
if (profile.executionPlacement === undefined || profile.backend !== "cursor_cloud_api" ||
|
|
binding.companyId !== run.companyId || binding.runId !== run.id || binding.issueId !== run.nativeIssueId ||
|
|
binding.agentId !== run.agentId || session.normalizedSessionId !== run.nativeSessionId || session.driverKind !== "cursor_cloud_api") return false;
|
|
try {
|
|
if (!resolveProviderApiPlacement({ run, agent: { adapterType: "process", adapterConfig: {} } })) return false;
|
|
} catch { return false; }
|
|
const [lease] = await db.select({ id: environmentLeases.id }).from(environmentLeases).where(and(
|
|
eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id),
|
|
)).limit(1);
|
|
if (lease || !await hasNativeLocalProcessStop(db, run.companyId, run.id)) return false;
|
|
const [terminal] = await db.select().from(heartbeatRunEvents).where(and(
|
|
eq(heartbeatRunEvents.companyId, run.companyId), eq(heartbeatRunEvents.runId, run.id),
|
|
isNotNull(heartbeatRunEvents.sourceEventId),
|
|
inArray(heartbeatRunEvents.eventType, ["turn.started", "turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"]),
|
|
)).orderBy(desc(heartbeatRunEvents.seq)).limit(1);
|
|
const validated = validatePrpEvent(parseObject(terminal?.payload).prpEvent);
|
|
if (!terminal || !validated.ok || terminal.sourcePayloadSha256 !== nativeSha256(validated.event)) return false;
|
|
const event = validated.event;
|
|
return isConfirmedCursorCloudTerminalObservation({ eventType: terminal.eventType, payload: event.payload }) && terminal.agentId === run.agentId &&
|
|
event.schema === "paperclip.prp.event.v1" && [1, 2].includes(Number(event.schemaVersion)) &&
|
|
terminal.protocolSchemaVersion === event.schemaVersion && event.eventType === terminal.eventType &&
|
|
event.sourceKind === "runner" && event.runId === run.id && event.normalizedSessionId === run.nativeSessionId &&
|
|
event.sourceInstanceId === run.runnerInstanceId && terminal.sourceInstanceId === event.sourceInstanceId &&
|
|
terminal.sourceEventId === event.sourceEventId && terminal.sourceSeq === event.sourceSeq &&
|
|
Number.isSafeInteger(event.sourceSeq) && Number(event.sourceSeq) > 0 &&
|
|
typeof event.turnId === "string" && Boolean(event.turnId);
|
|
}
|
|
|
|
/** A Stop acknowledgement alone does not prove provider cleanup completed. */
|
|
export async function acknowledgedNativeStopExecutionHasStopped(db: Db, run: Pick<typeof heartbeatRuns.$inferSelect,
|
|
"id" | "companyId" | "nativeIssueId" | "status" | "resultJson" | "processPid" | "processGroupId">) {
|
|
if (!isAcknowledgedNativeStop(run)) return false;
|
|
const leases = await db.select().from(environmentLeases).where(and(
|
|
eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id),
|
|
));
|
|
// Provider PIDs are meaningful only on their own host. A recovered sandbox
|
|
// queued for destroy keeps its verified stop; the destroy is only cleanup.
|
|
if (leases.some(lease => lease.provider !== "local"))
|
|
return leases.every(lease => hasRemoteTerminationReceipt(lease) || hasRecoveredCleanupStopReceipt(lease));
|
|
if (leases.some(lease => !lease.releasedAt || lease.cleanupStatus === "failed")) return false;
|
|
if (!run.processPid && !run.processGroupId) return hasNativeLocalProcessStop(db, run.companyId, run.id);
|
|
const absent = (pid: number) => {
|
|
try { process.kill(pid, 0); return false; }
|
|
catch (error) { return (error as NodeJS.ErrnoException).code === "ESRCH"; }
|
|
};
|
|
return (!run.processPid || absent(run.processPid)) && (!run.processGroupId || absent(-run.processGroupId));
|
|
}
|