mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-25 09:41:04 +01:00
Keep native reassignment stops out of recovery and retries
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
510e51c6d6
commit
baec6fe37f
8 files changed
+75
-16
No files matched your search
@@ -108,7 +108,9 @@ documents, dependencies, scope, priority, and history.
|
||||
|
||||
The old execution and any live goal must stop before ownership changes. Active
|
||||
work returns to todo; backlog and blocked work retain their status and are not
|
||||
started. Assignment changes advance the observed version, including ordinary
|
||||
started. An audited reassignment stop does not create a recovery blocker or
|
||||
retry the outgoing run when it ends without a semantic completion result.
|
||||
Assignment changes advance the observed version, including ordinary
|
||||
API changes, to reject stale handoffs. A durable audit receipt makes retries
|
||||
idempotent across caller runs; retrying after failed dispatch repairs the wake
|
||||
with a stable key and an owner/status/version guard.
|
||||
|
||||
@@ -318,6 +318,30 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => {
|
||||
});
|
||||
}
|
||||
|
||||
it("releases an acknowledged native handoff without blocking or restarting the old owner", async () => {
|
||||
const companyId = await seedCompany();
|
||||
const agentId = await seedAgent({ companyId });
|
||||
const issueId = await seedIssue({ companyId, assigneeAgentId: agentId, status: "in_progress" });
|
||||
const runId = await seedRun({ companyId, agentId, status: "cancelled", contextSnapshot: { issueId } });
|
||||
await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: issueId, resultJson: {
|
||||
reassignmentStopRequested: true,
|
||||
nativeCancellation: { schema: "paperclip.native-cancellation.v1", runId, companyId, issueId,
|
||||
scope: "run", reasonCode: "cancellation_run_only", dispatchState: "acknowledged", dispatched: true,
|
||||
intentAuditId: randomUUID(), acknowledgementAuditId: randomUUID() },
|
||||
} }).where(eq(heartbeatRuns.id, runId));
|
||||
await db.update(issues).set({ executionRunId: runId, checkoutRunId: runId }).where(eq(issues.id, issueId));
|
||||
const before = (await db.select().from(issues).where(eq(issues.id, issueId)))[0];
|
||||
const wakeId = await seedDeferredWake({ companyId, agentId, issueId });
|
||||
const adapter = createPostgresWakeQueueAdapter(db, stubDeps);
|
||||
const result = await adapter.withIssueExecutionLock({ companyId, runId, now: new Date() }, async () => { throw new Error("must not restart the outgoing owner"); });
|
||||
expect(result).toMatchObject({ outcome: { kind: "released" }, postCommitEffects: [] });
|
||||
expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0]).toMatchObject({
|
||||
status: "in_progress", statusVersion: before.statusVersion, assigneeAgentId: agentId, executionRunId: null, checkoutRunId: null,
|
||||
});
|
||||
expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeId)))[0].status).toBe("deferred_issue_execution");
|
||||
expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId))).toHaveLength(0);
|
||||
});
|
||||
|
||||
it.each(["in_progress", "blocked"])("preserves recovery ownership and queued messages when a native task fails from %s", async (status) => {
|
||||
const companyId = await seedCompany();
|
||||
const agentId = await seedAgent({ companyId });
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { isAcknowledgedNativeStop } from "../../../services/acknowledged-native-stop.js";
|
||||
import { isAcknowledgedNativeReassignmentStop, isAcknowledgedNativeStop } from "../../../services/acknowledged-native-stop.js";
|
||||
import { instanceSettingsService } from "../../../services/instance-settings.js";
|
||||
import { currentConversationCommentCondition } from "../../../services/agent-conversations.js";
|
||||
import { getExecutionBlocker } from "../../../services/execution-blocker.js";
|
||||
@@ -718,7 +718,7 @@ async function recordNativeTerminalRecoveryIfNeeded(tx: Db, run: HeartbeatRunRow
|
||||
["failed", "timed_out", "interrupted", "cancelled"].includes(run.status) &&
|
||||
issue.assigneeAgentId === run.agentId &&
|
||||
!["done", "cancelled"].includes(issue.status);
|
||||
if (!applies || isAcknowledgedNativeStop(run)) return false;
|
||||
if (!applies || isAcknowledgedNativeStop(run) || isAcknowledgedNativeReassignmentStop(run)) return false;
|
||||
|
||||
const existing = await tx
|
||||
.select({ id: issueRecoveryActions.id, evidence: issueRecoveryActions.evidence })
|
||||
@@ -1104,9 +1104,10 @@ export function createPostgresWakeQueueAdapter(db: Db, deps: WakeQueuePostgresAd
|
||||
// next explicit wake adopts those messages atomically when it
|
||||
// queues a run.
|
||||
executionCancellationAcknowledged:
|
||||
run.status === "cancelled" &&
|
||||
isAcknowledgedNativeReassignmentStop(run) ||
|
||||
(run.status === "cancelled" &&
|
||||
(parseObject(run.resultJson?.executionCancellation).state === "acknowledged" || isAcknowledgedNativeStop(run)) &&
|
||||
!interruptedQueue,
|
||||
!interruptedQueue),
|
||||
};
|
||||
const preDrain = decidePreDrain(preDrainFacts);
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { expect, it } from "vitest";
|
||||
import { isAcknowledgedNativeStop } from "./acknowledged-native-stop.js";
|
||||
import { isAcknowledgedNativeReassignmentStop, isAcknowledgedNativeStop } from "./acknowledged-native-stop.js";
|
||||
const run = { id: "run", companyId: "company", nativeIssueId: "issue", status: "cancelled", resultJson: {
|
||||
cancelledByActorType: "user", cancelledByUserId: "board", nativeCancellation: {
|
||||
schema: "paperclip.native-cancellation.v1", runId: "run", companyId: "company", issueId: "issue",
|
||||
@@ -15,3 +15,16 @@ it.each([{ runId: "other" }, { companyId: "other" }, { issueId: "other" }, { dis
|
||||
expect(isAcknowledgedNativeStop({ ...run, resultJson: { ...run.resultJson,
|
||||
nativeCancellation: { ...run.resultJson.nativeCancellation, ...change } } })).toBe(false);
|
||||
});
|
||||
|
||||
it("recognizes an audited handoff without granting operator Stop semantics", () => {
|
||||
const handoff = { ...run, resultJson: { reassignmentStopRequested: true, nativeCancellation: run.resultJson.nativeCancellation } };
|
||||
expect(isAcknowledgedNativeReassignmentStop(handoff)).toBe(true);
|
||||
expect(isAcknowledgedNativeStop(handoff)).toBe(false);
|
||||
expect(isAcknowledgedNativeReassignmentStop({ ...handoff, status: "running" })).toBe(false);
|
||||
expect(isAcknowledgedNativeReassignmentStop(run)).toBe(false);
|
||||
});
|
||||
it.each([{ runId: "other" }, { companyId: "other" }, { issueId: "other" }, { dispatchState: "pending" },
|
||||
{ scope: "subtree" }, { acknowledgementAuditId: undefined }])("requires an audited handoff receipt %j", change => {
|
||||
expect(isAcknowledgedNativeReassignmentStop({ ...run, resultJson: { reassignmentStopRequested: true,
|
||||
nativeCancellation: { ...run.resultJson.nativeCancellation, ...change } } })).toBe(false);
|
||||
});
|
||||
@@ -4,21 +4,34 @@ import { hasNativeLocalProcessStop } from "./native-local-process-stop.js";
|
||||
import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js";
|
||||
|
||||
/** A server-recorded run-only Stop must not manufacture a recovery incident. */
|
||||
export function hasAcknowledgedNativeStopIntent(run: {
|
||||
function hasAcknowledgedNativeRunStopReceipt(run: {
|
||||
id: string; companyId: string; status: string; nativeIssueId: string | null;
|
||||
resultJson: Record<string, unknown> | null;
|
||||
}): boolean {
|
||||
const result = run.resultJson;
|
||||
const intent = result?.nativeCancellation as Record<string, unknown> | undefined;
|
||||
return result?.cancelledByActorType === "user" &&
|
||||
typeof result.cancelledByUserId === "string" && Boolean(result.cancelledByUserId) &&
|
||||
intent?.schema === "paperclip.native-cancellation.v1" && intent.runId === run.id &&
|
||||
return intent?.schema === "paperclip.native-cancellation.v1" && intent.runId === run.id &&
|
||||
intent.companyId === run.companyId && intent.issueId === run.nativeIssueId &&
|
||||
intent.scope === "run" && intent.reasonCode === "cancellation_run_only" &&
|
||||
intent.dispatchState === "acknowledged" && intent.dispatched === true &&
|
||||
typeof intent.intentAuditId === "string" && typeof intent.acknowledgementAuditId === "string";
|
||||
}
|
||||
|
||||
export function hasAcknowledgedNativeStopIntent(run: Parameters<typeof hasAcknowledgedNativeRunStopReceipt>[0]): boolean {
|
||||
return run.resultJson?.cancelledByActorType === "user" &&
|
||||
typeof run.resultJson.cancelledByUserId === "string" && Boolean(run.resultJson.cancelledByUserId) &&
|
||||
hasAcknowledgedNativeRunStopReceipt(run);
|
||||
}
|
||||
|
||||
/** An intentional handoff retains task state and deferred work for its successor. */
|
||||
export function hasAcknowledgedNativeReassignmentStopIntent(run: Parameters<typeof hasAcknowledgedNativeRunStopReceipt>[0]): boolean {
|
||||
return run.resultJson?.reassignmentStopRequested === true && hasAcknowledgedNativeRunStopReceipt(run);
|
||||
}
|
||||
|
||||
export function isAcknowledgedNativeReassignmentStop(run: Parameters<typeof hasAcknowledgedNativeRunStopReceipt>[0]): boolean {
|
||||
return run.status === "cancelled" && hasAcknowledgedNativeReassignmentStopIntent(run);
|
||||
}
|
||||
|
||||
export function isAcknowledgedNativeStop(run: Parameters<typeof hasAcknowledgedNativeStopIntent>[0]): boolean {
|
||||
return run.status === "cancelled" && hasAcknowledgedNativeStopIntent(run);
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { readQueuedInteractionResponse } from "./queued-interaction-response.js";
|
||||
import { AGENT_CHAT_DIRECTIVE, conversationReplay, isConversation, isConversationExecutionWake, isWaitingConversation, prepareConversationTurn, settleConversationTurn } from "./agent-conversations.js";
|
||||
import { PROCESS_IDENTITY_RECORDED, recordNativeLocalProcessStop } from "./native-local-process-stop.js";
|
||||
import { hasAcknowledgedNativeStopIntent, isAcknowledgedNativeStop, acknowledgedNativeStopExecutionHasStopped } from "./acknowledged-native-stop.js";
|
||||
import { hasAcknowledgedNativeReassignmentStopIntent, hasAcknowledgedNativeStopIntent, isAcknowledgedNativeStop, acknowledgedNativeStopExecutionHasStopped } from "./acknowledged-native-stop.js";
|
||||
import { legacyControllerBootId, legacyControllerClaim, renewLegacyControllerLease, hasLiveLegacyController, revokeExpiredLegacyController, watchLegacyControllerLease } from "./legacy-controller-lease.js";
|
||||
import { completeTerminatedRemoteNativeSessionCleanup } from "../vendor/paperclip-runner/index.js";
|
||||
import { hasRemoteTerminationReceipt, remoteExecutionHasStopped, remoteTerminationReceipt, stoppedRemoteCleanupScopes } from "./remote-execution-termination.js";
|
||||
@@ -12849,7 +12849,7 @@ export function heartbeatService(
|
||||
|
||||
// Teardown can beat the cancellation finalizer. Preserve the acknowledged
|
||||
// user intent instead of reporting an infrastructure interruption.
|
||||
if (hasAcknowledgedNativeStopIntent(run)) terminalStatus = "cancelled";
|
||||
if (hasAcknowledgedNativeStopIntent(run) || hasAcknowledgedNativeReassignmentStopIntent(run)) terminalStatus = "cancelled";
|
||||
|
||||
const message = `run terminalized on environment lease release: heartbeat_runs.status was still ${run.status} at teardown`;
|
||||
// Match both "running" and "queued". A queued run has released its lease but
|
||||
@@ -28615,7 +28615,11 @@ export function heartbeatService(
|
||||
// additional durable fence before its existing cancellation path.
|
||||
if (run.runtimeMode === "native" || (!run.runtimeModeResolvedAt && !running && !control)) {
|
||||
const [fenced] = await db.update(heartbeatRuns).set({
|
||||
// Record handoff intent before native cancellation can finalize and release
|
||||
// the run. Only its audited stop acknowledgement suppresses recovery.
|
||||
resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) ||
|
||||
${JSON.stringify(options.errorCode === "issue_reassigned" && options.resultJson?.reassignmentStopConfirmed === true
|
||||
? { reassignmentStopRequested: true } : {})}::jsonb ||
|
||||
jsonb_build_object('startupCancellation', jsonb_build_object(
|
||||
'requestedAt', ${new Date().toISOString()}::text,
|
||||
'beforeNativeSelection', ${heartbeatRuns.runtimeMode} = 'legacy'
|
||||
|
||||
@@ -6059,11 +6059,13 @@ describe("native warm session supervision", () => {
|
||||
});
|
||||
|
||||
describe("native session bounded recovery", () => {
|
||||
it("does not turn an acknowledged Stop before completion into a failure or a retry", async () => {
|
||||
it.each(["operator", "reassignment"])("does not turn an acknowledged %s Stop before completion into a failure or a retry", async (source) => {
|
||||
const updates: Array<{ table: unknown; values: Record<string, unknown> }> = [];
|
||||
const stop: Record<string, unknown> = {};
|
||||
state.execute.mockReset().mockImplementationOnce(async () => {
|
||||
Object.assign(stop, { cancelledByActorType: "user", cancelledByUserId: "board", nativeCancellation: {
|
||||
Object.assign(stop, { ...(source === "operator"
|
||||
? { cancelledByActorType: "user", cancelledByUserId: "board" }
|
||||
: { reassignmentStopRequested: true }), nativeCancellation: {
|
||||
schema: "paperclip.native-cancellation.v1", ...execution.binding, scope: "run", reasonCode: "cancellation_run_only",
|
||||
dispatched: true, dispatchState: "acknowledged", intentAuditId: "intent", acknowledgementAuditId: "ack",
|
||||
} });
|
||||
|
||||
@@ -2,7 +2,7 @@ import { createLocalNativeQuestionBridge } from "./local-native-question-bridge.
|
||||
import { readVerifiedRemoteWorkspaceFile } from "./remote-deliverable-file.js";
|
||||
import { copyBackCodexAuth } from "@paperclipai/adapter-codex-local/server";
|
||||
import { nativeCompletionFeedback } from "./native-completion-feedback.js";
|
||||
import { hasAcknowledgedNativeStopIntent } from "../acknowledged-native-stop.js";
|
||||
import { hasAcknowledgedNativeReassignmentStopIntent, hasAcknowledgedNativeStopIntent } from "../acknowledged-native-stop.js";
|
||||
import { stoppedCodexTurnIsTextOnly } from "./stopped-codex-turn.js";
|
||||
import { prepareVerifiedRemoteProviderPack } from "./remote-provider-pack.js";
|
||||
import { readNativeLocalProcessStop, PROCESS_START_REQUESTED } from "../native-local-process-stop.js";
|
||||
@@ -8194,7 +8194,7 @@ async function executePaperclipNativeSessionWithinScope(
|
||||
eq(heartbeatRuns.agentId, input.execution.binding.agentId),
|
||||
eq(heartbeatRuns.nativeIssueId, input.execution.binding.issueId),
|
||||
)).limit(1);
|
||||
if (stoppedRun && hasAcknowledgedNativeStopIntent(stoppedRun)) {
|
||||
if (stoppedRun && (hasAcknowledgedNativeStopIntent(stoppedRun) || hasAcknowledgedNativeReassignmentStopIntent(stoppedRun))) {
|
||||
const [settled] = await input.db.update(nativeRunFinalizations).set({
|
||||
phase: "terminal_failure", failureCode: "native_retry_cancelled", nextAttemptAt: null,
|
||||
leaseOwner: null, leaseExpiresAt: null, controlDeadlineAt: null, recoveryState: null,
|
||||
|
||||
Reference in new issue
Block a user