mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-29 21:07:40 +01:00
Keep an accepted turn's starting input in task history, show its admission wait, and resume existing queued runs promptly when maintenance ends or expires. Verified with behavioral red/green regressions, real PostgreSQL and process execution, browser/native Runner acceptance with a deterministic provider, workspace typecheck, production build, and current-head CI/review. Co-Authored-By: Paperclip <noreply@paperclip.ing>
285 lines
12 KiB
TypeScript
285 lines
12 KiB
TypeScript
import { describe, expect, it } from "vitest";
|
|
import { projectExecution } from "./execution-projection.js";
|
|
import { startTaskDrain, stopTaskDrain } from "./task-admission.js";
|
|
|
|
type Run = Parameters<typeof projectExecution>[0];
|
|
type Coordinator = NonNullable<Parameters<typeof projectExecution>[1]>;
|
|
const now = new Date("2026-09-08T12:00:00Z");
|
|
const run = (values: Partial<Run> = {}) =>
|
|
({
|
|
status: "running",
|
|
runtimeMode: "native",
|
|
errorCode: null,
|
|
nextAction: null,
|
|
lastUsefulActionAt: now,
|
|
lastOutputAt: null,
|
|
startedAt: now,
|
|
scheduledRetryAt: null,
|
|
scheduledRetryAttempt: 1,
|
|
retryOfRunId: null,
|
|
executionControlDeadlineAt: null,
|
|
...values,
|
|
}) as Run;
|
|
const coordinator = (values: Partial<Coordinator> = {}) =>
|
|
({
|
|
phase: "observed",
|
|
attempt: 1,
|
|
failureCode: null,
|
|
failureDetail: {},
|
|
nextAttemptAt: null,
|
|
leaseExpiresAt: new Date(now.getTime() + 30_000),
|
|
controlDeadlineAt: null,
|
|
...values,
|
|
}) as Coordinator;
|
|
const project = (
|
|
r = run(),
|
|
c: Coordinator | undefined = coordinator(),
|
|
pending: { kind: string }[] = [],
|
|
) => projectExecution(r, c, pending, undefined, now);
|
|
|
|
describe("execution truth projection", () => {
|
|
it("shows maintenance only for admission that has not started", () => {
|
|
startTaskDrain();
|
|
try {
|
|
expect(projectExecution(run({ status: "queued", startedAt: null }), undefined, [], undefined, now))
|
|
.toMatchObject({ phase: "queued", label: "Waiting for maintenance", cause: "task_drain" });
|
|
expect(project().phase).toBe("working");
|
|
} finally { stopTaskDrain(); }
|
|
expect(projectExecution(run({ status: "queued", startedAt: null }), undefined, [], undefined, now))
|
|
.toMatchObject({ phase: "queued", label: "Queued", cause: null });
|
|
});
|
|
it.each(["queued", "recovering", "blocked", "recovered"] as const)("shows the exact saved workspace recovery state instead of stale repair instructions: %s", status => {
|
|
const sourceRunId = "ec036e06-90a3-4b69-81db-a8eab7dbfbac";
|
|
const source = run({ id: sourceRunId, status: "failed", errorCode: "native_continuation_requires_reconciliation" });
|
|
const action = {
|
|
status: "resolved" as const,
|
|
cause: "native_continuation_requires_reconciliation",
|
|
nextAction: "The original sandbox is retained for workspace repair.",
|
|
evidence: {
|
|
runId: sourceRunId,
|
|
automaticRecovery: { policy: "preserve_without_replay_v1", actionOutcome: "unknown", replay: "blocked" },
|
|
continuationDelivery: "invalidated",
|
|
workspaceRecovery: {
|
|
schema: "paperclip.workspace-recovery.v1", runId: sourceRunId, status,
|
|
stage: "saving_receipts", attempt: 1, requestIds: [],
|
|
...(status === "blocked" ? { message: "The saved source cannot be read safely." } : {}),
|
|
},
|
|
},
|
|
};
|
|
const projected = projectExecution(source, coordinator({ phase: "terminal_failure" }), [], action, now);
|
|
expect(projected).toMatchObject({
|
|
phase: "recovery_needed", label: "Stopped", cause: action.cause,
|
|
nextAction: status === "recovered" ? "Saved work recovered. Send a message to continue."
|
|
: status === "blocked" ? "The saved source cannot be read safely." : "Recovering saved work…",
|
|
permittedActions: ["inspect_run"], successorRunId: null,
|
|
});
|
|
expect(source.status).toBe("failed");
|
|
expect(action.evidence.automaticRecovery.replay).toBe("blocked");
|
|
expect(projectExecution(source, undefined, [], { ...action,
|
|
evidence: { ...action.evidence, workspaceRecovery: { ...action.evidence.workspaceRecovery,
|
|
runId: "5f227d75-b314-4511-bf23-9c615277e69b" } },
|
|
}, now).nextAction).toBe(action.nextAction);
|
|
});
|
|
|
|
it("keeps cleanup quarantine visible without a coordinator or recovery row", () => {
|
|
const stopped = run({ id: "stopped", status: "failed", errorCode: "native_session_cleanup_quarantined" });
|
|
expect(projectExecution(stopped, undefined, [], undefined, now)).toMatchObject({
|
|
phase: "recovery_needed", cause: "native_session_cleanup_quarantined", recoveryOwner: "board",
|
|
permittedActions: ["inspect_run", "inspect_recovery"],
|
|
});
|
|
expect(projectExecution(stopped, coordinator({ phase: "retryable_failure" }), [], undefined, now).phase).toBe("recovery_needed");
|
|
expect(projectExecution({ ...stopped, finishedAt: now }, coordinator({
|
|
phase: "terminal_failure", failureCode: "native_provider_terminal_failed",
|
|
}), [], undefined, now)).toMatchObject({ phase: "recovery_needed", recoveryOwner: "board" });
|
|
expect(projectExecution({ ...stopped, errorCode: "provider_frame_too_large" }, undefined, [], undefined, now).phase).toBe("failed");
|
|
expect(projectExecution(stopped, undefined, [], { status: "resolved", cause: "native_session_cleanup_quarantined", nextAction: "Verify prior execution",
|
|
evidence: { executionReconciliation: { providerStopped: true }, continuationRunId: "successor" } }, now))
|
|
.toMatchObject({ phase: "completed", successorRunId: "successor", cause: null });
|
|
});
|
|
it("shows subscription contention as a resource wait without failed provider attempts", () => {
|
|
expect(projectExecution(run({ runtimeMode: "legacy", status: "scheduled_retry", scheduledRetryReason: "ai_connection_busy",
|
|
scheduledRetryAttempt: 12, contextSnapshot: { failureRetriesBeforeAiConnectionWait: 0 } }), undefined, [], undefined, now))
|
|
.toMatchObject({ label: "Waiting for AI subscription", phase: "retry_scheduled", attempt: 1, recoveryOwner: null });
|
|
});
|
|
it("shows a workspace wait without presenting its deferral count as failed attempts", () => {
|
|
expect(projectExecution(run({ runtimeMode: "legacy", status: "scheduled_retry", scheduledRetryReason: "workspace_busy",
|
|
scheduledRetryAttempt: 12, contextSnapshot: { failureRetriesBeforeWorkspaceWait: 1 } }), undefined, [], undefined, now))
|
|
.toMatchObject({ label: "Waiting for workspace", phase: "retry_scheduled", attempt: 2, recoveryOwner: null });
|
|
});
|
|
it.each([
|
|
{ status: "resolved", previousRunId: "old", nextRunId: "next", continued: true },
|
|
{ status: "active", previousRunId: "old", nextRunId: "next", continued: false },
|
|
{ status: "resolved", previousRunId: "other", nextRunId: "next", continued: false },
|
|
{ status: "resolved", previousRunId: "old", nextRunId: "old", continued: false },
|
|
{ status: "resolved", previousRunId: "old", nextRunId: null, continued: false },
|
|
])("projects the recorded explicit successor without hiding unresolved recovery: $status/$previousRunId/$nextRunId", ({ status, previousRunId, nextRunId, continued }) => {
|
|
const source = run({ id: "old", status: "failed", errorCode: "adapter_failed" });
|
|
expect(projectExecution(source, coordinator({ phase: "terminal_failure" }), [], {
|
|
status,
|
|
cause: "native_continuation_requires_reconciliation",
|
|
nextAction: "Inspect the stopped execution.",
|
|
evidence: {
|
|
automaticRecovery: { policy: "preserve_without_replay_v1" },
|
|
explicitUserContinuation: { previousRunId, runId: nextRunId },
|
|
},
|
|
}, now)).toMatchObject(continued ? {
|
|
phase: "completed", label: "Continued in another run", successorRunId: "next",
|
|
cause: "native_continuation_requires_reconciliation", nextAction: null,
|
|
} : { phase: "recovery_needed", successorRunId: null });
|
|
expect(source.status).toBe("failed");
|
|
});
|
|
|
|
it("shows a reconciled continuation as queued until its durable delivery is recorded", () => {
|
|
const action = {
|
|
cause: "native_session_retry_exhausted",
|
|
nextAction: "Old recovery action",
|
|
status: "resolved" as const,
|
|
evidence: {
|
|
executionReconciliation: { runId: "old" },
|
|
continuationDelivery: "pending",
|
|
},
|
|
};
|
|
expect(
|
|
projectExecution(
|
|
run({ status: "failed" }),
|
|
coordinator({ phase: "terminal_failure" }),
|
|
[],
|
|
action,
|
|
now,
|
|
),
|
|
).toMatchObject({
|
|
phase: "queued",
|
|
label: "Continuation queued",
|
|
cause: null,
|
|
});
|
|
expect(
|
|
projectExecution(
|
|
run({ status: "failed" }),
|
|
coordinator({ phase: "terminal_failure" }),
|
|
[],
|
|
{
|
|
...action,
|
|
evidence: { ...action.evidence, continuationRunId: "next" },
|
|
},
|
|
now,
|
|
),
|
|
).toMatchObject({ phase: "completed", successorRunId: "next" });
|
|
expect(
|
|
projectExecution(
|
|
run({ status: "failed" }),
|
|
coordinator({ phase: "terminal_failure" }),
|
|
[],
|
|
{
|
|
...action,
|
|
evidence: { ...action.evidence, continuationDelivery: "invalidated" },
|
|
},
|
|
now,
|
|
),
|
|
).toMatchObject({ phase: "failed", label: "Continuation cancelled" });
|
|
});
|
|
it("requires current execution evidence, without treating output silence as failure", () => {
|
|
expect(project(run({ lastOutputAt: new Date(0) })).phase).toBe("working");
|
|
expect(
|
|
project(run(), coordinator({ leaseExpiresAt: new Date(0) })),
|
|
).toMatchObject({ phase: "reconnecting", label: "Confirming execution" });
|
|
expect(
|
|
project(
|
|
run({ runtimeMode: "legacy", processPid: process.pid }),
|
|
undefined,
|
|
).phase,
|
|
).toBe("working");
|
|
});
|
|
it("distinguishes provider work, finalization and timed retry", () => {
|
|
expect(
|
|
project(
|
|
run({ executionControlDeadlineAt: new Date(now.getTime() + 60_000) }),
|
|
).phase,
|
|
).toBe("finishing");
|
|
expect(
|
|
project(
|
|
run({ status: "failed" }),
|
|
coordinator({
|
|
phase: "retryable_failure",
|
|
nextAttemptAt: new Date(now.getTime() + 30_000),
|
|
attempt: 2,
|
|
}),
|
|
),
|
|
).toMatchObject({ phase: "retry_scheduled", attempt: 2, maxAttempts: 3 });
|
|
expect(
|
|
project(
|
|
run({ status: "failed" }),
|
|
coordinator({ phase: "retryable_failure", nextAttemptAt: new Date(0) }),
|
|
).phase,
|
|
).toBe("reconnecting");
|
|
});
|
|
it("includes the original legacy attempt in the displayed incident budget", () => {
|
|
expect(projectExecution(run({ runtimeMode: "legacy", status: "scheduled_retry", scheduledRetryAttempt: 1 }), undefined, [], undefined, now)).toMatchObject({ attempt: 2, maxAttempts: 3 });
|
|
});
|
|
it("shows pending interaction only after productive work has stopped", () => {
|
|
expect(
|
|
project(run(), coordinator(), [{ kind: "connection_intent" }]).phase,
|
|
).toBe("working");
|
|
expect(
|
|
project(run({ status: "succeeded" }), undefined, [
|
|
{ kind: "connection_intent" },
|
|
]).phase,
|
|
).toBe("waiting_for_access");
|
|
expect(
|
|
project(run({ status: "succeeded" }), undefined, [
|
|
{ kind: "ask_user_questions" },
|
|
]).phase,
|
|
).toBe("waiting_for_answer");
|
|
});
|
|
it("retains replacement lineage and exposes operator-owned failures for legacy runs", () => {
|
|
expect(
|
|
project(
|
|
run({ status: "failed" }),
|
|
coordinator({
|
|
phase: "terminal_failure",
|
|
failureDetail: { successorRunId: "replacement" },
|
|
}),
|
|
),
|
|
).toMatchObject({ phase: "completed", successorRunId: "replacement" });
|
|
expect(
|
|
projectExecution(
|
|
run({ runtimeMode: "legacy", status: "failed" }),
|
|
undefined,
|
|
[],
|
|
{
|
|
cause: "uncertain_external_action",
|
|
nextAction: "Reconcile the email delivery.",
|
|
},
|
|
now,
|
|
),
|
|
).toMatchObject({
|
|
phase: "recovery_needed",
|
|
recoveryOwner: "board",
|
|
nextAction: "Reconcile the email delivery.",
|
|
});
|
|
});
|
|
it("bounds recovery checking and surfaces its next action if verification never finishes", () => {
|
|
const c = coordinator({
|
|
phase: "terminal_failure",
|
|
failureCode: "native_provider_terminal_failed",
|
|
});
|
|
expect(project(run({ status: "failed", finishedAt: now }), c).label).toBe(
|
|
"Checking recovery",
|
|
);
|
|
expect(
|
|
project(
|
|
run({ status: "failed", finishedAt: new Date(now.getTime() - 60_000) }),
|
|
c,
|
|
).phase,
|
|
).toBe("recovery_needed");
|
|
expect(
|
|
project(
|
|
run({ status: "failed", finishedAt: now }),
|
|
coordinator({
|
|
...c,
|
|
failureDetail: { replacementDenied: "uncertain_provider_action" },
|
|
}),
|
|
).phase,
|
|
).toBe("recovery_needed");
|
|
});
|
|
});
|