mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-07 07:55:47 +01:00
250 lines
17 KiB
TypeScript
250 lines
17 KiB
TypeScript
import { afterEach, describe, expect, it, vi } from "vitest";
|
|
import type { ControllerOutageReceipt } from "./controller-outage.js";
|
|
import { storyLifecycleChecks, type StoryRun } from "./everyday-observations.js";
|
|
import { gradeRemoteRestart, observeRemoteRestartBoundary, projectRemoteRestartEvents, projectRemoteRestartRun, waitForRemoteRestartBoundary } from "./everyday-remote-restart.js";
|
|
|
|
const run: StoryRun = {
|
|
id: "run", companyId: "company", agentId: "lead", nativeIssueId: "task",
|
|
status: "succeeded", runtimeMode: "native", runnerInstanceId: "runner", nativeSessionId: "session",
|
|
};
|
|
|
|
function fixture() {
|
|
const active = { ...run, status: "running", processPid: 42, processStartedAt: "2026-10-14T00:00:00.000Z" };
|
|
const scope = { companyId: "company", agentId: "lead", issueId: "task", environmentId: "environment" };
|
|
const leases = [{ id: "lease", ...scope, heartbeatRunId: "run", provider: "daytona", providerLeaseId: "private-sandbox",
|
|
status: "active", releasedAt: null, executionWorkspaceId: "workspace" }];
|
|
const event = (seq: number, eventType: string, payload: Record<string, unknown> = {}, sourceKind = "runner", sourceSeq = seq) => {
|
|
const sourceInstanceId = sourceKind === "runner" ? "runner" : "runner:control";
|
|
return { seq, runId: "run", companyId: "company", eventType, protocolSchemaVersion: 1, payload: { prpEvent: {
|
|
schema: "paperclip.prp.event.v1", schemaVersion: 1, runId: "run", normalizedSessionId: "session", sourceKind,
|
|
sourceInstanceId, sourceSeq, sourceEventId: `${sourceInstanceId}:run:${sourceSeq}`, eventType,
|
|
turnId: eventType === "turn.submitted" ? null : "turn", payload,
|
|
} } };
|
|
};
|
|
const rows = [event(1, "turn.submitted"), event(2, "turn.started"),
|
|
event(3, "tool.execution.started", { executionId: "tool", status: "running" }),
|
|
event(4, "tool.execution.completed", { executionId: "tool", status: "completed", output: "private tool output" })];
|
|
const before = observeRemoteRestartBoundary(active, rows, leases, scope);
|
|
const afterRows = [...rows, event(5, "item.delta", { text: "private provider text" })];
|
|
const after = observeRemoteRestartBoundary(active, afterRows, leases, scope);
|
|
const finished: StoryRun = { ...run, nativePhase: "committed", resultJson: {
|
|
finalizationPhase: "committed", workspaceFinalizeStatus: "succeeded", nextAttemptAt: null, failureCode: null,
|
|
} };
|
|
const finalRows = [...afterRows, event(6, "run.result.proposed"), event(7, "turn.completed"),
|
|
event(8, "run.result.accepted", {}, "control_plane", 1), event(9, "run.terminal", {}, "control_plane", 2)];
|
|
const final = { observedAt: "2026-10-22T00:02:00.000Z", run: finished, events: projectRemoteRestartEvents(finished, finalRows),
|
|
operations: [{ companyId: "company", heartbeatRunId: "run", issueId: "task", executionWorkspaceId: "workspace",
|
|
phase: "workspace_finalize", status: "succeeded" }], scheduledRetry: null as unknown, recoveryActions: [] as Record<string, unknown>[] };
|
|
const outage: ControllerOutageReceipt = { schema: "paperclip.e2e.controller-outage/v1", requestId: "story-test",
|
|
signal: "SIGKILL", holdMs: 1000, previousPid: 100, replacementPid: 200,
|
|
startedAt: "2026-10-22T00:00:00.000Z", stoppedAfterMs: 100, cacheDiscarded: true,
|
|
replacementStartedAfterMs: 1100, readyAfterMs: 4000,
|
|
recovery: { binding: { companyId: "company", issueId: "task", runId: "run" }, runnerInstanceId: "runner", nativeSessionId: "session",
|
|
previousGeneration: 1, generation: 2, previousBootId: "old", bootId: "new", previousLeaseExpiresAt: "2026-10-22T00:01:00.000Z",
|
|
ownershipAcquiredAt: "2026-10-22T00:01:00.001Z", ownershipObservedAfterMs: 60_010,
|
|
authenticatedAttachmentObservedAfterMs: 61_000, resumedProcessingObservedAfterMs: 61_000,
|
|
runnerSourceSeqBefore: 4, runnerSourceSeqAfter: 5, previousAuthenticatedConnectionCount: 1,
|
|
authenticatedConnectionCount: 2, attachmentEvidence: "runner_checkpoint_authenticated_connection" } };
|
|
return { active, scope, leases, event, rows, afterRows, finalRows, evidence: { before, after, final, outage } };
|
|
}
|
|
|
|
describe("remote restart boundary polling", () => {
|
|
afterEach(() => vi.useRealTimers());
|
|
|
|
it.each([undefined, "succeeded", "failed", "cancelled", "timed_out"] as const)("stops immediately when the original run is %s", async status => {
|
|
vi.useFakeTimers();
|
|
const now = Date.now();
|
|
const load = vi.fn(async () => ({ run: status === undefined ? undefined : { ...run, status } }));
|
|
const outcome = waitForRemoteRestartBoundary({ deadlineAt: now + 60_000, load }).then(
|
|
() => undefined, error => error as Error,
|
|
);
|
|
await vi.runAllTimersAsync();
|
|
expect((await outcome)?.message).toBe("Stopped waiting for active remote native turn with completed tool receipt: Remote restart boundary unexercised: original run ended before fault injection");
|
|
expect(load).toHaveBeenCalledTimes(1);
|
|
expect(Date.now()).toBe(now);
|
|
});
|
|
|
|
it("waits for useful work while the original run remains active", async () => {
|
|
vi.useFakeTimers();
|
|
const { active, evidence } = fixture();
|
|
const load = vi.fn().mockResolvedValueOnce({ run: active })
|
|
.mockResolvedValueOnce({ run: active, boundary: evidence.before });
|
|
const waiting = waitForRemoteRestartBoundary({ deadlineAt: Date.now() + 60_000, load });
|
|
await vi.runAllTimersAsync();
|
|
expect(await waiting).toEqual(evidence.before);
|
|
expect(load).toHaveBeenCalledTimes(2);
|
|
});
|
|
});
|
|
|
|
describe("remote restart lifecycle evidence", () => {
|
|
it("accepts a plugin task with no controller PID and permits completion during the outage", () => {
|
|
const f = fixture();
|
|
const task = { runnerId: "runner", leaseId: "lease", runId: "run", sessionId: "session", turnId: "turn", itemId: "item" };
|
|
const lease = { ...f.leases[0]!, provider: "plugin", metadata: { runnerTaskDispatched: true, runnerTaskIdentity: task } };
|
|
const run = { ...f.active, processPid: null, processStartedAt: null };
|
|
const before = observeRemoteRestartBoundary(run, f.rows, [lease], f.scope);
|
|
const completedLease = { ...lease, status: "released", metadata: {
|
|
...lease.metadata, runnerTaskDispatched: false, runnerTaskExecutionStopped: true,
|
|
} };
|
|
const after = observeRemoteRestartBoundary({ ...run, status: "succeeded" }, f.finalRows, [completedLease], f.scope, true);
|
|
expect(before.identity).toEqual(after.identity);
|
|
expect(before.identity.processPid).toBeNull();
|
|
expect(before.identity.taskIdentitySha256).toMatch(/^[a-f0-9]{64}$/);
|
|
expect(() => observeRemoteRestartBoundary({ ...run, processPid: 123 }, f.rows, [lease], f.scope)).toThrow("not the controller PID");
|
|
for (const runnerTaskExecutionStopped of [undefined, false]) {
|
|
const unconfirmed = { ...completedLease, metadata: { ...completedLease.metadata, runnerTaskExecutionStopped } };
|
|
expect(() => observeRemoteRestartBoundary({ ...run, status: "succeeded" }, f.finalRows, [unconfirmed], f.scope, true))
|
|
.toThrow("admitted plugin task identity");
|
|
}
|
|
const stoppedActiveLease = { ...lease, metadata: completedLease.metadata };
|
|
expect(() => observeRemoteRestartBoundary(run, f.rows, [stoppedActiveLease], f.scope, true))
|
|
.toThrow("admitted plugin task identity");
|
|
const foreignTask = { ...completedLease, metadata: { ...completedLease.metadata, runnerTaskIdentity: { ...task, runId: "replacement" } } };
|
|
expect(() => observeRemoteRestartBoundary({ ...run, status: "succeeded" }, f.finalRows, [foreignTask], f.scope, true))
|
|
.toThrow("admitted plugin task identity");
|
|
});
|
|
it("does not confuse unchanged metadata or HTTP readiness with authenticated recovery", () => {
|
|
const { evidence } = fixture();
|
|
delete evidence.outage.recovery;
|
|
expect(gradeRemoteRestart(evidence).find(check => check.id === "remote-restart-authenticated-recovery")?.passed).toBe(false);
|
|
});
|
|
it("requires every requested immutable replacement instead of counting one restart as rollback", () => {
|
|
const { evidence } = fixture();
|
|
const A = { revision: "a".repeat(40), treeSha256: "a".repeat(64), version: "1.0.0", evidence: "development-build" as const };
|
|
const B = { ...A, revision: "b".repeat(40), treeSha256: "b".repeat(64) };
|
|
const first = { before: evidence.before, after: evidence.after, outage: { ...evidence.outage, previousArtifact: A, replacementArtifact: B } };
|
|
const second = { before: evidence.after, after: evidence.after, outage: { ...evidence.outage, previousArtifact: B, replacementArtifact: A } };
|
|
const grade = (replacements: typeof first[]) => gradeRemoteRestart({ ...evidence, expectedReplacements: 2, replacements })
|
|
.find(check => check.id === "remote-restart-all-replacements")!.passed;
|
|
expect(grade([first])).toBe(false);
|
|
expect(grade([first, second])).toBe(true);
|
|
for (const field of ["runnerInstanceId", "nativeSessionId"] as const) {
|
|
expect(grade([{ ...first, outage: { ...first.outage, recovery: { ...first.outage.recovery!, [field]: "another" } } }, second])).toBe(false);
|
|
}
|
|
for (const field of ["companyId", "issueId"] as const) {
|
|
expect(grade([{ ...first, outage: { ...first.outage, recovery: { ...first.outage.recovery!,
|
|
binding: { ...first.outage.recovery!.binding, [field]: "another" } } } }, second])).toBe(false);
|
|
}
|
|
expect(grade([first, { ...second, outage: { ...second.outage, recovery: undefined } }])).toBe(false);
|
|
expect(grade([first, { ...second, outage: { ...second.outage, previousArtifact: A } }])).toBe(false);
|
|
});
|
|
it("rejects otherwise successful recovery without a complete actual-server fault receipt", () => {
|
|
const { evidence } = fixture();
|
|
for (const outage of [undefined, { ...evidence.outage, replacementPid: 100 },
|
|
{ ...evidence.outage, replacementStartedAfterMs: 1099 }, { ...evidence.outage, cacheDiscarded: false }]) {
|
|
expect(gradeRemoteRestart({ ...evidence, outage }).every(check => check.passed)).toBe(false);
|
|
}
|
|
});
|
|
it("does not accept a successful replacement as proof that the original remote runner recovered", () => {
|
|
const { evidence } = fixture();
|
|
const replacement = { ...evidence.final.run, id: "replacement", runnerInstanceId: "replacement-runner" };
|
|
evidence.final.run = replacement;
|
|
const checks = storyLifecycleChecks({ issues: [{ id: "task", companyId: "company", title: "task", status: "done" }],
|
|
runs: [replacement], parentId: "task", leadId: "lead", requireRemoteRestart: true, remoteRestart: evidence,
|
|
} as Parameters<typeof storyLifecycleChecks>[0]);
|
|
expect(checks.every(check => check.passed)).toBe(false);
|
|
});
|
|
|
|
it("does not accept a terminal native result without its workspace finalization receipt", () => {
|
|
const { evidence } = fixture();
|
|
evidence.final.operations = [];
|
|
const checks = storyLifecycleChecks({ issues: [{ id: "task", companyId: "company", title: "task", status: "done" }],
|
|
runs: [evidence.final.run], parentId: "task", leadId: "lead", requireRemoteRestart: true, remoteRestart: evidence,
|
|
} as Parameters<typeof storyLifecycleChecks>[0]);
|
|
expect(checks.every(check => check.passed)).toBe(false);
|
|
});
|
|
|
|
it("accepts the original live process and lease followed by committed native and workspace receipts", () => {
|
|
const { evidence } = fixture();
|
|
expect(storyLifecycleChecks({ issues: [{ id: "task", companyId: "company", title: "task", status: "done" }],
|
|
runs: [evidence.final.run], parentId: "task", leadId: "lead", requireRemoteRestart: true, remoteRestart: evidence,
|
|
}).every(check => check.passed)).toBe(true);
|
|
});
|
|
|
|
it.each(["runId", "runnerInstanceId", "nativeSessionId", "processPid", "processStartedAt"] as const)("rejects a changed %s after restart", field => {
|
|
const { evidence } = fixture();
|
|
const identity = evidence.after.identity as Record<string, unknown>;
|
|
identity[field] = field === "processPid" ? 43 : "different";
|
|
expect(gradeRemoteRestart(evidence)[0]!.passed).toBe(false);
|
|
});
|
|
|
|
it("rejects a replacement sandbox and changed or missing receipt prefixes", () => {
|
|
const changedLease = fixture().evidence;
|
|
changedLease.after.lease.providerLeaseSha256 = "0".repeat(64);
|
|
expect(gradeRemoteRestart(changedLease)[0]!.passed).toBe(false);
|
|
const changedPrefix = fixture().evidence;
|
|
changedPrefix.after.events[0]!.sha256 = "0".repeat(64);
|
|
expect(gradeRemoteRestart(changedPrefix)[0]!.passed).toBe(false);
|
|
const missingPrefix = fixture().evidence;
|
|
missingPrefix.final.events.shift();
|
|
expect(gradeRemoteRestart(missingPrefix)[1]!.passed).toBe(false);
|
|
});
|
|
|
|
it("rejects failed native tool calls even when the original turn eventually succeeds", () => {
|
|
const { evidence, event } = fixture();
|
|
const rows = [...fixture().afterRows, event(6, "tool.execution.completed", { executionId: "failed-write", status: "failed", output: "private timeout" }),
|
|
event(7, "turn.completed"), event(8, "run.result.accepted", {}, "control_plane", 1), event(9, "run.terminal", {}, "control_plane", 2)];
|
|
evidence.final.events = projectRemoteRestartEvents(evidence.final.run, rows);
|
|
expect(gradeRemoteRestart(evidence).every(check => check.passed)).toBe(false);
|
|
});
|
|
|
|
it("accepts identical activity receipt redelivery without treating it as another submission", () => {
|
|
const { evidence } = fixture();
|
|
evidence.final.events.push({ ...evidence.final.events[1]!, seq: 10, sourceSeq: 10, sourceEventId: "runner:run:10" },
|
|
{ ...evidence.final.events[3]!, seq: 11, sourceSeq: 11, sourceEventId: "runner:run:11" });
|
|
expect(gradeRemoteRestart(evidence).every(check => check.passed)).toBe(true);
|
|
});
|
|
|
|
it("accepts the same start notification after reattachment and rejects a different turn or payload", () => {
|
|
const f = fixture();
|
|
const replay = f.event(5, "turn.started");
|
|
const after = [...f.rows, replay];
|
|
expect(observeRemoteRestartBoundary(f.active, after, f.leases, f.scope, true).identity).toEqual(f.evidence.before.identity);
|
|
expect(() => observeRemoteRestartBoundary(f.active, after, f.leases, f.scope)).toThrow("active original provider turn");
|
|
expect(() => observeRemoteRestartBoundary(f.active, [...f.rows, f.event(5, "turn.started", { changed: true })], f.leases, f.scope, true))
|
|
.toThrow("active original provider turn");
|
|
replay.payload.prpEvent.turnId = "another-turn";
|
|
expect(() => observeRemoteRestartBoundary(f.active, after, f.leases, f.scope, true)).toThrow("active original provider turn");
|
|
});
|
|
|
|
it.each(["replacement", "second submission", "changed tool receipt", "changed start receipt", "missing terminal"])("rejects %s at completion", fault => {
|
|
const { evidence } = fixture();
|
|
if (fault === "replacement") evidence.final.run.id = "replacement";
|
|
if (fault === "second submission") evidence.final.events.push({ ...evidence.final.events[0]!, seq: 10, sourceSeq: 10, sourceEventId: "runner:run:10" });
|
|
if (fault === "changed tool receipt") evidence.final.events.push({ ...evidence.final.events[3]!, seq: 10, sourceSeq: 10, sourceEventId: "runner:run:10", payloadSha256: "0".repeat(64) });
|
|
if (fault === "changed start receipt") evidence.final.events.push({ ...evidence.final.events[1]!, seq: 10, sourceSeq: 10, sourceEventId: "runner:run:10", payloadSha256: "0".repeat(64) });
|
|
if (fault === "missing terminal") evidence.final.events.pop();
|
|
expect(gradeRemoteRestart(evidence)[1]!.passed).toBe(false);
|
|
});
|
|
|
|
it.each(["no receipt", "foreign receipt", "wrong phase", "pending operation", "failed copyback", "retry", "recovery"])("rejects %s after useful work", fault => {
|
|
const { evidence } = fixture();
|
|
if (fault === "no receipt") evidence.final.operations = [];
|
|
if (fault === "foreign receipt") evidence.final.operations[0]!.heartbeatRunId = "other";
|
|
if (fault === "wrong phase") evidence.final.operations[0]!.phase = "workspace_prepare";
|
|
if (fault === "pending operation") evidence.final.operations.push({ ...evidence.final.operations[0]!, status: "running" });
|
|
if (fault === "failed copyback") evidence.final.run.resultJson!.workspaceFinalizeStatus = "failed";
|
|
if (fault === "retry") evidence.final.scheduledRetry = { runId: "run" };
|
|
if (fault === "recovery") evidence.final.recoveryActions.push({ status: "active" });
|
|
expect(gradeRemoteRestart(evidence)[2]!.passed).toBe(false);
|
|
});
|
|
|
|
it("rejects unexercised or foreign boundaries before any restart", () => {
|
|
const f = fixture();
|
|
expect(() => observeRemoteRestartBoundary({ ...f.active, status: "succeeded" }, f.rows, f.leases, f.scope)).toThrow("live native process");
|
|
expect(() => observeRemoteRestartBoundary(f.active, f.rows.slice(0, 3), f.leases, f.scope)).toThrow("completed tool");
|
|
expect(() => observeRemoteRestartBoundary(f.active, f.rows, [{ ...f.leases[0]!, companyId: "foreign" }], f.scope)).toThrow("exact active sandbox");
|
|
const foreign = structuredClone(f.rows);
|
|
foreign[0]!.payload.prpEvent.normalizedSessionId = "foreign";
|
|
expect(() => observeRemoteRestartBoundary(f.active, foreign, f.leases, f.scope)).toThrow("foreign, duplicate or invalid");
|
|
});
|
|
|
|
it("exports hashes instead of provider text, tool output or sandbox IDs", () => {
|
|
const { evidence } = fixture();
|
|
const privateRun = { ...evidence.final.run, error: "private provider failure", runnerProfileJson: { provider: "private-provider" },
|
|
resultJson: { ...evidence.final.run.resultJson, nextAttemptAt: "private-time", failureCode: "private-failure", summary: "private summary" } };
|
|
expect(JSON.stringify(projectRemoteRestartRun(privateRun))).not.toContain("private");
|
|
expect(JSON.stringify(evidence)).not.toContain("private");
|
|
});
|
|
});
|