mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-29 12:59:13 +01:00
test(e2e): require remote restart identity and copyback receipts (#16344)
Require original remote run identity, sandbox lease continuity, native receipt continuity and successful workspace finalization in the Codex Daytona controller-restart cell. Add positive and negative calibrations, including immediate rejection when the original run ends before fault injection. All current-head CI checks pass and Greptile is 5/5. Live Daytona qualification remains separate. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
ff0f976de1
commit
ecc0ab8174
8 files changed
+418
-2
No files matched your search
@@ -157,6 +157,11 @@ permission, and requester scope decisions through saved browser interactions.
|
||||
|
||||
## Validation ladder
|
||||
|
||||
The native Codex Daytona `recover-controller` cell in Everyday Workflows
|
||||
requires active process and lease continuity, native receipt continuity and
|
||||
workspace finalization. See the [restart evidence contract](../tests/runner-e2e/EVERYDAY-WORKFLOWS.md).
|
||||
Catalog discovery and deterministic calibration do not establish live recovery.
|
||||
|
||||
The explicit-only [public MCP suite](../tests/runner-e2e/PUBLIC-MCP.md) evaluates
|
||||
paid assistant delegation, later retrieval, feedback, review, uncertain retries
|
||||
and permission boundaries. It uses the Product E2E fixtures, launcher, evidence
|
||||
|
||||
@@ -47,6 +47,31 @@ restart, a verified first download supplies the persistence checkpoint; the
|
||||
controller is interrupted during a subsequent revision with another queued
|
||||
requirement.
|
||||
|
||||
The native Codex Daytona restart cell now requires a completed native tool
|
||||
receipt while that revision is still running. Before and after restart it pins
|
||||
the same run, normalized session, Runner identity, PID/start time, and active
|
||||
sandbox lease. The original receipt prefix must remain unchanged. A run that
|
||||
ends before either active observation leaves the fault boundary unexercised;
|
||||
it cannot pass from the final ZIP alone. The cell no longer exempts an original
|
||||
Codex remote run that fails with `process_lost` and is replaced.
|
||||
|
||||
Completion requires one original provider turn and control-plane terminal,
|
||||
the same native receipt prefix, no duplicate completed tool execution IDs,
|
||||
a committed run and successful `workspace_finalize` receipt, and no pending
|
||||
workspace operation or recovery. Existing independent ZIP and preserved-download
|
||||
checks still apply. These receipt checks do not prove that arbitrary external
|
||||
effects with different tool IDs cannot repeat.
|
||||
|
||||
`remote-restart-before.json`, `remote-restart-after.json` and
|
||||
`remote-restart-final.json` retain receipt identities and hashes. They exclude
|
||||
provider text, tool arguments, command output, and raw provider sandbox IDs.
|
||||
The suite definition version and harness digest change with this oracle.
|
||||
Its deterministic calibration is not a claim that a live Daytona run passed.
|
||||
|
||||
```sh
|
||||
pnpm test:e2e:runner -- --id everyday-workflows.runner-codex.daytona.recover-controller --max-automatic-retries 0
|
||||
```
|
||||
|
||||
```sh
|
||||
pnpm test:e2e:runner -- --list --suite everyday-workflows
|
||||
pnpm test:e2e:runner -- --suite everyday-workflows --environment local --max-parallel 2
|
||||
|
||||
@@ -32,6 +32,14 @@ changes.
|
||||
|
||||
## GitHub authorization
|
||||
|
||||
The Everyday remote Codex restart oracle reads only the existing public run,
|
||||
event, environment-lease, workspace-operation and recovery APIs. It does not
|
||||
execute commands in a provider sandbox or signal a provider process. Its new
|
||||
evidence projects closed receipt metadata and hashes the provider sandbox ID
|
||||
and full native event content. Provider text, tool arguments and command output
|
||||
do not enter these snapshots. Existing image admission, credentials, billing,
|
||||
retention and cleanup rules continue to apply.
|
||||
|
||||
Set `RUNNER_E2E_ALLOWED_ACTOR_IDS` to a non-empty JSON array of numeric GitHub
|
||||
user IDs. Keep the list equal to the owners of `.github/**` in
|
||||
`.github/CODEOWNERS`. For example, use `[123456,789012]`. Resolve each ID from
|
||||
|
||||
@@ -1367,7 +1367,8 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
|
||||
...(["hire-reuse", "delegate-feedback"].includes(task.id) ? [] : [`everyday-workflows.runner-opencode.local.${task.id}`]),
|
||||
]),
|
||||
],
|
||||
definitionMetadata: { version: 6, instructions: "production", grading: "outcome-and-invariants", scheduling: "explicit-only",
|
||||
definitionMetadata: { version: 7, instructions: "production", grading: "outcome-and-invariants", scheduling: "explicit-only",
|
||||
remoteCodexRestart: "same-active-process-lease-native-prefix-and-workspace-receipt",
|
||||
procedureCases: { ids: ["hire-reuse", "delegate-feedback"], maximumAttemptsPerCell: 1, companyAndLeadBudgetCents: 1_000 } },
|
||||
},
|
||||
{
|
||||
|
||||
@@ -19,6 +19,8 @@ import { submitTaskReply } from "./user-actions.js";
|
||||
import { createTaskFromPromptThroughUi } from "./plan-task-ui.js";
|
||||
import { waitForTaskChatRendered, waitForTaskIdentity } from "./continuation-screenshot.js";
|
||||
import { hasPersistedSource, isSavedSourceCheckpoint } from "./everyday-interruption.js";
|
||||
import { collectRunEvents } from "./run-observations.js";
|
||||
import { gradeRemoteRestart, observeRemoteRestartBoundary, projectRemoteRestartEvents, projectRemoteRestartRun, waitForRemoteRestartBoundary, type RemoteRestartEvidence } from "./everyday-remote-restart.js";
|
||||
import { setupAggregatorFixture } from "./aggregator-fixture.js";
|
||||
import { gradeProviderChoice, gradeProviderOutcome, requireProviderAccessCard } from "./connection-routing-evidence.js";
|
||||
import { setupConnectionReview } from "./connection-reviews.js";
|
||||
@@ -76,6 +78,7 @@ export interface EverydayEvidence {
|
||||
agents: Row[];
|
||||
downloads: Row[];
|
||||
allowedInterruptedRuns: string[];
|
||||
remoteRestart?: RemoteRestartEvidence;
|
||||
}
|
||||
interface Input {
|
||||
page: Page;
|
||||
@@ -165,6 +168,8 @@ export async function runEverydayFlow(input: Input) {
|
||||
let review: Awaited<ReturnType<typeof setupConnectionReview>> | undefined;
|
||||
let project = fixtures.project;
|
||||
const caseId = execution.task.id;
|
||||
const requireRemoteRestart = caseId === "recover-controller" && execution.environment.id === "daytona"
|
||||
&& execution.profile.generation === "native" && execution.profile.provider === "codex";
|
||||
const providerChoice = caseId === "provider-decline" || caseId === "provider-second";
|
||||
const nativeProviderCase = caseId === "provider-native";
|
||||
let aggregatorFixture: Awaited<ReturnType<typeof setupAggregatorFixture>> | undefined;
|
||||
@@ -544,6 +549,19 @@ export async function runEverydayFlow(input: Input) {
|
||||
async function sourceReady() {
|
||||
await recordSource("source-saved-before-interruption", "source-before-interruption.json", true);
|
||||
}
|
||||
async function remoteRestartBoundary(runId?: string) {
|
||||
await refresh();
|
||||
const active = ev.runs.filter(run => run.status === "running" && run.nativeIssueId === parent!.id
|
||||
&& (!runId || run.id === runId));
|
||||
if (active.length !== 1) throw new Error("Remote restart boundary unexercised: the original run is not active");
|
||||
const run = active[0]!;
|
||||
const [rows, leases] = await Promise.all([
|
||||
collectRunEvents<Row>((afterSeq, limit) => api.get(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=${limit}`)),
|
||||
api.get<Row[]>(`/api/environments/${fixtures.environment.id}/leases`),
|
||||
]);
|
||||
return observeRemoteRestartBoundary(run, rows, leases, { companyId: fixtures.company.id,
|
||||
agentId: fixtures.agent.id, issueId: parent!.id, environmentId: fixtures.environment.id });
|
||||
}
|
||||
async function prepareStopBoundary() {
|
||||
// Providers can finish a short first turn before the browser can click
|
||||
// Stop. If that happens, submit one ordinary user follow-up through the
|
||||
@@ -576,6 +594,9 @@ export async function runEverydayFlow(input: Input) {
|
||||
"agentmail-setup-evidence.ts",
|
||||
"everyday-delivery.ts",
|
||||
"everyday-observations.ts",
|
||||
"everyday-remote-restart.ts",
|
||||
"native-event-envelope.ts",
|
||||
"run-observations.ts",
|
||||
"everyday-artifact.py",
|
||||
"user-actions.ts",
|
||||
"runner.spec.ts",
|
||||
@@ -915,10 +936,28 @@ export async function runEverydayFlow(input: Input) {
|
||||
} else {
|
||||
await reply(LATE_REQUIREMENT);
|
||||
note("followup-submitted-before-interruption");
|
||||
ev.allowedInterruptedRuns.push(active.id);
|
||||
if (requireRemoteRestart) {
|
||||
const before = await waitForRemoteRestartBoundary({
|
||||
deadlineAt: input.deadlineAt, load: async () => {
|
||||
await refresh();
|
||||
const original = ev.runs.find(run => run.id === active.id);
|
||||
if (!original || original.status !== "running") return { run: original };
|
||||
const rows = await collectRunEvents<Row>((afterSeq, limit) => api.get(`/api/heartbeat-runs/${active.id}/events?afterSeq=${afterSeq}&limit=${limit}`));
|
||||
if (!rows.some(row => row.payload?.prpEvent?.eventType === "tool.execution.completed" && row.payload.prpEvent.payload?.status === "completed")) return { run: original };
|
||||
return { run: original, boundary: await remoteRestartBoundary(active.id) };
|
||||
} });
|
||||
ev.remoteRestart = { before };
|
||||
await input.evidence("remote-restart-before.json", before);
|
||||
} else ev.allowedInterruptedRuns.push(active.id);
|
||||
await input.restart();
|
||||
note("controller-restarted");
|
||||
await openParent();
|
||||
if (requireRemoteRestart) {
|
||||
const after = await remoteRestartBoundary(active.id);
|
||||
ev.remoteRestart!.after = after;
|
||||
await input.evidence("remote-restart-after.json", after);
|
||||
if (!gradeRemoteRestart(ev.remoteRestart)[0]!.passed) throw new Error("Remote restart changed the original native process, lease or receipt prefix");
|
||||
}
|
||||
}
|
||||
}
|
||||
if (providerChoice) {
|
||||
@@ -1583,6 +1622,30 @@ export async function runEverydayFlow(input: Input) {
|
||||
),
|
||||
"Successful runs retain the native terminal contract.",
|
||||
);
|
||||
if (requireRemoteRestart) {
|
||||
const runId = ev.remoteRestart?.before?.identity.runId;
|
||||
if (!runId) throw new Error("Remote restart has no saved original run boundary");
|
||||
const final = await pollUntil({ label: "original remote run committed with workspace receipt",
|
||||
deadlineAt: input.deadlineAt, load: async () => {
|
||||
const [run, rows, operationRows, recovery, issue] = await Promise.all([
|
||||
api.get<StoryRun>(`/api/heartbeat-runs/${runId}`),
|
||||
collectRunEvents<Row>((afterSeq, limit) => api.get(`/api/heartbeat-runs/${runId}/events?afterSeq=${afterSeq}&limit=${limit}`)),
|
||||
api.get<Row[]>(`/api/heartbeat-runs/${runId}/workspace-operations`),
|
||||
api.get<{ active: Row | null; actions: Row[] }>(`/api/issues/${parent!.id}/recovery-actions`),
|
||||
api.get<StoryIssue>(`/api/issues/${parent!.id}`),
|
||||
]);
|
||||
if (!Array.isArray(operationRows) || !Array.isArray(recovery.actions) || !("active" in recovery)) throw new Error("Remote restart finalization evidence is incomplete");
|
||||
const final = { run: projectRemoteRestartRun(run), events: projectRemoteRestartEvents(run, rows),
|
||||
operations: operationRows.map(operation => ({ companyId: operation.companyId, heartbeatRunId: operation.heartbeatRunId,
|
||||
issueId: operation.issueId, executionWorkspaceId: operation.executionWorkspaceId, phase: operation.phase, status: operation.status })),
|
||||
scheduledRetry: issue.scheduledRetry,
|
||||
recoveryActions: [recovery.active, ...recovery.actions].filter((action): action is Row => action !== null).map(action => ({ status: action.status })) };
|
||||
ev.remoteRestart!.final = final;
|
||||
return final;
|
||||
}, accept: final => gradeRemoteRestart({ ...ev.remoteRestart, final }).every(check => check.passed),
|
||||
reject: final => ["failed", "cancelled", "timed_out"].includes(final.run.status) ? "Original remote run failed after restart" : undefined });
|
||||
await input.evidence("remote-restart-final.json", final);
|
||||
}
|
||||
ev.checks.push(
|
||||
...storyLifecycleChecks({
|
||||
issues: ev.issues as StoryIssue[],
|
||||
@@ -1590,6 +1653,8 @@ export async function runEverydayFlow(input: Input) {
|
||||
parentId: parent!.id,
|
||||
leadId: fixtures.agent.id,
|
||||
allowedInterruptedRuns: ev.allowedInterruptedRuns,
|
||||
requireRemoteRestart,
|
||||
remoteRestart: ev.remoteRestart,
|
||||
}),
|
||||
);
|
||||
check(
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { IssueComment } from "../../packages/shared/src/types/issue.js";
|
||||
import { gradeRemoteRestart, type RemoteRestartEvidence } from "./everyday-remote-restart.js";
|
||||
|
||||
export interface StoryCheck {
|
||||
id: string;
|
||||
@@ -140,6 +141,7 @@ export interface StoryRun {
|
||||
lastOutputSeq?: number | null;
|
||||
errorCode?: string | null;
|
||||
error?: string | null;
|
||||
nativePhase?: string | null;
|
||||
}
|
||||
|
||||
export type StoryArtifactGradeMode = "base" | "separator" | "max-length";
|
||||
@@ -272,6 +274,8 @@ export function storyLifecycleChecks(input: {
|
||||
parentId: string;
|
||||
leadId: string;
|
||||
allowedInterruptedRuns?: string[];
|
||||
requireRemoteRestart?: boolean;
|
||||
remoteRestart?: RemoteRestartEvidence;
|
||||
}): StoryCheck[] {
|
||||
const executed = input.runs.filter((r) => !isStoryWorkspaceDeferral(r));
|
||||
const allowed = input.allowedInterruptedRuns ?? [];
|
||||
@@ -281,6 +285,7 @@ export function storyLifecycleChecks(input: {
|
||||
detail,
|
||||
});
|
||||
return [
|
||||
...(input.requireRemoteRestart ? gradeRemoteRestart(input.remoteRestart) : []),
|
||||
check(
|
||||
"tasks-done",
|
||||
input.issues.length > 0 && input.issues.every((i) => i.status === "done"),
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { storyLifecycleChecks, type StoryRun } from "./everyday-observations.js";
|
||||
import { gradeRemoteRestart, observeRemoteRestartBoundary, projectRemoteRestartEvents, projectRemoteRestartRun, waitForRemoteRestartBoundary } from "./everyday-remote-restart.js";
|
||||
|
||||
const run: StoryRun = {
|
||||
id: "run", companyId: "company", agentId: "lead", nativeIssueId: "task",
|
||||
status: "succeeded", runtimeMode: "native", runnerInstanceId: "runner", nativeSessionId: "session",
|
||||
};
|
||||
|
||||
function fixture() {
|
||||
const active = { ...run, status: "running", processPid: 42, processStartedAt: "2026-10-14T00:00:00.000Z" };
|
||||
const scope = { companyId: "company", agentId: "lead", issueId: "task", environmentId: "environment" };
|
||||
const leases = [{ id: "lease", ...scope, heartbeatRunId: "run", provider: "daytona", providerLeaseId: "private-sandbox",
|
||||
status: "active", releasedAt: null, executionWorkspaceId: "workspace" }];
|
||||
const event = (seq: number, eventType: string, payload: Record<string, unknown> = {}, sourceKind = "runner", sourceSeq = seq) => {
|
||||
const sourceInstanceId = sourceKind === "runner" ? "runner" : "runner:control";
|
||||
return { seq, runId: "run", companyId: "company", eventType, protocolSchemaVersion: 1, payload: { prpEvent: {
|
||||
schema: "paperclip.prp.event.v1", schemaVersion: 1, runId: "run", normalizedSessionId: "session", sourceKind,
|
||||
sourceInstanceId, sourceSeq, sourceEventId: `${sourceInstanceId}:run:${sourceSeq}`, eventType,
|
||||
turnId: eventType === "turn.submitted" ? null : "turn", payload,
|
||||
} } };
|
||||
};
|
||||
const rows = [event(1, "turn.submitted"), event(2, "turn.started"),
|
||||
event(3, "tool.execution.started", { executionId: "tool", status: "running" }),
|
||||
event(4, "tool.execution.completed", { executionId: "tool", status: "completed", output: "private tool output" })];
|
||||
const before = observeRemoteRestartBoundary(active, rows, leases, scope);
|
||||
const afterRows = [...rows, event(5, "item.delta", { text: "private provider text" })];
|
||||
const after = observeRemoteRestartBoundary(active, afterRows, leases, scope);
|
||||
const finished: StoryRun = { ...run, nativePhase: "committed", resultJson: {
|
||||
finalizationPhase: "committed", workspaceFinalizeStatus: "succeeded", nextAttemptAt: null, failureCode: null,
|
||||
} };
|
||||
const finalRows = [...afterRows, event(6, "run.result.proposed"), event(7, "turn.completed"),
|
||||
event(8, "run.result.accepted", {}, "control_plane", 1), event(9, "run.terminal", {}, "control_plane", 2)];
|
||||
const final = { run: finished, events: projectRemoteRestartEvents(finished, finalRows),
|
||||
operations: [{ companyId: "company", heartbeatRunId: "run", issueId: "task", executionWorkspaceId: "workspace",
|
||||
phase: "workspace_finalize", status: "succeeded" }], scheduledRetry: null as unknown, recoveryActions: [] as Record<string, unknown>[] };
|
||||
return { active, scope, leases, event, rows, afterRows, finalRows, evidence: { before, after, final } };
|
||||
}
|
||||
|
||||
describe("remote restart boundary polling", () => {
|
||||
afterEach(() => vi.useRealTimers());
|
||||
|
||||
it.each([undefined, "succeeded", "failed", "cancelled", "timed_out"] as const)("stops immediately when the original run is %s", async status => {
|
||||
vi.useFakeTimers();
|
||||
const now = Date.now();
|
||||
const load = vi.fn(async () => ({ run: status === undefined ? undefined : { ...run, status } }));
|
||||
const outcome = waitForRemoteRestartBoundary({ deadlineAt: now + 60_000, load }).then(
|
||||
() => undefined, error => error as Error,
|
||||
);
|
||||
await vi.runAllTimersAsync();
|
||||
expect((await outcome)?.message).toBe("Stopped waiting for active remote native turn with completed tool receipt: Remote restart boundary unexercised: original run ended before fault injection");
|
||||
expect(load).toHaveBeenCalledTimes(1);
|
||||
expect(Date.now()).toBe(now);
|
||||
});
|
||||
|
||||
it("waits for useful work while the original run remains active", async () => {
|
||||
vi.useFakeTimers();
|
||||
const { active, evidence } = fixture();
|
||||
const load = vi.fn().mockResolvedValueOnce({ run: active })
|
||||
.mockResolvedValueOnce({ run: active, boundary: evidence.before });
|
||||
const waiting = waitForRemoteRestartBoundary({ deadlineAt: Date.now() + 60_000, load });
|
||||
await vi.runAllTimersAsync();
|
||||
expect(await waiting).toEqual(evidence.before);
|
||||
expect(load).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe("remote restart lifecycle evidence", () => {
|
||||
it("does not accept a successful replacement as proof that the original remote runner recovered", () => {
|
||||
const { evidence } = fixture();
|
||||
const replacement = { ...evidence.final.run, id: "replacement", runnerInstanceId: "replacement-runner" };
|
||||
evidence.final.run = replacement;
|
||||
const checks = storyLifecycleChecks({ issues: [{ id: "task", companyId: "company", title: "task", status: "done" }],
|
||||
runs: [replacement], parentId: "task", leadId: "lead", requireRemoteRestart: true, remoteRestart: evidence,
|
||||
} as Parameters<typeof storyLifecycleChecks>[0]);
|
||||
expect(checks.every(check => check.passed)).toBe(false);
|
||||
});
|
||||
|
||||
it("does not accept a terminal native result without its workspace finalization receipt", () => {
|
||||
const { evidence } = fixture();
|
||||
evidence.final.operations = [];
|
||||
const checks = storyLifecycleChecks({ issues: [{ id: "task", companyId: "company", title: "task", status: "done" }],
|
||||
runs: [evidence.final.run], parentId: "task", leadId: "lead", requireRemoteRestart: true, remoteRestart: evidence,
|
||||
} as Parameters<typeof storyLifecycleChecks>[0]);
|
||||
expect(checks.every(check => check.passed)).toBe(false);
|
||||
});
|
||||
|
||||
it("accepts the original live process and lease followed by committed native and workspace receipts", () => {
|
||||
const { evidence } = fixture();
|
||||
expect(storyLifecycleChecks({ issues: [{ id: "task", companyId: "company", title: "task", status: "done" }],
|
||||
runs: [evidence.final.run], parentId: "task", leadId: "lead", requireRemoteRestart: true, remoteRestart: evidence,
|
||||
}).every(check => check.passed)).toBe(true);
|
||||
});
|
||||
|
||||
it.each(["runId", "runnerInstanceId", "nativeSessionId", "processPid", "processStartedAt"] as const)("rejects a changed %s after restart", field => {
|
||||
const { evidence } = fixture();
|
||||
const identity = evidence.after.identity as Record<string, unknown>;
|
||||
identity[field] = field === "processPid" ? 43 : "different";
|
||||
expect(gradeRemoteRestart(evidence)[0]!.passed).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects a replacement sandbox and changed or missing receipt prefixes", () => {
|
||||
const changedLease = fixture().evidence;
|
||||
changedLease.after.lease.providerLeaseSha256 = "0".repeat(64);
|
||||
expect(gradeRemoteRestart(changedLease)[0]!.passed).toBe(false);
|
||||
const changedPrefix = fixture().evidence;
|
||||
changedPrefix.after.events[0]!.sha256 = "0".repeat(64);
|
||||
expect(gradeRemoteRestart(changedPrefix)[0]!.passed).toBe(false);
|
||||
const missingPrefix = fixture().evidence;
|
||||
missingPrefix.final.events.shift();
|
||||
expect(gradeRemoteRestart(missingPrefix)[1]!.passed).toBe(false);
|
||||
});
|
||||
|
||||
it.each(["replacement", "second submission", "duplicate tool receipt", "missing terminal"])("rejects %s at completion", fault => {
|
||||
const { evidence } = fixture();
|
||||
if (fault === "replacement") evidence.final.run.id = "replacement";
|
||||
if (fault === "second submission") evidence.final.events.push({ ...evidence.final.events[0]!, seq: 10, sourceSeq: 10, sourceEventId: "runner:run:10" });
|
||||
if (fault === "duplicate tool receipt") evidence.final.events.push({ ...evidence.final.events[3]!, seq: 10, sourceSeq: 10, sourceEventId: "runner:run:10" });
|
||||
if (fault === "missing terminal") evidence.final.events.pop();
|
||||
expect(gradeRemoteRestart(evidence)[1]!.passed).toBe(false);
|
||||
});
|
||||
|
||||
it.each(["no receipt", "foreign receipt", "wrong phase", "pending operation", "failed copyback", "retry", "recovery"])("rejects %s after useful work", fault => {
|
||||
const { evidence } = fixture();
|
||||
if (fault === "no receipt") evidence.final.operations = [];
|
||||
if (fault === "foreign receipt") evidence.final.operations[0]!.heartbeatRunId = "other";
|
||||
if (fault === "wrong phase") evidence.final.operations[0]!.phase = "workspace_prepare";
|
||||
if (fault === "pending operation") evidence.final.operations.push({ ...evidence.final.operations[0]!, status: "running" });
|
||||
if (fault === "failed copyback") evidence.final.run.resultJson!.workspaceFinalizeStatus = "failed";
|
||||
if (fault === "retry") evidence.final.scheduledRetry = { runId: "run" };
|
||||
if (fault === "recovery") evidence.final.recoveryActions.push({ status: "active" });
|
||||
expect(gradeRemoteRestart(evidence)[2]!.passed).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects unexercised or foreign boundaries before any restart", () => {
|
||||
const f = fixture();
|
||||
expect(() => observeRemoteRestartBoundary({ ...f.active, status: "succeeded" }, f.rows, f.leases, f.scope)).toThrow("live native process");
|
||||
expect(() => observeRemoteRestartBoundary(f.active, f.rows.slice(0, 3), f.leases, f.scope)).toThrow("completed tool");
|
||||
expect(() => observeRemoteRestartBoundary(f.active, f.rows, [{ ...f.leases[0]!, companyId: "foreign" }], f.scope)).toThrow("exact active sandbox");
|
||||
const foreign = structuredClone(f.rows);
|
||||
foreign[0]!.payload.prpEvent.normalizedSessionId = "foreign";
|
||||
expect(() => observeRemoteRestartBoundary(f.active, foreign, f.leases, f.scope)).toThrow("foreign, duplicate or invalid");
|
||||
});
|
||||
|
||||
it("exports hashes instead of provider text, tool output or sandbox IDs", () => {
|
||||
const { evidence } = fixture();
|
||||
const privateRun = { ...evidence.final.run, error: "private provider failure", runnerProfileJson: { provider: "private-provider" },
|
||||
resultJson: { ...evidence.final.run.resultJson, nextAttemptAt: "private-time", failureCode: "private-failure", summary: "private summary" } };
|
||||
expect(JSON.stringify(projectRemoteRestartRun(privateRun))).not.toContain("private");
|
||||
expect(JSON.stringify(evidence)).not.toContain("private");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,155 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { canonicalJson } from "../../packages/shared/src/portability-hash.js";
|
||||
import { isValidNativePrpEnvelope } from "./native-event-envelope.js";
|
||||
import type { StoryCheck, StoryRun } from "./everyday-observations.js";
|
||||
import { pollUntil } from "./api.js";
|
||||
|
||||
type Row = Record<string, unknown>;
|
||||
const record = (value: unknown): Row => value !== null && typeof value === "object" && !Array.isArray(value) ? value as Row : {};
|
||||
const id = (value: unknown): value is string => typeof value === "string" && value.length > 0 && value.length <= 256 && !/[\u0000-\u001f\u007f]/u.test(value);
|
||||
const digest = (value: unknown) => createHash("sha256").update(canonicalJson(value)).digest("hex");
|
||||
|
||||
export interface RemoteRestartEvent {
|
||||
seq: number; sourceSeq: number; sourceEventId: string; eventType: string;
|
||||
sourceKind: "runner" | "control_plane"; turnId: string | null;
|
||||
toolExecutionId: string | null; toolCompleted: boolean; sha256: string;
|
||||
}
|
||||
export interface RemoteRestartBoundary {
|
||||
identity: { runId: string; companyId: string; agentId: string; issueId: string;
|
||||
runnerInstanceId: string; nativeSessionId: string; processPid: number; processStartedAt: string };
|
||||
lease: { id: string; environmentId: string; executionWorkspaceId: string | null; providerLeaseSha256: string };
|
||||
events: RemoteRestartEvent[];
|
||||
}
|
||||
export interface RemoteRestartEvidence {
|
||||
before?: RemoteRestartBoundary;
|
||||
after?: RemoteRestartBoundary;
|
||||
final?: { run: StoryRun; events: RemoteRestartEvent[]; operations: Row[];
|
||||
scheduledRetry: unknown; recoveryActions: Row[] };
|
||||
}
|
||||
|
||||
export async function waitForRemoteRestartBoundary(input: {
|
||||
deadlineAt: number;
|
||||
load: () => Promise<{ run: StoryRun | undefined; boundary?: RemoteRestartBoundary }>;
|
||||
}): Promise<RemoteRestartBoundary> {
|
||||
const observed = await pollUntil({
|
||||
label: "active remote native turn with completed tool receipt",
|
||||
deadlineAt: input.deadlineAt,
|
||||
load: input.load,
|
||||
accept: state => state.run?.status === "running" && state.boundary !== undefined,
|
||||
reject: state => !state.run || state.run.status !== "running"
|
||||
? "Remote restart boundary unexercised: original run ended before fault injection" : undefined,
|
||||
});
|
||||
return observed.boundary!;
|
||||
}
|
||||
|
||||
export function projectRemoteRestartRun(run: StoryRun): StoryRun {
|
||||
return { id: run.id, companyId: run.companyId, agentId: run.agentId, nativeIssueId: run.nativeIssueId,
|
||||
status: run.status, runtimeMode: run.runtimeMode, runnerInstanceId: run.runnerInstanceId, nativeSessionId: run.nativeSessionId,
|
||||
nativePhase: run.nativePhase === "committed" ? "committed" : null,
|
||||
resultJson: { finalizationPhase: run.resultJson?.finalizationPhase === "committed" ? "committed" : null,
|
||||
workspaceFinalizeStatus: run.resultJson?.workspaceFinalizeStatus === "succeeded" ? "succeeded" : null,
|
||||
nextAttemptAt: run.resultJson?.nextAttemptAt == null ? null : "scheduled",
|
||||
failureCode: run.resultJson?.failureCode == null ? null : "present" } };
|
||||
}
|
||||
|
||||
/** Keep only receipt identities and hashes; never retain provider text or tool arguments. */
|
||||
export function projectRemoteRestartEvents(run: StoryRun, rows: readonly Row[]): RemoteRestartEvent[] {
|
||||
const seen = new Set<string>();
|
||||
const cursors = new Map<string, number>();
|
||||
let lastSeq = 0;
|
||||
return rows.filter(row => record(row.payload).prpEvent !== undefined).map(row => {
|
||||
const event = record(record(row.payload).prpEvent);
|
||||
const source = event.sourceInstanceId;
|
||||
const runnerSource = event.sourceKind === "runner" && source === run.runnerInstanceId;
|
||||
const controllerSource = event.sourceKind === "control_plane" && source === `${run.runnerInstanceId}:control`
|
||||
&& ["run.result.accepted", "run.terminal"].includes(String(event.eventType));
|
||||
if (row.companyId !== run.companyId || row.runId !== run.id || !isValidNativePrpEnvelope(event, row.protocolSchemaVersion)
|
||||
|| event.runId !== run.id || event.normalizedSessionId !== run.nativeSessionId || event.eventType !== row.eventType
|
||||
|| (!runnerSource && !controllerSource) || !id(source) || !id(event.sourceEventId)
|
||||
|| !Number.isSafeInteger(row.seq) || Number(row.seq) <= lastSeq || !Number.isSafeInteger(event.sourceSeq)
|
||||
|| Number(event.sourceSeq) <= (cursors.get(source) ?? 0) || seen.has(event.sourceEventId)
|
||||
|| event.sourceEventId !== `${source}:${run.id}:${event.sourceSeq}`) {
|
||||
throw new Error("Remote restart has foreign, duplicate or invalid native receipts");
|
||||
}
|
||||
seen.add(event.sourceEventId); cursors.set(source, Number(event.sourceSeq)); lastSeq = Number(row.seq);
|
||||
const payload = record(event.payload);
|
||||
return { seq: lastSeq, sourceSeq: Number(event.sourceSeq), sourceEventId: event.sourceEventId,
|
||||
eventType: String(event.eventType), sourceKind: event.sourceKind as RemoteRestartEvent["sourceKind"],
|
||||
turnId: id(event.turnId) ? event.turnId : null,
|
||||
toolExecutionId: id(payload.executionId) ? payload.executionId : null,
|
||||
toolCompleted: event.eventType === "tool.execution.completed" && payload.status === "completed",
|
||||
sha256: digest(event) };
|
||||
});
|
||||
}
|
||||
|
||||
export function observeRemoteRestartBoundary(run: StoryRun, rows: readonly Row[], leases: readonly Row[], scope: {
|
||||
companyId: string; agentId: string; issueId: string; environmentId: string;
|
||||
}): RemoteRestartBoundary {
|
||||
if (run.companyId !== scope.companyId || run.agentId !== scope.agentId || run.nativeIssueId !== scope.issueId
|
||||
|| run.status !== "running" || run.runtimeMode !== "native" || !id(run.id) || !id(run.runnerInstanceId)
|
||||
|| !id(run.nativeSessionId) || !Number.isSafeInteger(run.processPid) || Number(run.processPid) <= 0
|
||||
|| !id(run.processStartedAt) || !Number.isFinite(Date.parse(run.processStartedAt))) {
|
||||
throw new Error("Remote restart boundary requires the original live native process");
|
||||
}
|
||||
const owned = leases.filter(lease => lease.companyId === scope.companyId && lease.environmentId === scope.environmentId
|
||||
&& lease.issueId === scope.issueId && lease.heartbeatRunId === run.id && lease.provider === "daytona"
|
||||
&& lease.status === "active" && lease.releasedAt == null && id(lease.id) && id(lease.providerLeaseId));
|
||||
if (owned.length !== 1) throw new Error("Remote restart boundary requires one exact active sandbox lease");
|
||||
const lease = owned[0]!;
|
||||
const events = projectRemoteRestartEvents(run, rows);
|
||||
const starts = events.filter(event => event.eventType === "turn.started" && event.sourceKind === "runner");
|
||||
const completed = events.filter(event => event.toolCompleted && event.sourceKind === "runner" && event.toolExecutionId);
|
||||
if (starts.length !== 1 || !id(starts[0]!.turnId) || completed.length === 0
|
||||
|| completed.some(event => event.turnId !== starts[0]!.turnId)
|
||||
|| events.some(event => ["turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted", "run.terminal"].includes(event.eventType))) {
|
||||
throw new Error("Remote restart boundary has no completed tool in an active original provider turn");
|
||||
}
|
||||
return {
|
||||
identity: { runId: run.id, companyId: scope.companyId, agentId: scope.agentId, issueId: scope.issueId,
|
||||
runnerInstanceId: run.runnerInstanceId, nativeSessionId: run.nativeSessionId,
|
||||
processPid: Number(run.processPid), processStartedAt: run.processStartedAt },
|
||||
lease: { id: String(lease.id), environmentId: scope.environmentId,
|
||||
executionWorkspaceId: typeof lease.executionWorkspaceId === "string" ? lease.executionWorkspaceId : null,
|
||||
providerLeaseSha256: digest(lease.providerLeaseId) },
|
||||
events,
|
||||
};
|
||||
}
|
||||
|
||||
function retainsPrefix(before: readonly RemoteRestartEvent[], after: readonly RemoteRestartEvent[]): boolean {
|
||||
return before.length > 0 && after.length >= before.length && before.every((event, index) => digest(event) === digest(after[index]));
|
||||
}
|
||||
|
||||
export function gradeRemoteRestart(evidence: RemoteRestartEvidence | undefined): StoryCheck[] {
|
||||
const { before, after, final } = evidence ?? {};
|
||||
const check = (id: string, passed: boolean, detail: string) => ({ id, passed, detail });
|
||||
const sameIdentity = Boolean(before && after && digest(before.identity) === digest(after.identity)
|
||||
&& digest(before.lease) === digest(after.lease) && retainsPrefix(before.events, after.events));
|
||||
const run = final?.run;
|
||||
const identity = before?.identity;
|
||||
const finalIdentity = Boolean(identity && run && run.id === identity.runId && run.companyId === identity.companyId
|
||||
&& run.agentId === identity.agentId && run.nativeIssueId === identity.issueId && run.runtimeMode === "native"
|
||||
&& run.runnerInstanceId === identity.runnerInstanceId && run.nativeSessionId === identity.nativeSessionId);
|
||||
const events = final?.events ?? [];
|
||||
const turnId = before?.events.find(event => event.eventType === "turn.started")?.turnId;
|
||||
const once = (kind: string, source: RemoteRestartEvent["sourceKind"]) => events.filter(event => event.eventType === kind && event.sourceKind === source).length === 1;
|
||||
const completedTools = events.filter(event => event.toolCompleted).map(event => event.toolExecutionId);
|
||||
const nativeCompletion = finalIdentity && Boolean(before && after && retainsPrefix(after.events, events))
|
||||
&& once("turn.submitted", "runner") && once("turn.started", "runner") && once("turn.completed", "runner")
|
||||
&& once("run.result.accepted", "control_plane") && once("run.terminal", "control_plane")
|
||||
&& events.filter(event => event.eventType.startsWith("turn.")).every(event => event.turnId === turnId || (event.eventType === "turn.submitted" && event.turnId === null))
|
||||
&& new Set(completedTools).size === completedTools.length;
|
||||
const operations = final?.operations ?? [];
|
||||
const saved = finalIdentity && run?.status === "succeeded" && run.nativePhase === "committed"
|
||||
&& run.resultJson?.finalizationPhase === "committed" && run.resultJson.workspaceFinalizeStatus === "succeeded"
|
||||
&& run.resultJson.nextAttemptAt == null && run.resultJson.failureCode == null
|
||||
&& operations.some(operation => operation.companyId === identity?.companyId && operation.heartbeatRunId === identity?.runId
|
||||
&& operation.issueId === identity?.issueId && operation.phase === "workspace_finalize" && operation.status === "succeeded"
|
||||
&& operation.executionWorkspaceId === before?.lease.executionWorkspaceId)
|
||||
&& !operations.some(operation => ["pending", "queued", "running"].includes(String(operation.status)))
|
||||
&& final?.scheduledRetry == null && !final?.recoveryActions.some(action => ["active", "pending"].includes(String(action.status)));
|
||||
return [
|
||||
check("remote-restart-same-process-and-lease", sameIdentity, "The same run, native session, Runner process and sandbox lease remain live after restart."),
|
||||
check("remote-restart-native-completion-once", nativeCompletion, "The original provider turn completes once and retains its native receipt prefix."),
|
||||
check("remote-restart-files-finalized", Boolean(saved), "The original run commits its workspace receipt and leaves no active operation or recovery."),
|
||||
];
|
||||
}
|
||||
Reference in new issue
Block a user