mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-27 02:44:50 +01:00
fix: continue historical local runs from retained stop evidence
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
1c4bcff2b1
commit
7c246bb34c
9 files changed
+699
-4
No files matched your search
@@ -3,7 +3,7 @@ import { recordNativeLocalProcessStop, hasNativeLocalProcessStop, PROCESS_START_
|
||||
import { remoteTerminationReceipt } from "./remote-execution-termination.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { beforeAll, afterAll, describe, it, expect } from "vitest";
|
||||
import { beforeAll, afterAll, describe, it, expect, vi } from "vitest";
|
||||
import {
|
||||
approvals, issueApprovals, issueThreadInteractions,
|
||||
agentWakeupRequests, agents, companies, createDb, heartbeatRunEvents, heartbeatRuns, issueComments, issueRecoveryActions,
|
||||
@@ -49,6 +49,33 @@ const support = await getEmbeddedPostgresTestSupport();
|
||||
agentId: f.agentId, status: "queued", contextSnapshot: { issueId: f.issueId, previousRunId: result.previousRunId, forceFreshSession: true } });
|
||||
return result;
|
||||
});
|
||||
it("reconciles legacy stop evidence transactionally and never uses it for a newer launch", async () => {
|
||||
const runtime = await import("./native-runtime/native-session-executor.js");
|
||||
const verify = vi.spyOn(runtime, "verifyRetainedLocalProcessStop").mockReturnValue({
|
||||
fingerprint: "verified-retained-snapshot", providerProcessIds: [999999998], controllerPid: 999999999,
|
||||
});
|
||||
try {
|
||||
const f = await seed();
|
||||
await db.update(heartbeatRuns).set({ processPid: null }).where(eq(heartbeatRuns.id, f.sourceRunId));
|
||||
const [environment] = await db.insert(environments).values({ name: "Local legacy", driver: "process" }).returning();
|
||||
await db.insert(environmentLeases).values({ companyId: f.companyId, heartbeatRunId: f.sourceRunId,
|
||||
environmentId: environment.id, provider: "local", status: "failed", leasePolicy: "ephemeral", releasedAt: new Date() });
|
||||
expect(await admit(f, true)).toMatchObject({ previousRunId: f.sourceRunId });
|
||||
expect(await hasNativeLocalProcessStop(db, f.companyId, f.sourceRunId)).toBe(false);
|
||||
expect(await admit(f)).toMatchObject({ previousRunId: f.sourceRunId });
|
||||
expect(await hasNativeLocalProcessStop(db, f.companyId, f.sourceRunId)).toBe(true);
|
||||
expect(await admit(f)).toBeNull();
|
||||
|
||||
const next = await seed();
|
||||
await db.update(heartbeatRuns).set({ processPid: null }).where(eq(heartbeatRuns.id, next.sourceRunId));
|
||||
await appendHeartbeatRunEvent(db, { companyId: next.companyId, runId: next.sourceRunId,
|
||||
agentId: next.agentId, eventType: PROCESS_START_REQUESTED });
|
||||
verify.mockClear();
|
||||
expect(await admit(next)).toBeNull();
|
||||
expect(verify).not.toHaveBeenCalled();
|
||||
} finally { verify.mockRestore(); }
|
||||
});
|
||||
|
||||
it("preserves local stop proof after process metadata is cleared and invalidates it on another launch", async () => {
|
||||
const f = await seed();
|
||||
const [source] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.sourceRunId));
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { hasNativeLocalProcessStop } from "./native-local-process-stop.js";
|
||||
import { hasNativeLocalProcessStop, reconcileLegacyNativeLocalStop } from "./native-local-process-stop.js";
|
||||
import { completeTerminatedRemoteNativeSessionCleanup } from "../vendor/paperclip-runner/index.js";
|
||||
import { hasRemoteTerminationReceipt, remoteLeaseCleanupScope } from "./remote-execution-termination.js";
|
||||
import { z } from "zod";
|
||||
@@ -103,7 +103,8 @@ export async function admitExplicitNativeContinuation(input: {
|
||||
if (!unusedAdmission) {
|
||||
// A missing process identity is not evidence that a provider exited.
|
||||
if (!run.processPid && !run.processGroupId &&
|
||||
!await hasNativeLocalProcessStop(db, companyId, run.id)) return blocked("process_identity_missing", "The previous run has no verified stop record. Paperclip cannot start this message yet.");
|
||||
!await hasNativeLocalProcessStop(db, companyId, run.id) &&
|
||||
!await reconcileLegacyNativeLocalStop(db, run, coordinator, input.dryRun === true)) return blocked("process_identity_missing", "The previous run has no verified stop record. Paperclip cannot start this message yet.");
|
||||
if (run.processPid && !processStopped(run.processPid)) return blocked("process_running", "Waiting for the previous process to stop. Your message will start automatically.");
|
||||
if (run.processGroupId && !processStopped(-run.processGroupId)) return blocked("process_running", "Waiting for the previous process to stop. Your message will start automatically.");
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { and, desc, eq, inArray, isNull } from "drizzle-orm";
|
||||
import { environmentLeases, heartbeatRunEvents, heartbeatRuns, type Db } from "@paperclipai/db";
|
||||
import { environmentLeases, heartbeatRunEvents, heartbeatRuns, nativeRunFinalizations, type Db } from "@paperclipai/db";
|
||||
import { appendHeartbeatRunEvent } from "./heartbeat-run-events.js";
|
||||
|
||||
export const PROCESS_START_REQUESTED = "native.process_start_requested";
|
||||
@@ -58,3 +58,34 @@ export async function hasNativeLocalProcessStop(db: Db, companyId: string, runId
|
||||
.limit(1);
|
||||
return event?.eventType === LOCAL_PROCESS_STOPPED;
|
||||
}
|
||||
|
||||
/** Upgrade old cleared identities only from an exact, closed retained session.
|
||||
* New-format launches must use their normal stop receipt, never this fallback.
|
||||
*/
|
||||
export async function reconcileLegacyNativeLocalStop(
|
||||
db: Db, run: typeof heartbeatRuns.$inferSelect,
|
||||
coordinator: typeof nativeRunFinalizations.$inferSelect | undefined,
|
||||
dryRun: boolean,
|
||||
): Promise<boolean> {
|
||||
if (!coordinator) return false;
|
||||
const [newFormat] = await db.select({ id: heartbeatRunEvents.id }).from(heartbeatRunEvents).where(and(
|
||||
eq(heartbeatRunEvents.companyId, run.companyId), eq(heartbeatRunEvents.runId, run.id),
|
||||
isNull(heartbeatRunEvents.sourceEventId),
|
||||
inArray(heartbeatRunEvents.eventType, [PROCESS_START_REQUESTED, PROCESS_IDENTITY_RECORDED, LOCAL_PROCESS_STOPPED]),
|
||||
)).limit(1);
|
||||
if (newFormat) return false;
|
||||
const leases = await db.select().from(environmentLeases).where(and(
|
||||
eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id),
|
||||
));
|
||||
if (!leases.length || leases.some(lease => lease.provider !== "local" || !lease.releasedAt || lease.cleanupStatus === "failed")) return false;
|
||||
const { verifyRetainedLocalProcessStop } = await import("./native-runtime/native-session-executor.js");
|
||||
const proof = verifyRetainedLocalProcessStop(run, coordinator);
|
||||
if (!proof) return false;
|
||||
if (!dryRun) await appendHeartbeatRunEvent(db, {
|
||||
companyId: run.companyId, runId: run.id, agentId: run.agentId,
|
||||
eventType: LOCAL_PROCESS_STOPPED, stream: "system", level: "info",
|
||||
message: "Verified the stopped local provider from its retained session before continuing the user message.",
|
||||
payload: { ...proof, source: "retained_local_session_v1" },
|
||||
});
|
||||
return true;
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import { retainedLocalProviderProcesses } from "./retained-local-stop.js";
|
||||
import { PROCESS_START_REQUESTED } from "../native-local-process-stop.js";
|
||||
import { remoteLeaseCleanupScope } from "../remote-execution-termination.js";
|
||||
import { resolveConnectorAssignments, isConnectorSkill } from "../connector-runtime.js";
|
||||
@@ -1635,6 +1636,55 @@ function cleanupStateSnapshot(root: string) {
|
||||
};
|
||||
}
|
||||
|
||||
/** Read-only compatibility proof for failed local Codex runs whose old recovery
|
||||
* cleared process metadata. Caller holds the issue/coordinator locks and checks
|
||||
* local leases and the absence of newer server launch records. No old command
|
||||
* is executed, no session is resumed, and no unknown action outcome is changed.
|
||||
*/
|
||||
export function verifyRetainedLocalProcessStop(
|
||||
run: typeof heartbeatRuns.$inferSelect,
|
||||
coordinator: typeof nativeRunFinalizations.$inferSelect,
|
||||
): { fingerprint: string; providerProcessIds: number[]; controllerPid: number } | null {
|
||||
try {
|
||||
if (run.runtimeMode !== "native" || run.status !== "failed" || !run.finishedAt ||
|
||||
run.errorCode !== "native_runner_process_exited" || run.processPid || run.processGroupId ||
|
||||
!run.nativeIssueId || !run.nativeSessionId || !run.runnerInstanceId ||
|
||||
coordinator.companyId !== run.companyId || coordinator.runId !== run.id ||
|
||||
coordinator.issueId !== run.nativeIssueId || coordinator.phase !== "terminal_failure" ||
|
||||
coordinator.leaseOwner || coordinator.resultId || coordinator.nextAttemptAt ||
|
||||
!Number.isSafeInteger(coordinator.controllerPid) || Number(coordinator.controllerPid) <= 0) return null;
|
||||
// A stopped control-plane process cannot mint another reconnect ticket.
|
||||
try { process.kill(coordinator.controllerPid!, 0); return null; }
|
||||
catch (error) { if ((error as NodeJS.ErrnoException).code !== "ESRCH") return null; }
|
||||
const execution = parseNativeExecutionInput(record(run.runnerProfileJson).nativeExecutionInput);
|
||||
if (execution.binding.companyId !== run.companyId || execution.binding.agentId !== run.agentId ||
|
||||
execution.binding.issueId !== run.nativeIssueId || execution.binding.runId !== run.id ||
|
||||
nativeSessionKey(execution) !== run.nativeSessionId || execution.provider.kind !== "codex" ||
|
||||
execution.session.driverKind !== "codex_app_server") return null;
|
||||
const scope = nativeSessionScopeKey(execution);
|
||||
if (activeNativeSessions.has(run.id) || executingRunnerdSessionScopes.has(scope) ||
|
||||
initializingSessionToolAuthorities.has(scope) || warmNativeSessions.has(scope)) return null;
|
||||
// Never migrate, search other scopes, or activate a retained directory here.
|
||||
const root = scopedRunnerdStateRoot(execution);
|
||||
const snapshot = cleanupStateSnapshot(root);
|
||||
const identity = record(snapshot.control.identity);
|
||||
if (identity.runnerInstanceId !== run.runnerInstanceId ||
|
||||
identity.environmentLeaseId !== execution.binding.executionWorkspaceId ||
|
||||
identity.runId !== run.id || identity.normalizedSessionId !== run.nativeSessionId ||
|
||||
typeof identity.turnId !== "string" || typeof identity.itemId !== "string") return null;
|
||||
const processes = retainedLocalProviderProcesses({ snapshot, now: new Date(), identity: {
|
||||
runnerInstanceId: run.runnerInstanceId, environmentLeaseId: execution.binding.executionWorkspaceId,
|
||||
runId: run.id, normalizedSessionId: run.nativeSessionId,
|
||||
turnId: identity.turnId, itemId: identity.itemId,
|
||||
} });
|
||||
if (!processes || !processes.every(cleanupProcessAbsent)) return null;
|
||||
if (cleanupStateSnapshot(root).fingerprint !== snapshot.fingerprint) return null;
|
||||
return { fingerprint: snapshot.fingerprint, providerProcessIds: processes, controllerPid: coordinator.controllerPid! };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function cleanupProcessAbsent(pid: unknown): pid is number {
|
||||
if (
|
||||
process.platform === "win32" ||
|
||||
|
||||
@@ -0,0 +1,359 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { retainedLocalProviderProcesses } from "./retained-local-stop.js";
|
||||
import { validatePrpEvent } from "../../vendor/paperclip-runner/index.js";
|
||||
|
||||
export function retainedStopFixture() {
|
||||
const identity = {
|
||||
runnerInstanceId: "runner-1",
|
||||
environmentLeaseId: "workspace-1",
|
||||
runId: "run-1",
|
||||
normalizedSessionId: "session-1",
|
||||
turnId: "turn-1",
|
||||
itemId: "item-1",
|
||||
};
|
||||
const events = ["intent", "spawned", "intent", "spawned"].map(
|
||||
(phase, index) => {
|
||||
const generation = index < 2 ? 2 : 3;
|
||||
const event = {
|
||||
schema: "paperclip.prp.event.v1",
|
||||
schemaVersion: 1,
|
||||
eventType: "harness.diagnostic",
|
||||
sourceKind: "runner",
|
||||
sourceInstanceId: identity.runnerInstanceId,
|
||||
sourceEventId: `event-${index + 1}`,
|
||||
sourceSeq: index + 1,
|
||||
runId: identity.runId,
|
||||
normalizedSessionId: identity.normalizedSessionId,
|
||||
turnId: identity.turnId,
|
||||
itemId: identity.itemId,
|
||||
priority: 1,
|
||||
emittedAt: "2026-09-11T10:00:00.000Z",
|
||||
payload: {
|
||||
startup: {
|
||||
schema: "paperclip.provider_startup.v1",
|
||||
origin: { ...identity },
|
||||
launchId: `launch-${generation}`,
|
||||
attemptedProcessGeneration: generation,
|
||||
phase,
|
||||
processId: phase === "spawned" ? 999999990 + generation : null,
|
||||
processGroupId: phase === "spawned" ? 999999990 + generation : null,
|
||||
},
|
||||
},
|
||||
};
|
||||
return {
|
||||
sourceSeq: index + 1,
|
||||
envelope: { ...identity, payload: event },
|
||||
};
|
||||
},
|
||||
);
|
||||
const snapshot = {
|
||||
fingerprint: "fixture",
|
||||
fileSha256: ["a", "b", "c"] as const,
|
||||
control: {
|
||||
schema: "paperclip.runner.durable.control-plane-state.v1",
|
||||
identity: { ...identity },
|
||||
tickets: { ticket: { expiresAt: "2026-09-11T10:01:00Z" } },
|
||||
leases: { lease: { expiresAt: "2026-09-11T11:00:00Z" } },
|
||||
commands: [
|
||||
{ type: "session.snapshot", status: "pending", controllerSeq: 14 },
|
||||
],
|
||||
committedEvents: events,
|
||||
ackedSourceSeq: events.length,
|
||||
},
|
||||
runner: {
|
||||
schema: "paperclip.runner.durable.state.v1",
|
||||
...identity,
|
||||
lifecycle: "suspended",
|
||||
pendingTerminalDelivery: null as unknown,
|
||||
pendingProviderCleanup: null as unknown,
|
||||
outbox: [] as unknown[],
|
||||
ackedSourceSeq: events.length,
|
||||
nextSourceSeq: events.length + 1,
|
||||
lastControllerCommandSeq: 13,
|
||||
},
|
||||
provider: {
|
||||
schema: "paperclip.runner.codex-provider-state.v1",
|
||||
lifecycle: "prepared",
|
||||
startupAttempt: null as unknown,
|
||||
providerProcessGeneration: 3,
|
||||
pendingEvents: [] as unknown[],
|
||||
queuedEvents: [] as unknown[],
|
||||
toolBridge: { pending: {} },
|
||||
activeProviderTurnId: null as string | null,
|
||||
ambiguousTurnStartPending: false,
|
||||
},
|
||||
};
|
||||
return { identity, snapshot, now: new Date("2026-09-11T12:00:00Z") };
|
||||
}
|
||||
|
||||
describe("retained local provider inventory", () => {
|
||||
it("recovers every recorded launch from the complete suspended journal", () => {
|
||||
const f = retainedStopFixture();
|
||||
for (const entry of f.snapshot.control.committedEvents)
|
||||
expect(
|
||||
validatePrpEvent(entry.envelope.payload),
|
||||
JSON.stringify(validatePrpEvent(entry.envelope.payload)),
|
||||
).toMatchObject({ ok: true });
|
||||
expect(retainedLocalProviderProcesses(f)).toEqual([999999992, 999999993]);
|
||||
});
|
||||
it.each([
|
||||
"foreign_scope",
|
||||
"foreign_event",
|
||||
"foreign_launch",
|
||||
"live_runner",
|
||||
"pending_delivery",
|
||||
"outbox",
|
||||
"live_provider",
|
||||
"startup_intent",
|
||||
"active_turn",
|
||||
"ambiguous_turn",
|
||||
"ticket",
|
||||
"lease",
|
||||
"missing_event",
|
||||
"new_generation",
|
||||
"missing_spawn",
|
||||
"bad_pid",
|
||||
"wrong_group",
|
||||
"pending_launch",
|
||||
"bad_schema",
|
||||
"missing_credentials",
|
||||
"duplicate_sequence",
|
||||
"provider_cleanup",
|
||||
"uncommitted_provider_event",
|
||||
])("rejects incomplete or unsettled evidence: %s", (kind) => {
|
||||
const f = retainedStopFixture();
|
||||
const { control, runner, provider } = f.snapshot;
|
||||
const event = control.committedEvents[0]!.envelope.payload;
|
||||
const spawned =
|
||||
control.committedEvents[3]!.envelope.payload.payload.startup;
|
||||
switch (kind) {
|
||||
case "uncommitted_provider_event":
|
||||
provider.pendingEvents.push({ eventType: "harness.diagnostic" });
|
||||
break;
|
||||
case "foreign_scope":
|
||||
control.identity.runId = "other";
|
||||
break;
|
||||
case "foreign_event":
|
||||
event.sourceInstanceId = "other";
|
||||
break;
|
||||
case "foreign_launch":
|
||||
event.payload.startup.origin.runId = "other";
|
||||
break;
|
||||
case "live_runner":
|
||||
runner.lifecycle = "ready";
|
||||
break;
|
||||
case "pending_delivery":
|
||||
runner.pendingTerminalDelivery = {};
|
||||
break;
|
||||
case "provider_cleanup":
|
||||
runner.pendingProviderCleanup = {};
|
||||
break;
|
||||
case "outbox":
|
||||
runner.outbox.push({});
|
||||
break;
|
||||
case "live_provider":
|
||||
provider.lifecycle = "turn_active";
|
||||
break;
|
||||
case "startup_intent":
|
||||
provider.startupAttempt = {};
|
||||
break;
|
||||
case "active_turn":
|
||||
provider.activeProviderTurnId = "turn";
|
||||
break;
|
||||
case "ambiguous_turn":
|
||||
provider.ambiguousTurnStartPending = true;
|
||||
break;
|
||||
case "ticket":
|
||||
control.tickets.ticket.expiresAt = "2026-09-12T00:00:00Z";
|
||||
break;
|
||||
case "lease":
|
||||
control.leases.lease.expiresAt = "invalid";
|
||||
break;
|
||||
case "missing_credentials":
|
||||
control.leases = {} as typeof control.leases;
|
||||
break;
|
||||
case "missing_event":
|
||||
control.committedEvents.shift();
|
||||
break;
|
||||
case "new_generation":
|
||||
provider.providerProcessGeneration++;
|
||||
break;
|
||||
case "missing_spawn":
|
||||
spawned.phase = "intent";
|
||||
break;
|
||||
case "bad_pid":
|
||||
spawned.processId = -1;
|
||||
break;
|
||||
case "wrong_group":
|
||||
spawned.processGroupId = 42;
|
||||
break;
|
||||
case "pending_launch":
|
||||
control.commands[0]!.type = "run.attach";
|
||||
break;
|
||||
case "bad_schema":
|
||||
runner.schema = "unknown";
|
||||
break;
|
||||
case "duplicate_sequence":
|
||||
event.sourceSeq = 2;
|
||||
break;
|
||||
}
|
||||
expect(retainedLocalProviderProcesses(f)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("retained local stop host verification", () => {
|
||||
it.each([
|
||||
"stopped",
|
||||
"live_controller",
|
||||
"live_provider",
|
||||
"permission_denied",
|
||||
"foreign_company",
|
||||
"newer_lease",
|
||||
"symlink",
|
||||
"missing_file",
|
||||
])(
|
||||
"checks persisted scope and real host evidence: %s",
|
||||
async (kind) => {
|
||||
const { mkdtempSync, mkdirSync, writeFileSync, rmSync, symlinkSync } =
|
||||
await import("node:fs");
|
||||
const { tmpdir } = await import("node:os");
|
||||
const { join } = await import("node:path");
|
||||
const { createHash } = await import("node:crypto");
|
||||
const { canonicalNativeJson } = await import("./canonical.js");
|
||||
const { verifyRetainedLocalProcessStop } =
|
||||
await import("./native-session-executor.js");
|
||||
const f = retainedStopFixture();
|
||||
const base = mkdtempSync(join(tmpdir(), "retained-stop-"));
|
||||
const previousBase = process.env.PAPERCLIP_RUNNER_STATE_DIR;
|
||||
process.env.PAPERCLIP_RUNNER_STATE_DIR = base;
|
||||
const execution = {
|
||||
schema: "paperclip.native-execution-input.v1",
|
||||
provider: { kind: "codex", model: null },
|
||||
binding: {
|
||||
companyId: "company",
|
||||
agentId: "agent",
|
||||
issueId: "issue",
|
||||
runId: "run-1",
|
||||
executionWorkspaceId: "workspace-1",
|
||||
},
|
||||
task: {
|
||||
identifier: "TEST-1",
|
||||
title: "Continue",
|
||||
description: null,
|
||||
prompt: "Continue",
|
||||
workMode: "standard",
|
||||
},
|
||||
workspace: {
|
||||
cwd: base,
|
||||
repoUrl: null,
|
||||
repoRef: null,
|
||||
branchName: null,
|
||||
},
|
||||
session: {
|
||||
normalizedSessionId: "session-1",
|
||||
driverKind: "codex_app_server",
|
||||
protocolVersion: 1,
|
||||
lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null },
|
||||
},
|
||||
completionContract: {
|
||||
id: "contract",
|
||||
sha256: "sha",
|
||||
schemaVersion: "paperclip.completion-contract.v1",
|
||||
contract: {
|
||||
revision: "1",
|
||||
objective: "Continue",
|
||||
criteria: [{ id: "objective", requirement: "Finish" }],
|
||||
},
|
||||
},
|
||||
interactionResponses: [],
|
||||
credentialBindings: [],
|
||||
};
|
||||
const run = {
|
||||
id: "run-1",
|
||||
companyId: "company",
|
||||
agentId: "agent",
|
||||
nativeIssueId: "issue",
|
||||
runtimeMode: "native",
|
||||
status: "failed",
|
||||
finishedAt: new Date("2026-09-11T11:00:00Z"),
|
||||
errorCode: "native_runner_process_exited",
|
||||
processPid: null,
|
||||
processGroupId: null,
|
||||
nativeSessionId: "session-1",
|
||||
runnerInstanceId: "runner-1",
|
||||
runnerProfileJson: { nativeExecutionInput: execution },
|
||||
} as unknown as Parameters<typeof verifyRetainedLocalProcessStop>[0];
|
||||
const coordinator = {
|
||||
companyId: "company",
|
||||
issueId: "issue",
|
||||
runId: "run-1",
|
||||
phase: "terminal_failure",
|
||||
controllerPid: kind === "live_controller" ? process.pid : 999999999,
|
||||
} as Parameters<typeof verifyRetainedLocalProcessStop>[1];
|
||||
const scope = canonicalNativeJson({
|
||||
schema: "paperclip.native-session-scope.v2",
|
||||
companyId: "company",
|
||||
agentId: "agent",
|
||||
workspace: { kind: "managed", executionWorkspaceId: "workspace-1" },
|
||||
provider: {
|
||||
driverKind: "codex_app_server",
|
||||
identity: { kind: "codex" },
|
||||
},
|
||||
normalizedSessionId: "session-1",
|
||||
});
|
||||
const root = join(base, createHash("sha256").update(scope).digest("hex"));
|
||||
mkdirSync(join(root, "runner"), { recursive: true });
|
||||
mkdirSync(join(root, "control-plane"));
|
||||
if (kind === "live_provider") {
|
||||
const startup =
|
||||
f.snapshot.control.committedEvents[3]!.envelope.payload.payload
|
||||
.startup;
|
||||
startup.processId = process.pid;
|
||||
startup.processGroupId = process.pid;
|
||||
}
|
||||
if (kind === "foreign_company") execution.binding.companyId = "other";
|
||||
if (kind === "newer_lease") coordinator.leaseOwner = "active";
|
||||
for (const [name, value] of [
|
||||
["control-plane/control-plane-state.json", f.snapshot.control],
|
||||
["runner/runner-state.json", f.snapshot.runner],
|
||||
["runner/codex-provider-state.json", f.snapshot.provider],
|
||||
] as const)
|
||||
writeFileSync(join(root, name), JSON.stringify(value));
|
||||
if (kind === "symlink" || kind === "missing_file") {
|
||||
rmSync(join(root, "runner/codex-provider-state.json"));
|
||||
if (kind === "symlink") {
|
||||
writeFileSync(
|
||||
join(base, "provider.json"),
|
||||
JSON.stringify(f.snapshot.provider),
|
||||
);
|
||||
symlinkSync(
|
||||
join(base, "provider.json"),
|
||||
join(root, "runner/codex-provider-state.json"),
|
||||
);
|
||||
}
|
||||
}
|
||||
const kill =
|
||||
kind === "permission_denied"
|
||||
? vi.spyOn(process, "kill").mockImplementation(() => {
|
||||
throw Object.assign(new Error("denied"), { code: "EPERM" });
|
||||
})
|
||||
: null;
|
||||
try {
|
||||
const result = verifyRetainedLocalProcessStop(run, coordinator);
|
||||
if (kind === "stopped")
|
||||
expect(result).toMatchObject({
|
||||
providerProcessIds: [999999992, 999999993],
|
||||
controllerPid: 999999999,
|
||||
});
|
||||
else expect(result).toBeNull();
|
||||
} finally {
|
||||
kill?.mockRestore();
|
||||
if (previousBase === undefined)
|
||||
delete process.env.PAPERCLIP_RUNNER_STATE_DIR;
|
||||
else process.env.PAPERCLIP_RUNNER_STATE_DIR = previousBase;
|
||||
rmSync(base, { recursive: true, force: true });
|
||||
}
|
||||
},
|
||||
30000,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,187 @@
|
||||
import { validatePrpEvent } from "../../vendor/paperclip-runner/index.js";
|
||||
import type {
|
||||
RetainedMaintenanceIdentity,
|
||||
RetainedMaintenanceSnapshot,
|
||||
} from "./native-maintenance-no-launch.js";
|
||||
|
||||
const record = (value: unknown): Record<string, unknown> => {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value))
|
||||
throw new Error("invalid");
|
||||
return value as Record<string, unknown>;
|
||||
};
|
||||
const array = (value: unknown): unknown[] => {
|
||||
if (!Array.isArray(value) || value.length > 100_000)
|
||||
throw new Error("invalid");
|
||||
return value;
|
||||
};
|
||||
function requireProof(value: unknown): asserts value {
|
||||
if (!value) throw new Error("unproven");
|
||||
}
|
||||
const identityKeys = [
|
||||
"runnerInstanceId",
|
||||
"environmentLeaseId",
|
||||
"runId",
|
||||
"normalizedSessionId",
|
||||
"turnId",
|
||||
"itemId",
|
||||
] as const;
|
||||
|
||||
/** Inventory only. The caller must separately verify local host ownership,
|
||||
* terminal coordinator/runner exit, absent processes, and unchanged files.
|
||||
* Missing launch records never count as evidence that a process stopped.
|
||||
*/
|
||||
export function retainedLocalProviderProcesses(input: {
|
||||
snapshot: RetainedMaintenanceSnapshot;
|
||||
identity: RetainedMaintenanceIdentity;
|
||||
now: Date;
|
||||
}): number[] | null {
|
||||
try {
|
||||
const { control, runner, provider } = input.snapshot;
|
||||
const { identity } = input;
|
||||
const now = input.now.getTime();
|
||||
requireProof(Number.isFinite(now));
|
||||
requireProof(
|
||||
control.schema === "paperclip.runner.durable.control-plane-state.v1",
|
||||
);
|
||||
requireProof(runner.schema === "paperclip.runner.durable.state.v1");
|
||||
requireProof(
|
||||
provider.schema === "paperclip.runner.codex-provider-state.v1",
|
||||
);
|
||||
for (const key of identityKeys) {
|
||||
requireProof(
|
||||
typeof identity[key] === "string" && identity[key].length > 0,
|
||||
);
|
||||
requireProof(
|
||||
record(control.identity)[key] === identity[key] &&
|
||||
runner[key] === identity[key],
|
||||
);
|
||||
}
|
||||
requireProof(
|
||||
runner.lifecycle === "suspended" &&
|
||||
runner.pendingTerminalDelivery === null,
|
||||
);
|
||||
requireProof(
|
||||
runner.pendingProviderCleanup == null &&
|
||||
array(runner.outbox).length === 0,
|
||||
);
|
||||
requireProof(
|
||||
provider.lifecycle === "prepared" && provider.startupAttempt === null,
|
||||
);
|
||||
requireProof(
|
||||
array(provider.pendingEvents).length === 0 &&
|
||||
array(provider.queuedEvents).length === 0,
|
||||
);
|
||||
requireProof(
|
||||
Object.keys(record(record(provider.toolBridge).pending)).length === 0,
|
||||
);
|
||||
requireProof(
|
||||
provider.activeProviderTurnId === null &&
|
||||
provider.ambiguousTurnStartPending === false,
|
||||
);
|
||||
// An old controller or a still-valid reconnect credential must not revive
|
||||
// this authority after the process inventory has been checked.
|
||||
for (const name of ["tickets", "leases"]) {
|
||||
const credentials = Object.values(record(control[name]));
|
||||
requireProof(credentials.length > 0 && credentials.length <= 4096);
|
||||
for (const raw of credentials) {
|
||||
const expiry = Date.parse(String(record(raw).expiresAt));
|
||||
requireProof(Number.isFinite(expiry) && expiry <= now);
|
||||
}
|
||||
}
|
||||
for (const raw of array(control.commands)) {
|
||||
const command = record(raw);
|
||||
requireProof(
|
||||
["completed", "failed", "pending"].includes(String(command.status)),
|
||||
);
|
||||
if (command.status === "pending") {
|
||||
requireProof(
|
||||
["session.snapshot", "runner.suspend"].includes(String(command.type)),
|
||||
);
|
||||
requireProof(
|
||||
Number(command.controllerSeq) >
|
||||
Number(runner.lastControllerCommandSeq),
|
||||
);
|
||||
}
|
||||
}
|
||||
const events = array(control.committedEvents);
|
||||
requireProof(events.length > 0 && control.ackedSourceSeq === events.length);
|
||||
requireProof(
|
||||
runner.ackedSourceSeq === events.length &&
|
||||
runner.nextSourceSeq === events.length + 1,
|
||||
);
|
||||
const launches = new Map<
|
||||
string,
|
||||
{ generation: number; pid: number | null }
|
||||
>();
|
||||
let lastGeneration = 0;
|
||||
for (const [index, raw] of events.entries()) {
|
||||
const wrapper = record(raw);
|
||||
const envelope = record(wrapper.envelope);
|
||||
const event = record(envelope.payload);
|
||||
requireProof(validatePrpEvent(event).ok);
|
||||
for (const key of identityKeys)
|
||||
requireProof(envelope[key] === identity[key]);
|
||||
requireProof(
|
||||
event.runId === identity.runId &&
|
||||
event.normalizedSessionId === identity.normalizedSessionId,
|
||||
);
|
||||
requireProof(
|
||||
event.sourceKind === "runner" &&
|
||||
event.sourceInstanceId === identity.runnerInstanceId,
|
||||
);
|
||||
requireProof(
|
||||
event.turnId === identity.turnId && event.itemId === identity.itemId,
|
||||
);
|
||||
requireProof(
|
||||
event.sourceSeq === index + 1 && wrapper.sourceSeq === event.sourceSeq,
|
||||
);
|
||||
const startupValue = record(event.payload).startup;
|
||||
if (startupValue === undefined) continue;
|
||||
const startup = record(startupValue);
|
||||
requireProof(
|
||||
event.eventType === "harness.diagnostic" &&
|
||||
startup.schema === "paperclip.provider_startup.v1",
|
||||
);
|
||||
for (const key of identityKeys.filter(
|
||||
(key) => key !== "environmentLeaseId",
|
||||
)) {
|
||||
requireProof(record(startup.origin)[key] === identity[key]);
|
||||
}
|
||||
const launchId = startup.launchId;
|
||||
const generation = startup.attemptedProcessGeneration;
|
||||
requireProof(typeof launchId === "string" && launchId.length > 0);
|
||||
requireProof(Number.isSafeInteger(generation) && Number(generation) > 0);
|
||||
if (startup.phase === "intent") {
|
||||
requireProof(
|
||||
!launches.has(launchId) && Number(generation) > lastGeneration,
|
||||
);
|
||||
requireProof(
|
||||
startup.processId === null && startup.processGroupId === null,
|
||||
);
|
||||
launches.set(launchId, { generation: Number(generation), pid: null });
|
||||
lastGeneration = Number(generation);
|
||||
} else {
|
||||
requireProof(startup.phase === "spawned");
|
||||
const launch = launches.get(launchId);
|
||||
requireProof(
|
||||
launch && launch.generation === generation && launch.pid === null,
|
||||
);
|
||||
requireProof(
|
||||
Number.isSafeInteger(startup.processId) &&
|
||||
Number(startup.processId) > 0,
|
||||
);
|
||||
requireProof(startup.processGroupId === startup.processId);
|
||||
launch.pid = Number(startup.processId);
|
||||
}
|
||||
}
|
||||
requireProof(
|
||||
launches.size > 0 &&
|
||||
provider.providerProcessGeneration === lastGeneration,
|
||||
);
|
||||
const processes = [...launches.values()].map((launch) => launch.pid);
|
||||
requireProof(processes.every((pid): pid is number => pid !== null));
|
||||
return [...new Set(processes)];
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user