mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-13 13:38:01 +02:00
fix(runtime): verify retired computer owners after warm expiry
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
ff26b3af9b
commit
d6363b41d4
2 files changed
+111
No files matched your search
@@ -28,6 +28,7 @@ import { inspect } from "node:util";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
heartbeatRuns,
|
||||
environmentLeases,
|
||||
heartbeatRunEvents,
|
||||
issues,
|
||||
nativeRunFinalizations,
|
||||
@@ -82,6 +83,9 @@ vi.mock("node:fs", async (importOriginal) => {
|
||||
return { ...actual, readFileSync: vi.fn(actual.readFileSync) };
|
||||
});
|
||||
|
||||
const computerRetirement = vi.hoisted(() => ({ isRetired: vi.fn() }));
|
||||
vi.mock("../../modules/computers/index.js", () => ({ computerService: () => computerRetirement }));
|
||||
|
||||
const githubAccess = vi.hoisted(() => ({
|
||||
activate: vi.fn((_binding: { runId: string }) => vi.fn()),
|
||||
stop: vi.fn(async () => undefined),
|
||||
@@ -13438,6 +13442,65 @@ describe("runnerd provider runtime wiring", () => {
|
||||
} finally { await rm(root, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ condition: "retired", expected: "verified" },
|
||||
{ condition: "uncertain", expected: "terminal_state_indeterminate" },
|
||||
{ condition: "stale-generation", expected: "terminal_state_indeterminate" },
|
||||
{ condition: "wrong-placement", expected: "scope_mismatch" },
|
||||
{ condition: "wrong-lease", expected: "scope_mismatch" },
|
||||
{ condition: "active-run", expected: "active" },
|
||||
{ condition: "unavailable", expected: "unavailable" },
|
||||
{ condition: "corrupt-local", expected: "terminal_state_indeterminate" },
|
||||
])("verifies remote idle retirement before a fresh same-session turn ($condition)", async ({ condition, expected }) => {
|
||||
const root = await mkdtemp(join(tmpdir(), "paperclip-computer-idle-retirement-"));
|
||||
const prior = execution;
|
||||
const current = { ...execution, binding: { ...execution.binding, runId: "after-idle" } };
|
||||
const owner = { computerId: "computer", ownerId: "retired-owner", generation: 2, listenerPort: 43127 };
|
||||
const descriptor = { kind: "remote-persistent", leaseId: "prior-lease", providerLeaseId: owner.ownerId,
|
||||
remoteCwd: "/remote/agent", computerId: owner.computerId, ownerGeneration: owner.generation,
|
||||
listenerPort: owner.listenerPort, placementId: "placement" };
|
||||
const db = { select: () => ({ from: (table: unknown) => ({ where: () => ({ limit: async () =>
|
||||
table === environmentLeases ? condition === "wrong-lease" ? [] : [{
|
||||
providerLeaseId: owner.ownerId,
|
||||
metadata: { agentId: prior.binding.agentId, computerOwner: { ...owner, generation: 1 } },
|
||||
}] : [{ status: condition === "active-run" ? "running" : "succeeded",
|
||||
runnerProfileJson: { nativeExecutionInput: prior, nativeComputerWorkspace: descriptor } }],
|
||||
}) }) }) } as unknown as Db;
|
||||
computerRetirement.isRetired.mockReset().mockImplementation(async ({ owner: requested }) => {
|
||||
if (condition === "unavailable") throw new Error("ledger unavailable");
|
||||
return condition !== "uncertain" && requested.generation === (condition === "stale-generation" ? 3 : 2);
|
||||
});
|
||||
const input = { db, root, execution: current, identity: { runId: prior.binding.runId,
|
||||
normalizedSessionId: prior.session.normalizedSessionId!, runnerInstanceId: "prior-runner",
|
||||
environmentLeaseId: condition === "wrong-lease" ? "other-lease" : "initial-physical-lease" },
|
||||
allowVerifiedBackup: false, remoteRunnerState: true,
|
||||
runnerExecutionTarget: { kind: "remote", transport: "computer", environmentId: "environment",
|
||||
remoteCwd: "/remote/agent", resourceAuthority: { ...owner, ownerId: "successor-owner", generation: 1 },
|
||||
fileAuthority: { placementId: condition === "wrong-placement" ? "other-placement" : "placement" } } as never };
|
||||
try {
|
||||
if (condition === "corrupt-local") {
|
||||
await mkdir(join(root, "runner"));
|
||||
await writeFile(join(root, "runner", "runner-state.json"), "invalid state");
|
||||
}
|
||||
if (condition === "retired") {
|
||||
// Both live warm handoffs use their retained owner. After idle expiry,
|
||||
// the next turn must instead prove the exact generation was retired.
|
||||
for (let turn = 0; turn < 2; turn += 1) {
|
||||
expect(await verifyPriorRunnerdStateForSessionScope({ ...input, allowRetainedWarmRunner: true }))
|
||||
.toBe("retained_warm_runner");
|
||||
}
|
||||
expect(computerRetirement.isRetired).not.toHaveBeenCalled();
|
||||
}
|
||||
expect(await verifyPriorRunnerdStateForSessionScope({ ...input, allowRetainedWarmRunner: false })).toBe(expected);
|
||||
if (["retired", "uncertain", "stale-generation", "unavailable"].includes(condition)) {
|
||||
expect(computerRetirement.isRetired).toHaveBeenCalledExactlyOnceWith({
|
||||
companyId: current.binding.companyId, environmentId: "environment", agentId: current.binding.agentId,
|
||||
runId: prior.binding.runId, owner,
|
||||
});
|
||||
} else expect(computerRetirement.isRetired).not.toHaveBeenCalled();
|
||||
} finally { await rm(root, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ override: undefined, expected: "@openai/codex@0.160.0" },
|
||||
{ override: "@openai/codex@0.159.0", expected: "@openai/codex@0.159.0" },
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { computerService } from "../../modules/computers/index.js";
|
||||
import { readNativeComputerWorkspaceReference } from "./native-workspace-sync.js";
|
||||
import { adapterExecutionTargetIsCommandBacked, type AdapterComputerExecutionTarget } from "@paperclipai/adapter-utils/execution-target";
|
||||
import { configuredEnvironment } from "../../vendor/paperclip-runner/index.js";
|
||||
import { agents } from "@paperclipai/db";
|
||||
@@ -4295,6 +4297,7 @@ export async function verifyPriorRunnerdStateForSessionScope(input: {
|
||||
allowVerifiedBackup: boolean;
|
||||
allowRetainedWarmRunner: boolean;
|
||||
remoteRunnerState?: boolean;
|
||||
runnerExecutionTarget?: AdapterExecutionTarget | null;
|
||||
}): Promise<PriorRunnerdStateVerification> {
|
||||
let priorRun: {
|
||||
status: string;
|
||||
@@ -4352,6 +4355,49 @@ export async function verifyPriorRunnerdStateForSessionScope(input: {
|
||||
) {
|
||||
return "retained_warm_runner";
|
||||
}
|
||||
const target = input.runnerExecutionTarget;
|
||||
if (input.remoteRunnerState && directLifecycle === "absent" &&
|
||||
target?.kind === "remote" && target.transport === "computer" && target.environmentId) {
|
||||
const descriptor = record(priorRun.runnerProfileJson).nativeComputerWorkspace;
|
||||
const priorWorkspace = readNativeComputerWorkspaceReference(descriptor);
|
||||
if (!priorWorkspace || priorWorkspace.computerOwner.computerId !== target.resourceAuthority.computerId ||
|
||||
priorWorkspace.remoteCwd !== target.remoteCwd ||
|
||||
record(descriptor).placementId !== target.fileAuthority.placementId) return "scope_mismatch";
|
||||
// Persistent runners keep their state remotely. Once the exact prior
|
||||
// owner has completed retirement, its terminal run and fenced process
|
||||
// are authoritative even though no local runner-state.json exists.
|
||||
// This read cannot retire a live owner or accept a newer generation.
|
||||
try {
|
||||
// Warm turns advance the attempt lease while the physical runner keeps
|
||||
// its initial PRP lease identity. Verify that original lease belongs to
|
||||
// this exact owner lineage instead of requiring the latest attempt id.
|
||||
const [physicalLease] = await input.db.select({
|
||||
providerLeaseId: environmentLeases.providerLeaseId,
|
||||
metadata: environmentLeases.metadata,
|
||||
}).from(environmentLeases).where(and(
|
||||
eq(environmentLeases.id, input.identity.environmentLeaseId),
|
||||
eq(environmentLeases.companyId, input.execution.binding.companyId),
|
||||
eq(environmentLeases.environmentId, target.environmentId),
|
||||
)).limit(1);
|
||||
const leaseMetadata = record(physicalLease?.metadata);
|
||||
const leaseOwner = record(leaseMetadata.computerOwner);
|
||||
if (physicalLease?.providerLeaseId !== priorWorkspace.computerOwner.ownerId ||
|
||||
leaseMetadata.agentId !== input.execution.binding.agentId ||
|
||||
leaseOwner.computerId !== priorWorkspace.computerOwner.computerId ||
|
||||
leaseOwner.ownerId !== priorWorkspace.computerOwner.ownerId ||
|
||||
!Number.isSafeInteger(leaseOwner.generation) || Number(leaseOwner.generation) < 1 ||
|
||||
Number(leaseOwner.generation) > priorWorkspace.computerOwner.generation) return "scope_mismatch";
|
||||
if (await computerService(input.db).isRetired({
|
||||
companyId: input.execution.binding.companyId,
|
||||
environmentId: target.environmentId,
|
||||
agentId: input.execution.binding.agentId,
|
||||
runId: input.identity.runId,
|
||||
owner: priorWorkspace.computerOwner,
|
||||
})) return "verified";
|
||||
} catch {
|
||||
return "unavailable";
|
||||
}
|
||||
}
|
||||
return "terminal_state_indeterminate";
|
||||
} catch {
|
||||
return "authority_indeterminate";
|
||||
@@ -4896,6 +4942,7 @@ async function migrateRunnerdStateRootForExecution(input: {
|
||||
allowRetainedWarmRunner: input.allowRetainedWarmRunner,
|
||||
remoteRunnerState: input.runnerExecutionTarget?.kind === "remote" &&
|
||||
input.runnerExecutionTarget.transport === "computer",
|
||||
runnerExecutionTarget: input.runnerExecutionTarget,
|
||||
});
|
||||
if (
|
||||
verification !== "verified" &&
|
||||
@@ -4940,6 +4987,7 @@ async function migrateRunnerdStateRootForExecution(input: {
|
||||
allowRetainedWarmRunner: input.allowRetainedWarmRunner,
|
||||
remoteRunnerState: input.runnerExecutionTarget?.kind === "remote" &&
|
||||
input.runnerExecutionTarget.transport === "computer",
|
||||
runnerExecutionTarget: input.runnerExecutionTarget,
|
||||
});
|
||||
if (
|
||||
verification !== "verified" &&
|
||||
|
||||
Reference in new issue
Block a user