mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-14 08:26:46 +02:00
fix: verify original workspace lineage after warm retirement
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
43b818a39a
commit
d3fc9db89e
2 files changed
+62
-13
No files matched your search
@@ -13457,6 +13457,13 @@ describe("runnerd provider runtime wiring", () => {
|
||||
it.each([
|
||||
{ condition: "retired", expected: "verified" },
|
||||
{ condition: "workspace-coordinate", expected: "verified" },
|
||||
{ condition: "warm-origin", expected: "verified" },
|
||||
{ condition: "origin-other-session", expected: "scope_mismatch" },
|
||||
{ condition: "origin-other-owner", expected: "scope_mismatch" },
|
||||
{ condition: "origin-newer-generation", expected: "scope_mismatch" },
|
||||
{ condition: "origin-active", expected: "scope_mismatch" },
|
||||
{ condition: "origin-ambiguous", expected: "scope_mismatch" },
|
||||
{ condition: "origin-wrong-lease-run", expected: "scope_mismatch" },
|
||||
{ condition: "wrong-workspace-run", expected: "scope_mismatch" },
|
||||
{ condition: "uncertain", expected: "terminal_state_indeterminate" },
|
||||
{ condition: "stale-generation", expected: "terminal_state_indeterminate" },
|
||||
@@ -13467,20 +13474,28 @@ describe("runnerd provider runtime wiring", () => {
|
||||
{ condition: "corrupt-local", expected: "terminal_state_indeterminate" },
|
||||
])("verifies remote idle retirement before a fresh same-session turn ($condition)", async ({ condition, expected }) => {
|
||||
const root = await mkdtemp(join(tmpdir(), "paperclip-computer-idle-retirement-"));
|
||||
const prior = execution;
|
||||
const current = { ...execution, binding: { ...execution.binding, runId: "after-idle" } };
|
||||
const originCase = condition === "warm-origin" || condition.startsWith("origin-");
|
||||
const prior = originCase ? { ...execution, binding: { ...execution.binding, executionWorkspaceId: execution.binding.runId } } : execution;
|
||||
const current = { ...prior, binding: { ...prior.binding, runId: "after-idle", ...(originCase ? { executionWorkspaceId: "after-idle" } : {}) } };
|
||||
const owner = { computerId: "computer", ownerId: "retired-owner", generation: 2, listenerPort: 43127 };
|
||||
const descriptor = { kind: "remote-persistent", leaseId: "prior-lease", providerLeaseId: owner.ownerId,
|
||||
remoteCwd: "/remote/agent", computerId: owner.computerId, ownerGeneration: owner.generation,
|
||||
listenerPort: owner.listenerPort, placementId: "placement" };
|
||||
let leaseQuery: SQL | undefined;
|
||||
const originExecution = { ...prior, binding: { ...prior.binding, runId: "origin-workspace", executionWorkspaceId: "origin-workspace" },
|
||||
session: { ...prior.session, normalizedSessionId: condition === "origin-other-session" ? "other-session" : prior.session.normalizedSessionId } };
|
||||
const originDescriptor = { ...descriptor, leaseId: "origin-lease",
|
||||
providerLeaseId: condition === "origin-other-owner" ? "other-owner" : owner.ownerId,
|
||||
ownerGeneration: condition === "origin-newer-generation" ? 3 : 1 };
|
||||
const originRow = { status: condition === "origin-active" ? "running" : "succeeded",
|
||||
runnerProfileJson: { nativeExecutionInput: originExecution, nativeComputerWorkspace: originDescriptor } };
|
||||
const workspaceCoordinate = ["workspace-coordinate", "wrong-workspace-run"].includes(condition);
|
||||
const db = { select: () => ({ from: (table: unknown) => ({ where: (query: SQL) => { if (table === environmentLeases) leaseQuery = query; return ({ limit: async () =>
|
||||
const db = { select: () => ({ from: (table: unknown) => ({ where: (query: SQL) => { if (table === environmentLeases) leaseQuery = query; return ({ limit: async (limit: number) =>
|
||||
table === environmentLeases ? condition === "wrong-lease" ? [] : [{
|
||||
providerLeaseId: owner.ownerId,
|
||||
heartbeatRunId: condition === "wrong-workspace-run" ? "unrelated-run" : prior.binding.runId,
|
||||
heartbeatRunId: condition === "wrong-workspace-run" || condition === "origin-wrong-lease-run" ? "unrelated-run" : originCase ? "origin-workspace" : prior.binding.runId,
|
||||
metadata: { agentId: prior.binding.agentId, computerOwner: { ...owner, generation: 1 } },
|
||||
}] : [{ status: condition === "active-run" ? "running" : "succeeded",
|
||||
}] : limit === 2 ? (originCase ? condition === "origin-ambiguous" ? [originRow, originRow] : [originRow] : []) : [{ status: condition === "active-run" ? "running" : "succeeded",
|
||||
runnerProfileJson: { nativeExecutionInput: prior, nativeComputerWorkspace: descriptor } }],
|
||||
}); } }) }) } as unknown as Db;
|
||||
computerRetirement.isRetired.mockReset().mockImplementation(async ({ owner: requested }) => {
|
||||
@@ -13489,7 +13504,7 @@ describe("runnerd provider runtime wiring", () => {
|
||||
});
|
||||
const input = { db, root, execution: current, identity: { runId: prior.binding.runId,
|
||||
normalizedSessionId: prior.session.normalizedSessionId!, runnerInstanceId: "prior-runner",
|
||||
environmentLeaseId: workspaceCoordinate ? prior.binding.executionWorkspaceId : condition === "wrong-lease" ? "other-lease" : "initial-physical-lease" },
|
||||
environmentLeaseId: originCase ? "origin-workspace" : workspaceCoordinate ? prior.binding.executionWorkspaceId : condition === "wrong-lease" ? "other-lease" : "initial-physical-lease" },
|
||||
allowVerifiedBackup: false, remoteRunnerState: true,
|
||||
runnerExecutionTarget: { kind: "remote", transport: "computer", environmentId: "environment",
|
||||
remoteCwd: "/remote/agent", resourceAuthority: { ...owner, ownerId: "successor-owner", generation: 1 },
|
||||
@@ -13509,12 +13524,17 @@ describe("runnerd provider runtime wiring", () => {
|
||||
expect(computerRetirement.isRetired).not.toHaveBeenCalled();
|
||||
}
|
||||
expect(await verifyPriorRunnerdStateForSessionScope({ ...input, allowRetainedWarmRunner: false })).toBe(expected);
|
||||
if (["retired", "workspace-coordinate", "uncertain", "stale-generation", "unavailable"].includes(condition)) {
|
||||
if (["retired", "workspace-coordinate", "warm-origin", "uncertain", "stale-generation", "unavailable"].includes(condition)) {
|
||||
expect(computerRetirement.isRetired).toHaveBeenCalledExactlyOnceWith({
|
||||
companyId: current.binding.companyId, environmentId: "environment", agentId: current.binding.agentId,
|
||||
runId: prior.binding.runId, owner,
|
||||
});
|
||||
} else expect(computerRetirement.isRetired).not.toHaveBeenCalled();
|
||||
if (condition === "warm-origin") {
|
||||
const query = new PgDialect().sqlToQuery(leaseQuery!);
|
||||
expect(query.params).toContain("origin-lease");
|
||||
expect(query.params).not.toContain("origin-workspace");
|
||||
}
|
||||
if (workspaceCoordinate) {
|
||||
const query = new PgDialect().sqlToQuery(leaseQuery!);
|
||||
expect(query.params).toContain(descriptor.leaseId);
|
||||
|
||||
@@ -4389,11 +4389,40 @@ export async function verifyPriorRunnerdStateForSessionScope(input: {
|
||||
// environmentLeaseId. Computer placement records the actual lease. Only
|
||||
// this exact prior workspace coordinate may resolve through that record.
|
||||
// Explicit physical lease identities still preserve their warm lineage.
|
||||
const workspaceCoordinate = input.identity.environmentLeaseId ===
|
||||
priorExecution.binding.executionWorkspaceId;
|
||||
const physicalLeaseId = workspaceCoordinate
|
||||
? priorWorkspace.leaseId
|
||||
: input.identity.environmentLeaseId;
|
||||
let workspaceRunId: string | null = input.identity.environmentLeaseId ===
|
||||
priorExecution.binding.executionWorkspaceId ? input.identity.runId : null;
|
||||
let physicalLeaseId = workspaceRunId ? priorWorkspace.leaseId : input.identity.environmentLeaseId;
|
||||
if (!workspaceRunId) {
|
||||
// Warm attachment advances runId while retaining the process's original
|
||||
// workspace coordinate. Resolve that recorded origin, never an arbitrary
|
||||
// run ID, before checking the physical lease and latest retirement.
|
||||
const origins = await input.db.select({
|
||||
status: heartbeatRuns.status,
|
||||
runnerProfileJson: heartbeatRuns.runnerProfileJson,
|
||||
}).from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.companyId, input.execution.binding.companyId),
|
||||
eq(heartbeatRuns.agentId, input.execution.binding.agentId),
|
||||
eq(sql<string>`${heartbeatRuns.runnerProfileJson}->'nativeExecutionInput'->'binding'->>'executionWorkspaceId'`,
|
||||
input.identity.environmentLeaseId),
|
||||
)).limit(2);
|
||||
if (origins.length > 1) return "scope_mismatch";
|
||||
if (origins.length === 1) {
|
||||
const origin = origins[0]!;
|
||||
const originProfile = record(origin.runnerProfileJson);
|
||||
const originExecution = parseNativeExecutionInput(originProfile.nativeExecutionInput);
|
||||
const originWorkspace = readNativeComputerWorkspaceReference(originProfile.nativeComputerWorkspace);
|
||||
if (!TERMINAL_HEARTBEAT_RUN_STATUSES.has(origin.status) ||
|
||||
originExecution.binding.executionWorkspaceId !== input.identity.environmentLeaseId ||
|
||||
nativeSessionScopeKey(originExecution) !== nativeSessionScopeKey(priorExecution) ||
|
||||
!originWorkspace || originWorkspace.remoteCwd !== priorWorkspace.remoteCwd ||
|
||||
record(originProfile.nativeComputerWorkspace).placementId !== record(descriptor).placementId ||
|
||||
originWorkspace.computerOwner.computerId !== priorWorkspace.computerOwner.computerId ||
|
||||
originWorkspace.computerOwner.ownerId !== priorWorkspace.computerOwner.ownerId ||
|
||||
originWorkspace.computerOwner.generation > priorWorkspace.computerOwner.generation) return "scope_mismatch";
|
||||
workspaceRunId = originExecution.binding.runId;
|
||||
physicalLeaseId = originWorkspace.leaseId;
|
||||
}
|
||||
}
|
||||
const [physicalLease] = await input.db.select({
|
||||
providerLeaseId: environmentLeases.providerLeaseId,
|
||||
heartbeatRunId: environmentLeases.heartbeatRunId,
|
||||
@@ -4405,7 +4434,7 @@ export async function verifyPriorRunnerdStateForSessionScope(input: {
|
||||
)).limit(1);
|
||||
const leaseMetadata = record(physicalLease?.metadata);
|
||||
const leaseOwner = record(leaseMetadata.computerOwner);
|
||||
if ((workspaceCoordinate && physicalLease?.heartbeatRunId !== input.identity.runId) ||
|
||||
if ((workspaceRunId && physicalLease?.heartbeatRunId !== workspaceRunId) ||
|
||||
physicalLease?.providerLeaseId !== priorWorkspace.computerOwner.ownerId ||
|
||||
leaseMetadata.agentId !== input.execution.binding.agentId ||
|
||||
leaseOwner.computerId !== priorWorkspace.computerOwner.computerId ||
|
||||
|
||||
Reference in new issue
Block a user