diff --git a/server/src/services/heartbeat/runtime-selection.test.ts b/server/src/services/heartbeat/runtime-selection.test.ts index dcb96533ada..d1b313c1d72 100644 --- a/server/src/services/heartbeat/runtime-selection.test.ts +++ b/server/src/services/heartbeat/runtime-selection.test.ts @@ -27,6 +27,7 @@ import { claimNativeRestartRecoveries } from "../native-runtime/native-restart-r import { buildNativeModelEnvelope, canonicalNativeRuntimeContextDigest } from "../../vendor/paperclip-runner/index.js"; import * as runner from "../../vendor/paperclip-runner/index.js"; import { prepareAndSelectHeartbeatRuntime, selectHeartbeatRuntime, type HeartbeatRuntimeSelectionInput } from "./runtime-selection.js"; +import { ConfigurationIncompleteFailure } from "./run-preparation.js"; import { resolveRunnerEnvironmentForRun, resolveWarmRunnerIdleTimeoutMsForRun } from "../runner-environment-lifecycle.js"; import { reserveRunBudget } from "../budget-reservations.js"; import { legacyControllerClaim } from "../legacy-controller-lease.js"; @@ -772,6 +773,41 @@ describe.skipIf(!support.supported)("heartbeat runtime selection boundary", () = expect((await read(run.id)).contextSnapshot?.skillVersions).toEqual({ [key]: versionId }); }); + it.each(["missing", "available"] as const)("distinguishes an unprepared permitted skill from revoked access on resume (%s)", async sourceStatus => { + const { input, run } = await fixture(); + const key = "company/test/project-skill", versionId = randomUUID(); + await db.update(heartbeatRuns).set({ contextSnapshot: { skillVersions: { [key]: versionId } } }).where(eq(heartbeatRuns.id, run.id)); + input.run = await read(run.id); + vi.mocked(buildNativeRuntimeContext).mockImplementationOnce(async () => { + const context = nativeRuntimeContextFixture(); + context.instructions.bundle.rootPath = instructionRoot; + context.skills = [{ key, runtimeName: "project-skill", versionId, bundle: { ...context.instructions.bundle } }]; + return { ...context, aggregateDigest: canonicalNativeRuntimeContextDigest(context) }; + }); + const selected = await selectHeartbeatRuntime(db, input); + if (!selected.selected || !selected.nativeExecution) throw new Error("Expected native selection"); + const saved = await read(run.id); + input.run = saved; + input.session.persistedNativeExecutionInput = selected.nativeExecution; + input.session.persistedRunnerProfile = saved.runnerProfileJson ?? {}; + const detail = "Failed to materialize the selected skill version: synthetic storage read timed out"; + input.config.runtimeSkillEntries = [{ key, runtimeName: "project-skill", versionId, source: "/unavailable/project-skill", sourceStatus, + missingDetail: sourceStatus === "missing" ? detail : null }]; + const resuming = selectHeartbeatRuntime(db, input); + if (sourceStatus === "missing") { + const error = await resuming.then(() => null, (failure: unknown) => failure); + expect(error).toBeInstanceOf(Error); + expect(error).not.toBeInstanceOf(ConfigurationIncompleteFailure); + expect((error as Error).message).not.toContain("access changed"); + expect((error as Error).message).toContain(key); + expect((error as Error).message).toContain(detail); + expect((await read(run.id)).runnerProfileJson).toEqual(saved.runnerProfileJson); + } else { + expect(await resuming).toMatchObject({ selected: true, nativeExecution: selected.nativeExecution }); + } + expect((await read(run.id)).contextSnapshot?.skillVersions).toEqual({ [key]: versionId }); + }); + it.each(["exact", "fresh", "bootstrap", "dead-runner", "expired", "foreign-owner", "missing-audit"] as const)( "admits only exact pending Stop cleanup at runtime selection (%s)", async mode => { const { input, run, issue } = await fixture(); diff --git a/server/src/services/heartbeat/runtime-selection.ts b/server/src/services/heartbeat/runtime-selection.ts index 2462b365c50..fd10d8ad1c1 100644 --- a/server/src/services/heartbeat/runtime-selection.ts +++ b/server/src/services/heartbeat/runtime-selection.ts @@ -610,13 +610,19 @@ export async function selectHeartbeatRuntime(db: Db, input: HeartbeatRuntimeSele ); if ("runtimeContext" in nativeExecution) { const savedVersions = parseObject(run.contextSnapshot?.skillVersions); - const accessibleKeys = new Set(runtimeSkillEntries.filter(entry => entry.sourceStatus !== "missing").map(entry => entry.key)); - if (nativeExecution.runtimeContext.skills.some(skill => - Object.prototype.hasOwnProperty.call(savedVersions, skill.key) && !accessibleKeys.has(skill.key))) { + const currentEntries = new Map(runtimeSkillEntries.map(entry => [entry.key, entry])); + const savedSkills = nativeExecution.runtimeContext.skills.filter(skill => + Object.prototype.hasOwnProperty.call(savedVersions, skill.key)); + if (savedSkills.some(skill => !currentEntries.has(skill.key))) { throw new ConfigurationIncompleteFailure("Project skill access changed while this execution was suspended. Restore access or start a new execution.", { configurationIncomplete: { reason: "skill_access_changed", actionUrl: "/skills" }, }); } + const unprepared = savedSkills.find(skill => currentEntries.get(skill.key)?.sourceStatus === "missing"); + if (unprepared) { + const detail = currentEntries.get(unprepared.key)?.missingDetail?.trim() || "its saved files are unavailable"; + throw new Error(`Saved skill "${unprepared.key}" could not be prepared for this resumed execution: ${detail}`); + } } // Recover only the originally admitted request. A stored idle // checkpoint can precede an already-started provider turn, so even