diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 74e550ad52a..faf96be0013 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -6204,6 +6204,20 @@ can be accepted without a terminal acknowledgement. The selected run and its history refresh after either outcome. Stop does not resume the provider or change a manual task hold. +### Provider-pack admission during restart recovery + +Provider-pack admission keeps a digest-bound historical proof when a controller +claim changes its current admission to pending. If a restart or a failed recovery +barrier interrupts that claim before its admission callback, the next same-run +claim can reuse that proof only for the unchanged receipt. Normal session, +company, agent, Runner and environment-lease checks still apply. Current artifact +retention remains blocked until the new claim publishes verified admission. +Unknown admission state discards the historical proof; a callback clears it when +it records its decision. This adds no background work or provider replay. +Previous-release verified receipts seed this proof on their first new claim. +Older pending or unknown records without it remain unverified, as before. Older +readers ignore the added profile field and retain their existing refusal behavior. + ### Native session diagnostic ownership Native session diagnostics follow the current execution's recorder epoch, including diff --git a/server/src/services/native-runtime/native-provider-pack-admission.test.ts b/server/src/services/native-runtime/native-provider-pack-admission.test.ts index da6f2f9a312..204663dab7c 100644 --- a/server/src/services/native-runtime/native-provider-pack-admission.test.ts +++ b/server/src/services/native-runtime/native-provider-pack-admission.test.ts @@ -5,7 +5,7 @@ import { agents, companies, createDb, heartbeatRuns, issues, nativeRunFinalizati import type { NativeSession } from "../../vendor/paperclip-runner/index.js"; import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; import { canonicalProviderPackJson, type ProviderPackIdentity } from "./remote-provider-pack-verification.js"; -import { admittedNativeProviderPack, nativeProviderPackAdmissionPending, recordNativeProviderPackAdmission, +import { admittedNativeProviderPack, nativeProviderPackAdmissionPending, prepareNativeProviderPackAdmission, recordNativeProviderPackAdmission, rememberAdmittedNativeProviderPack, rememberRecoveredNativeProviderPack } from "./native-provider-pack-admission.js"; const hash = (value: string) => `sha256:${createHash("sha256").update(value).digest("hex")}`; @@ -32,6 +32,41 @@ function receiptDb(receipt: unknown, state: unknown = verified): Db { } describe("provider pack admission identity", () => { + it("reattaches after a previous claim was interrupted before its admission callback", async () => { + const original = sessionFixture(); remember(original.session); + const receipt = admittedNativeProviderPack(original.session, initial)!; + const profile = { nativeProviderPackAdmission: receipt, nativeProviderPackAdmissionState: verified }; + const first = prepareNativeProviderPackAdmission(initial.runId, profile, randomUUID()); + // The claim commits, then snapshot/drain fails before the callback. The + // replacement controller has only persisted data, not the first's locals. + const pending = JSON.parse(JSON.stringify({ ...profile, ...first.patch })); + expect(pending.nativeProviderPackAdmissionState.state).toBe("pending"); + const second = prepareNativeProviderPackAdmission(initial.runId, pending, randomUUID()); + const recovered = sessionFixture(); + await expect(rememberRecoveredNativeProviderPack(receiptDb(receipt, pending.nativeProviderPackAdmissionState), recovered.session, + { identity: initial, sourceRunId: initial.runId, ...binding, source: second.source })).resolves.toBe(true); + expect(admittedNativeProviderPack(recovered.session, initial)).toEqual(receipt); + }); + it.each(["unknown", "missing", "different receipt", "different run", "different runner", "different lease"])( + "does not reuse a pending recovery proof with %s", async change => { + const original = sessionFixture(); remember(original.session); + const receipt = admittedNativeProviderPack(original.session, initial)!; + const profile = { nativeProviderPackAdmission: receipt, nativeProviderPackAdmissionState: verified }; + const first = prepareNativeProviderPackAdmission(initial.runId, profile, randomUUID()); + const pending = JSON.parse(JSON.stringify({ ...profile, ...first.patch })); + if (change === "unknown") pending.nativeProviderPackAdmissionState.state = "unknown"; + if (change === "missing") delete pending.nativeProviderPackAdmissionRecovery; + if (change === "different receipt") pending.nativeProviderPackAdmission.providerPackDigest = hash("replacement"); + if (change === "different run") pending.nativeProviderPackAdmissionRecovery.runId = "other-run"; + const next = prepareNativeProviderPackAdmission(initial.runId, pending, randomUUID()); + const recovered = sessionFixture(); + await expect(rememberRecoveredNativeProviderPack(receiptDb(null), recovered.session, { + identity: initial, sourceRunId: initial.runId, ...binding, source: next.source, + ...(change === "different runner" ? { runnerInstanceId: "replacement" } : {}), + ...(change === "different lease" ? { environmentLeaseId: "replacement" } : {}), + })).resolves.toBe(false); + expect(admittedNativeProviderPack(recovered.session, initial)).toBeNull(); + }); it("does not publish selected configuration before successful session admission", async () => { const { session } = sessionFixture(); expect(admittedNativeProviderPack(session, initial)).toBeNull(); @@ -179,6 +214,32 @@ describe("provider pack admission PostgreSQL fencing", () => { await expect(recordNativeProviderPackAdmission(db, { ...f, claim: { ...f.claim, [key]: changed } as typeof f.claim })).resolves.toBe(false); expect((await read(f.identity.runId)).runnerProfileJson).toEqual(f.profile); }); + it("republishes admission after two persisted interrupted claims without admitting a different Runner", async () => { + const f = await fixture(); + await expect(recordNativeProviderPackAdmission(db, f)).resolves.toBe(true); + for (let i = 0; i < 2; i++) { + const prior = (await read(f.identity.runId)).runnerProfileJson!; + const claim = prepareNativeProviderPackAdmission(f.identity.runId, prior, randomUUID()); + await db.update(heartbeatRuns).set({ runnerProfileJson: { ...prior, ...claim.patch } }).where(eq(heartbeatRuns.id, f.identity.runId)); + } + const prior = (await read(f.identity.runId)).runnerProfileJson!; + const claim = prepareNativeProviderPackAdmission(f.identity.runId, prior, randomUUID()); + const recovered = sessionFixture(f.identity); + await expect(rememberRecoveredNativeProviderPack(db, recovered.session, { + identity: f.identity, sourceRunId: f.identity.runId, ...binding, source: claim.source, + runnerInstanceId: "replacement", + })).resolves.toBe(false); + await expect(rememberRecoveredNativeProviderPack(db, recovered.session, { + identity: f.identity, sourceRunId: f.identity.runId, ...binding, source: claim.source, + })).resolves.toBe(true); + await db.update(heartbeatRuns).set({ runnerProfileJson: { ...prior, ...claim.patch } }).where(eq(heartbeatRuns.id, f.identity.runId)); + await expect(recordNativeProviderPackAdmission(db, { ...f, session: recovered.session, + admissionEpoch: claim.patch.nativeProviderPackAdmissionState.admissionEpoch })).resolves.toBe(true); + const final = (await read(f.identity.runId)).runnerProfileJson!; + expect(final.nativeProviderPackAdmissionState).toMatchObject({ state: "verified" }); + expect(final.nativeProviderPackAdmission).toEqual(prior.nativeProviderPackAdmission); + expect(final.nativeProviderPackAdmissionRecovery).toBeNull(); + }); it("rejects an expired claim and a terminal run", async () => { const f = await fixture(); await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(0) }).where(eq(nativeRunFinalizations.runId, f.identity.runId)); diff --git a/server/src/services/native-runtime/native-provider-pack-admission.ts b/server/src/services/native-runtime/native-provider-pack-admission.ts index 6de5cd840a9..6a2e3d05f32 100644 --- a/server/src/services/native-runtime/native-provider-pack-admission.ts +++ b/server/src/services/native-runtime/native-provider-pack-admission.ts @@ -41,11 +41,39 @@ export function nativeProviderPackAdmissionPending(admissionEpoch: string) { return { schema: "paperclip.native-provider-pack-admission-state.v1" as const, state: "pending" as const, admissionEpoch }; } +/** Called under the executor's claim lock, before provider admission. */ +export function prepareNativeProviderPackAdmission(runId: string, profile: Record | null | undefined, admissionEpoch: string) { + const receipt = profile?.nativeProviderPackAdmission; + let state = profile?.nativeProviderPackAdmissionState; + const saved = profile?.nativeProviderPackAdmissionRecovery as Record | null | undefined; + const receiptDigest = receipt && typeof receipt === "object" && !Array.isArray(receipt) + ? `sha256:${createHash("sha256").update(canonicalProviderPackJson(receipt)).digest("hex")}` : null; + // A pending claim has not selected a different pack. It can carry forward + // the exact verified proof that the previous claim captured under its lock. + // Unknown state means selection was refused and must never regain that proof. + if (admissionState(state, "pending") + && saved?.schema === "paperclip.native-provider-pack-admission-recovery.v1" + && Object.keys(saved).sort().join(",") === "receiptDigest,runId,schema,state" + && saved.runId === runId && receiptDigest !== null && saved.receiptDigest === receiptDigest + && verifiedState(saved.state)) state = saved.state; + const recovery = receiptDigest !== null && verifiedState(state) + ? { schema: "paperclip.native-provider-pack-admission-recovery.v1", runId, receiptDigest, state } : null; + return { + source: { runId, receipt, state, artifacts: profile?.nativeProviderArtifacts }, + patch: { nativeProviderPackAdmissionState: nativeProviderPackAdmissionPending(admissionEpoch), + nativeProviderPackAdmissionRecovery: recovery }, + }; +} + function verifiedState(value: unknown): boolean { + return admissionState(value, "verified"); +} + +function admissionState(value: unknown, expected: "pending" | "verified"): boolean { if (!value || typeof value !== "object" || Array.isArray(value)) return false; const state = value as Record; return Object.keys(state).sort().join(",") === "admissionEpoch,schema,state" - && state.schema === "paperclip.native-provider-pack-admission-state.v1" && state.state === "verified" + && state.schema === "paperclip.native-provider-pack-admission-state.v1" && state.state === expected && typeof state.admissionEpoch === "string" && /^[a-f0-9-]{36}$/.test(state.admissionEpoch); } @@ -210,6 +238,7 @@ async function recordNativeProviderPackAdmissionOnce(db: Db, input: AdmissionInp === canonicalProviderPackJson({ ...receipt, inheritedFromRunId: null })); if (verified && current.state === "verified") return true; const patch = { ...(verified && previous == null ? { nativeProviderPackAdmission: receipt } : {}), + ...(run.profile?.nativeProviderPackAdmissionRecovery != null ? { nativeProviderPackAdmissionRecovery: null } : {}), nativeProviderPackAdmissionState: { ...nativeProviderPackAdmissionPending(input.admissionEpoch), state: verified ? "verified" : "unknown" } }; const saved = await tx.update(heartbeatRuns).set({ runnerProfileJson: sql`coalesce(${heartbeatRuns.runnerProfileJson}, '{}'::jsonb) || ${JSON.stringify(patch)}::jsonb`, diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 898df3a20e6..d2efd0d2b06 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -10683,9 +10683,10 @@ describe("native session execution lease fencing", () => { }); await executePaperclipNativeSession({ db: leaseDb(execution, {}, {}, updates, profile), execution, runnerInstanceId: "runner" }); expect(state.execute).toHaveBeenCalledOnce(); - expect(observed).toEqual({ nativeProviderPackAdmissionState: { schema: "paperclip.native-provider-pack-admission-state.v1", + expect(observed).toMatchObject({ nativeProviderPackAdmissionState: { schema: "paperclip.native-provider-pack-admission-state.v1", state: "pending", admissionEpoch: expect.any(String) } }); - expect(JSON.stringify(observed)).not.toContain(profile.nativeProviderPackAdmissionState.admissionEpoch); + expect((observed as unknown as Record).nativeProviderPackAdmissionState.admissionEpoch) + .not.toBe(profile.nativeProviderPackAdmissionState.admissionEpoch); expect(profile.nativeProviderPackAdmissionState.state).toBe("verified"); }); it("does not admit a session if mandatory claim invalidation fails", async () => { diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index a2d43d50212..64ff926b87f 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -45,7 +45,7 @@ import { notifyUnsafeWorkspaceExportRecovery } from "../unsafe-workspace-export- import { notifyCancelledWorkspacePreservation } from "../cancelled-workspace-preservation-notifications.js"; import { notifyInstructionCollection } from "../instruction-collection-notifications.js"; import { providerPackWorkspaceId, recordAdmittedRemoteProviderPack, resolveRemoteProviderPackAuthority } from "./remote-provider-pack-authority.js"; -import { nativeProviderPackAdmissionFailureCode, nativeProviderPackAdmissionPending, rememberAdmittedNativeProviderPack, rememberRecoveredNativeProviderPack, recordNativeProviderPackAdmission, type NativeProviderPackAdmissionSource } from "./native-provider-pack-admission.js"; +import { nativeProviderPackAdmissionFailureCode, prepareNativeProviderPackAdmission, rememberAdmittedNativeProviderPack, rememberRecoveredNativeProviderPack, recordNativeProviderPackAdmission, type NativeProviderPackAdmissionSource } from "./native-provider-pack-admission.js"; import { type ProviderPackInstallation, verifyRemoteProviderPack as verifyRemoteProviderPackIdentity } from "./remote-provider-pack-verification.js"; import { writeEnvironmentCleanup } from "../environment-cleanup-notifications.js"; import { NATIVE_SESSION_EXECUTION_LEASE_TTL_MS, lockNativeExecutionLeaseForReacquire, nativeExecutionLeaseHeldCondition, releaseUnstartedNativeExecutionClaim, startNativeSessionExecutionLeaseRenewal, type NativeExecutionLeaseRenewal } from "./native-execution-lease.js"; @@ -11259,10 +11259,8 @@ async function executePaperclipNativeSessionAttempt( .returning({ runId: nativeRunFinalizations.runId }) .then((rows) => rows[0] ?? null); if (!claimed) throw new Error("native_session_lease_lost"); - providerPackAdmissionSource = { runId: coordinator.runId, - receipt: boundRun.runnerProfileJson?.nativeProviderPackAdmission, - state: boundRun.runnerProfileJson?.nativeProviderPackAdmissionState, - artifacts: boundRun.runnerProfileJson?.nativeProviderArtifacts }; + const providerPackAdmission = prepareNativeProviderPackAdmission(coordinator.runId, boundRun.runnerProfileJson, providerPackAdmissionEpoch); + providerPackAdmissionSource = providerPackAdmission.source; controllerGeneration = nextControllerGeneration; claimedStopCleanup = stopRecovery; await tx @@ -11270,7 +11268,7 @@ async function executePaperclipNativeSessionAttempt( .set({ nativePhase: "observed", nativePhaseUpdatedAt: leaseNow, - runnerProfileJson: sql`coalesce(${heartbeatRuns.runnerProfileJson}, '{}'::jsonb) || ${JSON.stringify({ nativeProviderPackAdmissionState: nativeProviderPackAdmissionPending(providerPackAdmissionEpoch), + runnerProfileJson: sql`coalesce(${heartbeatRuns.runnerProfileJson}, '{}'::jsonb) || ${JSON.stringify({ ...providerPackAdmission.patch, ...(claimedStartupContext ? { nativeStartupContext: { schema: "paperclip.native-startup-context.v1", companyId: input.execution.binding.companyId, agentId: input.execution.binding.agentId, runId: input.execution.binding.runId, contextId: claimedStartupContext.id, deadlineAtMs: claimedStartupContext.deadlineAtMs, leaseId: target?.kind === "remote" ? target.leaseId : null,