From f434f0e7bd72c0d880daa9c6ef915c913e989f1c Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Sat, 21 Nov 2026 04:28:57 -0800 Subject: [PATCH] fix(native): let a fresh replacement Runner publish its verified pack admission (#21498) A new claim's own fresh Runner selection now supersedes the earlier claim's admission receipt while no provider artifacts are retained under it, so a refused reattachment no longer fails before the first provider turn as native_provider_artifacts_admission_unverified. Co-Authored-By: Paperclip Co-Authored-By: Claude Opus 5.5 --- .../failed-before-provider-turn.test.ts | 23 ++++++++++ .../failed-before-provider-turn.ts | 3 +- .../native-provider-pack-admission.test.ts | 43 +++++++++++++++++-- .../native-provider-pack-admission.ts | 11 +++-- 4 files changed, 72 insertions(+), 8 deletions(-) create mode 100644 server/src/services/native-runtime/failed-before-provider-turn.test.ts diff --git a/server/src/services/native-runtime/failed-before-provider-turn.test.ts b/server/src/services/native-runtime/failed-before-provider-turn.test.ts new file mode 100644 index 00000000000..aef3b66cd06 --- /dev/null +++ b/server/src/services/native-runtime/failed-before-provider-turn.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from "vitest"; +import { NativeFailedBeforeProviderTurnError } from "./failed-before-provider-turn.js"; +import { fail } from "./native-provider-artifact-files.js"; + +const causeOf = (thrown: () => unknown) => { + try { thrown(); } catch (error) { return error; } + throw new Error("expected a throw"); +}; + +describe("NativeFailedBeforeProviderTurnError technical cause", () => { + it("names a provider-artifact failure that carries its code only as the message", () => { + const error = new NativeFailedBeforeProviderTurnError(null, causeOf(() => fail("admission_unverified"))); + expect(error.technicalCause).toBe("native_provider_artifacts_admission_unverified"); + }); + it("prefers an explicit code and keeps free-form messages out of the cause", () => { + expect(new NativeFailedBeforeProviderTurnError(null, Object.assign(new Error("anything"), { code: "native_x" })).technicalCause) + .toBe("native_x"); + expect(new NativeFailedBeforeProviderTurnError(null, new Error("native_provider at /home/user/secret")).technicalCause) + .toBe("unknown"); + expect(new NativeFailedBeforeProviderTurnError(null, new Error("connect ECONNREFUSED")).technicalCause).toBe("unknown"); + expect(new NativeFailedBeforeProviderTurnError(null, "native_string_cause").technicalCause).toBe("unknown"); + }); +}); diff --git a/server/src/services/native-runtime/failed-before-provider-turn.ts b/server/src/services/native-runtime/failed-before-provider-turn.ts index fd6b79dd01f..5a66622890e 100644 --- a/server/src/services/native-runtime/failed-before-provider-turn.ts +++ b/server/src/services/native-runtime/failed-before-provider-turn.ts @@ -14,7 +14,8 @@ const SLOW_SETUP_CODES = new Set(["native_provider_artifacts_transfer_timeout", function technicalCauseCode(cause: unknown): string { const code = cause && typeof cause === "object" ? (cause as { code?: unknown }).code : undefined; - return typeof code === "string" && /^[a-z][a-z0-9_]{0,79}$/.test(code) ? code : "unknown"; + if (typeof code === "string" && /^[a-z][a-z0-9_]{0,79}$/.test(code)) return code; + return cause instanceof Error && /^native_[a-z0-9_]{1,72}$/.test(cause.message) ? cause.message : "unknown"; } export function readFailedBeforeProviderTurnRecord(profile: unknown): FailedBeforeProviderTurnRecord | null { diff --git a/server/src/services/native-runtime/native-provider-pack-admission.test.ts b/server/src/services/native-runtime/native-provider-pack-admission.test.ts index 204663dab7c..f130adfa39d 100644 --- a/server/src/services/native-runtime/native-provider-pack-admission.test.ts +++ b/server/src/services/native-runtime/native-provider-pack-admission.test.ts @@ -278,19 +278,56 @@ describe("provider pack admission PostgreSQL fencing", () => { await expect(recordNativeProviderPackAdmission(db, f)).resolves.toBe(false); expect((await read(f.identity.runId)).runnerProfileJson).toEqual(next); }); - it.each(["runner", "lease", "unsupported"])("keeps the first receipt historical after an authorized %s replacement", async change => { + async function replacementClaim(retainedArtifacts: boolean) { const f = await fixture(); await recordNativeProviderPackAdmission(db, f); - const previous = (await read(f.identity.runId)).runnerProfileJson!; + const recorded = (await read(f.identity.runId)).runnerProfileJson!; + const previous = retainedArtifacts ? { ...recorded, nativeProviderArtifacts: { schema: "retained-under-the-first-pack" } } : recorded; const admissionEpoch = randomUUID(); await db.update(heartbeatRuns).set({ runnerProfileJson: { ...previous, nativeProviderPackAdmissionState: nativeProviderPackAdmissionPending(admissionEpoch) } }).where(eq(heartbeatRuns.id, f.identity.runId)); - const replacement = sessionFixture(f.identity); + return { f, previous, admissionEpoch, replacement: sessionFixture(f.identity) }; + } + it.each([ + ["runner", { runnerInstanceId: "replacement" }, manifest()], + ["lease", { environmentLeaseId: "replacement" }, manifest()], + ["release", { runnerInstanceId: "replacement" }, manifest("b".repeat(40))], + ] as const)("publishes a fresh %s replacement's verified selection when no provider artifacts were retained", async (_change, change, selected) => { + const { f, previous, admissionEpoch, replacement } = await replacementClaim(false); + rememberAdmittedNativeProviderPack(replacement.session, { identity: f.identity, manifest: selected, ...binding, ...change }); + await expect(recordNativeProviderPackAdmission(db, { ...f, admissionEpoch, session: replacement.session })).resolves.toBe(true); + const published = (await read(f.identity.runId)).runnerProfileJson; + expect(published).toEqual({ ...previous, + nativeProviderPackAdmission: admittedNativeProviderPack(replacement.session, f.identity), + nativeProviderPackAdmissionState: { ...nativeProviderPackAdmissionPending(admissionEpoch), state: "verified" } }); + await expect(recordNativeProviderPackAdmission(db, { ...f, admissionEpoch, session: replacement.session })).resolves.toBe(true); + await expect(recordNativeProviderPackAdmission(db, { ...f, claim: { ...f.claim, controllerGeneration: 1 } })).resolves.toBe(false); + expect((await read(f.identity.runId)).runnerProfileJson).toEqual(published); + }); + it.each(["runner", "lease", "unsupported"])("keeps the first receipt historical after a %s replacement once provider artifacts were retained", async change => { + const { f, previous, admissionEpoch, replacement } = await replacementClaim(true); if (change !== "unsupported") rememberAdmittedNativeProviderPack(replacement.session, { identity: f.identity, manifest: manifest(), ...binding, ...(change === "runner" ? { runnerInstanceId: "replacement" } : { environmentLeaseId: "replacement" }) }); await expect(recordNativeProviderPackAdmission(db, { ...f, admissionEpoch, session: replacement.session })).resolves.toBe(false); expect((await read(f.identity.runId)).runnerProfileJson).toEqual({ ...previous, nativeProviderPackAdmissionState: { ...nativeProviderPackAdmissionPending(admissionEpoch), state: "unknown" } }); }); + it("leaves an unsupported replacement unknown even without retained provider artifacts", async () => { + const { f, previous, admissionEpoch, replacement } = await replacementClaim(false); + await expect(recordNativeProviderPackAdmission(db, { ...f, admissionEpoch, session: replacement.session })).resolves.toBe(false); + expect((await read(f.identity.runId)).runnerProfileJson).toEqual({ ...previous, + nativeProviderPackAdmissionState: { ...nativeProviderPackAdmissionPending(admissionEpoch), state: "unknown" } }); + }); + it("does not let a retained attachment from another run replace this run's receipt", async () => { + const { f, previous, admissionEpoch } = await replacementClaim(false); + const source = { ...f.identity, runId: randomUUID() }; + const inherited = sessionFixture(source); + rememberAdmittedNativeProviderPack(inherited.session, { identity: source, manifest: manifest("b".repeat(40)), ...binding }); + await inherited.attach(f.identity); + expect(admittedNativeProviderPack(inherited.session, f.identity)).toMatchObject({ inheritedFromRunId: source.runId }); + await expect(recordNativeProviderPackAdmission(db, { ...f, admissionEpoch, session: inherited.session })).resolves.toBe(false); + expect((await read(f.identity.runId)).runnerProfileJson).toEqual({ ...previous, + nativeProviderPackAdmissionState: { ...nativeProviderPackAdmissionPending(admissionEpoch), state: "unknown" } }); + }); it("cannot publish while another transaction holds the coordinator claim", async () => { const f = await fixture(); await recordNativeProviderPackAdmission(db, f); diff --git a/server/src/services/native-runtime/native-provider-pack-admission.ts b/server/src/services/native-runtime/native-provider-pack-admission.ts index 6a2e3d05f32..2d98194a271 100644 --- a/server/src/services/native-runtime/native-provider-pack-admission.ts +++ b/server/src/services/native-runtime/native-provider-pack-admission.ts @@ -233,11 +233,14 @@ async function recordNativeProviderPackAdmissionOnce(db: Db, input: AdmissionInp || !["pending", "verified", "unknown"].includes(String(current.state))) return false; if (current.state === "unknown") return false; const previous = run.profile?.nativeProviderPackAdmission; - const verified = receipt != null && (previous == null ? current.state === "pending" - : canonicalProviderPackJson({ ...(previous as Record), inheritedFromRunId: null }) - === canonicalProviderPackJson({ ...receipt, inheritedFromRunId: null })); + const unchanged = previous != null && receipt != null + && canonicalProviderPackJson({ ...(previous as Record), inheritedFromRunId: null }) + === canonicalProviderPackJson({ ...receipt, inheritedFromRunId: null }); + const superseded = previous != null && !unchanged && receipt != null && current.state === "pending" + && receipt.inheritedFromRunId === undefined && run.profile?.nativeProviderArtifacts == null; + const verified = receipt != null && (previous == null ? current.state === "pending" : unchanged || superseded); if (verified && current.state === "verified") return true; - const patch = { ...(verified && previous == null ? { nativeProviderPackAdmission: receipt } : {}), + const patch = { ...(verified && (previous == null || superseded) ? { nativeProviderPackAdmission: receipt } : {}), ...(run.profile?.nativeProviderPackAdmissionRecovery != null ? { nativeProviderPackAdmissionRecovery: null } : {}), nativeProviderPackAdmissionState: { ...nativeProviderPackAdmissionPending(input.admissionEpoch), state: verified ? "verified" : "unknown" } }; const saved = await tx.update(heartbeatRuns).set({