mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-17 21:34:48 +01:00
## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - OpenCode turns run in a remote Runner. When a server restarts, the new server reconnects to the Runner and reuses the provider pack admission that the run saved earlier. > - On staging, a run reconnected cleanly after an upgrade but then failed with `native_provider_artifacts_admission_unverified`. Its saved admission state stayed "pending". > - The reuse step refuses a saved admission for several reasons. It returns only false and logs nothing, so we cannot tell which check refused it. > - This pull request adds diagnostics only. It names the refused check in the run log and the server log. It also logs when the admission stays unverified for a session that needs it. > - The benefit is that the next upgrade audit shows the real cause, so the next fix can target it. ## Linked Issues or Issue Description **What happened?** After an automatic staging upgrade, a run reconnected to its live Runner without a new process. The next checkpoint failed with `native_provider_artifacts_admission_unverified`, which used one of the run's 3 attempts. The run log had no line that explained why the saved admission was not reused. **Expected behavior** When a saved admission is not reused, the run log and the server log name the check that refused it. **Steps to reproduce** 1. Start an OpenCode run on a remote sandbox. 2. Restart the server, so the run reconnects to its Runner. 3. If the saved admission is refused, the run fails later with `native_provider_artifacts_admission_unverified` and no reason in the logs. ## What Changed - `native-provider-pack-admission.ts`: `rememberRecoveredNativeProviderPack` accepts an optional `onRefused` callback. It is called with one reason: `source_state_pending`, `source_state_unknown`, `source_state_missing`, `receipt_missing`, `runner_instance_differs`, `environment_lease_differs`, `receipt_invalid`, or a session identity reason. The result of every check stays the same. An error in the callback is ignored. - `native-session-executor.ts`: the reattach path logs the reason to the run log and the server log. When `recordNativeProviderPackAdmission` returns false for a session that requires admission, it logs that too. - `native-provider-pack-admission.test.ts`: each refusal names its reason, and a reused admission reports nothing. ## Verification - `npx vitest run src/services/native-runtime/native-provider-pack-admission.test.ts` passes 46/46. The new reason tests fail on the old code because `onRefused` is never called. - `npx vitest run src/services/native-runtime/native-session-executor.test.ts` passes 1412/1412. - `npx tsc --noEmit -p server/tsconfig.json` passes. ## Risks - Low risk. This change adds diagnostics only, and admission decisions do not change. The new log lines contain only a fixed reason code and run ids. They do not contain secrets or file paths. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Claude Code agent with tool use (shell, code search, test execution). ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
359 lines
25 KiB
TypeScript
359 lines
25 KiB
TypeScript
import { createHash, randomUUID } from "node:crypto";
|
|
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
|
import { eq, sql } from "drizzle-orm";
|
|
import { agents, companies, createDb, heartbeatRuns, issues, nativeRunFinalizations, type Db } from "@paperclipai/db";
|
|
import type { NativeSession } from "../../vendor/paperclip-runner/index.js";
|
|
import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
|
|
import { canonicalProviderPackJson, type ProviderPackIdentity } from "./remote-provider-pack-verification.js";
|
|
import { admittedNativeProviderPack, nativeProviderPackAdmissionPending, recordNativeProviderPackAdmission,
|
|
rememberAdmittedNativeProviderPack, rememberRecoveredNativeProviderPack } from "./native-provider-pack-admission.js";
|
|
|
|
const hash = (value: string) => `sha256:${createHash("sha256").update(value).digest("hex")}`;
|
|
function manifest(revision = "a".repeat(40)): ProviderPackIdentity {
|
|
const artifact = { path: "/private/provider-path", sha256: hash(revision) };
|
|
const payload = { runnerSourceRevision: revision, pins: { opencode: "1.18.34" },
|
|
target: { platform: "linux", architecture: "x64" },
|
|
artifacts: { nodeCommand: artifact, opencodeCommand: artifact, opencodeProxy: artifact } };
|
|
return { schema: "paperclip-runner/remote-provider-pack/v1", payload, digest: hash(canonicalProviderPackJson(payload)) };
|
|
}
|
|
const initial = { companyId: "company", agentId: "agent", issueId: "issue", runId: "source-run", sessionId: "session" };
|
|
function sessionFixture(identity = initial) {
|
|
let current = { ...identity };
|
|
const session = { identity: () => ({ ...current }), attachRun: async ({ identity: next }: { identity: typeof initial }) => { current = { ...next }; } } as NativeSession;
|
|
return { session, attach: (next: typeof initial) => session.attachRun!({ identity: next }) };
|
|
}
|
|
const binding = { runnerInstanceId: "runner", environmentLeaseId: "lease" };
|
|
function remember(session: NativeSession, identity = initial, selected = manifest()) {
|
|
return rememberAdmittedNativeProviderPack(session, { identity, ...binding, manifest: selected });
|
|
}
|
|
const verified = { ...nativeProviderPackAdmissionPending(randomUUID()), state: "verified" };
|
|
function receiptDb(receipt: unknown, state: unknown = verified): Db {
|
|
return { select: () => ({ from: () => ({ where: () => ({ limit: async () => [{ receipt, state }] }) }) }) } as unknown as Db;
|
|
}
|
|
|
|
describe("provider pack admission identity", () => {
|
|
it("does not publish selected configuration before successful session admission", async () => {
|
|
const { session } = sessionFixture();
|
|
expect(admittedNativeProviderPack(session, initial)).toBeNull();
|
|
});
|
|
it("keeps a closed original pack identity through an actual target attachment", async () => {
|
|
const { session, attach } = sessionFixture();
|
|
const original = manifest();
|
|
expect(remember(session, initial, original)).toBe(true);
|
|
const before = admittedNativeProviderPack(session, initial)!;
|
|
const target = { ...initial, runId: "target-run", issueId: "target-issue" };
|
|
expect(admittedNativeProviderPack(session, target)).toBeNull();
|
|
await attach(target);
|
|
const next = admittedNativeProviderPack(session, target)!;
|
|
expect(next).toEqual({ ...before, runId: target.runId, inheritedFromRunId: initial.runId });
|
|
expect(JSON.stringify(next)).not.toContain("private/provider-path");
|
|
expect(Object.keys(next).sort()).toEqual(["schema", "stage", "authority", "companyId", "agentId", "runId", "sessionId",
|
|
"runnerInstanceId", "runnerEnvironmentLeaseId", "providerPackDigest", "opencodeProxySha256", "sourceRevision", "inheritedFromRunId"].sort());
|
|
expect(remember(session, target, original)).toBe(true);
|
|
expect(admittedNativeProviderPack(session, target)).toEqual(next);
|
|
expect(remember(session, target, manifest("b".repeat(40)))).toBe(true);
|
|
expect(admittedNativeProviderPack(session, target)).toMatchObject({ sourceRevision: "b".repeat(40), runId: target.runId });
|
|
});
|
|
it.each(["companyId", "agentId", "issueId", "runId", "sessionId"] as const)("rejects a returned session with a different %s", key => {
|
|
const { session } = sessionFixture();
|
|
expect(remember(session, { ...initial, [key]: "foreign" })).toBe(false);
|
|
expect(admittedNativeProviderPack(session, initial)).toBeNull();
|
|
});
|
|
it("rejects stale manifest digests and unavailable source identity", () => {
|
|
const { session } = sessionFixture();
|
|
const stale = manifest(); stale.payload.artifacts.opencodeProxy.sha256 = hash("changed");
|
|
expect(remember(session, initial, stale)).toBe(false);
|
|
expect(remember(session, initial, manifest("unknown"))).toBe(false);
|
|
expect(admittedNativeProviderPack(session, initial)).toBeNull();
|
|
});
|
|
it("does not reuse an earlier in-memory proof for an unsupported new admission", async () => {
|
|
const { session } = sessionFixture(); remember(session);
|
|
expect(remember(session, initial, manifest("unknown"))).toBe(false);
|
|
expect(admittedNativeProviderPack(session, initial)).toBeNull();
|
|
remember(session);
|
|
await expect(rememberRecoveredNativeProviderPack(receiptDb(undefined), session,
|
|
{ identity: initial, sourceRunId: initial.runId, ...binding })).resolves.toBe(false);
|
|
expect(admittedNativeProviderPack(session, initial)).toBeNull();
|
|
});
|
|
it("restores only the exact prior closed Core receipt and leaves legacy absence unknown", async () => {
|
|
const source = sessionFixture(); remember(source.session);
|
|
const saved = admittedNativeProviderPack(source.session, initial)!;
|
|
const recovered = sessionFixture();
|
|
await expect(rememberRecoveredNativeProviderPack(receiptDb(undefined), recovered.session,
|
|
{ identity: initial, sourceRunId: initial.runId, ...binding })).resolves.toBe(false);
|
|
expect(admittedNativeProviderPack(recovered.session, initial)).toBeNull();
|
|
await expect(rememberRecoveredNativeProviderPack(receiptDb(saved), recovered.session,
|
|
{ identity: initial, sourceRunId: initial.runId, ...binding })).resolves.toBe(true);
|
|
const target = { ...initial, runId: "target-run" };
|
|
expect(admittedNativeProviderPack(recovered.session, target)).toBeNull();
|
|
await recovered.attach(target);
|
|
expect(admittedNativeProviderPack(recovered.session, target)).toEqual({ ...saved, runId: target.runId, inheritedFromRunId: initial.runId });
|
|
});
|
|
it.each(["companyId", "agentId", "runId", "sessionId", "runnerInstanceId", "runnerEnvironmentLeaseId"] as const)("rejects a prior receipt with a different %s", async key => {
|
|
const source = sessionFixture(); remember(source.session);
|
|
const saved = { ...admittedNativeProviderPack(source.session, initial)!, [key]: "foreign" };
|
|
const recovered = sessionFixture();
|
|
await expect(rememberRecoveredNativeProviderPack(receiptDb(saved), recovered.session,
|
|
{ identity: initial, sourceRunId: initial.runId, ...binding })).resolves.toBe(false);
|
|
expect(admittedNativeProviderPack(recovered.session, initial)).toBeNull();
|
|
});
|
|
it.each([undefined, { ...verified, state: "pending" }, { ...verified, state: "unknown" }])("does not revive a historical receipt without verified current provenance: %j", async state => {
|
|
const source = sessionFixture(); remember(source.session);
|
|
const recovered = sessionFixture();
|
|
const absentState = state === undefined ? null : state;
|
|
await expect(rememberRecoveredNativeProviderPack(receiptDb(admittedNativeProviderPack(source.session, initial), absentState), recovered.session,
|
|
{ identity: initial, sourceRunId: initial.runId, ...binding })).resolves.toBe(false);
|
|
expect(admittedNativeProviderPack(recovered.session, initial)).toBeNull();
|
|
});
|
|
it.each([
|
|
["source_state_pending", {}, { ...verified, state: "pending" }],
|
|
["source_state_unknown", {}, { ...verified, state: "unknown" }],
|
|
["source_state_missing", {}, null],
|
|
["runner_instance_differs", { runnerInstanceId: "other-runner" }, verified],
|
|
["environment_lease_differs", { runnerEnvironmentLeaseId: "other-lease" }, verified],
|
|
["receipt_invalid", { sessionId: "foreign" }, verified],
|
|
] as const)("names the refused check %s without admitting", async (reason, change, state) => {
|
|
const source = sessionFixture(); remember(source.session);
|
|
const saved = { ...admittedNativeProviderPack(source.session, initial)!, ...change };
|
|
const recovered = sessionFixture();
|
|
const onRefused = vi.fn();
|
|
await expect(rememberRecoveredNativeProviderPack(receiptDb(saved, state), recovered.session,
|
|
{ identity: initial, sourceRunId: initial.runId, ...binding, onRefused })).resolves.toBe(false);
|
|
expect(onRefused).toHaveBeenCalledExactlyOnceWith(reason);
|
|
expect(admittedNativeProviderPack(recovered.session, initial)).toBeNull();
|
|
});
|
|
it("reports nothing when the saved admission is reused", async () => {
|
|
const source = sessionFixture(); remember(source.session);
|
|
const recovered = sessionFixture();
|
|
const onRefused = vi.fn();
|
|
await expect(rememberRecoveredNativeProviderPack(receiptDb(admittedNativeProviderPack(source.session, initial)), recovered.session,
|
|
{ identity: initial, sourceRunId: initial.runId, ...binding, onRefused })).resolves.toBe(true);
|
|
expect(onRefused).not.toHaveBeenCalled();
|
|
});
|
|
it("uses only the exact same-run locked pre-invalidation source snapshot", async () => {
|
|
const source = sessionFixture(); remember(source.session);
|
|
const saved = admittedNativeProviderPack(source.session, initial)!;
|
|
const recovered = sessionFixture();
|
|
const db = receiptDb(saved, nativeProviderPackAdmissionPending(randomUUID()));
|
|
await expect(rememberRecoveredNativeProviderPack(db, recovered.session,
|
|
{ identity: initial, sourceRunId: initial.runId, ...binding, source: { runId: "foreign-run", receipt: saved, state: verified } })).resolves.toBe(false);
|
|
await expect(rememberRecoveredNativeProviderPack(db, recovered.session,
|
|
{ identity: initial, sourceRunId: initial.runId, ...binding, source: { runId: initial.runId, receipt: saved, state: verified } })).resolves.toBe(true);
|
|
});
|
|
});
|
|
|
|
describe("provider pack admission PostgreSQL fencing", () => {
|
|
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
|
let db: ReturnType<typeof createDb>;
|
|
beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("pack-admission-"); db = createDb(database.connectionString); }, 90_000);
|
|
afterAll(async () => { await database?.cleanup(); });
|
|
async function fixture() {
|
|
const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID();
|
|
await db.insert(companies).values({ id: companyId, name: "Pack admission", issuePrefix: randomUUID().slice(0, 8) });
|
|
await db.insert(agents).values({ id: agentId, companyId, name: "Pack agent" });
|
|
await db.insert(issues).values({ id: issueId, companyId, title: "Pack task", assigneeAgentId: agentId });
|
|
const identity = { companyId, agentId, issueId, runId, sessionId: `session-${runId}` };
|
|
const admissionEpoch = randomUUID();
|
|
const profile = { preserved: { value: "unchanged" }, nativeExecutionInput: { binding: identity },
|
|
nativeProviderPackAdmissionState: nativeProviderPackAdmissionPending(admissionEpoch) };
|
|
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: issueId, runtimeMode: "native", status: "running", runnerProfileJson: profile });
|
|
const claim = { leaseOwner: randomUUID(), attempt: 1, controllerGeneration: 2, controllerBootId: randomUUID() };
|
|
await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId, phase: "observed", ...claim,
|
|
leaseExpiresAt: new Date(Date.now() + 60_000) });
|
|
const { session } = sessionFixture(identity); remember(session, identity);
|
|
return { identity, session, claim, profile, admissionEpoch };
|
|
}
|
|
const read = async (runId: string) => (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)))[0]!;
|
|
it("merges the exact closed receipt without changing other profile data and is idempotent", async () => {
|
|
const f = await fixture();
|
|
await expect(recordNativeProviderPackAdmission(db, f)).resolves.toBe(true);
|
|
const saved = await read(f.identity.runId);
|
|
expect(saved.runnerProfileJson).toEqual({ ...f.profile, nativeProviderPackAdmission: admittedNativeProviderPack(f.session, f.identity),
|
|
nativeProviderPackAdmissionState: { ...f.profile.nativeProviderPackAdmissionState, state: "verified" } });
|
|
await expect(recordNativeProviderPackAdmission(db, f)).resolves.toBe(true);
|
|
expect((await read(f.identity.runId)).runnerProfileJson).toEqual(saved.runnerProfileJson);
|
|
});
|
|
it.each(["leaseOwner", "attempt", "controllerGeneration", "controllerBootId"] as const)("rejects stale %s without publishing a receipt", async key => {
|
|
const f = await fixture();
|
|
const changed = typeof f.claim[key] === "number" ? 99 : randomUUID();
|
|
await expect(recordNativeProviderPackAdmission(db, { ...f, claim: { ...f.claim, [key]: changed } as typeof f.claim })).resolves.toBe(false);
|
|
expect((await read(f.identity.runId)).runnerProfileJson).toEqual(f.profile);
|
|
});
|
|
it("rejects an expired claim and a terminal run", async () => {
|
|
const f = await fixture();
|
|
await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(0) }).where(eq(nativeRunFinalizations.runId, f.identity.runId));
|
|
await expect(recordNativeProviderPackAdmission(db, f)).resolves.toBe(false);
|
|
await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() + 60_000) }).where(eq(nativeRunFinalizations.runId, f.identity.runId));
|
|
await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, f.identity.runId));
|
|
await expect(recordNativeProviderPackAdmission(db, f)).resolves.toBe(false);
|
|
expect((await read(f.identity.runId)).runnerProfileJson).toEqual(f.profile);
|
|
});
|
|
it("preserves the original immutable receipt when another pack is proposed", async () => {
|
|
const f = await fixture(); await recordNativeProviderPackAdmission(db, f);
|
|
const before = (await read(f.identity.runId)).runnerProfileJson!;
|
|
const other = sessionFixture(f.identity); remember(other.session, f.identity, manifest("b".repeat(40)));
|
|
await expect(recordNativeProviderPackAdmission(db, { ...f, session: other.session })).resolves.toBe(false);
|
|
expect((await read(f.identity.runId)).runnerProfileJson).toEqual({ ...before,
|
|
nativeProviderPackAdmissionState: { ...f.profile.nativeProviderPackAdmissionState, state: "unknown" } });
|
|
await expect(recordNativeProviderPackAdmission(db, f)).resolves.toBe(false);
|
|
});
|
|
it("preserves the original inheritance when the same run recovers the same verified pack", async () => {
|
|
const f = await fixture(), sourceRunId = randomUUID();
|
|
const receipt = admittedNativeProviderPack(f.session, f.identity)!;
|
|
await db.insert(heartbeatRuns).values({ id: sourceRunId, companyId: f.identity.companyId, agentId: f.identity.agentId,
|
|
nativeIssueId: f.identity.issueId, runtimeMode: "native", status: "succeeded",
|
|
runnerProfileJson: { nativeProviderPackAdmission: { ...receipt, runId: sourceRunId } } });
|
|
const original = { ...f.profile, nativeProviderPackAdmission: { ...receipt, inheritedFromRunId: sourceRunId } };
|
|
await db.update(heartbeatRuns).set({ runnerProfileJson: original }).where(eq(heartbeatRuns.id, f.identity.runId));
|
|
await expect(recordNativeProviderPackAdmission(db, f)).resolves.toBe(true);
|
|
expect((await read(f.identity.runId)).runnerProfileJson).toEqual({ ...original,
|
|
nativeProviderPackAdmissionState: { ...f.profile.nativeProviderPackAdmissionState, state: "verified" } });
|
|
});
|
|
it("leaves a newer pending epoch untouched by an older same-claim callback", async () => {
|
|
const f = await fixture(); await recordNativeProviderPackAdmission(db, f);
|
|
const previous = (await read(f.identity.runId)).runnerProfileJson!;
|
|
const next = { ...previous, nativeProviderPackAdmissionState: nativeProviderPackAdmissionPending(randomUUID()) };
|
|
await db.update(heartbeatRuns).set({ runnerProfileJson: next }).where(eq(heartbeatRuns.id, f.identity.runId));
|
|
await expect(recordNativeProviderPackAdmission(db, f)).resolves.toBe(false);
|
|
expect((await read(f.identity.runId)).runnerProfileJson).toEqual(next);
|
|
});
|
|
it.each(["runner", "lease", "unsupported"])("keeps the first receipt historical after an authorized %s replacement", async change => {
|
|
const f = await fixture(); await recordNativeProviderPackAdmission(db, f);
|
|
const previous = (await read(f.identity.runId)).runnerProfileJson!;
|
|
const admissionEpoch = randomUUID();
|
|
await db.update(heartbeatRuns).set({ runnerProfileJson: { ...previous,
|
|
nativeProviderPackAdmissionState: nativeProviderPackAdmissionPending(admissionEpoch) } }).where(eq(heartbeatRuns.id, f.identity.runId));
|
|
const replacement = sessionFixture(f.identity);
|
|
if (change !== "unsupported") rememberAdmittedNativeProviderPack(replacement.session, { identity: f.identity, manifest: manifest(),
|
|
...binding, ...(change === "runner" ? { runnerInstanceId: "replacement" } : { environmentLeaseId: "replacement" }) });
|
|
await expect(recordNativeProviderPackAdmission(db, { ...f, admissionEpoch, session: replacement.session })).resolves.toBe(false);
|
|
expect((await read(f.identity.runId)).runnerProfileJson).toEqual({ ...previous,
|
|
nativeProviderPackAdmissionState: { ...nativeProviderPackAdmissionPending(admissionEpoch), state: "unknown" } });
|
|
});
|
|
it("cannot publish while another transaction holds the coordinator claim", async () => {
|
|
const f = await fixture();
|
|
await recordNativeProviderPackAdmission(db, f);
|
|
const previous = (await read(f.identity.runId)).runnerProfileJson!;
|
|
f.admissionEpoch = randomUUID();
|
|
const pending = { ...previous, nativeProviderPackAdmissionState: nativeProviderPackAdmissionPending(f.admissionEpoch) };
|
|
await db.update(heartbeatRuns).set({ runnerProfileJson: pending }).where(eq(heartbeatRuns.id, f.identity.runId));
|
|
let release!: () => void, locked!: () => void;
|
|
const acquired = new Promise<void>(resolve => { locked = resolve; });
|
|
const hold = new Promise<void>(resolve => { release = resolve; });
|
|
const owner = db.transaction(async tx => {
|
|
await tx.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.identity.runId)).for("update");
|
|
locked(); await hold;
|
|
});
|
|
try {
|
|
await acquired;
|
|
await expect(recordNativeProviderPackAdmission(db, f)).rejects.toThrow();
|
|
expect((await read(f.identity.runId)).runnerProfileJson).toEqual(pending);
|
|
} finally { release(); await owner; }
|
|
await expect(recordNativeProviderPackAdmission(db, f)).resolves.toBe(true);
|
|
});
|
|
it.each([false, true])("waits for a concurrent run-event append's run-row lock instead of losing the admission (waitForClaim: %s)", async waitForClaim => {
|
|
// Staging failure: a run-event append locks the run row
|
|
// (FOR NO KEY UPDATE, as appendHeartbeatRunEvent does) for milliseconds. A
|
|
// NOWAIT admission write failed, left the state pending, and artifact
|
|
// retention then failed the run as native_provider_artifacts_admission_unverified.
|
|
const f = await fixture();
|
|
let release!: () => void, acquired!: () => void;
|
|
const holding = new Promise<void>(resolve => { acquired = resolve; });
|
|
const gate = new Promise<void>(resolve => { release = resolve; });
|
|
const append = db.transaction(async tx => {
|
|
await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(eq(heartbeatRuns.id, f.identity.runId)).for("no key update");
|
|
acquired(); await gate;
|
|
});
|
|
await holding;
|
|
let settled = false;
|
|
const recording = recordNativeProviderPackAdmission(db, { ...f, waitForClaim } as Parameters<typeof recordNativeProviderPackAdmission>[1])
|
|
.then(value => { settled = true; return { value }; }, error => { settled = true; return { error }; });
|
|
try {
|
|
// Release only after the admission is proven to wait on the run row, so
|
|
// the old NOWAIT code cannot pass by running after the append commits.
|
|
await vi.waitFor(async () => {
|
|
const [row] = await db.execute<{ waiting: boolean }>(sql`select exists(select 1 from pg_stat_activity
|
|
where datname = current_database() and wait_event_type = 'Lock' and state = 'active'
|
|
and query like '%heartbeat_runs%') as waiting`);
|
|
expect(settled || row.waiting).toBe(true);
|
|
}, { timeout: 800 });
|
|
} finally { release(); await append; }
|
|
expect(await recording).toEqual({ value: true });
|
|
expect((await read(f.identity.runId)).runnerProfileJson?.nativeProviderPackAdmissionState)
|
|
.toEqual({ ...f.profile.nativeProviderPackAdmissionState, state: "verified" });
|
|
});
|
|
it("keeps the admission when a checkpoint write holds the claim longer than one lock wait", async () => {
|
|
// Staging run 3f317172 (OpenCode, fresh Daytona sandbox): the pack upload
|
|
// and runner start succeeded, the runner began writing durable checkpoints,
|
|
// and the admission write lost one 1000 ms lock wait on the coordinator row
|
|
// a checkpoint transaction held. The state stayed pending, artifact
|
|
// retention failed the run as native_provider_artifacts_admission_unverified
|
|
// and the task rested blocked before its first turn.
|
|
const f = await fixture();
|
|
let release!: () => void, acquired!: () => void;
|
|
const holding = new Promise<void>(resolve => { acquired = resolve; });
|
|
const gate = new Promise<void>(resolve => { release = resolve; });
|
|
const checkpoint = db.transaction(async tx => {
|
|
await tx.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.identity.runId)).for("update");
|
|
acquired(); await gate;
|
|
});
|
|
await holding;
|
|
const recording = recordNativeProviderPackAdmission(db, { ...f, waitForClaim: true } as Parameters<typeof recordNativeProviderPackAdmission>[1])
|
|
.then(value => ({ value }), error => ({ error }));
|
|
// Longer than one lock wait, shorter than the bounded retry window.
|
|
const held = setTimeout(release, 1_600);
|
|
try {
|
|
expect(await recording).toEqual({ value: true });
|
|
} finally { clearTimeout(held); release(); await checkpoint; }
|
|
expect((await read(f.identity.runId)).runnerProfileJson?.nativeProviderPackAdmissionState)
|
|
.toEqual({ ...f.profile.nativeProviderPackAdmissionState, state: "verified" });
|
|
});
|
|
it("gives up after the bounded retry window while the claim stays held", async () => {
|
|
const f = await fixture();
|
|
let release!: () => void, acquired!: () => void;
|
|
const holding = new Promise<void>(resolve => { acquired = resolve; });
|
|
const gate = new Promise<void>(resolve => { release = resolve; });
|
|
const checkpoint = db.transaction(async tx => {
|
|
await tx.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.identity.runId)).for("update");
|
|
acquired(); await gate;
|
|
});
|
|
await holding;
|
|
const started = Date.now();
|
|
try {
|
|
await expect(recordNativeProviderPackAdmission(db, { ...f, waitForClaim: true } as Parameters<typeof recordNativeProviderPackAdmission>[1]))
|
|
.rejects.toMatchObject({ cause: { code: "55P03" } });
|
|
expect(Date.now() - started).toBeLessThan(15_000);
|
|
} finally { release(); await checkpoint; }
|
|
expect((await read(f.identity.runId)).runnerProfileJson).toEqual(f.profile);
|
|
}, 30_000);
|
|
it.each([false, true])("waits within the durable-capture bound and rechecks the claim after its lock (revoked: %s)", async revoked => {
|
|
const f = await fixture();
|
|
let release!: () => void, acquired!: () => void;
|
|
const holding = new Promise<void>(resolve => { acquired = resolve; });
|
|
const gate = new Promise<void>(resolve => { release = resolve; });
|
|
const owner = db.transaction(async tx => {
|
|
await tx.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.identity.runId)).for("update");
|
|
acquired(); await gate;
|
|
if (revoked) await tx.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 1000) })
|
|
.where(eq(nativeRunFinalizations.runId, f.identity.runId));
|
|
});
|
|
let settled = false;
|
|
await holding;
|
|
const recording = recordNativeProviderPackAdmission(db, { ...f, waitForClaim: true } as Parameters<typeof recordNativeProviderPackAdmission>[1])
|
|
.then(value => { settled = true; return { value }; }, error => { settled = true; return { error }; });
|
|
try {
|
|
await vi.waitFor(async () => {
|
|
const [row] = await db.execute<{ waiting: boolean }>(sql`select exists(select 1 from pg_stat_activity
|
|
where datname = current_database() and wait_event_type = 'Lock' and state = 'active'
|
|
and query like '%native_run_finalizations%') as waiting`);
|
|
expect(settled || row.waiting).toBe(true);
|
|
}, { timeout: 800 });
|
|
expect(settled).toBe(false);
|
|
} finally { release(); await owner; }
|
|
if (revoked) {
|
|
expect(await recording).toHaveProperty("error");
|
|
expect((await read(f.identity.runId)).runnerProfileJson?.nativeProviderPackAdmissionState).toEqual(f.profile.nativeProviderPackAdmissionState);
|
|
} else expect(await recording).toEqual({ value: true });
|
|
});
|
|
});
|