mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-03 09:37:19 +01:00
Merge qualified backup expiration parent into GitHub connection work
Co-Authored-By: Paperclip <noreply@paperclip.ing> * commit 'aae32f2f493848199f6cb8c0a9996af3ee0061b1': (30 commits) Scope status-card refresh tasks to their claim revision test(chat): retain completion identity before link metadata loads test(chat): locate completion through its rendered task link fix(chat): recover committed native Agent Chat replies (#18465) perf(heartbeat): drive retry promotion from durable deadlines (#18387) fix(ui): make the Coordinator composer footer opaque (#18455) fix(testops): verify unexpanded result job runs (#18462) test(server): observe restored Retry before draining unrelated work test(runner): own listener cleanup across setup failures test(runner): buffer upgrade frames until the wire subscribes fix(runner): preserve OpenCode written-answer defaults (#18456) fix(native): fence retries when replaying an audited Stop (#18457) test: await async outcomes and drive saved deadlines deterministically (#18416) fix(ui): retain startup autosave through session recovery (#18412) perf(server): recover independent native replies concurrently (#18421) perf(chat): scope task questions and batch inbox queue checks (#18422) fix(native): recover controllers across webserver replacement (#18343) fix(ui): restore task document authoring and add persistence journey (#18356) fix(tools): stabilize tied profile entry order (#18427) fix(adapter-utils): tolerate entries removed during workspace snapshots (#18426) ...
This commit is contained in:
commit
f522ab07e2
484 files changed
+33834
-1620
No files matched your search
@@ -1,6 +1,7 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { ACPX_QUALIFICATION_ENV, resolveAcpxQualification } from "./acpx-qualification.js";
|
||||
import { ACPX_QUALIFICATION_ENV, ANTIGRAVITY_CANDIDATE_ADMISSION_ENV, resolveAcpxQualification, resolveKimiCandidateAdmission } from "./acpx-qualification.js";
|
||||
import { QUALIFIED_ACPX_PROFILES, KIMI_DISTRIBUTION_CLOSURE_SHA256 } from "../../vendor/paperclip-runner/index.js";
|
||||
import type { NativeExecutionInput } from "../../vendor/paperclip-runner/index.js";
|
||||
import { resolvePaperclipRunnerProviderProfile, resolvePaperclipRunnerNativeProviderInput } from "./provider-profile.js";
|
||||
|
||||
@@ -9,6 +10,20 @@ const authorize = (value: unknown) => ({ [ACPX_QUALIFICATION_ENV]: JSON.stringif
|
||||
describe("host ACPX qualification admission", () => {
|
||||
afterEach(() => vi.unstubAllEnvs());
|
||||
|
||||
it.each([undefined, "malformed", JSON.stringify(kimiAdmission("linux", "x64"))])("admits released Kimi without candidate authority (%s)", authority => {
|
||||
vi.stubEnv(ANTIGRAVITY_CANDIDATE_ADMISSION_ENV, authority);
|
||||
vi.stubEnv(ACPX_QUALIFICATION_ENV, "malformed-unrelated-authority");
|
||||
const config = { provider: "acpx", acpxAgent: "kimi", model: "kimi-for-coding" };
|
||||
expect(QUALIFIED_ACPX_PROFILES.kimi.agentProfileVersion).toBe(17);
|
||||
expect(QUALIFIED_ACPX_PROFILES.kimi.qualificationStatus).not.toBe("pending");
|
||||
expect(resolveKimiCandidateAdmission({ companyId: "candidate-company", model: config.model })).toBeUndefined();
|
||||
expect(resolvePaperclipRunnerProviderProfile(config, { companyId: "candidate-company" })).toMatchObject(config);
|
||||
expect(resolvePaperclipRunnerNativeProviderInput({ companyId: "candidate-company", backend: "acpx_runtime", adapterConfig: config }))
|
||||
.toMatchObject({ ...config, acpxPermissionMode: "approve-all" });
|
||||
expect(() => resolvePaperclipRunnerProviderProfile({ ...config, model: "" }))
|
||||
.toThrow(expect.objectContaining({ code: "paperclip_runner_acpx_model_required" }));
|
||||
});
|
||||
|
||||
it.each([undefined, "malformed", JSON.stringify([{ agent: "antigravity", model: "exact-model" }])])(
|
||||
"keeps pending Antigravity closed despite host authorization (%s)", host => {
|
||||
vi.stubEnv(ACPX_QUALIFICATION_ENV, host);
|
||||
@@ -97,3 +112,9 @@ it.each([undefined, "off", "low", "high", "max"] as const)("validates saved Pi t
|
||||
expect(resolvePaperclipRunnerNativeProviderInput({ backend: "acpx_runtime", adapterConfig })).toMatchObject({ piThinkingLevel: piThinkingLevel ?? "low" });
|
||||
for (const alias of ["medium", "minimal", "xhigh"]) expect(() => resolvePaperclipRunnerProviderProfile({ ...adapterConfig, piThinkingLevel: alias })).toThrow(expect.objectContaining({ code: "paperclip_runner_pi_thinking_invalid" }));
|
||||
});
|
||||
|
||||
function kimiAdmission(platform: "darwin" | "linux", architecture: "arm64" | "x64") {
|
||||
return { schema: "paperclip.acpx-candidate-admission.v1", companyId: "candidate-company", agent: "kimi", model: "kimi-for-coding",
|
||||
platform, architecture, expiresAt: Date.now() + 60_000, profileDigest: QUALIFIED_ACPX_PROFILES.kimi.commandDigest,
|
||||
nativeClosureDigest: `sha256:${KIMI_DISTRIBUTION_CLOSURE_SHA256[`${platform}-${architecture}` as "darwin-arm64" | "linux-x64"]}` };
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import { PAPERCLIP_RUNNER_ACPX_PROFILES } from "@paperclipai/adapter-utils";
|
||||
import { parseAcpxCandidateAdmission, type AcpxCandidateAdmission, type NativeExecutionInput } from "../../vendor/paperclip-runner/index.js";
|
||||
import { parseAcpxCandidateAdmission, QUALIFIED_ACPX_PROFILES, type AcpxCandidateAdmission, type NativeExecutionInput } from "../../vendor/paperclip-runner/index.js";
|
||||
|
||||
/** Trusted operator startup authority, separate from saved agent configuration. */
|
||||
export const ANTIGRAVITY_CANDIDATE_ADMISSION_ENV = "PAPERCLIP_ACPX_CANDIDATE_ADMISSION";
|
||||
@@ -23,9 +23,31 @@ export function resolveAntigravityCandidateAdmission(
|
||||
});
|
||||
}
|
||||
|
||||
export function resolveKimiCandidateAdmission(
|
||||
input: { companyId?: string; model?: string; target?: { platform: string; architecture: string } },
|
||||
hostEnvironment: NodeJS.ProcessEnv = process.env,
|
||||
): AcpxCandidateAdmission | undefined {
|
||||
if (QUALIFIED_ACPX_PROFILES.kimi.qualificationStatus !== "pending") return undefined;
|
||||
const encoded = hostEnvironment[ANTIGRAVITY_CANDIDATE_ADMISSION_ENV];
|
||||
if (!encoded || !input.companyId) return undefined;
|
||||
let value: unknown;
|
||||
try { value = JSON.parse(encoded); } catch { throw new Error("Invalid Kimi candidate admission"); }
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("Invalid Kimi candidate admission");
|
||||
const record = value as Record<string, unknown>;
|
||||
if (record.companyId !== input.companyId || record.agent !== "kimi") return undefined;
|
||||
const admission = parseAcpxCandidateAdmission(value, {
|
||||
companyId: input.companyId, agent: "kimi",
|
||||
model: input.model ?? (typeof record.model === "string" ? record.model : ""),
|
||||
});
|
||||
if (input.target && (admission.platform !== input.target.platform || admission.architecture !== input.target.architecture)) {
|
||||
throw new Error("Kimi candidate admission does not match the execution target");
|
||||
}
|
||||
return admission;
|
||||
}
|
||||
|
||||
/** Operator-only admission for an exact candidate/model during qualification. */
|
||||
export const ACPX_QUALIFICATION_ENV = "PAPERCLIP_RUNNER_ACPX_QUALIFICATION";
|
||||
export type AcpxQualificationCandidate = "cursor" | "copilot" | "pi";
|
||||
export type AcpxQualificationCandidate = "cursor" | "copilot" | "pi" | "kimi";
|
||||
|
||||
export function resolveAcpxQualification(
|
||||
provider: NativeExecutionInput["provider"] | { kind: "acpx"; agent: string; model: string },
|
||||
@@ -45,14 +67,14 @@ export function resolveAcpxQualification(
|
||||
if (!encoded) return undefined;
|
||||
let entries: unknown;
|
||||
try { entries = JSON.parse(encoded); } catch { throw new Error("Invalid ACPX qualification authorization"); }
|
||||
if (!Array.isArray(entries) || entries.length < 1 || entries.length > 3) {
|
||||
if (!Array.isArray(entries) || entries.length < 1 || entries.length > 4) {
|
||||
throw new Error("Invalid ACPX qualification authorization");
|
||||
}
|
||||
const agents = new Set<string>();
|
||||
for (const entry of entries) {
|
||||
if (!entry || typeof entry !== "object" || Array.isArray(entry)
|
||||
|| Object.keys(entry).sort().join(",") !== "agent,model"
|
||||
|| !["cursor", "copilot", "pi"].includes(entry.agent)
|
||||
|| !["cursor", "copilot", "pi", "kimi"].includes(entry.agent)
|
||||
|| typeof entry.model !== "string" || !entry.model.trim() || entry.model !== entry.model.trim()
|
||||
|| agents.has(entry.agent)) throw new Error("Invalid ACPX qualification authorization");
|
||||
agents.add(entry.agent);
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { agents, authUsers, companies, completionContracts, createDb, heartbeatRunEvents, heartbeatRuns, issueComments, issues,
|
||||
nativeRunFinalizations, nativeRunResults, type Db } from "@paperclipai/db";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
|
||||
import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js";
|
||||
import { NativeRunCoordinatorStore } from "./native-run-coordinator-store.js";
|
||||
import { finalizeNativeRun, recoverCommittedNativeChatResponse } from "./native-run-finalizer.js";
|
||||
import { nativeResponsePresentationWork } from "./native-response-presentation-work.js";
|
||||
|
||||
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_AGENT_CHAT_RESPONSE_TEST_DATABASE_URL");
|
||||
const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
describe.skipIf(!support.supported)("committed Agent Chat reply recovery", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
|
||||
let db: Db;
|
||||
beforeAll(async () => {
|
||||
if (!external) database = await startEmbeddedPostgresTestDatabase("agent-chat-response-");
|
||||
db = createDb(external ?? database!.connectionString);
|
||||
}, 60_000);
|
||||
afterAll(async () => { await db?.$client.end(); await database?.cleanup(); });
|
||||
|
||||
async function fixture(conversation = true) {
|
||||
const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID();
|
||||
const sessionId = randomUUID(), runnerId = randomUUID(), userId = `fixture-${randomUUID()}`, wakeId = randomUUID();
|
||||
await db.insert(authUsers).values({ id: userId, name: "Conversation owner", email: `${userId}@example.test`,
|
||||
createdAt: new Date("2020-01-01T00:00:00Z"), updatedAt: new Date("2020-01-01T00:00:00Z") });
|
||||
await db.insert(companies).values({ id: companyId, name: "Reply recovery fixture", issuePrefix: companyId.slice(0, 8) });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Conversation agent", adapterType: "paperclip_runner" });
|
||||
await db.insert(issues).values({ id: issueId, companyId, title: "Conversation reply", status: "in_progress", assigneeAgentId: agentId,
|
||||
...(conversation ? { conversationAgentId: agentId, conversationUserId: userId, conversationState: "active", conversationSessionGeneration: 0 } : {}) });
|
||||
await db.insert(issueComments).values({ id: wakeId, companyId, issueId, authorUserId: userId, body: "Hire one teammate and tell me when ready.",
|
||||
createdAt: new Date("2020-01-01T00:00:00Z") });
|
||||
const [contract] = await db.insert(completionContracts).values({ companyId, issueId, revision: 1,
|
||||
schemaVersion: "paperclip.completion-contract.v1", policyVersion: "test", risk: "standard", completionAuthority: "server_arbiter",
|
||||
incompleteCriteriaPolicy: "preserve_non_terminal", contractJson: { revision: "test", objective: "Answer the user", criteria: [] },
|
||||
canonicalSha256: "fixture-conversation-contract", createdByActorType: "system", createdByActorId: "test" }).returning();
|
||||
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: issueId, nativeSessionId: sessionId,
|
||||
runnerInstanceId: runnerId, completionContractId: contract.id, completionContractSha256: contract.canonicalSha256,
|
||||
status: "running", runtimeMode: "native", responsibleUserId: userId,
|
||||
contextSnapshot: { issueId, source: "issue.comment", wakeCommentId: wakeId, ...(conversation ? { conversationSessionGeneration: 0 } : {}) } });
|
||||
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId));
|
||||
const port = new NativeRunCoordinatorStore(db, { companyId, issueId, runId, agentId, normalizedSessionId: sessionId,
|
||||
runnerSourceInstanceId: runnerId, completionContractId: contract.id, completionContractSha256: contract.canonicalSha256,
|
||||
completionContractRevision: "test", completionContractCriterionIds: [] });
|
||||
const reply = "The teammate is hired and ready for your next instruction.";
|
||||
const prpEvent = { runId, normalizedSessionId: sessionId, turnId: "hire-turn", sourceEventId: "fixture-final-answer",
|
||||
payload: { kind: "agentMessage", channel: "final", providerPhase: "final", text: reply } };
|
||||
await db.insert(heartbeatRunEvents).values({ companyId, runId, agentId, seq: 170, eventType: "item.completed", payload: { prpEvent } });
|
||||
await port.completeRun({ result: { schema: "paperclip.run_result.v1", reportedWorkDisposition: "done",
|
||||
summary: "A completed hire, with its final message preserved separately.", completionClaim: { contractRevision: "test",
|
||||
objectiveSatisfied: true, criteria: [], remainingWork: [] }, evidence: [], verification: [], attentionRequests: [], artifacts: [] },
|
||||
terminal: { schema: "paperclip.prp.terminal.v1", runTerminalState: "succeeded", turnTerminalState: "completed", reportedWorkDisposition: "done" }, turnId: "hire-turn" });
|
||||
await finalizeNativeRun({ db, runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: false });
|
||||
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date("2020-01-01T00:02:00Z") }).where(eq(heartbeatRuns.id, runId));
|
||||
expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)))[0].phase).toBe("committed");
|
||||
const input = { companyId, issueId, runId };
|
||||
return { ...input, input, agentId, userId, wakeId, prpEvent, reply };
|
||||
}
|
||||
const comments = (runId: string) => db.select().from(issueComments).where(eq(issueComments.createdByRunId, runId));
|
||||
const issue = async (issueId: string) => (await db.select().from(issues).where(eq(issues.id, issueId)))[0];
|
||||
|
||||
it("finds a committed final Agent Chat turn, persists the exact reply once and only then settles waiting", async () => {
|
||||
const f = await fixture();
|
||||
expect(await comments(f.runId)).toEqual([]);
|
||||
expect(await issue(f.issueId)).toMatchObject({ status: "in_progress", conversationState: "active" });
|
||||
const work = nativeResponsePresentationWork(db), entries = await work.load(f.input);
|
||||
expect(entries).toEqual([expect.objectContaining({ id: f.runId })]);
|
||||
await expect(work.check(entries[0])).resolves.toBeNull();
|
||||
expect(await comments(f.runId)).toEqual([expect.objectContaining({ body: f.reply, authorAgentId: f.agentId, createdByRunId: f.runId })]);
|
||||
expect(await issue(f.issueId)).toMatchObject({ status: "in_review", conversationState: "waiting" });
|
||||
expect(await work.load(f.input)).toEqual([]);
|
||||
await expect(recoverCommittedNativeChatResponse(db, f.input)).resolves.toBe(false);
|
||||
expect(await comments(f.runId)).toHaveLength(1);
|
||||
});
|
||||
it("keeps a later user message actionable after recovering the earlier exact reply", async () => {
|
||||
const f = await fixture();
|
||||
await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId, authorUserId: f.userId, body: "Another instruction",
|
||||
createdAt: new Date("2020-01-01T00:01:00Z") });
|
||||
await expect(recoverCommittedNativeChatResponse(db, f.input)).resolves.toBe(true);
|
||||
expect(await comments(f.runId)).toHaveLength(1);
|
||||
expect(await issue(f.issueId)).toMatchObject({ status: "in_progress", conversationState: "active" });
|
||||
});
|
||||
it("does not duplicate a reply under concurrent recovery", async () => {
|
||||
const f = await fixture();
|
||||
expect((await Promise.all([recoverCommittedNativeChatResponse(db, f.input), recoverCommittedNativeChatResponse(db, f.input)])).sort())
|
||||
.toEqual([false, true]);
|
||||
expect(await comments(f.runId)).toHaveLength(1);
|
||||
});
|
||||
it.each(["skip", "generation", "agent", "user", "execution", "waiting", "turn", "event-run", "session", "channel", "kind", "digest", "partial"])(
|
||||
"keeps %s mismatches unchanged instead of publishing an unauthorized or incomplete reply", async fault => {
|
||||
const f = await fixture();
|
||||
if (["skip", "generation"].includes(fault)) await db.update(heartbeatRuns).set({ contextSnapshot: { issueId: f.issueId,
|
||||
source: "issue.comment", wakeCommentId: f.wakeId, conversationSessionGeneration: fault === "generation" ? 1 : 0,
|
||||
...(fault === "skip" ? { skipIssueComment: true } : {}) } }).where(eq(heartbeatRuns.id, f.runId));
|
||||
if (fault === "user") await db.update(heartbeatRuns).set({ responsibleUserId: "another-user" }).where(eq(heartbeatRuns.id, f.runId));
|
||||
if (fault === "agent") {
|
||||
const other = randomUUID(); await db.insert(agents).values({ id: other, companyId: f.companyId, name: "Other agent" });
|
||||
await db.update(issues).set({ conversationAgentId: other, assigneeAgentId: other }).where(eq(issues.id, f.issueId));
|
||||
}
|
||||
if (fault === "execution") {
|
||||
const other = randomUUID(); await db.insert(heartbeatRuns).values({ id: other, companyId: f.companyId, agentId: f.agentId });
|
||||
await db.update(issues).set({ executionRunId: other }).where(eq(issues.id, f.issueId));
|
||||
}
|
||||
if (fault === "waiting") await db.update(issues).set({ executionRunId: null, status: "in_review", conversationState: "waiting" }).where(eq(issues.id, f.issueId));
|
||||
if (["turn", "event-run", "session", "channel", "kind"].includes(fault)) {
|
||||
const event = structuredClone(f.prpEvent);
|
||||
if (fault === "turn") event.turnId = "foreign-turn";
|
||||
if (fault === "event-run") event.runId = randomUUID();
|
||||
if (fault === "session") event.normalizedSessionId = randomUUID();
|
||||
if (fault === "channel") event.payload.channel = "analysis";
|
||||
if (fault === "kind") event.payload.kind = "reasoning";
|
||||
await db.update(heartbeatRunEvents).set({ payload: { prpEvent: event } }).where(and(eq(heartbeatRunEvents.runId, f.runId), eq(heartbeatRunEvents.seq, 170)));
|
||||
}
|
||||
if (fault === "digest") await db.update(nativeRunResults).set({ canonicalSha256: "invalid-digest" }).where(eq(nativeRunResults.runId, f.runId));
|
||||
if (fault === "partial") await db.update(nativeRunFinalizations).set({ phase: "copying_back" }).where(eq(nativeRunFinalizations.runId, f.runId));
|
||||
const before = await issue(f.issueId);
|
||||
await expect(recoverCommittedNativeChatResponse(db, f.input)).resolves.toBe(false);
|
||||
expect(await comments(f.runId)).toEqual([]); expect(await issue(f.issueId)).toEqual(before);
|
||||
},
|
||||
);
|
||||
it.each([false, true])("preserves a selected reply without replacing or resurrecting it (deleted=%s)", async deleted => {
|
||||
const f = await fixture();
|
||||
const [saved] = await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId, authorAgentId: f.agentId,
|
||||
createdByRunId: f.runId, body: "Already selected", ...(deleted ? { deletedAt: new Date("2020-01-01T00:03:00Z") } : {}) }).returning();
|
||||
await db.update(heartbeatRuns).set({ resultJson: { presentationDecision: { commentId: saved.id, commentAction: "create" } } }).where(eq(heartbeatRuns.id, f.runId));
|
||||
await expect(recoverCommittedNativeChatResponse(db, f.input)).resolves.toBe(false);
|
||||
expect(await comments(f.runId)).toEqual([saved]);
|
||||
});
|
||||
it("never crosses company scope and keeps ordinary internal task response recovery intact", async () => {
|
||||
const f = await fixture(false);
|
||||
await expect(recoverCommittedNativeChatResponse(db, { ...f.input, companyId: randomUUID() })).resolves.toBe(false);
|
||||
const work = nativeResponsePresentationWork(db), entries = await work.load(f.input);
|
||||
expect(entries).toHaveLength(1); await expect(work.check(entries[0])).resolves.toBeNull();
|
||||
expect(await comments(f.runId)).toEqual([expect.objectContaining({ body: f.reply })]);
|
||||
expect((await issue(f.issueId)).conversationAgentId).toBeNull();
|
||||
});
|
||||
it("rolls back a partial reply write without changing the waiting state or consuming recovery", async () => {
|
||||
const f = await fixture(), abort = new Error("fixture-commit-aborted");
|
||||
const failing = new Proxy(db, { get(target, key) { return key === "transaction"
|
||||
? (callback: Parameters<Db["transaction"]>[0]) => db.transaction(async tx => { await callback(tx); throw abort; })
|
||||
: Reflect.get(target, key); } });
|
||||
await expect(recoverCommittedNativeChatResponse(failing, f.input)).rejects.toBe(abort);
|
||||
expect(await comments(f.runId)).toEqual([]);
|
||||
expect(await issue(f.issueId)).toMatchObject({ status: "in_progress", conversationState: "active" });
|
||||
await expect(recoverCommittedNativeChatResponse(db, f.input)).resolves.toBe(true);
|
||||
expect(await comments(f.runId)).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,61 @@
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { sql } from "drizzle-orm";
|
||||
import { inspectNativeRunnerUpgradeHeader } from "./native-runner-checkpoint.js";
|
||||
|
||||
/** An explicit, paginated diagnostic. Never called by an idle/background scan.
|
||||
* This inventories selected retained task-session pointers, not all history.
|
||||
* Header evidence is preliminary: full snapshot/journal validation still occurs
|
||||
* only when ordinary admission selects its exact continuation source. */
|
||||
export async function readNativeControllerRecoveryInventory(db: Db, input: { afterId?: string; limit?: number } = {}) {
|
||||
if (input.limit !== undefined && !Number.isFinite(input.limit)) throw new Error("native_recovery_inventory_limit_invalid");
|
||||
const limit = Math.min(128, Math.max(1, Math.floor(input.limit ?? 128)));
|
||||
const rows = await db.transaction(async tx => {
|
||||
await tx.execute(sql`set local statement_timeout = '2s'`);
|
||||
return tx.execute<{
|
||||
id: string; runId: string | null; companyId: string; issueId: string | null; sessionId: string | null;
|
||||
runnerId: string | null; metadata: Record<string, unknown> | null; material: Record<string, unknown> | null;
|
||||
pending: boolean;
|
||||
}>(sql`with selected as (
|
||||
select id, company_id, agent_id, last_run_id from agent_task_sessions
|
||||
where adapter_type = 'paperclip_runner' ${input.afterId ? sql`and id > ${input.afterId}::uuid` : sql``}
|
||||
order by id limit ${limit + 1}
|
||||
) select s.id, s.company_id as "companyId", r.id as "runId", r.native_issue_id as "issueId",
|
||||
r.native_session_id as "sessionId", r.runner_instance_id as "runnerId",
|
||||
case when pg_column_size(r.runner_profile_json->'nativeControllerRecovery') <= 4096
|
||||
then r.runner_profile_json->'nativeControllerRecovery' else null end as metadata,
|
||||
case when pg_column_size(c.material) <= 262144 and sum(pg_column_size(c.material)) over () <= 8388608
|
||||
then c.material else null end as material,
|
||||
(r.finished_at is null or r.status not in ('succeeded', 'failed', 'cancelled', 'interrupted', 'timed_out')
|
||||
or f.run_id is null or f.lease_owner is not null or f.recovery_state is not null
|
||||
or f.phase not in ('committed', 'terminal_failure')) is true as pending
|
||||
from selected s left join heartbeat_runs r on r.id = s.last_run_id and r.company_id = s.company_id and r.agent_id = s.agent_id
|
||||
left join native_run_finalizations f on f.run_id = r.id and f.company_id = s.company_id
|
||||
left join native_runner_checkpoints c on c.run_id = r.id and c.company_id = s.company_id
|
||||
and c.issue_id = r.native_issue_id and c.part = 'header'
|
||||
order by s.id`);
|
||||
}, { accessMode: "read only" });
|
||||
const truncated = rows.length > limit;
|
||||
const page = rows.slice(0, limit);
|
||||
const reasons: Record<string, number> = {};
|
||||
let selectedRetainedEvidence = 0, outstandingObligations = 0;
|
||||
for (const row of page) {
|
||||
let reason: string | null = null;
|
||||
if (row.pending) { outstandingObligations++; reason = "persistence_or_handoff_pending"; }
|
||||
else if (!row.runId || !row.issueId || !row.sessionId) reason = "selected_binding_missing";
|
||||
else if (row.metadata?.schema !== "paperclip.native-controller-recovery.v1") reason = "metadata_unsupported_or_missing";
|
||||
else {
|
||||
const checkpoint = row.metadata.checkpoint as Record<string, unknown> | null;
|
||||
const header = row.material ? inspectNativeRunnerUpgradeHeader({ companyId: row.companyId, issueId: row.issueId, runId: row.runId }, row.material) : null;
|
||||
if (!checkpoint || checkpoint.runId !== row.runId || checkpoint.runnerInstanceId !== row.runnerId
|
||||
|| checkpoint.normalizedSessionId !== row.sessionId || !header
|
||||
|| header.runnerInstanceId !== row.runnerId || header.normalizedSessionId !== row.sessionId) reason = "checkpoint_binding_missing";
|
||||
else selectedRetainedEvidence++;
|
||||
}
|
||||
if (reason) reasons[reason] = (reasons[reason] ?? 0) + 1;
|
||||
}
|
||||
return { version: 1 as const, observedAt: new Date().toISOString(), inspected: page.length,
|
||||
coverage: truncated || input.afterId ? "partial" as const : "selected_task_session_pointers" as const,
|
||||
truncated, nextAfterId: truncated ? page.at(-1)!.id : null,
|
||||
selectedRetainedEvidence, outstandingObligations, reasons,
|
||||
fullPayloadsValidated: false as const };
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { agentTaskSessions, agents, companies, createDb, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults } from "@paperclipai/db";
|
||||
import { DurablePrpControlPlane, type PersistedNativeSession } from "../../vendor/paperclip-runner/index.js";
|
||||
import { startRecoveryPostgres, recoveryPostgresSupport } from "../../__tests__/helpers/recovery-postgres.js";
|
||||
import { createIsolatedTestHome } from "../../__tests__/helpers/isolated-test-home.js";
|
||||
import { prepareNativeHeartbeatRun } from "./prepare-native-run.js";
|
||||
import { nativeRunnerCheckpointWriter } from "./native-runner-checkpoint.js";
|
||||
import { createNativeControllerRecovery, deferNativeControllerRecovery, importNativeControllerRecovery, NATIVE_RETAINED_OWNER_WAIT, resolveNativeControllerRecovery } from "./native-controller-recovery.js";
|
||||
import { claimNativeSessionResumptions } from "./native-finalization-reconciler.js";
|
||||
import { claimNativeRestartRecoveries } from "./native-restart-recovery.js";
|
||||
import { readNativeControllerRecoveryInventory } from "./native-controller-recovery-inventory.js";
|
||||
|
||||
describe.skipIf(!recoveryPostgresSupport.supported)("native controller durable discovery and import", () => {
|
||||
let database: Awaited<ReturnType<typeof startRecoveryPostgres>>;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let home: ReturnType<typeof createIsolatedTestHome>;
|
||||
let root: string;
|
||||
const previousHome = process.env.PAPERCLIP_HOME;
|
||||
const companyId = randomUUID(), agentId = randomUUID();
|
||||
const compatibility = { configDigest: `sha256:${"a".repeat(64)}`, processConfigDigest: null };
|
||||
beforeAll(async () => {
|
||||
home = createIsolatedTestHome();
|
||||
process.env.PAPERCLIP_HOME = home.home;
|
||||
root = await mkdtemp(join(tmpdir(), "native-controller-discovery-"));
|
||||
database = await startRecoveryPostgres("native-controller-discovery-");
|
||||
db = createDb(database.connectionString);
|
||||
await db.insert(companies).values({ id: companyId, name: "Controller discovery", issuePrefix: "NCD" });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Runner", adapterType: "paperclip_runner", adapterConfig: { provider: "codex" } });
|
||||
}, 60_000);
|
||||
afterAll(async () => {
|
||||
await database?.cleanup();
|
||||
if (root) await rm(root, { recursive: true, force: true });
|
||||
home?.cleanup();
|
||||
if (previousHome === undefined) delete process.env.PAPERCLIP_HOME;
|
||||
else process.env.PAPERCLIP_HOME = previousHome;
|
||||
});
|
||||
|
||||
async function seed() {
|
||||
const issueId = randomUUID(), runId = randomUUID();
|
||||
await db.insert(issues).values({ id: issueId, companyId, title: "Continue retained execution", assigneeAgentId: agentId, status: "in_progress" });
|
||||
const [run] = await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running", runtimeMode: "native", nativeIssueId: issueId }).returning();
|
||||
const native = await prepareNativeHeartbeatRun({ db, run, issue: { id: issueId, title: "Continue retained execution", description: null, reviewPolicy: null }, environmentLeaseId: randomUUID() });
|
||||
const owner = { companyId, issueId, runId, authorityRunId: runId, bootId: randomUUID(), generation: 1, leaseOwner: randomUUID() };
|
||||
await db.insert(nativeRunFinalizations).values({ companyId, issueId, runId, phase: "observed", controllerBootId: owner.bootId,
|
||||
controllerGeneration: 1, leaseOwner: owner.leaseOwner, leaseExpiresAt: new Date(Date.now() + 60_000) });
|
||||
const core = new DurablePrpControlPlane({ stateDirectory: join(root, runId),
|
||||
identity: { runId, normalizedSessionId: native.normalizedSessionId, runnerInstanceId: native.runnerInstanceId,
|
||||
environmentLeaseId: native.environmentLeaseId, turnId: native.turnId, itemId: native.itemId },
|
||||
expectedRunnerVersion: "discovery-fixture", expectedRunnerDigest: `sha256:${"a".repeat(64)}` });
|
||||
core.setCheckpointPersistence(nativeRunnerCheckpointWriter(db, owner));
|
||||
await core.flushCheckpoint();
|
||||
await core.stop();
|
||||
const snapshot: PersistedNativeSession = { backendKind: "mock", sessionId: native.normalizedSessionId,
|
||||
identity: { companyId, agentId, issueId, runId, sessionId: native.normalizedSessionId } };
|
||||
const [prepared] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
|
||||
const [result] = await db.insert(nativeRunResults).values({ companyId, issueId, runId,
|
||||
completionContractId: prepared.completionContractId!, serverFingerprint: "fixture", schemaStatus: "accepted",
|
||||
resultJson: {}, canonicalSha256: "fixture" }).returning();
|
||||
await db.update(nativeRunFinalizations).set({ phase: "committed", resultId: result.id, leaseOwner: null, leaseExpiresAt: null }).where(eq(nativeRunFinalizations.runId, runId));
|
||||
const profile = { ...prepared.runnerProfileJson, sessionCheckpoint: snapshot };
|
||||
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date(), runnerProfileJson: profile }).where(eq(heartbeatRuns.id, runId));
|
||||
const scope = { companyId, agentId, issueId, runId, normalizedSessionId: native.normalizedSessionId };
|
||||
return { scope, native, profile, snapshot, owner,
|
||||
supervisor: { schema: "paperclip.native-session-supervisor.v1", ...compatibility, snapshot } };
|
||||
}
|
||||
|
||||
it("imports a retained source once, rejects stale input, and resolves it without any local projection", async () => {
|
||||
const fixture = await seed();
|
||||
const input = { ...fixture.scope, expectedConfigDigest: compatibility.configDigest, supervisor: fixture.supervisor };
|
||||
expect(await importNativeControllerRecovery(db, { ...input, supervisor: { ...input.supervisor, snapshot: { ...fixture.snapshot, providerSessionId: "stale" } } }))
|
||||
.toEqual({ kind: "missing_evidence", reason: "stale_supervisor_snapshot" });
|
||||
expect(await importNativeControllerRecovery(db, input)).toEqual({ kind: "imported" });
|
||||
expect(await importNativeControllerRecovery(db, input)).toEqual({ kind: "unchanged" });
|
||||
await rm(join(root, fixture.scope.runId), { recursive: true });
|
||||
const currentRunId = randomUUID();
|
||||
await db.insert(heartbeatRuns).values({ id: currentRunId, companyId, agentId, nativeIssueId: fixture.scope.issueId,
|
||||
nativeSessionId: fixture.scope.normalizedSessionId, status: "running", runtimeMode: "native" });
|
||||
const recovered = await resolveNativeControllerRecovery(db, { ...fixture.scope, currentRunId });
|
||||
expect(recovered.kind).toBe("recoverable");
|
||||
if (recovered.kind !== "recoverable") throw new Error(JSON.stringify(recovered));
|
||||
expect(recovered.snapshot).toEqual(fixture.snapshot);
|
||||
expect(recovered.checkpoint.recoveryEvidence.runnerInstanceId).toBe(fixture.native.runnerInstanceId);
|
||||
const deadline = new Date(Date.now() + 60_000);
|
||||
await db.update(nativeRunFinalizations).set({ leaseOwner: "other-controller", leaseExpiresAt: deadline }).where(eq(nativeRunFinalizations.runId, fixture.scope.runId));
|
||||
expect(await importNativeControllerRecovery(db, input)).toEqual({ kind: "waiting_for_owner", deadline });
|
||||
expect(await resolveNativeControllerRecovery(db, { ...fixture.scope, currentRunId })).toEqual({ kind: "waiting_for_owner", deadline });
|
||||
});
|
||||
|
||||
it("holds mismatched snapshots and runner bindings instead of falling back to fresh execution", async () => {
|
||||
const fixture = await seed(), currentRunId = randomUUID();
|
||||
const metadata = createNativeControllerRecovery(fixture.snapshot, compatibility, { runId: fixture.scope.runId,
|
||||
runnerInstanceId: fixture.native.runnerInstanceId, normalizedSessionId: fixture.scope.normalizedSessionId });
|
||||
await db.insert(heartbeatRuns).values({ id: currentRunId, companyId, agentId, nativeIssueId: fixture.scope.issueId,
|
||||
nativeSessionId: fixture.scope.normalizedSessionId, status: "running", runtimeMode: "native" });
|
||||
for (const envelope of [{ ...metadata, snapshotSha256: "wrong" }, { ...metadata, checkpoint: { ...metadata.checkpoint!, runnerInstanceId: randomUUID() } }]) {
|
||||
await db.update(heartbeatRuns).set({ runnerProfileJson: { ...fixture.profile, nativeControllerRecovery: envelope } }).where(eq(heartbeatRuns.id, fixture.scope.runId));
|
||||
expect(await resolveNativeControllerRecovery(db, { ...fixture.scope, currentRunId }))
|
||||
.toMatchObject({ kind: "missing_evidence", reason: "snapshot_digest_or_checkpoint_binding" });
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps an unstarted follow-up on the same run until the retained owner's deadline", async () => {
|
||||
const fixture = await seed(), runId = randomUUID();
|
||||
const scope = { ...fixture.scope, runId };
|
||||
const deadline = new Date(Date.now() + 60_000);
|
||||
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: scope.issueId,
|
||||
nativeSessionId: scope.normalizedSessionId, status: "running", runtimeMode: "native",
|
||||
// The admission boundary has already stored its input; the deferral
|
||||
// and resumption owners never execute this transport-independent fixture.
|
||||
runnerProfileJson: { nativeExecutionInput: { binding: scope } } });
|
||||
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, scope.issueId));
|
||||
await db.insert(nativeRunFinalizations).values({ companyId, issueId: scope.issueId, runId, phase: "observed" });
|
||||
await deferNativeControllerRecovery(db, { ...scope, retryAt: deadline });
|
||||
const read = () => db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId));
|
||||
expect((await read())[0]).toMatchObject({ phase: "retryable_failure", attempt: 0, leaseOwner: null,
|
||||
nextAttemptAt: deadline, failureCode: NATIVE_RETAINED_OWNER_WAIT });
|
||||
expect(await claimNativeSessionResumptions({ db, runnerInstanceId: "successor", runIds: [runId] })).toEqual([]);
|
||||
// Even after expiry, restart adoption cannot reinterpret this pre-provider
|
||||
// wait as a crashed provider. The ordinary resumption owner consumes it.
|
||||
expect(await claimNativeRestartRecoveries({ db, runIds: [runId], restartKind: "hard", now: deadline })).toEqual([]);
|
||||
const claims = await claimNativeSessionResumptions({ db, runnerInstanceId: "successor", runIds: [runId], now: deadline });
|
||||
expect(claims).toHaveLength(1);
|
||||
const nextDeadline = new Date(deadline.getTime() + 60_000);
|
||||
await deferNativeControllerRecovery(db, { ...scope, leaseOwner: claims[0].leaseOwner, retryAt: nextDeadline });
|
||||
expect((await read())[0]).toMatchObject({ attempt: 0, leaseOwner: null, nextAttemptAt: nextDeadline });
|
||||
await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, runId));
|
||||
expect(await claimNativeSessionResumptions({ db, runnerInstanceId: "successor", runIds: [runId], now: nextDeadline })).toEqual([]);
|
||||
});
|
||||
|
||||
it.each(["live-owner", "expired-owner", "provider-attempt", "stopped", "operator-stop"])("does not publish an admission wait after authority changes: %s", async scenario => {
|
||||
const fixture = await seed(), runId = randomUUID();
|
||||
const scope = { ...fixture.scope, runId };
|
||||
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: scope.issueId,
|
||||
status: scenario === "stopped" ? "cancelled" : "running", runtimeMode: "native",
|
||||
resultJson: scenario === "operator-stop" ? { cancellation: { source: "operator", expected: true,
|
||||
initiator: { type: "user" }, reason: "Stop", recordedAt: new Date().toISOString() } } : null,
|
||||
runnerProfileJson: { nativeExecutionInput: { binding: scope } } });
|
||||
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, scope.issueId));
|
||||
await db.insert(nativeRunFinalizations).values({ companyId, issueId: scope.issueId, runId, phase: "observed",
|
||||
attempt: scenario === "provider-attempt" ? 1 : 0,
|
||||
leaseOwner: scenario.endsWith("-owner") ? "other-controller" : null,
|
||||
leaseExpiresAt: scenario === "expired-owner" ? new Date(Date.now() - 1_000) : null });
|
||||
await expect(deferNativeControllerRecovery(db, { ...scope,
|
||||
leaseOwner: scenario === "expired-owner" ? "other-controller" : undefined,
|
||||
retryAt: new Date(Date.now() + 60_000) }))
|
||||
.rejects.toThrow("wait_authority_changed");
|
||||
expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)))[0].phase).toBe("observed");
|
||||
});
|
||||
|
||||
it("reports bounded retained-pointer coverage without loading historical payloads", async () => {
|
||||
const fixture = await seed();
|
||||
await importNativeControllerRecovery(db, { ...fixture.scope, expectedConfigDigest: compatibility.configDigest, supervisor: fixture.supervisor });
|
||||
await db.insert(agentTaskSessions).values([0, 1].map(index => ({ companyId, agentId, taskKey: `${fixture.scope.issueId}-${index}`,
|
||||
adapterType: "paperclip_runner", lastRunId: fixture.scope.runId })));
|
||||
const first = await readNativeControllerRecoveryInventory(db, { limit: 1 });
|
||||
expect(first).toMatchObject({ inspected: 1, coverage: "partial", truncated: true, selectedRetainedEvidence: 1, fullPayloadsValidated: false });
|
||||
const next = await readNativeControllerRecoveryInventory(db, { limit: 1, afterId: first.nextAfterId! });
|
||||
expect(next).toMatchObject({ inspected: 1, coverage: "partial", truncated: false, selectedRetainedEvidence: 1 });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,43 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { createNativeControllerRecovery, readNativeControllerRecovery } from "./native-controller-recovery.js";
|
||||
import { nativeSha256 } from "./canonical.js";
|
||||
import { redactRegisteredSecretValues } from "../run-secret-redaction.js";
|
||||
import type { PersistedNativeSession } from "../../vendor/paperclip-runner/index.js";
|
||||
|
||||
const snapshot: PersistedNativeSession = { backendKind: "mock", sessionId: "provider",
|
||||
identity: { companyId: "company", agentId: "agent", issueId: "issue", runId: "run", sessionId: "session" } };
|
||||
const compatibility = { configDigest: `sha256:${"a".repeat(64)}`, processConfigDigest: null };
|
||||
const binding = { runId: "run", runnerInstanceId: "runner", normalizedSessionId: "session" };
|
||||
|
||||
describe("native controller recovery envelope", () => {
|
||||
it("binds the stored snapshot, including redaction, without serializing its contents twice", () => {
|
||||
const stored = { ...snapshot, providerSessionId: "[REDACTED]" };
|
||||
const envelope = createNativeControllerRecovery(stored, compatibility, binding);
|
||||
expect(envelope.snapshotSha256).toBe(nativeSha256(stored));
|
||||
expect(readNativeControllerRecovery(envelope, JSON.parse(JSON.stringify(stored)))).toEqual(envelope);
|
||||
expect(readNativeControllerRecovery(envelope, { ...stored, providerSessionId: "secret-before-redaction" })).toBeNull();
|
||||
expect(JSON.stringify(envelope)).not.toContain("[REDACTED]");
|
||||
expect(Buffer.byteLength(JSON.stringify(envelope))).toBeLessThan(4096);
|
||||
});
|
||||
|
||||
it("does not certify a missing PRP publication or unknown format", () => {
|
||||
const pending = createNativeControllerRecovery(snapshot, compatibility, null);
|
||||
expect(readNativeControllerRecovery(pending, snapshot)).toBeNull();
|
||||
const published = createNativeControllerRecovery(snapshot, compatibility, binding);
|
||||
expect(readNativeControllerRecovery({ ...published, schema: "future" }, snapshot)).toBeNull();
|
||||
expect(readNativeControllerRecovery({ ...published, processConfigDigest: "unverified" }, snapshot)).toBeNull();
|
||||
});
|
||||
|
||||
it("enforces the metadata byte bound on both write and read", () => {
|
||||
expect(() => createNativeControllerRecovery(snapshot, { ...compatibility, configDigest: "x".repeat(4096) }, binding))
|
||||
.toThrow("metadata_too_large");
|
||||
expect(readNativeControllerRecovery({ ...createNativeControllerRecovery(snapshot, compatibility, binding), extra: "x".repeat(4096) }, snapshot))
|
||||
.toBeNull();
|
||||
});
|
||||
|
||||
it("keeps compatibility metadata out of public run projections", () => {
|
||||
expect(redactRegisteredSecretValues({ runnerProfileJson: { preserved: true,
|
||||
nativeControllerRecovery: createNativeControllerRecovery(snapshot, compatibility, binding) } }, []))
|
||||
.toEqual({ runnerProfileJson: { preserved: true } });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,270 @@
|
||||
import { and, eq, ne, sql } from "drizzle-orm";
|
||||
import { environmentLeases, heartbeatRuns, issues, nativeRunFinalizations, nativeRunnerCheckpoints, type Db } from "@paperclipai/db";
|
||||
import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target";
|
||||
import type { DurablePrpCheckpoint, PersistedNativeSession } from "../../vendor/paperclip-runner/index.js";
|
||||
import { notifyNativeFinalizationWork } from "../native-finalization-notifications.js";
|
||||
import { readRunCancellation } from "../run-cancellation.js";
|
||||
import { nativeSha256 } from "./canonical.js";
|
||||
import { findNativeSessionResumeRun, isNativeSessionId, isUnusedNativeSessionBootstrap, nativeSessionProviderEvidence } from "./native-session-resume.js";
|
||||
import { loadNativeRunnerCheckpoint } from "./native-runner-checkpoint.js";
|
||||
import { redactRunnerApiPayloads } from "./runner-api-log-redaction.js";
|
||||
|
||||
export const NATIVE_CONTROLLER_RECOVERY_MAX_BYTES = 4 * 1024;
|
||||
|
||||
/** Private run-profile metadata. No credentials or controller-local paths. */
|
||||
export interface NativeControllerRecovery {
|
||||
schema: "paperclip.native-controller-recovery.v1";
|
||||
snapshotSha256: string;
|
||||
configDigest: string;
|
||||
processConfigDigest: string | null;
|
||||
checkpoint: {
|
||||
runId: string;
|
||||
runnerInstanceId: string;
|
||||
normalizedSessionId: string;
|
||||
} | null;
|
||||
}
|
||||
|
||||
export type NativeControllerCompatibility = Pick<NativeControllerRecovery, "configDigest" | "processConfigDigest">;
|
||||
|
||||
export function createNativeControllerRecovery(
|
||||
snapshot: PersistedNativeSession,
|
||||
compatibility: NativeControllerCompatibility,
|
||||
checkpoint: NativeControllerRecovery["checkpoint"],
|
||||
): NativeControllerRecovery {
|
||||
const value: NativeControllerRecovery = {
|
||||
schema: "paperclip.native-controller-recovery.v1",
|
||||
snapshotSha256: nativeSha256(snapshot),
|
||||
configDigest: compatibility.configDigest,
|
||||
processConfigDigest: compatibility.processConfigDigest,
|
||||
checkpoint,
|
||||
};
|
||||
if (Buffer.byteLength(JSON.stringify(value)) > NATIVE_CONTROLLER_RECOVERY_MAX_BYTES) {
|
||||
throw new Error("native_controller_recovery_metadata_too_large");
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function record(value: unknown): Record<string, unknown> {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||||
}
|
||||
|
||||
export function readNativeControllerRecovery(value: unknown, snapshot: unknown): NativeControllerRecovery | null {
|
||||
const envelope = record(value), checkpoint = record(envelope.checkpoint);
|
||||
if (Buffer.byteLength(JSON.stringify(value) ?? "null") > NATIVE_CONTROLLER_RECOVERY_MAX_BYTES
|
||||
|| envelope.schema !== "paperclip.native-controller-recovery.v1"
|
||||
|| typeof envelope.snapshotSha256 !== "string" || envelope.snapshotSha256 !== nativeSha256(snapshot)
|
||||
|| typeof envelope.configDigest !== "string" || !/^sha256:[a-f0-9]{64}$/.test(envelope.configDigest)
|
||||
|| !(envelope.processConfigDigest === null || (typeof envelope.processConfigDigest === "string" && /^[a-f0-9]{64}$/.test(envelope.processConfigDigest)))
|
||||
|| ![checkpoint.runId, checkpoint.runnerInstanceId, checkpoint.normalizedSessionId].every(v => typeof v === "string" && v.length > 0)) return null;
|
||||
return envelope as unknown as NativeControllerRecovery;
|
||||
}
|
||||
|
||||
export type NativeControllerRecoveryOutcome =
|
||||
| { kind: "recoverable"; snapshot: PersistedNativeSession; metadata: NativeControllerRecovery; checkpoint: DurablePrpCheckpoint }
|
||||
| { kind: "waiting_for_owner"; deadline: Date }
|
||||
| { kind: "missing_evidence" | "unsupported" | "dependency_unavailable"; reason: string; cause?: unknown }
|
||||
| { kind: "no_prior_execution" };
|
||||
|
||||
export const NATIVE_RETAINED_OWNER_WAIT = "native_controller_recovery_waiting_for_owner";
|
||||
|
||||
export class NativeControllerRecoveryWaitingError extends Error {
|
||||
constructor(readonly retryAt: Date) {
|
||||
super(NATIVE_RETAINED_OWNER_WAIT);
|
||||
this.name = "NativeControllerRecoveryWaitingError";
|
||||
}
|
||||
}
|
||||
|
||||
/** Admission has not crossed the provider boundary. Retain this same business
|
||||
* run on the existing retry owner; never turn an ownership wait into a failure
|
||||
* or invent a new provider attempt. The usual resumption gates recheck Stop,
|
||||
* permissions and task ownership when this deadline arrives. */
|
||||
export async function deferNativeControllerRecovery(db: Db, input: {
|
||||
companyId: string; agentId: string; issueId: string; runId: string;
|
||||
leaseOwner?: string; retryAt: Date;
|
||||
}): Promise<void> {
|
||||
await db.transaction(async tx => {
|
||||
const [task] = await tx.select({ executionRunId: issues.executionRunId }).from(issues)
|
||||
.where(and(eq(issues.companyId, input.companyId), eq(issues.id, input.issueId))).for("update");
|
||||
const [owner] = await tx.select().from(nativeRunFinalizations).where(and(
|
||||
eq(nativeRunFinalizations.companyId, input.companyId), eq(nativeRunFinalizations.issueId, input.issueId),
|
||||
eq(nativeRunFinalizations.runId, input.runId))).for("update");
|
||||
const [run] = await tx.select().from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.id, input.runId), eq(heartbeatRuns.companyId, input.companyId),
|
||||
eq(heartbeatRuns.agentId, input.agentId), eq(heartbeatRuns.nativeIssueId, input.issueId))).for("update");
|
||||
const [authority] = await tx.select({ runId: nativeRunnerCheckpoints.runId }).from(nativeRunnerCheckpoints)
|
||||
.where(and(eq(nativeRunnerCheckpoints.companyId, input.companyId), eq(nativeRunnerCheckpoints.runId, input.runId))).limit(1);
|
||||
const progress = await nativeSessionProviderEvidence(tx as unknown as Db, [input.runId]);
|
||||
const cancellation = record(run?.resultJson?.nativeCancellation);
|
||||
const stop = readRunCancellation(run?.resultJson);
|
||||
const [clock] = await tx.execute<{ now: Date }>(sql`select clock_timestamp() as now`);
|
||||
if (!run || run.runtimeMode !== "native" || run.status !== "running" || task?.executionRunId !== input.runId
|
||||
|| !owner || owner.phase !== "observed" || owner.resultId || owner.attempt !== 0
|
||||
|| owner.controllerGeneration !== 0 || owner.leaseOwner !== (input.leaseOwner ?? null)
|
||||
|| (input.leaseOwner && (!owner.leaseExpiresAt || owner.leaseExpiresAt <= new Date(clock.now)))
|
||||
|| run.processPid || run.processGroupId || run.processStartedAt || authority || progress.size
|
||||
|| !record(run.runnerProfileJson).nativeExecutionInput || run.resultJson?.startupCancellation
|
||||
|| (stop?.source === "operator" && stop.expected)
|
||||
|| (["run", "issue"].includes(String(cancellation.scope)) && ["pending", "acknowledged"].includes(String(cancellation.dispatchState)))) {
|
||||
throw new Error("native_controller_recovery_wait_authority_changed");
|
||||
}
|
||||
await notifyNativeFinalizationWork(tx, input);
|
||||
await tx.update(nativeRunFinalizations).set({
|
||||
phase: "retryable_failure", leaseOwner: null, leaseExpiresAt: null, controlDeadlineAt: null,
|
||||
nextAttemptAt: input.retryAt, recoveryState: "resuming_session", failureCode: NATIVE_RETAINED_OWNER_WAIT,
|
||||
failureDetail: { originalFailureCode: NATIVE_RETAINED_OWNER_WAIT,
|
||||
message: "Waiting for the retained native controller owner to release its lease.",
|
||||
recoveryMode: record(run.runnerProfileJson).sessionCheckpoint ? "same_session_resume" : "bootstrap_retry",
|
||||
providerSessionEstablished: false, providerEventsExist: false },
|
||||
updatedAt: new Date(),
|
||||
}).where(eq(nativeRunFinalizations.runId, input.runId));
|
||||
});
|
||||
}
|
||||
|
||||
/** Bind a read of retained Runner lifecycle to the surviving sandbox. The
|
||||
* ordinary admission owner still checks current permissions before execution. */
|
||||
export async function assertNativeRetainedSandboxBinding(db: Db, input: {
|
||||
companyId: string; issueId: string; runId: string;
|
||||
target: Extract<AdapterExecutionTarget, { kind: "remote" }>;
|
||||
}): Promise<void> {
|
||||
const { target } = input;
|
||||
if (target.transport !== "sandbox" || target.sandboxLeaseAcquisition?.outcome !== "resumed"
|
||||
|| !target.providerKey || !target.environmentId || !target.leaseId) {
|
||||
throw new Error("native_controller_recovery_dependency_unavailable:retained_sandbox");
|
||||
}
|
||||
const [owner] = await db.select().from(nativeRunFinalizations).where(and(
|
||||
eq(nativeRunFinalizations.companyId, input.companyId), eq(nativeRunFinalizations.issueId, input.issueId),
|
||||
eq(nativeRunFinalizations.runId, input.runId)));
|
||||
const leases = await db.select({ metadata: environmentLeases.metadata, status: environmentLeases.status }).from(environmentLeases).where(and(
|
||||
eq(environmentLeases.companyId, input.companyId), eq(environmentLeases.issueId, input.issueId),
|
||||
eq(environmentLeases.heartbeatRunId, input.runId), eq(environmentLeases.environmentId, target.environmentId),
|
||||
eq(environmentLeases.provider, target.providerKey), eq(environmentLeases.providerLeaseId, target.sandboxLeaseAcquisition.providerLeaseId)));
|
||||
if (!owner || owner.leaseOwner || owner.phase !== "committed" || !owner.resultId || owner.recoveryState
|
||||
|| leases.length !== 1 || leases[0].metadata?.remoteCwd !== target.remoteCwd
|
||||
|| !["active", "released", "expired"].includes(leases[0].status)) {
|
||||
throw new Error("native_controller_recovery_missing_evidence:retained_sandbox_binding");
|
||||
}
|
||||
}
|
||||
|
||||
/** Only the existing bounded selector chooses history. Never search backwards
|
||||
* again when its selected record is incomplete or incompatible. This operation
|
||||
* reads evidence; it grants neither command nor acknowledgement authority. */
|
||||
export async function resolveNativeControllerRecovery(db: Db, input: {
|
||||
companyId: string; agentId: string; issueId: string; normalizedSessionId: string; currentRunId: string;
|
||||
}): Promise<NativeControllerRecoveryOutcome> {
|
||||
if (!isNativeSessionId(input.normalizedSessionId)) return { kind: "missing_evidence", reason: "session_binding" };
|
||||
const [current] = await db.select({ createdAt: heartbeatRuns.createdAt, processPid: heartbeatRuns.processPid,
|
||||
processGroupId: heartbeatRuns.processGroupId, processStartedAt: heartbeatRuns.processStartedAt,
|
||||
runnerProfileJson: heartbeatRuns.runnerProfileJson }).from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.id, input.currentRunId), eq(heartbeatRuns.companyId, input.companyId),
|
||||
eq(heartbeatRuns.agentId, input.agentId), eq(heartbeatRuns.nativeIssueId, input.issueId),
|
||||
eq(heartbeatRuns.nativeSessionId, input.normalizedSessionId)));
|
||||
if (!current) return { kind: "missing_evidence", reason: "current_binding" };
|
||||
const source = await findNativeSessionResumeRun(db, { ...input, beforeCreatedAt: current.createdAt, includeProgressBarrier: true });
|
||||
if (!source) {
|
||||
// An exhausted candidate window is unknown history, not fresh-bootstrap
|
||||
// proof. Read only IDs, and retain the selector's existing 32-row boundary.
|
||||
const prior = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(
|
||||
ne(heartbeatRuns.id, input.currentRunId), eq(heartbeatRuns.companyId, input.companyId),
|
||||
eq(heartbeatRuns.agentId, input.agentId), eq(heartbeatRuns.nativeIssueId, input.issueId),
|
||||
eq(heartbeatRuns.nativeSessionId, input.normalizedSessionId))).limit(33);
|
||||
if (prior.length > 32) return { kind: "missing_evidence", reason: "candidate_window_exhausted" };
|
||||
const progress = await nativeSessionProviderEvidence(db, [input.currentRunId]);
|
||||
if (!isUnusedNativeSessionBootstrap(current, progress.has(input.currentRunId))) {
|
||||
return { kind: "missing_evidence", reason: "current_execution_progress" };
|
||||
}
|
||||
// The normal admission and remote untouched-session claim still execute.
|
||||
return { kind: "no_prior_execution" };
|
||||
}
|
||||
const [owner] = await db.select({ leaseOwner: nativeRunFinalizations.leaseOwner, leaseExpiresAt: nativeRunFinalizations.leaseExpiresAt }).from(nativeRunFinalizations)
|
||||
.where(and(eq(nativeRunFinalizations.runId, source.id), eq(nativeRunFinalizations.companyId, input.companyId),
|
||||
eq(nativeRunFinalizations.issueId, input.issueId)));
|
||||
if (owner?.leaseOwner && owner.leaseExpiresAt && owner.leaseExpiresAt.getTime() > Date.now()) {
|
||||
return { kind: "waiting_for_owner", deadline: owner.leaseExpiresAt };
|
||||
}
|
||||
if (source.runtimeMode !== "native" || !["succeeded", "failed", "cancelled", "interrupted", "timed_out"].includes(source.status)
|
||||
|| source.createdAt > current.createdAt) return { kind: "missing_evidence", reason: "newer_progress_barrier" };
|
||||
const profile = record(source.runnerProfileJson), snapshot = record(profile.sessionCheckpoint);
|
||||
const identity = record(snapshot.identity);
|
||||
if (identity.companyId !== input.companyId || identity.agentId !== input.agentId
|
||||
|| identity.issueId !== input.issueId || identity.runId !== source.id || identity.sessionId !== input.normalizedSessionId) {
|
||||
return { kind: "missing_evidence", reason: "snapshot_binding" };
|
||||
}
|
||||
if (record(profile.nativeControllerRecovery).schema !== "paperclip.native-controller-recovery.v1") {
|
||||
return { kind: "unsupported", reason: "metadata_format" };
|
||||
}
|
||||
const metadata = readNativeControllerRecovery(profile.nativeControllerRecovery, snapshot);
|
||||
if (!metadata?.checkpoint || metadata.checkpoint.runId !== source.id
|
||||
|| metadata.checkpoint.normalizedSessionId !== input.normalizedSessionId
|
||||
|| metadata.checkpoint.runnerInstanceId !== source.runnerInstanceId) {
|
||||
return { kind: "missing_evidence", reason: "snapshot_digest_or_checkpoint_binding" };
|
||||
}
|
||||
let checkpoint: DurablePrpCheckpoint | null;
|
||||
try {
|
||||
checkpoint = await loadNativeRunnerCheckpoint(db, { companyId: input.companyId, issueId: input.issueId, runId: source.id });
|
||||
} catch (cause) {
|
||||
return { kind: "dependency_unavailable", reason: "checkpoint_authority_unreadable", cause };
|
||||
}
|
||||
if (!checkpoint || checkpoint.recoveryEvidence.runnerInstanceId !== metadata.checkpoint.runnerInstanceId
|
||||
|| checkpoint.recoveryEvidence.normalizedSessionId !== input.normalizedSessionId) {
|
||||
return { kind: "missing_evidence", reason: "checkpoint_binding" };
|
||||
}
|
||||
return { kind: "recoverable", snapshot: snapshot as unknown as PersistedNativeSession, metadata, checkpoint };
|
||||
}
|
||||
|
||||
/** Bounded internal migration of ONE retained source while its disk is available.
|
||||
* The migration owner supplies compatibility computed from the admitted input.
|
||||
* This never repairs/replaces the saved snapshot or claims a live execution. */
|
||||
export async function importNativeControllerRecovery(db: Db, input: {
|
||||
companyId: string; agentId: string; issueId: string; runId: string; normalizedSessionId: string;
|
||||
expectedConfigDigest: string;
|
||||
supervisor: unknown;
|
||||
}): Promise<{ kind: "imported" | "unchanged" } | Exclude<NativeControllerRecoveryOutcome, { kind: "recoverable" } | { kind: "no_prior_execution" }>> {
|
||||
const supervisor = record(input.supervisor);
|
||||
if (supervisor.schema !== "paperclip.native-session-supervisor.v1" || supervisor.configDigest !== input.expectedConfigDigest) {
|
||||
return { kind: "unsupported", reason: "supervisor_compatibility" };
|
||||
}
|
||||
return db.transaction(async tx => {
|
||||
const [task] = await tx.select({ executionRunId: issues.executionRunId }).from(issues)
|
||||
.where(and(eq(issues.companyId, input.companyId), eq(issues.id, input.issueId))).for("share");
|
||||
const [owner] = await tx.select().from(nativeRunFinalizations).where(and(
|
||||
eq(nativeRunFinalizations.companyId, input.companyId), eq(nativeRunFinalizations.issueId, input.issueId),
|
||||
eq(nativeRunFinalizations.runId, input.runId))).for("update");
|
||||
if (!task || !owner) return { kind: "missing_evidence" as const, reason: "owner_binding" };
|
||||
if (owner.leaseOwner || task.executionRunId) {
|
||||
return owner.leaseExpiresAt && owner.leaseExpiresAt > new Date()
|
||||
? { kind: "waiting_for_owner" as const, deadline: owner.leaseExpiresAt }
|
||||
: { kind: "missing_evidence" as const, reason: "normal_ownership_recovery_required" };
|
||||
}
|
||||
if (owner.phase !== "committed" || !owner.resultId || owner.recoveryState) {
|
||||
return { kind: "missing_evidence" as const, reason: "pending_finalization" };
|
||||
}
|
||||
const [run] = await tx.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, input.runId),
|
||||
eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.agentId, input.agentId),
|
||||
eq(heartbeatRuns.nativeIssueId, input.issueId), eq(heartbeatRuns.nativeSessionId, input.normalizedSessionId))).for("update");
|
||||
const profile = record(run?.runnerProfileJson), snapshot = record(profile.sessionCheckpoint), identity = record(snapshot.identity);
|
||||
if (run?.runtimeMode !== "native" || run.status !== "succeeded" || !run.finishedAt
|
||||
|| identity.runId !== input.runId || identity.companyId !== input.companyId || identity.issueId !== input.issueId
|
||||
|| identity.agentId !== input.agentId || identity.sessionId !== input.normalizedSessionId
|
||||
|| nativeSha256(snapshot) !== nativeSha256(redactRunnerApiPayloads(supervisor.snapshot))) {
|
||||
return { kind: "missing_evidence" as const, reason: "stale_supervisor_snapshot" };
|
||||
}
|
||||
const checkpoint = await loadNativeRunnerCheckpoint(tx, input);
|
||||
if (!checkpoint || checkpoint.recoveryEvidence.normalizedSessionId !== input.normalizedSessionId
|
||||
|| checkpoint.recoveryEvidence.runnerInstanceId !== run.runnerInstanceId) {
|
||||
return { kind: "missing_evidence" as const, reason: "checkpoint_binding" };
|
||||
}
|
||||
const metadata = createNativeControllerRecovery(snapshot as unknown as PersistedNativeSession, {
|
||||
configDigest: input.expectedConfigDigest,
|
||||
processConfigDigest: typeof supervisor.processConfigDigest === "string" ? supervisor.processConfigDigest : null,
|
||||
}, { runId: input.runId, runnerInstanceId: checkpoint.recoveryEvidence.runnerInstanceId, normalizedSessionId: input.normalizedSessionId });
|
||||
if (!readNativeControllerRecovery(metadata, snapshot)) return { kind: "unsupported" as const, reason: "metadata_format" };
|
||||
if (profile.nativeControllerRecovery) {
|
||||
return nativeSha256(profile.nativeControllerRecovery) === nativeSha256(metadata)
|
||||
? { kind: "unchanged" as const }
|
||||
: { kind: "missing_evidence" as const, reason: "existing_metadata_conflict" };
|
||||
}
|
||||
await tx.update(heartbeatRuns).set({ runnerProfileJson: { ...profile, nativeControllerRecovery: metadata }, updatedAt: new Date() })
|
||||
.where(eq(heartbeatRuns.id, input.runId));
|
||||
return { kind: "imported" as const };
|
||||
});
|
||||
}
|
||||
@@ -189,6 +189,8 @@ async function sessionRetentionBlocker(
|
||||
id: heartbeatRuns.id,
|
||||
status: heartbeatRuns.status,
|
||||
finishedAt: heartbeatRuns.finishedAt,
|
||||
incompleteRecovery: sql<boolean>`(${heartbeatRuns.runnerProfileJson}->'nativeControllerRecovery'->>'schema' = 'paperclip.native-controller-recovery.v1'
|
||||
and ${heartbeatRuns.runnerProfileJson}->'nativeControllerRecovery'->'checkpoint' = 'null'::jsonb) is true`,
|
||||
issueId: sql<string | null>`coalesce(${heartbeatRuns.nativeIssueId}, ${heartbeatRuns.issueId})`,
|
||||
})
|
||||
.from(heartbeatRuns)
|
||||
@@ -202,6 +204,7 @@ async function sessionRetentionBlocker(
|
||||
if (runs.some((run) => LIVE_RUN_STATUSES.includes(run.status) || !run.finishedAt)) {
|
||||
return { reason: "live_run" };
|
||||
}
|
||||
if (runs.some(run => run.incompleteRecovery)) return { reason: "native_recovery_incomplete" };
|
||||
if (input.checkpoint && runs.some(run => run.finishedAt! >= input.closedBefore)) return { reason: "recent_run" };
|
||||
const issueIds = [...new Set(runs.map((run) => run.issueId).filter((id): id is string => Boolean(id)))];
|
||||
if (issueIds.length === 0) return { reason: "issue_unknown" };
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { PgDialect } from "drizzle-orm/pg-core";
|
||||
import type { SQL } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { DurableRecoveryIdentity, PrpEvent } from "../../vendor/paperclip-runner/index.js";
|
||||
import { buildNativeExecutionInput } from "./native-execution-input.js";
|
||||
import { nativeRuntimeContextFixture } from "./runtime-context.test-fixture.js";
|
||||
import { createNativeRecoveryUsageDelivery } from "./native-recovery-usage-delivery.js";
|
||||
|
||||
const accounting = vi.hoisted(() => ({ open: vi.fn(), capture: vi.fn() }));
|
||||
vi.mock("./native-usage-delivery.js", () => ({ createNativeUsageContinuationDelivery: accounting.open }));
|
||||
beforeEach(() => { vi.clearAllMocks(); accounting.capture.mockResolvedValue(undefined); accounting.open.mockResolvedValue({ capture: accounting.capture }); });
|
||||
|
||||
function fixture() {
|
||||
const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), sessionId = randomUUID();
|
||||
const source: DurableRecoveryIdentity = { runId: randomUUID(), runnerInstanceId: randomUUID(), environmentLeaseId: randomUUID(),
|
||||
normalizedSessionId: sessionId, turnId: randomUUID(), itemId: randomUUID() };
|
||||
const build = (runId: string) => buildNativeExecutionInput({ companyId, agentId, runId, normalizedSessionId: sessionId,
|
||||
provider: "codex", issue: { id: issueId, identifier: "REC-1", title: "Continue", description: null, workMode: "standard" },
|
||||
taskPrompt: "Continue once", workspace: { id: randomUUID(), cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null },
|
||||
completionContract: { id: randomUUID(), sha256: `sha256:${"a".repeat(64)}`, schemaVersion: "paperclip.run-result.v1",
|
||||
contract: { revision: "1", objective: "Continue", criteria: [{ id: "objective", requirement: "Preserve the result" }] } },
|
||||
runtimeContext: nativeRuntimeContextFixture() });
|
||||
const execution = build(randomUUID()), previous = build(source.runId);
|
||||
const row = { run: { status: "succeeded", finishedAt: new Date() as Date | null, runtimeMode: "native", runnerInstanceId: source.runnerInstanceId,
|
||||
nativeSessionId: sessionId, runnerProfileJson: { nativeExecutionInput: previous } },
|
||||
owner: { phase: "committed", resultId: randomUUID() as string | null, leaseOwner: null as string | null, companyId, issueId } };
|
||||
const where = vi.fn(async (_predicate: SQL) => [row]);
|
||||
const select = vi.fn(() => ({ from: () => ({ innerJoin: () => ({ where }) }) }));
|
||||
const db = { select } as unknown as Db;
|
||||
const matchesExecution = vi.fn(() => true);
|
||||
const deliver = () => createNativeRecoveryUsageDelivery({ db, execution, sourceIdentity: { ...source }, matchesExecution });
|
||||
const event: PrpEvent = { schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", sourceInstanceId: source.runnerInstanceId,
|
||||
sourceEventId: "source-usage", sourceSeq: 1, runId: source.runId, normalizedSessionId: sessionId, turnId: source.turnId,
|
||||
eventType: "usage.reported", priority: 0, emittedAt: new Date().toISOString(), payload: {} };
|
||||
return { source, execution, previous, row, select, where, matchesExecution, deliver, event };
|
||||
}
|
||||
|
||||
describe("source accounting recovery authority", () => {
|
||||
it("pins the relational source tuple and hydrates once without binding target accounting", async () => {
|
||||
const f = fixture(), deliver = f.deliver();
|
||||
await deliver(f.event, f.source);
|
||||
await deliver({ ...f.event, sourceEventId: "source-terminal", sourceSeq: 2, eventType: "turn.completed" }, f.source);
|
||||
expect(f.select).toHaveBeenCalledOnce();
|
||||
expect(new PgDialect().sqlToQuery(f.where.mock.calls[0][0]).params).toEqual([
|
||||
f.source.runId, f.execution.binding.companyId, f.execution.binding.agentId, f.execution.binding.issueId,
|
||||
]);
|
||||
expect(f.matchesExecution).toHaveBeenCalledWith(f.previous);
|
||||
expect(accounting.open).toHaveBeenCalledExactlyOnceWith(expect.anything(), {
|
||||
companyId: f.execution.binding.companyId, runId: f.source.runId, runnerInstanceId: f.source.runnerInstanceId, provider: f.previous.provider,
|
||||
});
|
||||
expect(accounting.capture).toHaveBeenCalledTimes(2);
|
||||
expect(accounting.capture).toHaveBeenCalledWith(f.event);
|
||||
});
|
||||
|
||||
it.each(["runId", "runnerInstanceId", "environmentLeaseId", "normalizedSessionId", "turnId", "itemId"] as const)(
|
||||
"rejects a changed pinned %s before source hydration", async key => {
|
||||
const f = fixture();
|
||||
await expect(f.deliver()(f.event, { ...f.source, [key]: randomUUID() })).rejects.toThrow("native_recovery_usage_source_unproven");
|
||||
expect(f.select).not.toHaveBeenCalled();
|
||||
expect(accounting.open).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(["runId", "sourceInstanceId", "normalizedSessionId"] as const)("rejects mismatched event %s before source hydration", async key => {
|
||||
const f = fixture();
|
||||
await expect(f.deliver()({ ...f.event, [key]: randomUUID() }, f.source)).rejects.toThrow("native_recovery_usage_source_unproven");
|
||||
expect(f.select).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(["nonterminal", "unfinished", "uncommitted", "no-result", "held", "company", "issue", "scope"])(
|
||||
"keeps an unproven source held: %s", async change => {
|
||||
const f = fixture();
|
||||
if (change === "nonterminal") f.row.run.status = "running";
|
||||
if (change === "unfinished") f.row.run.finishedAt = null;
|
||||
if (change === "uncommitted") f.row.owner.phase = "observed";
|
||||
if (change === "no-result") f.row.owner.resultId = null;
|
||||
if (change === "held") f.row.owner.leaseOwner = "live-owner";
|
||||
if (change === "company") f.row.owner.companyId = randomUUID();
|
||||
if (change === "issue") f.row.owner.issueId = randomUUID();
|
||||
if (change === "scope") f.matchesExecution.mockReturnValue(false);
|
||||
await expect(f.deliver()(f.event, f.source)).rejects.toThrow("native_recovery_usage_source_unproven");
|
||||
expect(accounting.open).not.toHaveBeenCalled();
|
||||
expect(accounting.capture).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps a durable source receipt failure unacknowledged", async () => {
|
||||
const f = fixture();
|
||||
accounting.capture.mockRejectedValue(new Error("native_usage_after_finalization_requires_reconciliation"));
|
||||
await expect(f.deliver()(f.event, f.source)).rejects.toThrow("native_usage_after_finalization_requires_reconciliation");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,47 @@
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { heartbeatRuns, nativeRunFinalizations, type Db } from "@paperclipai/db";
|
||||
import { parseNativeExecutionInput, type DurableRecoveryIdentity, type NativeExecutionInput, type PrpEvent } from "../../vendor/paperclip-runner/index.js";
|
||||
import { createNativeUsageContinuationDelivery } from "./native-usage-delivery.js";
|
||||
|
||||
const identityKeys = ["runId", "runnerInstanceId", "environmentLeaseId", "normalizedSessionId", "turnId", "itemId"] as const;
|
||||
|
||||
/** A cold handoff may discover accounting that belongs to its completed source.
|
||||
* Hydrate that producer once; never route its receipts through the new run. */
|
||||
export function createNativeRecoveryUsageDelivery(input: {
|
||||
db: Db;
|
||||
execution: NativeExecutionInput;
|
||||
sourceIdentity: Record<string, unknown>;
|
||||
matchesExecution(previous: NativeExecutionInput): boolean;
|
||||
}) {
|
||||
let delivery: ReturnType<typeof createNativeUsageContinuationDelivery> | undefined;
|
||||
const deny = (): never => { throw new Error("native_recovery_usage_source_unproven"); };
|
||||
const open = async (source: DurableRecoveryIdentity) => {
|
||||
const { binding } = input.execution;
|
||||
const [prior] = await input.db.select({ run: heartbeatRuns, owner: nativeRunFinalizations })
|
||||
.from(heartbeatRuns).innerJoin(nativeRunFinalizations, eq(nativeRunFinalizations.runId, heartbeatRuns.id))
|
||||
.where(and(eq(heartbeatRuns.id, source.runId), eq(heartbeatRuns.companyId, binding.companyId),
|
||||
eq(heartbeatRuns.agentId, binding.agentId), eq(heartbeatRuns.nativeIssueId, binding.issueId)));
|
||||
if (!prior || prior.run.status !== "succeeded" || prior.run.finishedAt === null ||
|
||||
prior.run.runtimeMode !== "native" || prior.run.runnerInstanceId !== source.runnerInstanceId ||
|
||||
prior.run.nativeSessionId !== source.normalizedSessionId || prior.owner.phase !== "committed" ||
|
||||
!prior.owner.resultId || prior.owner.leaseOwner || prior.owner.companyId !== binding.companyId ||
|
||||
prior.owner.issueId !== binding.issueId) return deny();
|
||||
const previous = parseNativeExecutionInput(prior.run.runnerProfileJson?.nativeExecutionInput);
|
||||
if (previous.binding.companyId !== binding.companyId || previous.binding.agentId !== binding.agentId ||
|
||||
previous.binding.issueId !== binding.issueId || previous.binding.runId !== source.runId ||
|
||||
!input.matchesExecution(previous)) return deny();
|
||||
return createNativeUsageContinuationDelivery(input.db, {
|
||||
companyId: binding.companyId, runId: source.runId,
|
||||
runnerInstanceId: source.runnerInstanceId, provider: previous.provider,
|
||||
});
|
||||
};
|
||||
return async (event: PrpEvent, source: DurableRecoveryIdentity) => {
|
||||
if (source.runId === input.execution.binding.runId ||
|
||||
identityKeys.some(key => typeof input.sourceIdentity[key] !== "string" ||
|
||||
!input.sourceIdentity[key] || source[key] !== input.sourceIdentity[key]) ||
|
||||
event.runId !== source.runId || event.sourceInstanceId !== source.runnerInstanceId ||
|
||||
event.normalizedSessionId !== source.normalizedSessionId) return deny();
|
||||
delivery ??= open(source);
|
||||
await (await delivery).capture(event);
|
||||
};
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
import { QueryBuilder } from "drizzle-orm/pg-core";
|
||||
import { and, asc, eq, gt, inArray, isNotNull, isNull, notExists, or, sql } from "drizzle-orm";
|
||||
import { heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults, statusDecisions, statusDecisionEffects } from "@paperclipai/db";
|
||||
import { and, asc, eq, exists, gt, inArray, isNotNull, isNull, notExists, or, sql } from "drizzle-orm";
|
||||
import { heartbeatRunEvents, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults, statusDecisions, statusDecisionEffects } from "@paperclipai/db";
|
||||
import type { NativeResponsePresentationScope } from "../native-response-presentation-notifications.js";
|
||||
|
||||
export function nativeResponsePresentationCandidates(scope?: NativeResponsePresentationScope, after?: string, runId?: string) {
|
||||
@@ -30,7 +30,24 @@ export function nativeResponsePresentationCandidates(scope?: NativeResponsePrese
|
||||
sql`${nativeRunResults.resultJson} #>> '{terminal,turnTerminalState}' = 'completed'`,
|
||||
or(
|
||||
and(eq(heartbeatRuns.status, "succeeded"),
|
||||
sql`(coalesce(${issues.conversationAgentId}::text, '') = '' or coalesce(${issues.conversationUserId}, '') = '')`,
|
||||
or(sql`(coalesce(${issues.conversationAgentId}::text, '') = '' or coalesce(${issues.conversationUserId}, '') = '')`,
|
||||
and(eq(statusDecisions.reasonCode, "conversation_turn_finished"),
|
||||
eq(issues.conversationAgentId, heartbeatRuns.agentId), eq(issues.conversationUserId, heartbeatRuns.responsibleUserId),
|
||||
eq(issues.status, "in_progress"), eq(issues.conversationState, "active"),
|
||||
or(isNull(issues.executionRunId), eq(issues.executionRunId, heartbeatRuns.id)),
|
||||
sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issues.id}::text`,
|
||||
sql`jsonb_typeof(${heartbeatRuns.contextSnapshot}->'conversationSessionGeneration') = 'number'`,
|
||||
sql`${heartbeatRuns.contextSnapshot}->>'conversationSessionGeneration' = ${issues.conversationSessionGeneration}::text`,
|
||||
exists(db.select({ id: heartbeatRunEvents.id }).from(heartbeatRunEvents).where(and(
|
||||
eq(heartbeatRunEvents.companyId, heartbeatRuns.companyId), eq(heartbeatRunEvents.runId, heartbeatRuns.id),
|
||||
eq(heartbeatRunEvents.agentId, heartbeatRuns.agentId), eq(heartbeatRunEvents.eventType, "item.completed"),
|
||||
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,runId}' = ${heartbeatRuns.id}::text`,
|
||||
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,normalizedSessionId}' = ${heartbeatRuns.nativeSessionId}::text`,
|
||||
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,turnId}' = ${nativeRunResults.turnId}`,
|
||||
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,payload,kind}' = 'agentMessage'`,
|
||||
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,payload,channel}' = 'final'`,
|
||||
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,payload,text}' ~ '[^[:space:]]'`,
|
||||
))))),
|
||||
sql`${nativeRunResults.resultJson} #>> '{result,reportedWorkDisposition}' = 'done'`,
|
||||
sql`${nativeRunResults.resultJson} #>> '{terminal,reportedWorkDisposition}' = 'done'`),
|
||||
and(sql`${nativeRunResults.resultJson} #>> '{result,reportedWorkDisposition}' = 'yielded'`,
|
||||
|
||||
@@ -56,6 +56,7 @@ export function nativeResponsePresentationWork(db: Db) {
|
||||
function register(coordinator: ReturnType<typeof createDeliveryWorkCoordinator>, canRun: () => boolean) {
|
||||
const worker = registerScopedDeadlineWork(coordinator, {
|
||||
queue: DELIVERY_QUEUES.nativeResponsePresentation,
|
||||
concurrentChecks: { limit: 4, key: (entry: Entry) => `${entry.companyId}:${entry.issueId}` },
|
||||
decodeScope: (hint: string): NativeResponsePresentationScope | undefined => hint === "*" ? undefined : JSON.parse(hint),
|
||||
matches: (entry: Entry, scope?: NativeResponsePresentationScope) => !scope || entry.companyId === scope.companyId && (!scope.issueId || entry.issueId === scope.issueId) && (!scope.runId || entry.id === scope.runId) && (!scope.agentId || entry.agentId === scope.agentId),
|
||||
load: (scope, signal) => load(scope, signal, canRun),
|
||||
|
||||
@@ -8,6 +8,18 @@ import {
|
||||
} from "./native-restart-recovery.js";
|
||||
|
||||
describe("native restart recovery classification", () => {
|
||||
it.each([false, true])("recognizes verified staged remote evidence with target snapshot=%s", hasCheckpoint => {
|
||||
const evidence = { remoteSandbox: true, runnerPidAlive: false, runnerGroupAlive: false,
|
||||
processStartMatches: false, hasCheckpoint, hasProviderEvidence: false };
|
||||
expect(classifyNativeRunnerRecoveryEvidence(evidence).claimKind).toBeNull();
|
||||
expect(classifyNativeRunnerRecoveryEvidence({ ...evidence, stagedAttachmentVerified: true })).toEqual({
|
||||
claimKind: "reattach_remote_runner", reason: "staged_remote_attachment_requires_sandbox_verification",
|
||||
});
|
||||
expect(classifyNativeRunnerRecoveryEvidence({ ...evidence, stagedAttachmentVerified: true,
|
||||
checkpointFailed: true }).claimKind).toBeNull();
|
||||
expect(classifyNativeRunnerRecoveryEvidence({ ...evidence, remoteSandbox: false,
|
||||
stagedAttachmentVerified: true }).claimKind).toBe(hasCheckpoint ? null : "bootstrap_incomplete");
|
||||
});
|
||||
it.each([true, false])("uses the remote fenced lease instead of the old container PID (live=%s)", async live => {
|
||||
const now = new Date();
|
||||
const probe = vi.fn(() => { throw new Error("must not probe another container's PID"); });
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { NATIVE_RETAINED_OWNER_WAIT } from "./native-controller-recovery.js";
|
||||
import { notifyNativeEnvironmentSettlement } from "../native-environment-settlement-notifications.js";
|
||||
import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
|
||||
import { notifyNativeStatusReassessment } from "../native-status-reassessment-notifications.js";
|
||||
@@ -6,7 +7,8 @@ import { notifyCancelledWorkspacePreservation } from "../cancelled-workspace-pre
|
||||
import { notifyInstructionCollection } from "../instruction-collection-notifications.js";
|
||||
import { notifyExecutionRunWork } from "../execution-control-notifications.js";
|
||||
import { preserveNativeFailedWorkspace } from "./native-failed-workspace-recovery.js";
|
||||
import { hasNativeRunnerCheckpoint } from "./native-runner-checkpoint.js";
|
||||
import { hasNativeRunnerCheckpoint, loadNativeRunnerCheckpointHeader } from "./native-runner-checkpoint.js";
|
||||
import { resolveNativeStagedAttachmentRecovery, type NativeStagedAttachmentRecovery } from "./native-staged-attachment-recovery.js";
|
||||
import { readNativeWorkspaceSyncReference } from "./native-workspace-sync.js";
|
||||
import { activeNativeExecutionRunIds, NATIVE_PREPARATION_FENCING_VERSION } from "./native-active-runs.js";
|
||||
import {
|
||||
@@ -15,7 +17,7 @@ import {
|
||||
recordNativeLocalProcessStop,
|
||||
} from "../native-local-process-stop.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { and, desc, eq, inArray, isNotNull, isNull, lte, or, sql } from "drizzle-orm";
|
||||
import { and, desc, eq, inArray, isNotNull, isNull, lte, ne, or, sql } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import {
|
||||
agents,
|
||||
@@ -68,6 +70,8 @@ export type NativeRestartRecoveryClaim =
|
||||
recoveryRequestId: string | null;
|
||||
remote: { kind?: "sandbox"; providerLeaseId: string; remoteCwd: string }
|
||||
| { kind: "plugin_task"; leaseId: string };
|
||||
/** Discovery evidence only; executor revalidates the encrypted header. */
|
||||
stagedAttachment?: Pick<NativeStagedAttachmentRecovery, "sourceIdentity" | "targetIdentity" | "templateDigest">;
|
||||
}
|
||||
| {
|
||||
kind: "resume_dead_runner";
|
||||
@@ -94,6 +98,7 @@ export type NativeRestartRecoveryDisposition =
|
||||
kind: "awaiting_evidence" | "blocked" | "already_finalized";
|
||||
runId: string;
|
||||
reason: string;
|
||||
retryAt?: Date;
|
||||
};
|
||||
|
||||
export function nextNativeProviderAttempt(
|
||||
@@ -320,6 +325,7 @@ export function classifyNativeRunnerRecoveryEvidence(input: {
|
||||
checkpointIdentityMatches?: boolean;
|
||||
hasProviderEvidence: boolean;
|
||||
checkpointFailed?: boolean;
|
||||
stagedAttachmentVerified?: boolean;
|
||||
providerAttempt?: number;
|
||||
}): {
|
||||
claimKind: NativeRestartRecoveryClaim["kind"] | null;
|
||||
@@ -334,6 +340,9 @@ export function classifyNativeRunnerRecoveryEvidence(input: {
|
||||
// inspection; the executor must prove the original lease and authenticate
|
||||
// the surviving runner before it can resume delivery.
|
||||
if (input.remoteSandbox) {
|
||||
if (input.stagedAttachmentVerified === true) {
|
||||
return { claimKind: "reattach_remote_runner", reason: "staged_remote_attachment_requires_sandbox_verification" };
|
||||
}
|
||||
if (input.unclaimedBootstrap === true && !input.hasCheckpoint && !input.hasProviderEvidence) {
|
||||
return { claimKind: "bootstrap_incomplete", reason: "remote_execution_never_claimed" };
|
||||
}
|
||||
@@ -514,6 +523,9 @@ export async function claimNativeRestartRecoveries(input: {
|
||||
eq(agents.adapterType, "paperclip_runner"),
|
||||
inArray(heartbeatRuns.status, ["running", "failed"]),
|
||||
isNull(nativeRunFinalizations.resultId),
|
||||
// A pre-provider retained-owner wait belongs to normal same-run
|
||||
// resumption, including when its deadline elapsed while we were down.
|
||||
or(isNull(nativeRunFinalizations.failureCode), ne(nativeRunFinalizations.failureCode, NATIVE_RETAINED_OWNER_WAIT)),
|
||||
or(
|
||||
eq(nativeRunFinalizations.phase, "observed"),
|
||||
and(
|
||||
@@ -668,9 +680,21 @@ export async function claimNativeRestartRecoveries(input: {
|
||||
kind: "awaiting_evidence",
|
||||
runId: row.run.id,
|
||||
reason: takeover.reason,
|
||||
...(durableAuthority && row.coordinator.leaseExpiresAt ? { retryAt: row.coordinator.leaseExpiresAt } : {}),
|
||||
} as const;
|
||||
}
|
||||
|
||||
// Only the classic remote path stages attachments. After the owner is
|
||||
// eligible for takeover, inspect its bounded header even if activation
|
||||
// already published a runtime snapshot. Never scan command/event payloads.
|
||||
const stagedHeader = remoteWorkspace && !pluginTask && durableAuthority
|
||||
? await loadNativeRunnerCheckpointHeader(tx, {
|
||||
companyId: row.run.companyId, issueId: row.coordinator.issueId, runId: row.run.id,
|
||||
}) : null;
|
||||
const stagedAttachment = stagedHeader ? await resolveNativeStagedAttachmentRecovery({
|
||||
db: tx, currentRun: row.run, header: stagedHeader,
|
||||
}) : null;
|
||||
|
||||
const runnerPidAlive =
|
||||
!remoteProcess && processIsAlive(row.run.processPid);
|
||||
const runnerGroupAlive =
|
||||
@@ -859,6 +883,7 @@ export async function claimNativeRestartRecoveries(input: {
|
||||
knownProviderProcessIdentityAmbiguous:
|
||||
providerProcesses.ambiguousLivePids.length > 0,
|
||||
hasCheckpoint,
|
||||
stagedAttachmentVerified: stagedAttachment !== null,
|
||||
checkpointIdentityMatches:
|
||||
checkpointIdentityMatches &&
|
||||
(!remoteSandbox || remoteWorkspace !== null),
|
||||
@@ -1109,6 +1134,10 @@ export async function claimNativeRestartRecoveries(input: {
|
||||
return {
|
||||
kind: claimKind,
|
||||
...common,
|
||||
...(stagedAttachment ? { stagedAttachment: {
|
||||
sourceIdentity: stagedAttachment.sourceIdentity, targetIdentity: stagedAttachment.targetIdentity,
|
||||
templateDigest: stagedAttachment.templateDigest,
|
||||
} } : {}),
|
||||
remote: pluginTask ? { kind: "plugin_task", leaseId: pluginTask.id } : {
|
||||
providerLeaseId: remoteWorkspace!.providerLeaseId,
|
||||
remoteCwd: remoteWorkspace!.remoteCwd,
|
||||
|
||||
@@ -29,7 +29,7 @@ import { readPersistedNativeFailureEvidence } from "./native-provider-failure-ev
|
||||
import { dismissAutomaticCompletionReviews } from "./automatic-completion-reviews.js";
|
||||
import { findSupersededExternalReviewRequests } from "./superseded-external-review-requests.js";
|
||||
import { getNativeReviewAssignment, readNativeReviewAssignmentContext } from "./native-review-participant.js";
|
||||
import { conversationNativeDecision, isConversation } from "../agent-conversations.js";
|
||||
import { conversationNativeDecision, isConversation, settleConversationTurnForRun } from "../agent-conversations.js";
|
||||
import { issueTreeControlService } from "../issue-tree-control.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { preserveNativeWorkspaceExportLease } from "./native-workspace-export-resume.js";
|
||||
@@ -747,6 +747,7 @@ async function recoverCommittedNativeChatResponseInScope(
|
||||
): Promise<boolean> {
|
||||
let agentId: string | null = null;
|
||||
let materialized = false;
|
||||
let conversationReply = false;
|
||||
const entered = beginRunPhase("response.transaction.entry");
|
||||
try {
|
||||
materialized = await measureRunPhase("response.transaction", () => db.transaction(async (transaction) => {
|
||||
@@ -789,6 +790,7 @@ async function recoverCommittedNativeChatResponseInScope(
|
||||
id: heartbeatRuns.id,
|
||||
companyId: heartbeatRuns.companyId,
|
||||
agentId: heartbeatRuns.agentId,
|
||||
responsibleUserId: heartbeatRuns.responsibleUserId,
|
||||
nativeIssueId: heartbeatRuns.nativeIssueId,
|
||||
nativeSessionId: heartbeatRuns.nativeSessionId,
|
||||
completionContractId: heartbeatRuns.completionContractId,
|
||||
@@ -803,6 +805,8 @@ async function recoverCommittedNativeChatResponseInScope(
|
||||
contextSnapshot: sql<Record<string, unknown> | null>`case when ${heartbeatRuns.contextSnapshot} is null then null
|
||||
else jsonb_build_object(
|
||||
'skipIssueComment', ${heartbeatRuns.contextSnapshot}->'skipIssueComment',
|
||||
'issueId', ${heartbeatRuns.contextSnapshot}->'issueId',
|
||||
'conversationSessionGeneration', ${heartbeatRuns.contextSnapshot}->'conversationSessionGeneration',
|
||||
'source', ${heartbeatRuns.contextSnapshot}->'source',
|
||||
'externalChatContinuation', ${heartbeatRuns.contextSnapshot}->'externalChatContinuation',
|
||||
'paperclipWake', jsonb_build_object('externalInteractionContinuation',
|
||||
@@ -909,12 +913,16 @@ async function recoverCommittedNativeChatResponseInScope(
|
||||
agentId = run.agentId;
|
||||
return true;
|
||||
}
|
||||
// Copyback can be owned by reconciliation before the live heartbeat
|
||||
// reaches its presentation step. Recover a completed internal task's
|
||||
// exact final reply from the same accepted turn, without rerunning work
|
||||
// or granting external-chat publication authority.
|
||||
const conversation = isConversation(issue);
|
||||
const currentConversation = conversation && decision.reasonCode === "conversation_turn_finished"
|
||||
&& issue.conversationAgentId === run.agentId && issue.conversationUserId === run.responsibleUserId
|
||||
&& issue.status === "in_progress" && issue.conversationState === "active"
|
||||
&& (!issue.executionRunId || issue.executionRunId === run.id)
|
||||
&& run.contextSnapshot?.issueId === issue.id
|
||||
&& typeof run.contextSnapshot.conversationSessionGeneration === "number"
|
||||
&& run.contextSnapshot.conversationSessionGeneration === issue.conversationSessionGeneration;
|
||||
if (
|
||||
!isConversation(issue) &&
|
||||
(!conversation || currentConversation) &&
|
||||
run.status === "succeeded" &&
|
||||
record(run.contextSnapshot).skipIssueComment !== true &&
|
||||
!isExternalChatPresentationContext(run.contextSnapshot) &&
|
||||
@@ -929,6 +937,9 @@ async function recoverCommittedNativeChatResponseInScope(
|
||||
eq(heartbeatRunEvents.companyId, input.companyId),
|
||||
eq(heartbeatRunEvents.runId, run.id),
|
||||
eq(heartbeatRunEvents.eventType, "item.completed"),
|
||||
conversation ? and(eq(heartbeatRunEvents.agentId, run.agentId),
|
||||
sql`${heartbeatRunEvents.payload}->'prpEvent'->>'runId' = ${run.id}`,
|
||||
sql`${heartbeatRunEvents.payload}->'prpEvent'->>'normalizedSessionId' = ${run.nativeSessionId}`) : undefined,
|
||||
sql`${heartbeatRunEvents.payload}->'prpEvent'->>'turnId' = ${accepted.turnId}`,
|
||||
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,payload,kind}' = 'agentMessage'`,
|
||||
sql`${heartbeatRunEvents.payload} #>> '{prpEvent,payload,channel}' = 'final'`,
|
||||
@@ -963,6 +974,7 @@ async function recoverCommittedNativeChatResponseInScope(
|
||||
})}::jsonb`, updatedAt: new Date(),
|
||||
}).where(eq(heartbeatRuns.id, run.id));
|
||||
agentId = run.agentId;
|
||||
conversationReply = conversation;
|
||||
return true;
|
||||
}
|
||||
if (
|
||||
@@ -1039,12 +1051,14 @@ async function recoverCommittedNativeChatResponseInScope(
|
||||
} finally {
|
||||
entered();
|
||||
}
|
||||
if (materialized && agentId)
|
||||
if (materialized && agentId) {
|
||||
if (conversationReply) await settleConversationTurnForRun(db, input.companyId, input.runId);
|
||||
publishChatPublicationCommitSignal({
|
||||
...input,
|
||||
agentId,
|
||||
eventType: "run.presentation.resolved",
|
||||
});
|
||||
}
|
||||
return materialized;
|
||||
}
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ import { createLocalEncryptedCipher } from "../../secrets/local-encrypted-provid
|
||||
import { legacyControllerBootId } from "../legacy-controller-lease.js";
|
||||
|
||||
type Binding = { companyId: string; issueId: string; runId: string };
|
||||
type Owner = Binding & { bootId: string; leaseOwner: string; generation: number; authorityRunId: string };
|
||||
export type NativeRunnerCheckpointOwner = Binding & { bootId: string; leaseOwner: string; generation: number; authorityRunId: string };
|
||||
function encryptParts(binding: Binding, records: ReadonlyMap<string, string>) {
|
||||
if (!records.size) return [];
|
||||
const cipher = createLocalEncryptedCipher("write");
|
||||
@@ -25,49 +25,79 @@ function decryptPart(cipher: ReturnType<typeof createLocalEncryptedCipher>, bind
|
||||
return envelope.contents;
|
||||
}
|
||||
|
||||
export function nativeRunnerCheckpointWriter(db: Db, owner: Owner) {
|
||||
type CheckpointTransaction = Parameters<Parameters<Db["transaction"]>[0]>[0];
|
||||
|
||||
/** A dependent run-lock query can consume this CTE to preserve coordinator-first
|
||||
* lock order without adding a second database round trip. */
|
||||
export function nativeRunnerCheckpointOwnerLock(tx: CheckpointTransaction, owner: NativeRunnerCheckpointOwner) {
|
||||
return tx.$with("checkpoint_owner").as(tx.select({
|
||||
leaseOwner: nativeRunFinalizations.leaseOwner,
|
||||
leaseExpiresAt: nativeRunFinalizations.leaseExpiresAt,
|
||||
resultId: nativeRunFinalizations.resultId,
|
||||
}).from(nativeRunFinalizations).where(and(
|
||||
eq(nativeRunFinalizations.companyId, owner.companyId), eq(nativeRunFinalizations.issueId, owner.issueId),
|
||||
eq(nativeRunFinalizations.runId, owner.authorityRunId), eq(nativeRunFinalizations.controllerBootId, owner.bootId),
|
||||
eq(nativeRunFinalizations.controllerGeneration, owner.generation))).for("update"));
|
||||
}
|
||||
|
||||
/** Locks the coordinator before callers lock a run. A caller that subsequently
|
||||
* waits for another lock must recheck the fence before writing. */
|
||||
export async function lockNativeRunnerCheckpointOwner(
|
||||
tx: CheckpointTransaction, owner: NativeRunnerCheckpointOwner,
|
||||
) {
|
||||
// A locking CTE acquires ownership before the outer projection samples
|
||||
// wall time. Sampling in the locking SELECT can precede its lock wait.
|
||||
// Keep both steps in one statement without another database round trip.
|
||||
const locked = nativeRunnerCheckpointOwnerLock(tx, owner);
|
||||
const [owned] = await tx.with(locked).select({ leaseOwner: locked.leaseOwner,
|
||||
leaseExpiresAt: locked.leaseExpiresAt, resultId: locked.resultId, now: sql<Date>`clock_timestamp()` }).from(locked);
|
||||
if (!owned) throw new Error("native_runner_checkpoint_owner_lost");
|
||||
return owned;
|
||||
}
|
||||
|
||||
export async function validateNativeRunnerCheckpointOwner(
|
||||
tx: CheckpointTransaction, owner: NativeRunnerCheckpointOwner,
|
||||
owned: Awaited<ReturnType<typeof lockNativeRunnerCheckpointOwner>>, newWork: boolean,
|
||||
): Promise<void> {
|
||||
const current = owned;
|
||||
const activeOwner = current.leaseOwner === owner.leaseOwner
|
||||
&& current.leaseExpiresAt !== null && current.leaseExpiresAt > new Date(owned.now);
|
||||
if (!activeOwner) {
|
||||
const [task] = await tx.select({ executionRunId: issues.executionRunId }).from(issues)
|
||||
.where(and(eq(issues.companyId, owner.companyId), eq(issues.id, owner.issueId)));
|
||||
const [successor] = task?.executionRunId && task.executionRunId !== owner.runId
|
||||
? await tx.select({ runId: nativeRunFinalizations.runId }).from(nativeRunFinalizations).where(and(
|
||||
eq(nativeRunFinalizations.runId, task.executionRunId), eq(nativeRunFinalizations.companyId, owner.companyId),
|
||||
eq(nativeRunFinalizations.issueId, owner.issueId), eq(nativeRunFinalizations.controllerBootId, owner.bootId),
|
||||
gt(nativeRunFinalizations.leaseExpiresAt, sql`clock_timestamp()`))) : [];
|
||||
const [preparingSuccessor] = !successor && task?.executionRunId && task.executionRunId !== owner.runId
|
||||
? await tx.select({ runId: heartbeatRuns.id }).from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.id, task.executionRunId), eq(heartbeatRuns.companyId, owner.companyId),
|
||||
eq(heartbeatRuns.issueId, owner.issueId), eq(heartbeatRuns.runtimeMode, "legacy"),
|
||||
eq(heartbeatRuns.status, "running"), eq(heartbeatRuns.executionStage, "preparing"),
|
||||
eq(heartbeatRuns.controllerBootId, legacyControllerBootId),
|
||||
gt(heartbeatRuns.controllerLeaseExpiresAt, sql`clock_timestamp()`),
|
||||
notExists(tx.select({ runId: nativeRunFinalizations.runId }).from(nativeRunFinalizations)
|
||||
.where(eq(nativeRunFinalizations.runId, heartbeatRuns.id))))) : [];
|
||||
if (!current.resultId || newWork || !task || (task.executionRunId && task.executionRunId !== owner.runId && !successor && !preparingSuccessor)) {
|
||||
throw new Error("native_runner_checkpoint_owner_lost");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function assertNativeRunnerCheckpointOwner(tx: CheckpointTransaction, owner: NativeRunnerCheckpointOwner, newWork: boolean) {
|
||||
const locked = await lockNativeRunnerCheckpointOwner(tx, owner);
|
||||
await validateNativeRunnerCheckpointOwner(tx, owner, locked, newWork);
|
||||
}
|
||||
|
||||
export function nativeRunnerCheckpointWriter(db: Db, owner: NativeRunnerCheckpointOwner) {
|
||||
const persistenceDb = checkpointDb(db);
|
||||
let previousWork = new Set<string>();
|
||||
return async (checkpoint: DurablePrpCheckpoint): Promise<void> => {
|
||||
if (checkpoint.runId !== owner.runId) throw new Error("native_runner_checkpoint_binding_mismatch");
|
||||
const changed = encryptParts(owner, checkpoint.changedRecords);
|
||||
await persistenceDb.transaction(async tx => {
|
||||
// A locking CTE acquires ownership before the outer projection samples
|
||||
// wall time. Sampling in the locking SELECT can precede its lock wait.
|
||||
// Keep both steps in one statement without another database round trip.
|
||||
const locked = tx.$with("checkpoint_owner").as(tx.select().from(nativeRunFinalizations)
|
||||
.where(and(eq(nativeRunFinalizations.companyId, owner.companyId),
|
||||
eq(nativeRunFinalizations.issueId, owner.issueId), eq(nativeRunFinalizations.runId, owner.authorityRunId),
|
||||
eq(nativeRunFinalizations.controllerBootId, owner.bootId),
|
||||
eq(nativeRunFinalizations.controllerGeneration, owner.generation))).for("update"));
|
||||
const [owned] = await tx.with(locked).select({ leaseOwner: locked.leaseOwner,
|
||||
leaseExpiresAt: locked.leaseExpiresAt, resultId: locked.resultId, now: sql<Date>`clock_timestamp()` }).from(locked);
|
||||
if (!owned) throw new Error("native_runner_checkpoint_owner_lost");
|
||||
const current = owned;
|
||||
const activeOwner = current.leaseOwner === owner.leaseOwner
|
||||
&& current.leaseExpiresAt !== null && current.leaseExpiresAt > new Date(owned.now);
|
||||
if (!activeOwner) {
|
||||
const newWork = checkpoint.workCommandIds.some(id => !previousWork.has(id));
|
||||
const [task] = await tx.select({ executionRunId: issues.executionRunId }).from(issues)
|
||||
.where(and(eq(issues.companyId, owner.companyId), eq(issues.id, owner.issueId)));
|
||||
const [successor] = task?.executionRunId && task.executionRunId !== owner.runId
|
||||
? await tx.select({ runId: nativeRunFinalizations.runId }).from(nativeRunFinalizations).where(and(
|
||||
eq(nativeRunFinalizations.runId, task.executionRunId), eq(nativeRunFinalizations.companyId, owner.companyId),
|
||||
eq(nativeRunFinalizations.issueId, owner.issueId), eq(nativeRunFinalizations.controllerBootId, owner.bootId),
|
||||
gt(nativeRunFinalizations.leaseExpiresAt, sql`clock_timestamp()`))) : [];
|
||||
const [preparingSuccessor] = !successor && task?.executionRunId && task.executionRunId !== owner.runId
|
||||
? await tx.select({ runId: heartbeatRuns.id }).from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.id, task.executionRunId), eq(heartbeatRuns.companyId, owner.companyId),
|
||||
eq(heartbeatRuns.issueId, owner.issueId), eq(heartbeatRuns.runtimeMode, "legacy"),
|
||||
eq(heartbeatRuns.status, "running"), eq(heartbeatRuns.executionStage, "preparing"),
|
||||
eq(heartbeatRuns.controllerBootId, legacyControllerBootId),
|
||||
gt(heartbeatRuns.controllerLeaseExpiresAt, sql`clock_timestamp()`),
|
||||
notExists(tx.select({ runId: nativeRunFinalizations.runId }).from(nativeRunFinalizations)
|
||||
.where(eq(nativeRunFinalizations.runId, heartbeatRuns.id))))) : [];
|
||||
if (!current.resultId || newWork || !task || (task.executionRunId && task.executionRunId !== owner.runId && !successor && !preparingSuccessor)) {
|
||||
throw new Error("native_runner_checkpoint_owner_lost");
|
||||
}
|
||||
}
|
||||
await assertNativeRunnerCheckpointOwner(tx, owner, checkpoint.workCommandIds.some(id => !previousWork.has(id)));
|
||||
if (checkpoint.predecessor) {
|
||||
const [prior] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.id, checkpoint.predecessor.runId), eq(heartbeatRuns.companyId, owner.companyId),
|
||||
@@ -150,7 +180,17 @@ export async function registerNativeRunnerCheckpoint(input: {
|
||||
await input.authority.flushCheckpoint();
|
||||
}
|
||||
|
||||
export async function loadNativeRunnerCheckpoint(db: Db, binding: Binding): Promise<DurablePrpCheckpoint | null> {
|
||||
/** Exact-run bounded metadata read; never loads command/event payload records. */
|
||||
export async function loadNativeRunnerCheckpointHeader(db: Pick<Db, "select">, binding: Binding): Promise<string | null> {
|
||||
const [row] = await db.select({ material: nativeRunnerCheckpoints.material }).from(nativeRunnerCheckpoints).where(and(
|
||||
eq(nativeRunnerCheckpoints.companyId, binding.companyId), eq(nativeRunnerCheckpoints.issueId, binding.issueId),
|
||||
eq(nativeRunnerCheckpoints.runId, binding.runId), eq(nativeRunnerCheckpoints.part, "header"))).limit(1);
|
||||
if (!row) return null;
|
||||
try { return decryptPart(createLocalEncryptedCipher("read"), binding, "header", row.material); }
|
||||
catch { return null; }
|
||||
}
|
||||
|
||||
export async function loadNativeRunnerCheckpoint(db: Pick<Db, "select">, binding: Binding): Promise<DurablePrpCheckpoint | null> {
|
||||
const rows = await db.select().from(nativeRunnerCheckpoints).where(and(
|
||||
eq(nativeRunnerCheckpoints.companyId, binding.companyId), eq(nativeRunnerCheckpoints.issueId, binding.issueId),
|
||||
eq(nativeRunnerCheckpoints.runId, binding.runId)));
|
||||
|
||||
@@ -6,9 +6,12 @@ import { prepareNativeRemoteRunner, createNativeRunnerExecutionDriver } from "./
|
||||
const transport = vi.hoisted(() => ({
|
||||
register: vi.fn(),
|
||||
connect: vi.fn(),
|
||||
checkpoint: vi.fn(),
|
||||
checkpointRegistration: vi.fn(),
|
||||
}));
|
||||
vi.mock("../../realtime/runner-prp-ws.js", () => ({ registerRunnerPrpAuthority: transport.register }));
|
||||
vi.mock("../../realtime/runner-prp-outbound.js", () => ({ connectRunnerPrpIngress: transport.connect }));
|
||||
vi.mock("./native-runner-checkpoint.js", () => ({ nativeRunnerCheckpointRegistration: transport.checkpointRegistration }));
|
||||
|
||||
const binding = { companyId: "company", issueId: "issue", agentId: "agent", runId: "run-a" };
|
||||
const authority = {} as Parameters<NonNullable<ReturnType<typeof createNativeRunnerExecutionDriver>["controlPlaneRegistration"]>>[0];
|
||||
@@ -30,9 +33,41 @@ beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
transport.register.mockResolvedValue({ connectUrl: "ws://127.0.0.1/core", release: vi.fn() });
|
||||
transport.connect.mockReturnValue({ ready: Promise.resolve(), close: vi.fn(async () => {}) });
|
||||
transport.checkpointRegistration.mockReturnValue(transport.checkpoint);
|
||||
transport.checkpoint.mockResolvedValue(undefined);
|
||||
});
|
||||
|
||||
describe("shared Runner execution driver", () => {
|
||||
it("installs both staged checkpoint writers under the business lease without acquiring endpoints", async () => {
|
||||
const prepare = vi.fn();
|
||||
const persistence = { db: {} as never, bootId: "boot-c", authorityRunId: "run-c" };
|
||||
const driver = createNativeRunnerExecutionDriver({ target: sandbox(), binding: { ...binding, runId: "run-c" },
|
||||
controllerPersistence: persistence, prepareRemote: prepare });
|
||||
const sourceIdentity = { runnerInstanceId: "runner", environmentLeaseId: "lease", normalizedSessionId: "session",
|
||||
runId: "run-a", turnId: "turn-a", itemId: "item-a" };
|
||||
const targetIdentity = { ...sourceIdentity, runId: "run-c", turnId: "turn-c", itemId: "item-c" };
|
||||
await driver.onStagedRunAttachment!(authority, { sourceIdentity, targetIdentity, templateDigest: `sha256:${"a".repeat(64)}` });
|
||||
expect(transport.checkpointRegistration).toHaveBeenCalledExactlyOnceWith({ ...persistence,
|
||||
binding: { companyId: binding.companyId, issueId: binding.issueId, runId: "run-c" } });
|
||||
expect(transport.checkpoint.mock.calls).toEqual([[authority, sourceIdentity], [authority, targetIdentity]]);
|
||||
expect(prepare).not.toHaveBeenCalled();
|
||||
expect(transport.register).not.toHaveBeenCalled();
|
||||
expect(transport.connect).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("installs the target writer first when restoring an already persisted stage", async () => {
|
||||
const sourceIdentity = { runnerInstanceId: "runner", environmentLeaseId: "lease", normalizedSessionId: "session",
|
||||
runId: "run-a", turnId: "turn-a", itemId: "item-a" };
|
||||
const targetIdentity = { ...sourceIdentity, runId: "run-c", turnId: "turn-c", itemId: "item-c" };
|
||||
const restored = { store: { state: { stagedRunAttachment: { sourceIdentity, targetIdentity } } } } as unknown as typeof authority;
|
||||
const driver = createNativeRunnerExecutionDriver({ target: sandbox(), binding: { ...binding, runId: "run-c" },
|
||||
controllerPersistence: { db: {} as never, bootId: "boot-c", authorityRunId: "run-c" } });
|
||||
await driver.onStagedRunAttachment!(restored, { sourceIdentity, targetIdentity, templateDigest: `sha256:${"a".repeat(64)}` });
|
||||
expect(transport.checkpoint.mock.calls).toEqual([[restored, targetIdentity], [restored, sourceIdentity]]);
|
||||
expect(transport.register).not.toHaveBeenCalled();
|
||||
expect(transport.connect).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps local execution on the registered loopback route", async () => {
|
||||
const driver = createNativeRunnerExecutionDriver({ target: { kind: "local" }, binding });
|
||||
const connection = await driver.controlPlaneRegistration!(authority);
|
||||
|
||||
@@ -92,6 +92,20 @@ export async function prepareNativeRemoteRunner(input: RemotePreparation) {
|
||||
export function createNativeRunnerExecutionDriver(input: NativeRunnerExecutionDriverInput): RunnerExecutionDriver {
|
||||
return {
|
||||
...input.process,
|
||||
onStagedRunAttachment: input.controllerPersistence && input.binding.issueId
|
||||
? async (authority, proof) => {
|
||||
// Both journals use the current business lease. Install persistence
|
||||
// before staging; endpoint acquisition still belongs to attachment.
|
||||
const register = nativeRunnerCheckpointRegistration({ ...input.controllerPersistence!,
|
||||
binding: { companyId: input.binding.companyId, issueId: input.binding.issueId!, runId: input.binding.runId },
|
||||
});
|
||||
// A restored stage already publishes under the target. Its writer
|
||||
// must exist before registration flushes that restored revision.
|
||||
const identities = authority.store?.state.stagedRunAttachment
|
||||
? [proof.targetIdentity, proof.sourceIdentity] : [proof.sourceIdentity, proof.targetIdentity];
|
||||
for (const identity of identities) await register(authority, identity);
|
||||
}
|
||||
: undefined,
|
||||
controlPlaneRegistration: async (authority, attachmentIdentity) => {
|
||||
await input.beforeConnect?.(attachmentIdentity, authority);
|
||||
if (input.controllerPersistence && input.binding.issueId) {
|
||||
|
||||
+239
-57
@@ -60,6 +60,7 @@ import { readProcessStartedAt } from "../hot-restart.js";
|
||||
import { resolvePaperclipInstanceRoot } from "../../home-paths.js";
|
||||
import { prepareNativeHeartbeatRun } from "./prepare-native-run.js";
|
||||
import { buildNativeExecutionInput } from "./native-execution-input.js";
|
||||
import { buildNativeExecutionWithCheckpoint, findNativeSessionResumeRun, nativeToolContractFingerprintForTarget } from "./native-session-resume.js";
|
||||
import { nativeRuntimeContextFixture } from "./runtime-context.test-fixture.js";
|
||||
import { materializeAsset } from "./runtime-context.js";
|
||||
import { createRunnerTestTlsTunnel, dockerCommandRunner } from "../../__tests__/helpers/docker-command-runner.js";
|
||||
@@ -321,8 +322,10 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
};
|
||||
}
|
||||
|
||||
it.skipIf(!process.env.PAPERCLIP_TEST_RUNNER_CONTAINER).each(["warm", "hard", "legacy", "changed", "same-server"])("preserves remote warm continuity across completed turns: %s", async (scenario) => {
|
||||
it.skipIf(!process.env.PAPERCLIP_TEST_RUNNER_CONTAINER).each(["warm", "hard", "legacy", "changed", "same-server", "diskless", "diskless-per-turn", "diskless-per-turn-read-retry"])("preserves remote warm continuity across completed turns: %s", async (scenario) => {
|
||||
const hardRestart = !["warm", "same-server"].includes(scenario);
|
||||
const diskless = scenario.startsWith("diskless");
|
||||
const perTurn = scenario.startsWith("diskless-per-turn");
|
||||
const container = process.env.PAPERCLIP_TEST_RUNNER_CONTAINER!;
|
||||
const tunnel = await createRunnerTestTlsTunnel({ port: Number(process.env.PAPERCLIP_TEST_RUNNER_PORT),
|
||||
key: process.env.PAPERCLIP_TEST_RUNNER_TLS_KEY!, cert: process.env.NODE_EXTRA_CA_CERTS! });
|
||||
@@ -346,12 +349,13 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
taskPrompt: "Complete the first turn.",
|
||||
workspace: { id: randomUUID(), cwd: workspace, repoUrl: null, repoRef: null, branchName: null },
|
||||
normalizedSessionId: fixture.native.normalizedSessionId,
|
||||
lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 },
|
||||
lifecyclePolicy: perTurn ? { mode: "per_turn", idleTimeoutMs: null } : { mode: "warm", idleTimeoutMs: 60_000 },
|
||||
runtimeContext: context,
|
||||
completionContract: await boundContract(fixture.runId),
|
||||
});
|
||||
await fixture.db.update(heartbeatRuns).set({ runnerProfileJson: {
|
||||
...fixture.preparedProfile, nativeExecutionInput: execution,
|
||||
nativeToolContractFingerprint: nativeToolContractFingerprintForTarget("remote", execution.provider.kind),
|
||||
} }).where(eq(heartbeatRuns.id, fixture.runId));
|
||||
const remoteCwd = `/tmp/paperclip-continuity-${fixture.issueId}`;
|
||||
expect((await runner.execute({ command: "mkdir", args: ["-p", remoteCwd] })).exitCode).toBe(0);
|
||||
@@ -385,6 +389,26 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
onLog: async (_stream: "stdout" | "stderr", text: string) => { logs.push(text); },
|
||||
};
|
||||
let controller: ReturnType<typeof spawn> | undefined;
|
||||
const executeInController = (executionOptions: Parameters<typeof executePaperclipNativeSession>[0]) => new Promise<{ exitCode: number | null }>((resolveResult, reject) => {
|
||||
controller = spawn(process.execPath, ["--import", createRequire(import.meta.url).resolve("tsx"), resolve("server/src/__tests__/helpers/remote-warm-controller.test-worker.ts")],
|
||||
{ stdio: ["ignore", "pipe", "pipe", "ipc"], env: process.env });
|
||||
let timer = setTimeout(() => { controller?.kill("SIGKILL"); reject(new Error("Controller fixture imports exceeded 120 seconds")); }, 120_000);
|
||||
controller.stdout?.on("data", data => logs.push(String(data)));
|
||||
controller.stderr?.on("data", data => logs.push(String(data)));
|
||||
controller.on("error", error => { clearTimeout(timer); reject(error); });
|
||||
controller.on("exit", (code, signal) => { clearTimeout(timer); reject(new Error(`Warm controller exited: ${code}/${signal}: ${logs.join("").slice(-3_000)}`)); });
|
||||
controller.on("message", (message: { ready?: boolean; log?: { text: string }; result?: { exitCode: number | null }; error?: string }) => {
|
||||
if (message.ready) {
|
||||
clearTimeout(timer);
|
||||
timer = setTimeout(() => { controller?.kill("SIGKILL"); reject(new Error(`Warm controller admission exceeded 60 seconds: ${logs.join("").slice(-3_000)}`)); }, 60_000);
|
||||
controller!.send?.({ connectionString: temporary.connectionString, container, ingressUrl: `wss://127.0.0.1:${tunnel.port}`,
|
||||
options: { ...executionOptions, db: undefined, onLog: undefined, runnerExecutionTarget: { ...executionOptions.runnerExecutionTarget, runner: undefined } } });
|
||||
}
|
||||
if (message.log) logs.push(message.log.text);
|
||||
if (message.error) { clearTimeout(timer); reject(new Error(message.error)); }
|
||||
if (message.result) { clearTimeout(timer); resolveResult(message.result); }
|
||||
});
|
||||
});
|
||||
try {
|
||||
const firstOptions = { ...options, execution, runnerExecutionTarget: target };
|
||||
const executeFirst = async () => {
|
||||
@@ -454,28 +478,13 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
expect(nextProcess.stdout).toBe(priorProcess.stdout);
|
||||
return result;
|
||||
};
|
||||
const first = hardRestart ? await new Promise<{ exitCode: number | null }>((resolveResult, reject) => {
|
||||
controller = spawn(process.execPath, ["--import", createRequire(import.meta.url).resolve("tsx"), resolve("server/src/__tests__/helpers/remote-warm-controller.test-worker.ts")],
|
||||
{ stdio: ["ignore", "pipe", "pipe", "ipc"], env: process.env });
|
||||
const timer = setTimeout(() => { controller?.kill("SIGKILL"); reject(new Error("Warm controller startup exceeded 60 seconds")); }, 60_000);
|
||||
controller.stdout?.on("data", data => logs.push(String(data)));
|
||||
controller.stderr?.on("data", data => logs.push(String(data)));
|
||||
controller.on("error", error => { clearTimeout(timer); reject(error); });
|
||||
controller.on("exit", (code, signal) => { clearTimeout(timer); reject(new Error(`Warm controller exited: ${code}/${signal}: ${logs.join("").slice(-3_000)}`)); });
|
||||
controller.on("message", (message: { log?: { text: string }; result?: { exitCode: number | null }; error?: string }) => {
|
||||
if (message.log) logs.push(message.log.text);
|
||||
if (message.error) { clearTimeout(timer); reject(new Error(message.error)); }
|
||||
if (message.result) { clearTimeout(timer); resolveResult(message.result); }
|
||||
});
|
||||
controller.send?.({ connectionString: temporary.connectionString, container, ingressUrl: `wss://127.0.0.1:${tunnel.port}`,
|
||||
options: { ...firstOptions, db: undefined, onLog: undefined, runnerExecutionTarget: { ...target, runner: undefined } } });
|
||||
}) : await executeFirst();
|
||||
const first = hardRestart ? await executeInController(firstOptions) : await executeFirst();
|
||||
expect(first.exitCode).toBe(0);
|
||||
await finalizeNativeRun({ db: fixture.db, runId: fixture.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
|
||||
const snapshot = await runner.execute({ command: "sh", args: ["-c", `cat '${remoteCwd}'/.paperclip-runtime/paperclip-runner/sessions/*/runner/runner-state.json`] });
|
||||
expect(snapshot.exitCode).toBe(0);
|
||||
const saved = JSON.parse(snapshot.stdout);
|
||||
expect(saved.lifecycle).toBe("ready");
|
||||
expect(saved.lifecycle).toBe(perTurn ? "suspended" : "ready");
|
||||
const processBefore = await runner.execute({ command: "sh", args: ["-c", `cat '${remoteCwd}'/.paperclip-runtime/paperclip-runner/sessions/*/runner/runner-process.identity`] });
|
||||
expect(processBefore.exitCode).toBe(0);
|
||||
const historyBefore = await runner.execute({ command: "sh", args: ["-c", `cat '${remoteCwd}'/.paperclip-runtime/paperclip-runner/sessions/*/filesystem/codex-home/fake-codex-state.json`] });
|
||||
@@ -502,12 +511,11 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
runnerExecutionTarget: { ...target, leaseId: nextLeaseId, sandboxLeaseAcquisition: { outcome: "resumed" as const, providerLeaseId: container } },
|
||||
};
|
||||
if (controller) {
|
||||
await expect(executePaperclipNativeSession(nextOptions)).rejects.toThrow("prior_owner_terminal_state_indeterminate");
|
||||
const exited = new Promise<void>(resolveExit => controller!.once("exit", () => resolveExit()));
|
||||
controller.kill("SIGKILL");
|
||||
(controller as ReturnType<typeof spawn>).kill("SIGKILL");
|
||||
await exited;
|
||||
controller = undefined;
|
||||
await expect(executePaperclipNativeSession({ ...nextOptions,
|
||||
if (!perTurn) await expect(executePaperclipNativeSession({ ...nextOptions,
|
||||
runnerExecutionTarget: { ...nextOptions.runnerExecutionTarget,
|
||||
sandboxLeaseAcquisition: { outcome: "replacement", providerLeaseId: "unrelated-sandbox" } },
|
||||
})).rejects.toThrow("prior_owner_terminal_state_indeterminate");
|
||||
@@ -526,10 +534,43 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
}
|
||||
expect(changed).toBe(1);
|
||||
}
|
||||
const [nextPrepared] = await fixture.db.select({ profile: heartbeatRuns.runnerProfileJson }).from(heartbeatRuns).where(eq(heartbeatRuns.id, nextRunId));
|
||||
await fixture.db.update(heartbeatRuns).set({ runnerProfileJson: { ...nextPrepared.profile,
|
||||
nativeExecutionInput: nextOptions.execution } }).where(eq(heartbeatRuns.id, nextRunId));
|
||||
const second = await executePaperclipNativeSession(nextOptions);
|
||||
if (diskless) {
|
||||
// Preserve sandbox/provider storage and workspace dependencies. Delete
|
||||
// both native controller projections; admission must discover the source.
|
||||
await rm(resolve(resolvePaperclipInstanceRoot(), "runtime", "paperclip-runner"), { recursive: true, force: true });
|
||||
}
|
||||
// Exercise the existing admission selector/rebinder, which supplies per-turn
|
||||
// runtime snapshots. It does not restore PRP or supervisor files.
|
||||
const admitContinuation = async (current: typeof execution) => {
|
||||
const [row] = await fixture.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, current.binding.runId));
|
||||
const selected = await findNativeSessionResumeRun(fixture.db, { ...current.binding, currentRunId: row.id,
|
||||
normalizedSessionId: fixture.native.normalizedSessionId, beforeCreatedAt: row.createdAt });
|
||||
const admission = buildNativeExecutionWithCheckpoint({ previousRun: selected,
|
||||
normalizedSessionId: fixture.native.normalizedSessionId, executionTargetKind: "remote", buildExecution: () => current });
|
||||
if (perTurn) expect(admission.checkpoint).not.toBeNull();
|
||||
await fixture.db.update(heartbeatRuns).set({ runnerProfileJson: { ...row.runnerProfileJson,
|
||||
nativeExecutionInput: current, nativeToolContractFingerprint: nativeToolContractFingerprintForTarget("remote", current.provider.kind),
|
||||
...(perTurn ? { sessionCheckpoint: admission.checkpoint } : {}) } }).where(eq(heartbeatRuns.id, row.id));
|
||||
};
|
||||
await admitContinuation(nextOptions.execution);
|
||||
if (scenario === "diskless-per-turn-read-retry") {
|
||||
let rejectedReads = 0;
|
||||
await expect(executePaperclipNativeSession({ ...nextOptions,
|
||||
runnerExecutionTarget: { ...nextOptions.runnerExecutionTarget, runner: { ...runner,
|
||||
execute: async request => {
|
||||
if (request.args?.some(arg => arg.includes("base64 <") && arg.includes("runner-state.json"))) {
|
||||
rejectedReads++;
|
||||
throw new Error("fixture_remote_state_timeout");
|
||||
}
|
||||
return runner.execute(request);
|
||||
},
|
||||
} },
|
||||
})).rejects.toThrow("fixture_remote_state_timeout");
|
||||
expect(rejectedReads).toBe(1);
|
||||
// Retry with the partial PRP projection left by the failed read. No
|
||||
// fixture writes a suspended receipt or changes the surviving Runner.
|
||||
}
|
||||
const second = diskless ? await executeInController(nextOptions) : await executePaperclipNativeSession(nextOptions);
|
||||
expect(second.exitCode).toBe(0);
|
||||
const historyAfter = await runner.execute({ command: "sh", args: ["-c", `cat '${remoteCwd}'/.paperclip-runtime/paperclip-runner/sessions/*/filesystem/codex-home/fake-codex-state.json`] });
|
||||
const providerAfter = JSON.parse(historyAfter.stdout);
|
||||
@@ -540,12 +581,19 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
expect(calls.stdout.split("\n").filter(line => line === "thread/start")).toHaveLength(1);
|
||||
const providerStarts = calls.stdout.split("\n").filter(line => line === "process-start").length;
|
||||
expect(providerStarts).toBeGreaterThanOrEqual(1);
|
||||
expect(providerStarts).toBeLessThanOrEqual(["legacy", "changed"].includes(scenario) ? 3 : hardRestart ? 2 : 1);
|
||||
expect(providerStarts).toBeLessThanOrEqual(perTurn || ["legacy", "changed"].includes(scenario) ? 3 : hardRestart ? 2 : 1);
|
||||
const processAfter = await runner.execute({ command: "sh", args: ["-c", `cat '${remoteCwd}'/.paperclip-runtime/paperclip-runner/sessions/*/runner/runner-process.identity`] });
|
||||
expect(processAfter.exitCode).toBe(0);
|
||||
if (["legacy", "changed"].includes(scenario)) expect(processAfter.stdout).not.toBe(processBefore.stdout);
|
||||
if (perTurn || ["legacy", "changed"].includes(scenario)) expect(processAfter.stdout).not.toBe(processBefore.stdout);
|
||||
else expect(processAfter.stdout).toBe(processBefore.stdout);
|
||||
await finalizeNativeRun({ db: fixture.db, runId: nextRunId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
|
||||
if (diskless && controller) {
|
||||
const exited = new Promise<void>(resolveExit => controller!.once("exit", () => resolveExit()));
|
||||
(controller as ReturnType<typeof spawn>).kill("SIGKILL");
|
||||
await exited;
|
||||
controller = undefined;
|
||||
await rm(resolve(resolvePaperclipInstanceRoot(), "runtime", "paperclip-runner"), { recursive: true, force: true });
|
||||
}
|
||||
const thirdRunId = randomUUID();
|
||||
const thirdLeaseId = randomUUID();
|
||||
const [thirdRun] = await fixture.db.insert(heartbeatRuns).values({
|
||||
@@ -562,9 +610,11 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
issue: { id: fixture.issueId, title: "Continue the same conversation", description: null, reviewPolicy: null },
|
||||
environmentLeaseId: thirdLeaseId });
|
||||
await fixture.db.insert(nativeRunFinalizations).values({ runId: thirdRunId, companyId, issueId: fixture.issueId, phase: "observed" });
|
||||
const thirdExecution = { ...nextOptions.execution, completionContract: await boundContract(thirdRunId),
|
||||
binding: { ...execution.binding, runId: thirdRunId }, task: { ...execution.task, prompt: "Complete the third conversation turn." } };
|
||||
await admitContinuation(thirdExecution);
|
||||
const third = await executePaperclipNativeSession({ ...nextOptions, runnerInstanceId: thirdNative.runnerInstanceId,
|
||||
execution: { ...nextOptions.execution, completionContract: await boundContract(thirdRunId),
|
||||
binding: { ...execution.binding, runId: thirdRunId }, task: { ...execution.task, prompt: "Complete the third conversation turn." } },
|
||||
execution: thirdExecution,
|
||||
runnerExecutionTarget: { ...nextOptions.runnerExecutionTarget, leaseId: thirdLeaseId },
|
||||
});
|
||||
expect(third.exitCode).toBe(0);
|
||||
@@ -573,10 +623,11 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
const finalCalls = await runner.execute({ command: "cat", args: [`${remoteCwd}/provider-calls.log`] });
|
||||
expect(finalCalls.stdout.split("\n").filter(line => line === "turn/start")).toHaveLength(3);
|
||||
expect(finalCalls.stdout.split("\n").filter(line => line === "thread/start")).toHaveLength(1);
|
||||
expect(finalCalls.stdout.split("\n").filter(line => line === "process-start")).toHaveLength(providerStarts);
|
||||
expect(finalCalls.stdout.split("\n").filter(line => line === "process-start")).toHaveLength(providerStarts + (perTurn ? 1 : 0));
|
||||
const finalProcess = await runner.execute({ command: "sh", args: ["-c", `cat '${remoteCwd}'/.paperclip-runtime/paperclip-runner/sessions/*/runner/runner-process.identity`] });
|
||||
expect(finalProcess.stdout).toBe(processAfter.stdout);
|
||||
expect(await closeIdleWarmNativeSessionsForRestart()).toEqual({ closed: 1, busy: 0, failed: 0 });
|
||||
if (perTurn) expect(finalProcess.stdout).not.toBe(processAfter.stdout);
|
||||
else expect(finalProcess.stdout).toBe(processAfter.stdout);
|
||||
expect(await closeIdleWarmNativeSessionsForRestart()).toEqual({ closed: perTurn ? 0 : 1, busy: 0, failed: 0 });
|
||||
const suspended = await runner.execute({ command: "sh", args: ["-c", `cat '${remoteCwd}'/.paperclip-runtime/paperclip-runner/sessions/*/runner/runner-state.json`] });
|
||||
expect(suspended.exitCode).toBe(0);
|
||||
expect(JSON.parse(suspended.stdout).lifecycle).toBe("suspended");
|
||||
@@ -588,7 +639,7 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
await closeIdleWarmNativeSessionsForRestart();
|
||||
await tunnel.close();
|
||||
}
|
||||
}, 180_000);
|
||||
}, 360_000);
|
||||
|
||||
async function persistRestartEvidence(input: {
|
||||
fixture: Awaited<ReturnType<typeof seedRun>>;
|
||||
@@ -1277,10 +1328,11 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
} finally { await authority.stop(); }
|
||||
});
|
||||
|
||||
it.each(["idle", "preparing_successor", "unclaimed_successor", "claimed_successor", "expired_preparation", "dispatching_successor", "foreign_preparation", "foreign_native_preparation", "wrong_issue", "new_work", "replaced_controller"] as const)(
|
||||
"settles a completed warm checkpoint during early retirement without granting work: %s",
|
||||
async scenario => {
|
||||
const fixture = await seedRun(`WARM-EARLY-RETIREMENT-${scenario}`);
|
||||
it.each((["idle", "preparing_successor", "unclaimed_successor", "claimed_successor", "expired_preparation", "dispatching_successor", "foreign_preparation", "foreign_native_preparation", "wrong_issue", "new_work", "replaced_controller", "unaccepted"] as const).flatMap(scenario =>
|
||||
(["runner.suspend", "turn.stop"] as const).map(command => ({ scenario, command }))))(
|
||||
"settles a completed warm checkpoint during early retirement without granting work: $scenario/$command",
|
||||
async ({ scenario, command }) => {
|
||||
const fixture = await seedRun(`WARM-EARLY-RETIREMENT-${scenario}-${command}`);
|
||||
const binding = { companyId, issueId: fixture.issueId, runId: fixture.runId };
|
||||
const identity = transportOptions(fixture, runtimeRoot).prpIdentity;
|
||||
const authority = new DurablePrpControlPlane({
|
||||
@@ -1305,7 +1357,7 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
});
|
||||
await fixture.db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() })
|
||||
.where(eq(heartbeatRuns.id, fixture.runId));
|
||||
await fixture.db.update(nativeRunFinalizations).set({ phase: "committed", resultId,
|
||||
await fixture.db.update(nativeRunFinalizations).set({ phase: "committed", resultId: scenario === "unaccepted" ? null : resultId,
|
||||
leaseOwner: null, leaseExpiresAt: null,
|
||||
...(scenario === "replaced_controller" ? { controllerBootId: randomUUID(), controllerGeneration: 2 } : {}),
|
||||
}).where(eq(nativeRunFinalizations.runId, fixture.runId));
|
||||
@@ -1314,7 +1366,7 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
if (scenario !== "idle") {
|
||||
nextRunId = randomUUID();
|
||||
const preparing = scenario !== "claimed_successor" && scenario !== "unclaimed_successor";
|
||||
const otherIssue = scenario === "wrong_issue" ? (await seedRun("WARM-OTHER-ISSUE")).issueId : fixture.issueId;
|
||||
const otherIssue = scenario === "wrong_issue" ? (await seedRun(`WARM-OTHER-ISSUE-${command}`)).issueId : fixture.issueId;
|
||||
await fixture.db.insert(heartbeatRuns).values({ id: nextRunId, companyId, agentId,
|
||||
issueId: otherIssue, scopeKind: "issue", contextSnapshot: { issueId: otherIssue },
|
||||
...(preparing ? { runtimeMode: "legacy", executionStage: scenario === "dispatching_successor" ? "dispatching" : "preparing",
|
||||
@@ -1331,7 +1383,7 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
}
|
||||
await fixture.db.update(issues).set({ executionRunId: nextRunId }).where(eq(issues.id, fixture.issueId));
|
||||
const before = (await loadNativeRunnerCheckpoint(fixture.db, binding))!;
|
||||
authority.queueCommand(scenario === "new_work" ? "turn.start" : "runner.suspend", {}, `early-retirement-${scenario}`);
|
||||
authority.queueCommand(scenario === "new_work" ? "turn.start" : command, {}, `early-retirement-${scenario}`);
|
||||
if (!["idle", "preparing_successor", "claimed_successor"].includes(scenario)) {
|
||||
await expect(authority.flushCheckpoint()).rejects.toThrow("native_runner_checkpoint_owner_lost");
|
||||
expect((await loadNativeRunnerCheckpoint(fixture.db, binding))?.records).toEqual(before.records);
|
||||
@@ -1342,7 +1394,7 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
saved.restoreSessionDirectory(restoredRoot);
|
||||
const restored = JSON.parse(await readFile(resolve(restoredRoot, "control-plane", "control-plane-state.json"), "utf8"));
|
||||
expect(restored.identity).toEqual(identity);
|
||||
expect(restored.commands).toContainEqual(expect.objectContaining({ type: "runner.suspend", status: "pending" }));
|
||||
expect(restored.commands).toContainEqual(expect.objectContaining({ type: command, status: "pending" }));
|
||||
}
|
||||
const [task] = await fixture.db.select({ executionRunId: issues.executionRunId }).from(issues)
|
||||
.where(eq(issues.id, fixture.issueId));
|
||||
@@ -1351,6 +1403,104 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
},
|
||||
);
|
||||
|
||||
realProcessIt("durably retires an accepted warm Runner while its same-controller successor prepares", async () => {
|
||||
const fixture = await seedRun("REAL-WARM-RETIREMENT");
|
||||
const binding = { companyId, issueId: fixture.issueId, runId: fixture.runId };
|
||||
const root = resolve(runtimeRoot, fixture.runId, "real-retirement");
|
||||
const options = transportOptions(fixture, root);
|
||||
const retirementController = await currentNativeControllerIdentity();
|
||||
const diagnostics: string[] = [], checkpointFailures: unknown[] = [];
|
||||
let authority!: DurablePrpControlPlane;
|
||||
await fixture.db.update(nativeRunFinalizations).set({ controllerBootId: retirementController.bootId,
|
||||
controllerGeneration: 1, leaseOwner: "real-retirement-owner", leaseExpiresAt: new Date(Date.now() + 60_000),
|
||||
}).where(eq(nativeRunFinalizations.runId, fixture.runId));
|
||||
const transport = createRunnerdCodexTransport({ ...options,
|
||||
codexArgs: [...options.codexArgs, "--linger-after-turn-start"],
|
||||
onDiagnostic: message => diagnostics.push(message),
|
||||
controlPlaneRegistration: async core => {
|
||||
authority = core;
|
||||
const write = nativeRunnerCheckpointWriter(fixture.db, { ...binding, authorityRunId: fixture.runId,
|
||||
bootId: retirementController.bootId, generation: 1, leaseOwner: "real-retirement-owner" });
|
||||
core.setCheckpointPersistence(async checkpoint => {
|
||||
try { await write(checkpoint); }
|
||||
catch (error) { checkpointFailures.push(error); throw error; }
|
||||
});
|
||||
return registerRunnerPrpAuthority({ ...binding, authority: core });
|
||||
},
|
||||
});
|
||||
let runnerPid: number | null = null, providerPid: number | null = null;
|
||||
try {
|
||||
await transport.transport.request("thread/start", { cwd: root, dynamicTools: [] });
|
||||
await transport.transport.request("turn/start", { input: [{ type: "text", text: "Keep the provider alive after accepting its result." }] });
|
||||
runnerPid = transport.evidence().runnerPid;
|
||||
providerPid = transport.evidence().providerPid;
|
||||
expect(runnerPid).toEqual(expect.any(Number));
|
||||
expect(providerPid).toEqual(expect.any(Number));
|
||||
await waitForCondition("active real provider before accepted-result retirement", () =>
|
||||
authority.store.state.committedEvents.some(event => event.eventType === "turn.started"));
|
||||
await authority.flushCheckpoint();
|
||||
const [run] = await fixture.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId));
|
||||
const resultId = randomUUID();
|
||||
await fixture.db.insert(nativeRunResults).values({ id: resultId, ...binding, turnId: options.prpIdentity.turnId,
|
||||
completionContractId: run.completionContractId!, serverFingerprint: "real-retirement-accepted",
|
||||
schemaStatus: "accepted", canonicalSha256: "a".repeat(64),
|
||||
resultJson: { schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", summary: "Accepted before provider cleanup." },
|
||||
});
|
||||
await fixture.db.update(nativeRunFinalizations).set({ phase: "committed", resultId, leaseOwner: null, leaseExpiresAt: null })
|
||||
.where(eq(nativeRunFinalizations.runId, fixture.runId));
|
||||
await fixture.db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() })
|
||||
.where(eq(heartbeatRuns.id, fixture.runId));
|
||||
const nextRunId = randomUUID();
|
||||
const [nextRun] = await fixture.db.insert(heartbeatRuns).values({ id: nextRunId, companyId, agentId,
|
||||
issueId: fixture.issueId, nativeIssueId: fixture.issueId, scopeKind: "issue", status: "running", runtimeMode: "native",
|
||||
invocationSource: "assignment", contextSnapshot: { issueId: fixture.issueId } }).returning();
|
||||
const next = await prepareNativeHeartbeatRun({ db: fixture.db, run: nextRun,
|
||||
issue: { id: fixture.issueId, title: "Next conversation", description: null }, environmentLeaseId: randomUUID() });
|
||||
await fixture.db.insert(nativeRunFinalizations).values({ runId: nextRunId, companyId, issueId: fixture.issueId, phase: "observed" });
|
||||
await fixture.db.update(issues).set({ executionRunId: nextRunId }).where(eq(issues.id, fixture.issueId));
|
||||
const [contract] = await fixture.db.select().from(completionContracts).where(eq(completionContracts.id, run.completionContractId!));
|
||||
const nextExecution = buildNativeExecutionInput({ companyId, agentId, runId: nextRunId,
|
||||
issue: { id: fixture.issueId, identifier: "NRR-NATIVE-RETIREMENT", title: "Next conversation", description: null, workMode: "standard" },
|
||||
taskPrompt: "Start only after the previous provider durably suspends.",
|
||||
workspace: { id: randomUUID(), cwd: root, repoUrl: null, repoRef: null, branchName: null },
|
||||
normalizedSessionId: next.normalizedSessionId, runtimeContext: nativeRuntimeContextFixture(),
|
||||
completionContract: { id: contract.id, sha256: contract.canonicalSha256, schemaVersion: contract.schemaVersion, contract: contract.contractJson as never },
|
||||
});
|
||||
const fixtureStoppedAfterRealClaim = new Error("fixture stopped before next provider dispatch");
|
||||
await expect(executePaperclipNativeSession({ db: fixture.db, execution: nextExecution, runnerInstanceId: next.runnerInstanceId,
|
||||
onExecutionLeaseAcquired: async () => {
|
||||
const [claimed] = await fixture.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, nextRunId));
|
||||
expect(claimed.controllerBootId).toBe(retirementController.bootId);
|
||||
expect(claimed.leaseOwner).toEqual(expect.any(String));
|
||||
expect(claimed.leaseExpiresAt!.getTime()).toBeGreaterThan(Date.now());
|
||||
await expect(transport.transport.close(), diagnostics.join("\n")).resolves.toBeUndefined();
|
||||
throw fixtureStoppedAfterRealClaim;
|
||||
},
|
||||
})).rejects.toBe(fixtureStoppedAfterRealClaim);
|
||||
expect(checkpointFailures).toEqual([]);
|
||||
const state = JSON.parse(await readFile(resolve(root, "runner", "runner-state.json"), "utf8"));
|
||||
expect(state).toMatchObject({ lifecycle: "suspended", runId: fixture.runId,
|
||||
normalizedSessionId: options.prpIdentity.normalizedSessionId, runnerInstanceId: options.prpIdentity.runnerInstanceId });
|
||||
const saved = (await loadNativeRunnerCheckpoint(fixture.db, binding))!;
|
||||
const commands = [...saved.records].filter(([key]) => key.startsWith("command:")).map(([, value]) => JSON.parse(value));
|
||||
for (const type of ["turn.stop", "runner.drain", "runner.suspend"]) {
|
||||
expect(commands, diagnostics.join("\n")).toContainEqual(expect.objectContaining({ type, status: "completed" }));
|
||||
}
|
||||
expect(commands.filter(command => command.type === "turn.start")).toHaveLength(1);
|
||||
const [task] = await fixture.db.select().from(issues).where(eq(issues.id, fixture.issueId));
|
||||
expect(task.executionRunId).toBe(nextRunId);
|
||||
} catch (error) {
|
||||
throw new Error(`${String(error)}; owned retirement diagnostics ${JSON.stringify({
|
||||
checkpointFailures: checkpointFailures.map(String), diagnostics,
|
||||
commands: authority?.store.state.commands.map(({ type, status }) => ({ type, status })),
|
||||
})}`);
|
||||
} finally {
|
||||
await transport.transport.close().catch(() => undefined);
|
||||
if (providerPid && processAlive(providerPid)) process.kill(providerPid, "SIGKILL");
|
||||
await stopOwnedProcessGroup(runnerPid, root).catch(() => undefined);
|
||||
}
|
||||
}, 45_000);
|
||||
|
||||
it("applies checkpoint deltas in Postgres without rewriting authority and removes only evicted events", async () => {
|
||||
const fixture = await seedRun("CHECKPOINT-DELTA");
|
||||
const root = resolve(runtimeRoot, fixture.runId);
|
||||
@@ -1463,8 +1613,8 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
realProcessIt("keeps checkpoint SQL independent of streamed output volume with a real Postgres writer", async () => {
|
||||
const samples: Array<{ chunks: number; checkpointWrites: number; transactions: number;
|
||||
changedRows: number; checkpointBytes: number; walBytes: number; elapsedMs: number; idleSql: number;
|
||||
streamSql: number; streamTransactions: number; streamChangedRows: number }> = [];
|
||||
for (const chunks of [8, 512]) {
|
||||
streamSql: number; streamTransactions: number; streamChangedRows: number; renewalSql: number }> = [];
|
||||
for (const chunks of [8, 512, 8192]) {
|
||||
const fixture = await seedRun(`CHECKPOINT-COST-${chunks}`);
|
||||
const root = resolve(runtimeRoot, fixture.runId);
|
||||
const releaseOutput = resolve(root, "release-output");
|
||||
@@ -1472,19 +1622,31 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
const owner = { ...binding, authorityRunId: fixture.runId, bootId: successor.bootId,
|
||||
generation: 1, leaseOwner: "cost-controller" };
|
||||
await fixture.db.update(nativeRunFinalizations).set({ controllerBootId: owner.bootId,
|
||||
controllerGeneration: 1, leaseOwner: owner.leaseOwner, leaseExpiresAt: new Date(Date.now() + 120_000),
|
||||
controllerPid: successor.pid, controllerProcessStartedAt: successor.processStartedAt,
|
||||
controllerGeneration: 1, leaseOwner: owner.leaseOwner, leaseExpiresAt: new Date(Date.now() + 60_000),
|
||||
}).where(eq(nativeRunFinalizations.runId, fixture.runId));
|
||||
let authority!: DurablePrpControlPlane;
|
||||
let measuring = false;
|
||||
let transactions = 0, changedRows = 0, checkpointBytes = 0;
|
||||
const statements: string[] = [];
|
||||
const capturedClients = [fixture.db.$client, checkpointDb(fixture.db).$client];
|
||||
const renewalStatements: string[] = [];
|
||||
let leaseLost = false;
|
||||
const renewal = startNativeSessionExecutionLeaseRenewal({ db: fixture.db, ...binding, leaseOwner: owner.leaseOwner,
|
||||
attempt: 0, controller: successor, remote: true, onLost: async () => { leaseLost = true; } });
|
||||
const capturedClients = [fixture.db.$client, checkpointDb(fixture.db).$client, livenessDb(fixture.db).$client];
|
||||
const priorDebug = capturedClients.map(client => client.options.debug);
|
||||
// Capture statements only, never SQL parameters (encrypted state or secrets).
|
||||
for (const client of capturedClients) {
|
||||
client.options.debug = (_connection, query) => { if (measuring) statements.push(query); };
|
||||
}
|
||||
capturedClients.forEach((client, index) => {
|
||||
client.options.debug = (_connection, query) => {
|
||||
if (!measuring) return;
|
||||
// Renewals are existing liveness traffic, reported separately. A
|
||||
// checkpoint query on any pool still counts against the zero oracle.
|
||||
if (index === 2 && !query.includes("native_runner_checkpoints")) renewalStatements.push(query);
|
||||
else statements.push(query);
|
||||
};
|
||||
});
|
||||
const write = nativeRunnerCheckpointWriter(fixture.db, owner);
|
||||
let costFailure: unknown;
|
||||
const transport = createRunnerdCodexTransport({ ...transportOptions(fixture, root),
|
||||
codexArgs: ["--state-file", resolve(root, "provider.json"), "--linger-after-turn-start",
|
||||
"--held-output", "--held-output-count", String(chunks), "--held-output-release-file", releaseOutput],
|
||||
@@ -1516,7 +1678,7 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
const beforeStream = { sql: statements.length, transactions, changedRows };
|
||||
await writeFile(releaseOutput, "release\n", { flag: "wx" });
|
||||
await waitForCondition(`the final scripted text chunk of ${chunks}`, () => authority.store.state.committedEvents
|
||||
.some(event => event.eventType === "item.delta" && JSON.stringify(event.envelope).includes(`still computing ${chunks - 1} `)), 60_000)
|
||||
.some(event => event.eventType === "item.delta" && JSON.stringify(event.envelope).includes(`still computing ${chunks - 1} `)), chunks > 4096 ? 480_000 : 60_000)
|
||||
.catch(error => { console.info("checkpoint-cost-stream-diagnostic", JSON.stringify({ chunks,
|
||||
cursor: authority.store.state.ackedSourceSeq,
|
||||
receivedTextEvents: authority.store.state.committedEvents.filter(event => event.eventType === "item.delta").length })); throw error; });
|
||||
@@ -1528,13 +1690,14 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
&& query.includes("native_runner_checkpoints")).length;
|
||||
expect(checkpointWrites).toBeGreaterThan(0); // The collector must see real control writes.
|
||||
const snapshot = { chunks, checkpointWrites, transactions, changedRows, checkpointBytes,
|
||||
walBytes: Number(walEnd.bytes), elapsedMs, idleSql: 0,
|
||||
walBytes: Number(walEnd.bytes), elapsedMs, idleSql: 0, renewalSql: renewalStatements.length,
|
||||
streamSql: statements.length - beforeStream.sql,
|
||||
streamTransactions: transactions - beforeStream.transactions,
|
||||
streamChangedRows: changedRows - beforeStream.changedRows };
|
||||
// Setup control messages may coalesce differently. The released text-only
|
||||
// interval must add exactly zero work, independent of that scheduling.
|
||||
expect(snapshot).toMatchObject({ streamSql: 0, streamTransactions: 0, streamChangedRows: 0 });
|
||||
expect(leaseLost).toBe(false);
|
||||
statements.length = 0;
|
||||
measuring = true;
|
||||
await new Promise(done => setTimeout(done, 100));
|
||||
@@ -1547,20 +1710,34 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
expect(statements.some(query => /insert into "native_runner_checkpoints"/i.test(query))).toBe(true);
|
||||
measuring = false;
|
||||
samples.push(snapshot);
|
||||
console.info("native-checkpoint-cost-sample", JSON.stringify(snapshot));
|
||||
} catch (error) {
|
||||
costFailure = error;
|
||||
throw error;
|
||||
} finally {
|
||||
measuring = false;
|
||||
capturedClients.forEach((client, index) => { client.options.debug = priorDebug[index]; });
|
||||
await transport.transport.close();
|
||||
try { await transport.transport.close(); }
|
||||
catch (error) {
|
||||
if (costFailure) throw new AggregateError([costFailure, error], "Checkpoint cost measurement and cleanup failed");
|
||||
throw error;
|
||||
}
|
||||
finally {
|
||||
await renewal.stop().catch(error => {
|
||||
if (costFailure) throw new AggregateError([costFailure, error], "Checkpoint cost measurement and lease renewal failed");
|
||||
throw error;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
const report = { schema: "paperclip.checkpoint-cost/v1", samples,
|
||||
scope: "isolated Postgres; control operations included; WAL is diagnostic; no production capacity claim" };
|
||||
if (process.env.PAPERCLIP_TEST_CHECKPOINT_COST_REPORT) await writeFile(process.env.PAPERCLIP_TEST_CHECKPOINT_COST_REPORT,
|
||||
`${JSON.stringify(report, null, 2)}\n`, { flag: "wx", mode: 0o600 });
|
||||
// Both stream sizes were checked independently; total setup counts remain
|
||||
// Each stream size was checked independently; total setup counts remain
|
||||
// diagnostic because control-message batching is scheduling-dependent.
|
||||
console.info("native-checkpoint-cost", JSON.stringify(report));
|
||||
}, 120_000);
|
||||
}, 540_000);
|
||||
|
||||
realProcessIt.each([false, true])("adopts the same runner from Postgres after losing the controller filesystem (prepared task: %s)", async preparedTask => {
|
||||
const fixture = await seedRun(`EMPTY-CONTROLLER-DISK-${preparedTask}`);
|
||||
@@ -1579,9 +1756,10 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
heartbeatRunId: fixture.runId, issueId: fixture.issueId, status: "active", leasePolicy: "ephemeral",
|
||||
provider: "plugin", providerLeaseId: "saved-task", metadata: { agentId, runnerTaskLease: true, runnerTaskDispatched: true } });
|
||||
}
|
||||
const originalLeaseExpiresAt = new Date(Date.now() + 60_000);
|
||||
await fixture.db.update(nativeRunFinalizations).set({ controllerBootId: bootId,
|
||||
controllerGeneration: 1, controllerPid: process.pid, controllerProcessStartedAt: new Date(),
|
||||
leaseOwner: "original-controller", leaseExpiresAt: new Date(Date.now() + 60_000),
|
||||
leaseOwner: "original-controller", leaseExpiresAt: originalLeaseExpiresAt,
|
||||
}).where(eq(nativeRunFinalizations.runId, fixture.runId));
|
||||
const options = {
|
||||
...transportOptions(fixture, controllerRoot),
|
||||
@@ -1647,9 +1825,13 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
|
||||
await new Promise(resolveOutage => setTimeout(resolveOutage, outageMs));
|
||||
expect(processAlive(runnerPid)).toBe(true);
|
||||
expect(processAlive(providerPid)).toBe(true);
|
||||
await fixture.db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 1) })
|
||||
.where(eq(nativeRunFinalizations.runId, fixture.runId));
|
||||
bootId = randomUUID();
|
||||
if (originalLeaseExpiresAt.getTime() > Date.now()) {
|
||||
const [waiting] = await claimNativeRestartRecoveries({ db: fixture.db, controller: { ...successor, bootId },
|
||||
restartKind: "hard", runIds: [fixture.runId] });
|
||||
expect(waiting).toMatchObject({ kind: "awaiting_evidence", reason: "remote_controller_lease_live", retryAt: originalLeaseExpiresAt });
|
||||
await new Promise(resolveExpiry => setTimeout(resolveExpiry, Math.max(0, originalLeaseExpiresAt.getTime() - Date.now()) + 20));
|
||||
}
|
||||
const [claim] = await claimNativeRestartRecoveries({ db: fixture.db, controller: { ...successor, bootId },
|
||||
restartKind: "hard", runIds: [fixture.runId] });
|
||||
expect(claim).toMatchObject({ kind: "reattach_remote_runner", providerAttempt: 0, controllerGeneration: 2 });
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { onRetainedNativeSessionCleanupRelease } from "../retained-native-session-cleanup-notifications.js";
|
||||
import { readIdleLocalWork, markIdleStartupComplete, markIdleIngressTracked } from "../idle-local-work.js";
|
||||
import * as runnerCheckpoint from "./native-runner-checkpoint.js";
|
||||
import * as stagedAttachmentRecovery from "./native-staged-attachment-recovery.js";
|
||||
import * as controllerRecovery from "./native-controller-recovery.js";
|
||||
import { DurablePrpControlPlane } from "../../vendor/paperclip-runner/index.js";
|
||||
import * as environmentTaskRuntime from "../environment-task-runtime.js";
|
||||
import * as pluginTaskRuntime from "./plugin-runner-task.js";
|
||||
@@ -47,6 +49,7 @@ import {
|
||||
import {
|
||||
canonicalNativeRuntimeContextDigest,
|
||||
CURSOR_DISTRIBUTION_PINS,
|
||||
KIMI_DISTRIBUTION_CLOSURE_SHA256,
|
||||
QUALIFIED_ACPX_PROFILES,
|
||||
acpxRuntimeSessionDirectoryName,
|
||||
resolveAcpxRuntimeRoot,
|
||||
@@ -96,6 +99,7 @@ import { buildNativeHeartbeatPreparationSpans, createNativeRunTrace } from "./na
|
||||
import { NativeRunnerOwnershipUnverifiedError } from "./native-runner-ownership.js";
|
||||
import type { AdapterRuntimeEvent } from "../../adapters/index.js";
|
||||
import type { AdapterUsageCheckpoint } from "@paperclipai/adapter-utils";
|
||||
import { acknowledgedNativeStopExecutionHasStopped, isAcknowledgedNativeStop } from "../acknowledged-native-stop.js";
|
||||
|
||||
vi.mock("node:fs", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("node:fs")>();
|
||||
@@ -391,6 +395,8 @@ import {
|
||||
REMOTE_RUNNER_CHILD_LAUNCH_SCRIPT,
|
||||
verifyRemoteRunnerReattachment,
|
||||
readRemoteProviderPackManifest,
|
||||
prepareRemoteProviderPackOnTarget,
|
||||
resolveRemoteProviderPackSource,
|
||||
readBundledRemoteProviderPackManifest,
|
||||
providerSessionIdentityFromDurableProviderState,
|
||||
durableProviderCheckpointFailureReason,
|
||||
@@ -416,6 +422,10 @@ import {
|
||||
import { onNativeRunnerEvidence } from "./native-runner-evidence-signal.js";
|
||||
|
||||
beforeEach(() => {
|
||||
// These transport-wiring fixtures use a partial Db double and synthetic IDs.
|
||||
// The real selector/bootstrap proof is exercised with Postgres by the durable
|
||||
// discovery and diskless admission suites, not by this backend unit boundary.
|
||||
vi.spyOn(controllerRecovery, "resolveNativeControllerRecovery").mockResolvedValue({ kind: "no_prior_execution" });
|
||||
state.drainMaintenance.mockReset().mockResolvedValue(undefined);
|
||||
usageRecorder.capture.mockReset().mockImplementation(async receipt => receipt);
|
||||
usageRecorder.create.mockReset().mockResolvedValue({ capture: usageRecorder.capture,
|
||||
@@ -1946,6 +1956,40 @@ describe("remote provider pack manifest", () => {
|
||||
expect(readRemoteProviderPackManifest(root).payload.pins.opencode).toBe(
|
||||
"1.18.34",
|
||||
);
|
||||
// Both execution and setup Test consume this same normal installer. Reuse
|
||||
// the complete controller fixture here instead of inventing a thin pack.
|
||||
const selectedPack = resolveRemoteProviderPackSource(root);
|
||||
for (const mode of ["staged", "preinstalled", "uploaded", "failed-final", "missing-transport"] as const) {
|
||||
let transferred = false;
|
||||
const syncIn = vi.fn(async (operations: { files: { sourcePath?: string; targetPath: string; kind: string }[] }[]) => {
|
||||
expect(operations[0]?.files[0]).toMatchObject({ sourcePath: root,
|
||||
targetPath: "/workspace/.paperclip-runtime/paperclip-runner/provider-pack", kind: "directory" });
|
||||
transferred = true;
|
||||
return { operations: [] };
|
||||
});
|
||||
const execute = vi.fn(async (command: { command: string; args?: string[]; timeoutMs?: number }) => {
|
||||
expect(command.timeoutMs).toBeGreaterThan(0);
|
||||
expect(command.timeoutMs).toBeLessThanOrEqual(30_000);
|
||||
const verification = command.args?.[0] === "-e";
|
||||
const packRoot = command.args?.[2];
|
||||
const valid = mode === "staged" || mode === "preinstalled" && packRoot === "/opt/paperclip-runner/provider-pack"
|
||||
|| transferred && mode !== "failed-final";
|
||||
return { pid: null, startedAt: null, exitCode: verification && !valid ? 1 : 0, timedOut: false, signal: null,
|
||||
stdout: command.args?.[0] === "--version" ? "1.18.34\n"
|
||||
: command.args?.some(value => value.includes("for candidate in")) ? "/opt/paperclip-runner/provider-pack\n" : "",
|
||||
stderr: verification && !valid ? "manifest mismatch" : "" };
|
||||
});
|
||||
const runner = { execute, ...(mode === "missing-transport" ? {} : { syncIn }) };
|
||||
const preparation = prepareRemoteProviderPackOnTarget({
|
||||
target: { kind: "remote", transport: "sandbox", remoteCwd: "/workspace", runner } as never,
|
||||
runner, configuredProviderPackRoot: selectedPack.configuredRoot, manifest: selectedPack.manifest,
|
||||
});
|
||||
if (mode === "failed-final") await expect(preparation).rejects.toThrow("provider pack verification");
|
||||
else if (mode === "missing-transport") await expect(preparation).rejects.toThrow("transport cannot stage a provider pack");
|
||||
else await expect(preparation).resolves.toMatchObject({ source: mode, root: "/workspace/.paperclip-runtime/paperclip-runner/provider-pack",
|
||||
manifest: selectedPack.manifest });
|
||||
expect(syncIn).toHaveBeenCalledTimes(mode === "uploaded" || mode === "failed-final" ? 1 : 0);
|
||||
}
|
||||
const cursorPath = "provider-assets/cursor/linux-x64";
|
||||
await mkdir(join(root, cursorPath), { recursive: true });
|
||||
await writeFile(join(root, cursorPath, "runtime"), "pinned Cursor runtime");
|
||||
@@ -1980,6 +2024,78 @@ describe("remote provider pack manifest", () => {
|
||||
await writeFile(join(root, cursorPath, "runtime"), "substitute Cursor runtime");
|
||||
expect(() => readRemoteProviderPackManifest(root)).toThrow("asset tree digest mismatch");
|
||||
await writeFile(join(root, cursorPath, "runtime"), "pinned Cursor runtime");
|
||||
const kimiPath = "provider-assets/kimi/linux-x64";
|
||||
await mkdir(join(root, kimiPath), { recursive: true });
|
||||
await writeFile(join(root, kimiPath, "runtime"), "pinned Kimi runtime");
|
||||
const kimi = { version: QUALIFIED_ACPX_PROFILES.kimi.agentServerVersion,
|
||||
profileDigest: QUALIFIED_ACPX_PROFILES.kimi.commandDigest,
|
||||
closureDigest: `sha256:${KIMI_DISTRIBUTION_CLOSURE_SHA256["linux-x64"]}`,
|
||||
qualification: "qualified", path: kimiPath, sha256: sha256DirectoryTree(join(root, kimiPath)) };
|
||||
Object.assign(payload, { providers: { cursor, kimi } });
|
||||
await mkdir(releaseMetadata, { recursive: true });
|
||||
const writeKimiManifest = async () => {
|
||||
await writeManifest();
|
||||
await cp(join(root, "provider-pack.json"), manifestPath);
|
||||
};
|
||||
await writeKimiManifest();
|
||||
expect(readRemoteProviderPackManifest(root).payload.providers?.kimi).toEqual(kimi);
|
||||
expect(readBundledRemoteProviderPackManifest(manifestPath).payload.providers?.kimi?.qualification).toBe("qualified");
|
||||
for (const field of ["version", "profileDigest", "closureDigest"] as const) {
|
||||
const previous = kimi[field];
|
||||
kimi[field] = field === "version" ? "older-kimi" : digest(`old-kimi-${field}`);
|
||||
await writeKimiManifest();
|
||||
expect(() => readRemoteProviderPackManifest(root)).toThrow("Kimi profile or closure");
|
||||
expect(() => readBundledRemoteProviderPackManifest(manifestPath)).toThrow("Kimi profile or closure");
|
||||
kimi[field] = previous;
|
||||
}
|
||||
kimi.closureDigest = `sha256:${KIMI_DISTRIBUTION_CLOSURE_SHA256["darwin-arm64"]}`;
|
||||
await writeKimiManifest();
|
||||
expect(() => readBundledRemoteProviderPackManifest(manifestPath)).toThrow("Kimi profile or closure");
|
||||
kimi.closureDigest = `sha256:${KIMI_DISTRIBUTION_CLOSURE_SHA256["linux-x64"]}`;
|
||||
for (const target of [{ platform: "linux", architecture: "arm64" }, { platform: "darwin", architecture: "arm64" }]) {
|
||||
payload.target = target;
|
||||
kimi.path = `provider-assets/kimi/${target.platform}-${target.architecture}`;
|
||||
Object.assign(payload, { providers: {} });
|
||||
await writeKimiManifest();
|
||||
expect(() => readBundledRemoteProviderPackManifest(manifestPath)).toThrow("bundled image pack must target linux-x64");
|
||||
}
|
||||
payload.target = originalTarget;
|
||||
kimi.path = kimiPath;
|
||||
Object.assign(payload, { providers: { cursor, kimi } });
|
||||
kimi.qualification = "pending";
|
||||
await writeKimiManifest();
|
||||
expect(() => readBundledRemoteProviderPackManifest(manifestPath)).toThrow("invalid candidate identity");
|
||||
kimi.qualification = "qualified";
|
||||
Object.assign(payload, { providers: { cursor, kimi }, candidateProviders: { kimi } });
|
||||
await writeKimiManifest();
|
||||
expect(() => readBundledRemoteProviderPackManifest(manifestPath)).toThrow("Kimi profile or closure");
|
||||
Object.assign(payload, { providers: { cursor, kimi } });
|
||||
Reflect.deleteProperty(payload, "candidateProviders");
|
||||
await writeKimiManifest();
|
||||
await writeFile(join(root, kimiPath, "runtime"), "substitute Kimi runtime");
|
||||
expect(() => readRemoteProviderPackManifest(root)).toThrow("candidate asset tree digest mismatch");
|
||||
await writeFile(join(root, kimiPath, "runtime"), "pinned Kimi runtime");
|
||||
const threeCandidates = { cursor,
|
||||
pi: { ...kimi, version: "1.0.0", qualification: "qualified", path: "provider-assets/pi/linux-x64" },
|
||||
copilot: { ...kimi, qualification: "pending", version: QUALIFIED_ACPX_PROFILES.copilot.agentServerVersion,
|
||||
profileDigest: QUALIFIED_ACPX_PROFILES.copilot.commandDigest,
|
||||
closureDigest: "sha256:a3d8f4367cfa1694d79e3f8b7930b6fbfe42a229c272b375b11951d631db8d07",
|
||||
path: "provider-assets/copilot/linux-x64" } };
|
||||
for (const candidate of [threeCandidates.pi, threeCandidates.copilot]) {
|
||||
await mkdir(join(root, candidate.path), { recursive: true });
|
||||
await writeFile(join(root, candidate.path, "runtime"), "candidate runtime");
|
||||
candidate.sha256 = sha256DirectoryTree(join(root, candidate.path));
|
||||
}
|
||||
Object.assign(payload, { candidateProviders: threeCandidates });
|
||||
await writeKimiManifest();
|
||||
expect(Object.keys(readRemoteProviderPackManifest(root).payload.candidateProviders!)).toHaveLength(3);
|
||||
Object.assign(payload, { candidateProviders: { ...threeCandidates, unknown: kimi } });
|
||||
await writeKimiManifest();
|
||||
expect(() => readBundledRemoteProviderPackManifest(manifestPath)).toThrow("invalid candidate inventory");
|
||||
Object.assign(payload, { candidateProviders: { unknown: kimi } });
|
||||
await writeKimiManifest();
|
||||
expect(() => readBundledRemoteProviderPackManifest(manifestPath)).toThrow("invalid candidate identity");
|
||||
await rm(releaseMetadata, { recursive: true, force: true });
|
||||
const candidatePath = "provider-assets/pi/linux-x64";
|
||||
await mkdir(join(root, candidatePath), { recursive: true });
|
||||
await writeFile(join(root, candidatePath, "runtime"), "pinned runtime");
|
||||
@@ -3791,10 +3907,10 @@ describe("runtime question fallback", () => {
|
||||
});
|
||||
|
||||
describe("native provider bootstrap environment", () => {
|
||||
it.each(["pi", "cursor", "copilot", "antigravity"] as const)("does not promote ambient %s credentials when bindings are omitted", agent => {
|
||||
it.each(["pi", "cursor", "copilot", "antigravity", "kimi"] as const)("does not promote ambient %s credentials when bindings are omitted", agent => {
|
||||
const provider = { kind: "acpx", agent } as NativeExecutionInput["provider"];
|
||||
const host = { PATH: "/host/bin", HOME: "/host/home", OPENROUTER_API_KEY: "ambient-pi",
|
||||
CURSOR_API_KEY: "ambient-cursor", CURSOR_AUTH_TOKEN: "ambient-cursor-login", COPILOT_GITHUB_TOKEN: "ambient-copilot",
|
||||
CURSOR_API_KEY: "ambient-cursor", CURSOR_AUTH_TOKEN: "ambient-cursor-login", COPILOT_GITHUB_TOKEN: "ambient-copilot", KIMI_API_KEY: "ambient-kimi",
|
||||
GEMINI_API_KEY: "ambient-google", GOOGLE_API_KEY: "ambient-other-google", GEMINI_HOME: "/ambient-google-home",
|
||||
GOOGLE_APPLICATION_CREDENTIALS: "/ambient-google-credential.json", PAPERCLIP_ACPX_ANTIGRAVITY_AUTH_JSON_SECRET: "ambient-google-subscription",
|
||||
PAPERCLIP_ACPX_CREDENTIAL_BINDING: "ambient-forged-binding" };
|
||||
@@ -7754,6 +7870,27 @@ describe("native session cancellation", () => {
|
||||
expect(state.persistActivity).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ companyId: "another-company" }, { runId: "another-run" }, { issueId: "another-issue" },
|
||||
{ scope: "issue" }, { effects: ["cancel_issue"] }, { intentAuditId: null },
|
||||
])("does not fence a failed retry for a foreign or stale Stop receipt %j", async changed => {
|
||||
const intent = { schema: "paperclip.native-cancellation.v1", intentId: "native-cancellation:earlier-stop",
|
||||
...execution.binding, scope: "run", reasonCode: "cancellation_run_only", effects: ["release_run_resources"],
|
||||
dispatchState: "acknowledged", dispatched: true, intentAuditId: "earlier-intent-audit",
|
||||
acknowledgementAuditId: "earlier-ack-audit", ...changed };
|
||||
const coordinator = { runId: execution.binding.runId, assessmentId: null, phase: "retryable_failure" };
|
||||
const persistence = cancellationDb({ status: "failed", coordinator, resultJson: { nativeCancellation: intent } });
|
||||
const before = structuredClone(persistence.getResultJson());
|
||||
state.cancel.mockClear();
|
||||
state.persistActivity.mockClear();
|
||||
await expect(cancelNativeSession(execution.binding.runId, "retry Stop", { db: persistence.db, scope: "run" })).rejects.toThrow();
|
||||
expect(persistence.updates).toEqual([]);
|
||||
expect(coordinator.phase).toBe("retryable_failure");
|
||||
expect(persistence.getResultJson()).toEqual(before);
|
||||
expect(state.cancel).not.toHaveBeenCalled();
|
||||
expect(state.persistActivity).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not mistake a recovered running attempt for the failed retry that reserved Stop", async () => {
|
||||
const id = "11111111-1111-4111-8111-111111111111";
|
||||
const persistence = cancellationDb({ status: "running",
|
||||
@@ -7779,6 +7916,39 @@ describe("native session cancellation", () => {
|
||||
expect(persistence.getResultJsonUpdateCount()).toBe(writes);
|
||||
});
|
||||
|
||||
it("fences a failed retry whose earlier Stop is already acknowledged without changing its receipts", async () => {
|
||||
const intent = { schema: "paperclip.native-cancellation.v1", intentId: "native-cancellation:earlier-stop",
|
||||
...execution.binding, scope: "run", reasonCode: "cancellation_run_only",
|
||||
effects: ["release_run_resources"], dispatchState: "acknowledged", dispatched: true,
|
||||
intentAuditId: "earlier-intent-audit", acknowledgementAuditId: "earlier-ack-audit",
|
||||
acknowledgedAt: "2026-01-01T00:00:00.000Z", decisionId: null, priorCoordinatorDecisionId: null };
|
||||
const coordinator = { runId: execution.binding.runId, assessmentId: null, phase: "retryable_failure",
|
||||
nextAttemptAt: new Date(Date.now() + 30_000) };
|
||||
const persistence = cancellationDb({ status: "failed", coordinator,
|
||||
resultJson: { nativeCancellation: intent, executionCancellation: { state: "requested", requestedAt: "later-stop" } } });
|
||||
const before = structuredClone(persistence.getResultJson());
|
||||
state.cancel.mockClear();
|
||||
state.persistActivity.mockClear();
|
||||
for (let attempt = 0; attempt < 2; attempt++) {
|
||||
await expect(cancelNativeSession(execution.binding.runId, "retry Stop", { db: persistence.db, scope: "run" }))
|
||||
.resolves.toMatchObject({ dispatched: true, auditId: intent.intentAuditId });
|
||||
expect(coordinator).toMatchObject({ phase: "terminal_failure", failureCode: "native_retry_cancelled", nextAttemptAt: null });
|
||||
expect(persistence.getResultJson()).toEqual(before);
|
||||
}
|
||||
expect(persistence.updates.filter(update => update.table === nativeRunFinalizations)).toHaveLength(1);
|
||||
expect(persistence.getResultJsonUpdateCount()).toBe(0);
|
||||
expect(state.cancel).not.toHaveBeenCalled();
|
||||
expect(state.persistActivity).not.toHaveBeenCalled();
|
||||
const stopped = { id: execution.binding.runId, companyId: execution.binding.companyId,
|
||||
nativeIssueId: execution.binding.issueId, status: "cancelled", processPid: null, processGroupId: null,
|
||||
resultJson: { ...persistence.getResultJson(), cancelledByActorType: "user", cancelledByUserId: "operator" } };
|
||||
const lease = { provider: "daytona", providerLeaseId: "unconfirmed-provider-resource", status: "active",
|
||||
releasedAt: null, cleanupStatus: null, metadata: {} };
|
||||
const leaseDb = { select: () => ({ from: () => ({ where: async () => [lease] }) }) } as unknown as Db;
|
||||
expect(isAcknowledgedNativeStop(stopped)).toBe(true);
|
||||
await expect(acknowledgedNativeStopExecutionHasStopped(leaseDb, stopped)).resolves.toBe(false);
|
||||
});
|
||||
|
||||
it("uses the reserved caller UUID and permits same-intent and default retries", async () => {
|
||||
const id = "11111111-1111-4111-8111-111111111111";
|
||||
const persistence = cancellationDb({ resultJson: { startupCancellation: { cancellationRequestId: id } } });
|
||||
@@ -9202,6 +9372,44 @@ describe("native warm session supervision", () => {
|
||||
expect(state.execute).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("claims native execution authority before retiring a superseded warm identity", async () => {
|
||||
const { base, next, run } = fixture("identity-native-claim-order");
|
||||
const updates: Array<{ table: unknown; values: Record<string, unknown> }> = [];
|
||||
let inspected = false;
|
||||
const close = vi.fn(async () => {
|
||||
if (inspected) return;
|
||||
inspected = true;
|
||||
expect(updates.some(update => update.table === nativeRunFinalizations
|
||||
&& update.values.leaseOwner && update.values.controllerBootId)).toBe(true);
|
||||
});
|
||||
state.execute.mockReset().mockImplementationOnce(async options => {
|
||||
options.onSession?.({ close }); return result;
|
||||
}).mockImplementationOnce(async () => result);
|
||||
await run(base);
|
||||
// Normal runtime selection already converted preparation to native and
|
||||
// inserted an unclaimed observed coordinator before calling this facade.
|
||||
await executePaperclipNativeSession({ db: leaseDb(next, {}, {}, updates),
|
||||
execution: next, runnerInstanceId: "runner", useRunnerd: true });
|
||||
expect(close).toHaveBeenCalledOnce();
|
||||
expect(state.execute).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it.each(["cancelled", "foreign-owner"] as const)("does not retire or stage a warm successor after claim denial: %s", async denial => {
|
||||
const { base, next, run } = fixture(`identity-native-claim-denied-${denial}`);
|
||||
const close = vi.fn(async () => undefined);
|
||||
state.execute.mockReset().mockImplementationOnce(async options => {
|
||||
options.onSession?.({ close }); return result;
|
||||
});
|
||||
await run(base);
|
||||
state.stageNativeRunnerWakeAttachments.mockClear();
|
||||
const db = leaseDb(next, denial === "foreign-owner" ? { leaseOwner: "different-controller" } : {},
|
||||
{}, [], {}, denial === "cancelled" ? "cancelled" : "running");
|
||||
await expect(executePaperclipNativeSession({ db, execution: next, runnerInstanceId: "runner", useRunnerd: true })).rejects.toThrow();
|
||||
expect(close).not.toHaveBeenCalled();
|
||||
expect(state.stageNativeRunnerWakeAttachments).not.toHaveBeenCalled();
|
||||
expect(state.execute).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not retire an active turn when a new session identity arrives", async () => {
|
||||
const { base, next, run } = fixture("identity-active");
|
||||
let finish!: () => void;
|
||||
@@ -12183,6 +12391,55 @@ describe("runnerd provider runtime wiring", () => {
|
||||
).toThrow("scope_closed");
|
||||
});
|
||||
|
||||
it.each(["attached", "changed-admission", "changed-template", "changed-template-digest", "changed-preparation"] as const)(
|
||||
"verifies frozen staged admission after normal wake attachment preparation: %s", async mode => {
|
||||
const current = scopedNativeExecution(`staged-attachment-${mode}`);
|
||||
current.task.prompt = "Inspect attachment ref:original";
|
||||
const root = await initializeScopedRunnerRoot(current, "attachment-runner");
|
||||
const sourceIdentity = { runnerInstanceId: "attachment-runner", environmentLeaseId: "attachment-lease",
|
||||
runId: "attachment-source", normalizedSessionId: current.session.normalizedSessionId!, turnId: "source-turn", itemId: "source-item" };
|
||||
const targetIdentity = { ...sourceIdentity, runId: current.binding.runId, turnId: "target-turn", itemId: "target-item" };
|
||||
const controller = new DurablePrpControlPlane({ stateDirectory: join(root, "control-plane"), identity: sourceIdentity,
|
||||
expectedRunnerVersion: "fixture", expectedRunnerDigest: `sha256:${"a".repeat(64)}` });
|
||||
controller.stageRunAttachment(targetIdentity, { provider: { kind: "codex", instructions: "prepared attachment grant" } });
|
||||
const path = join(root, "control-plane/control-plane-state.json");
|
||||
const saved = JSON.parse(await readFile(path, "utf8"));
|
||||
const canonicalTemplateDigest = saved.stagedRunAttachment.templateDigest;
|
||||
if (mode === "changed-template" || mode === "changed-template-digest") {
|
||||
saved.stagedRunAttachment.runAttachTemplate.provider.instructions = "substituted grant";
|
||||
if (mode === "changed-template-digest") saved.stagedRunAttachment.templateDigest = `sha256:${nativeSha256(saved.stagedRunAttachment.runAttachTemplate)}`;
|
||||
await writeFile(path, JSON.stringify(saved));
|
||||
}
|
||||
state.stageNativeRunnerWakeAttachments.mockImplementationOnce(async () => {
|
||||
if (mode === "changed-preparation") current.task.prompt = "Inspect attachment ref:mutated-during-preparation";
|
||||
return { attachments: [{ id: "attachment-original" }], cleanup: async () => undefined } as never;
|
||||
});
|
||||
state.renderNativeRunnerStagedAttachmentPrompt.mockReturnValueOnce("Materialized attachment: .paperclip-inbound/current/original");
|
||||
const admission = structuredClone(current);
|
||||
if (mode === "changed-admission") admission.task.prompt = "Inspect attachment ref:changed";
|
||||
const reached = new Error("verified staged admission reached recheck");
|
||||
const restore = vi.spyOn(runnerCheckpoint, "restoreNativeRunnerCheckpoint").mockResolvedValue(null);
|
||||
const header = vi.spyOn(runnerCheckpoint, "loadNativeRunnerCheckpointHeader").mockResolvedValue("staged header");
|
||||
const verify = vi.spyOn(stagedAttachmentRecovery, "resolveNativeStagedAttachmentRecovery")
|
||||
.mockResolvedValueOnce({ currentExecution: admission, sourceIdentity, targetIdentity,
|
||||
templateDigest: canonicalTemplateDigest } as never)
|
||||
.mockRejectedValueOnce(reached);
|
||||
try {
|
||||
const resume = executePaperclipNativeSession({ db: leaseDb(current), execution: current, useRunnerd: true,
|
||||
runnerInstanceId: "attachment-runner", restartRecovery: { kind: "reattach_remote_runner", runId: current.binding.runId,
|
||||
leaseOwner: "controller", controllerGeneration: 2, providerAttempt: 1, restartKind: "hard", recoveryRequestId: null,
|
||||
remote: { kind: "sandbox", leaseId: "attachment-lease", providerLeaseId: "sandbox", remoteCwd: "/workspace" } } as never,
|
||||
runnerExecutionTarget: { kind: "remote", transport: "sandbox", leaseId: "attachment-lease", remoteCwd: "/workspace",
|
||||
runner: { execute: vi.fn() }, sandboxLeaseAcquisition: { outcome: "resumed", providerLeaseId: "sandbox" } } as never });
|
||||
if (mode === "attached") { await expect(resume).rejects.toBe(reached); expect(verify).toHaveBeenCalledTimes(2); }
|
||||
else await expect(resume).rejects.toThrow(mode === "changed-template" ? "staged_attachment_binding_mismatch"
|
||||
: mode === "changed-preparation" ? "native_attachment_admission_changed" : "staged_attachment_recovery_unproven");
|
||||
expect(state.stageNativeRunnerWakeAttachments).toHaveBeenCalledWith(expect.objectContaining({ binding: expect.objectContaining({
|
||||
companyId: current.binding.companyId, issueId: current.binding.issueId, agentId: current.binding.agentId, runId: current.binding.runId,
|
||||
}) }));
|
||||
} finally { restore.mockRestore(); header.mockRestore(); verify.mockRestore(); await controller.stop(); }
|
||||
});
|
||||
|
||||
it("passes the run checkpoint active turn into restart recovery", async () => {
|
||||
state.createBackend.mockClear();
|
||||
state.createTransport.mockClear();
|
||||
@@ -12213,6 +12470,27 @@ describe("runnerd provider runtime wiring", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it.each(["warm", "per_turn"] as const)("defers a %s retained-owner wait before claiming a provider attempt", async mode => {
|
||||
const fresh = scopedNativeExecution(`retained-owner-${mode}`);
|
||||
fresh.session.lifecyclePolicy = mode === "warm"
|
||||
? { mode: "warm", idleTimeoutMs: 300_000 } : { mode: "per_turn", idleTimeoutMs: null };
|
||||
const deadline = new Date(Date.now() + 60_000);
|
||||
vi.mocked(controllerRecovery.resolveNativeControllerRecovery).mockResolvedValueOnce({ kind: "waiting_for_owner", deadline });
|
||||
const defer = vi.spyOn(controllerRecovery, "deferNativeControllerRecovery").mockResolvedValueOnce();
|
||||
const onExecutionLeaseAcquired = vi.fn();
|
||||
const updates: Array<{ table: unknown; values: Record<string, unknown> }> = [];
|
||||
const db = leaseDb(fresh, {}, {}, updates);
|
||||
await expect(executePaperclipNativeSession({ db, execution: fresh, runnerInstanceId: "waiting-controller",
|
||||
useRunnerd: true, onExecutionLeaseAcquired,
|
||||
runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: "/workspace",
|
||||
environmentId: "retained-environment", leaseId: "retained-lease" },
|
||||
})).rejects.toMatchObject({ name: "NativeControllerRecoveryWaitingError", retryAt: deadline });
|
||||
expect(defer).toHaveBeenCalledWith(db, expect.objectContaining({ ...fresh.binding, retryAt: deadline }));
|
||||
expect(onExecutionLeaseAcquired).not.toHaveBeenCalled();
|
||||
expect(updates.some(update => update.table === nativeRunFinalizations && update.values.attempt !== undefined)).toBe(false);
|
||||
expect(state.createBackend).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects overlapping runs for the same runnerd provider session scope", async () => {
|
||||
const first = {
|
||||
...execution,
|
||||
@@ -12576,7 +12854,7 @@ describe("runnerd provider runtime wiring", () => {
|
||||
} as NativeExecutionInputV1;
|
||||
const identity = { runId: current.binding.runId, normalizedSessionId: current.session.normalizedSessionId!,
|
||||
runnerInstanceId: `admitted-${provider}`, environmentLeaseId: "admitted-lease",
|
||||
turnId: `turn-${current.binding.runId}`, itemId: `item-${current.binding.runId}` };
|
||||
turnId: `turn_attachment_${randomUUID()}`, itemId: `item_attachment_${randomUUID()}` };
|
||||
const root = await initializeScopedRunnerRoot(current, identity.runnerInstanceId);
|
||||
await mkdir(join(root, "control-plane"), { recursive: true });
|
||||
await writeFile(join(root, "control-plane", "control-plane-state.json"), JSON.stringify(durableControlPlaneState(identity)));
|
||||
@@ -12606,9 +12884,56 @@ describe("runnerd provider runtime wiring", () => {
|
||||
runnerArtifact: artifact, adoptExistingRunner: { pid: 123, startedAt: "2026-09-19T10:00:00.000Z" } });
|
||||
expect(state.resolveRemoteRunnerBinary).not.toHaveBeenCalled();
|
||||
expect(execute.mock.calls.some(([request]) => request.args.includes("paperclip-runner-launch") || request.args.includes("--build-metadata"))).toBe(false);
|
||||
const options = state.createTransport.mock.calls.at(-1)![0] as RunnerTransportOptions & { prepareExternalRunnerState: () => Promise<void> };
|
||||
await expect(options.prepareExternalRunnerState()).resolves.toBeUndefined();
|
||||
expect(execute.mock.calls.some(([request]) => request.args.includes("paperclip-runner-launch")
|
||||
|| request.args.includes("paperclip-runner-claim-unstarted-session"))).toBe(false);
|
||||
});
|
||||
|
||||
it.each(["claude", "codex"].flatMap(provider => (["per_turn", "warm"] as const).flatMap(policy => ["canonical", "local-receipt", "local-receipt-missing-terminal", "local-receipt-pending-command", "quarantined", "empty-runner", "empty-runner-quarantined", "nonempty-runner", "wrong-identity", "active", "foreign-scope", "no-stop-proof", "not-suspended", "pending-outbox", "pending-command", "missing-terminal", "ambiguous", "contradictory"].map(mode => [provider, policy, mode] as const))))(
|
||||
it.each(["verified", "no-proof", "dead", "stale"] as const)("reuses only verified idle warm process during actual remote preparation: %s", async mode => {
|
||||
const previous = scopedNativeExecution(`warm-live-preparation-${mode}`);
|
||||
previous.session.lifecyclePolicy = { mode: "warm", idleTimeoutMs: 300_000 };
|
||||
const current = { ...previous, binding: { ...previous.binding, runId: `${previous.binding.runId}-next` } };
|
||||
const identity = { runId: previous.binding.runId, normalizedSessionId: previous.session.normalizedSessionId!,
|
||||
runnerInstanceId: "retained-warm-runner", environmentLeaseId: "retained-lease", turnId: "prior-turn", itemId: "prior-item" };
|
||||
const root = await initializeScopedRunnerRoot(previous, identity.runnerInstanceId);
|
||||
await writeRunnerAuthority(root, identity, "suspended");
|
||||
const projections = vi.spyOn(controllerRecovery, "resolveNativeControllerRecovery").mockResolvedValue({ kind: "no_prior_execution" });
|
||||
onTestFinished(() => projections.mockRestore());
|
||||
const db = leaseDb(current), ordinarySelect = db.select.bind(db);
|
||||
db.select = ((fields?: Record<string, unknown>) => fields?.status === heartbeatRuns.status && fields?.runnerProfileJson === heartbeatRuns.runnerProfileJson
|
||||
? priorNativeRunDb(previous).select(fields as never) : ordinarySelect(fields as never)) as Db["select"];
|
||||
let alive = true;
|
||||
const execute = vi.fn(async (request: { args: string[] }) => ({ exitCode: request.args[2] === "paperclip-runner-recovery-check" && !alive ? 45 : 0,
|
||||
timedOut: false, stderr: "", stdout: request.args[0] === "--build-metadata" ? JSON.stringify({
|
||||
schema: "paperclip-runner/runnerd-build-metadata/v1", binaryName: "paperclip-runnerd",
|
||||
packageName: "@paperclipai/paperclip-runner", binaryContractVersion: 2,
|
||||
durableSessionCapabilities: ["unlimited_runtime", "connection_lease_renewal"], prpTransportModes: ["listen_ws"],
|
||||
}) : request.args[0] === "--version" ? "codex-cli 0.160.0" : request.args[1]?.includes("base64")
|
||||
? Buffer.from(JSON.stringify({ ...identity, lifecycle: "ready", outbox: [], pendingTerminalDelivery: null })).toString("base64")
|
||||
: request.args[1]?.includes("cat --") ? `nonce\n123\n2026-09-19T10:00:00.000Z\n${identity.runnerInstanceId}\nlinux:abcd-1234:100\n` : "" }));
|
||||
const revalidate = vi.fn(async () => undefined);
|
||||
const syncIn = vi.fn(async () => undefined);
|
||||
await createRunnerdBackend({ db, execution: current, runnerInstanceId: identity.runnerInstanceId,
|
||||
runnerIngressAuthorized: true,
|
||||
...(mode === "no-proof" ? {} : { remoteWarmRecovery: { identity, reuseProcess: true, previousExecution: previous,
|
||||
checkpoint: {} as never, remote: { providerLeaseId: "retained-sandbox", remoteCwd: "/workspace" }, revalidate } }),
|
||||
runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: "/workspace", environmentId: "environment",
|
||||
leaseId: identity.environmentLeaseId, providerKey: "daytona", effectiveCapabilities: { runnerWebSocketIngress: true },
|
||||
sandboxLeaseAcquisition: { outcome: "resumed", providerLeaseId: "retained-sandbox" }, runner: { execute, syncIn } } as never });
|
||||
state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!();
|
||||
const options = state.createTransport.mock.calls.at(-1)![0] as RunnerTransportOptions & { prepareExternalRunnerState: () => Promise<void> };
|
||||
if (mode === "dead") alive = false;
|
||||
if (mode === "stale") revalidate.mockRejectedValueOnce(new Error("fixture_warm_owner_changed"));
|
||||
if (mode === "verified") await expect(options.prepareExternalRunnerState()).resolves.toBeUndefined();
|
||||
else await expect(options.prepareExternalRunnerState()).rejects.toThrow(mode === "no-proof" ? "resumed_not_suspended"
|
||||
: mode === "dead" ? "runner_remote_process_identity_unavailable" : "fixture_warm_owner_changed");
|
||||
expect(syncIn).not.toHaveBeenCalled();
|
||||
expect(execute.mock.calls.some(([request]) => request.args.includes("paperclip-runner-launch")
|
||||
|| request.args.includes("paperclip-runner-claim-unstarted-session"))).toBe(false);
|
||||
});
|
||||
|
||||
it.each(["claude", "codex"].flatMap(provider => (["per_turn", "warm"] as const).flatMap(policy => ["canonical", "resumed", "resumed-timeout", "local-receipt", "local-receipt-missing-terminal", "local-receipt-pending-command", "quarantined", "empty-runner", "empty-runner-quarantined", "nonempty-runner", "wrong-identity", "active", "foreign-scope", "no-stop-proof", "not-suspended", "pending-outbox", "pending-command", "missing-terminal", "ambiguous", "contradictory"].filter(mode => policy === "per_turn" || !mode.startsWith("resumed")).map(mode => [provider, policy, mode] as const))))(
|
||||
"recovers a blocked %s %s plugin checkpoint on a replacement lease: %s", async (provider, policy, mode) => {
|
||||
const { permissionPolicy: _permissionPolicy, ...claudeProfile } = QUALIFIED_ACPX_PROFILES.claude;
|
||||
const first = { ...scopedNativeExecution(`plugin-${provider}-${policy}-${mode}`),
|
||||
@@ -12656,8 +12981,13 @@ describe("runnerd provider runtime wiring", () => {
|
||||
if (mode === "no-stop-proof") throw new Error("Plugin Runner retained session is not authorized");
|
||||
return "old-task";
|
||||
});
|
||||
let timedOut = false;
|
||||
const task = vi.fn(async (operation: any) => {
|
||||
environmentTaskOperationSchema.parse(operation);
|
||||
if (mode === "resumed-timeout" && operation.kind === "read_state" && !timedOut) {
|
||||
timedOut = true;
|
||||
throw new Error("fixture_remote_state_timeout");
|
||||
}
|
||||
if (operation.kind === "read_state") return { kind: "state", taskId: "replacement-task", state: {
|
||||
...durableRunnerState(identity, mode === "not-suspended" ? "ready" : "suspended"),
|
||||
outbox: mode === "pending-outbox" ? [{}] : [], pendingTerminalDelivery: null,
|
||||
@@ -12676,7 +13006,7 @@ describe("runnerd provider runtime wiring", () => {
|
||||
managedAiCredentialHome: home, managedAiCredentialIdentity: "revision", managedAiAccountIdentity: "account",
|
||||
runnerEnvironment: { ANTHROPIC_API_KEY: "synthetic-fixture" },
|
||||
runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: ".", leaseId: "replacement-lease", runnerTask: task,
|
||||
sandboxLeaseAcquisition: { outcome: "replacement", providerLeaseId: "replacement-task" } } as never };
|
||||
sandboxLeaseAcquisition: { outcome: mode.startsWith("resumed") ? "resumed" : "replacement", providerLeaseId: "replacement-task" } } as never };
|
||||
if (throughExecutor) {
|
||||
const validated = validatePrpStructuredRunResult({ schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", summary: "done",
|
||||
completionClaim: { contractRevision: next.completionContract.contract.revision, objectiveSatisfied: true, criteria: [], remainingWork: [] },
|
||||
@@ -12686,12 +13016,19 @@ describe("runnerd provider runtime wiring", () => {
|
||||
normalizedSessionId: next.session.normalizedSessionId, providerSessionId: "original-provider-history", driverKind: next.session.driverKind,
|
||||
driverVersion: "1", nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null });
|
||||
}
|
||||
if (mode === "resumed-timeout") {
|
||||
await expect(createRunnerdBackend(launch)).rejects.toThrow("fixture_remote_state_timeout");
|
||||
expect(existsSync(join(scoped, "plugin-runner-state.json"))).toBe(false);
|
||||
expect(await readFile(join(scoped, "control-plane/control-plane-state.json"), "utf8")).toBe(journal);
|
||||
}
|
||||
const backend = throughExecutor ? executePaperclipNativeSession({ ...launch, useRunnerd: true }) : createRunnerdBackend(launch);
|
||||
if (["canonical", "local-receipt", "quarantined", "empty-runner", "empty-runner-quarantined"].includes(mode)) {
|
||||
if (["canonical", "resumed", "resumed-timeout", "local-receipt", "quarantined", "empty-runner", "empty-runner-quarantined"].includes(mode)) {
|
||||
await expect(backend).resolves.toBeDefined();
|
||||
expect(state.createBackend).toHaveBeenCalledOnce();
|
||||
expect(owner).toHaveBeenCalledWith(expect.objectContaining({ resumeFrom: "old-task", priorIdentity: identity }));
|
||||
expect(resume).toHaveBeenCalledWith(db, expect.objectContaining({ leaseId: "replacement-lease", priorIdentity: identity, inspectCheckpoint: true }));
|
||||
expect(owner).toHaveBeenCalledWith(expect.objectContaining({ priorIdentity: identity,
|
||||
...(!mode.startsWith("resumed") ? { resumeFrom: "old-task" } : {}) }));
|
||||
if (!mode.startsWith("resumed")) expect(resume).toHaveBeenCalledWith(db, expect.objectContaining({ leaseId: "replacement-lease", priorIdentity: identity, inspectCheckpoint: true }));
|
||||
else expect(task).toHaveBeenCalledWith({ kind: "read_state", identity });
|
||||
expect(JSON.parse(await readFile(join(scoped, "plugin-runner-state.json"), "utf8"))).toMatchObject({ ...identity, lifecycle: "suspended" });
|
||||
expect(await readFile(join(scoped, "control-plane/control-plane-state.json"), "utf8")).toBe(journal);
|
||||
} else {
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -5,6 +5,7 @@ import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
|
||||
import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js";
|
||||
import { findNativeSessionResumeRun } from "./native-session-resume.js";
|
||||
import { resolveNativeControllerRecovery } from "./native-controller-recovery.js";
|
||||
|
||||
const externalUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_TEST_DATABASE_URL");
|
||||
const supported = Boolean(externalUrl) || (await getEmbeddedPostgresTestSupport()).supported;
|
||||
@@ -58,7 +59,7 @@ const supported = Boolean(externalUrl) || (await getEmbeddedPostgresTestSupport(
|
||||
const saved = source();
|
||||
await db.insert(heartbeatRuns).values([saved, run(1), run(2)]);
|
||||
const result = await select();
|
||||
expect(result).toEqual({ ...saved, runnerProfileJson: { nativeExecutionInput, sessionCheckpoint: checkpoint, nativeToolContractFingerprint: "fingerprint" },
|
||||
expect(result).toEqual({ ...saved, runnerProfileJson: { nativeExecutionInput, sessionCheckpoint: checkpoint, nativeToolContractFingerprint: "fingerprint", nativeControllerRecovery: null },
|
||||
contextSnapshot: { paperclipWake, paperclipIssue } });
|
||||
const stored = await db.select({ profile: heartbeatRuns.runnerProfileJson, context: heartbeatRuns.contextSnapshot }).from(heartbeatRuns).where(eq(heartbeatRuns.id, saved.id));
|
||||
expect(stored[0].profile).toEqual(profile);
|
||||
@@ -80,6 +81,16 @@ const supported = Boolean(externalUrl) || (await getEmbeddedPostgresTestSupport(
|
||||
}
|
||||
});
|
||||
|
||||
it("does not call missing or newer incompatible recovery material a fresh session", async () => {
|
||||
const saved = source();
|
||||
await db.insert(heartbeatRuns).values([saved, run(30, { id: currentRunId, status: "running" })]);
|
||||
const { beforeCreatedAt: _before, ...binding } = input;
|
||||
expect(await resolveNativeControllerRecovery(db, binding)).toEqual({ kind: "missing_evidence", reason: "snapshot_binding" });
|
||||
// The newest progress barrier remains authoritative even without a snapshot.
|
||||
await db.insert(heartbeatRuns).values(run(10, { runtimeMode: "legacy" }));
|
||||
expect(await resolveNativeControllerRecovery(db, binding)).toEqual({ kind: "missing_evidence", reason: "newer_progress_barrier" });
|
||||
});
|
||||
|
||||
it("skips only unused SQL-null, JSON-null, scalar, array and checkpoint-null profiles", async () => {
|
||||
const saved = source(), newer = run(1, { runnerProfileJson: null });
|
||||
await db.insert(heartbeatRuns).values([saved, newer]);
|
||||
|
||||
@@ -223,6 +223,8 @@ export async function findNativeSessionResumeRun(
|
||||
normalizedSessionId: string;
|
||||
currentRunId: string;
|
||||
beforeCreatedAt: Date;
|
||||
/** Recovery discovery must retain an ineligible newest progress barrier. */
|
||||
includeProgressBarrier?: boolean;
|
||||
},
|
||||
) {
|
||||
if (!isNativeSessionId(input.normalizedSessionId)) return null;
|
||||
@@ -287,10 +289,12 @@ export async function findNativeSessionResumeRun(
|
||||
nativeToolContractFingerprint: unknown;
|
||||
nativeExecutionInput: unknown;
|
||||
sessionCheckpoint: unknown;
|
||||
nativeControllerRecovery: unknown;
|
||||
}>`jsonb_build_object(
|
||||
'nativeToolContractFingerprint', ${heartbeatRuns.runnerProfileJson}->'nativeToolContractFingerprint',
|
||||
'nativeExecutionInput', ${heartbeatRuns.runnerProfileJson}->'nativeExecutionInput',
|
||||
'sessionCheckpoint', ${heartbeatRuns.runnerProfileJson}->'sessionCheckpoint'
|
||||
'sessionCheckpoint', ${heartbeatRuns.runnerProfileJson}->'sessionCheckpoint',
|
||||
'nativeControllerRecovery', ${heartbeatRuns.runnerProfileJson}->'nativeControllerRecovery'
|
||||
)`,
|
||||
contextSnapshot: sql<{
|
||||
paperclipWake: unknown;
|
||||
@@ -306,14 +310,16 @@ export async function findNativeSessionResumeRun(
|
||||
.from(heartbeatRuns)
|
||||
.innerJoin(barrier, eq(heartbeatRuns.id, barrier.id))
|
||||
.where(and(
|
||||
eq(barrier.runtimeMode, "native"),
|
||||
inArray(barrier.status, terminalStatuses),
|
||||
eq(barrier.hasCheckpoint, true),
|
||||
...(input.includeProgressBarrier ? [] : [
|
||||
eq(barrier.runtimeMode, "native"),
|
||||
inArray(barrier.status, terminalStatuses),
|
||||
eq(barrier.hasCheckpoint, true),
|
||||
sql`date_trunc('milliseconds', ${barrier.createdAt}) <= ${input.beforeCreatedAt.toISOString()}::timestamptz`,
|
||||
]),
|
||||
sql`${heartbeatRuns.companyId}::text = ${input.companyId}`,
|
||||
sql`${heartbeatRuns.agentId}::text = ${input.agentId}`,
|
||||
sql`${heartbeatRuns.nativeIssueId}::text = ${input.issueId}`,
|
||||
sql`${heartbeatRuns.nativeSessionId}::text = ${input.normalizedSessionId}`,
|
||||
sql`date_trunc('milliseconds', ${barrier.createdAt}) <= ${input.beforeCreatedAt.toISOString()}::timestamptz`,
|
||||
));
|
||||
return run ?? null;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,187 @@
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { DurablePrpControlPlane, type DurablePrpCheckpoint } from "../../vendor/paperclip-runner/index.js";
|
||||
import { createLocalEncryptedCipher } from "../../secrets/local-encrypted-provider.js";
|
||||
import { buildNativeExecutionInput } from "./native-execution-input.js";
|
||||
import { nativeToolContractFingerprintForTarget } from "./native-session-resume.js";
|
||||
import { nativeRuntimeContextFixture } from "./runtime-context.test-fixture.js";
|
||||
import { resolveNativeStagedAttachmentRecovery } from "./native-staged-attachment-recovery.js";
|
||||
|
||||
const companyId = "10000000-0000-4000-8000-000000000001";
|
||||
const issueId = "20000000-0000-4000-8000-000000000002";
|
||||
const agentId = "30000000-0000-4000-8000-000000000003";
|
||||
const sourceRunId = "40000000-0000-4000-8000-000000000004";
|
||||
const currentRunId = "50000000-0000-4000-8000-000000000005";
|
||||
const normalizedSessionId = "60000000-0000-4000-8000-000000000006";
|
||||
const runnerInstanceId = "70000000-0000-4000-8000-000000000007";
|
||||
const sourceIdentity = { runId: sourceRunId, normalizedSessionId, runnerInstanceId,
|
||||
environmentLeaseId: "environment-1", turnId: "opaque-source-turn", itemId: "opaque-source-item" };
|
||||
const targetIdentity = { ...sourceIdentity, runId: currentRunId, turnId: "opaque-target-turn", itemId: "opaque-target-item" };
|
||||
const roots: string[] = [];
|
||||
afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); });
|
||||
|
||||
function execution(runId: string) {
|
||||
return buildNativeExecutionInput({ companyId, runId, agentId, normalizedSessionId,
|
||||
issue: { id: issueId, identifier: "TEST-1", title: "Continue", description: null, workMode: "standard" },
|
||||
taskPrompt: "Continue normally", provider: "codex", workspace: { id: "workspace-1", cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null },
|
||||
completionContract: { id: "80000000-0000-4000-8000-000000000008", sha256: `sha256:${"a".repeat(64)}`,
|
||||
schemaVersion: "paperclip.run-result.v1", contract: { revision: "1", objective: "Continue", criteria: [{ id: "done", requirement: "Answer" }] } },
|
||||
runtimeContext: nativeRuntimeContextFixture(),
|
||||
});
|
||||
}
|
||||
|
||||
async function fixture() {
|
||||
const root = await mkdtemp(join(tmpdir(), "native-staged-proof-")); roots.push(root);
|
||||
const core = new DurablePrpControlPlane({ stateDirectory: root, identity: sourceIdentity,
|
||||
expectedRunnerVersion: "0.0.0", expectedRunnerDigest: `sha256:${"f".repeat(64)}` });
|
||||
let saved!: DurablePrpCheckpoint;
|
||||
core.setCheckpointPersistence(async checkpoint => { saved = checkpoint; });
|
||||
core.persistRunAttachTemplate({ provider: { provider: "codex", command: "codex" }, authorizedTools: {} });
|
||||
await core.flushCheckpoint();
|
||||
const sourceHeader = saved.records.get("header")!;
|
||||
core.stageRunAttachment(targetIdentity, { provider: { provider: "codex", command: "codex" }, authorizedTools: {} });
|
||||
await core.flushCheckpoint();
|
||||
const header = saved.records.get("header")!;
|
||||
const base = { companyId, agentId, issueId, nativeIssueId: issueId, runnerInstanceId,
|
||||
nativeSessionId: normalizedSessionId, runtimeMode: "native", status: "running", finishedAt: null as Date | null };
|
||||
const currentRun = { ...base, id: currentRunId, runnerProfileJson: { nativeExecutionInput: execution(currentRunId) } };
|
||||
const sourceRun = { ...base, id: sourceRunId, status: "succeeded", finishedAt: new Date(), runnerProfileJson: {
|
||||
nativeExecutionInput: execution(sourceRunId), nativeToolContractFingerprint: nativeToolContractFingerprintForTarget("remote", "codex"),
|
||||
sessionCheckpoint: { sessionId: "provider-thread-1", providerSessionId: "provider-thread-1", identity: {
|
||||
companyId, issueId, agentId, runId: sourceRunId, sessionId: normalizedSessionId },
|
||||
},
|
||||
} };
|
||||
const source = { run: sourceRun, coordinator: { runId: sourceRunId, companyId, issueId,
|
||||
phase: "committed", resultId: "result-1", leaseOwner: null, leaseExpiresAt: null }, resultId: "result-1", resultStatus: "accepted" };
|
||||
function database(sourceRows = [source], contents = sourceHeader, envelope = {}) {
|
||||
const material = createLocalEncryptedCipher("write").encrypt(JSON.stringify({ companyId, issueId, runId: sourceRunId, part: "header", contents, ...envelope }));
|
||||
const batches: unknown[][] = [sourceRows, [{ material }]];
|
||||
const select = vi.fn(() => {
|
||||
const chain = { from: () => chain, innerJoin: () => chain, where: () => chain,
|
||||
limit: vi.fn(async () => batches.shift() ?? []) };
|
||||
return chain;
|
||||
});
|
||||
return { db: { select } as unknown as Pick<Db, "select">, select };
|
||||
}
|
||||
return { header, sourceHeader, source, sourceRun, currentRun, database };
|
||||
}
|
||||
|
||||
describe("private staged attachment restart evidence", () => {
|
||||
it("proves both frozen run bindings before the target has a session snapshot", async () => {
|
||||
const f = await fixture(); const { db, select } = f.database();
|
||||
const matchesExecution = vi.fn(() => true);
|
||||
const result = await resolveNativeStagedAttachmentRecovery({ db, header: f.header, currentRun: f.currentRun,
|
||||
expectedTarget: targetIdentity, matchesExecution });
|
||||
expect(result).toMatchObject({ sourceIdentity, targetIdentity, sourceRun: { id: sourceRunId }, hasTargetCheckpoint: false, targetCheckpoint: null,
|
||||
protocolIdentity: sourceIdentity, phase: "source_bootstrap", receipt: null });
|
||||
expect(matchesExecution).toHaveBeenCalledOnce();
|
||||
expect(select).toHaveBeenCalledTimes(2); // one source point lookup and one encrypted header point lookup
|
||||
expect(f.currentRun.runnerProfileJson).not.toHaveProperty("sessionCheckpoint");
|
||||
});
|
||||
|
||||
it("retains staged evidence when the target runtime has already published its snapshot", async () => {
|
||||
const f = await fixture();
|
||||
const snapshot = structuredClone(f.sourceRun.runnerProfileJson.sessionCheckpoint);
|
||||
snapshot.identity.runId = currentRunId;
|
||||
Object.assign(snapshot, { cursor: "target-cursor", activeTurnId: "target-current-turn", pendingRuntimeRequests: [{ requestId: "target-request" }] });
|
||||
Object.assign(f.currentRun.runnerProfileJson, { sessionCheckpoint: snapshot });
|
||||
const result = await resolveNativeStagedAttachmentRecovery({ ...f.database(), header: f.header, currentRun: f.currentRun });
|
||||
expect(result).toMatchObject({ sourceIdentity, targetIdentity, checkpoint: { identity: { runId: currentRunId } },
|
||||
hasTargetCheckpoint: true, targetCheckpoint: snapshot });
|
||||
expect(f.currentRun.runnerProfileJson).toHaveProperty("sessionCheckpoint", snapshot);
|
||||
});
|
||||
|
||||
it("returns the codec's queued phase without assuming the Runner has activated the target", async () => {
|
||||
const f = await fixture(); const header = JSON.parse(f.header);
|
||||
// Header inspection deliberately does not certify separate command records.
|
||||
// Full restoration/authentication owns that later gate; no RPC runs here.
|
||||
header.state.stagedRunAttachment.commandId = "staged-attach-command";
|
||||
const result = await resolveNativeStagedAttachmentRecovery({ ...f.database(), header: JSON.stringify(header), currentRun: f.currentRun });
|
||||
expect(result).toMatchObject({ commandId: "staged-attach-command", protocolIdentity: sourceIdentity,
|
||||
phase: "attachment_queued", receipt: null, targetIdentity });
|
||||
});
|
||||
|
||||
it.each(["runId", "companyId", "issueId", "agentId", "sessionId", "providerSessionId"])(
|
||||
"rejects a present target checkpoint with changed %s instead of replaying the source", async key => {
|
||||
const f = await fixture(); const snapshot = structuredClone(f.sourceRun.runnerProfileJson.sessionCheckpoint);
|
||||
snapshot.identity.runId = currentRunId;
|
||||
if (key === "providerSessionId") snapshot.providerSessionId = "foreign";
|
||||
else Object.assign(snapshot.identity, { [key]: "foreign" });
|
||||
Object.assign(f.currentRun.runnerProfileJson, { sessionCheckpoint: snapshot });
|
||||
expect(await resolveNativeStagedAttachmentRecovery({ ...f.database(), header: f.header, currentRun: f.currentRun })).toBeNull();
|
||||
});
|
||||
|
||||
it.each(["runId", "runnerInstanceId", "environmentLeaseId", "normalizedSessionId", "turnId", "itemId"] as const)(
|
||||
"rejects a different expected target %s before querying the source", async key => {
|
||||
const f = await fixture(); const { db, select } = f.database();
|
||||
expect(await resolveNativeStagedAttachmentRecovery({ db, header: f.header, currentRun: f.currentRun,
|
||||
expectedTarget: { ...targetIdentity, [key]: "foreign" } })).toBeNull();
|
||||
expect(select).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(["agentId", "companyId", "nativeIssueId", "nativeSessionId", "runnerInstanceId"] as const)(
|
||||
"rejects mixed source DB %s", async key => {
|
||||
const f = await fixture(); Object.assign(f.source.run, { [key]: "foreign" });
|
||||
expect(await resolveNativeStagedAttachmentRecovery({ ...f.database(), header: f.header, currentRun: f.currentRun })).toBeNull();
|
||||
});
|
||||
|
||||
it.each(["binding", "schema", "runner", "session"])("rejects current frozen evidence mismatch: %s", async kind => {
|
||||
const f = await fixture(); const { db, select } = f.database();
|
||||
if (kind === "binding") f.currentRun.runnerProfileJson.nativeExecutionInput.binding.agentId = sourceRunId;
|
||||
if (kind === "schema") Object.assign(f.currentRun.runnerProfileJson.nativeExecutionInput, { schema: "unsupported" });
|
||||
if (kind === "runner") f.currentRun.runnerInstanceId = sourceRunId;
|
||||
if (kind === "session") f.currentRun.nativeSessionId = sourceRunId;
|
||||
expect(await resolveNativeStagedAttachmentRecovery({ db, header: f.header, currentRun: f.currentRun })).toBeNull();
|
||||
expect(select).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(["phase", "leaseOwner", "resultStatus", "status", "finishedAt", "frozenRun", "workspace", "provider", "sourceTuple", "artifactVersion", "artifactDigest"])(
|
||||
"rejects unsupported or stale source evidence: %s", async kind => {
|
||||
const f = await fixture(); let sourceHeader = f.sourceHeader;
|
||||
if (kind === "phase") f.source.coordinator.phase = "observed";
|
||||
if (kind === "leaseOwner") Object.assign(f.source.coordinator, { leaseOwner: "another-owner", leaseExpiresAt: new Date(0) });
|
||||
if (kind === "resultStatus") f.source.resultStatus = "rejected";
|
||||
if (kind === "status") f.source.run.status = "running";
|
||||
if (kind === "finishedAt") Object.assign(f.source.run, { finishedAt: null });
|
||||
if (kind === "frozenRun") f.source.run.runnerProfileJson.nativeExecutionInput.binding.runId = currentRunId;
|
||||
if (kind === "workspace") f.source.run.runnerProfileJson.nativeExecutionInput.binding.executionWorkspaceId = "foreign";
|
||||
if (kind === "provider") Object.assign(f.source.run.runnerProfileJson.nativeExecutionInput.provider, { model: "another-model" });
|
||||
if (kind === "sourceTuple") { const h = JSON.parse(sourceHeader); h.state.identity.turnId = "foreign"; sourceHeader = JSON.stringify(h); }
|
||||
if (kind === "artifactVersion") { const h = JSON.parse(sourceHeader); h.runnerVersion = "other-version"; sourceHeader = JSON.stringify(h); }
|
||||
if (kind === "artifactDigest") { const h = JSON.parse(sourceHeader); h.runnerDigest = `sha256:${"d".repeat(64)}`; sourceHeader = JSON.stringify(h); }
|
||||
expect(await resolveNativeStagedAttachmentRecovery({ ...f.database([f.source], sourceHeader), header: f.header, currentRun: f.currentRun })).toBeNull();
|
||||
});
|
||||
|
||||
it("keeps private executor scope/config verification mandatory when supplied", async () => {
|
||||
const f = await fixture();
|
||||
expect(await resolveNativeStagedAttachmentRecovery({ ...f.database(), header: f.header, currentRun: f.currentRun,
|
||||
matchesExecution: () => false })).toBeNull();
|
||||
});
|
||||
|
||||
it.each(["companyId", "issueId", "runId", "part"])("rejects encrypted source header with another %s", async key => {
|
||||
const f = await fixture();
|
||||
expect(await resolveNativeStagedAttachmentRecovery({ ...f.database([f.source], f.sourceHeader, { [key]: "foreign" }),
|
||||
header: f.header, currentRun: f.currentRun })).toBeNull();
|
||||
});
|
||||
|
||||
it("does not accept a source lacking the current tool contract or provider identity", async () => {
|
||||
const f = await fixture();
|
||||
f.source.run.runnerProfileJson.nativeToolContractFingerprint = "old-contract";
|
||||
expect(await resolveNativeStagedAttachmentRecovery({ ...f.database(), header: f.header, currentRun: f.currentRun })).toBeNull();
|
||||
f.source.run.runnerProfileJson.nativeToolContractFingerprint = nativeToolContractFingerprintForTarget("remote", "codex");
|
||||
f.source.run.runnerProfileJson.sessionCheckpoint.providerSessionId = "";
|
||||
expect(await resolveNativeStagedAttachmentRecovery({ ...f.database(), header: f.header, currentRun: f.currentRun })).toBeNull();
|
||||
});
|
||||
|
||||
it("rejects corrupted or ordinary headers instead of promoting them to staged evidence", async () => {
|
||||
const f = await fixture();
|
||||
for (const header of [f.sourceHeader, "{}", "not-json"]) {
|
||||
const { db, select } = f.database();
|
||||
expect(await resolveNativeStagedAttachmentRecovery({ db, header, currentRun: f.currentRun })).toBeNull();
|
||||
expect(select).not.toHaveBeenCalled();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,124 @@
|
||||
import { isDeepStrictEqual } from "node:util";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { heartbeatRuns, nativeRunFinalizations, nativeRunResults, type Db } from "@paperclipai/db";
|
||||
import {
|
||||
inspectDurablePrpCheckpointHeader,
|
||||
inspectStagedRunAttachmentHeader,
|
||||
parseNativeExecutionInput,
|
||||
type DurableRecoveryIdentity,
|
||||
type NativeExecutionInput,
|
||||
type PersistedNativeSession,
|
||||
type StagedRunAttachment,
|
||||
} from "../../vendor/paperclip-runner/index.js";
|
||||
import { loadNativeRunnerCheckpointHeader } from "./native-runner-checkpoint.js";
|
||||
import { rebindNativeSessionCheckpoint } from "./native-session-resume.js";
|
||||
|
||||
type FrozenRun = Pick<typeof heartbeatRuns.$inferSelect,
|
||||
"id" | "companyId" | "agentId" | "nativeIssueId" | "nativeSessionId" | "runnerInstanceId" |
|
||||
"runtimeMode" | "status" | "runnerProfileJson" | "finishedAt">;
|
||||
|
||||
type StagedHeaderProof = NonNullable<ReturnType<typeof inspectStagedRunAttachmentHeader>>;
|
||||
export type NativeStagedAttachmentRecovery = Omit<StagedRunAttachment, "runAttachTemplate"> &
|
||||
Pick<StagedHeaderProof, "protocolIdentity" | "phase" | "receipt"> & {
|
||||
sourceRun: FrozenRun;
|
||||
sourceExecution: NativeExecutionInput;
|
||||
currentExecution: NativeExecutionInput;
|
||||
/** In-memory input only. It is not a published/resumable target snapshot. */
|
||||
checkpoint: PersistedNativeSession;
|
||||
/** A published target snapshot takes precedence over the source rebound. */
|
||||
hasTargetCheckpoint: boolean;
|
||||
targetCheckpoint: PersistedNativeSession | null;
|
||||
runnerVersion: string;
|
||||
runnerDigest: string;
|
||||
};
|
||||
|
||||
function record(value: unknown): Record<string, unknown> {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value)
|
||||
? value as Record<string, unknown> : {};
|
||||
}
|
||||
|
||||
function boundTargetCheckpoint(run: FrozenRun, source: PersistedNativeSession): PersistedNativeSession | null {
|
||||
const snapshot = record(run.runnerProfileJson?.sessionCheckpoint), identity = record(snapshot.identity);
|
||||
if (typeof snapshot.sessionId !== "string" || !snapshot.sessionId || identity.runId !== run.id ||
|
||||
identity.companyId !== run.companyId || identity.issueId !== run.nativeIssueId || identity.agentId !== run.agentId ||
|
||||
identity.sessionId !== run.nativeSessionId) return null;
|
||||
// A target checkpoint may advance cursor/goal/tool progress, but cannot change
|
||||
// the provider session retained by this attachment. Do not rebound it again.
|
||||
if ((source.providerSessionId != null && source.providerSessionId !== snapshot.providerSessionId) ||
|
||||
(source.providerIdentity != null && !isDeepStrictEqual(source.providerIdentity, snapshot.providerIdentity))) return null;
|
||||
return structuredClone(snapshot) as unknown as PersistedNativeSession;
|
||||
}
|
||||
|
||||
function frozenExecution(run: FrozenRun): NativeExecutionInput | null {
|
||||
try {
|
||||
const execution = parseNativeExecutionInput(run.runnerProfileJson?.nativeExecutionInput);
|
||||
return run.runtimeMode === "native" && execution.binding.runId === run.id &&
|
||||
execution.binding.companyId === run.companyId && execution.binding.agentId === run.agentId &&
|
||||
execution.binding.issueId === run.nativeIssueId && execution.session.normalizedSessionId === run.nativeSessionId
|
||||
? execution : null;
|
||||
} catch { return null; }
|
||||
}
|
||||
|
||||
function matchesRun(identity: DurableRecoveryIdentity, run: FrozenRun): boolean {
|
||||
return identity.runId === run.id && identity.runnerInstanceId === run.runnerInstanceId &&
|
||||
identity.normalizedSessionId === run.nativeSessionId;
|
||||
}
|
||||
|
||||
/** Staging proves which prior attachment may be inspected, not that it is alive
|
||||
* or that work may be dispatched. Classification reads only two indexed point
|
||||
* lookups: the named source/result and its encrypted header. The executor must
|
||||
* re-read this proof with its private scope/config matcher before any RPC, then
|
||||
* independently prove the physical Runner and authenticate the target tuple. */
|
||||
export async function resolveNativeStagedAttachmentRecovery(input: {
|
||||
db: Pick<Db, "select">;
|
||||
currentRun: FrozenRun;
|
||||
header: string;
|
||||
expectedTarget?: DurableRecoveryIdentity;
|
||||
matchesExecution?: (previous: NativeExecutionInput, current: NativeExecutionInput) => boolean;
|
||||
}): Promise<NativeStagedAttachmentRecovery | null> {
|
||||
const proof = inspectStagedRunAttachmentHeader(input.header);
|
||||
const current = frozenExecution(input.currentRun);
|
||||
if (!proof || !current || current.provider.kind !== "codex" || !["running", "failed"].includes(input.currentRun.status) ||
|
||||
!matchesRun(proof.attachment.targetIdentity, input.currentRun) ||
|
||||
(input.expectedTarget && !isDeepStrictEqual(proof.attachment.targetIdentity, input.expectedTarget))) return null;
|
||||
const { attachment } = proof;
|
||||
const [source] = await input.db.select({ run: heartbeatRuns, coordinator: nativeRunFinalizations,
|
||||
resultId: nativeRunResults.id, resultStatus: nativeRunResults.schemaStatus }).from(heartbeatRuns)
|
||||
.innerJoin(nativeRunFinalizations, and(eq(nativeRunFinalizations.runId, heartbeatRuns.id),
|
||||
eq(nativeRunFinalizations.companyId, heartbeatRuns.companyId), eq(nativeRunFinalizations.issueId, heartbeatRuns.nativeIssueId)))
|
||||
.innerJoin(nativeRunResults, and(eq(nativeRunResults.id, nativeRunFinalizations.resultId),
|
||||
eq(nativeRunResults.runId, heartbeatRuns.id), eq(nativeRunResults.companyId, heartbeatRuns.companyId),
|
||||
eq(nativeRunResults.issueId, heartbeatRuns.nativeIssueId)))
|
||||
.where(and(eq(heartbeatRuns.id, attachment.sourceIdentity.runId), eq(heartbeatRuns.companyId, input.currentRun.companyId),
|
||||
eq(heartbeatRuns.agentId, input.currentRun.agentId), eq(heartbeatRuns.nativeIssueId, current.binding.issueId)))
|
||||
.limit(1);
|
||||
if (!source || source.run.companyId !== current.binding.companyId || source.run.agentId !== current.binding.agentId ||
|
||||
source.run.nativeIssueId !== current.binding.issueId || !matchesRun(attachment.sourceIdentity, source.run) ||
|
||||
source.run.status !== "succeeded" || source.run.finishedAt === null || source.coordinator.phase !== "committed" ||
|
||||
source.coordinator.companyId !== current.binding.companyId || source.coordinator.issueId !== current.binding.issueId ||
|
||||
source.coordinator.runId !== source.run.id || !source.coordinator.resultId ||
|
||||
source.coordinator.resultId !== source.resultId || source.resultStatus !== "accepted" ||
|
||||
source.coordinator.leaseOwner !== null || source.coordinator.leaseExpiresAt !== null) return null;
|
||||
const previous = frozenExecution(source.run);
|
||||
if (!previous || (input.matchesExecution && !input.matchesExecution(previous, current))) return null;
|
||||
// Reuse the existing provider, tool-contract, workspace, runtime-context and
|
||||
// normalized-session compatibility owner; no second set of digest formulas.
|
||||
const checkpoint = rebindNativeSessionCheckpoint({ previousRun: source.run, currentExecution: current, executionTargetKind: "remote" });
|
||||
if (!checkpoint || !(typeof checkpoint.providerSessionId === "string" && checkpoint.providerSessionId.length > 0 ||
|
||||
checkpoint.providerIdentity && Object.keys(checkpoint.providerIdentity).length > 0)) return null;
|
||||
const hasTargetCheckpoint = input.currentRun.runnerProfileJson?.sessionCheckpoint != null;
|
||||
const targetCheckpoint = hasTargetCheckpoint ? boundTargetCheckpoint(input.currentRun, checkpoint) : null;
|
||||
if (hasTargetCheckpoint && !targetCheckpoint) return null;
|
||||
const sourceHeader = await loadNativeRunnerCheckpointHeader(input.db, {
|
||||
companyId: source.run.companyId, issueId: current.binding.issueId, runId: source.run.id,
|
||||
});
|
||||
if (!sourceHeader || !isDeepStrictEqual(inspectDurablePrpCheckpointHeader(sourceHeader), attachment.sourceIdentity)) return null;
|
||||
// The codec above has validated this bounded header's artifact fields. A
|
||||
// staged target cannot silently select different bytes from its source.
|
||||
const sourceArtifact = JSON.parse(sourceHeader) as { runnerVersion: string; runnerDigest: string };
|
||||
if (sourceArtifact.runnerVersion !== proof.runnerVersion || sourceArtifact.runnerDigest !== proof.runnerDigest) return null;
|
||||
return { ...attachment, sourceRun: source.run, sourceExecution: previous, currentExecution: current,
|
||||
checkpoint, hasTargetCheckpoint, targetCheckpoint,
|
||||
protocolIdentity: proof.protocolIdentity, phase: proof.phase, receipt: proof.receipt,
|
||||
runnerVersion: proof.runnerVersion, runnerDigest: proof.runnerDigest };
|
||||
}
|
||||
@@ -0,0 +1,319 @@
|
||||
import { createRequire } from "node:module";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { drizzle } from "drizzle-orm/postgres-js";
|
||||
import { agents, agentTaskSessions, companies, completionContracts, createDb, heartbeatRuns, issues,
|
||||
nativeRunFinalizations, nativeRunResults, type Db } from "@paperclipai/db";
|
||||
import { CONTROL_PLANE_CONFORMANCE_OPEN, CONTROL_PLANE_CONFORMANCE_RESULT,
|
||||
CONTROL_PLANE_CONFORMANCE_TERMINAL } from "../../vendor/paperclip-runner/testing.js";
|
||||
import type { PersistedNativeSession } from "../../vendor/paperclip-runner/index.js";
|
||||
import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js";
|
||||
import { startRecoveryPostgres } from "../../__tests__/helpers/recovery-postgres.js";
|
||||
import { redactRegisteredSecretValues } from "../run-secret-redaction.js";
|
||||
import { nativeSha256 } from "./canonical.js";
|
||||
import { PaperclipControlPlanePort, type PaperclipControlPlaneBinding } from "./paperclip-control-plane-port.js";
|
||||
import { nativeTaskSessionPublicationHandled, publishNativeTaskSessionInTransaction, NATIVE_TASK_SESSION_PUBLICATION_KEY,
|
||||
type NativeTaskSessionPublicationContext } from "./native-task-session-publication.js";
|
||||
|
||||
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_NATIVE_PUBLICATION_TEST_DATABASE_URL");
|
||||
describe("accepted native task session publication", () => {
|
||||
let temporary: Awaited<ReturnType<typeof startRecoveryPostgres>> | null = null;
|
||||
let db: Db;
|
||||
let measured: Db;
|
||||
let client: Db["$client"];
|
||||
const statements: string[] = [];
|
||||
beforeAll(async () => {
|
||||
if (!external) temporary = await startRecoveryPostgres("native-publication-");
|
||||
const url = external ?? temporary!.connectionString;
|
||||
db = createDb(url, { maxConnections: 2 });
|
||||
// Test-only wire observer. Production still uses the existing Db/pools.
|
||||
const postgres = createRequire(import.meta.resolve("@paperclipai/db"))("postgres") as
|
||||
(url: string, options: object) => Db["$client"];
|
||||
client = postgres(url, { max: 1, prepare: false, connection: { application_name: "native-publication-observer" },
|
||||
debug: (_id: number, query: string) => { statements.push(query); } });
|
||||
measured = drizzle(client) as unknown as Db;
|
||||
});
|
||||
afterAll(async () => {
|
||||
await client?.end({ timeout: 1 });
|
||||
await db?.$client.end({ timeout: 1 });
|
||||
await temporary?.cleanup();
|
||||
});
|
||||
|
||||
async function fixture(source: "runner" | "control" = "runner", accepted = true) {
|
||||
const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID();
|
||||
const sessionId = randomUUID(), contractId = randomUUID(), runner = randomUUID(), control = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Native pointer", issuePrefix: `P${companyId.slice(0, 7)}` });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Native pointer", adapterType: "codex_local", status: "running" });
|
||||
await db.insert(issues).values({ id: issueId, companyId, title: "Native pointer", status: "in_progress", assigneeAgentId: agentId });
|
||||
await db.insert(completionContracts).values({ id: contractId, companyId, issueId, revision: 1,
|
||||
schemaVersion: "paperclip.completion-contract.v1", policyVersion: "standalone-v1", risk: "standard",
|
||||
completionAuthority: "server_arbiter", incompleteCriteriaPolicy: "preserve_non_terminal",
|
||||
contractJson: { revision: "standalone-v1", objective: "Conformance", criteria: [{ id: "objective", requirement: "Complete" }] },
|
||||
canonicalSha256: "publication-contract", createdByActorType: "system", createdByActorId: "test" });
|
||||
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running", runtimeMode: "native",
|
||||
nativeIssueId: issueId, nativeSessionId: sessionId, runnerInstanceId: runner, completionContractId: contractId,
|
||||
completionContractSha256: "publication-contract", startedAt: new Date(Date.now() - 2_000), runnerProfileJson: {} });
|
||||
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId));
|
||||
const owner = { companyId, issueId, runId, authorityRunId: runId, bootId: "publication-boot", generation: 1, leaseOwner: "publication-owner" };
|
||||
await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId, phase: "observed",
|
||||
controllerBootId: owner.bootId, controllerGeneration: owner.generation, leaseOwner: owner.leaseOwner,
|
||||
leaseExpiresAt: new Date(Date.now() + 60_000) });
|
||||
const binding: PaperclipControlPlaneBinding = { companyId, agentId, issueId, runId, sessionId,
|
||||
completionContractId: contractId, completionContractSha256: "publication-contract", sourceInstanceId: runner,
|
||||
controlPlaneSourceInstanceId: control };
|
||||
const context: NativeTaskSessionPublicationContext = { taskKey: issueId, adapterType: "codex_local",
|
||||
sessionParamsJson: { cwd: "/working", paperclipSessionConfigFingerprint: "prepared-once", opaque: "private-prepared-param" } };
|
||||
const snapshot: PersistedNativeSession = { backendKind: "mock", sessionId, identity: { companyId, agentId, issueId, runId, sessionId },
|
||||
providerSessionId: "provider-thread", activeTurnId: null, pendingRuntimeRequests: [], goal: null,
|
||||
semanticResult: CONTROL_PLANE_CONFORMANCE_RESULT, terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL };
|
||||
const port = new PaperclipControlPlanePort(measured, binding, { taskSessionPublication: context, taskSessionPublicationOwner: owner });
|
||||
await port.openRun({ ...CONTROL_PLANE_CONFORMANCE_OPEN, identity: snapshot.identity, sourceInstanceId: runner });
|
||||
if (accepted) await new PaperclipControlPlanePort(db, binding).completeRun({ result: CONTROL_PLANE_CONFORMANCE_RESULT,
|
||||
terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, turnId: "publication-turn" });
|
||||
await db.insert(agentTaskSessions).values({ companyId, agentId, adapterType: context.adapterType, taskKey: issueId,
|
||||
goalSourceId: `${source === "runner" ? runner : control}:${runId}`, goalSourceCursor: 4 });
|
||||
const read = async () => {
|
||||
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
|
||||
const [session] = await db.select().from(agentTaskSessions).where(eq(agentTaskSessions.taskKey, issueId));
|
||||
return { run, session };
|
||||
};
|
||||
return { binding, owner, context, snapshot, port, read };
|
||||
}
|
||||
|
||||
it.each(["runner", "control"] as const)("publishes from the exact accepted %s epoch in one checkpoint transaction", async source => {
|
||||
const f = await fixture(source); statements.length = 0;
|
||||
await f.port.checkpointSession(f.snapshot);
|
||||
const queries = [...statements], { run, session } = await f.read();
|
||||
expect(session).toMatchObject({ sessionParamsJson: { ...f.context.sessionParamsJson, sessionId: f.binding.sessionId },
|
||||
sessionDisplayId: "provider-thread", lastRunId: f.binding.runId });
|
||||
expect(nativeTaskSessionPublicationHandled(run.runnerProfileJson, { ...f.binding, ...f.context })).toBe(true);
|
||||
expect(queries.filter(query => /^begin\b/i.test(query.trim()))).toHaveLength(1);
|
||||
expect(queries.filter(query => /^commit\b/i.test(query.trim()))).toHaveLength(1);
|
||||
expect(queries.filter(query => /native_run_results/i.test(query))).toHaveLength(1);
|
||||
expect(queries.filter(query => /^select .*from "agent_task_sessions"/i.test(query.trim()))).toHaveLength(1);
|
||||
expect(queries.filter(query => /^update "agent_task_sessions"/i.test(query.trim()))).toHaveLength(1);
|
||||
expect(queries).toHaveLength(7);
|
||||
expect(JSON.stringify(run.runnerProfileJson)).not.toContain("private-prepared-param");
|
||||
expect(JSON.stringify({ [NATIVE_TASK_SESSION_PUBLICATION_KEY]: run.runnerProfileJson![NATIVE_TASK_SESSION_PUBLICATION_KEY] }).length).toBeLessThan(4_096);
|
||||
expect(redactRegisteredSecretValues(run.runnerProfileJson, [])).not.toHaveProperty(NATIVE_TASK_SESSION_PUBLICATION_KEY);
|
||||
expect(nativeTaskSessionPublicationHandled(run.runnerProfileJson, { ...f.binding, ...f.context, companyId: randomUUID() })).toBe(false);
|
||||
expect(nativeTaskSessionPublicationHandled(run.runnerProfileJson, { ...f.binding, ...f.context, agentId: randomUUID() })).toBe(false);
|
||||
const malformed = structuredClone(run.runnerProfileJson!);
|
||||
(malformed[NATIVE_TASK_SESSION_PUBLICATION_KEY] as Record<string, unknown>).resultId = "not-an-id";
|
||||
expect(nativeTaskSessionPublicationHandled(malformed, { ...f.binding, ...f.context })).toBe(false);
|
||||
});
|
||||
|
||||
it("does not recreate a reset pointer or query task state on repeated terminal maintenance", async () => {
|
||||
const f = await fixture(); await f.port.checkpointSession(f.snapshot);
|
||||
await db.delete(agentTaskSessions).where(eq(agentTaskSessions.taskKey, f.binding.issueId)); statements.length = 0;
|
||||
await f.port.checkpointSession(f.snapshot);
|
||||
const repeated = [...statements];
|
||||
expect((await f.read()).session).toBeUndefined();
|
||||
expect(repeated.filter(query => /agent_task_sessions|native_run_results/i.test(query))).toHaveLength(0);
|
||||
expect(repeated.filter(query => /^begin\b/i.test(query.trim()))).toHaveLength(1);
|
||||
expect(repeated.filter(query => /^update /i.test(query.trim()))).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("persists a pre-acceptance terminal observation and publishes only at the existing accepted checkpoint", async () => {
|
||||
const f = await fixture("runner", false);
|
||||
// Driver event/startup checkpoints can expose a semantic terminal observation
|
||||
// before the runtime calls completeRun. That observation is not acceptance.
|
||||
await f.port.checkpointSession(f.snapshot);
|
||||
const observed = await f.read();
|
||||
expect(observed.run.runnerProfileJson?.sessionCheckpoint).toMatchObject({ semanticResult: CONTROL_PLANE_CONFORMANCE_RESULT });
|
||||
expect(observed.run.runnerProfileJson).not.toHaveProperty(NATIVE_TASK_SESSION_PUBLICATION_KEY);
|
||||
expect(observed.session?.sessionParamsJson).toBeNull();
|
||||
expect(nativeTaskSessionPublicationHandled(observed.run.runnerProfileJson, { ...f.binding, ...f.context })).toBe(false);
|
||||
await f.port.completeRun({ result: CONTROL_PLANE_CONFORMANCE_RESULT,
|
||||
terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, turnId: "publication-turn" });
|
||||
expect((await f.read()).session?.sessionParamsJson).toBeNull();
|
||||
await f.port.checkpointSession(f.snapshot);
|
||||
const completed = await f.read();
|
||||
expect(completed.session?.lastRunId).toBe(f.binding.runId);
|
||||
expect(nativeTaskSessionPublicationHandled(completed.run.runnerProfileJson, { ...f.binding, ...f.context })).toBe(true);
|
||||
});
|
||||
|
||||
it("does not hold the issue parent lock while a result replay waits for the status coordinator", async () => {
|
||||
const f = await fixture();
|
||||
let committed = 0;
|
||||
const port = new PaperclipControlPlanePort(db, f.binding, { nativeRecovery: { owner: f.owner,
|
||||
compatibility: { configDigest: nativeSha256("config"), processConfigDigest: null } },
|
||||
onResultCommitted: async () => { committed++; } });
|
||||
await port.openRun({ ...CONTROL_PLANE_CONFORMANCE_OPEN, identity: f.snapshot.identity, sourceInstanceId: f.binding.sourceInstanceId });
|
||||
let replay: Promise<unknown> | undefined;
|
||||
let statusLockError: unknown = null;
|
||||
try {
|
||||
await db.transaction(async tx => {
|
||||
// This is commitNativeStatusDecision's actual lock prefix. Hold its
|
||||
// coordinator first, then overlap an identical remote result replay.
|
||||
await tx.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.binding.runId)).for("update");
|
||||
const [backend] = await tx.execute(sql`select pg_backend_pid() as pid`);
|
||||
replay = port.completeRun({ result: CONTROL_PLANE_CONFORMANCE_RESULT,
|
||||
terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, turnId: "publication-turn" }).then(() => null, error => error);
|
||||
let waiting = false;
|
||||
for (let attempt = 0; attempt < 100; attempt++) {
|
||||
const [state] = await measured.execute(sql`select exists (
|
||||
select 1 from pg_stat_activity where ${backend.pid} = any(pg_blocking_pids(pid))) as waiting`);
|
||||
if (state.waiting) { waiting = true; break; }
|
||||
await new Promise(resolve => setTimeout(resolve, 10));
|
||||
}
|
||||
expect(waiting).toBe(true);
|
||||
// A bounded test-local timeout reports the inverse-order cycle without
|
||||
// relying on which transaction PostgreSQL chooses as deadlock victim.
|
||||
await tx.execute(sql`set local lock_timeout = '500ms'`);
|
||||
await tx.select().from(issues).where(eq(issues.id, f.binding.issueId)).for("update");
|
||||
});
|
||||
} catch (error) { statusLockError = error; }
|
||||
const replayError = await replay;
|
||||
expect(statusLockError).toBeNull();
|
||||
expect(replayError).toBeNull();
|
||||
expect(committed).toBe(0);
|
||||
expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, f.binding.runId))).toHaveLength(1);
|
||||
await db.update(nativeRunFinalizations).set({ controllerGeneration: 2 }).where(eq(nativeRunFinalizations.runId, f.binding.runId));
|
||||
await expect(port.completeRun({ result: CONTROL_PLANE_CONFORMANCE_RESULT,
|
||||
terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, turnId: "publication-turn" })).rejects.toThrow("checkpoint_owner_lost");
|
||||
});
|
||||
|
||||
it.each(["missing", "foreign_epoch", "newer_pointer", "newer_execution", "generation_reset", "stop"])("fences %s and records a handled suppression", async kind => {
|
||||
const f = await fixture();
|
||||
if (kind === "missing") await db.delete(agentTaskSessions).where(eq(agentTaskSessions.taskKey, f.binding.issueId));
|
||||
if (kind === "foreign_epoch") await db.update(agentTaskSessions).set({ goalSourceId: `${randomUUID()}:${f.binding.runId}` }).where(eq(agentTaskSessions.taskKey, f.binding.issueId));
|
||||
if (kind === "newer_pointer" || kind === "newer_execution") {
|
||||
const newer = randomUUID();
|
||||
await db.insert(heartbeatRuns).values({ id: newer, companyId: f.binding.companyId, agentId: f.binding.agentId,
|
||||
status: "running", issueId: f.binding.issueId, startedAt: new Date() });
|
||||
if (kind === "newer_pointer") await db.update(agentTaskSessions).set({ lastRunId: newer, sessionParamsJson: { sessionId: "newer" } }).where(eq(agentTaskSessions.taskKey, f.binding.issueId));
|
||||
else await db.update(issues).set({ executionRunId: newer }).where(eq(issues.id, f.binding.issueId));
|
||||
}
|
||||
if (kind === "generation_reset") await db.update(issues).set({ conversationAgentId: f.binding.agentId,
|
||||
conversationUserId: "publication-test-user", conversationState: "active", conversationSessionGeneration: 2 }).where(eq(issues.id, f.binding.issueId));
|
||||
if (kind === "stop") await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, f.binding.runId));
|
||||
await f.port.checkpointSession(f.snapshot);
|
||||
const { run, session } = await f.read();
|
||||
expect(nativeTaskSessionPublicationHandled(run.runnerProfileJson, { ...f.binding, ...f.context })).toBe(true);
|
||||
expect(session?.sessionParamsJson?.sessionId).not.toBe(f.binding.sessionId);
|
||||
});
|
||||
|
||||
it("keeps unsupported task keys and missing remote recovery headers outside handled publication", async () => {
|
||||
const f = await fixture();
|
||||
const context = { ...f.context, taskKey: "custom-task" };
|
||||
const port = new PaperclipControlPlanePort(measured, f.binding, { taskSessionPublication: context, taskSessionPublicationOwner: f.owner });
|
||||
await port.openRun({ ...CONTROL_PLANE_CONFORMANCE_OPEN, identity: f.snapshot.identity, sourceInstanceId: f.binding.sourceInstanceId });
|
||||
await port.checkpointSession(f.snapshot);
|
||||
const { run, session } = await f.read();
|
||||
expect(session?.sessionParamsJson).toBeNull();
|
||||
expect(nativeTaskSessionPublicationHandled(run.runnerProfileJson, { ...f.binding, ...context })).toBe(false);
|
||||
const remote = await fixture();
|
||||
const remotePort = new PaperclipControlPlanePort(measured, remote.binding, { taskSessionPublication: remote.context,
|
||||
taskSessionPublicationOwner: remote.owner, nativeRecovery: { owner: remote.owner,
|
||||
compatibility: { configDigest: nativeSha256("config"), processConfigDigest: null } } });
|
||||
await remotePort.openRun({ ...CONTROL_PLANE_CONFORMANCE_OPEN, identity: remote.snapshot.identity, sourceInstanceId: remote.binding.sourceInstanceId });
|
||||
await remotePort.checkpointSession(remote.snapshot);
|
||||
expect((await remote.read()).session?.sessionParamsJson).toBeNull();
|
||||
expect(nativeTaskSessionPublicationHandled((await remote.read()).run.runnerProfileJson, { ...remote.binding, ...remote.context })).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects stale owners and a different accepted result without partially publishing", async () => {
|
||||
const stale = await fixture();
|
||||
await db.update(nativeRunFinalizations).set({ controllerGeneration: 2 }).where(eq(nativeRunFinalizations.runId, stale.binding.runId));
|
||||
await expect(stale.port.checkpointSession(stale.snapshot)).rejects.toThrow("checkpoint_owner_lost");
|
||||
expect((await stale.read()).session?.sessionParamsJson).toBeNull();
|
||||
const wrong = await fixture();
|
||||
await expect(wrong.port.checkpointSession({ ...wrong.snapshot, semanticResult: { ...CONTROL_PLANE_CONFORMANCE_RESULT, summary: "different" } })).rejects.toThrow("publication_result_mismatch");
|
||||
expect((await wrong.read()).session?.sessionParamsJson).toBeNull();
|
||||
const foreign = await fixture();
|
||||
const foreignResult = await fixture();
|
||||
const [otherCoordinator] = await db.select({ resultId: nativeRunFinalizations.resultId }).from(nativeRunFinalizations)
|
||||
.where(eq(nativeRunFinalizations.runId, foreignResult.binding.runId));
|
||||
await expect(db.update(nativeRunFinalizations).set({ resultId: otherCoordinator.resultId })
|
||||
.where(eq(nativeRunFinalizations.runId, foreign.binding.runId)))
|
||||
.rejects.toMatchObject({ cause: { constraint_name: "native_run_finalizations_result_owner_fk" } });
|
||||
expect((await foreign.read()).session?.sessionParamsJson).toBeNull();
|
||||
await db.update(nativeRunFinalizations).set({ resultId: null }).where(eq(nativeRunFinalizations.runId, foreign.binding.runId));
|
||||
// The CP defers preacceptance snapshots, while the publication helper itself
|
||||
// still requires accepted-result authority if called directly.
|
||||
await expect(publishNativeTaskSessionInTransaction({} as never, { binding: foreign.binding,
|
||||
context: foreign.context, snapshot: foreign.snapshot, run: (await foreign.read()).run,
|
||||
issue: null, resultId: null })).rejects.toThrow("publication_result_missing");
|
||||
});
|
||||
|
||||
it("adds no publication SQL for nonterminal or goal snapshots", async () => {
|
||||
const f = await fixture(); statements.length = 0;
|
||||
await f.port.checkpointSession({ ...f.snapshot, semanticResult: null, terminal: null });
|
||||
expect(statements.some(query => /agent_task_sessions|native_run_results|publication_issue/i.test(query))).toBe(false);
|
||||
statements.length = 0;
|
||||
await f.port.checkpointSession({ ...f.snapshot, goal: {} as NonNullable<PersistedNativeSession["goal"]> });
|
||||
expect(statements.some(query => /agent_task_sessions|native_run_results|publication_issue/i.test(query))).toBe(false);
|
||||
expect((await f.read()).session?.sessionParamsJson).toBeNull();
|
||||
});
|
||||
|
||||
it.each(["completed_predecessor", "expired_accepted_lease"])("preserves legitimate %s maintenance", async kind => {
|
||||
const f = await fixture();
|
||||
if (kind === "completed_predecessor") {
|
||||
const prior = randomUUID();
|
||||
await db.insert(heartbeatRuns).values({ id: prior, companyId: f.binding.companyId, agentId: f.binding.agentId,
|
||||
issueId: f.binding.issueId, status: "succeeded", finishedAt: new Date(Date.now() - 5_000) });
|
||||
await db.update(agentTaskSessions).set({ lastRunId: prior, sessionParamsJson: { sessionId: "prior" } })
|
||||
.where(eq(agentTaskSessions.taskKey, f.binding.issueId));
|
||||
} else {
|
||||
await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 1_000) })
|
||||
.where(eq(nativeRunFinalizations.runId, f.binding.runId));
|
||||
}
|
||||
await f.port.checkpointSession(f.snapshot);
|
||||
expect((await f.read()).session?.lastRunId).toBe(f.binding.runId);
|
||||
});
|
||||
|
||||
it("does not block completeRun's issue KEY SHARE while publishing", async () => {
|
||||
const f = await fixture(); let release!: () => void; let locked!: () => void;
|
||||
const ready = new Promise<void>(resolve => { locked = resolve; });
|
||||
const hold = new Promise<void>(resolve => { release = resolve; });
|
||||
const blocker = db.transaction(async tx => {
|
||||
await tx.select().from(issues).where(eq(issues.id, f.binding.issueId)).for("key share"); locked(); await hold;
|
||||
});
|
||||
await ready;
|
||||
try { await f.port.checkpointSession(f.snapshot); }
|
||||
finally { release(); await blocker; }
|
||||
expect((await f.read()).session?.lastRunId).toBe(f.binding.runId);
|
||||
});
|
||||
|
||||
it("rechecks the owner clock and issue authority after an issue lock wait", async () => {
|
||||
const f = await fixture(); let release!: () => void; let locked!: () => void;
|
||||
await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() + 500) }).where(eq(nativeRunFinalizations.runId, f.binding.runId));
|
||||
const ready = new Promise<void>(resolve => { locked = resolve; });
|
||||
const hold = new Promise<void>(resolve => { release = resolve; });
|
||||
const newer = randomUUID();
|
||||
await db.insert(heartbeatRuns).values({ id: newer, companyId: f.binding.companyId, agentId: f.binding.agentId,
|
||||
issueId: f.binding.issueId, status: "running", startedAt: new Date() });
|
||||
const blocker = db.transaction(async tx => {
|
||||
await tx.select().from(issues).where(eq(issues.id, f.binding.issueId)).for("no key update");
|
||||
await tx.update(issues).set({ executionRunId: newer }).where(eq(issues.id, f.binding.issueId));
|
||||
locked(); await hold;
|
||||
});
|
||||
await ready;
|
||||
const publication = f.port.checkpointSession(f.snapshot);
|
||||
const rejected = expect(publication).rejects.toThrow("checkpoint_owner_lost");
|
||||
try {
|
||||
let waiting = false;
|
||||
for (let attempt = 0; attempt < 100; attempt++) {
|
||||
const rows = await db.execute(sql`select 1 from pg_stat_activity where application_name = 'native-publication-observer'
|
||||
and wait_event_type = 'Lock' and query like '%publication_issue%'`);
|
||||
if (rows.length) { waiting = true; break; }
|
||||
await new Promise(resolve => setTimeout(resolve, 10));
|
||||
}
|
||||
expect(waiting).toBe(true);
|
||||
await new Promise(resolve => setTimeout(resolve, 600));
|
||||
} finally { release(); await blocker; }
|
||||
await rejected;
|
||||
expect((await f.read()).session?.sessionParamsJson).toBeNull();
|
||||
});
|
||||
|
||||
it("rolls back the pointer if combined private metadata exceeds its existing bound", async () => {
|
||||
const f = await fixture();
|
||||
await db.update(heartbeatRuns).set({ runnerProfileJson: { nativeControllerRecovery: { padding: "x".repeat(4_096) } } }).where(eq(heartbeatRuns.id, f.binding.runId));
|
||||
await expect(f.port.checkpointSession(f.snapshot)).rejects.toThrow("publication_metadata_too_large");
|
||||
const { run, session } = await f.read();
|
||||
expect(session?.sessionParamsJson).toBeNull();
|
||||
expect(run.runnerProfileJson).not.toHaveProperty(NATIVE_TASK_SESSION_PUBLICATION_KEY);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,145 @@
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { agentTaskSessions, heartbeatRuns, nativeRunResults, type Db, type issues } from "@paperclipai/db";
|
||||
import { validatePrpStructuredRunResult, type PersistedNativeSession } from "../../vendor/paperclip-runner/index.js";
|
||||
import { nativeSha256 } from "./canonical.js";
|
||||
import type { PaperclipControlPlaneBinding } from "./paperclip-control-plane-port.js";
|
||||
import { readRunCancellation } from "../run-cancellation.js";
|
||||
|
||||
/** Prepared once by heartbeat; never copied into recovery metadata or receipts. */
|
||||
export interface NativeTaskSessionPublicationContext {
|
||||
taskKey: string;
|
||||
adapterType: string;
|
||||
sessionParamsJson: Record<string, unknown> | null;
|
||||
}
|
||||
|
||||
export const NATIVE_TASK_SESSION_PUBLICATION_KEY = "nativeTaskSessionPublication";
|
||||
type Transaction = Parameters<Parameters<Db["transaction"]>[0]>[0];
|
||||
type Outcome = "published" | "suppressed_reset" | "suppressed_newer" | "suppressed_authority" | "missing_stub" | "unsupported";
|
||||
interface Receipt {
|
||||
schema: "paperclip.native-task-session-publication.v1";
|
||||
companyId: string;
|
||||
issueId: string;
|
||||
agentId: string;
|
||||
runId: string;
|
||||
taskKey: string;
|
||||
adapterType: string;
|
||||
sessionId: string;
|
||||
resultId: string;
|
||||
outcome: Outcome;
|
||||
}
|
||||
|
||||
function record(value: unknown): Record<string, unknown> {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||||
}
|
||||
const uuid = (value: unknown): value is string => typeof value === "string" && /^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/i.test(value);
|
||||
const outcomes: readonly string[] = ["published", "suppressed_reset", "suppressed_newer", "suppressed_authority", "missing_stub", "unsupported"];
|
||||
|
||||
function receiptFromProfile(profile: unknown): Receipt | null {
|
||||
const root = record(profile), value = record(root[NATIVE_TASK_SESSION_PUBLICATION_KEY]);
|
||||
if (value.schema !== "paperclip.native-task-session-publication.v1"
|
||||
|| ![value.companyId, value.issueId, value.agentId, value.runId, value.sessionId, value.resultId].every(uuid)
|
||||
|| typeof value.taskKey !== "string" || value.taskKey.length > 512
|
||||
|| typeof value.adapterType !== "string" || !value.adapterType.length || value.adapterType.length > 128
|
||||
|| !outcomes.includes(String(value.outcome)) || Object.keys(value).length !== 10) return null;
|
||||
const snapshot = record(root.sessionCheckpoint), identity = record(snapshot.identity);
|
||||
if (identity.runId !== value.runId || identity.companyId !== value.companyId || identity.issueId !== value.issueId
|
||||
|| identity.agentId !== value.agentId || identity.sessionId !== value.sessionId) return null;
|
||||
return value as unknown as Receipt;
|
||||
}
|
||||
|
||||
/** Only the exact server-written outcome may suppress heartbeat's late fallback. */
|
||||
export function nativeTaskSessionPublicationHandled(profile: unknown, binding: {
|
||||
companyId: string; agentId: string; runId: string; taskKey: string; adapterType: string;
|
||||
}): boolean {
|
||||
const receipt = receiptFromProfile(profile);
|
||||
return !!receipt && receipt.companyId === binding.companyId && receipt.agentId === binding.agentId
|
||||
&& receipt.runId === binding.runId && receipt.taskKey === binding.taskKey
|
||||
&& receipt.adapterType === binding.adapterType && ["published", "suppressed_reset", "suppressed_newer", "suppressed_authority", "missing_stub"].includes(receipt.outcome);
|
||||
}
|
||||
|
||||
export function eligibleNativeTaskSessionPublication(snapshot: PersistedNativeSession): boolean {
|
||||
return snapshot.goal == null && snapshot.semanticResult != null && snapshot.terminal?.runTerminalState === "succeeded"
|
||||
&& snapshot.activeTurnId == null && !(snapshot.pendingRuntimeRequests?.length);
|
||||
}
|
||||
|
||||
export type NativeTaskSessionPublicationRun = Pick<typeof heartbeatRuns.$inferSelect,
|
||||
"id" | "companyId" | "agentId" | "nativeIssueId" | "nativeSessionId" | "status" | "resultJson" | "contextSnapshot" | "startedAt" | "createdAt" | "runnerProfileJson">;
|
||||
export type NativeTaskSessionPublicationIssue = Pick<typeof issues.$inferSelect,
|
||||
"id" | "companyId" | "executionRunId" | "conversationAgentId" | "conversationSessionGeneration">;
|
||||
|
||||
/** Caller holds coordinator → issue NO KEY UPDATE → run. This helper owns only
|
||||
* the task-row lock and UPDATE; never starts a transaction or creates a stub. */
|
||||
export async function publishNativeTaskSessionInTransaction(tx: Transaction, input: {
|
||||
binding: PaperclipControlPlaneBinding;
|
||||
context: NativeTaskSessionPublicationContext;
|
||||
snapshot: PersistedNativeSession;
|
||||
run: NativeTaskSessionPublicationRun;
|
||||
issue: NativeTaskSessionPublicationIssue | null;
|
||||
resultId: string | null;
|
||||
recoveryReady?: boolean;
|
||||
}): Promise<Receipt> {
|
||||
const { binding, context, snapshot, run, issue } = input;
|
||||
const existingReceipt = receiptFromProfile(run.runnerProfileJson);
|
||||
if (existingReceipt) {
|
||||
if (existingReceipt.companyId !== binding.companyId || existingReceipt.issueId !== binding.issueId
|
||||
|| existingReceipt.agentId !== binding.agentId || existingReceipt.runId !== binding.runId
|
||||
|| existingReceipt.sessionId !== snapshot.identity.sessionId || existingReceipt.taskKey !== context.taskKey
|
||||
|| existingReceipt.adapterType !== context.adapterType || existingReceipt.resultId !== input.resultId) {
|
||||
throw new Error("native_task_session_publication_receipt_mismatch");
|
||||
}
|
||||
return existingReceipt;
|
||||
}
|
||||
if (record(run.runnerProfileJson)[NATIVE_TASK_SESSION_PUBLICATION_KEY] !== undefined) {
|
||||
throw new Error("native_task_session_publication_receipt_invalid");
|
||||
}
|
||||
if (!input.resultId) throw new Error("native_task_session_publication_result_missing");
|
||||
const [accepted] = await tx.select({ id: nativeRunResults.id, canonicalSha256: nativeRunResults.canonicalSha256,
|
||||
turnId: nativeRunResults.turnId }).from(nativeRunResults).where(and(
|
||||
eq(nativeRunResults.id, input.resultId), eq(nativeRunResults.runId, binding.runId),
|
||||
eq(nativeRunResults.companyId, binding.companyId), eq(nativeRunResults.issueId, binding.issueId),
|
||||
eq(nativeRunResults.completionContractId, binding.completionContractId), eq(nativeRunResults.schemaStatus, "accepted"),
|
||||
)).limit(1);
|
||||
const validated = validatePrpStructuredRunResult(snapshot.semanticResult);
|
||||
if (!accepted || !validated.ok || accepted.canonicalSha256 !== nativeSha256({
|
||||
binding, result: validated.result, terminal: snapshot.terminal, turnId: accepted.turnId ?? null,
|
||||
})) throw new Error("native_task_session_publication_result_mismatch");
|
||||
const receipt = (outcome: Outcome): Receipt => ({ schema: "paperclip.native-task-session-publication.v1",
|
||||
companyId: binding.companyId, issueId: binding.issueId, agentId: binding.agentId, runId: binding.runId,
|
||||
taskKey: context.taskKey, adapterType: context.adapterType, sessionId: snapshot.identity.sessionId,
|
||||
resultId: accepted.id, outcome });
|
||||
if (context.taskKey !== binding.issueId || input.recoveryReady === false) return receipt("unsupported");
|
||||
const cancellation = record(run.resultJson?.nativeCancellation), stop = readRunCancellation(run.resultJson);
|
||||
if (!issue || ["cancelled", "interrupted", "failed", "timed_out"].includes(run.status) || issue.companyId !== binding.companyId
|
||||
|| run.resultJson?.startupCancellation || (stop?.source === "operator" && stop.expected)
|
||||
|| (["run", "issue"].includes(String(cancellation.scope)) && ["pending", "acknowledged"].includes(String(cancellation.dispatchState)))
|
||||
|| (issue.conversationAgentId && issue.conversationAgentId !== binding.agentId)) return receipt("suppressed_authority");
|
||||
if (issue.conversationAgentId
|
||||
&& run.contextSnapshot?.conversationSessionGeneration !== issue.conversationSessionGeneration) return receipt("suppressed_reset");
|
||||
if (issue.executionRunId && issue.executionRunId !== binding.runId) return receipt("suppressed_newer");
|
||||
const [session] = await tx.select({ id: agentTaskSessions.id, goalSourceId: agentTaskSessions.goalSourceId,
|
||||
lastRunId: agentTaskSessions.lastRunId }).from(agentTaskSessions).where(and(
|
||||
eq(agentTaskSessions.companyId, binding.companyId), eq(agentTaskSessions.agentId, binding.agentId),
|
||||
eq(agentTaskSessions.adapterType, context.adapterType), eq(agentTaskSessions.taskKey, context.taskKey),
|
||||
)).for("update").limit(1);
|
||||
// Supported native turns synchronously project a stub before this terminal
|
||||
// snapshot. Absence can be an operator deletion; never recreate it later.
|
||||
if (!session) return receipt("missing_stub");
|
||||
const boundSources = [binding.sourceInstanceId, binding.controlPlaneSourceInstanceId]
|
||||
.filter((value): value is string => typeof value === "string" && value.length > 0)
|
||||
.map(source => `${source}:${binding.runId}`);
|
||||
if (!session.goalSourceId || !boundSources.includes(session.goalSourceId)) return receipt("suppressed_newer");
|
||||
if (session.lastRunId && session.lastRunId !== binding.runId) {
|
||||
const [prior] = await tx.select({ companyId: heartbeatRuns.companyId, agentId: heartbeatRuns.agentId,
|
||||
issueId: heartbeatRuns.issueId, nativeIssueId: heartbeatRuns.nativeIssueId, finishedAt: heartbeatRuns.finishedAt,
|
||||
}).from(heartbeatRuns).where(eq(heartbeatRuns.id, session.lastRunId)).limit(1);
|
||||
if (!prior || prior.companyId !== binding.companyId || prior.agentId !== binding.agentId
|
||||
|| (prior.nativeIssueId ?? prior.issueId) !== binding.issueId || !prior.finishedAt || !run.startedAt
|
||||
|| prior.finishedAt > run.startedAt) return receipt("suppressed_newer");
|
||||
}
|
||||
await tx.update(agentTaskSessions).set({
|
||||
sessionParamsJson: { ...context.sessionParamsJson, sessionId: snapshot.identity.sessionId },
|
||||
sessionDisplayId: snapshot.providerSessionId ?? snapshot.sessionId, lastRunId: binding.runId,
|
||||
lastError: null, updatedAt: new Date(),
|
||||
}).where(eq(agentTaskSessions.id, session.id));
|
||||
return receipt("published");
|
||||
}
|
||||
@@ -12,8 +12,8 @@ import { agents, budgetPolicies, companies, costEvents, createDb, heartbeatRuns,
|
||||
import { type PrpEvent } from "../../vendor/paperclip-runner/index.js";
|
||||
import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
|
||||
import { accountRunCost } from "../run-cost-accounting.js";
|
||||
import { usageReceiptSpoolPath } from "../usage-receipts.js";
|
||||
import { createNativeUsageDelivery } from "./native-usage-delivery.js";
|
||||
import { usageReceiptSpoolPath, type UsageReceiptEnvelope } from "../usage-receipts.js";
|
||||
import { createNativeUsageContinuationDelivery, createNativeUsageDelivery } from "./native-usage-delivery.js";
|
||||
|
||||
const runnerRoot = fileURLToPath(new URL("../../../../packages/paperclip-runner/runner/", import.meta.url));
|
||||
const fixtureProfile = process.env.CI ? "release" : "debug";
|
||||
@@ -44,6 +44,89 @@ describe("producer-owned native usage", () => {
|
||||
payload: type === "usage.reported" ? { runDeltaAvailable: true, runDeltaBasis: "run", runDelta: { inputTokens: 120, outputTokens: 10, providerCostUsd: amount } } : {},
|
||||
} as PrpEvent);
|
||||
|
||||
const finalizedSource = async () => {
|
||||
const input = await seed();
|
||||
await db.update(heartbeatRuns).set({ runnerInstanceId: input.runnerInstanceId }).where(eq(heartbeatRuns.id, input.runId));
|
||||
const delivery = await createNativeUsageDelivery(db, input);
|
||||
const report = event(input, 1, "usage.reported"), done = event(input, 2, "turn.completed");
|
||||
await delivery.capture(report);
|
||||
await delivery.capture(done);
|
||||
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, input.runId));
|
||||
expect(await accountRunCost(db, input.runId)).toBe(true);
|
||||
const { billing: _billing, ...source } = input;
|
||||
return { input, source, report, done };
|
||||
};
|
||||
|
||||
it("continues exact finalized source receipts without acquiring ownership or resetting accounting", async () => {
|
||||
const { input, source, report, done } = await finalizedSource();
|
||||
const before = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId));
|
||||
await expect(createNativeUsageDelivery(db, input)).rejects.toThrow("native_usage_run_not_available");
|
||||
const continuation = await createNativeUsageContinuationDelivery(db, source);
|
||||
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId))).toEqual(before);
|
||||
await continuation.capture(report);
|
||||
await continuation.capture(done);
|
||||
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId))).toEqual(before);
|
||||
expect(await db.select().from(runUsageReceipts).where(eq(runUsageReceipts.runId, input.runId))).toHaveLength(2);
|
||||
expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId))).toHaveLength(1);
|
||||
await expect(continuation.capture({ ...report, sourceInstanceId: randomUUID() })).rejects.toThrow("native_usage_source_mismatch");
|
||||
await expect(continuation.capture({ ...report, payload: { ...report.payload as object, runDeltaAvailable: false } }))
|
||||
.rejects.toThrow("native_usage_source_conflict");
|
||||
});
|
||||
|
||||
it("journals an equal finalized source tail using the source billing without repricing", async () => {
|
||||
const { input, source } = await finalizedSource();
|
||||
const before = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId));
|
||||
// Even a structurally supplied extra current-run classification cannot own
|
||||
// the old receipt: the continuation derives billing from saved source data.
|
||||
const withCurrentBilling = { ...source, billing: { provider: "anthropic", biller: "current-grant", billingType: "subscription_included" } };
|
||||
const continuation = await createNativeUsageContinuationDelivery(db, withCurrentBilling);
|
||||
await continuation.capture(event(input, 3, "usage.reported"));
|
||||
const receipts = await db.select().from(runUsageReceipts).where(eq(runUsageReceipts.runId, input.runId));
|
||||
expect(receipts).toHaveLength(3);
|
||||
expect(receipts.find(row => row.sequence === 3)?.receiptJson.receipt).toMatchObject(billing);
|
||||
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId))).toEqual(before);
|
||||
expect((await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId)))[0].costCents).toBe(25);
|
||||
});
|
||||
|
||||
it.each(["tokens", "cost", "incomplete", "unpriceable", "basis"])(
|
||||
"holds a differing finalized source tail before acknowledgement: %s", async change => {
|
||||
const { input, source } = await finalizedSource();
|
||||
const continuation = await createNativeUsageContinuationDelivery(db, source);
|
||||
const late = event(input, 3, "usage.reported");
|
||||
if (change === "incomplete") (late.payload as Record<string, unknown>).runDeltaAvailable = false;
|
||||
if (change === "basis") delete (late.payload as Record<string, unknown>).runDeltaBasis;
|
||||
if (["tokens", "cost", "unpriceable"].includes(change)) (late.payload as Record<string, unknown>).runDelta = {
|
||||
inputTokens: change === "tokens" ? 150 : change === "unpriceable" ? "unknown" : 120,
|
||||
outputTokens: 10, providerCostUsd: change === "cost" ? 0.5 : 0.25,
|
||||
};
|
||||
await expect(continuation.capture(late)).rejects.toThrow("native_usage_after_finalization_requires_reconciliation");
|
||||
await expect(continuation.capture(late)).rejects.toThrow("native_usage_after_finalization_requires_reconciliation");
|
||||
expect(await db.select().from(runUsageReceipts).where(eq(runUsageReceipts.runId, input.runId))).toHaveLength(2);
|
||||
expect((await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId)))[0].costCents).toBe(25);
|
||||
});
|
||||
|
||||
it.each(["company", "run", "runner", "source"])("rejects a changed continuation binding without mutation: %s", async change => {
|
||||
const { input, source } = await finalizedSource();
|
||||
if (change === "source") await db.update(heartbeatRuns).set({ usageJson: sql`${heartbeatRuns.usageJson} || ${JSON.stringify({ accountingReceiptSourceId: randomUUID() })}::jsonb` })
|
||||
.where(eq(heartbeatRuns.id, input.runId));
|
||||
const before = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId));
|
||||
await expect(createNativeUsageContinuationDelivery(db, { ...source,
|
||||
...(change === "company" ? { companyId: randomUUID() } : {}),
|
||||
...(change === "run" ? { runId: randomUUID() } : {}),
|
||||
...(change === "runner" ? { runnerInstanceId: randomUUID() } : {}),
|
||||
})).rejects.toThrow("native_usage_continuation_source_mismatch");
|
||||
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId))).toEqual(before);
|
||||
});
|
||||
|
||||
it("rejects changed immutable source receipt facts during hydration", async () => {
|
||||
const { input, source } = await finalizedSource();
|
||||
const [receipt] = await db.select().from(runUsageReceipts).where(and(eq(runUsageReceipts.runId, input.runId), eq(runUsageReceipts.sequence, 1)));
|
||||
const changed = structuredClone(receipt.receiptJson) as UsageReceiptEnvelope;
|
||||
changed.producerEvent!.usage!.runDelta = { inputTokens: 999, outputTokens: 10 };
|
||||
await db.update(runUsageReceipts).set({ receiptJson: changed }).where(eq(runUsageReceipts.id, receipt.id));
|
||||
await expect(createNativeUsageContinuationDelivery(db, source)).rejects.toThrow("native_usage_source_conflict");
|
||||
});
|
||||
|
||||
it("retains the same Runner report through a failed Postgres receipt and an empty replacement receiver", async () => {
|
||||
const input = await seed();
|
||||
expect(existsSync(usageReceiptSpoolPath())).toBe(false);
|
||||
|
||||
@@ -16,11 +16,13 @@ const uuid = (identity: string) => {
|
||||
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-5${hex.slice(13, 16)}-a${hex.slice(17, 20)}-${hex.slice(20, 32)}`;
|
||||
};
|
||||
|
||||
export async function createNativeUsageDelivery(db: Db, input: {
|
||||
type NativeUsageDeliveryInput = {
|
||||
companyId: string; runId: string; runnerInstanceId: string;
|
||||
provider: NativeExecutionInput["provider"];
|
||||
billing: Pick<AdapterUsageCheckpoint, "provider" | "biller" | "billingType">;
|
||||
}) {
|
||||
};
|
||||
|
||||
export async function createNativeUsageDelivery(db: Db, input: NativeUsageDeliveryInput) {
|
||||
const sourceId = uuid(`${input.runId}:${input.runnerInstanceId}`);
|
||||
const [bound] = await db.update(heartbeatRuns).set({ usageJson: sql`
|
||||
coalesce(${heartbeatRuns.usageJson}, '{}'::jsonb) || '{"accountingReceiptOwner":"native_producer"}'::jsonb ||
|
||||
@@ -29,9 +31,39 @@ export async function createNativeUsageDelivery(db: Db, input: {
|
||||
}).where(and(eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.runId),
|
||||
isNull(heartbeatRuns.costAccountedAt))).returning({ id: heartbeatRuns.id });
|
||||
if (!bound) throw new Error("native_usage_run_not_available");
|
||||
const saved = await db.select({ receipt: runUsageReceipts.receiptJson }).from(runUsageReceipts).where(and(
|
||||
return createNativeUsageReceiver(db, input, sourceId, await loadNativeUsageReceipts(db, input, sourceId));
|
||||
}
|
||||
|
||||
/** Drain the authenticated previous producer's accounting without acquiring or
|
||||
* resetting its ownership. Finalized sources retain normal receipt equality
|
||||
* and reconciliation rules; this grants no business/tool execution authority. */
|
||||
export async function createNativeUsageContinuationDelivery(db: Db, input: Omit<NativeUsageDeliveryInput, "billing">) {
|
||||
const sourceId = uuid(`${input.runId}:${input.runnerInstanceId}`);
|
||||
const [source] = await db.select({ runnerInstanceId: heartbeatRuns.runnerInstanceId, usage: heartbeatRuns.usageJson })
|
||||
.from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.runId)));
|
||||
if (source?.runnerInstanceId !== input.runnerInstanceId || source.usage?.accountingReceiptOwner !== "native_producer"
|
||||
|| source.usage.accountingReceiptSourceId !== sourceId) throw new Error("native_usage_continuation_source_mismatch");
|
||||
const saved = await loadNativeUsageReceipts(db, input, sourceId);
|
||||
if (!saved.length) throw new Error("native_usage_continuation_receipts_missing");
|
||||
const classification = (receipt: AdapterUsageCheckpoint) => ({
|
||||
provider: receipt.provider, biller: receipt.biller, billingType: receipt.billingType,
|
||||
});
|
||||
const receipts = saved.map(row => (row.receipt as UsageReceiptEnvelope).receipt as AdapterUsageCheckpoint);
|
||||
const billing = classification(receipts[0]);
|
||||
if (receipts.some(receipt => receiptFingerprint(classification(receipt)) !== receiptFingerprint(billing)
|
||||
|| receipt.model !== (input.provider.model ?? "unknown"))) throw new Error("native_usage_source_conflict");
|
||||
return createNativeUsageReceiver(db, { ...input, billing }, sourceId, saved, true);
|
||||
}
|
||||
|
||||
function loadNativeUsageReceipts(db: Db, input: Pick<NativeUsageDeliveryInput, "companyId" | "runId">, sourceId: string) {
|
||||
return db.select({ receipt: runUsageReceipts.receiptJson, hash: runUsageReceipts.receiptHash,
|
||||
sequence: runUsageReceipts.sequence }).from(runUsageReceipts).where(and(
|
||||
eq(runUsageReceipts.companyId, input.companyId), eq(runUsageReceipts.runId, input.runId),
|
||||
eq(runUsageReceipts.sourceId, sourceId))).orderBy(asc(runUsageReceipts.sequence));
|
||||
}
|
||||
|
||||
function createNativeUsageReceiver(db: Db, input: NativeUsageDeliveryInput, sourceId: string,
|
||||
saved: Awaited<ReturnType<typeof loadNativeUsageReceipts>>, verifySaved = false) {
|
||||
const accepted = new Map<string, UsageReceiptEnvelope>();
|
||||
const turnAccounting = ["opencode", "acpx"].includes(input.provider.kind) ? createNativeTurnAccounting(input.provider) : null;
|
||||
let observedUsage: Record<string, unknown> | null = null;
|
||||
@@ -60,6 +92,12 @@ export async function createNativeUsageDelivery(db: Db, input: {
|
||||
for (const row of saved) {
|
||||
const envelope = row.receipt as UsageReceiptEnvelope;
|
||||
if (!envelope.producerEvent) throw new Error("native_usage_receipt_missing_producer");
|
||||
if (verifySaved && (envelope.companyId !== input.companyId || envelope.runId !== input.runId
|
||||
|| envelope.sourceId !== sourceId || envelope.sequence !== row.sequence || envelope.adapterType !== "paperclip_runner"
|
||||
|| envelope.id !== uuid(`${input.runId}:${input.runnerInstanceId}:${envelope.producerEvent.sourceEventId}`)
|
||||
|| receiptFingerprint(envelope) !== row.hash || accepted.has(envelope.producerEvent.sourceEventId))) {
|
||||
throw new Error("native_usage_source_conflict");
|
||||
}
|
||||
accepted.set(envelope.producerEvent.sourceEventId, envelope);
|
||||
observe(envelope);
|
||||
}
|
||||
|
||||
@@ -48,6 +48,7 @@ import {
|
||||
import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
|
||||
import { startRecoveryPostgres } from "../../__tests__/helpers/recovery-postgres.js";
|
||||
import { PaperclipControlPlanePort } from "./paperclip-control-plane-port.js";
|
||||
import { nativeSha256 } from "./canonical.js";
|
||||
import { NativeRunCoordinatorStore } from "./native-run-coordinator-store.js";
|
||||
import { finalizeNativeRun, repairCommittedNativeChatResponse } from "./native-run-finalizer.js";
|
||||
import { nativeRuntimeContextFixture } from "./runtime-context.test-fixture.js";
|
||||
@@ -313,6 +314,63 @@ describe("PaperclipControlPlanePort conformance", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("publishes native recovery metadata in the existing snapshot transaction and fences stale generations", async () => {
|
||||
const identity = { ...CONTROL_PLANE_CONFORMANCE_OPEN.identity, runId: randomUUID(), sessionId: randomUUID() };
|
||||
const runnerId = randomUUID();
|
||||
const owner = { ...identity, authorityRunId: identity.runId, bootId: randomUUID(), generation: 1, leaseOwner: "snapshot-owner" };
|
||||
await db.insert(heartbeatRuns).values({ id: identity.runId, companyId: identity.companyId, agentId: identity.agentId,
|
||||
status: "running", runtimeMode: "native", nativeIssueId: identity.issueId, nativeSessionId: identity.sessionId,
|
||||
runnerInstanceId: runnerId, completionContractId: contractId, completionContractSha256: contractSha,
|
||||
runnerProfileJson: { unrelated: "preserve" } });
|
||||
await db.insert(nativeRunFinalizations).values({ runId: identity.runId, companyId: identity.companyId, issueId: identity.issueId,
|
||||
phase: "observed",
|
||||
controllerBootId: owner.bootId, controllerGeneration: owner.generation, leaseOwner: owner.leaseOwner,
|
||||
leaseExpiresAt: new Date(Date.now() + 60_000) });
|
||||
const port = new PaperclipControlPlanePort(db, { ...identity, completionContractId: contractId,
|
||||
completionContractSha256: contractSha, sourceInstanceId: runnerId, controlPlaneSourceInstanceId: "snapshot-control" }, {
|
||||
nativeRecovery: { owner, compatibility: { configDigest: `sha256:${"a".repeat(64)}`, processConfigDigest: null } },
|
||||
});
|
||||
await port.openRun({ identity, backendKind: "mock", sourceInstanceId: runnerId });
|
||||
const snapshot = { backendKind: "mock" as const, sessionId: identity.sessionId, identity };
|
||||
const statements: string[] = [];
|
||||
const previousDebug = db.$client.options.debug;
|
||||
db.$client.options.debug = (_connection, statement) => { statements.push(statement); };
|
||||
try { await port.checkpointSession(snapshot); }
|
||||
finally { db.$client.options.debug = previousDebug; }
|
||||
expect(statements.filter(statement => /^begin\b/i.test(statement.trim()))).toHaveLength(1);
|
||||
expect(statements.filter(statement => /^update\s+"heartbeat_runs"/i.test(statement.trim()))).toHaveLength(1);
|
||||
expect(statements.filter(statement => /^(?:with|select)\b/i.test(statement.trim()))).toHaveLength(1);
|
||||
const [saved] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, identity.runId));
|
||||
expect(saved.runnerProfileJson).toMatchObject({ unrelated: "preserve", sessionCheckpoint: snapshot,
|
||||
nativeControllerRecovery: { schema: "paperclip.native-controller-recovery.v1", snapshotSha256: nativeSha256(saved.runnerProfileJson?.sessionCheckpoint),
|
||||
checkpoint: null, processConfigDigest: null } });
|
||||
// Missing PRP is an explicit incomplete publication, never fabricated authority.
|
||||
await db.update(nativeRunFinalizations).set({ controllerGeneration: 2 }).where(eq(nativeRunFinalizations.runId, identity.runId));
|
||||
await expect(port.checkpointSession({ ...snapshot, providerSessionId: "stale" })).rejects.toThrow("checkpoint_owner_lost");
|
||||
await expect(port.completeRun({ result: CONTROL_PLANE_CONFORMANCE_RESULT, terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL }))
|
||||
.rejects.toThrow("checkpoint_owner_lost");
|
||||
const [unchanged] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, identity.runId));
|
||||
expect(unchanged.runnerProfileJson).toEqual(saved.runnerProfileJson);
|
||||
// A valid coordinator lock does not certify a lease that expires while the
|
||||
// subsequent run-row lock waits. Recheck time at the actual write boundary.
|
||||
await db.update(nativeRunFinalizations).set({ controllerGeneration: 1, leaseExpiresAt: new Date(Date.now() + 150) })
|
||||
.where(eq(nativeRunFinalizations.runId, identity.runId));
|
||||
let locked!: () => void, release!: () => void;
|
||||
const acquired = new Promise<void>(resolve => { locked = resolve; });
|
||||
const held = new Promise<void>(resolve => { release = resolve; });
|
||||
const blocker = db.transaction(async tx => {
|
||||
await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(eq(heartbeatRuns.id, identity.runId)).for("update");
|
||||
locked();
|
||||
await held;
|
||||
});
|
||||
await acquired;
|
||||
const waiting = port.checkpointSession(snapshot);
|
||||
const rejected = expect(waiting).rejects.toThrow("checkpoint_owner_lost");
|
||||
try { await new Promise(resolve => setTimeout(resolve, 200)); }
|
||||
finally { release(); await blocker; }
|
||||
await rejected;
|
||||
});
|
||||
|
||||
it("permits only read-only admission while an existing startup drains", async () => {
|
||||
const identity = { ...CONTROL_PLANE_CONFORMANCE_OPEN.identity, runId: randomUUID(), sessionId: randomUUID() };
|
||||
const runnerId = randomUUID();
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { notifyNativeResultCommit } from "../native-result-commit-notifications.js";
|
||||
import { notifyNativeWorkspaceFinalization } from "../native-workspace-finalization-notifications.js";
|
||||
import { createAgentIdentityRedactor } from "../agent-identity-redaction.js";
|
||||
import { and, asc, eq, gt, or } from "drizzle-orm";
|
||||
import { and, asc, eq, gt, or, sql } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import {
|
||||
heartbeatRunEvents,
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
issues,
|
||||
nativeRunFinalizations,
|
||||
nativeRunResults,
|
||||
nativeRunnerCheckpoints,
|
||||
} from "@paperclipai/db";
|
||||
import type {
|
||||
CompleteControlPlaneRunInput,
|
||||
@@ -30,6 +31,10 @@ import { appendHeartbeatRunEvent } from "../heartbeat-run-events.js";
|
||||
import { publishLiveEvent } from "../live-events.js";
|
||||
import { nativeSha256 } from "./canonical.js";
|
||||
import { redactRunnerApiEvent, redactRunnerApiPayloads } from "./runner-api-log-redaction.js";
|
||||
import { createNativeControllerRecovery, NATIVE_CONTROLLER_RECOVERY_MAX_BYTES, type NativeControllerCompatibility } from "./native-controller-recovery.js";
|
||||
import { eligibleNativeTaskSessionPublication, publishNativeTaskSessionInTransaction, NATIVE_TASK_SESSION_PUBLICATION_KEY,
|
||||
type NativeTaskSessionPublicationContext, type NativeTaskSessionPublicationIssue } from "./native-task-session-publication.js";
|
||||
import { lockNativeRunnerCheckpointOwner, nativeRunnerCheckpointOwnerLock, validateNativeRunnerCheckpointOwner, inspectNativeRunnerUpgradeHeader, type NativeRunnerCheckpointOwner } from "./native-runner-checkpoint.js";
|
||||
|
||||
export interface PaperclipControlPlaneBinding {
|
||||
companyId: string;
|
||||
@@ -90,6 +95,9 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
|
||||
readonly #onEventPersistence?: (event: PrpEvent, startedAtMs: number, endedAtMs: number) => Promise<void>;
|
||||
readonly #onCommittedEvent?: (event: PrpEvent, cursor: number) => Promise<void>;
|
||||
readonly #onDuplicateEvent?: (event: PrpEvent, cursor: number) => Promise<void>;
|
||||
readonly #nativeRecovery?: { compatibility: NativeControllerCompatibility; owner: NativeRunnerCheckpointOwner };
|
||||
readonly #taskSessionPublication?: NativeTaskSessionPublicationContext;
|
||||
readonly #taskSessionPublicationOwner?: NativeRunnerCheckpointOwner;
|
||||
|
||||
constructor(
|
||||
db: Db,
|
||||
@@ -105,6 +113,9 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
|
||||
onEventPersistence?: (event: PrpEvent, startedAtMs: number, endedAtMs: number) => Promise<void>;
|
||||
onCommittedEvent?: (event: PrpEvent, cursor: number) => Promise<void>;
|
||||
onDuplicateEvent?: (event: PrpEvent, cursor: number) => Promise<void>;
|
||||
nativeRecovery?: { compatibility: NativeControllerCompatibility; owner: NativeRunnerCheckpointOwner };
|
||||
taskSessionPublication?: NativeTaskSessionPublicationContext;
|
||||
taskSessionPublicationOwner?: NativeRunnerCheckpointOwner;
|
||||
} = {},
|
||||
) {
|
||||
this.#db = db;
|
||||
@@ -116,6 +127,9 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
|
||||
this.#onEventPersistence = options.onEventPersistence;
|
||||
this.#onResultCommitted = options.onResultCommitted;
|
||||
this.#onDuplicateEvent = options.onDuplicateEvent;
|
||||
this.#nativeRecovery = options.nativeRecovery;
|
||||
this.#taskSessionPublication = options.taskSessionPublication ? structuredClone(options.taskSessionPublication) : undefined;
|
||||
this.#taskSessionPublicationOwner = options.taskSessionPublicationOwner;
|
||||
}
|
||||
|
||||
#assertActive(options?: { signal: AbortSignal }): void {
|
||||
@@ -123,7 +137,9 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
|
||||
options?.signal.throwIfAborted();
|
||||
}
|
||||
|
||||
#matchesPersistedBinding(run: typeof heartbeatRuns.$inferSelect): boolean {
|
||||
#matchesPersistedBinding(run: Pick<typeof heartbeatRuns.$inferSelect,
|
||||
"companyId" | "agentId" | "runtimeMode" | "nativeIssueId" | "nativeSessionId" |
|
||||
"runnerInstanceId" | "completionContractId" | "completionContractSha256">): boolean {
|
||||
return run.companyId === this.#binding.companyId
|
||||
&& run.agentId === this.#binding.agentId
|
||||
&& run.runtimeMode === "native"
|
||||
@@ -200,19 +216,105 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
|
||||
|| identity.agentId !== this.#binding.agentId
|
||||
|| this.#sessionId !== identity.sessionId
|
||||
) throw new Error("native_session_checkpoint_binding_mismatch");
|
||||
const publication = this.#taskSessionPublication && eligibleNativeTaskSessionPublication(snapshot)
|
||||
? this.#taskSessionPublication : null;
|
||||
if (publication && !this.#taskSessionPublicationOwner) throw new Error("native_task_session_publication_owner_required");
|
||||
await this.#db.transaction(async (tx) => {
|
||||
const run = await tx.select().from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, this.#binding.runId)).for("update").limit(1)
|
||||
const recovery = this.#nativeRecovery;
|
||||
const owner = recovery?.owner ?? (publication ? this.#taskSessionPublicationOwner : undefined);
|
||||
if (publication && recovery && nativeSha256(recovery.owner) !== nativeSha256(this.#taskSessionPublicationOwner)) {
|
||||
throw new Error("native_task_session_publication_owner_mismatch");
|
||||
}
|
||||
const ownerLock = owner ? nativeRunnerCheckpointOwnerLock(tx, owner) : null;
|
||||
// NO KEY UPDATE serializes reset/admission fields while remaining compatible
|
||||
// with completeRun's parent KEY SHARE. Keep coordinator → issue → run order.
|
||||
const publicationIssue = publication ? tx.$with("publication_issue").as(tx.select({
|
||||
id: issues.id, companyId: issues.companyId, executionRunId: issues.executionRunId,
|
||||
conversationAgentId: issues.conversationAgentId, conversationSessionGeneration: issues.conversationSessionGeneration,
|
||||
}).from(issues).where(and(eq(issues.id, this.#binding.issueId), eq(issues.companyId, this.#binding.companyId),
|
||||
ownerLock ? sql`exists (select 1 from ${ownerLock})` : undefined)).for("no key update")) : null;
|
||||
const lockedRun = tx.$with("snapshot_run").as(tx.select({
|
||||
id: heartbeatRuns.id, status: heartbeatRuns.status,
|
||||
contextSnapshot: (publication ? sql<Record<string, unknown>>`jsonb_build_object('conversationSessionGeneration', ${heartbeatRuns.contextSnapshot}->'conversationSessionGeneration')` : sql<null>`null`).as("publication_context"),
|
||||
resultJson: (publication ? sql<Record<string, unknown>>`jsonb_build_object('nativeCancellation', ${heartbeatRuns.resultJson}->'nativeCancellation',
|
||||
'cancellation', ${heartbeatRuns.resultJson}->'cancellation', 'startupCancellation', ${heartbeatRuns.resultJson}->'startupCancellation')` : sql<null>`null`).as("publication_cancellation"),
|
||||
startedAt: heartbeatRuns.startedAt, createdAt: heartbeatRuns.createdAt,
|
||||
companyId: heartbeatRuns.companyId, agentId: heartbeatRuns.agentId,
|
||||
runtimeMode: heartbeatRuns.runtimeMode, nativeIssueId: heartbeatRuns.nativeIssueId,
|
||||
nativeSessionId: heartbeatRuns.nativeSessionId, runnerInstanceId: heartbeatRuns.runnerInstanceId,
|
||||
completionContractId: heartbeatRuns.completionContractId,
|
||||
completionContractSha256: heartbeatRuns.completionContractSha256,
|
||||
runnerProfileJson: heartbeatRuns.runnerProfileJson,
|
||||
}).from(heartbeatRuns)
|
||||
.where(and(eq(heartbeatRuns.id, this.#binding.runId), ownerLock
|
||||
? sql`exists (select 1 from ${ownerLock})` : undefined,
|
||||
publicationIssue ? sql`exists (select 1 from ${publicationIssue})` : undefined)).for("update"));
|
||||
// Project the locking CTE's tuples, including any update committed while
|
||||
// the lock waited. A second scan of heartbeat_runs could see an old MVCC tuple.
|
||||
// The run CTE depends on the coordinator CTE: both locks precede the
|
||||
// outer clock sample, including when either lock waits.
|
||||
const selected = await tx.with(...(ownerLock ? [ownerLock] : []), ...(publicationIssue ? [publicationIssue] : []), lockedRun).select({ run: {
|
||||
id: lockedRun.id, status: lockedRun.status, contextSnapshot: lockedRun.contextSnapshot, resultJson: lockedRun.resultJson,
|
||||
startedAt: lockedRun.startedAt, createdAt: lockedRun.createdAt,
|
||||
companyId: lockedRun.companyId, agentId: lockedRun.agentId,
|
||||
runtimeMode: lockedRun.runtimeMode, nativeIssueId: lockedRun.nativeIssueId,
|
||||
nativeSessionId: lockedRun.nativeSessionId, runnerInstanceId: lockedRun.runnerInstanceId,
|
||||
completionContractId: lockedRun.completionContractId,
|
||||
completionContractSha256: lockedRun.completionContractSha256,
|
||||
runnerProfileJson: lockedRun.runnerProfileJson,
|
||||
}, now: sql<Date>`clock_timestamp()`, ...(ownerLock ? {
|
||||
leaseOwner: sql<string | null>`(select ${ownerLock.leaseOwner} from ${ownerLock})`,
|
||||
leaseExpiresAt: sql<Date | null>`(select ${ownerLock.leaseExpiresAt} from ${ownerLock})`.mapWith(nativeRunFinalizations.leaseExpiresAt),
|
||||
resultId: sql<string | null>`(select ${ownerLock.resultId} from ${ownerLock})`,
|
||||
} : {}), ...(recovery ? {
|
||||
checkpointHeader: sql<Record<string, unknown> | null>`(select material from ${nativeRunnerCheckpoints}
|
||||
where ${nativeRunnerCheckpoints.runId} = ${this.#binding.runId}
|
||||
and ${nativeRunnerCheckpoints.companyId} = ${this.#binding.companyId}
|
||||
and ${nativeRunnerCheckpoints.issueId} = ${this.#binding.issueId}
|
||||
and ${nativeRunnerCheckpoints.part} = 'header')`,
|
||||
} : {}), ...(publicationIssue ? { publicationIssue: sql<NativeTaskSessionPublicationIssue>`(select jsonb_build_object(
|
||||
'id', ${publicationIssue.id}, 'companyId', ${publicationIssue.companyId}, 'executionRunId', ${publicationIssue.executionRunId},
|
||||
'conversationAgentId', ${publicationIssue.conversationAgentId}, 'conversationSessionGeneration', ${publicationIssue.conversationSessionGeneration}) from ${publicationIssue})` } : {}),
|
||||
}).from(lockedRun).limit(1)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
const run = selected?.run;
|
||||
this.#assertActive(options);
|
||||
if (owner && !selected) throw new Error("native_runner_checkpoint_owner_lost");
|
||||
if (!run || !this.#matchesPersistedBinding(run)) {
|
||||
throw new Error("native_session_checkpoint_binding_mismatch");
|
||||
}
|
||||
await tx.update(heartbeatRuns).set({
|
||||
runnerProfileJson: {
|
||||
// The run row lock may have waited beyond our lease. Generation cannot
|
||||
// change while we hold the coordinator, but wall-clock expiry still can.
|
||||
if (owner && selected) await validateNativeRunnerCheckpointOwner(tx, owner,
|
||||
{ leaseOwner: selected.leaseOwner ?? null, leaseExpiresAt: selected.leaseExpiresAt ?? null,
|
||||
resultId: selected.resultId ?? null, now: selected.now }, !snapshot.semanticResult);
|
||||
const header = recovery && selected?.checkpointHeader
|
||||
? inspectNativeRunnerUpgradeHeader(this.#binding, selected.checkpointHeader) : null;
|
||||
if (header && (header.normalizedSessionId !== identity.sessionId || header.runnerInstanceId !== this.#binding.sourceInstanceId)) {
|
||||
throw new Error("native_session_checkpoint_prp_binding_mismatch");
|
||||
}
|
||||
const profile: Record<string, unknown> = {
|
||||
...record(run.runnerProfileJson),
|
||||
sessionCheckpoint: structuredClone(snapshot) as unknown as Record<string, unknown>,
|
||||
},
|
||||
...(recovery ? { nativeControllerRecovery: createNativeControllerRecovery(snapshot, recovery.compatibility,
|
||||
header ? { runId: this.#binding.runId, runnerInstanceId: header.runnerInstanceId,
|
||||
normalizedSessionId: header.normalizedSessionId } : null) } : {}),
|
||||
};
|
||||
// Driver/live snapshots may already describe a terminal result before
|
||||
// completeRun accepts it. Persist that observation without a handled
|
||||
// receipt; the existing post-acceptance checkpoint publishes the pointer.
|
||||
if (publication && selected?.resultId) {
|
||||
const receipt = await publishNativeTaskSessionInTransaction(tx, { binding: this.#binding, context: publication,
|
||||
snapshot, run, issue: selected.publicationIssue ?? null, resultId: selected.resultId ?? null,
|
||||
recoveryReady: !recovery || header !== null });
|
||||
profile[NATIVE_TASK_SESSION_PUBLICATION_KEY] = receipt;
|
||||
if (Buffer.byteLength(JSON.stringify({ nativeControllerRecovery: profile.nativeControllerRecovery,
|
||||
[NATIVE_TASK_SESSION_PUBLICATION_KEY]: receipt })) > NATIVE_CONTROLLER_RECOVERY_MAX_BYTES) {
|
||||
throw new Error("native_task_session_publication_metadata_too_large");
|
||||
}
|
||||
}
|
||||
await tx.update(heartbeatRuns).set({
|
||||
runnerProfileJson: profile,
|
||||
sessionIdAfter: snapshot.providerSessionId ?? snapshot.sessionId,
|
||||
nativePhase: snapshot.semanticResult ? "workspace_finalizing" : "observed",
|
||||
nativePhaseUpdatedAt: new Date(),
|
||||
@@ -386,9 +488,12 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
|
||||
});
|
||||
|
||||
const insertedResult = await this.#db.transaction(async (tx) => {
|
||||
// Result insertion checks the task foreign key. Acquire that parent lock
|
||||
// before the run, matching task mutations that subsequently update a run.
|
||||
// Otherwise concurrent chat/status writes can deadlock after generation.
|
||||
// Status commit and snapshot publication lock coordinator → issue → run.
|
||||
// A replay must follow that order too: holding the issue while waiting
|
||||
// for the coordinator deadlocks a status commit's issue UPDATE lock.
|
||||
const recovery = this.#nativeRecovery;
|
||||
const owner = recovery ? await lockNativeRunnerCheckpointOwner(tx, recovery.owner) : null;
|
||||
// Result insertion checks the task FK. Keep its parent lock before run.
|
||||
await tx.select({ id: issues.id }).from(issues)
|
||||
.where(and(eq(issues.id, this.#binding.issueId), eq(issues.companyId, this.#binding.companyId)))
|
||||
.for("key share");
|
||||
@@ -399,6 +504,10 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
|
||||
if (!run || !this.#matchesPersistedBinding(run)) {
|
||||
throw new Error("native_result_binding_mismatch");
|
||||
}
|
||||
if (recovery && owner) {
|
||||
const [clock] = await tx.execute<{ now: Date }>(sql`select clock_timestamp() as now`);
|
||||
await validateNativeRunnerCheckpointOwner(tx, recovery.owner, { ...owner, now: clock.now }, false);
|
||||
}
|
||||
|
||||
const callerConditions = [eq(nativeRunResults.serverFingerprint, serverFingerprint)];
|
||||
if (value.callerResultId) callerConditions.push(eq(nativeRunResults.callerResultId, value.callerResultId));
|
||||
@@ -411,6 +520,7 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
|
||||
if (existing.canonicalSha256 !== canonicalSha256) throw new Error("structured_result_replay_conflict");
|
||||
return false;
|
||||
}
|
||||
if (owner?.resultId) throw new Error("structured_result_replay_conflict");
|
||||
const resultJson = {
|
||||
result: validated.result,
|
||||
terminal: value.terminal,
|
||||
|
||||
@@ -120,7 +120,7 @@ describe("PaperclipRunnerToolAuthority", () => {
|
||||
issueId,
|
||||
runId,
|
||||
});
|
||||
expect(authority.definitions()).toHaveLength(50);
|
||||
expect(authority.definitions()).toHaveLength(51);
|
||||
expect(authority.definitions().map(tool => tool.name)).toEqual(expect.arrayContaining([
|
||||
"get_workspace", "list_workspaces", "select_workspace", "prepare_repository",
|
||||
]));
|
||||
|
||||
@@ -28,6 +28,7 @@ import { createReadStream } from "node:fs";
|
||||
import { publicChatTaskUrl } from "../chat-task-url.js";
|
||||
import { agentCommentaryToolInputSchema } from "@paperclipai/shared";
|
||||
import { submitAgentCommentary } from "../agent-commentary.js";
|
||||
import { sendAgentChatMessage } from "../agent-chat-messages.js";
|
||||
import type { createAssignedMcpTools } from "./assigned-mcp-tools.js";
|
||||
import { captureRunnerToolResult, fitRunnerToolResultPage, READ_TOOL_RESULT, READ_TOOL_RESULT_DEFINITION, readToolResultSchema, RUNNER_TOOL_RESULT_MAX_BYTES } from "./runner-tool-results.js";
|
||||
import { assertAssignableAgent } from "../agent-assignability.js";
|
||||
@@ -126,6 +127,7 @@ import {
|
||||
|
||||
const EXTERNAL_WORKSPACE_OPERATIONS = new Set(["get_workspace", "list_workspaces", "select_workspace", "prepare_repository", "register_deliverable"]);
|
||||
const IMPLEMENTED_OPERATIONS = new Set([
|
||||
"send_chat_message",
|
||||
"submit_complaint", "submit_suggestion",
|
||||
"read_agent_instructions", "update_agent_instructions", "get_agent_instruction_history", "restore_agent_instructions",
|
||||
"search_api", "call_api", "hire_agent",
|
||||
@@ -434,6 +436,7 @@ export class PaperclipRunnerToolAuthority {
|
||||
// retry, without ever falling through to an uncertain mutation.
|
||||
let recovered: unknown;
|
||||
if (call.tool === "set_task_title") recovered = await this.#setTaskTitle(input, true);
|
||||
else if (call.tool === "send_chat_message") recovered = await this.#sendChatMessage(input, this.#privacyActor(run), true);
|
||||
else if (call.tool === "submit_complaint" || call.tool === "submit_suggestion") recovered = await this.#submitCommentary(call.tool, input, true);
|
||||
else if (call.tool === "create_task") recovered = await this.#createTask(input, null, true);
|
||||
else if (call.tool === "request_human_input") recovered = await this.#requestHumanInput(input, null, true);
|
||||
@@ -959,6 +962,7 @@ export class PaperclipRunnerToolAuthority {
|
||||
return { approval, tasks: tasks.filter(row => !readable || readable.has(row.issue.id)).map(row => row.issue) };
|
||||
}
|
||||
case "report_progress": return this.#reportProgress(input);
|
||||
case "send_chat_message": return this.#sendChatMessage(input, this.#privacyActor(context.run));
|
||||
case "request_human_input": return this.#requestHumanInput(input,
|
||||
(await captureRunIdentity(this.db, this.binding, { readScope: "identity" })).context?.id ?? null);
|
||||
case "create_task": return this.#createTask(input,
|
||||
@@ -1486,6 +1490,14 @@ export class PaperclipRunnerToolAuthority {
|
||||
return { interactionId: review.interaction.id, status: expectedStatus };
|
||||
}
|
||||
|
||||
async #sendChatMessage(input: Record<string, unknown>, actor: AuthorizationActor, receiptOnly = false) {
|
||||
const result = await sendAgentChatMessage(this.db, { ...this.binding, keyScope: actor.keyScope,
|
||||
onBehalfOfUserId: actor.onBehalfOfUserId, authorizingUserId: actor.authorizingUserId, source: actor.source }, input, {
|
||||
authorizeNative: tx => this.#lockAuthorizedMutationContext(tx), receiptOnly,
|
||||
});
|
||||
return result ?? undefined;
|
||||
}
|
||||
|
||||
async #reportProgress(input: Record<string, unknown>): Promise<unknown> {
|
||||
const body = typeof input.body === "string" ? input.body.trim() : "";
|
||||
const idempotencyKey = typeof input.idempotencyKey === "string" ? input.idempotencyKey.trim() : "";
|
||||
|
||||
@@ -91,6 +91,23 @@ it.skipIf(!tar || !hasZstd)("reuses only byte-matched image files, keeps modes,
|
||||
expect(await fs.readlink(path.join(targetPath, "tool"))).toBe("bin/tool");
|
||||
});
|
||||
|
||||
it.skipIf(!tar || !hasZstd)("stages a complete layered pack into a fresh nested runtime directory", async () => {
|
||||
const { root, pack, archive } = await layeredFixture();
|
||||
const remote = await runner(root);
|
||||
const workspace = path.join(root, "workspace");
|
||||
await fs.mkdir(workspace);
|
||||
const targetPath = path.join(workspace, ".paperclip-runtime", "paperclip-runner", "provider-pack");
|
||||
await expect(fs.lstat(path.dirname(targetPath))).rejects.toMatchObject({ code: "ENOENT" });
|
||||
expect(await stageProviderPackArchive({ archive, runner: remote, targetPath, reuseRoots: [pack] })).toBe(true);
|
||||
expect(await fs.readFile(path.join(targetPath, "bin/large"))).toEqual(await fs.readFile(path.join(pack, "bin/large")));
|
||||
expect(await fs.readFile(path.join(targetPath, "provider-pack.json"))).toEqual(await fs.readFile(path.join(pack, "provider-pack.json")));
|
||||
expect(await fs.readlink(path.join(targetPath, "tool"))).toBe("bin/tool");
|
||||
for (const directory of [targetPath, path.dirname(targetPath), path.dirname(path.dirname(targetPath))]) {
|
||||
expect((await fs.stat(directory)).mode & 0o777).toBe(0o700);
|
||||
}
|
||||
expect((await fs.stat(path.join(targetPath, "bin/large"))).ino).not.toBe((await fs.stat(path.join(pack, "bin/large"))).ino);
|
||||
});
|
||||
|
||||
it.skipIf(!tar || !hasZstd).each(["missing", "wrong-digest", "symlink", "ancestor-symlink"])("transfers a large layer when the image file is %s", async reason => {
|
||||
const { root, pack, archive } = await layeredFixture();
|
||||
const remote = await runner(root);
|
||||
@@ -135,13 +152,18 @@ it.skipIf(!tar || !hasZstd)("rejects a corrupt missing layer before transfer and
|
||||
expect(remote.syncIn).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.skipIf(!tar || !hasZstd)("does not follow a layered target symlink or accept invalid reuse receipts", async () => {
|
||||
it.skipIf(!tar || !hasZstd).each(["directory", "file", "symlink"])("rejects an existing layered target %s or invalid reuse receipts", async kind => {
|
||||
const { root, archive } = await layeredFixture();
|
||||
const remote = await runner(root);
|
||||
const targetPath = path.join(root, "target"), other = path.join(root, "other");
|
||||
await fs.mkdir(other); await fs.symlink(other, targetPath);
|
||||
await fs.mkdir(other);
|
||||
if (kind === "directory") await fs.mkdir(targetPath);
|
||||
else if (kind === "file") await fs.writeFile(targetPath, "untouched");
|
||||
else await fs.symlink(other, targetPath);
|
||||
await expect(stageProviderPackArchive({ archive, runner: remote, targetPath })).rejects.toThrow("runner_remote_provider_archive_reuse_failed");
|
||||
expect(await fs.readdir(other)).toEqual([]);
|
||||
if (kind === "directory") expect(await fs.readdir(targetPath)).toEqual([]);
|
||||
if (kind === "file") expect(await fs.readFile(targetPath, "utf8")).toBe("untouched");
|
||||
remote.execute.mockResolvedValue({ exitCode: 0, timedOut: false, stdout: "[999999]", stderr: "", signal: null, pid: null, startedAt: null });
|
||||
await expect(stageProviderPackArchive({ archive, runner: remote, targetPath: `${targetPath}-2` })).rejects.toThrow("runner_remote_provider_archive_reuse_failed");
|
||||
expect(remote.syncIn).not.toHaveBeenCalled();
|
||||
|
||||
@@ -70,7 +70,8 @@ const reuseScript = `
|
||||
(async()=>{
|
||||
const fs=require('node:fs'),p=require('node:path'),c=require('node:crypto');
|
||||
const spec=JSON.parse(Buffer.from(process.argv[1],'base64').toString('utf8'));
|
||||
const f=fs.promises;await f.mkdir(spec.target,{mode:448});
|
||||
const f=fs.promises;await f.mkdir(p.dirname(spec.target),{recursive:true,mode:448});
|
||||
await f.mkdir(spec.target,{mode:448});
|
||||
const missing=[];
|
||||
const regularPath=async(root,relative)=>{
|
||||
let current=root;if(!(await f.lstat(current)).isDirectory())return false;
|
||||
|
||||
@@ -18,7 +18,7 @@ import {
|
||||
AGENTCORE_QUALIFIED_MODEL,
|
||||
CLAUDE_MANAGED_QUALIFIED_MODEL,
|
||||
} from "../provider-profile-qualification.js";
|
||||
import { resolveAcpxQualification, resolveAntigravityCandidateAdmission, type AcpxQualificationCandidate } from "./acpx-qualification.js";
|
||||
import { resolveAcpxQualification, resolveAntigravityCandidateAdmission, resolveKimiCandidateAdmission, type AcpxQualificationCandidate } from "./acpx-qualification.js";
|
||||
import { compatibleCodexModel } from "./codex-model-fallback.js";
|
||||
|
||||
export const QUALIFIED_OPENCODE_RUNNER_VERSION = "1.18.34" as const;
|
||||
@@ -34,6 +34,7 @@ export const DEFAULT_ACPX_RUNNER_MODELS = {
|
||||
cursor: null,
|
||||
copilot: null,
|
||||
pi: null,
|
||||
kimi: null,
|
||||
} as const;
|
||||
|
||||
export type QualifiedPaperclipRunnerAcpxAgent =
|
||||
@@ -564,6 +565,15 @@ export function resolvePaperclipRunnerProviderProfile(
|
||||
throw new PaperclipRunnerProviderProfileError("paperclip_runner_acpx_qualification_invalid", "Antigravity candidate admission requires the authorized company, exact model and runtime identity.");
|
||||
}
|
||||
}
|
||||
if (pendingAcpxProfile.value === "kimi") {
|
||||
try {
|
||||
if (resolveKimiCandidateAdmission({ companyId: context.companyId, model }, process.env)) {
|
||||
return { provider: "acpx", backend: "acpx_runtime", model, acpxAgent: "kimi" };
|
||||
}
|
||||
} catch {
|
||||
throw new PaperclipRunnerProviderProfileError("paperclip_runner_acpx_qualification_invalid", "Kimi candidate admission requires the authorized company, exact model, target and runtime identity.");
|
||||
}
|
||||
}
|
||||
// Qualification authority belongs to the isolated server operator. Never
|
||||
// read this allowlist from adapter config, credential refs, or run env.
|
||||
let qualification: AcpxQualificationCandidate | undefined;
|
||||
@@ -577,10 +587,10 @@ export function resolvePaperclipRunnerProviderProfile(
|
||||
}
|
||||
throw new PaperclipRunnerProviderProfileError("paperclip_runner_acpx_agent_unavailable", `${pendingAcpxProfile.label} is awaiting native Runner acceptance. Its profile is not enabled for production runs.`);
|
||||
}
|
||||
if ((acpxAgent === "cursor" || acpxAgent === "pi") && !model) {
|
||||
if ((acpxAgent === "cursor" || acpxAgent === "pi" || acpxAgent === "kimi") && !model) {
|
||||
throw new PaperclipRunnerProviderProfileError("paperclip_runner_acpx_model_required", "This ACPX agent requires an explicit model ID; there is no default model.");
|
||||
}
|
||||
if (acpxAgent !== "claude" && acpxAgent !== "codex" && acpxAgent !== "grok" && acpxAgent !== "cursor" && acpxAgent !== "copilot" && acpxAgent !== "pi") {
|
||||
if (acpxAgent !== "claude" && acpxAgent !== "codex" && acpxAgent !== "grok" && acpxAgent !== "cursor" && acpxAgent !== "copilot" && acpxAgent !== "pi" && acpxAgent !== "kimi") {
|
||||
throw new PaperclipRunnerProviderProfileError(
|
||||
"paperclip_runner_acpx_agent_unavailable",
|
||||
"Paperclip Runner ACPX requires a qualified agent profile.",
|
||||
|
||||
@@ -10,7 +10,7 @@ export function shouldUseBundledAcpxImageAssets(input: {
|
||||
}): boolean {
|
||||
return input.requiresRemoteProviderPack && !input.configuredProviderPackRoot &&
|
||||
input.provider.kind === "acpx" &&
|
||||
["cursor", "copilot"].includes(input.provider.agent);
|
||||
["cursor", "copilot", "kimi"].includes(input.provider.agent);
|
||||
}
|
||||
|
||||
type VerificationCommandResult = {
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { isSettledRemoteWarmControlState } from "./remote-warm-session-recovery.js";
|
||||
|
||||
const identity = { runId: "run", normalizedSessionId: "session", runnerInstanceId: "runner", environmentLeaseId: "lease", turnId: "turn", itemId: "item" };
|
||||
const terminal = { eventType: "run.terminal", envelope: { ...identity } };
|
||||
const goal = { schema: "paperclip.session_goal.snapshot.v1", goal: null, workingNow: false };
|
||||
const notification = { schema: "paperclip.prp.event.v1", eventType: "session.goal.snapshot", runId: identity.runId,
|
||||
normalizedSessionId: identity.normalizedSessionId, sourceInstanceId: "runner", sourceKind: "runner", payload: goal };
|
||||
const snapshot = { eventType: "session.goal.snapshot", envelope: { ...identity, payload: notification } };
|
||||
const state = (events: unknown[]) => ({ identity, commands: [{ status: "completed" }], committedEvents: events });
|
||||
|
||||
describe("settled remote warm control evidence", () => {
|
||||
it.each(["paperclip.prp.event.v1", "paperclip.prp.event.v2"])("accepts the terminal and an authenticated %s idle goal-read suffix without changing its order", schema => {
|
||||
const suffix = { ...snapshot, envelope: { ...snapshot.envelope, payload: { ...notification, schema } } };
|
||||
const control = state([terminal, suffix]);
|
||||
expect(isSettledRemoteWarmControlState(control, identity)).toBe(true);
|
||||
expect(control.committedEvents).toEqual([terminal, suffix]);
|
||||
expect(isSettledRemoteWarmControlState(state([terminal]), identity)).toBe(true);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["active goal", { ...snapshot, envelope: { ...snapshot.envelope, payload: { ...notification, payload: { ...goal, goal: { status: "active" } } } } }],
|
||||
["provider work", { ...snapshot, envelope: { ...snapshot.envelope, payload: { ...notification, payload: { ...goal, workingNow: true } } } }],
|
||||
["unknown event", { ...snapshot, eventType: "unknown" }],
|
||||
["unknown protocol version", { ...snapshot, envelope: { ...snapshot.envelope, payload: { ...notification, schema: "paperclip.prp.event.v3" } } }],
|
||||
["new turn", { ...snapshot, eventType: "turn.started" }],
|
||||
["foreign run", { ...snapshot, envelope: { ...snapshot.envelope, runId: "other" } }],
|
||||
["foreign session", { ...snapshot, envelope: { ...snapshot.envelope, normalizedSessionId: "other" } }],
|
||||
["foreign source", { ...snapshot, envelope: { ...snapshot.envelope, payload: { ...notification, sourceInstanceId: "other" } } }],
|
||||
["foreign lease", { ...snapshot, envelope: { ...snapshot.envelope, environmentLeaseId: "other" } }],
|
||||
["foreign notification", { ...snapshot, envelope: { ...snapshot.envelope, payload: { ...notification, runId: "other" } } }],
|
||||
["missing payload", { ...snapshot, envelope: { ...identity } }],
|
||||
])("holds a %s suffix", (_name, suffix) => {
|
||||
expect(isSettledRemoteWarmControlState(state([terminal, suffix]), identity)).toBe(false);
|
||||
});
|
||||
|
||||
it("holds missing terminals, pending commands and mixed core identities", () => {
|
||||
expect(isSettledRemoteWarmControlState(state([snapshot]), identity)).toBe(false);
|
||||
expect(isSettledRemoteWarmControlState({ ...state([terminal, snapshot]), commands: [{ status: "dispatched" }] }, identity)).toBe(false);
|
||||
expect(isSettledRemoteWarmControlState({ ...state([terminal, snapshot]), identity: { ...identity, runnerInstanceId: "other" } }, identity)).toBe(false);
|
||||
expect(isSettledRemoteWarmControlState(state([{ ...terminal, envelope: { ...identity, runId: "other" } }, snapshot]), identity)).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -3,6 +3,7 @@ import { agents, companies, environmentLeases, heartbeatRuns, issues, nativeRunF
|
||||
import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target";
|
||||
import { parseNativeExecutionInput, type NativeExecutionInput, type PersistedNativeSession } from "../../vendor/paperclip-runner/index.js";
|
||||
import { currentNativeControllerIdentity, evaluateNativeControllerTakeover } from "./native-restart-recovery.js";
|
||||
import { readNativeControllerRecovery } from "./native-controller-recovery.js";
|
||||
|
||||
export type RemoteWarmSessionRecovery = {
|
||||
identity: Record<string, string>;
|
||||
@@ -13,6 +14,36 @@ export type RemoteWarmSessionRecovery = {
|
||||
revalidate: (leaseOwner: string) => Promise<void>;
|
||||
};
|
||||
|
||||
/** A goal read can publish an idle snapshot after the terminal. It carries no
|
||||
* new work; every other suffix still requires reconciliation before handoff. */
|
||||
export function isSettledRemoteWarmControlState(control: Record<string, unknown>, identity: Record<string, string>): boolean {
|
||||
const record = (value: unknown): Record<string, unknown> =>
|
||||
value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||||
const commands = control.commands;
|
||||
const events = control.committedEvents;
|
||||
if (!Array.isArray(commands) || commands.some(command => record(command).status !== "completed") ||
|
||||
!Array.isArray(events) || Object.entries(identity).some(([key, value]) => record(control.identity)[key] !== value)) return false;
|
||||
const terminalIndex = events.findLastIndex(event => record(event).eventType === "run.terminal");
|
||||
if (terminalIndex < 0) return false;
|
||||
const matchesSession = (event: unknown) => {
|
||||
const envelope = record(record(event).envelope);
|
||||
return envelope.runId === identity.runId && envelope.normalizedSessionId === identity.normalizedSessionId;
|
||||
};
|
||||
if (!matchesSession(events[terminalIndex])) return false;
|
||||
return events.slice(terminalIndex + 1).every(event => {
|
||||
const envelope = record(record(event).envelope);
|
||||
const notification = record(envelope.payload);
|
||||
const payload = record(notification.payload);
|
||||
return record(event).eventType === "session.goal.snapshot" && matchesSession(event) &&
|
||||
Object.entries(identity).every(([key, value]) => envelope[key] === value) &&
|
||||
(notification.schema === "paperclip.prp.event.v1" || notification.schema === "paperclip.prp.event.v2") &&
|
||||
notification.eventType === "session.goal.snapshot" &&
|
||||
notification.sourceInstanceId === identity.runnerInstanceId && notification.sourceKind === "runner" &&
|
||||
notification.runId === identity.runId && notification.normalizedSessionId === identity.normalizedSessionId &&
|
||||
payload.schema === "paperclip.session_goal.snapshot.v1" && payload.goal === null && payload.workingNow === false;
|
||||
});
|
||||
}
|
||||
|
||||
export async function inspectRemoteWarmSessionRecovery(input: {
|
||||
db: Db;
|
||||
execution: NativeExecutionInput;
|
||||
@@ -52,15 +83,14 @@ export async function inspectRemoteWarmSessionRecovery(input: {
|
||||
checkpoint.driverKind !== previous.session.driverKind || !checkpoint.providerSessionId ||
|
||||
checkpoint.activeTurnId !== null || checkpoint.pendingRuntimeRequests?.length !== 0) return deny();
|
||||
const control = input.readControlState();
|
||||
const commands = control.commands;
|
||||
const events = control.committedEvents;
|
||||
const terminal = Array.isArray(events) ? events.at(-1) : undefined;
|
||||
if (!Array.isArray(commands) || commands.some(command => command.status !== "completed") ||
|
||||
terminal?.eventType !== "run.terminal" || terminal?.envelope?.runId !== identity.runId ||
|
||||
terminal?.envelope?.normalizedSessionId !== identity.normalizedSessionId ||
|
||||
Object.entries(identity).some(([key, value]) => (control.identity as Record<string, unknown>)?.[key] !== value)) return deny();
|
||||
if (!isSettledRemoteWarmControlState(control, identity)) return deny();
|
||||
const assertPriorStopped = async () => {
|
||||
const stopped = await evaluateNativeControllerTakeover({ owner: prior.owner, now: new Date(), requireStoppedController: true });
|
||||
const persistedRecovery = readNativeControllerRecovery(prior.run.runnerProfileJson?.nativeControllerRecovery,
|
||||
prior.run.runnerProfileJson?.sessionCheckpoint);
|
||||
// A PID in another webserver container cannot prove life or death. New
|
||||
// durable controllers are fenced by the database lease and current turn.
|
||||
const stopped = await evaluateNativeControllerTakeover({ owner: prior.owner, now: new Date(), requireStoppedController: true,
|
||||
remoteLeaseFenced: persistedRecovery !== null });
|
||||
if (!stopped.allowed) return deny();
|
||||
};
|
||||
await assertPriorStopped();
|
||||
|
||||
@@ -41,6 +41,7 @@ function words(text: string): string[] {
|
||||
}
|
||||
|
||||
function dedicatedTools(method: string, path: string): string[] {
|
||||
if (method === "POST" && path === "/api/agents/me/chat/messages") return ["send_chat_message"];
|
||||
if (/\/companies\/\{[^}]+\}\/workflows$/.test(path)) return method === "GET" ? ["list_workflows"] : method === "POST" ? ["create_workflow"] : [];
|
||||
if (/\/workflows\/\{[^}]+\}$/.test(path)) return method === "GET" ? ["read_workflow"] : method === "PATCH" ? ["update_workflow"] : [];
|
||||
if (/\/issues\/\{[^}]+\}\/title$/.test(path) && method === "PUT") return ["set_task_title"];
|
||||
|
||||
Reference in new issue
Block a user