diff --git a/doc/DATABASE.md b/doc/DATABASE.md index f4fbdea8348..5b1ecb4c9b1 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -2516,6 +2516,14 @@ error in `cause`, and is logged with both. A run that fails this way keeps the generic `adapter_failed` code, so existing retry paths apply, and records that bounded message with the SQLSTATE as its failure cause. +Prompt-injection classifier admission also uses READ COMMITTED. Its company, +project, configuration, agent, run and secret share locks retain the admission +fences. Run status and execution identity are read under the run lock, so a +concurrent run-event append cannot invalidate an earlier snapshot and discard +a connector result before classification. The budget check, invocation and +reservation still commit together. Classifier settlement keeps its separate +repeatable-read transaction and receipt replay rules. + Upgrade all controllers/workers to this accounting contract and drain incompatible pre-upgrade database transactions before enabling independent writers. Do not run pre-upgrade synchronous counter/evaluation writers beside unlocked writers. diff --git a/server/src/__tests__/prompt-injection-gateway.test.ts b/server/src/__tests__/prompt-injection-gateway.test.ts index f4a6cf4b6af..098f71a435c 100644 --- a/server/src/__tests__/prompt-injection-gateway.test.ts +++ b/server/src/__tests__/prompt-injection-gateway.test.ts @@ -20,6 +20,7 @@ import { providerUsageReceipt } from "../services/provider-usage.js"; import { adapterExecutionControls, createAdapterExecutionControl, registerAdapterExecutionControl } from "../services/adapter-execution-control.js"; import type { PluginToolDispatcher } from "../services/plugin-tool-dispatcher.js"; import { promptInjectionCorpus } from "./fixtures/prompt-injection-corpus.js"; +import { appendHeartbeatRunEvent } from "../services/heartbeat-run-events.js"; let db: ReturnType, database: Awaited>, home: string; beforeAll(async () => { home = await realpath(await mkdtemp(path.join(os.tmpdir(), "paperclip-guard-gateway-"))); @@ -106,7 +107,7 @@ async function createRemoteMcpToolFixture(db: ReturnType, compa const benign = promptInjectionCorpus[0]!.content; const safe = { verdict: { violation: false, findings: [] }, receipt: { ...providerUsageReceipt(), inputTokens: 100, outputTokens: 10, costStatus: "reported", costCents: "0.0020000", pricingProvenance: { source: "provider_reported" } } }; -async function fixture() { +async function fixture(gatewayDb?: (run: typeof heartbeatRuns.$inferSelect) => typeof db) { const f = await createRunFixture(db); await db.insert(companyMemberships).values({ companyId: f.company.id, principalType: "user", principalId: "alice", membershipRole: "owner", status: "active" }); const binding = await aiConnectionService(db).save(f.company.id, "alice", { provider: "openrouter", method: "api_key", name: "Classifier", ownership: "shared", apiKey: "fixture-key", agentIds: [], allAgents: true }, "fixture-key"); @@ -121,7 +122,7 @@ async function fixture() { ? { protocolVersion: "2025-06-18", capabilities: { tools: {} }, serverInfo: { name: "fixture", version: "1" } } : { content: [{ type: "text", text: benign }] } }); }); - const gateway = createToolGatewayService(db, { promptInjectionProvider: classifier, remoteHttpRequest, toolActionSigningSecret: "signing-fixture" }); + const gateway = createToolGatewayService(gatewayDb?.(f.run) ?? db, { promptInjectionProvider: classifier, remoteHttpRequest, toolActionSigningSecret: "signing-fixture" }); const session = await gateway.createSession({ companyId: f.company.id, agentId: f.agent.id, runId: f.run.id }); const tool = (await gateway.listToolsForSession(session.token)).find(t => t.connectionId === remote.connection.id)!; return { ...f, remote, classifier, remoteHttpRequest, gateway, session, tool }; @@ -134,7 +135,88 @@ async function evidence(companyId: string, toolInvocationId: string) { const events = await db.select().from(toolCallEvents).where(eq(toolCallEvents.invocationId, toolInvocationId)); expect(events).toContainEqual(expect.objectContaining({ eventType: "call_completed", metadata: expect.objectContaining({ promptInjection: expect.objectContaining({ detector: "model", invocationId: invocations[0].id }) }) })); } + +/** Commit a real competing write after each accounting snapshot starts. + * The admission must still read current authority when it obtains its locks. */ +function interleaveAccountingSnapshot(advance: () => Promise) { + return new Proxy(db, { get(target, key) { + if (key !== "transaction") return Reflect.get(target, key); + return (callback: Parameters[0], options: Parameters[1]) => { + let first = Boolean(options?.isolationLevel); + return target.transaction(tx => callback(new Proxy(tx, { get(transaction, property) { + if (property !== "select") return Reflect.get(transaction, property); + return (...fields: Parameters) => { + const builder = transaction.select(...fields), from = builder.from.bind(builder); + builder.from = ((...args: Parameters) => { + const query = from(...args); + if (args[0] === companies && first) { + first = false; + const execute = query.execute.bind(query); + query.execute = async (...executeArgs: Parameters) => { + const rows = await execute(...executeArgs); + await advance(); + return rows; + }; + } + return query; + }) as typeof builder.from; + return builder; + }; + } })), options); + }; + } }); +} describe("model classification at gateway result boundaries", () => { + it("returns a screened connector result while run events advance the admission snapshot", async () => { + let interleavings = 0; + const f = await fixture(run => interleaveAccountingSnapshot(async () => { + await appendHeartbeatRunEvent(db, { companyId: run.companyId, runId: run.id, + agentId: run.agentId, eventType: "lifecycle", message: "stream advanced during classifier admission" }); + interleavings++; + })); + const result = await f.gateway.executeTool({ sessionToken: f.session.token, tool: f.tool.name, parameters: {} }); + expect(JSON.stringify(result.result)).toContain("secret.txt"); + expect(interleavings).toBeGreaterThan(0); + expect(f.classifier).toHaveBeenCalledOnce(); + await evidence(f.company.id, result.invocationId); + const holds = await db.select().from(budgetReservations).where(eq(budgetReservations.companyId, f.company.id)); + expect(holds).toHaveLength(1); + expect(holds[0].state).toBe("settled"); + expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toHaveLength(1); + const [current] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)); + expect(current.nextEventSeq).toBeGreaterThan(f.run.nextEventSeq); + }); + it.each(["stopped", "steered"] as const)("refuses classifier dispatch when the run is %s after the admission snapshot", async change => { + const f = await fixture(run => interleaveAccountingSnapshot(async () => { + await db.update(heartbeatRuns).set(change === "stopped" ? { status: "cancelled" } : { responsibleUserId: "alice" }) + .where(eq(heartbeatRuns.id, run.id)); + })); + await expect(f.gateway.executeTool({ sessionToken: f.session.token, tool: f.tool.name, parameters: {} })).rejects.toMatchObject({ + status: 422, reasonCode: "prompt_injection_blocked", + }); + expect(f.classifier).not.toHaveBeenCalled(); + expect(await db.select().from(promptInjectionInvocations).where(eq(promptInjectionInvocations.companyId, f.company.id))).toHaveLength(0); + expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toHaveLength(0); + }); + it("screens concurrent connector reads and saves each charge once while run events advance", async () => { + const f = await fixture(run => interleaveAccountingSnapshot(async () => { + await appendHeartbeatRunEvent(db, { companyId: run.companyId, runId: run.id, agentId: run.agentId, + eventType: "lifecycle", message: "parallel connector progress" }); + })); + const results = await Promise.all(Array.from({ length: 4 }, () => f.gateway.executeTool({ + sessionToken: f.session.token, tool: f.tool.name, parameters: {}, + }))); + expect(f.classifier).toHaveBeenCalledTimes(4); + expect(new Set(results.map(result => result.invocationId)).size).toBe(4); + for (const result of results) { + expect(JSON.stringify(result.result)).toContain("secret.txt"); + await evidence(f.company.id, result.invocationId); + } + const holds = await db.select().from(budgetReservations).where(eq(budgetReservations.companyId, f.company.id)); + expect(holds).toHaveLength(4); + expect(holds.every(hold => hold.state === "settled")).toBe(true); + expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toHaveLength(4); + }); it.each([ ["direct", false], ["test", false], ["approved", false], ["direct", true], ["test", true], ["approved", true], diff --git a/server/src/services/prompt-injection-prevention.ts b/server/src/services/prompt-injection-prevention.ts index fbd140b6d49..6957ba44320 100644 --- a/server/src/services/prompt-injection-prevention.ts +++ b/server/src/services/prompt-injection-prevention.ts @@ -195,6 +195,9 @@ export function promptInjectionPreventionService(db: Db, options: { try { credential = await aiConnectionService(db).credential(selected.connection, 0, { responsibleUserId: selected.userId, actorType: selected.actorType, actorId: selected.actorId, issueId: selected.issueId, heartbeatRunId: selected.runId }); } catch (error) { const reason = unavailableReason(error); if (reason) return { status: "unavailable", reason }; throw error; } + // The shared admission fences and run lock protect the live authority + // checks below. A snapshot taken before the run lock would go stale on + // every streamed run-event append, losing an otherwise valid tool result. const admission = await withOperationAccountingTransaction(db, context.companyId, async (tx, publications) => { const [company] = await tx.select({ status: companies.status }).from(companies) .where(eq(companies.id, context.companyId)).for("share"); @@ -244,7 +247,7 @@ export function promptInjectionPreventionService(db: Db, options: { projectId: current.projectId, amountCents: unitsToCents(amount), providerStartedAt: invocation.startedAt }); await queueBudgetEvaluation(tx, { companyId: context.companyId, agentId: current.agentId, projectId: current.projectId }); return { invocation }; - }); + }, { isolationLevel: "read committed" }); if (admission.reason) return { status: "unavailable", reason: admission.reason }; const invocation = admission.invocation!; let outcome: PromptInjectionProviderOutcome;