diff --git a/server/src/app.ts b/server/src/app.ts index e9acf5e28a9..a3682ea2a26 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -98,6 +98,7 @@ import { readinessRoutes } from "./routes/ready.js"; import { livenessRoutes } from "./routes/liveness.js"; import { cloudRuntimeIdentityMiddleware } from "./middleware/cloud-runtime-identity.js"; import { cloudControlMiddleware } from "./middleware/cloud-control.js"; +import { runnerCallOutcomeMiddleware } from "./middleware/runner-call-outcome.js"; import { cloudRoutes } from "./routes/cloud.js"; import { companyRoutes } from "./routes/companies.js"; import { companySkillRoutes } from "./routes/company-skills.js"; @@ -623,6 +624,7 @@ export async function createApp( // REPLACES whatever actor the request otherwise resolved to, and only on // the one endpoint it authorizes (see the middleware for the contract). app.use(cloudControlMiddleware()); + app.use(runnerCallOutcomeMiddleware(db)); app.use(connectionEventWakeRoutes(db)); app.use(skillAccessRequestContext); app.use("/api/auth", authRoutes(db)); diff --git a/server/src/middleware/runner-call-outcome.test.ts b/server/src/middleware/runner-call-outcome.test.ts new file mode 100644 index 00000000000..961e047b8c5 --- /dev/null +++ b/server/src/middleware/runner-call-outcome.test.ts @@ -0,0 +1,54 @@ +import type { Request, Response } from "express"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { Db } from "@paperclipai/db"; + +const persistActivity = vi.hoisted(() => vi.fn(async () => ({}))); +vi.mock("../services/activity-log.js", () => ({ persistActivity })); + +import { runnerCallOutcomeMiddleware } from "./runner-call-outcome.js"; + +const runId = "11111111-2222-4333-8444-555555555555"; +const callKey = "d".repeat(64); +const agent = { type: "agent", source: "agent_jwt", agentId: "agent-a", companyId: "company-a", runId } as const; + +function invoke(actor: unknown, options: { method?: string; key?: string | null } = {}) { + const key = options.key === undefined ? callKey : options.key; + const end = vi.fn(); + const req = { method: options.method ?? "PATCH", actor, + header: (name: string) => (name.toLowerCase() === "x-paperclip-runner-call" ? key ?? undefined : undefined) } as unknown as Request; + const res = { statusCode: 200, end } as unknown as Response; + const next = vi.fn(); + runnerCallOutcomeMiddleware({} as Db)(req, res, next); + return { res, end, next }; +} + +describe("runner call outcome", () => { + beforeEach(() => persistActivity.mockClear()); + + it.each([201, 422, 500])("records the final status %s of an agent run's keyed mutation once", (status) => { + const { res, end, next } = invoke(agent); + expect(next).toHaveBeenCalledOnce(); + res.statusCode = status; + res.end(); + res.end(); + expect(end).toHaveBeenCalledTimes(2); + expect(persistActivity).toHaveBeenCalledExactlyOnceWith({}, { companyId: "company-a", actorType: "agent", actorId: "agent-a", + agentId: "agent-a", runId, action: "runner.api_completed", entityType: "heartbeat_run", entityId: runId, + details: { callKey, status } }); + }); + + it.each([ + ["a read", agent, { method: "GET" }], + ["a missing call key", agent, { key: null }], + ["a malformed call key", agent, { key: "not-a-key" }], + ["an agent API key", { ...agent, source: "agent_key" }, {}], + ["a board actor", { type: "board", source: "session", runId }, {}], + ["a non-UUID run", { ...agent, runId: "run-a" }, {}], + ] as const)("ignores %s", (_case, actor, options) => { + const { res, end, next } = invoke(actor, options); + expect(next).toHaveBeenCalledOnce(); + res.end(); + expect(end).toHaveBeenCalledOnce(); + expect(persistActivity).not.toHaveBeenCalled(); + }); +}); diff --git a/server/src/middleware/runner-call-outcome.ts b/server/src/middleware/runner-call-outcome.ts new file mode 100644 index 00000000000..5a6ced45e16 --- /dev/null +++ b/server/src/middleware/runner-call-outcome.ts @@ -0,0 +1,41 @@ +import type { RequestHandler } from "express"; +import type { Db } from "@paperclipai/db"; +import { RUNNER_API_CALL_HEADER } from "../services/native-runtime/runner-api-client.js"; +import { persistActivity } from "../services/activity-log.js"; +import { RUNNER_API_COMPLETED_ACTION } from "../services/recovery-effect-receipts.js"; +import { logger } from "./logger.js"; + +const CALL_KEY = /^[0-9a-f]{64}$/; +const RUN_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +const READ_METHODS = new Set(["GET", "HEAD", "OPTIONS"]); + +export async function recordRunnerApiOutcome(db: Db, input: { + companyId: string; agentId: string; runId: string; callKey: string; status: number; +}) { + await persistActivity(db, { companyId: input.companyId, actorType: "agent", actorId: input.agentId, + agentId: input.agentId, runId: input.runId, action: RUNNER_API_COMPLETED_ACTION, + entityType: "heartbeat_run", entityId: input.runId, details: { callKey: input.callKey, status: input.status } }); +} + +export function runnerCallOutcomeMiddleware(db: Db): RequestHandler { + return (req, res, next) => { + const callKey = req.header(RUNNER_API_CALL_HEADER); + const { type, source, agentId, companyId, runId } = req.actor ?? {}; + if (!callKey || !CALL_KEY.test(callKey) || READ_METHODS.has(req.method) || type !== "agent" + || source !== "agent_jwt" || !agentId || !companyId || !runId || !RUN_ID.test(runId)) { + next(); + return; + } + const end = res.end; + let recorded = false; + res.end = function (this: typeof res, ...args: unknown[]) { + if (!recorded) { + recorded = true; + recordRunnerApiOutcome(db, { companyId, agentId, runId, callKey, status: res.statusCode }) + .catch((err) => logger.warn({ err, runId }, "Failed to record a runner API call outcome")); + } + return (end as (...endArgs: unknown[]) => typeof res).apply(this, args); + } as typeof res.end; + next(); + }; +} diff --git a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts index e4a5bd44dcf..61f790329f4 100644 --- a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts +++ b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts @@ -1205,11 +1205,11 @@ export class PaperclipRunnerToolAuthority { if (!apiUrl) throw new Error("Paperclip API origin is unavailable"); const token = this.#apiToken(bound); if (!token) throw new Error("Paperclip run authentication is unavailable"); - const execute = async () => { + const execute = async (callKey?: string) => { const current = await this.#boundContext(); assertAvailable(current.issue.workMode); const result = await executeRunnerApi(input, { ...context, workMode: current.issue.workMode }, { - apiUrl, token, + apiUrl, token, callKey, beforeDispatch: async () => { const fresh = await this.#boundContext(); assertAvailable(fresh.issue.workMode); @@ -1253,7 +1253,7 @@ export class PaperclipRunnerToolAuthority { return null; }); if (prior) return prior.result; - const result = await execute(); + const result = await execute(key); const storedResult = provider ? { encryptedResult: (await provider.createSecret({ value: JSON.stringify(result) })).material } : { result }; diff --git a/server/src/services/native-runtime/runner-api-client.ts b/server/src/services/native-runtime/runner-api-client.ts index 0d4ae28abe2..65c4b6e4009 100644 --- a/server/src/services/native-runtime/runner-api-client.ts +++ b/server/src/services/native-runtime/runner-api-client.ts @@ -36,9 +36,12 @@ export interface RunnerApiContext { runId: string; workMode: string; } +export const RUNNER_API_CALL_HEADER = "X-Paperclip-Runner-Call"; + export interface RunnerApiIo { apiUrl: string; token: string; + callKey?: string; fetch?: typeof fetch; /** Server-owned page size for the dedicated saved-result reader. */ responseTextPageBytes?: number; @@ -209,6 +212,7 @@ export async function executeRunnerApi(input: RunnerApiCall, context: RunnerApiC const url = runnerApiUrl(operation, input, context, io.apiUrl); if (!io.token) throw new Error("Paperclip run authentication is unavailable"); const headers = new Headers({ Authorization: `Bearer ${io.token}`, "X-Paperclip-Run-Id": context.runId }); + if (io.callKey && !["GET", "HEAD", "OPTIONS"].includes(operation.method)) headers.set(RUNNER_API_CALL_HEADER, io.callKey); const textOffset = input.responseText?.offsetBytes ?? 0; const textLimit = input.responseText?.limitBytes ?? (input.responseText ? io.responseTextPageBytes : undefined) ?? RUNNER_API_INLINE_BYTES; diff --git a/server/src/services/native-runtime/runner-api.test.ts b/server/src/services/native-runtime/runner-api.test.ts index 2d9cebb96a4..6e83f4a2c66 100644 --- a/server/src/services/native-runtime/runner-api.test.ts +++ b/server/src/services/native-runtime/runner-api.test.ts @@ -7,6 +7,7 @@ import { import { executeRunnerApi, readBoundedResponse, + RUNNER_API_CALL_HEADER, runnerApiUrl, validateRunnerApiCall, type RunnerApiCall, @@ -79,6 +80,15 @@ describe("capped list pagination", () => { }); }); +describe("runner call identity", () => { + it.each([[createProject, "call-key-a"], [projects, null]] as const)("sends the call key only with mutations: %s", async (operationId, expected) => { + const fetcher = vi.fn(async (_url: string | URL | Request, _init?: RequestInit) => new Response("{}", { headers: { "content-type": "application/json" } })); + await executeRunnerApi({ operationId, ...(operationId === createProject ? { body: { name: "Keyed" } } : {}) }, context, + { ...io(fetcher as unknown as typeof fetch), callKey: "call-key-a" }); + expect(new Headers(fetcher.mock.calls[0]![1]!.headers).get(RUNNER_API_CALL_HEADER)).toBe(expected); + }); +}); + describe("bounded response capture receipts", () => { it.each([projects, createProject])("reports oversize evidence without retrying %s", async operationId => { const fetcher = vi.fn(async () => new Response("", { headers: { "content-length": String(RUNNER_API_RESPONSE_MAX_BYTES + 1) } })); diff --git a/server/src/services/recovery-effect-receipts.ts b/server/src/services/recovery-effect-receipts.ts index a9456768dc3..241954c64ec 100644 --- a/server/src/services/recovery-effect-receipts.ts +++ b/server/src/services/recovery-effect-receipts.ts @@ -1,11 +1,36 @@ -import { and, eq, sql, ne, or, inArray } from "drizzle-orm"; -import { heartbeatRuns, toolInvocations, type Db } from "@paperclipai/db"; +import { and, desc, eq, gt, sql, ne, or, inArray } from "drizzle-orm"; +import { activityLog, heartbeatRuns, toolInvocations, type Db } from "@paperclipai/db"; import { getSecretProvider } from "../secrets/provider-registry.js"; import { recoveryRunAncestry } from "./recovery-run-ancestry.js"; const record = (value: unknown): Record => value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +export const RUNNER_API_COMPLETED_ACTION = "runner.api_completed"; + +const UNKNOWN_API_OUTCOME = { ok: false, status: null, error: "api_outcome_unknown", outcome: "unknown", + guidance: "An earlier attempt of this operation may already have applied. Read the current state, and do not repeat the change if it is already there. Independent work and reads remain available." }; + +async function settleUnknownApiReceipt(db: Db, input: { companyId: string; issueId: string }, + source: { id: string; finishedAt: Date | null }, callKey: string, +): Promise<"fenced" | "superseded" | "rejected" | { status: number }> { + const [outcome] = await db.select({ details: activityLog.details }).from(activityLog).where(and( + eq(activityLog.companyId, input.companyId), eq(activityLog.runId, source.id), + eq(activityLog.action, RUNNER_API_COMPLETED_ACTION), sql`${activityLog.details}->>'callKey' = ${callKey}`, + )).orderBy(desc(activityLog.createdAt)).limit(1); + const status = record(outcome?.details).status; + if (typeof status === "number" && status >= 200 && status < 300) return { status }; + if (typeof status === "number" && status >= 400 && status < 500 && status !== 408) return "rejected"; + if (!source.finishedAt) return "fenced"; + const [human] = await db.select({ id: activityLog.id }).from(activityLog).where(and( + eq(activityLog.companyId, input.companyId), eq(activityLog.entityType, "issue"), + eq(activityLog.entityId, input.issueId), eq(activityLog.actorType, "user"), + inArray(activityLog.action, ["issue.updated", "issue.comment_added"]), + gt(activityLog.createdAt, source.finishedAt), + )).limit(1); + return human ? "superseded" : "fenced"; +} + /** Generic upstream tools have no common authoritative result-query protocol. * Preserve their uncertain mutation fence across recovery. Exact publication * protocols can reconcile through their existing owner instead. No transcript @@ -44,7 +69,8 @@ export async function priorRecoveryApiReceipt(db: Db, input: { issueId: input.issueId, sourceRunIds: [input.runId] })) .map(source => source.id).filter(id => id !== input.runId)); if (!ancestry.size) return null; - const sources = await db.select({ id: heartbeatRuns.id, result: heartbeatRuns.resultJson }).from(heartbeatRuns).where(and( + const sources = await db.select({ id: heartbeatRuns.id, result: heartbeatRuns.resultJson, + finishedAt: heartbeatRuns.finishedAt }).from(heartbeatRuns).where(and( eq(heartbeatRuns.companyId, input.companyId), sql`coalesce(${heartbeatRuns.nativeIssueId}::text, ${heartbeatRuns.issueId}::text, ${heartbeatRuns.contextSnapshot}->>'issueId') = ${input.issueId}`, @@ -53,23 +79,28 @@ export async function priorRecoveryApiReceipt(db: Db, input: { sql`${heartbeatRuns.createdAt} < (select created_at from heartbeat_runs where id = ${input.runId})`))); let completed: Record | null = null; for (const source of sources) { - for (const receipt of Object.values(record(source.result?.apiToolReceipts)).map(record)) { + for (const [callKey, value] of Object.entries(record(source.result?.apiToolReceipts))) { + const receipt = record(value); if (receipt.operationId !== input.operationId) continue; const result = receipt.encryptedResult ? record(JSON.parse(await getSecretProvider("local_encrypted").resolveVersion({ material: record(receipt.encryptedResult), externalRef: null }))) : record(receipt.result); if (receipt.state !== "completed" || result.outcome === "unknown" || result.error === "api_outcome_unknown") { - return { state: "pending", result: { ok: false, status: null, error: "api_outcome_unknown", - outcome: "unknown", guidance: "Reconcile the existing operation before repeating it. Independent work and reads remain available." } }; + const settlement = await settleUnknownApiReceipt(db, input, source, callKey); + if (settlement === "fenced") return { state: "pending", result: UNKNOWN_API_OUTCOME }; + if (typeof settlement === "object" && ancestry.has(source.id) && receipt.digest === input.digest) { + completed = { digest: receipt.digest, operationId: receipt.operationId, state: "completed", + result: { ok: true, status: settlement.status, operationId: receipt.operationId, outcome: "applied", + guidance: "An earlier attempt of this exact call was applied. Read the current state for its result." } }; + } + continue; } // A completed HTTP request is not necessarily a completed effect. A // definite client rejection may be retried after the user fixes its cause. // Preserve the transport owner's uncertainty classification above. if (result.ok === false && typeof result.status === "number" && result.status >= 400 && result.status < 500 && result.status !== 408) continue; - if (result.ok !== true) return { state: "pending", result: { ok: false, status: null, - error: "api_outcome_unknown", outcome: "unknown", - guidance: "Reconcile the existing operation before repeating it. Independent work and reads remain available." } }; + if (result.ok !== true) return { state: "pending", result: UNKNOWN_API_OUTCOME }; // Successful effects belong to their original intent, not every later // request on this task. Uncertain effects above retain their wider fence. if (ancestry.has(source.id) && receipt.digest === input.digest) completed = receipt; diff --git a/server/src/services/task-retry.test.ts b/server/src/services/task-retry.test.ts index cae346c9192..d26cfd34475 100644 --- a/server/src/services/task-retry.test.ts +++ b/server/src/services/task-retry.test.ts @@ -1,9 +1,13 @@ import { randomUUID } from "node:crypto"; +import type { AddressInfo } from "node:net"; +import express from "express"; import { and, eq, sql } from "drizzle-orm"; -import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { activityLog, agents, agentTaskSessions, agentWakeupRequests, chatActions, chatConversations, chatEndpoints, companies, budgetPolicies, costEvents, createDb, type Db, environmentLeases, environments, executionWorkspaces, executionWorkspaceSnapshots, workspaceOperations, heartbeatRunEvents, heartbeatRuns, issueComments, issueRelations, issues, approvals, issueApprovals, issueRecoveryActions, nativeRunFinalizations, toolApplications, toolConnections, toolInvocations, applyPendingMigrations } from "@paperclipai/db"; import { startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; import { priorRecoveryApiReceipt, unresolvedRecoveryToolInvocation } from "./recovery-effect-receipts.js"; +import { recordRunnerApiOutcome, runnerCallOutcomeMiddleware } from "../middleware/runner-call-outcome.js"; +import { RUNNER_API_CALL_HEADER } from "./native-runtime/runner-api-client.js"; import { requestTaskRetry } from "./task-retry.js"; import { HttpError } from "../errors.js"; import { NativeChatReviewPresentationContentionError } from "./native-runtime/native-chat-review-presentation.js"; @@ -1165,6 +1169,99 @@ describe("durable task Retry admission", () => { expect(await priorRecoveryApiReceipt(db, call)).toBeNull(); }); + it.each([[200, "applied"], [422, "rejected"], [500, "fenced"], [null, "fenced"]] as const)( + "settles an uncertain API receipt from the recorded server outcome %s of the same call", async (status, expected) => { + const { input, retry, automatic } = await automaticSuccessorOfTaskRetry(); + const callKey = "a".repeat(64); + const receipt = { operationId: "PATCH /api/issues/{id}", digest: "same-update", state: "completed", + result: { ok: false, status: null, error: "api_transport_failure", outcome: "unknown" } }; + await db.update(heartbeatRuns).set({ resultJson: { apiToolReceipts: { [callKey]: receipt } } }).where(eq(heartbeatRuns.id, retry.id)); + const outcome = { companyId: input.companyId, agentId: retry.agentId, runId: retry.id }; + if (status === null) await recordRunnerApiOutcome(db, { ...outcome, callKey: "b".repeat(64), status: 200 }); + else await recordRunnerApiOutcome(db, { ...outcome, callKey, status }); + const call = { companyId: input.companyId, issueId: input.issueId, runId: automatic.id, + operationId: receipt.operationId, digest: receipt.digest }; + const another = { ...call, digest: "another-update" }; + if (expected === "applied") { + expect(await priorRecoveryApiReceipt(db, call)).toMatchObject({ state: "completed", + result: { ok: true, status: 200, outcome: "applied" } }); + expect(await priorRecoveryApiReceipt(db, another)).toBeNull(); + } else if (expected === "rejected") { + expect(await priorRecoveryApiReceipt(db, call)).toBeNull(); + } else { + expect(await priorRecoveryApiReceipt(db, another)).toMatchObject({ state: "pending", result: { outcome: "unknown" } }); + } + }); + + it("records the final status of a mutation whose client disconnected and settles its receipt without repeating it", async () => { + const { input, retry, automatic } = await automaticSuccessorOfTaskRetry(); + const callKey = "e".repeat(64); + const receipt = { operationId: "PATCH /api/issues/{id}", digest: "late-handoff", state: "completed", + result: { ok: false, status: null, error: "api_transport_failure", outcome: "unknown" } }; + await db.update(heartbeatRuns).set({ resultJson: { apiToolReceipts: { [callKey]: receipt } } }).where(eq(heartbeatRuns.id, retry.id)); + let writes = 0; + let entered!: () => void; + const handlerEntered = new Promise(resolve => { entered = resolve; }); + let releaseHandler!: () => void; + const handlerReleased = new Promise(resolve => { releaseHandler = resolve; }); + const app = express(); + app.use((req, _res, next) => { + req.actor = { type: "agent", source: "agent_jwt", agentId: retry.agentId, companyId: input.companyId, runId: retry.id }; + next(); + }); + app.use(runnerCallOutcomeMiddleware(db)); + app.patch("/slow", async (req, res) => { + const clientGone = new Promise(resolve => req.once("close", () => resolve())); + entered(); + await Promise.all([clientGone, handlerReleased]); + writes += 1; + res.status(200).json({ ok: true }); + }); + const server = app.listen(0, "127.0.0.1"); + await new Promise(resolve => server.once("listening", () => resolve())); + try { + const { port } = server.address() as AddressInfo; + const client = new AbortController(); + const response = fetch(`http://127.0.0.1:${port}/slow`, { method: "PATCH", signal: client.signal, + headers: { [RUNNER_API_CALL_HEADER]: callKey, "content-type": "application/json" }, body: "{}" }); + await handlerEntered; + client.abort(); + await expect(response).rejects.toThrow(); + releaseHandler(); + const outcomes = () => db.select({ details: activityLog.details }).from(activityLog).where(and( + eq(activityLog.runId, retry.id), eq(activityLog.action, "runner.api_completed"))); + await vi.waitFor(async () => expect(await outcomes()).toHaveLength(1), { timeout: 5_000 }); + expect(await outcomes()).toEqual([{ details: { callKey, status: 200 } }]); + expect(writes).toBe(1); + expect(await priorRecoveryApiReceipt(db, { companyId: input.companyId, issueId: input.issueId, runId: automatic.id, + operationId: receipt.operationId, digest: receipt.digest })).toMatchObject({ state: "completed", + result: { ok: true, status: 200, outcome: "applied" } }); + } finally { + await new Promise(resolve => server.close(() => resolve())); + } + }); + + it.each(["user", "agent", "system"] as const)("lifts an uncertain API receipt fence only after a later %s change", async actorType => { + const { input, retry, automatic } = await automaticSuccessorOfTaskRetry(); + const receipt = { operationId: "PATCH /api/issues/{id}", digest: "handoff", state: "completed", + result: { ok: false, status: null, error: "api_transport_failure", outcome: "unknown" } }; + const call = { companyId: input.companyId, issueId: input.issueId, runId: automatic.id, + operationId: receipt.operationId, digest: "reassign" }; + const change = { companyId: input.companyId, actorType, actorId: actorType === "agent" ? retry.agentId : "board", + entityType: "issue", entityId: input.issueId }; + await db.update(heartbeatRuns).set({ resultJson: { apiToolReceipts: { ["c".repeat(64)]: receipt } } }) + .where(eq(heartbeatRuns.id, retry.id)); + await db.insert(activityLog).values({ ...change, action: "issue.comment_added", createdAt: new Date(Date.now() + 5_000) }); + expect(await priorRecoveryApiReceipt(db, call)).toMatchObject({ state: "pending", result: { outcome: "unknown" } }); + const finishedAt = new Date(Date.now() + 10_000); + await db.update(heartbeatRuns).set({ finishedAt }).where(eq(heartbeatRuns.id, retry.id)); + await db.insert(activityLog).values({ ...change, action: "issue.read_marked", createdAt: new Date(finishedAt.getTime() + 1_000) }); + expect(await priorRecoveryApiReceipt(db, call)).toMatchObject({ state: "pending", result: { outcome: "unknown" } }); + await db.insert(activityLog).values({ ...change, action: "issue.updated", createdAt: new Date(finishedAt.getTime() + 2_000) }); + if (actorType === "user") expect(await priorRecoveryApiReceipt(db, call)).toBeNull(); + else expect(await priorRecoveryApiReceipt(db, call)).toMatchObject({ state: "pending", result: { outcome: "unknown" } }); + }); + it("automatic successors retain uncertain upstream mutation fences after task Retry", async () => { const { input, retry, automatic } = await automaticSuccessorOfTaskRetry(); const [application] = await db.insert(toolApplications).values({ companyId: input.companyId,