diff --git a/docs/api/issues.md b/docs/api/issues.md index 209643be83c..4abe8e2b68b 100644 --- a/docs/api/issues.md +++ b/docs/api/issues.md @@ -343,6 +343,36 @@ GET /api/issues/{issueId}/documents GET /api/issues/{issueId}/documents/{key} ``` +### Bulk Get By Key + +``` +GET /api/companies/{companyId}/issue-documents?key=sentry-case&issueIds={uuid1},{uuid2} +``` + +Read one document key across 1–200 task UUIDs. Items follow the requested order; +duplicate IDs appear once. The result is `{ key, items: [{ issueId, document }] }`. +Each document has the same body and revision metadata as Get By Key. A readable +task without the key returns `document: null`. Missing, hidden, foreign-company, +and unreadable tasks are omitted without revealing whether they exist. Callers +must check that every requested ID is present before treating a scan as complete. +Task bridge keys cannot use this company-wide route. Break-glass reads are not supported. + +Use `includeBody=false` for current revision IDs, revision numbers, and update +times without loading document bodies. A helper with cached documents can compare +their revisions, then fetch bodies only for changed or newly added documents. +Task-row timestamps alone do not prove that a document is unchanged. + +Responses use `Cache-Control: private, no-store`. A response over 8 MiB returns +`413`; split the IDs into smaller requests. The response is never truncated. +Body sizes are checked in the database before loading them. Reads use one +snapshot so a concurrent edit cannot grow an admitted body set. At most two +reads per actor and eight per server instance can run at once; extra reads +return `429` with `Retry-After: 1`. +Native agents can discover this GET with `search_api`, save the whole batch with +one `save_api_response`, and transfer it with one `download_artifact`. The saved +artifact supplies its exact `artifactId`, `byteSize`, and `sha256` for the download. +This replaces per-task document relays; helpers must consume the batch result. + ### Create Or Update ``` diff --git a/packages/shared/src/constants.ts b/packages/shared/src/constants.ts index 97621602179..049c9b0cfd2 100644 --- a/packages/shared/src/constants.ts +++ b/packages/shared/src/constants.ts @@ -6,6 +6,8 @@ export const COMPANY_STATUSES = ["active", "paused", "archived"] as const; export const ISSUE_SUMMARIES_MAX_IDS = 50; /** Tasks per chat bootstrap request. The list preloads larger sets in chunks. */ export const ISSUE_CHAT_BOOTSTRAP_MAX_IDS = 10; +/** Tasks per keyed bulk document read; keeps a routine's case inventory in one request. */ +export const ISSUE_DOCUMENT_BATCH_MAX_IDS = 200; export type CompanyStatus = (typeof COMPANY_STATUSES)[number]; export const DEPLOYMENT_MODES = ["local_trusted", "authenticated"] as const; diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index bc4373e3956..414cc5429e2 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -426,6 +426,7 @@ export { ISSUE_DISPLAY_STATUSES, ISSUE_SUMMARIES_MAX_IDS, ISSUE_CHAT_BOOTSTRAP_MAX_IDS, + ISSUE_DOCUMENT_BATCH_MAX_IDS, INBOX_MINE_ISSUE_STATUSES, INBOX_MINE_ISSUE_STATUS_FILTER, ISSUE_PRIORITIES, @@ -1285,6 +1286,7 @@ export type { IssueThreadInteractionResult, IssueDocument, IssueDocumentSummary, + IssueDocumentBatch, DocumentRevision, ExactDocumentRevision, DocumentFormat, @@ -2176,6 +2178,7 @@ export { workspaceFileWorkspaceKindSchema, issueDocumentFormatSchema, issueDocumentKeySchema, + issueDocumentBatchQuerySchema, upsertIssueDocumentSchema, restoreIssueDocumentRevisionSchema, createIssueTreeHoldSchema, diff --git a/packages/shared/src/types/index.ts b/packages/shared/src/types/index.ts index e753d49a5cb..708503f0fe1 100644 --- a/packages/shared/src/types/index.ts +++ b/packages/shared/src/types/index.ts @@ -874,6 +874,7 @@ export type { IssueThreadInteractionResult, IssueDocument, IssueDocumentSummary, + IssueDocumentBatch, DocumentRevision, ExactDocumentRevision, DocumentFormat, diff --git a/packages/shared/src/types/issue.ts b/packages/shared/src/types/issue.ts index d594c0037a8..27ae2558644 100644 --- a/packages/shared/src/types/issue.ts +++ b/packages/shared/src/types/issue.ts @@ -198,6 +198,12 @@ export interface IssueDocument extends IssueDocumentSummary { body: string; } +/** Unreadable or nonexistent tasks are omitted. A readable task without the key has document: null. */ +export interface IssueDocumentBatch { + key: string; + items: Array<{ issueId: string; document: IssueDocument | IssueDocumentSummary | null }>; +} + export interface DocumentRevision { id: string; companyId: string; diff --git a/packages/shared/src/validators/index.ts b/packages/shared/src/validators/index.ts index c8cbf1aea75..450d622613c 100644 --- a/packages/shared/src/validators/index.ts +++ b/packages/shared/src/validators/index.ts @@ -517,6 +517,7 @@ export { createIssueAttachmentMetadataSchema, issueDocumentFormatSchema, issueDocumentKeySchema, + issueDocumentBatchQuerySchema, upsertIssueDocumentSchema, restoreIssueDocumentRevisionSchema, upsertIssueWatchdogSchema, diff --git a/packages/shared/src/validators/issue.test.ts b/packages/shared/src/validators/issue.test.ts index 9a10bf1112d..c66108342e6 100644 --- a/packages/shared/src/validators/issue.test.ts +++ b/packages/shared/src/validators/issue.test.ts @@ -4,6 +4,7 @@ import { addIssueCommentSchema, paperclipQuestionSetPayloadSchema, issueCommentMetadataSchema, + issueDocumentBatchQuerySchema, createIssueSchema, issueBlockedInboxAttentionSchema, resolveIssueRecoveryActionSchema, @@ -16,6 +17,16 @@ import { import { createAgentSchema } from "./agent.js"; describe("issue validators", () => { + it("bounds document batches before deduplication and normalizes keys and IDs", () => { + const id = "AAAAAAAA-AAAA-4AAA-8AAA-AAAAAAAAAAAA"; + expect(issueDocumentBatchQuerySchema.parse({ key: " SENTRY-CASE ", issueIds: ` ${id},${id} ` })).toEqual({ + key: "sentry-case", issueIds: [id.toLowerCase()], includeBody: true, + }); + expect(issueDocumentBatchQuerySchema.parse({ key: "notes", issueIds: Array(200).fill(id).join(","), includeBody: "false" }).includeBody).toBe(false); + expect(issueDocumentBatchQuerySchema.safeParse({ key: "notes", issueIds: Array(201).fill(id).join(",") }).success).toBe(false); + expect(issueDocumentBatchQuerySchema.safeParse({ key: "notes", issueIds: [id] }).success).toBe(false); + }); + it.each(["BASH_ENV", "ZDOTDIR", "GH_CONFIG_DIR", "PAPERCLIP_GITHUB_LAUNCHER_DIR"])("rejects task-owned managed shell selector %s", key => { for (const name of [key, key.toLowerCase()]) { const overrides = {adapterConfig:{env:{[name]:"/workspace/startup"}}}; diff --git a/packages/shared/src/validators/issue.ts b/packages/shared/src/validators/issue.ts index 9877f071ad9..1b016c9453f 100644 --- a/packages/shared/src/validators/issue.ts +++ b/packages/shared/src/validators/issue.ts @@ -2,7 +2,7 @@ import { CHAT_PROVIDERS } from "../types/chat-channels.js"; import { taskWorkspaceSelectionSchema } from "./task-workspace.js"; import { z } from "zod"; import { maxStringLength } from "./text.js"; -import { ISSUE_CHAT_BOOTSTRAP_MAX_IDS, ISSUE_SUMMARIES_MAX_IDS } from "../constants.js"; +import { ISSUE_CHAT_BOOTSTRAP_MAX_IDS, ISSUE_DOCUMENT_BATCH_MAX_IDS, ISSUE_SUMMARIES_MAX_IDS } from "../constants.js"; import { normalizeIssueIdentifier } from "../issue-references.js"; import { @@ -1336,6 +1336,15 @@ export const issueDocumentKeySchema = z "Document key must be lowercase letters, numbers, _ or -", ); +export const issueDocumentBatchQuerySchema = z.object({ + key: z.string().trim().toLowerCase().pipe(issueDocumentKeySchema), + issueIds: z.string().max(ISSUE_DOCUMENT_BATCH_MAX_IDS * 40) + .transform(value => value.split(",").map(id => id.trim().toLowerCase())) + .pipe(z.array(z.string().guid()).min(1).max(ISSUE_DOCUMENT_BATCH_MAX_IDS)) + .transform(ids => [...new Set(ids)]), + includeBody: z.enum(["true", "false"]).default("true").transform(value => value === "true"), +}).strict(); + export const suggestedTaskDraftSchema = z .object({ clientKey: z.string().trim().min(1).max(120), diff --git a/server/src/__tests__/heartbeat-run-event-atomic-append.test.ts b/server/src/__tests__/heartbeat-run-event-atomic-append.test.ts index 71433c1cb0f..c816499013c 100644 --- a/server/src/__tests__/heartbeat-run-event-atomic-append.test.ts +++ b/server/src/__tests__/heartbeat-run-event-atomic-append.test.ts @@ -66,6 +66,42 @@ describeEmbeddedPostgres("atomic ordinary run events", () => { } }); + it("appends ordinary read receipts in one statement with contiguous sequences under a concurrent writer", async () => { + const runId = await makeRun(); + const statements: string[] = []; + const recorded = drizzle(db.$client, { schema: db._.fullSchema, logger: { logQuery: query => statements.push(query) } }) as typeof db; + const inputs = Array.from({ length: 146 }, (_, index) => ({ + companyId, agentId, runId, eventType: "chat_handoff.completed_read", stream: "system", level: "info", + payload: { taskId: `task-${index}`, statusVersion: index, nested: ["before\u0000after"] }, + })); + const [receipts] = await Promise.all([ + appendHeartbeatRunEventBatch(recorded, inputs), + appendHeartbeatRunEvent(otherDb, { companyId, agentId, runId, eventType: "run.performance.span", payload: { concurrent: true } }), + ]); + expect(statements).toHaveLength(1); + expect(receipts).toHaveLength(146); + for (const [index, receipt] of receipts.entries()) { + expect(receipt).toMatchObject({ disposition: "committed", highestContiguousSourceSeq: 0, row: { payload: inputs[index].payload } }); + expect(receipt.row.seq).toBe(receipts[0].row.seq + index); + } + const events = await readEvents(runId); + expect(events.map(event => event.seq)).toEqual(Array.from({ length: 147 }, (_, index) => index + 1)); + expect((await readRun(runId)).nextEventSeq).toBe(148); + }); + + it("refuses ordinary batches with mixed bindings or special event effects before writing", async () => { + const runId = await makeRun(); + const input = { companyId, agentId, runId, eventType: "child_task.read" }; + for (const other of [ + { ...input, runId: "30000000-0000-4000-8000-000000000099" }, + { ...input, companyId: "30000000-0000-4000-8000-000000000099" }, + { ...input, eventType: "native.process_start_requested" }, + ]) await expect(appendHeartbeatRunEventBatch(db, [input, other])).rejects.toThrow("heartbeat_run_event_batch_unsupported"); + expect(await readEvents(runId)).toEqual([]); + expect((await readRun(runId)).nextEventSeq).toBe(1); + expect(await appendHeartbeatRunEventBatch(db, [])).toEqual([]); + }); + it("persists a decoded receipt in one executed statement", async () => { const runId = await makeRun(); const statements: string[] = []; diff --git a/server/src/__tests__/openapi-routes.test.ts b/server/src/__tests__/openapi-routes.test.ts index 72d5dfabb13..7d94265cc17 100644 --- a/server/src/__tests__/openapi-routes.test.ts +++ b/server/src/__tests__/openapi-routes.test.ts @@ -79,6 +79,7 @@ const apiPrefixes: Record = { "instance-server-metrics.ts": "/api", "instance-settings.ts": "/api", "issues.ts": "/api", + "issue-document-batch.ts": "/api", "issue-tree-control.ts": "/api", "llms.ts": "/api", "link-previews.ts": "/api", @@ -1325,6 +1326,18 @@ describe("openapi routes", () => { .toMatchObject({ type: "object", required: ["started"], properties: { started: { type: "boolean" } } }); }); + it("documents bounded bulk keyed document reads for native discovery", () => { + const { spec } = loadSpecRoutes(); + const operation = spec.paths["/api/companies/{companyId}/issue-documents"].get; + expect(operation.parameters).toEqual(expect.arrayContaining([ + expect.objectContaining({ in: "query", name: "key", required: true }), + expect.objectContaining({ in: "query", name: "issueIds", required: true, schema: expect.objectContaining({ type: "string", maxLength: 8000 }) }), + expect.objectContaining({ in: "query", name: "includeBody", required: false, schema: expect.objectContaining({ enum: ["true", "false"] }) }), + ])); + expect(operation.responses["413"]).toBeDefined(); + expect(operation.description).toContain("save_api_response"); + }); + it("covers the mounted server routes exactly", () => { const { routes: actualRoutes, diff --git a/server/src/routes/issue-document-batch.test.ts b/server/src/routes/issue-document-batch.test.ts new file mode 100644 index 00000000000..b307f1c15b3 --- /dev/null +++ b/server/src/routes/issue-document-batch.test.ts @@ -0,0 +1,91 @@ +import express from "express"; +import request from "supertest"; +import type { Db } from "@paperclipai/db"; +import { describe, expect, it, vi } from "vitest"; +import { registerIssueDocumentBatchRoute } from "./issue-document-batch.js"; + +vi.mock("../services/authorization.js", () => ({ issueReadSqlCondition: async () => undefined })); +vi.mock("../services/documents.js", () => ({ + ISSUE_DOCUMENT_BATCH_MAX_BYTES: 8 * 1024 * 1024, + documentService: () => ({ getIssueDocumentsByKey: async () => [] }), +})); + +const companyId = "11111111-1111-4111-8111-111111111111"; +const url = `/api/companies/${companyId}/issue-documents?key=notes&issueIds=22222222-2222-4222-8222-222222222222`; + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (error: unknown) => void; + const promise = new Promise((yes, no) => { resolve = yes; reject = no; }); + return { promise, resolve, reject }; +} + +function fixture() { + const app = express(), router = express.Router(); + const entered: Array<() => void> = []; + const gates: Array>> = []; + const pending: Array> = []; + const db = { select: () => ({ from: () => ({ where: async () => [] }) }) } as unknown as Db; + app.use((req, _res, next) => { + req.actor = { type: "agent", companyId, agentId: req.header("actor")!, source: "agent_key" }; + next(); + }); + registerIssueDocumentBatchRoute(router, db, { + filterReadable: async () => { + const gate = deferred<[]>(); + gates.push(gate); + entered.shift()!(); + return gate.promise; + }, + recordReads: async () => undefined, + }); + app.use("/api", router); + app.use((_error: unknown, _req: express.Request, res: express.Response, _next: express.NextFunction) => { res.status(500).json({ error: "test read failed" }); }); + return { + app, + async start(actor: string) { + const signal = deferred(); + entered.push(signal.resolve); + const response = request(app).get(url).set("actor", actor).then(value => value); + pending.push(response); + await signal.promise; + return { response, gate: gates.at(-1)! }; + }, + async close() { + for (const gate of gates) gate.resolve([]); + await Promise.allSettled(pending); + }, + }; +} + +describe("bulk document read admission", () => { + it("caps each actor and releases failed reads", async () => { + const f = fixture(); + try { + const first = await f.start("one"), second = await f.start("one"); + const refused = await request(f.app).get(url).set("actor", "one"); + expect(refused.status).toBe(429); + expect(refused.headers["retry-after"]).toBe("1"); + first.gate.reject(new Error("read failed")); + expect((await first.response).status).toBe(500); + const replacement = await f.start("one"); + replacement.gate.resolve([]); + second.gate.resolve([]); + expect((await replacement.response).status).toBe(200); + } finally { await f.close(); } + }); + + it("caps total reads across actors", async () => { + const f = fixture(); + try { + const reads = []; + for (let index = 0; index < 8; index++) reads.push(await f.start(`actor-${index}`)); + expect((await request(f.app).get(url).set("actor", "ninth")).status).toBe(429); + reads[0].gate.resolve([]); + await reads[0].response; + const replacement = await f.start("ninth"); + replacement.gate.resolve([]); + expect((await replacement.response).status).toBe(200); + } finally { await f.close(); } + }); +}); diff --git a/server/src/routes/issue-document-batch.ts b/server/src/routes/issue-document-batch.ts new file mode 100644 index 00000000000..e29caf36c42 --- /dev/null +++ b/server/src/routes/issue-document-batch.ts @@ -0,0 +1,96 @@ +import type { Request, Response, Router } from "express"; +import { and, eq, inArray, isNull } from "drizzle-orm"; +import { issues, type Db } from "@paperclipai/db"; +import { issueDocumentBatchQuerySchema } from "@paperclipai/shared"; +import { issueReadSqlCondition } from "../services/authorization.js"; +import { documentService, ISSUE_DOCUMENT_BATCH_MAX_BYTES } from "../services/documents.js"; +import { assertCompanyAccess } from "./authz.js"; + +type ReadableIssue = Pick; + +const MAX_ACTOR_READS = 2; +const MAX_SERVER_READS = 8; + +function createBatchReadAdmission() { + const active = new Map(); + let total = 0; + return (req: Request, res: Response): (() => void) | null => { + const actor = `${req.actor.type}:${req.actor.agentId ?? req.actor.userId ?? req.actor.source}`; + if (total >= MAX_SERVER_READS || (active.get(actor) ?? 0) >= MAX_ACTOR_READS) { + res.setHeader("Retry-After", "1"); + res.status(429).json({ error: "Too many document batch reads; retry after an active read finishes" }); + return null; + } + total++; + active.set(actor, (active.get(actor) ?? 0) + 1); + let workDone = false, responseDone = false, released = false; + const release = () => { + if (released || !workDone || !responseDone) return; + released = true; + total--; + const remaining = active.get(actor)! - 1; + if (remaining) active.set(actor, remaining); + else active.delete(actor); + }; + const finish = () => { responseDone = true; release(); }; + res.once("finish", finish); + res.once("close", finish); + // A disconnect cannot free database capacity while its read is still live. + return () => { workDone = true; release(); }; + }; +} + +export function registerIssueDocumentBatchRoute(router: Router, db: Db, options: { + filterReadable: (req: Request, rows: ReadableIssue[]) => Promise; + recordReads: (req: Request, issues: ReadableIssue[]) => Promise; +}) { + const admit = createBatchReadAdmission(); + router.get("/companies/:companyId/issue-documents", async (req, res) => { + const companyId = req.params.companyId as string; + assertCompanyAccess(req, companyId); + if (req.actor.type === "agent" && req.actor.source === "agent_key" && req.actor.keyScope?.kind === "task_bridge") { + res.status(403).json({ error: "Task bridge keys cannot use company-wide issue list APIs" }); + return; + } + const parsed = issueDocumentBatchQuerySchema.safeParse(req.query); + if (!parsed.success) { + res.status(400).json({ error: "Invalid issue document batch query", details: parsed.error.issues }); + return; + } + const { key, issueIds, includeBody } = parsed.data; + const settle = admit(req, res); + if (!settle) return; + try { + res.setHeader("Cache-Control", "private, no-store"); + const candidates = await db.select({ + id: issues.id, companyId: issues.companyId, projectId: issues.projectId, parentId: issues.parentId, + assigneeAgentId: issues.assigneeAgentId, assigneeUserId: issues.assigneeUserId, status: issues.status, + updatedAt: issues.updatedAt, lastStatusDecisionId: issues.lastStatusDecisionId, statusVersion: issues.statusVersion, + }).from(issues).where(and( + eq(issues.companyId, companyId), inArray(issues.id, issueIds), isNull(issues.hiddenAt), + await issueReadSqlCondition(db, req.actor), + )); + const readable = await options.filterReadable(req, candidates); + const byId = new Map(readable.map(issue => [issue.id, issue])); + const docs = await documentService(db).getIssueDocumentsByKey(companyId, readable.map(issue => issue.id), key, includeBody); + const byIssueId = new Map(docs.map(doc => [doc.issueId, doc])); + const items = issueIds.filter(id => byId.has(id)).map(issueId => ({ issueId, document: byIssueId.get(issueId) ?? null })); + const parts = [`{"key":${JSON.stringify(key)},"items":[`]; + let bytes = Buffer.byteLength(parts[0]) + 2; // closing array and object + for (const [index, item] of items.entries()) { + const part = `${index ? "," : ""}${JSON.stringify(item)}`; + bytes += Buffer.byteLength(part); + if (bytes > ISSUE_DOCUMENT_BATCH_MAX_BYTES) { + res.status(413).json({ error: "Document batch exceeds 8 MiB; split issueIds into smaller requests or use includeBody=false", maxResponseBytes: ISSUE_DOCUMENT_BATCH_MAX_BYTES }); + return; + } + parts.push(part); + } + if (includeBody) { + await options.recordReads(req, readable.filter(issue => byIssueId.has(issue.id))); + } + res.type("json").send(parts.join("") + "]}"); + } finally { settle(); } + }); +} diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.ts index f10691186e2..ca9217a229a 100644 --- a/server/src/routes/issues.ts +++ b/server/src/routes/issues.ts @@ -18,6 +18,7 @@ import { parseObject } from "../adapters/utils.js"; import { prepareRestrictedWorkspaceRetry } from "../services/restricted-workspace-retry.js"; import { ISSUE_SUMMARIES_MAX_IDS, issueChatBootstrapQuerySchema, issueSummariesQuerySchema, type IssueSummary } from "@paperclipai/shared"; import { readIssueChatBootstrapItems } from "../services/issue-chat-bootstrap.js"; +import { registerIssueDocumentBatchRoute } from "./issue-document-batch.js"; import { issueRunPresenceService } from "../services/issue-run-presence.js"; import { activityService } from "../services/activity.js"; import { listIssueNavigationState, listIssueSummaries } from "../services/issue-summaries.js"; @@ -25,8 +26,8 @@ import { listIssueStatusBlockerAttention } from "../services/issue-status-presen import { assertNoCaseHandoffMutation } from "../services/heartbeat/case-handoff-lifecycle.js"; import { assertHandedOffRunTaskOwnership } from "../run-task-ownership.js"; import { recordNativeHumanSelfHandoff } from "../services/native-runtime/native-human-self-handoff.js"; -import { recordChildTaskRead } from "../services/native-runtime/native-child-completion-delivery.js"; -import { recordCompletedHandoffRead } from "../services/chat-completion-delivery.js"; +import { recordChildTaskRead, recordChildTaskReads } from "../services/native-runtime/native-child-completion-delivery.js"; +import { recordCompletedHandoffRead, recordCompletedHandoffReads } from "../services/chat-completion-delivery.js"; import { recordDeclinedInteractionRead } from "../services/chat-interaction-outcome-report.js"; import { runRetainsIssueIdentity } from "../services/task-repository-acquisition.js"; import { withConversationReadState } from "../services/agent-chat-read-state.js"; @@ -11446,6 +11447,18 @@ export function issueRoutes( }, ); + registerIssueDocumentBatchRoute(router, db, { + filterReadable: filterIssuesForActor, + recordReads: async (req, issues) => { + if (req.actor.type !== "agent" || !req.actor.agentId || !req.actor.runId) return; + const input = { companyId: req.params.companyId as string, agentId: req.actor.agentId, runId: req.actor.runId }; + await recordCompletedHandoffReads(db, { ...input, tasks: issues }) + .catch(error => logger.warn({ err: error, runId: input.runId }, "could not record the completed handoffs a run read")); + await recordChildTaskReads(db, { ...input, children: issues }) + .catch(error => logger.warn({ err: error, runId: input.runId }, "could not record the child tasks a run read")); + }, + }); + router.get("/issues/:id/documents", async (req, res) => { const id = req.params.id as string; const issue = await getAccessibleResource( diff --git a/server/src/routes/openapi.ts b/server/src/routes/openapi.ts index a80a1a74074..4524123771e 100644 --- a/server/src/routes/openapi.ts +++ b/server/src/routes/openapi.ts @@ -1,4 +1,4 @@ -import { catalogTeamInstallSchema, catalogTeamPreviewSchema, updateWorkspacePreferenceSchema } from "@paperclipai/shared"; +import { catalogTeamInstallSchema, catalogTeamPreviewSchema, updateWorkspacePreferenceSchema, issueDocumentBatchQuerySchema } from "@paperclipai/shared"; import { MUSE_ASSET_VERSION, museStopBoundarySchema, museDetectorCleanupAttestationSchema, createGrokInvitationSchema, configureGrokRoutineSchema, verifyGrokRoutineSchema, recoverGrokRoutineSchema, pairGrokBindingSchema, stopGrokWorkSchema, retryGrokDeliverySchema, updateFastResponseSchema } from "@paperclipai/shared"; import { slackTestMessageSchema, slackTestMessageResultSchema, slackManagerAuthorizationSchema, slackSetupOptionsSchema, slackManagedProvisionSchema, slackManagedResultSchema, slackRegistrationSchema, slackSetupActionSchema, slackInstallAuthorizationSchema, slackRegistrationStateSchema, slackAppConfigurationSchema, slackAvatarStateSchema, slackAccountStateSchema } from "@paperclipai/shared"; import { experimentalApiMetadata } from "./experimental-api-metadata.js"; @@ -4800,6 +4800,16 @@ registry.registerPath({ responses: { 200: r.ok(), 401: r.unauthorized }, }); +registry.registerPath({ + method: "get", + path: "/api/companies/{companyId}/issue-documents", + tags: ["issues"], + summary: "Bulk read one document key across issue IDs", + description: "Read current documents for 1–200 comma-separated task UUIDs in one request. Company and task read permissions apply. Unreadable, hidden, foreign and nonexistent tasks are omitted; a readable task without this key returns document:null. Items follow first-requested ID order with duplicates removed. includeBody defaults to true; includeBody=false returns revision metadata and updatedAt without loading bodies, for cache validation. No break-glass access. Responses over 8 MiB fail with 413; split the IDs into smaller requests. Native helpers can use save_api_response then download_artifact to relay the complete batch without per-task calls. Verify every requested ID is present before treating the inventory as complete.", + request: { params: z.object({ companyId: z.string().uuid() }), query: issueDocumentBatchQuerySchema }, + responses: { 200: r.ok(), 400: r.badRequest, 401: r.unauthorized, 403: r.forbidden, 413: { description: "Split the document batch into smaller requests" }, 429: { description: "At most two active reads per actor and eight per server instance; retry after an active read finishes", headers: { "Retry-After": { schema: { type: "string", enum: ["1"] } } } } }, +}); + registry.registerPath({ method: "get", path: "/api/issues/{id}/documents", diff --git a/server/src/services/chat-completion-delivery.ts b/server/src/services/chat-completion-delivery.ts index c9df7255995..f9f4852604f 100644 --- a/server/src/services/chat-completion-delivery.ts +++ b/server/src/services/chat-completion-delivery.ts @@ -5,7 +5,7 @@ import { and, asc, desc, eq, gte, inArray, isNotNull, isNull, lte, ne, sql } fro import { agents, agentWakeupRequests, chatCompletionDeliveries as deliveries, chatTaskHandoffs as handoffs, heartbeatRunEvents, heartbeatRuns, issueComments, issueDocuments, issueThreadInteractions, issues, type Db } from "@paperclipai/db"; import { extractIssueReferenceMatches } from "@paperclipai/shared"; -import { appendHeartbeatRunEvent } from "./heartbeat-run-events.js"; +import { appendHeartbeatRunEventBatch } from "./heartbeat-run-events.js"; import { conflict, forbidden, notFound } from "../errors.js"; export const CHAT_COMPLETION_WAKE_REASON = "chat_task_completed"; @@ -148,13 +148,24 @@ export async function recordCompletedHandoffRead(db: Db, input: { companyId: string; agentId: string; runId: string; task: { id: string; status: string; statusVersion?: number | null }; }) { - if (input.task.status !== "done" || typeof input.task.statusVersion !== "number") return; - const [handoff] = await db.select({ agentId: handoffs.agentId }).from(handoffs) - .where(and(eq(handoffs.taskId, input.task.id), eq(handoffs.companyId, input.companyId))); - if (handoff?.agentId !== input.agentId) return; - await appendHeartbeatRunEvent(db, { companyId: input.companyId, runId: input.runId, agentId: input.agentId, + await recordCompletedHandoffReads(db, { ...input, tasks: [input.task] }); +} + +/** One lookup and one event append statement for a batch; payloads stay compatible with single reads. */ +export async function recordCompletedHandoffReads(db: Db, input: { + companyId: string; agentId: string; runId: string; + tasks: ReadonlyArray<{ id: string; status: string; statusVersion?: number | null }>; +}) { + const completed = input.tasks.filter(task => task.status === "done" && typeof task.statusVersion === "number"); + if (!completed.length) return; + const rows = await db.select({ taskId: handoffs.taskId }).from(handoffs) + .where(and(inArray(handoffs.taskId, completed.map(task => task.id)), eq(handoffs.companyId, input.companyId), eq(handoffs.agentId, input.agentId))); + const matching = new Set(rows.map(row => row.taskId)); + await appendHeartbeatRunEventBatch(db, completed.filter(task => matching.has(task.id)).map(task => ({ + companyId: input.companyId, runId: input.runId, agentId: input.agentId, eventType: COMPLETED_HANDOFF_READ_EVENT, stream: "system", level: "info", - payload: { taskId: input.task.id, statusVersion: input.task.statusVersion } }); + payload: { taskId: task.id, statusVersion: task.statusVersion }, + }))); } /** A reply in this conversation already reported this completion when a turn of the same diff --git a/server/src/services/documents.ts b/server/src/services/documents.ts index 96c916e0993..fdbe34c0794 100644 --- a/server/src/services/documents.ts +++ b/server/src/services/documents.ts @@ -2,15 +2,18 @@ import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { assertAgentInteraction } from "./agent-interaction-policy.js"; import { notifyIssueRecovery, notifyIssueGraphRecovery } from "./issue-recovery-notifications.js"; import { randomUUID } from "node:crypto"; -import { and, asc, desc, eq, sql } from "drizzle-orm"; +import { and, asc, desc, eq, inArray, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { documentRevisions, documents, issueDocuments, issues } from "@paperclipai/db"; -import { isSystemIssueDocumentKey, issueDocumentKeySchema } from "@paperclipai/shared"; -import { conflict, notFound, unprocessable } from "../errors.js"; +import { ISSUE_DOCUMENT_BATCH_MAX_IDS, isSystemIssueDocumentKey, issueDocumentKeySchema } from "@paperclipai/shared"; +import { conflict, notFound, payloadTooLarge, unprocessable } from "../errors.js"; import { isUniqueViolation } from "../db-errors.js"; import { insertRowsInChunks } from "./batch-insert.js"; import type { ImportIssueDocumentRow } from "./import-write-types.js"; +// Stay below download_artifact's 10 MiB limit, including JSON escaping and metadata. +export const ISSUE_DOCUMENT_BATCH_MAX_BYTES = 8 * 1024 * 1024; + function normalizeDocumentKey(key: string) { const normalized = key.trim().toLowerCase(); const parsed = issueDocumentKeySchema.safeParse(normalized); @@ -172,6 +175,34 @@ export function documentService(db: Db) { return row ? mapIssueDocumentRow(row, true) : null; }, + /** Caller authorizes the task set first. One query reads only the requested key. */ + getIssueDocumentsByKey: async (companyId: string, issueIds: string[], rawKey: string, includeBody = true) => { + const key = normalizeDocumentKey(rawKey); + if (issueIds.length > ISSUE_DOCUMENT_BATCH_MAX_IDS) throw unprocessable("Too many issue document IDs"); + if (!issueIds.length) return []; + const join = and(eq(issueDocuments.documentId, documents.id), eq(documents.companyId, companyId)); + const condition = and(eq(issueDocuments.companyId, companyId), inArray(issueDocuments.issueId, issueIds), eq(issueDocuments.key, key)); + const read = async (connection: Db) => { + if (includeBody) { + // PostgreSQL counts encoded JSON one body at a time; no bodies cross + // into the app before the entire set fits. The same snapshot below + // prevents a concurrent edit from growing the admitted response. + const [size] = await connection.select({ bytes: sql`coalesce(sum(octet_length(to_json(${documents.latestBody})::text)), 0)` }) + .from(issueDocuments).innerJoin(documents, join).where(condition); + if (Number(size.bytes) > ISSUE_DOCUMENT_BATCH_MAX_BYTES) { + throw payloadTooLarge("Document batch exceeds 8 MiB; split issueIds into smaller requests or use includeBody=false", { maxResponseBytes: ISSUE_DOCUMENT_BATCH_MAX_BYTES }); + } + } + const rows = await connection + .select({ ...issueDocumentSelect, latestBody: includeBody ? documents.latestBody : sql`''` }) + .from(issueDocuments).innerJoin(documents, join).where(condition); + return rows.map(row => mapIssueDocumentRow(row, includeBody)); + }; + return includeBody + ? db.transaction(tx => read(tx as unknown as Db), { isolationLevel: "repeatable read", accessMode: "read only" }) + : read(db); + }, + /** Bounded receipt read: never load every historical body for one revision. */ getIssueDocumentRevision: async (issueId: string, rawKey: string, revisionId: string) => { const key = normalizeDocumentKey(rawKey); diff --git a/server/src/services/heartbeat-run-events.ts b/server/src/services/heartbeat-run-events.ts index f248d3824cd..7d2b9a66fe7 100644 --- a/server/src/services/heartbeat-run-events.ts +++ b/server/src/services/heartbeat-run-events.ts @@ -180,7 +180,9 @@ async function appendRunEvent( } /** - * Appends native source events of one run and one source atomically, in the + * Appends ordinary non-waking events of one run in one statement, preserving + * each event's existing payload and sequence. Special writers keep the single + * append path. Also appends native source events of one run and one source in the * given order, and returns one receipt per event. A stream that continues its * source's stored cursor with new events takes one statement for the whole * batch. Any other batch (a replay, a gap, an event that registers a wake, a @@ -205,7 +207,38 @@ async function appendRunEventBatch( ): Promise { const first = inputs[0]!; const source = first.nativeSource; - if (!source || inputs.some(input => !input.nativeSource || input.runUpdate || input.retryExhaustion + if (!source) { + if (inputs.some(input => input.nativeSource || input.runUpdate || input.retryExhaustion || input.assertWritable + || input.companyId !== first.companyId || input.runId !== first.runId || input.agentId !== first.agentId + || Object.values(runEventWakes(input)).some(Boolean))) throw new Error("heartbeat_run_event_batch_unsupported"); + assertWritable?.(); + const rows = JSON.stringify(inputs.map(input => ({ + eventType: input.eventType, stream: input.stream ?? null, level: input.level ?? null, color: input.color ?? null, + message: input.message ?? null, payload: input.payload ? heartbeatRunEvents.payload.mapToDriverValue(input.payload) : null, + }))); + const inserted = await db.execute>(sql` + with allocated_run_events as ( + update heartbeat_runs + set next_event_seq = next_event_seq + ${inputs.length}::integer, + updated_at = ${sql.param(new Date(), heartbeatRuns.updatedAt)} + where id = ${first.runId} and company_id = ${first.companyId} and agent_id = ${first.agentId} + returning company_id, id as run_id, agent_id, next_event_seq - ${inputs.length}::integer as first_seq + ) + insert into heartbeat_run_events + (company_id, run_id, agent_id, seq, event_type, stream, level, color, message, payload) + select allocated.company_id, allocated.run_id, allocated.agent_id, allocated.first_seq + event.ordinal - 1, + event.value ->> 'eventType', event.value ->> 'stream', event.value ->> 'level', event.value ->> 'color', + event.value ->> 'message', (event.value ->> 'payload')::jsonb + from allocated_run_events allocated cross join jsonb_array_elements(${rows}::jsonb) with ordinality as event(value, ordinal) + order by event.ordinal + returning * + `); + if (inserted.length !== inputs.length) throw new Error("heartbeat_run_event_binding_mismatch"); + assertWritable?.(); + return inserted.map(decodeRunEventRow).sort((a, b) => a.seq - b.seq) + .map(row => ({ row, disposition: "committed" as const, highestContiguousSourceSeq: 0 })); + } + if (inputs.some(input => !input.nativeSource || input.runUpdate || input.retryExhaustion || input.companyId !== first.companyId || input.runId !== first.runId || input.agentId !== first.agentId || input.nativeSource.sourceInstanceId !== source.sourceInstanceId)) { throw new Error("heartbeat_run_event_batch_unsupported"); diff --git a/server/src/services/native-runtime/issue-document-batch.integration.test.ts b/server/src/services/native-runtime/issue-document-batch.integration.test.ts new file mode 100644 index 00000000000..90b9c4af7ad --- /dev/null +++ b/server/src/services/native-runtime/issue-document-batch.integration.test.ts @@ -0,0 +1,194 @@ +import { createHash, randomUUID } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { eq, sql } from "drizzle-orm"; +import { drizzle } from "drizzle-orm/postgres-js"; +import { agentApiKeys, assets, chatTaskHandoffs, documents, heartbeatRunEvents, issueDocuments, issues } from "@paperclipai/db"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { startRunnerApiTestServer } from "../../__tests__/helpers/runner-api-server.js"; +import { documentService } from "../documents.js"; +import { PaperclipRunnerToolAuthority } from "./paperclip-runner-tool-authority.js"; + +const operationId = "GET /api/companies/{companyId}/issue-documents"; + +describe("bulk issue documents through native API tools", () => { + let server: Awaited>; + const oldSecret = process.env.PAPERCLIP_AGENT_JWT_SECRET; + beforeAll(async () => { + process.env.PAPERCLIP_AGENT_JWT_SECRET = randomUUID(); + server = await startRunnerApiTestServer(); + }, 60000); + afterAll(async () => { + await server?.close(); + if (oldSecret === undefined) delete process.env.PAPERCLIP_AGENT_JWT_SECRET; + else process.env.PAPERCLIP_AGENT_JWT_SECRET = oldSecret; + }); + + async function saveCases(completed = false) { + const f = await server.fixture({ conversation: completed }); + const ids = Array.from({ length: 146 }, () => randomUUID()); + await server.db.insert(issues).values(ids.map(id => ({ id, companyId: f.companyId, title: "Sync case", status: completed ? "done" : "todo", ...(completed ? { parentId: f.issueId } : {}) }))); + if (completed) await server.db.insert(chatTaskHandoffs).values(ids.map(taskId => ({ + taskId, companyId: f.companyId, conversationId: f.issueId, agentId: f.agentId, sessionGeneration: 0, + }))); + const rows = await server.db.insert(documents).values(ids.map((id, index) => ({ + companyId: f.companyId, format: "markdown", title: "Case", latestBody: `Case ${index}: é🧭\n\"quoted\" \\escaped`, + }))).returning(); + await server.db.insert(issueDocuments).values(rows.map((row, index) => ({ companyId: f.companyId, issueId: ids[index], documentId: row.id, key: "sentry-case" }))); + const authority = new PaperclipRunnerToolAuthority(server.db, { ...f, workspaceRoot: f.workspace }); + const execute = (tool: string, args: object) => authority.execute({ tool, callId: randomUUID(), arguments: args }) as Promise; + const found = await execute("search_api", { query: "bulk issue documents by key", limit: 10 }); + expect(JSON.stringify(found)).toContain(operationId); + const queries: string[] = []; + const previousDebug = server.db.$client.options.debug; + server.db.$client.options.debug = (_connection, query) => { queries.push(query); }; + let saved: any; + try { + saved = await execute("save_api_response", { operationId, pathParams: { companyId: f.companyId }, query: { key: "sentry-case", issueIds: ids.join(",") } }); + } finally { server.db.$client.options.debug = previousDebug; } + expect(saved).toMatchObject({ ok: true, status: 200, artifact: { byteSize: expect.any(Number), sha256: expect.any(String) } }); + const [asset] = await server.db.select().from(assets).where(eq(assets.id, saved.artifact.artifactId)); + const object = await server.storage.getObject(f.companyId, asset.objectKey); + const chunks: Buffer[] = []; + for await (const chunk of object.stream) chunks.push(Buffer.from(chunk)); + return { f, ids, rows, saved, execute, queries, bytes: Buffer.concat(chunks) }; + } + + it("saves 146 complete documents in one lossless response artifact", async () => { + const { ids, rows, saved, bytes } = await saveCases(); + expect(bytes.length).toBe(saved.artifact.byteSize); + expect(createHash("sha256").update(bytes).digest("hex")).toBe(saved.artifact.sha256); + const result = JSON.parse(bytes.toString()); + expect(result.key).toBe("sentry-case"); + expect(result.items.map((item: any) => item.issueId)).toEqual(ids); + expect(result.items.map((item: any) => item.document.body)).toEqual(rows.map(row => row.latestBody)); + for (const item of result.items) expect(item.document).toMatchObject({ issueId: item.issueId, key: "sentry-case", latestRevisionNumber: expect.any(Number), updatedAt: expect.any(String) }); + }); + + it.skipIf(process.platform !== "linux")("downloads all 146 documents in one native transfer with the saved hash and size", async () => { + const { f, saved, execute, bytes } = await saveCases(); + await execute("download_artifact", { artifactId: saved.artifact.artifactId, byteSize: saved.artifact.byteSize, sha256: saved.artifact.sha256, destination: "cases.json" }); + expect(await readFile(join(f.workspace, "cases.json"))).toEqual(bytes); + }); + + it("reads 146 completed cases with one handoff lookup and batched receipt writes", async () => { + const { f, ids, queries } = await saveCases(true); + expect(queries.filter(query => /from "chat_task_handoffs"/.test(query))).toHaveLength(1); + const writes = queries.filter(query => /insert into heartbeat_run_events/.test(query)); + expect(writes).toHaveLength(2); + const events = await server.db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, f.runId)); + const handoffs = events.filter(event => event.eventType === "chat_handoff.completed_read"); + expect(new Set(handoffs.map(event => event.payload!.taskId))).toEqual(new Set(ids)); + const children = events.filter(event => event.eventType === "child_task.read"); + expect(children).toHaveLength(1); + expect(new Set(Object.keys(children[0].payload!.children as object))).toEqual(new Set(ids)); + expect(new Set(events.map(event => event.seq)).size).toBe(events.length); + }); + + it("returns current revision metadata without bodies and identifies readable tasks with no document", async () => { + const f = await server.fixture(); + const service = documentService(server.db); + const first = await service.getIssueDocumentByKey(f.issueId, "notes"); + await service.upsertIssueDocument({ issueId: f.issueId, key: "notes", format: "markdown", body: "Updated content", baseRevisionId: first!.latestRevisionId }); + const current = await service.getIssueDocumentByKey(f.issueId, "notes"); + const receipt = await f.authority.execute({ tool: "call_api", callId: randomUUID(), arguments: { + operationId, pathParams: { companyId: f.companyId }, query: { key: " NOTES ", issueIds: [f.blockerId, f.issueId, f.issueId].join(","), includeBody: false }, + } }) as any; + expect(receipt).toMatchObject({ ok: true, data: { key: "notes", items: [ + { issueId: f.blockerId, document: null }, + { issueId: f.issueId, document: { latestRevisionId: current!.latestRevisionId, latestRevisionNumber: 2 } }, + ] } }); + expect(receipt.data.items[1].document).not.toHaveProperty("body"); + }); + + it("omits private, hidden, foreign and missing tasks without disclosing which exist", async () => { + const f = await server.fixture(); + const foreign = await server.fixture(); + const privateId = randomUUID(), hiddenId = randomUUID(), missingId = randomUUID(); + await server.db.insert(issues).values([ + { id: privateId, companyId: f.companyId, title: "Private", visibility: "private", status: "todo" }, + { id: hiddenId, companyId: f.companyId, title: "Hidden", hiddenAt: new Date(), status: "todo" }, + ]); + for (const id of [privateId, hiddenId]) await documentService(server.db).upsertIssueDocument({ issueId: id, key: "notes", format: "markdown", body: "Never leak" }); + const result = await f.authority.execute({ tool: "call_api", callId: randomUUID(), arguments: { + operationId, pathParams: { companyId: f.companyId }, query: { key: "notes", issueIds: [f.issueId, privateId, hiddenId, foreign.issueId, missingId].join(",") }, + } }) as any; + expect(result).toMatchObject({ ok: true, data: { items: [{ issueId: f.issueId, document: { body: "Document verification code: silver-wren." } }] } }); + expect(JSON.stringify(result)).not.toContain("Never leak"); + await expect(f.authority.execute({ tool: "save_api_response", callId: randomUUID(), arguments: { + operationId, pathParams: { companyId: foreign.companyId }, query: { key: "notes", issueIds: foreign.issueId }, + } })).rejects.toMatchObject({ status: 403 }); + const anonymous = await fetch(`${server.apiUrl}/api/companies/${f.companyId}/issue-documents?key=notes&issueIds=${f.issueId}`); + expect(anonymous.status).toBe(401); + }); + + it("enforces company boundaries and denies task bridge keys on the HTTP route", async () => { + const f = await server.fixture({ connectionScenario: "fresh", disableWakeOnDemand: true }); + const token = `pcp_fixture_${randomUUID()}`; + const [key] = await server.db.insert(agentApiKeys).values({ + companyId: f.companyId, agentId: f.agentId, name: "Batch fixture", + keyHash: createHash("sha256").update(token).digest("hex"), responsibleUserId: f.responsibleUserId, + }).returning(); + const get = (companyId: string) => fetch(`${server.apiUrl}/api/companies/${companyId}/issue-documents?key=notes&issueIds=${f.issueId}`, { headers: { Authorization: `Bearer ${token}` } }); + expect((await get(f.companyId)).status).toBe(200); + expect((await get(f.foreignCompanyId)).status).toBe(403); + await server.db.update(agentApiKeys).set({ scopeConfig: { kind: "task_bridge", projectId: f.projectId } }).where(eq(agentApiKeys.id, key.id)); + expect((await get(f.companyId)).status).toBe(403); + }); + + it.each([ + {}, { key: "notes" }, { key: "notes", issueIds: "" }, { key: "notes", issueIds: "not-a-uuid" }, + { key: "notes", issueIds: Array.from({ length: 201 }, () => randomUUID()).join(",") }, + { key: "../notes", issueIds: randomUUID() }, { key: "notes", issueIds: randomUUID(), includeBody: "yes" }, + { key: "notes", issueIds: randomUUID(), breakGlass: true }, + ])("rejects invalid or unbounded query %#", async query => { + const f = await server.fixture(); + const result = await f.authority.execute({ tool: "call_api", callId: randomUUID(), arguments: { operationId, pathParams: { companyId: f.companyId }, query } }) as any; + expect(result).toMatchObject({ ok: false, status: 400 }); + }); + + it("rejects an oversized batch without saving a partial artifact, while metadata remains readable", async () => { + const f = await server.fixture(); + const document = await documentService(server.db).getIssueDocumentByKey(f.issueId, "notes"); + // Escaping expands these bodies in JSON. The cap must use wire bytes. + await server.db.update(documents).set({ latestBody: "\"".repeat(4 * 1024 * 1024) }).where(eq(documents.id, document!.id)); + const query = { key: "notes", issueIds: f.issueId }; + const result = await f.authority.execute({ tool: "save_api_response", callId: randomUUID(), arguments: { operationId, pathParams: { companyId: f.companyId }, query } }) as any; + expect(result).toMatchObject({ ok: false, status: 413 }); + expect(result).not.toHaveProperty("artifact"); + const metadata = await f.authority.execute({ tool: "call_api", callId: randomUUID(), arguments: { operationId, pathParams: { companyId: f.companyId }, query: { ...query, includeBody: false } } }) as any; + expect(metadata).toMatchObject({ ok: true, data: { items: [{ issueId: f.issueId, document: { latestRevisionId: document!.latestRevisionId } }] } }); + expect(metadata.data.items[0].document).not.toHaveProperty("body"); + }); + + it("rejects many large documents using a size-only query before loading any bodies", async () => { + const f = await server.fixture(); + const ids = Array.from({ length: 200 }, () => randomUUID()); + await server.db.insert(issues).values(ids.map(id => ({ id, companyId: f.companyId, title: "Large case", status: "todo" }))); + const rows = await server.db.insert(documents).values(ids.map(() => ({ companyId: f.companyId, latestBody: sql`repeat('x', 524288)` }))).returning({ id: documents.id }); + await server.db.insert(issueDocuments).values(rows.map((row, index) => ({ companyId: f.companyId, issueId: ids[index], documentId: row.id, key: "sentry-case" }))); + const queries: string[] = []; + const measured = drizzle(server.db.$client, { schema: server.db._.fullSchema, logger: { logQuery: query => queries.push(query) } }) as typeof server.db; + let status: number | undefined; + try { await documentService(measured).getIssueDocumentsByKey(f.companyId, ids, "sentry-case"); } + catch (error) { status = (error as { status: number }).status; } + expect(status).toBe(413); + const reads = queries.filter(query => /^select /i.test(query)); + expect(reads).toHaveLength(1); + expect(reads[0]).toContain("octet_length"); + expect(reads[0].split(" from ")[0]).not.toContain('"documents"."id"'); + const result = await f.authority.execute({ tool: "save_api_response", callId: randomUUID(), arguments: { + operationId, pathParams: { companyId: f.companyId }, query: { key: "sentry-case", issueIds: ids.join(",") }, + } }) as any; + expect(result).toMatchObject({ ok: false, status: 413 }); + expect(result).not.toHaveProperty("artifact"); + // Bodies alone fit, but their metadata pushes the response over budget. + // The incremental JSON builder must refuse this before joining a response. + await server.db.update(documents).set({ latestBody: "x".repeat(524000) }).where(eq(documents.companyId, f.companyId)); + const nearLimit = await f.authority.execute({ tool: "save_api_response", callId: randomUUID(), arguments: { + operationId, pathParams: { companyId: f.companyId }, query: { key: "sentry-case", issueIds: ids.slice(0, 16).join(",") }, + } }) as any; + expect(nearLimit).toMatchObject({ ok: false, status: 413 }); + expect(nearLimit).not.toHaveProperty("artifact"); + }); +}); diff --git a/server/src/services/native-runtime/native-child-completion-delivery.ts b/server/src/services/native-runtime/native-child-completion-delivery.ts index 7d367edfa59..cafc5fd018b 100644 --- a/server/src/services/native-runtime/native-child-completion-delivery.ts +++ b/server/src/services/native-runtime/native-child-completion-delivery.ts @@ -1,6 +1,6 @@ import { and, eq, inArray, isNull, ne, or, sql } from "drizzle-orm"; import { agentWakeupRequests, heartbeatRunEvents, heartbeatRuns, statusDecisions, type Db } from "@paperclipai/db"; -import { appendHeartbeatRunEvent } from "../heartbeat-run-events.js"; +import { appendHeartbeatRunEventBatch } from "../heartbeat-run-events.js"; export interface NativeChildCompletionRecipient { companyId: string; @@ -17,12 +17,25 @@ export async function recordChildTaskRead(db: Db, input: { children: ReadonlyArray<{ id: string; parentId: string | null; status: string; updatedAt: Date; lastStatusDecisionId?: string | null }>; }): Promise { const completed = input.children.filter(child => child.parentId === input.parentIssueId && child.status === "done"); - if (completed.length === 0) return; - await appendHeartbeatRunEvent(db, { companyId: input.companyId, runId: input.runId, agentId: input.agentId, - eventType: CHILD_TASK_READ_EVENT, stream: "system", level: "info", - payload: { parentIssueId: input.parentIssueId, - children: Object.fromEntries(completed.map(child => [child.id, { status: child.status, updatedAt: child.updatedAt.toISOString(), - statusDecisionId: child.lastStatusDecisionId ?? null }])) } }); + await recordChildTaskReads(db, { ...input, children: completed }); +} + +/** Group completed children by parent, then persist the existing receipt shape in one statement. */ +export async function recordChildTaskReads(db: Db, input: { + companyId: string; agentId: string; runId: string; + children: ReadonlyArray<{ id: string; parentId: string | null; status: string; updatedAt: Date; lastStatusDecisionId?: string | null }>; +}): Promise { + const groups = new Map>(); + for (const child of input.children) { + if (!child.parentId || child.status !== "done") continue; + const children = groups.get(child.parentId) ?? {}; + children[child.id] = { status: child.status, updatedAt: child.updatedAt.toISOString(), statusDecisionId: child.lastStatusDecisionId ?? null }; + groups.set(child.parentId, children); + } + await appendHeartbeatRunEventBatch(db, [...groups].map(([parentIssueId, children]) => ({ + companyId: input.companyId, runId: input.runId, agentId: input.agentId, + eventType: CHILD_TASK_READ_EVENT, stream: "system", level: "info", payload: { parentIssueId, children }, + }))); } /** New child results belong to a fresh parent turn, including across a Done claim. */ diff --git a/tests/runner-e2e/source-sdk-public-inputs.json b/tests/runner-e2e/source-sdk-public-inputs.json index 340998e9394..70d372906e9 100644 --- a/tests/runner-e2e/source-sdk-public-inputs.json +++ b/tests/runner-e2e/source-sdk-public-inputs.json @@ -229,7 +229,7 @@ "packages/shared/src/connector-availability.ts": "82aec0dfb2e9b8cb04e21c4bcd9aab74b087c76a", "packages/shared/src/connector-identity.test.ts": "d58357707faefa7c98bb906cdaeb4db39d31081a", "packages/shared/src/connector-identity.ts": "5ede83c9c5225dbe8af9cf3a49c74e477ef5b955", - "packages/shared/src/constants.ts": "9762160217907b1a3cc33f3dc9f86b46afae2cdd", + "packages/shared/src/constants.ts": "049c9b0cfd2f222aa361bbf07a75d6150760acdc", "packages/shared/src/crash-point-registry.ts": "3cbf2d290bc111061376b43020848623e493737c", "packages/shared/src/crash-points.test.ts": "3564a1f62d22f54a3c2fb0a72b3a6cde17c957d2", "packages/shared/src/crash-points.ts": "c15259f39610e352eac5423a491548ec1bae94fe", @@ -268,7 +268,7 @@ "packages/shared/src/home-paths.ts": "fa4f1c440653b76b3dbb7a51026d557287deb45d", "packages/shared/src/humanize-connection.test.ts": "ba9e90a44c63452b0bf9f733adb90241921e8eca", "packages/shared/src/humanize-connection.ts": "fbd414476a0ef7106a7c80d8572208781b091699", - "packages/shared/src/index.ts": "bc4373e3956c06613f3214706824a93050a757cf", + "packages/shared/src/index.ts": "414cc5429e2566c9f5cdc24379b05a5916385a3e", "packages/shared/src/issue-activity-identity.test.ts": "3dc294640f54c2be67943e3e0149cf4eff2c95ca", "packages/shared/src/issue-activity-identity.ts": "1b0f07ec3fe0506aaaf80a4fa9893f3baf8f0075", "packages/shared/src/issue-attribution.test.ts": "84563685e7f3c48029243cf1cc24a3a09ee3cde9", @@ -412,10 +412,10 @@ "packages/shared/src/types/inbox-agent-policy.ts": "cb9d41d9681e80b834889268ad96015a39df92f6", "packages/shared/src/types/inbox-dismissal.ts": "58fb7013847396ac9971eb5799dd0bf0486e0371", "packages/shared/src/types/inbox-mine.ts": "1d302b531d34842189704abd2f75f8743d09490d", - "packages/shared/src/types/index.ts": "e753d49a5cb6ae09a4b2d62bc7f832012b499c08", + "packages/shared/src/types/index.ts": "708503f0fe1ad9ed807ccb70cab4d77f3eac6cf2", "packages/shared/src/types/instance.ts": "d47ae3080fbf096e71c51e707efe7e827a6562c7", "packages/shared/src/types/issue-tree-control.ts": "75489fffd491b8350b05caf57d7796419797d9df", - "packages/shared/src/types/issue.ts": "d594c0037a8d9882dc35d3ecd7cb8c276d7e2831", + "packages/shared/src/types/issue.ts": "27ae255864444760df489d1fc2df081ccd214889", "packages/shared/src/types/live.ts": "b90e55e4de8600c6fc17d7fa8d89079d4ae655ec", "packages/shared/src/types/native-finalization.ts": "c026a73003ae232fea4f7624d561493b383df358", "packages/shared/src/types/plugin.ts": "af85e6aa845c2ef6956d0151ff11332947bf3949", @@ -489,13 +489,13 @@ "packages/shared/src/validators/folder.ts": "b486444ab09ec3d5a9afec7dbd2e71c1bc2beb67", "packages/shared/src/validators/goal.ts": "602724134915e1e83c50115abba36b288c0abd3d", "packages/shared/src/validators/inbox-agent-policy.ts": "29cef157f4d9b564fd813868038aa8c813df4a7b", - "packages/shared/src/validators/index.ts": "c8cbf1aea7577e7f7a07b4ca733dabb9b23b0458", + "packages/shared/src/validators/index.ts": "450d622613cece45c4e5cc968a7d8e68c7e903dc", "packages/shared/src/validators/instance.test.ts": "3faa9593a9a05f74429d9b1d2251906a03e9a903", "packages/shared/src/validators/instance.ts": "40c23e4579a1a777d77588a3058db03b46bccb14", "packages/shared/src/validators/issue-chat-bootstrap.test.ts": "e1ebb9aa94da22c825bc3ddd32ce2cfc98bfe645", "packages/shared/src/validators/issue-tree-control.ts": "7144ef86a83e008ac0626e1d62a1d6164c46f438", - "packages/shared/src/validators/issue.test.ts": "9a10bf1112d7c85f80cc1fd5db9920dda8974643", - "packages/shared/src/validators/issue.ts": "9877f071ad93952070507a79aec91fbf312d9a84", + "packages/shared/src/validators/issue.test.ts": "c66108342e62370064b9f3e92652a71c71528012", + "packages/shared/src/validators/issue.ts": "1b016c9453f5276eb61526de2b27d4841e88a008", "packages/shared/src/validators/json-schema.ts": "b6cba6a3f76b49afce1c7dc1c2f086ab946c6133", "packages/shared/src/validators/native-finalization.test.ts": "40c68bdd1d0e8a77cd70c4576fae53334fe5be0d", "packages/shared/src/validators/native-finalization.ts": "d4e674de6ac7d36385e6f6c2172fa16ffc9b1830",