diff --git a/server/src/__tests__/issues-list-query-parsing.test.ts b/server/src/__tests__/issues-list-query-parsing.test.ts index 9a369ce0afe..8aa58e266da 100644 --- a/server/src/__tests__/issues-list-query-parsing.test.ts +++ b/server/src/__tests__/issues-list-query-parsing.test.ts @@ -82,4 +82,66 @@ describeEmbeddedPostgres("issue list status query parsing", () => { await request(app).get(`${path}&sortDir=invalid`).expect(400); }); + it.each(["", "&view=compact", "&view=index", "&view=rows"])("reports a capped page and how to fetch the rest (%s)", async (projection) => { + const { companyId, actor } = await seedCompanyWithBoardAccess(ctx.db, "Capped pages"); + const ids = Array.from({ length: 5 }, (_, index) => `00000000-0000-4000-8000-00000000001${index}`); + await ctx.db.insert(issues).values(ids.map((id) => ({ id, companyId, title: "Launch task", status: "todo", priority: "medium" }))); + const app = routeApp(ctx.db, actor, issueRoutes); + const base = `/api/companies/${companyId}/issues?status=todo&sortField=id&sortDir=asc${projection}`; + + const first = await request(app).get(`${base}&limit=2`).expect(200); + expect(first.body).toHaveLength(2); + expect(first.headers["x-paperclip-has-more"]).toBe("true"); + expect(first.headers["x-paperclip-limit"]).toBe("2"); + expect(first.headers["x-paperclip-next-offset"]).toBe("2"); + expect(first.headers["x-paperclip-next-after-id"]).toBe(ids[1]); + + const byCursor = await request(app).get(`${base}&limit=2&afterId=${first.headers["x-paperclip-next-after-id"]}`).expect(200); + expect(byCursor.body.map((issue: { id: string }) => issue.id)).toEqual(ids.slice(2, 4)); + expect(byCursor.headers["x-paperclip-has-more"]).toBe("true"); + expect(byCursor.headers["x-paperclip-next-offset"]).toBeUndefined(); + expect(byCursor.headers["x-paperclip-next-after-id"]).toBe(ids[3]); + + const last = await request(app).get(`${base}&limit=2&offset=4`).expect(200); + expect(last.body.map((issue: { id: string }) => issue.id)).toEqual([ids[4]]); + expect(last.headers["x-paperclip-has-more"]).toBe("false"); + expect(last.headers["x-paperclip-next-offset"]).toBeUndefined(); + expect(last.headers["x-paperclip-next-after-id"]).toBeUndefined(); + + const exact = await request(app).get(`${base}&limit=5`).expect(200); + expect(exact.body).toHaveLength(5); + expect(exact.headers["x-paperclip-has-more"]).toBe("false"); + expect(exact.headers["x-paperclip-next-offset"]).toBeUndefined(); + }); + + it("pages one-child-per-parent lists by offset, not an id cursor", async () => { + const { companyId, actor } = await seedCompanyWithBoardAccess(ctx.db, "Grouped pages"); + const parentIds = ["00000000-0000-4000-8000-000000000021", "00000000-0000-4000-8000-000000000022"]; + await ctx.db.insert(issues).values(parentIds.map((id) => ({ id, companyId, title: "Parent", status: "todo", priority: "medium" }))); + // The first parent's child has the larger id, so an id cursor would skip the second parent. + await ctx.db.insert(issues).values([ + { id: "00000000-0000-4000-8000-000000000032", companyId, parentId: parentIds[0], title: "Child", status: "todo", priority: "medium" }, + { id: "00000000-0000-4000-8000-000000000031", companyId, parentId: parentIds[1], title: "Child", status: "todo", priority: "medium" }, + ]); + const res = await request(routeApp(ctx.db, actor, issueRoutes)) + .get(`/api/companies/${companyId}/issues?view=index&parentIds=${parentIds.join(",")}&oneChildPerParent=true&sortField=id&sortDir=asc&limit=1`) + .expect(200); + expect(res.body).toHaveLength(1); + expect(res.headers["x-paperclip-has-more"]).toBe("true"); + expect(res.headers["x-paperclip-next-offset"]).toBe("1"); + expect(res.headers["x-paperclip-next-after-id"]).toBeUndefined(); + }); + + it("keeps the default response an array and marks a full-default page as capped", async () => { + const { companyId, actor } = await seedCompanyWithBoardAccess(ctx.db, "Default cap"); + await ctx.db.insert(issues).values(Array.from({ length: 501 }, (_, index) => ({ companyId, title: `RUN-${index + 1}`, status: "todo", priority: "medium" }))); + const res = await request(routeApp(ctx.db, actor, issueRoutes)) + .get(`/api/companies/${companyId}/issues?status=todo&view=index`) + .expect(200); + expect(Array.isArray(res.body)).toBe(true); + expect(res.body).toHaveLength(500); + expect(res.headers["x-paperclip-has-more"]).toBe("true"); + expect(res.headers["x-paperclip-next-offset"]).toBe("500"); + }); + }); diff --git a/server/src/__tests__/low-trust-red-team-routes.test.ts b/server/src/__tests__/low-trust-red-team-routes.test.ts index d0562505fe2..1df8eabb106 100644 --- a/server/src/__tests__/low-trust-red-team-routes.test.ts +++ b/server/src/__tests__/low-trust-red-team-routes.test.ts @@ -1949,6 +1949,50 @@ describeEmbeddedPostgres( expect(lowTrustCount.body.count).toBe(1); }); + it("withholds raw-page pagination headers from a low-trust agent", async () => { + const fixture = await seedLowTrustFixture(db); + const hiddenId = "00000000-0000-4000-8000-000000000001"; + const visibleId = "00000000-0000-4000-8000-000000000002"; + await db.insert(issues).values([ + { + id: hiddenId, + companyId: fixture.company.id, + projectId: fixture.projects.outOfScope.id, + title: "Pagination probe hidden", + status: "todo", + priority: "medium", + }, + { + id: visibleId, + companyId: fixture.company.id, + projectId: fixture.projects.allowed.id, + parentId: fixture.issues.reviewRoot.id, + title: "Pagination probe visible", + status: "todo", + priority: "medium", + }, + ]); + const query = { q: "Pagination probe", sortField: "id", sortDir: "asc", limit: "1" }; + + const board = await request(createApp(db, boardActor(fixture))) + .get(`/api/companies/${fixture.company.id}/issues`) + .query(query); + expect(board.status, JSON.stringify(board.body)).toBe(200); + expect(board.headers["x-paperclip-has-more"]).toBe("true"); + expect(board.headers["x-paperclip-next-after-id"]).toBe(hiddenId); + + const lowTrust = await request(createApp(db, agentActor(fixture))) + .get(`/api/companies/${fixture.company.id}/issues`) + .query(query); + expect(lowTrust.status, JSON.stringify(lowTrust.body)).toBe(200); + expect(lowTrust.body).toEqual([]); + expect(lowTrust.headers["x-paperclip-has-more"]).toBeUndefined(); + expect(lowTrust.headers["x-paperclip-limit"]).toBeUndefined(); + expect(lowTrust.headers["x-paperclip-next-offset"]).toBeUndefined(); + expect(lowTrust.headers["x-paperclip-next-after-id"]).toBeUndefined(); + expect(JSON.stringify(lowTrust.headers)).not.toContain(hiddenId); + }); + it("redacts quarantined low-trust output from higher-trust wake and continuation contexts", async () => { const fixture = await seedLowTrustFixture(db); const lowTrustApp = createApp(db, agentActor(fixture)); diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.ts index 8b3d6254e3d..73c7c754ca5 100644 --- a/server/src/routes/issues.ts +++ b/server/src/routes/issues.ts @@ -3003,6 +3003,19 @@ const ISSUE_LIST_STORM_WINDOW_MS = 500; const ISSUE_LIST_STORM_THRESHOLD = 4; const ISSUE_LIST_MAX_ACTOR_CLIENT_INFLIGHT = 8; +/** Page position of a capped issue-list response, exposed as response headers. */ +export type IssueListPagination = { + limit: number; + hasMore: boolean; + nextOffset: number | null; + nextAfterId: string | null; +}; + +export const ISSUE_LIST_HAS_MORE_HEADER = "X-Paperclip-Has-More"; +export const ISSUE_LIST_LIMIT_HEADER = "X-Paperclip-Limit"; +export const ISSUE_LIST_NEXT_OFFSET_HEADER = "X-Paperclip-Next-Offset"; +export const ISSUE_LIST_NEXT_AFTER_ID_HEADER = "X-Paperclip-Next-After-Id"; + type IssueListPreparedResponse = | { kind: "inbox"; body: unknown } | { @@ -3010,12 +3023,22 @@ type IssueListPreparedResponse = body: CompactIssue[]; etag: string; cacheControl: string; + pagination?: IssueListPagination; } | { kind: "full"; body: unknown[]; + pagination?: IssueListPagination; }; +function setIssueListPaginationHeaders(res: Response, pagination: IssueListPagination | undefined) { + if (!pagination) return; + res.setHeader(ISSUE_LIST_HAS_MORE_HEADER, String(pagination.hasMore)); + res.setHeader(ISSUE_LIST_LIMIT_HEADER, String(pagination.limit)); + if (pagination.nextOffset !== null) res.setHeader(ISSUE_LIST_NEXT_OFFSET_HEADER, String(pagination.nextOffset)); + if (pagination.nextAfterId !== null) res.setHeader(ISSUE_LIST_NEXT_AFTER_ID_HEADER, pagination.nextAfterId); +} + type IssueListCacheStatus = "miss" | "hit" | "coalesced" | "stale" | "retry"; type IssueListStormEvent = { @@ -8530,16 +8553,47 @@ export function issueRoutes( res.json(summaries); }); - async function prepareIssueListForActor(req: Request, companyId: string, listFilters: IssueFilters, view: string | undefined): Promise { + async function prepareIssueListForActor( + req: Request, + companyId: string, + listFilters: IssueFilters, + view: string | undefined, + options: { reportPagination?: boolean } = {}, + ): Promise { const compactView = view === "compact" || view === "index" || view === "rows"; - const rawResult = await svc.list(companyId, listFilters); - const visibleRows = await actorCanReadCompanyScope(req, companyId) + const canReadCompanyScope = await actorCanReadCompanyScope(req, companyId); + // Pagination describes the raw query page. An actor whose rows are filtered + // after the query (for example a low-trust agent) gets no pagination: the + // raw page's cursor and has-more state would reveal tasks it cannot read. + const pageLimit = options.reportPagination && canReadCompanyScope && typeof listFilters.limit === "number" + ? listFilters.limit + : null; + // Probe one row past the page so a capped response can say more exist. + const probedResult = await svc.list( + companyId, + pageLimit === null ? listFilters : { ...listFilters, limit: pageLimit + 1 }, + ); + const hasMore = pageLimit !== null && probedResult.length > pageLimit; + const rawResult = hasMore ? probedResult.slice(0, pageLimit!) : probedResult; + // An id cursor is valid only when the query is ordered by id alone. + // oneChildPerParent orders by parentId first, so it pages by offset. + const cursorOrder = listFilters.sortField === "id" + && listFilters.sortDir === "asc" + && !listFilters.offset + && !listFilters.oneChildPerParent; + const pagination: IssueListPagination | undefined = pageLimit === null ? undefined : { + limit: pageLimit, + hasMore, + nextOffset: hasMore && !listFilters.afterId ? (listFilters.offset ?? 0) + pageLimit : null, + nextAfterId: hasMore && cursorOrder ? rawResult[rawResult.length - 1]?.id ?? null : null, + }; + const visibleRows = canReadCompanyScope ? rawResult : await filterIssuesForActor(req, rawResult); const result = await redactIssueRelationEdgesOnRows(req, companyId, visibleRows); if (view === "index") { const body = result.map((issue) => ({ ...toCompactIssue(issue), listProjection: "index" as const })); - return { kind: "compact", body, etag: compactIssueListEtag(body), cacheControl: "private, must-revalidate" }; + return { kind: "compact", body, etag: compactIssueListEtag(body), cacheControl: "private, must-revalidate", pagination }; } const issueIds = result.map((issue) => issue.id); const interruptions = await taskInterruptionPresentations(db, companyId, result); @@ -8577,6 +8631,7 @@ export function issueRoutes( body: compactResult, etag: compactIssueListEtag(compactResult), cacheControl: "private, must-revalidate", + pagination, }; } const [handoffStates, recoveryActionByIssue] = await Promise.all([ @@ -8607,6 +8662,7 @@ export function issueRoutes( activeRecoveryAction: recoveryActionByIssue.get(issue.id) ?? null, interruption: interruptions.get(issue.id) ?? null, })), + pagination, }; } @@ -8970,7 +9026,7 @@ export function issueRoutes( allowTtlCache: view === "compact", diagnostics: opts.issueListDiagnostics, compute: async () => { - return prepareIssueListForActor(req, companyId, listFilters, view); + return prepareIssueListForActor(req, companyId, listFilters, view, { reportPagination: true }); }, }); @@ -8996,6 +9052,9 @@ export function issueRoutes( return; } + if (coordinated.response.kind !== "inbox") { + setIssueListPaginationHeaders(res, coordinated.response.pagination); + } if (coordinated.response.kind === "compact") { res.setHeader("Cache-Control", coordinated.response.cacheControl); res.setHeader("ETag", coordinated.response.etag); diff --git a/server/src/routes/openapi.ts b/server/src/routes/openapi.ts index 548232c49ff..bb5a80234b1 100644 --- a/server/src/routes/openapi.ts +++ b/server/src/routes/openapi.ts @@ -4283,7 +4283,7 @@ registry.registerPath({ tags: ["issues"], summary: "List issues in a company", description: - "Use view=index for collection membership/order without task bodies or diagnostic enrichment; view=rows enriches visible rows without task bodies. These responses include listProjection and cannot stand in for a full issue. view=compact retains the board row compatibility contract. The default response remains the broad compatibility contract. All projections enforce company and task access.", + "Use view=index for collection membership/order without task bodies or diagnostic enrichment; view=rows enriches visible rows without task bodies. These responses include listProjection and cannot stand in for a full issue. view=compact retains the board row compatibility contract. The default response remains the broad compatibility contract. All projections enforce company and task access. Results are paged: limit defaults to 500 (maximum 1000) and offset to 0. For actors who can read the whole company's task list, every page sets X-Paperclip-Has-More (true when more results exist) and X-Paperclip-Limit; a capped page also sets X-Paperclip-Next-Offset, or X-Paperclip-Next-After-Id for sortField=id&sortDir=asc cursor paging (not with oneChildPerParent). Actors whose rows are filtered by task access, such as low-trust agents, get no pagination headers. Agent call_api results carry these as pagination.hasMore/nextOffset/nextAfterId. A page whose hasMore is true is not the complete list: repeat the request with the same filters and the next offset or afterId until hasMore is false.", request: { params: z.object({ companyId: z.string() }), query: z.object({ @@ -4291,6 +4291,11 @@ registry.registerPath({ ids: z.string().optional().describe("One to 100 comma-separated task UUIDs; requires index or rows without attention."), parentIds: z.string().optional().describe("One to 100 comma-separated parent UUIDs; requires index or rows without attention."), oneChildPerParent: z.boolean().optional().describe("Return at most one readable child per parent; requires view=index and parentIds."), + limit: z.number().int().min(1).max(1000).optional().describe("Page size; defaults to 500, at most 1000. Check X-Paperclip-Has-More (call_api: pagination.hasMore) for more results."), + offset: z.number().int().min(0).optional().describe("Rows to skip; use X-Paperclip-Next-Offset (call_api: pagination.nextOffset) from the previous page."), + sortField: z.enum(["updated", "created", "id"]).optional(), + sortDir: z.enum(["asc", "desc"]).optional(), + afterId: z.string().uuid().optional().describe("Cursor paging: return rows after this task UUID; requires sortField=id, sortDir=asc and no offset. Use X-Paperclip-Next-After-Id (call_api: pagination.nextAfterId)."), }).passthrough(), }, responses: { diff --git a/server/src/services/native-runtime/runner-api-client.ts b/server/src/services/native-runtime/runner-api-client.ts index 05f6737ba13..2f051b7309b 100644 --- a/server/src/services/native-runtime/runner-api-client.ts +++ b/server/src/services/native-runtime/runner-api-client.ts @@ -121,6 +121,34 @@ export async function readBoundedResponse(response: Response, maxBytes = RUNNER_ return Buffer.concat(chunks); } +/** + * Page position reported by capped list routes (for example + * GET /api/companies/{companyId}/issues) through X-Paperclip-Has-More and its + * sibling headers. Agents never see response headers, so surface it beside the + * data; otherwise a capped page reads as the complete list. + */ +export function runnerApiPagination(headers: Headers) { + const hasMoreHeader = headers.get("x-paperclip-has-more"); + if (hasMoreHeader !== "true" && hasMoreHeader !== "false") return undefined; + const hasMore = hasMoreHeader === "true"; + const limit = Number(headers.get("x-paperclip-limit")); + const offsetHeader = headers.get("x-paperclip-next-offset"); + const nextOffset = offsetHeader !== null && /^\d+$/.test(offsetHeader) ? Number(offsetHeader) : null; + const nextAfterId = hasMore ? headers.get("x-paperclip-next-after-id") : null; + const next = [ + ...(nextAfterId ? [`afterId=${nextAfterId}`] : []), + ...(nextOffset !== null ? [`offset=${nextOffset}`] : []), + ]; + return { + hasMore, + truncated: hasMore, + ...(Number.isSafeInteger(limit) && limit > 0 ? { limit } : {}), + nextOffset: hasMore ? nextOffset : null, + ...(nextAfterId ? { nextAfterId } : {}), + ...(hasMore ? { guidance: `This list is incomplete: the page limit was reached and more results exist. Repeat the same call with the same filters and ${next.length ? next.join(" or ") : "the next page"} in query; continue until pagination.hasMore is false before treating the list as complete.` } : {}), + }; +} + export async function executeRunnerApi(input: RunnerApiCall, context: RunnerApiContext, io: RunnerApiIo) { const inlineBytes = RUNNER_API_INLINE_BYTES; const { operation, input: validated } = validateRunnerApiCall(input, context); @@ -221,7 +249,8 @@ export async function executeRunnerApi(input: RunnerApiCall, context: RunnerApiC const uncertain = mutation && (response.status >= 500 || response.status === 408 || (response.status >= 300 && response.status < 400)); const uncertainty = { outcome: "unknown", guidance: "Inspect current state before retrying this mutation; it may already have succeeded." }; const etag = input.responseText && response.ok ? response.headers.get("etag") : null; - const base = { ok: response.ok, status: response.status, operationId: operation.operationId, contentType: type, retryAfter: response.headers.get("retry-after"), ...(etag ? { etag } : {}), ...(uncertain ? uncertainty : {}) }; + const pagination = response.ok ? runnerApiPagination(response.headers) : undefined; + const base = { ok: response.ok, status: response.status, operationId: operation.operationId, contentType: type, retryAfter: response.headers.get("retry-after"), ...(etag ? { etag } : {}), ...(pagination ? { pagination } : {}), ...(uncertain ? uncertainty : {}) }; if (response.status >= 300 && response.status < 400) return { ...base, ok: false, error: "api_redirect_not_followed" }; let totalBytes = captured?.byteSize ?? bytes.length; let baseOffset = captured && response.ok ? textOffset : 0; diff --git a/server/src/services/native-runtime/runner-api.test.ts b/server/src/services/native-runtime/runner-api.test.ts index 0c3f6640909..32b19d21090 100644 --- a/server/src/services/native-runtime/runner-api.test.ts +++ b/server/src/services/native-runtime/runner-api.test.ts @@ -39,6 +39,41 @@ const io = (fetcher: typeof fetch): RunnerApiIo => ({ }), }); +describe("capped list pagination", () => { + const listIssues = "GET /api/companies/{companyId}/issues"; + const page = (headers: Record, rows = [{ id: "a" }, { id: "b" }]) => + vi.fn(async () => new Response(JSON.stringify(rows), { headers: { "content-type": "application/json", ...headers } })); + + it("tells the agent a capped list is incomplete and how to fetch the next page", async () => { + const fetcher = page({ "x-paperclip-has-more": "true", "x-paperclip-limit": "2", "x-paperclip-next-offset": "2" }); + const result = await executeRunnerApi({ operationId: listIssues, query: { limit: 2 } }, context, io(fetcher)); + expect(result).toMatchObject({ + ok: true, + data: [{ id: "a" }, { id: "b" }], + pagination: { hasMore: true, truncated: true, limit: 2, nextOffset: 2 }, + }); + expect((result as unknown as { pagination: { guidance: string } }).pagination.guidance).toContain("offset=2"); + }); + + it("carries the cursor and the signal when the capped body is saved as an artifact", async () => { + const rows = Array.from({ length: 600 }, (_, index) => ({ id: `row-${index}`, title: "x".repeat(80) })); + const fetcher = page({ "x-paperclip-has-more": "true", "x-paperclip-limit": "600", "x-paperclip-next-offset": "600", "x-paperclip-next-after-id": "row-599" }, rows); + const result = await executeRunnerApi({ operationId: listIssues }, context, io(fetcher)); + expect(result).toMatchObject({ ok: true, artifact: { artifactId: "artifact-a" }, pagination: { hasMore: true, nextOffset: 600, nextAfterId: "row-599" } }); + }); + + it("reports a complete list as complete", async () => { + const result = await executeRunnerApi({ operationId: listIssues }, context, io(page({ "x-paperclip-has-more": "false", "x-paperclip-limit": "500" }))); + expect(result).toMatchObject({ ok: true, pagination: { hasMore: false, truncated: false, nextOffset: null } }); + expect((result as unknown as { pagination: object }).pagination).not.toHaveProperty("guidance"); + }); + + it("adds nothing for routes that do not page", async () => { + const result = await executeRunnerApi({ operationId: projects }, context, io(page({}))); + expect(result).not.toHaveProperty("pagination"); + }); +}); + describe("bounded response capture receipts", () => { it.each([projects, createProject])("reports oversize evidence without retrying %s", async operationId => { const fetcher = vi.fn(async () => new Response("", { headers: { "content-length": String(RUNNER_API_RESPONSE_MAX_BYTES + 1) } }));