diff --git a/.github/scripts/tests/ci-profile.test.mjs b/.github/scripts/tests/ci-profile.test.mjs index 9492dae5847..b6a931b6e3c 100644 --- a/.github/scripts/tests/ci-profile.test.mjs +++ b/.github/scripts/tests/ci-profile.test.mjs @@ -105,6 +105,21 @@ test("coverage is conservative after failures and does not merge framework invoc assert.equal(trace.events[0].data["ci.reported_invocations"], 1); }); +test("complete reporter pairs count without launcher intent, and long test names remain timed", () => { + const records = [ + record({ kind: "job" }), record({ kind: "job", job: "second" }), + record({ framework: "node", kind: "start" }), record({ framework: "node", kind: "end" }), + record({ kind: "test", id: "long", parent: "file", name: "x".repeat(5000), + file: "a.test.ts", start: Date.parse(at(4)), duration: 100, status: "passed" }), + record({ kind: "file", id: "file", name: "a.test.ts", file: "a.test.ts", + start: Date.parse(at(4)), duration: 100, status: "passed" }), + ]; + const trace = buildTrace({ ...run, conclusion: "success" }, jobs, records); + assert.equal(trace.events[0].data["ci.profile_complete"], true); + assert.equal(trace.events[0].data["ci.test_count"], 1); + assert.equal(trace.events.find((event) => event.data["ci.kind"] === "test").data.name.length, 256); +}); + test("trusted policy commands work with both current and pre-profile checkouts", () => { const source = readFileSync(new URL("../../workflows/pr-trusted.yml", import.meta.url), "utf8"); const block = source.split(" - name: Select compatible Node test command\n")[1].split("\n - name:")[0]; @@ -191,18 +206,30 @@ test("profiling uses only infrastructure CODEOWNERS resolved at trusted master", ]); }); -test("export checks per-event ingestion results, including partial failures", async () => { +test("export paces batches and retries only rejected transient events", async () => { const events = Array.from({ length: 501 }, () => ({ data: {} })); const sizes = []; + const pauses = []; await sendEvents(events, "test-only", "paperclip-ci", async (url, options) => { assert.equal(url, "https://api.honeycomb.io/1/batch/paperclip-ci"); assert.equal(options.redirect, "error"); const size = JSON.parse(options.body).length; sizes.push(size); return { ok: true, json: async () => Array.from({ length: size }, () => ({ status: 202 })) }; - }); - assert.deepEqual(sizes, [500, 1]); - await assert.rejects(sendEvents([events[0]], "test-only", "paperclip-ci", async () => ({ ok: true, json: async () => [{ status: 400 }] })), /every span/); + }, async (ms) => { pauses.push(ms); }); + assert.deepEqual(sizes, [250, 250, 1]); + assert.deepEqual(pauses, [1000, 1000]); + let calls = 0; + const retried = []; + await sendEvents([{ data: { id: 1 } }, { data: { id: 2 } }], "test-only", "paperclip-ci", async (_url, options) => { + const batch = JSON.parse(options.body); + retried.push(batch.map((event) => event.data.id)); + calls += 1; + return { ok: true, json: async () => calls === 1 ? [{ status: 202 }, { status: 429 }] : [{ status: 202 }] }; + }, async (ms) => { pauses.push(ms); }); + assert.deepEqual(retried, [[1, 2], [2]]); + assert.equal(pauses.at(-1), 1000); + await assert.rejects(sendEvents([events[0]], "test-only", "paperclip-ci", async () => ({ ok: true, json: async () => [{ status: 400 }] })), /status 400/); }); test("workflow exporter runs trusted code and keeps keys out of the PR jobs", () => { diff --git a/doc/ci-profiling.md b/doc/ci-profiling.md index d2031d9f7f8..9d836537bd8 100644 --- a/doc/ci-profiling.md +++ b/doc/ci-profiling.md @@ -17,7 +17,8 @@ The PR author, original workflow actor, and rerun actor must all be individual users in the trusted `master` branch's `.github/**` CODEOWNERS rule. GitHub user IDs are resolved from that rule; PR edits to CODEOWNERS cannot grant access. A label on an outside contributor's PR does not enable profiling, even when a -maintainer applies it. Failed identity lookups disable profiling. +maintainer applies it. The PR still runs normal CI and needs no Honeycomb +access. Failed identity lookups disable profiling. Ordinary PRs do not write timing artifacts or export traces. The exporter job is skipped for ordinary run names. Adding another label does not run the CI matrix. @@ -42,6 +43,10 @@ The expandable hierarchy is **workflow → job → step → test filename → te Vitest, Playwright, and Node file rows use repository-relative filenames, also available in `test.file`. Separate files retain their own children even when individual test names repeat across files. +Honeycomb shows only the 32,000 spans closest to the root in a very large trace. +All file rows are near the root, but the chart can omit some individual test +rows. Use **View events** to query all recorded test spans. Test names are +limited to 256 characters in the trace; file paths remain intact. - Compare the jobs that finish last (`ci.finishes_last`) with other overlapping work. A long job that finishes early is not necessarily delaying CI. @@ -79,7 +84,9 @@ artifacts, interrupted reporters, rejected data, and absent test details visible Test launchers write an expected-invocation record before starting each Vitest, Playwright, or Node process. Each reporter must write matching start/end records, so a missing reporter cannot hide behind another successful invocation in the -same job. Compare `ci.expected_invocations` with `ci.reported_invocations`. +same job. Direct package scripts can start a reporter without the launcher; +their start/end pair proves collection. Compare `ci.expected_invocations` with +`ci.reported_invocations`. Failed or cancelled workflows are conservatively incomplete because a failed command can prevent later tests or steps from running. @@ -90,9 +97,10 @@ unambiguous full-suite baseline. GitHub may carry earlier successful jobs into the new attempt's job inventory. Jobs completed before the new attempt started are excluded and counted in `ci.carried_job_count`. -If export fails, inspect **CI profile** and rerun that exporter after fixing the -cause. A partial HTTP export may already have sent some spans; retries use the -same IDs but the Events API does not promise deduplication. Timing artifacts +The exporter sends bounded batches with pauses and retries only events that +Honeycomb rejects temporarily. If export still fails, inspect **CI profile** +before rerunning it. A partial HTTP export may already have sent some spans; +reruns use the same IDs but the Events API does not promise deduplication. Timing artifacts remain available for 14 days. Invalid/oversized artifacts stop export instead of silently truncating the suite. No logs, environment dumps, error messages, stack traces, or credentials are included in the exported field allowlist. @@ -144,4 +152,5 @@ workflow. References: [Honeycomb trace fields](https://docs.honeycomb.io/get-started/best-practices/organizing-data/), [Events API](https://docs.honeycomb.io/api/events/create-events), +[large trace display limit](https://docs.honeycomb.io/troubleshoot/common-issues/data-in-honeycomb/#your-very-large-trace-displays-only-32000-spans), [GitHub workflow_run](https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#workflow_run). diff --git a/doc/plans/2026-10-12-claude-runner-default.md b/doc/plans/2026-10-12-claude-runner-default.md index 41ec65c6d30..1216508ea92 100644 --- a/doc/plans/2026-10-12-claude-runner-default.md +++ b/doc/plans/2026-10-12-claude-runner-default.md @@ -114,17 +114,21 @@ Update this record with the integrated source, package/image digests, preview, a ## Integrated implementation progress — 2026-10-15 -Current master `7fc2dc8b3236e1149ae32756b6153a503623404b` is merged forward through the three PRs. Protocol is `8b51137aed61bedcb0781bb82f31d8a6931f7304`; behavior is `77936b29731fd988226bb7cd9b5f5c459d45aa11`; platform is `84592191eb` (before this evidence update). Existing Codex behavior and master's landed ARM SDK admission are preserved. GitHub workflow scope is now verified and the cloud image consumer is published; the earlier access blocker is resolved. +Master through `7f9d3d1929` is merged through the three PRs. Protocol is `56f8365a19c33825897fe6b9c23e0eb2e5958303`; behavior is `f1dfde5a097fee41cda53fdb9a44b61d6722bc7a`; frozen platform package source is `bd74bf50596d099e138bc6824cf0230f1026bfe1`. Existing Codex behavior and master's landed ARM SDK admission remain intact. GitHub workflow scope is verified; the cloud image consumer is published. -Focused verification: 343 selection/configuration checks; 849 skills, comment-routing and native-session checks; 203 protocol checks plus the runtime-host file's independent 55-test confirmation; runner TypeScript build; server TypeScript; 15 image-consumer controls. CI exposed a pre-existing snapshot worker timing failure: its test now uses its configured 30-second command deadline; all four cases pass. A current review found onboarding could remain busy after discovery changed the Auto resolution. The fix includes resolvedRunner in setup identity, prevents stale hires, and requires a manual reconnect. Its regression fails before the change; all 148 onboarding tests pass after it. Protocol has current-head Greptile 5/5. Remaining current-head CI/review is pending. +Review corrections are committed and covered by failing-before/passing-after controls. Discovery changes invalidate stale onboarding probes. Finish setup now saves the exact configuration admitted during qualification rather than rebuilding Legacy defaults from an older render. V7 native input retains prepared transport, constraints, and completion-only second turns. The latter regression reproduces `session cannot start another turn` before the fix. All 160 focused protocol/session tests, all 71 New Agent tests, 148 onboarding tests, UI typecheck, and runner TypeScript build pass. Earlier focused verification also passed 343 selection/configuration checks, 849 skills/comment/native-session checks, 320 latest-master lifecycle/selection/snapshot checks, 148 platform-selection controls, 114 release controls, 16 image-consumer controls and token gates. Master's subsequently landed snapshot-worker test fix replaces the earlier local timeout change. All three heads have fresh 5/5 reviews with zero unresolved findings and passing required CI. One platform shard hit an ephemeral HTTP/2 EADDRINUSE test-port collision in unchanged code; a single provider-free rerun passed, preserving the original failure log at `/private/tmp/claude-v7-platform-workspaces-b-failure-20261015.log`. Final installed-package and remote execution gates remain pending. -Fresh preview: http://127.0.0.1:3105/CLA/issues/CLA-1. This is a new isolated data directory, `/private/tmp/claude-master-local-20261015`. Retained historical preview 3115 stays separate. Ordinary New Agent -> Claude Code -> saved Anthropic API key -> Model Default -> high effort -> Finish setup saved agent `5339638d-b41d-4a6f-a041-4e5d1698a8de` as `paperclip_runner`/`acpx`/`claude`/`claude-opus-5-5`. No runner/model was patched after creation. Only timeout (180 seconds) and concurrency (one) were bounded. Experimental native runner is false. The first existing CLI credential probe identified an outdated supplementary system CLI; the isolated PATH now uses the already-qualified 2.1.286 executable, preserving the user's CLI. +Preview: http://127.0.0.1:3105/CLA/issues/CLA-2, data `/private/tmp/claude-master-local-20261015`. Historical preview 3115 is separate. Ordinary New Agent -> Claude Code -> saved Anthropic API key -> Model Default -> high effort -> Finish setup originally saved agent `5339638d-b41d-4a6f-a041-4e5d1698a8de` as `paperclip_runner`/`acpx`/`claude`/`claude-opus-5-5`. No runner/model was patched after creation. Only timeout (180 seconds) and concurrency (one) were bounded. Experimental native runner remains false. The isolated PATH uses the already-qualified Claude 2.1.286 executable, preserving the user's system CLI. -First current-source run `58291208-459e-4de4-8b3e-5eff25d4e98c` succeeded on behavior `77936b2973`, API billing, high effort admitted, settled estimate $0.180427400. Its independently downloaded `claude-master-checklist-v1.md` is 160 bytes, SHA256 `fb94f834e1d33212aec13bc91e9b568e7161f96276bcd2525d812f4eb4a69616`. The follow-up `37e6164e-98e3-42fe-8847-2ee061629055` also succeeded after a controller restart, settled estimate $0.156077800. Its independently downloaded v2 is 217 bytes, SHA256 `60b89d58e6995d170b673a196995c7cf9021f9294a580b46cb82ef8ceeb2ed58`; v1 remains byte-identical. Task, provider and backend session IDs are identical across both runs. Both admitted high effort; saved configuration and Done status persist. Combined current-pair estimate $0.336505200; no pending/unpriced usage. Report: `/private/tmp/claude-master-local-proof-20261015/acceptance.json`. The local API gate is passed; final packaged proof remains separate. Evidence: `/private/tmp/claude-master-local-proof-20261015`; browser journal `/private/tmp/claude-master-live-ui-v3-20261015`. +The first integrated local pair passed on `77936b2973`: runs `58291208-459e-4de4-8b3e-5eff25d4e98c` and `37e6164e-98e3-42fe-8847-2ee061629055`, independent 160/217-byte artifacts, high effort, unchanged task/provider/backend session across controller restart, estimate $0.336505200. Its report remains `/private/tmp/claude-master-local-proof-20261015/acceptance.json` and attached to CLA-1. This is preserved historical evidence for that runtime. -The local daemon is reused from verified runtime `2646b94533bac1eae0d30b11d3774b104eeaf981`, with its original manifest intact; Rust/protocol trees are byte-identical to the integrated behavior source. Darwin ARM64 daemon SHA256 is `094e5e809843efc3d16c796ee4271ff11fcb550afb927b3fa0e745d964bcba2d`. Current-source JS was rebuilt locally. No local Docker or Rust build ran. Final packaged identity and other-target acceptance remain pending. +After rebuilding the corrected TypeScript runtime, the new pair passed on `f1dfde5a09`: runs `6bfd4019-4d7e-4be7-948e-b56f56d8ee3e` and `c761ac19-3e5f-4301-bf13-22a1f9ec0d32`. Independently downloaded `claude-v7-checklist-v1.md` is 134 bytes, SHA256 `3161571692c5d08dd80e91ce83f7ff42fb69f70d8f349084114707e27b8b8b82`; v2 is 180 bytes, SHA256 `3fe1b7f05f1f320246691d007576d0d25a40a1e91fd649c2496a8e56d1ddd299`. V1 is preserved. Task/provider/backend identity survived another controller restart; both turns admitted high effort, billed as metered API, and settled without pending/unpriced usage. Estimate $0.351676000. Report `/private/tmp/claude-v7-local-proof-20261015/acceptance.json`, also attached to CLA-2 as `56c7acb8-7079-4b46-a73f-b0b49e21c0e3`; screenshot attachment `615e3de4-38dd-4c0f-be94-0bb05f7b6757`. Task and agent settings survive reload. A fresh read-only browser check also passes light/dark desktop, 390-pixel mobile, and keyboard open/Escape/focus restoration for the shared Runner control. No horizontal overflow or mutations occurred. Screenshots and receipt: `/private/tmp/claude-v7-local-ui-20261015`. -Budget allocation remains $50 total: $0.80 unused allowance was transferred from an existing paused fixture (cap $1 -> $0.20) to the fresh local fixture (cap $0.80, per-run reservation $0.50). No failed holds were cleared; automatic paid retries and timer heartbeats remain disabled. Prior unknown charges and the $1 setup reservation remain conservative reservations. Allocation receipt: `/private/tmp/claude-master-local-allocation-20261015.json`. +The local daemon retains verified source `2646b94533bac1eae0d30b11d3774b104eeaf981` and its original manifest. Rust/protocol trees are unchanged; Darwin ARM64 daemon SHA256 is `094e5e809843efc3d16c796ee4271ff11fcb550afb927b3fa0e745d964bcba2d`. Current-source JS was rebuilt locally. No local Docker or Rust builds ran. Local source acceptance is passed; final installed-package proof remains separate. + +The first new cloud campaign [38734513491](https://github.com/paperclipai/paperclip/actions/runs/38734513491), source `64e721e5f5`, was cancelled after review identified the v7 session defect. Its ARM64 Linux, ARM64 Mac and x64 Linux daemon jobs succeeded; Intel and package assembly were cancelled. Those artifacts are preserved and are not final qualification. Following the corrected local pair, [campaign 38738177651](https://github.com/paperclipai/paperclip/actions/runs/38738177651) builds frozen source `bd74bf5059` in cloud workers. The exact-artifact image consumer and final installed Mac/Linux execution follow that successful producer. Live subscription acceptance remains unverified under the user's API-key direction. + +Budget remains $50 total with historical holds intact. The fresh fixture cap is now $2.30: $0.80 from the original safe reallocation, then $1.50 from unused allowance in held fixture `16cc3ada` while retaining its $1 pending reservation and $1 setup reserve. Donor remains hard-stopped and its agents paused. Receipts: `/private/tmp/claude-master-local-allocation-20261015.json` and `/private/tmp/claude-master-platform-allocation-20261015.json`. Fresh fixture observed usage is $0.688181200, pending/unpriced zero, leaving $1.611818800. Historical known rate-card estimate plus these runs is $3.742819800, excluding unknown charges. Per-run reservation remains $0.50; automatic paid retries and timer heartbeats remain disabled. --- diff --git a/doc/plans/2026-10-15-retained-reader-query-batches.md b/doc/plans/2026-10-15-retained-reader-query-batches.md new file mode 100644 index 00000000000..7528af7feda --- /dev/null +++ b/doc/plans/2026-10-15-retained-reader-query-batches.md @@ -0,0 +1,42 @@ +# Bound retained-reader authorization database work + +## Measured problem + +A synthetic local service fixture uses 383 workspace paths and 64 retained runs +with the same agent and responsible user. It exercises the production publication +guard against disposable PostgreSQL. It does not represent a live tenant or a +Product E2E evaluation. + +The original guard made 1,431 queries. A declared 4ms round-trip delay in a +loopback proxy increased one admission to 15.19 seconds and three concurrent +admissions to 54.69 seconds. The company root fence serializes those admissions. +With this change, one admission makes 680 queries and takes 7.13 seconds under +the same proxy; three take 20.93 seconds. The zero-injected-delay samples took +359ms before and 146ms after. Timing samples are controlled local evidence, not +an agent-response or hosted latency guarantee. + +## Change boundary + +Retained readers still need individual saved-run identity and execution-policy +checks. Current readers still need the existing write checks. Existing company +root fences, row locks, source privacy and publication ordering stay in place. + +Batch retained-reader project reads through the existing multi-actor read API. +Share only database read promises within one call. Keep actor-supplied membership +snapshots and per-actor privacy results separate. Key raw reads by their company, +principal and run as applicable. Each later call reads current grants again. +Select only the run identity and saved policy fields needed by the guard, +including both native and legacy issue-binding fallbacks. + +## Verification and remaining proof + +The query-count regression must fail on the original code without changing its +800-query bound. Distinct reader identities and stored policies must still fail +closed. Grant revocation must affect the next call. Supplied membership snapshots +must match individual decisions in both actor orders without leaking into database +snapshots. Run the complete affected authorization and source-trust suites. + +Full repository typecheck and build plus fresh current-head CI and review precede +merge. A real hosted warm agent reply remains the acceptance test for the separate +under-30-second objective. Neither this service benchmark nor a fresh local +provider run establishes that hosted result. diff --git a/scripts/ci-profile/export.mjs b/scripts/ci-profile/export.mjs index 48e726b42ed..2e5ab2ba2e5 100644 --- a/scripts/ci-profile/export.mjs +++ b/scripts/ci-profile/export.mjs @@ -14,19 +14,35 @@ const get = (endpoint) => JSON.parse(gh([endpoint]).toString()); const pages = (endpoint, field) => JSON.parse(gh(["--paginate", "--slurp", endpoint]).toString()) .flatMap((page) => field ? page[field] : page); -export async function sendEvents(events, apiKey, dataset, fetcher = fetch) { +export async function sendEvents(events, apiKey, dataset, fetcher = fetch, pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms))) { if (!apiKey) throw new Error("HONEYCOMB_CI_API_KEY is not configured"); - for (let offset = 0; offset < events.length; offset += 500) { - const batch = events.slice(offset, offset + 500); - const response = await fetcher(`https://api.honeycomb.io/1/batch/${encodeURIComponent(dataset)}`, { - method: "POST", headers: { "X-Honeycomb-Team": apiKey, "Content-Type": "application/json" }, - body: JSON.stringify(batch), redirect: "error", signal: AbortSignal.timeout(30_000), - }); - if (!response.ok) throw new Error(`Honeycomb export failed (HTTP ${response.status})`); - const results = await response.json(); - if (!Array.isArray(results) || results.length !== batch.length || results.some((result) => result.status !== 202)) { - throw new Error("Honeycomb did not accept every span in the batch"); + const transient = new Set([429, 500, 502, 503, 504]); + for (let offset = 0; offset < events.length; offset += 250) { + let pending = events.slice(offset, offset + 250); + for (let attempt = 0; pending.length; attempt += 1) { + if (attempt >= 8) throw new Error(`Honeycomb did not accept ${pending.length} spans after retries`); + const response = await fetcher(`https://api.honeycomb.io/1/batch/${encodeURIComponent(dataset)}`, { + method: "POST", headers: { "X-Honeycomb-Team": apiKey, "Content-Type": "application/json" }, + body: JSON.stringify(pending), redirect: "error", signal: AbortSignal.timeout(30_000), + }); + if (!response.ok) { + if (!transient.has(response.status)) throw new Error(`Honeycomb export failed (HTTP ${response.status})`); + await pause(Math.min(30_000, 1000 * 2 ** attempt)); + continue; + } + const results = await response.json(); + if (!Array.isArray(results) || results.length !== pending.length) { + throw new Error("Honeycomb returned an unexpected batch response"); + } + const rejected = results.flatMap((result, index) => result.status === 202 ? [] : [{ event: pending[index], status: result.status }]); + const permanent = rejected.find(({ status }) => !transient.has(status)); + if (permanent) throw new Error(`Honeycomb rejected ${rejected.length} spans (status ${permanent.status})`); + pending = rejected.map(({ event }) => event); + if (pending.length) await pause(Math.min(30_000, 1000 * 2 ** attempt)); } + // The full suite can exceed Honeycomb's burst allowance. Leave capacity + // for the team's other telemetry rather than exhausting it in one export. + if (offset + 250 < events.length) await pause(1000); } } diff --git a/scripts/ci-profile/trace.mjs b/scripts/ci-profile/trace.mjs index 99beeabad04..a36112d6ced 100644 --- a/scripts/ci-profile/trace.mjs +++ b/scripts/ci-profile/trace.mjs @@ -105,7 +105,8 @@ export function buildTrace(run, jobs, records = []) { invocations.set(invocation, state); continue; } - if (!["file", "test"].includes(record.kind) || !text(record.id) || !text(record.name) + if (!["file", "test"].includes(record.kind) || !text(record.id) + || (record.kind === "file" ? !text(record.name) : typeof record.name !== "string") || !text(record.file) || !["vitest", "playwright", "node"].includes(record.framework) || !Number.isFinite(record.start) || !Number.isFinite(record.duration) || record.duration < 0 || record.start < millis(job.started_at) - 1000 @@ -128,7 +129,7 @@ export function buildTrace(run, jobs, records = []) { && candidate.conclusion !== "skipped"); const parent = files.get(`${invocation}:${record.parent}`) || (step ? id(`${jobId}:step:${step.number}`) : jobId); - span(recordId(entry), parent, record.name, record.start, record.start + record.duration, { + span(recordId(entry), parent, record.kind === "test" ? record.name.slice(0, 256) : record.name, record.start, record.start + record.duration, { "ci.kind": record.kind, "ci.job": job.name, "test.framework": record.framework, "test.file": record.file, "test.status": typeof record.status === "string" ? record.status.slice(0, 32) : "unknown", error: failed(record.status), "test.retries": Number.isFinite(record.retries) ? record.retries : 0, @@ -140,7 +141,9 @@ export function buildTrace(run, jobs, records = []) { const expectedReceipts = jobs.filter((job) => job.steps?.some((step) => step.name === "Initialize CI profile")); const missing = expectedReceipts.filter((job) => !receipts.has(job.id)); if (missing.length) warnings.push(`Missing timing artifacts for ${missing.length} jobs`); - const incomplete = [...invocations.values()].filter((value) => !value.expected || !value.start || !value.end).length; + // Some package scripts invoke a reporter without using the launcher. Its + // start/end pair proves collection even when no intent record was written. + const incomplete = [...invocations.values()].filter((value) => !value.start || !value.end).length; if (incomplete) warnings.push(`${incomplete} test invocations did not finish reporting`); if (rejected) warnings.push(`${rejected} invalid timing records ignored`); if (run.conclusion !== "success") warnings.push("Workflow did not succeed; later tests or steps may not have run"); @@ -159,8 +162,8 @@ export function buildTrace(run, jobs, records = []) { if (expectedReceipts.length && missingTests.length) warnings.push(`Missing test detail for ${missingTests.length} test jobs (including jobs that failed before testing)`); root["ci.test_count"] = valid.filter(({ record }) => record.kind === "test").length; root["ci.file_count"] = valid.filter(({ record }) => record.kind === "file").length; - root["ci.expected_invocations"] = [...invocations.values()].filter((value) => value.expected).length; - root["ci.reported_invocations"] = [...invocations.values()].filter((value) => value.expected && value.start && value.end).length; + root["ci.expected_invocations"] = invocations.size; + root["ci.reported_invocations"] = [...invocations.values()].filter((value) => value.start && value.end).length; root["ci.profile_complete"] = warnings.length === 0 && run.run_attempt === 1; root["ci.coverage_warnings"] = warnings.join("; "); return { traceId, events, warnings, wallMs: end - start, jobCount: jobs.length }; diff --git a/server/src/__tests__/repository-root-reader-queries.test.ts b/server/src/__tests__/repository-root-reader-queries.test.ts new file mode 100644 index 00000000000..4433b2f50f0 --- /dev/null +++ b/server/src/__tests__/repository-root-reader-queries.test.ts @@ -0,0 +1,115 @@ +import { createHash, randomUUID } from "node:crypto"; +import { mkdtemp, realpath, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { and, eq, inArray } from "drizzle-orm"; +import { drizzle } from "drizzle-orm/postgres-js"; +import { afterAll, beforeAll, expect, it, vi } from "vitest"; +import { agents, authUsers, companies, companyMemberships, createDb, executionWorkspaceRepositoryAccess, + executionWorkspaces, heartbeatRuns, issues, projects, projectAccessMembers, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db"; +import { executionWorkspaceRepositoryService } from "../services/execution-workspace-repositories.js"; +import { authorizationService, type AuthorizationActor } from "../services/authorization.js"; +import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +let database: Awaited>; +let db: ReturnType; +let cwd: string; +const companyId = randomUUID(), agentId = randomUUID(), userId = randomUUID(), issueId = randomUUID(), projectId = randomUUID(); +const runIds = Array.from({ length: 64 }, () => randomUUID()); +let workspaceId: string; + +beforeAll(async () => { + vi.stubEnv("PAPERCLIP_ISSUE_PRIVACY_MODE", "enforce"); + database = await startEmbeddedPostgresTestDatabase("root-reader-query-"); + db = createDb(database.connectionString); + cwd = await realpath(await mkdtemp(path.join(tmpdir(), "root-reader-query-"))); + await db.insert(companies).values({ id: companyId, name: "Retained readers", issuePrefix: "RRQ" }); + await db.insert(authUsers).values({ id: userId, name: "Sponsor", email: `${userId}@example.test`, emailVerified: true, createdAt: new Date(), updatedAt: new Date() }); + await db.insert(agents).values({ id: agentId, companyId, name: "Worker", adapterType: "process" }); + await db.insert(companyMemberships).values([ + { companyId, principalType: "user", principalId: userId, membershipRole: "operator", status: "active" }, + { companyId, principalType: "agent", principalId: agentId, membershipRole: "member", status: "active" }, + ]); + await db.insert(projects).values({ id: projectId, companyId, name: "Source" }); + const [workspace] = await db.insert(executionWorkspaces).values({ companyId, projectId, cwd, name: "Task source", mode: "shared_workspace", strategyType: "operator_directory" }).returning(); + workspaceId = workspace.id; + await db.insert(issues).values({ id: issueId, companyId, projectId, title: "Task", assigneeAgentId: agentId, executionWorkspaceId: workspaceId }); + await db.insert(heartbeatRuns).values(runIds.map(id => ({ id, companyId, agentId, responsibleUserId: userId, + status: "succeeded", contextSnapshot: { issueId }, resultJson: { unusedPayload: "x".repeat(65536) } }))); + const rootKey = createHash("sha256").update(cwd).digest("hex"); + await db.insert(executionWorkspaceRepositoryAccess).values(runIds.map(scopeId => ({ companyId, rootKey, + canonicalCwd: cwd, kind: "reader", scopeId, issueId, agentId, responsibleUserId: userId }))); +}, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + +afterAll(async () => { + vi.unstubAllEnvs(); + await db?.$client.end({ timeout: 0 }); + await database?.cleanup(); + if (cwd) await rm(cwd, { recursive: true, force: true }); +}); + +const input = () => ({ companyId, issueId, workspaceId, cwd, agentId, runId: runIds[0] }); + +it("bounds repeated-principal queries without loading retained run payloads", async () => { + const queries: string[] = []; + const recorded = drizzle(db.$client, { schema: db._.fullSchema, logger: { logQuery: query => queries.push(query) } }) as unknown as typeof db; + await executionWorkspaceRepositoryService(recorded).guardForPublication(input()); + expect(queries.length).toBeLessThan(800); + const runs = queries.filter(query => query.includes('from "heartbeat_runs"')); + expect(runs.length).toBeGreaterThan(0); + expect(runs.every(query => !query.includes('"result_json"') && !query.includes('"runner_profile_json"'))).toBe(true); +}); + +it("keeps each retained run identity and policy distinct for the same principals", async () => { + const service = executionWorkspaceRepositoryService(db); + const invalidId = runIds[31]; + try { + await db.update(executionWorkspaceRepositoryAccess).set({ responsibleUserId: randomUUID() }).where(eq(executionWorkspaceRepositoryAccess.scopeId, invalidId)); + await expect(service.guardForPublication(input())).rejects.toThrow("identity"); + await db.update(executionWorkspaceRepositoryAccess).set({ responsibleUserId: userId }).where(eq(executionWorkspaceRepositoryAccess.scopeId, invalidId)); + await db.update(heartbeatRuns).set({ contextSnapshot: { issueId, executionPolicy: { trustPreset: "unsupported" } } }).where(eq(heartbeatRuns.id, invalidId)); + await expect(service.guardForPublication(input())).rejects.toThrow(); + } finally { + await db.update(executionWorkspaceRepositoryAccess).set({ responsibleUserId: userId }).where(eq(executionWorkspaceRepositoryAccess.scopeId, invalidId)); + await db.update(heartbeatRuns).set({ contextSnapshot: { issueId } }).where(eq(heartbeatRuns.id, invalidId)); + } + await expect(service.guardForPublication(input())).resolves.toBeUndefined(); +}); + +it("reloads project grants on every publication attempt", async () => { + const service = executionWorkspaceRepositoryService(db); + await db.update(projects).set({ visibility: "private" }).where(eq(projects.id, projectId)); + const grants = await db.insert(projectAccessMembers).values([ + { companyId, projectId, subjectType: "agent", subjectId: agentId }, + { companyId, projectId, subjectType: "user", subjectId: userId }, + ]).returning(); + try { + await expect(service.guardForPublication(input())).resolves.toBeUndefined(); + await db.delete(projectAccessMembers).where(and(eq(projectAccessMembers.projectId, projectId), eq(projectAccessMembers.subjectType, "user"))); + await expect(service.guardForPublication(input())).rejects.toThrow(); + } finally { + await db.delete(projectAccessMembers).where(inArray(projectAccessMembers.id, grants.map(grant => grant.id))); + await db.update(projects).set({ visibility: "open" }).where(eq(projects.id, projectId)); + } +}); + +it("keeps supplied membership snapshots separate from fresh database snapshots", async () => { + const actor: AuthorizationActor = { type: "agent", source: "agent_jwt", companyId, agentId, + runId: runIds[0], onBehalfOfUserId: userId }; + const supplied = { ...actor, onBehalfOfMemberships: [{ companyId, membershipRole: "operator", status: "active" }] }; + const service = authorizationService(db); + const resource = { type: "company" as const, companyId }; + await db.update(companyMemberships).set({ status: "inactive" }).where(and(eq(companyMemberships.companyId, companyId), eq(companyMemberships.principalId, userId))); + try { + for (const candidates of [[supplied, actor], [actor, supplied]]) { + const decisions = await service.decideReadsForActors(candidates, [resource]); + for (let index = 0; index < candidates.length; index++) { + expect(decisions[index][0]).toEqual(await service.decide({ actor: { ...candidates[index] }, action: "company_scope:read", resource })); + expect(decisions[index][0].code).toBe(candidates[index] === supplied + ? "RESPONSIBLE_USER_UNAUTHORIZED" : "RESPONSIBLE_USER_UNAVAILABLE"); + } + } + } finally { + await db.update(companyMemberships).set({ status: "active" }).where(and(eq(companyMemberships.companyId, companyId), eq(companyMemberships.principalId, userId))); + } +}); diff --git a/server/src/__tests__/vitest-chat-shards.test.ts b/server/src/__tests__/vitest-chat-shards.test.ts index 75de627551f..7c985762211 100644 --- a/server/src/__tests__/vitest-chat-shards.test.ts +++ b/server/src/__tests__/vitest-chat-shards.test.ts @@ -37,7 +37,7 @@ it("runs every active nested/parameterized fixture case exactly once through the const run = (index: number, count: number) => spawnSync(process.execPath, [ path.join(repoRoot, "scripts/run-vitest-stable.mjs"), "--mode", "general", "--group", "general-chat", "--shard-index", String(index), "--shard-count", String(count), - ], { cwd: root, env: { ...process.env, CI: "true" }, encoding: "utf8", timeout: 45_000, maxBuffer: 4 * 1024 * 1024 }); + ], { cwd: root, env: { ...process.env, CI: "true", PAPERCLIP_CI_PROFILE: "false" }, encoding: "utf8", timeout: 45_000, maxBuffer: 4 * 1024 * 1024 }); for (const index of [0, 1]) { const result = run(index, 2); expect(result.error, result.stderr).toBeUndefined(); diff --git a/server/src/services/authorization.ts b/server/src/services/authorization.ts index ce5810ece00..7f57d0bce93 100644 --- a/server/src/services/authorization.ts +++ b/server/src/services/authorization.ts @@ -755,7 +755,8 @@ function createAuthorizationService(db: Db | DbTransaction, readSnapshot?: ReadA async function decideReadsForActors(actors: AuthorizationActor[], resources: ReadAuthorizationResource[]): Promise { if (!actors.length) return []; - const snapshots: ReadAuthorizationSnapshot[] = actors.map(() => ({ issues: new Map(), projects: new Map(), memo: new Map() })); + const memo = new Map>(); + const snapshots: ReadAuthorizationSnapshot[] = actors.map(() => ({ issues: new Map(), projects: new Map(), memo })); const batchActors: ResponsibleUserActorWithMemo[] = actors.map(actor => ({ ...actor, __responsibleUserSnapshotMemo: undefined })); const privacyKey = (actor: AuthorizationActor) => JSON.stringify([actor.type, actor.agentId, actor.userId, actor.onBehalfOfUserId]); const privacyActors = [...new Map(batchActors.map(actor => [privacyKey(actor), actor])).values()]; @@ -892,7 +893,8 @@ function createAuthorizationService(db: Db | DbTransaction, readSnapshot?: ReadA return promise; } - const promise = loadResponsibleUserSnapshot(input.companyId, input.userId); + const promise = batchRead(`responsible:${input.companyId}:${input.userId}`, + () => loadResponsibleUserSnapshot(input.companyId, input.userId)); actorWithMemo.__responsibleUserSnapshotMemo.set(key, promise); void promise.catch(() => { if (actorWithMemo.__responsibleUserSnapshotMemo?.get(key) === promise) { diff --git a/server/src/services/execution-workspace-repositories.ts b/server/src/services/execution-workspace-repositories.ts index 27c762d112c..47d94e6e80e 100644 --- a/server/src/services/execution-workspace-repositories.ts +++ b/server/src/services/execution-workspace-repositories.ts @@ -29,6 +29,11 @@ type DbTransaction = Parameters[0]>[0]; const controllerAcquisitionMessage = "New controller repository acquisition requires a trusted task. Existing verified public checkouts remain available."; const lowTrustRepositoryMessage = "Low-trust repository reuse permits anonymous public URLs only; new acquisition is unavailable. Use an authorized configured source for private repositories."; +const rootReaderRunColumns = { + id: heartbeatRuns.id, companyId: heartbeatRuns.companyId, agentId: heartbeatRuns.agentId, + responsibleUserId: heartbeatRuns.responsibleUserId, nativeIssueId: heartbeatRuns.nativeIssueId, issueId: heartbeatRuns.issueId, + contextSnapshot: sql>`jsonb_build_object('issueId', ${heartbeatRuns.contextSnapshot}->'issueId', 'taskId', ${heartbeatRuns.contextSnapshot}->'taskId', 'executionPolicy', ${heartbeatRuns.contextSnapshot}->'executionPolicy')`, +}; function resolveAcquisitionSource(request: PrepareWorkspaceRepository["repository"], available: ProjectRepository[], anonymousOnly: boolean) { if (anonymousOnly && request.kind === "catalog") throw forbidden(lowTrustRepositoryMessage); @@ -180,8 +185,17 @@ async function assertRootProjectAccess(db: Db | DbTransaction, actor: Authorizat } } +async function assertRootProjectsReadAccessForActors(db: Db | DbTransaction, actors: AuthorizationActor[], companyId: string, projectIds: Set) { + if (!actors.length || !projectIds.size) return; + const decisions = await authorizationService(db).decideReadsForActors(actors, + [...projectIds].map(projectId => ({ type: "project" as const, companyId, projectId }))); + if (decisions.some(batch => batch.some(decision => !decision.allowed))) { + throw forbidden("Workspace source project is no longer accessible"); + } +} + async function retainedRootReader(tx: DbTransaction, companyId: string, reader: Pick) { - const [run] = await tx.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, reader.scopeId), eq(heartbeatRuns.companyId, companyId))); + const [run] = await tx.select(rootReaderRunColumns).from(heartbeatRuns).where(and(eq(heartbeatRuns.id, reader.scopeId), eq(heartbeatRuns.companyId, companyId))); const [agent] = reader.agentId ? await tx.select({ id: agents.id }).from(agents).where(and(eq(agents.id, reader.agentId), eq(agents.companyId, companyId))) : []; const [issue] = reader.issueId ? await tx.select().from(issues).where(and(eq(issues.id, reader.issueId), eq(issues.companyId, companyId))) : []; if (!run || !agent || !issue || run.agentId !== reader.agentId || run.responsibleUserId !== reader.responsibleUserId @@ -260,8 +274,8 @@ async function retainGitMetadataOwners(tx: DbTransaction, input: { for (const reader of closure.overlappingEntries.filter(entry => entry.kind === "reader")) { const { actor } = await retainedRootReader(tx, input.companyId, reader); readerActors.push(actor); - await assertRootProjectAccess(tx, actor, input.companyId, projectIds); } + await assertRootProjectsReadAccessForActors(tx, readerActors, input.companyId, projectIds); await assertTaskWorkspacesReadAccessForActors(tx, readerActors, input.companyId, workspaceIds); for (const [kind, ids] of [["source", workspaceIds], ["project_source", projectIds]] as const) { for (const scopeId of ids) if (!closure.entries.some(entry => entry.kind === kind && entry.scopeId === scopeId)) { @@ -356,8 +370,8 @@ export async function retainIsolatedWorkspaceSourceAccess(db: Db, input: { const { actor, issue } = await retainedRootReader(tx, input.companyId, reader); readerActors.push(actor); await assertRootProjectTrust(tx, { companyId: input.companyId, actor, issueId: issue.id, projectIds: target.sourceProjectIds }); - await assertRootProjectAccess(tx, actor, input.companyId, target.sourceProjectIds); } + await assertRootProjectsReadAccessForActors(tx, readerActors, input.companyId, target.sourceProjectIds); await assertTaskWorkspacesReadAccessForActors(tx, readerActors, input.companyId, target.sourceIds); } catch (error) { failure ??= error; } if (!failure && verifiedCopyProjectIds) { @@ -385,7 +399,7 @@ async function assertRootProjectTrust(db: Db | DbTransaction, input: { if (!restricted.length) return; const incompatible = () => forbidden("Physical workspace source trust is incompatible with the task's execution policy"); const runId = input.actor.runId; - const [run] = runId ? await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, input.companyId))) : []; + const [run] = runId ? await db.select(rootReaderRunColumns).from(heartbeatRuns).where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, input.companyId))) : []; const creation = !input.issueId && input.actor.type === "agent" && runId ? input.taskCreation : undefined; const runIssueId = run?.nativeIssueId ?? run?.contextSnapshot?.issueId ?? run?.contextSnapshot?.taskId ?? run?.issueId; const issueId = input.issueId ?? (creation && typeof runIssueId === "string" ? runIssueId : null); @@ -471,7 +485,7 @@ export function executionWorkspaceRepositoryService(db: Db) { const [project] = issue.projectId ? await reader.select().from(projects).where(and(eq(projects.id, issue.projectId), eq(projects.companyId, issue.companyId))) : []; const [workspaceSourceProject] = workspace?.projectId ? await reader.select().from(projects).where(and(eq(projects.id, workspace.projectId), eq(projects.companyId, issue.companyId))) : []; const runId = input.runId ?? issue.executionRunId; - const [run] = runId ? await reader.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, issue.companyId))) : []; + const [run] = runId ? await reader.select(rootReaderRunColumns).from(heartbeatRuns).where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, issue.companyId))) : []; if (runId && (!run || (input.runId && run.agentId !== agentId) || (run.nativeIssueId ?? run.contextSnapshot?.issueId ?? run.contextSnapshot?.taskId ?? run.issueId) !== issue.id)) { throw forbidden("Task repository run identity is unavailable"); } @@ -545,12 +559,14 @@ export function executionWorkspaceRepositoryService(db: Db) { const sourceProjects = sourceIds.size ? await tx.selectDistinct({ projectId: executionWorkspaces.projectId }).from(executionWorkspaces).where(and( eq(executionWorkspaces.companyId, input.companyId), inArray(executionWorkspaces.id, [...sourceIds]))) : []; const sourceActors: AuthorizationActor[] = []; + const projectReaders: AuthorizationActor[] = []; for (const reader of readers) { const { actor, issue } = await retainedRootReader(tx, input.companyId, reader); await assertRootProjectTrust(tx, { companyId: input.companyId, projectIds: sourceProjectIds, actor, issueId: issue.id, sourceWorkspaceId: input.reader?.runId === reader.scopeId ? input.sourceWorkspaceId : undefined }); - await assertRootProjectAccess(tx, actor, input.companyId, sourceProjectIds, - input.reader?.runId === reader.scopeId ? { sharedProjectIds, issueId: issue.id } : undefined); + if (input.reader?.runId === reader.scopeId) { + await assertRootProjectAccess(tx, actor, input.companyId, sourceProjectIds, { sharedProjectIds, issueId: issue.id }); + } else projectReaders.push(actor); if (ownAdmissionOnly) { const [ownSource] = await tx.select({ id: executionWorkspaces.id }).from(executionWorkspaces).where(and( eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, input.sourceWorkspaceId!), @@ -566,6 +582,7 @@ export function executionWorkspaceRepositoryService(db: Db) { } } } + await assertRootProjectsReadAccessForActors(tx, projectReaders, input.companyId, sourceProjectIds); if (input.actor) { if (!input.issueId) throw forbidden("Workspace write authorization requires a task"); await assertRootProjectTrust(tx, { ...input, projectIds: sourceProjectIds, actor: input.actor }); @@ -676,7 +693,7 @@ export function executionWorkspaceRepositoryService(db: Db) { throw forbidden("Task repository publication binding is unavailable"); } await requiresAnonymousAcquisition(tx, { issue, workspace, agentId: input.agentId, runId: input.runId }); - const [run] = await tx.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, input.runId), eq(heartbeatRuns.companyId, input.companyId))); + const [run] = await tx.select({ responsibleUserId: heartbeatRuns.responsibleUserId }).from(heartbeatRuns).where(and(eq(heartbeatRuns.id, input.runId), eq(heartbeatRuns.companyId, input.companyId))); const { actor } = await retainedRootReader(tx, input.companyId, { scopeId: input.runId, issueId: input.issueId, agentId: input.agentId, responsibleUserId: run?.responsibleUserId ?? null, });