From 0998ec77036af456532b2c52963a6f5f2a70ff1f Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 29 Sep 2026 16:20:50 -0500 Subject: [PATCH] feat(history): page run output and bound browser history Part 5 of the opt-in indexed history draft series. Validated at the integrated tip; activation remains disabled by default. Co-Authored-By: Paperclip --- doc/run-log-events.md | 84 +++ .../__tests__/agent-live-run-routes.test.ts | 27 +- .../src/__tests__/attention-service.test.ts | 9 +- .../chat-channels.integration.test.ts | 26 + .../__tests__/cleanup-removal-service.test.ts | 25 +- .../src/__tests__/environment-runtime.test.ts | 2 +- .../heartbeat-run-event-sequencing.test.ts | 92 ++- .../native-finalization-migration.test.ts | 77 ++- server/src/__tests__/openapi-routes.test.ts | 2 +- server/src/__tests__/redaction.test.ts | 11 + .../server-startup-feedback-export.test.ts | 8 + .../__tests__/storage-local-provider.test.ts | 16 + server/src/redaction.ts | 12 + server/src/routes/agents.ts | 31 +- server/src/routes/issues.ts | 4 +- server/src/routes/openapi.ts | 61 +- server/src/services/activity.ts | 9 +- .../src/services/attention-exhausted-runs.ts | 8 +- server/src/services/chat-channels.ts | 2 + .../chat-publication-reconciliation.ts | 7 + server/src/services/chat-run-publications.ts | 36 +- .../src/services/conversation-continuation.ts | 7 +- server/src/services/environment-runtime.ts | 4 +- server/src/services/feedback.ts | 39 +- server/src/services/heartbeat-run-events.ts | 170 +++-- server/src/services/heartbeat.ts | 159 ++--- server/src/services/run-event-excerpt.ts | 30 + server/src/services/run-event-history.test.ts | 90 +++ server/src/services/run-event-history.ts | 155 +++++ server/src/services/run-log-store.test.ts | 26 + server/src/services/run-log-store.ts | 107 ++- server/src/services/run-output-body.test.ts | 39 ++ server/src/services/run-output-body.ts | 90 +++ server/src/services/runner-goals.test.ts | 99 ++- server/src/services/runner-goals.ts | 51 +- ...mented-run-log-scale.qualification.test.ts | 60 ++ .../services/segmented-run-log-store.test.ts | 643 ++++++++++++++++++ .../src/services/segmented-run-log-store.ts | 603 ++++++++++++++++ server/src/services/work-products.ts | 13 +- server/src/storage/local-disk-provider.ts | 42 +- server/src/storage/s3-provider.ts | 18 + server/src/storage/types.ts | 5 + ui/src/adapters/transcript.ts | 2 +- ui/src/api/heartbeats.test.ts | 21 + ui/src/api/heartbeats.ts | 17 +- ui/src/components/LiveRunWidget.tsx | 1 + ui/src/components/RunOutputDownload.tsx | 12 + ui/src/components/RunnerInspector.test.tsx | 28 + ui/src/components/RunnerInspector.tsx | 52 +- ui/src/components/TaskChatThread.test.tsx | 31 + ui/src/components/TaskChatThread.tsx | 46 +- .../task-chat/RunnerGoalWidget.test.tsx | 30 +- .../components/task-chat/RunnerGoalWidget.tsx | 18 +- .../task-chat/task-protocol-surfaces.ts | 1 + .../transcript/native-run-events.test.ts | 14 +- .../transcript/native-run-events.ts | 32 +- .../transcript/useLiveRunTranscripts.test.tsx | 205 +++++- .../transcript/useLiveRunTranscripts.ts | 133 +++- .../useNativeRunTranscripts.test.tsx | 56 +- .../transcript/useNativeRunTranscripts.ts | 82 ++- .../components/useSummaryDraftStream.test.tsx | 6 +- ui/src/components/useSummaryDraftStream.ts | 22 +- ui/src/lib/issue-chat-messages.ts | 1 + ui/src/lib/issueChatTranscriptRuns.ts | 3 + ui/src/lib/run-event-pagination.test.ts | 58 ++ ui/src/lib/run-event-pagination.ts | 58 ++ ui/src/lib/run-log-chunks.test.ts | 62 +- ui/src/lib/run-log-chunks.ts | 70 +- ui/src/lib/run-log-position.test.ts | 12 + ui/src/lib/run-log-position.ts | 12 + .../pages/AgentDetail.log-visibility.test.tsx | 77 +++ ui/src/pages/AgentDetail.production.tsx | 157 +++-- ui/src/pages/AgentDetail.tsx | 154 +++-- 73 files changed, 3875 insertions(+), 597 deletions(-) create mode 100644 server/src/services/run-event-excerpt.ts create mode 100644 server/src/services/run-event-history.test.ts create mode 100644 server/src/services/run-event-history.ts create mode 100644 server/src/services/run-output-body.test.ts create mode 100644 server/src/services/run-output-body.ts create mode 100644 server/src/services/segmented-run-log-scale.qualification.test.ts create mode 100644 server/src/services/segmented-run-log-store.test.ts create mode 100644 server/src/services/segmented-run-log-store.ts create mode 100644 ui/src/components/RunOutputDownload.tsx create mode 100644 ui/src/lib/run-event-pagination.test.ts create mode 100644 ui/src/lib/run-event-pagination.ts create mode 100644 ui/src/lib/run-log-position.test.ts create mode 100644 ui/src/lib/run-log-position.ts diff --git a/doc/run-log-events.md b/doc/run-log-events.md index e468d7ff86f..a1aa6874710 100644 --- a/doc/run-log-events.md +++ b/doc/run-log-events.md @@ -5,6 +5,59 @@ Run-log events write to the `heartbeat_run_events` table Paperclip Telemetry events, and they are not OpenTelemetry exports. A run-log event needs no operator endpoint. +## Segmented stdout/stderr history + +The experimental `PAPERCLIP_NATIVE_INDEXED_STATE=1` lane also selects +`local_segments` for fresh stdout/stderr logs. Each segment is at most 32 MiB; +the current head contains exact decimal byte/revision counters and the current +tail digest. It contains no list of old segments. Segment lookup is arithmetic, +and append, reopen, mirroring and finalization never scan completed history. +Completed segments have independently stored SHA-256 references. Finalization +reports exact bytes and does not calculate a whole-history SHA-256. + +The existing authorized `GET /api/heartbeat-runs/:runId/log` accepts `cursor` +as an exact decimal byte offset. Responses include `cursor` and `hasMore`. +Clients retain that cursor verbatim, including when caught up; rendered string +length is not a byte offset. `offset`/`nextOffset` remain compatible within the +safe numeric range. Pages keep UTF-8 characters intact. The board's two log +viewers support both formats; feedback captures its existing bounded excerpt. + +Completed segments reach configured object storage before local publication. +Periodic mirroring uploads only a bounded tail and then its small head, using +a distinct tail identity per writer and revision. The v2 head is published with +an atomic S3 ETag condition; opening a replacement writer claims a new UUID so a +late old publication is rejected. Completed remote bodies use content-addressed +keys, with separately fenced per-segment references. An interrupted unpublished +segment can be replaced without overwriting bytes referenced by a committed +head. Existing v1 heads/references remain readable and upgrade when reopened for +writing; older readers reject the v2 head explicitly. + +Routine mirroring retires its previous tail through one durable pending-deletion +field. A reader whose old tail was removed retries once against newer content +and verifies the exact original prefix before returning its page. A deletion +failure fences further writes until recovery can finish it. Unreferenced objects +from unknown prepublication failures still require orphan maintenance. Shared +local paths continue to require exclusive run ownership; remote conditional +writes do not provide an operating-system lock on a shared filesystem. + +The configured in-flight mirror interval +still defines the unmirrored tail window on loss of the local volume. A failed +write/upload fences the writer; it must be reopened under verified run ownership. +Restoration reads the head and current tail, fetching older segments on demand. +Reopening never truncates a retained legacy log. Turning the experiment off does +not change the format of an existing segmented log. + +The local chunk path uses private files, sync and directory sync. Object uploads +carry exact length and SHA-256 checksums. A historical page validates its +segment reference and the complete bounded segment before returning bytes. The +first read can fetch up to 32 MiB; at most two verified segments are cached and +at most two distinct verification jobs run concurrently. Subsequent pages can +reuse that cache. Current tails verify the exact prefix declared by the head. +Storage qualification covers local files, fault fixtures, and a real local +MinIO server, including volume loss, continuation, corruption and missing old +history. Hosted object storage and remote runner lifecycles remain unqualified. These chunks hold historical output, not execution +authority, and do not externalize large PRP inputs/results. + ## Native PRP Run-Log Events The hidden native coordinator writes each validated PRP event to the bound @@ -34,6 +87,37 @@ credential material are never written to the run log. These records remain run-log events. They do not create an OpenTelemetry or Paperclip Telemetry export, and legacy adapters do not use this writer. +## Native output bodies and current process evidence + +Indexed output uses authenticated `output.body.chunk` events followed by an +immutable `paperclip.output.body.v1` reference on the message or tool event. +Chunks are at most 32 KiB; one admitted provider frame is at most 4 MiB. These +are per-frame limits, with no cumulative output limit. The small +`native_output_body_chunks` catalog is committed with run-event insertion before +ACK, enabling a company/run/body lookup without searching the transcript. +New chunk events contain scoped immutable payload references, not the output +text. The configured local/S3 provider must durably write and verify the bytes +before the event transaction publishes the reference. Retained inline chunk +events still work. Raw controller receipts larger than 16 KiB use the same +payload store, with original byte sizes and hashes retained for replay and +bounded page admission. +`GET /api/heartbeat-runs/:runId/output-bodies/:bodyId` verifies every chunk, +contiguous offsets, total bytes and SHA-256 before returning a download. It uses +the existing run-telemetry access policy. Public event pages contain chunk +metadata only; whole-body run-secret redaction happens before the authorized +download so a secret split between chunks is not exposed. The board exposes a +"Download full output" link beside the referenced event. + +Feedback captures a recent event excerpt bounded by both row count and encoded +bytes, and reports truncation. It never collects the whole event history. + +The `native_run_process_evidence` table stores the latest server-authored launch, +process-identity or stop observation per run, including a cached absence. It is +updated in the same transaction as the event. Older runs receive a single +backfill under the run lock. Repeated recovery checks use this current row; +provider-authored events cannot update it. It preserves the existing process +stop observation semantics and does not prove that escaped descendants stopped. + ## Native Restart Recovery Run-Log Event Paperclip writes a `native.recovery.transition` event for every native restart diff --git a/server/src/__tests__/agent-live-run-routes.test.ts b/server/src/__tests__/agent-live-run-routes.test.ts index 28919226bde..34641558a6f 100644 --- a/server/src/__tests__/agent-live-run-routes.test.ts +++ b/server/src/__tests__/agent-live-run-routes.test.ts @@ -3,6 +3,7 @@ import request from "supertest"; import { type SQL } from "drizzle-orm"; import { PgDialect } from "drizzle-orm/pg-core"; import { beforeEach, describe, expect, it, vi } from "vitest"; +import { runEventLane } from "../services/run-event-history.js"; const mockAgentService = vi.hoisted(() => ({ getById: vi.fn(), @@ -203,13 +204,29 @@ function createLiveRunsDbStub(rows: Array>) { } function createRuntimeRequestDbStub(row: Record) { + const executedStatements: SQL[] = []; + const execute = vi.fn(async (statement: SQL) => { + executedStatements.push(statement); + return [{ event_id: String(row.id) }]; + }); const query = { from: vi.fn().mockReturnThis(), where: vi.fn().mockReturnThis(), orderBy: vi.fn().mockReturnThis(), limit: vi.fn(async () => [row]), + then: (resolve: (value: Array>) => unknown) => + Promise.resolve([row]).then(resolve), }; - return { select: vi.fn(() => query) }; + return { select: vi.fn(() => query), execute, executedStatements }; +} + +function expectRuntimeRequestLaneBinding(db: ReturnType) { + expect(db.executedStatements.length).toBeGreaterThan(0); + for (const statement of db.executedStatements) { + const { params } = new PgDialect().sqlToQuery(statement); + expect(params).toContain("aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"); + expect(params).toContain(runEventLane("request", "approval-1")); + } } function createFailedChatRetryDb(chatBound = true) { @@ -1551,6 +1568,9 @@ describe("agent live run routes", () => { runtimeMode: "native", }); const db = createRuntimeRequestDbStub({ + id: "event-approval-1", + companyId: "company-1", + runId: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", eventType: "runtime_request.created", payload: { prpEvent: { @@ -1587,6 +1607,7 @@ describe("agent live run routes", () => { })); expect(res.status).toBe(403); + expectRuntimeRequestLaneBinding(db); expect(mockQueueRuntimeRequestResolution).not.toHaveBeenCalled(); }); @@ -1599,6 +1620,9 @@ describe("agent live run routes", () => { runtimeMode: "native", }); const db = createRuntimeRequestDbStub({ + id: "event-approval-1", + companyId: "company-1", + runId: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", eventType: "runtime_request.created", payload: { prpEvent: { @@ -1635,6 +1659,7 @@ describe("agent live run routes", () => { })); expect(res.status, JSON.stringify(res.body)).toBe(202); + expectRuntimeRequestLaneBinding(db); expect(mockQueueRuntimeRequestResolution).toHaveBeenCalledWith({ companyId: "company-1", runId: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", diff --git a/server/src/__tests__/attention-service.test.ts b/server/src/__tests__/attention-service.test.ts index 889b9a9ace6..7b87bb9ddc2 100644 --- a/server/src/__tests__/attention-service.test.ts +++ b/server/src/__tests__/attention-service.test.ts @@ -951,11 +951,16 @@ describeEmbeddedPostgres("attention service", () => { { companyId, agentId: workerId, runId: succeededId, seq: 1, eventType: "lifecycle", message: "Bounded retry exhausted success" }, { companyId, agentId: reviewerId, runId: terminatedId, seq: 1, eventType: "lifecycle", message: "Bounded retry exhausted terminated" }, { companyId: other.companyId, agentId: other.workerId, runId: foreignId, seq: 1, eventType: "lifecycle", message: "Bounded retry exhausted other company" }, - // A newer event must not replace the latest matching, company-scoped receipt. - { companyId: other.companyId, agentId: other.workerId, runId: failedId, seq: 2501, eventType: "lifecycle", message: "Bounded retry exhausted foreign receipt" }, + // A newer nonmatching event must not replace the exhaustion receipt. { companyId, agentId: workerId, runId: failedId, seq: 2502, eventType: "stdout", message: "Bounded retry exhausted quoted output" }, { companyId, agentId: workerId, runId: failedId, seq: 2503, eventType: "lifecycle", message: "Unrelated lifecycle event" }, ]); + // Run-event binding is now enforced at insertion as well as by the reader. + // An attempted foreign receipt cannot replace the company-scoped receipt. + await expect(db.insert(heartbeatRunEvents).values({ + companyId: other.companyId, agentId: other.workerId, runId: failedId, + seq: 2501, eventType: "lifecycle", message: "Bounded retry exhausted foreign receipt", + })).rejects.toMatchObject({ cause: { message: "heartbeat_run_event_binding_mismatch" } }); // Assert the database result itself: JavaScript feed deduplication used to // hide the thousands of full run contexts already loaded into memory. diff --git a/server/src/__tests__/chat-channels.integration.test.ts b/server/src/__tests__/chat-channels.integration.test.ts index 85d968cbc1c..987d5647a05 100644 --- a/server/src/__tests__/chat-channels.integration.test.ts +++ b/server/src/__tests__/chat-channels.integration.test.ts @@ -7,6 +7,7 @@ import { githubAutomaticReviewEvent } from "../services/chat-github-events.js"; import { githubBotToolsForSession } from "../services/chat-github-tools.js"; import { resolveGitHubOperationCredentials } from "../services/github-operation-credentials.js"; import { initializeRunIdentity } from "../services/run-identity.js"; +import { allocateRunEventPosition, runEventCursor } from "../services/run-event-history.js"; import { chatGitHubRegistrations, chatGitHubReviews, toolCatalogEntries } from "@paperclipai/db"; import { AsyncLocalStorage } from "node:async_hooks"; import * as cloudRuntimeIdentity from "../services/cloud-runtime-identity.js"; @@ -59101,6 +59102,31 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { }, ); + it("publishes current native progress through an event epoch boundary", async () => { + const context = await safeNativeProgressFixture("telegram", "803"); + try { + const run = await context.createRun("history epoch"); + await context.addEvent(run, "item.completed", 100); + await db.update(heartbeatRuns).set({ nextEventSeq: 101 }).where(eq(heartbeatRuns.id, run.runId)); + const position = await db.transaction(async tx => { + const allocated = await allocateRunEventPosition(tx as unknown as typeof db, run.runId, 100); + await tx.insert(heartbeatRunEvents).values({ + companyId: context.fixture.companyId, agentId: context.fixture.assignedAgentId, + runId: run.runId, ...allocated, eventType: "tool.execution.completed", + createdAt: new Date(run.baseCreatedAt.getTime() + 40_000), + }); + return allocated; + }); + expect(position.seq).toBe(1); + expect(position.eventEpoch).not.toBe(""); + await expect(enqueueChatRunMilestones(db, { since: new Date(0) })).resolves.toBe(1); + const rows = await db.select({ key: chatPublications.idempotencyKey }).from(chatPublications) + .where(like(chatPublications.idempotencyKey, `run:${run.runId}:working:${context.endpoint.id}:native:%`)); + expect(rows).toEqual([{ key: `run:${run.runId}:working:${context.endpoint.id}:native:using_tools:${runEventCursor(position)}` }]); + await expect(enqueueChatRunMilestones(db, { since: new Date(0) })).resolves.toBe(0); + } finally { await context.service.shutdown(); } + }); + it("enforces the native progress cadence boundary and one publication per phase", async () => { const context = await safeNativeProgressFixture("telegram", "8"); const run = await context.createRun("cadence boundaries"); diff --git a/server/src/__tests__/cleanup-removal-service.test.ts b/server/src/__tests__/cleanup-removal-service.test.ts index 68bf1ce2c17..f8ca84b5027 100644 --- a/server/src/__tests__/cleanup-removal-service.test.ts +++ b/server/src/__tests__/cleanup-removal-service.test.ts @@ -1,5 +1,5 @@ import { randomUUID } from "node:crypto"; -import { eq } from "drizzle-orm"; +import { eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; import { activityLog, @@ -244,14 +244,23 @@ describeEmbeddedPostgres("cleanup removal services", () => { requireBoardApprovalForNewAgents: false, }); - await db.insert(heartbeatRunEvents).values({ - companyId: otherCompanyId, - runId, - agentId, - seq: 1, - eventType: "output", - message: "event with mismatched company scope", + // Simulate a retained row written before the run-event binding trigger. + // This transaction-local setting affects only the isolated test database; + // ordinary writers must still reject this scope mismatch. + await db.transaction(async (tx) => { + await tx.execute(sql`SET LOCAL session_replication_role = replica`); + await tx.insert(heartbeatRunEvents).values({ + companyId: otherCompanyId, + runId, + agentId, + seq: 1, + eventType: "output", + message: "event with mismatched company scope", + }); }); + await expect(db.insert(heartbeatRunEvents).values({ + companyId: otherCompanyId, runId, agentId, seq: 2, eventType: "output", + })).rejects.toMatchObject({ cause: { message: "heartbeat_run_event_binding_mismatch" } }); const removed = await companyService(db).remove(companyId); diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index 8ec03b56d3a..b45b41115cd 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -5101,7 +5101,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, }, } as never; - const manifest = buildNativeHarnessBackupManifest({ + const manifest = await buildNativeHarnessBackupManifest({ backupRoot: current, execution, runnerInstanceId, diff --git a/server/src/__tests__/heartbeat-run-event-sequencing.test.ts b/server/src/__tests__/heartbeat-run-event-sequencing.test.ts index 28c2dd00d24..4f0f94f6a86 100644 --- a/server/src/__tests__/heartbeat-run-event-sequencing.test.ts +++ b/server/src/__tests__/heartbeat-run-event-sequencing.test.ts @@ -1,20 +1,80 @@ -import { describe, expect, it } from "vitest"; -import { eq } from "drizzle-orm"; +import { allocateRunEventPosition } from "../services/run-event-history.js"; +import { createHash } from "node:crypto"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { resolve } from "node:path"; +import { readRunOutputBody } from "../services/run-output-body.js"; +import { readRunEventExcerpt } from "../services/run-event-excerpt.js"; +import { describe, expect, it, vi } from "vitest"; +import { eq, sql } from "drizzle-orm"; import { agents, companies, createDb, + type Db, heartbeatRunEvents, heartbeatRuns, + nativeSourceCursors, + nativeOutputBodyChunks, } from "@paperclipai/db"; import { - allocateHeartbeatRunEventSeq, appendHeartbeatRunEvent, HeartbeatRunEventConflictError, } from "../services/heartbeat-run-events.js"; import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; describe("P6-11..13 / P6-17 canonical event allocator", () => { + it("commits the bounded output catalog with event ACKs and enforces company scope", async () => { + const storage = mkdtempSync(resolve(tmpdir(), "paperclip-output-objects-")); + vi.stubEnv("PAPERCLIP_STORAGE_PROVIDER", "local_disk"); + vi.stubEnv("PAPERCLIP_STORAGE_LOCAL_DIR", storage); + const temporary = await startEmbeddedPostgresTestDatabase("paperclip-output-body-"); + const db = createDb(temporary.connectionString); + const companyId = "20000000-0000-4000-8000-000000000001"; + const agentId = "20000000-0000-4000-8000-000000000002"; + const runId = "20000000-0000-4000-8000-000000000003"; + const text = "x\0".repeat(24_000); + const hash = (s: string) => createHash("sha256").update(s).digest("hex"); + const bodyId = hash(text), body = { schema: "paperclip.output.body.v1", bodyId, sha256: bodyId, + byteLength: String(Buffer.byteLength(text)), mediaType: "text/plain; charset=utf-8" }; + try { + await db.insert(companies).values({ id: companyId, name: "Output fixture", issuePrefix: "OUT" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Output agent" }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running" }); + let sourceSeq = 0; + for (let offset = 0; offset < text.length; offset += 8192) { + const part = text.slice(offset, offset + 8192); + const event = { payload: { schema: "paperclip.output.body.chunk.v1", body, offset: String(offset), sha256: hash(part), text: part } }; + const input = { companyId, runId, agentId, eventType: "output.body.chunk", payload: { prpEvent: event }, + nativeSource: { sourceInstanceId: "output-runner", sourceEventId: `chunk-${++sourceSeq}`, sourceSeq, protocolSchemaVersion: 1, canonicalPayload: event } }; + expect((await appendHeartbeatRunEvent(db, input)).disposition).toBe("committed"); + expect((await appendHeartbeatRunEvent(db, input)).disposition).toBe("duplicate"); + if (offset === 0) await expect(readRunOutputBody(db, companyId, runId, bodyId)).rejects.toThrow("not complete"); + } + expect(await readRunOutputBody(db, companyId, runId, bodyId)).toBe(text); + await expect(readRunOutputBody(db, "20000000-0000-4000-8000-000000000099", runId, bodyId)).rejects.toThrow("not found"); + expect(await db.select().from(nativeOutputBodyChunks)).toHaveLength(sourceSeq); + const excerpt = await readRunEventExcerpt(db, companyId, runId, { events: 2, bytes: 200_000 }); + expect(excerpt.truncated).toBe(true); + expect(excerpt.events.map(event => event.seq)).toEqual([sourceSeq - 1, sourceSeq]); + expect(JSON.stringify(excerpt.events)).not.toContain("\\u0000"); + expect(excerpt.events[0]?.payload).toMatchObject({ prpEvent: { payload: { body } } }); + const [last] = await db.select({ bytes: sql`octet_length(row_to_json(${heartbeatRunEvents})::text)` }).from(heartbeatRunEvents) + .where(eq(heartbeatRunEvents.seq, sourceSeq)); + expect((await readRunEventExcerpt(db, companyId, runId, { events: 2, bytes: last!.bytes + 3 })).events).toHaveLength(1); + const stored = await db.select().from(heartbeatRunEvents); + expect(JSON.stringify(stored)).not.toContain("\\u0000"); + expect(stored[0]?.payload).toMatchObject({ prpEvent: { payload: { textRef: { schema: "paperclip.history-payload.v1", companyId, runId } } } }); + expect(await readRunEventExcerpt(db, companyId, runId, { events: 2, bytes: 1 })).toEqual({ events: [], truncated: true }); + expect(await readRunEventExcerpt(db, "20000000-0000-4000-8000-000000000099", runId)).toEqual({ events: [], truncated: false }); + await expect(appendHeartbeatRunEvent(db, { companyId, runId, agentId, eventType: "output.body.chunk", + nativeSource: { sourceInstanceId: "output-runner", sourceEventId: "invalid-body", sourceSeq: sourceSeq + 1, protocolSchemaVersion: 1, canonicalPayload: { payload: { body: { bodyId: "invalid" } } } }, + })).rejects.toThrow("native_output_body_binding_invalid"); + expect(await db.select().from(heartbeatRunEvents)).toHaveLength(sourceSeq); + expect((await db.select().from(heartbeatRuns))[0]?.nextEventSeq).toBe(sourceSeq + 1); + } finally { await temporary.cleanup(); vi.unstubAllEnvs(); rmSync(storage, { recursive: true, force: true }); } + }, 60_000); + it.each([false, true])("deduplicates concurrent retry exhaustion (historical receipt: %s)", async (historical) => { const temporary = await startEmbeddedPostgresTestDatabase("paperclip-exhaustion-events-"); const db = createDb(temporary.connectionString); @@ -112,18 +172,18 @@ describe("P6-11..13 / P6-17 canonical event allocator", () => { expect((await db.select({ nextEventSeq: heartbeatRuns.nextEventSeq }).from(heartbeatRuns) .where(eq(heartbeatRuns.id, runId)))[0]?.nextEventSeq).toBe(33); - const legacyWrites = Array.from({ length: 16 }, (_, index) => (async () => { - const seq = await allocateHeartbeatRunEventSeq(db, runId); - await db.insert(heartbeatRunEvents).values({ + const legacyWrites = Array.from({ length: 16 }, (_, index) => db.transaction(async (tx) => { + const position = await allocateRunEventPosition(tx as unknown as Db, runId); + await tx.insert(heartbeatRunEvents).values({ companyId, runId, agentId, - seq, + ...position, eventType: "stdout", message: `legacy-event-${index + 1}`, }); - return seq; - })()); + return position.seq; + })); const legacySequences = await Promise.all(legacyWrites); expect([...legacySequences].sort((a, b) => a - b)).toEqual( Array.from({ length: 16 }, (_, index) => index + 33), @@ -161,6 +221,20 @@ describe("P6-11..13 / P6-17 canonical event allocator", () => { }, })).rejects.toBeInstanceOf(HeartbeatRunEventConflictError); expect(await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, runId))).toHaveLength(48); + + const gapEvent = (sourceSeq: number) => ({ + companyId, runId, agentId, eventType: "stdout", + nativeSource: { sourceInstanceId: "gap-writer", sourceEventId: `gap-${sourceSeq}`, sourceSeq, + protocolSchemaVersion: 1, canonicalPayload: { sourceSeq } }, + }); + // Cached cursors advance through the contiguous prefix, never across a + // gap. Old installations build the same cursor once from existing rows. + expect((await appendHeartbeatRunEvent(db, gapEvent(2))).highestContiguousSourceSeq).toBe(0); + expect((await appendHeartbeatRunEvent(db, gapEvent(4))).highestContiguousSourceSeq).toBe(0); + expect((await appendHeartbeatRunEvent(db, gapEvent(1))).highestContiguousSourceSeq).toBe(2); + expect((await appendHeartbeatRunEvent(db, gapEvent(3))).highestContiguousSourceSeq).toBe(4); + await db.delete(nativeSourceCursors).where(eq(nativeSourceCursors.sourceInstanceId, "gap-writer")); + expect((await appendHeartbeatRunEvent(db, gapEvent(1))).highestContiguousSourceSeq).toBe(4); } finally { await temporary.cleanup(); } diff --git a/server/src/__tests__/native-finalization-migration.test.ts b/server/src/__tests__/native-finalization-migration.test.ts index 6ba1a2cbd24..9f114c319f0 100644 --- a/server/src/__tests__/native-finalization-migration.test.ts +++ b/server/src/__tests__/native-finalization-migration.test.ts @@ -5,7 +5,6 @@ import { eq, sql } from "drizzle-orm"; import { applyPendingMigrations, createDb, - heartbeatRunEvents, heartbeatRuns, } from "@paperclipai/db"; import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; @@ -37,18 +36,36 @@ describe("P6-18 / MIG-01..04 native finalization migration", () => { native_run_finalizations, native_run_results, completion_contracts CASCADE; DROP TRIGGER IF EXISTS paperclip_issue_status_version_trigger ON issues; DROP FUNCTION IF EXISTS paperclip_bump_issue_status_version(); - DROP INDEX IF EXISTS heartbeat_run_events_run_seq_uq; - CREATE INDEX IF NOT EXISTS heartbeat_run_events_run_seq_idx + -- Recreate the original 0001 table shape. Downgrading columns in place + -- would leave later AFTER INSERT triggers installed against fields that + -- did not exist before 0227. + DROP TABLE heartbeat_run_events CASCADE; + CREATE TABLE heartbeat_run_events ( + id bigserial PRIMARY KEY NOT NULL, + company_id uuid NOT NULL, + run_id uuid NOT NULL, + agent_id uuid NOT NULL, + seq integer NOT NULL, + event_type text NOT NULL, + stream text, + level text, + color text, + message text, + payload jsonb, + created_at timestamptz DEFAULT now() NOT NULL, + CONSTRAINT heartbeat_run_events_company_id_companies_id_fk + FOREIGN KEY (company_id) REFERENCES companies(id), + CONSTRAINT heartbeat_run_events_run_id_heartbeat_runs_id_fk + FOREIGN KEY (run_id) REFERENCES heartbeat_runs(id), + CONSTRAINT heartbeat_run_events_agent_id_agents_id_fk + FOREIGN KEY (agent_id) REFERENCES agents(id) + ); + CREATE INDEX heartbeat_run_events_run_seq_idx ON heartbeat_run_events (run_id, seq); - DROP INDEX IF EXISTS heartbeat_run_events_run_source_event_uq; - DROP INDEX IF EXISTS heartbeat_run_events_run_source_seq_uq; - ALTER TABLE heartbeat_run_events - DROP COLUMN IF EXISTS source_instance_id, - DROP COLUMN IF EXISTS source_event_id, - DROP COLUMN IF EXISTS source_seq, - DROP COLUMN IF EXISTS source_payload_sha256, - DROP COLUMN IF EXISTS protocol_schema_version; - ALTER TABLE heartbeat_run_events ALTER COLUMN seq TYPE integer; + CREATE INDEX heartbeat_run_events_company_run_idx + ON heartbeat_run_events (company_id, run_id); + CREATE INDEX heartbeat_run_events_company_created_idx + ON heartbeat_run_events (company_id, created_at); ALTER TABLE heartbeat_runs DROP COLUMN IF EXISTS runtime_mode, DROP COLUMN IF EXISTS runtime_mode_resolver_version, @@ -99,32 +116,40 @@ describe("P6-18 / MIG-01..04 native finalization migration", () => { await applyPendingMigrations(temporary.connectionString); const db = createDb(temporary.connectionString); - const after = await db.select().from(heartbeatRunEvents) - .where(eq(heartbeatRunEvents.runId, runId)).orderBy(heartbeatRunEvents.id); - expect(after.map((row) => row.seq)).toEqual([1, 5, 10, 9]); + // The fixture deliberately stops at migration 0235 while the generated + // Drizzle schema includes later columns, so inspect this historical shape + // with SQL instead of selecting through the current schema definition. + const afterResult = await rawDb.execute(sql` + SELECT id::text AS id, company_id, run_id, agent_id, seq::integer AS seq, + event_type, stream, level, color, message, payload, created_at + FROM heartbeat_run_events WHERE run_id = ${runId} ORDER BY id + `); + const after = [...afterResult] as unknown as Record[]; + expect(after.map((row) => Number(row.seq))).toEqual([1, 5, 10, 9]); expect((await db.select({ nextEventSeq: heartbeatRuns.nextEventSeq }).from(heartbeatRuns) .where(eq(heartbeatRuns.id, runId)))[0]?.nextEventSeq).toBe(11); // The repaired duplicate's cursor is the only changed byte-equivalent read field. const legacyColumns = (row: Record) => ({ id: String(row.id), - companyId: row.companyId ?? row.company_id, - runId: row.runId ?? row.run_id, - agentId: row.agentId ?? row.agent_id, - eventType: row.eventType ?? row.event_type, + companyId: row.company_id, + runId: row.run_id, + agentId: row.agent_id, + eventType: row.event_type, stream: row.stream, level: row.level, message: row.message, payload: row.payload, - createdAt: new Date(String(row.createdAt ?? row.created_at)).toISOString(), + createdAt: new Date(String(row.created_at)).toISOString(), }); expect(after.map((row) => legacyColumns(row))).toEqual(before.map(legacyColumns)); - expect(after[0]?.seq).toBe(Number(before[0]?.seq)); - expect(after[1]?.seq).toBe(Number(before[1]?.seq)); - expect(after[3]?.seq).toBe(Number(before[3]?.seq)); - await expect(db.insert(heartbeatRunEvents).values({ - companyId, runId, agentId, seq: 5, eventType: "must-conflict", - })).rejects.toThrow(); + expect(Number(after[0]?.seq)).toBe(Number(before[0]?.seq)); + expect(Number(after[1]?.seq)).toBe(Number(before[1]?.seq)); + expect(Number(after[3]?.seq)).toBe(Number(before[3]?.seq)); + await expect(rawDb.execute(sql` + INSERT INTO heartbeat_run_events (company_id, run_id, agent_id, seq, event_type) + VALUES (${companyId}, ${runId}, ${agentId}, 5, 'must-conflict') + `)).rejects.toThrow(); } finally { await temporary.cleanup(); } diff --git a/server/src/__tests__/openapi-routes.test.ts b/server/src/__tests__/openapi-routes.test.ts index 479b1418a5d..9665e99745c 100644 --- a/server/src/__tests__/openapi-routes.test.ts +++ b/server/src/__tests__/openapi-routes.test.ts @@ -953,7 +953,7 @@ describe("heartbeat run ID OpenAPI contract", () => { checked++; } } - expect(checked).toBe(12); + expect(checked).toBe(13); }); }); diff --git a/server/src/__tests__/redaction.test.ts b/server/src/__tests__/redaction.test.ts index 3dc9e391f61..24979962583 100644 --- a/server/src/__tests__/redaction.test.ts +++ b/server/src/__tests__/redaction.test.ts @@ -11,6 +11,17 @@ import { } from "../redaction.js"; describe("redaction", () => { + it("keeps full-output references public but withholds fragments until whole-body redaction", () => { + const body = { schema: "paperclip.output.body.v1", bodyId: "a".repeat(64), sha256: "a".repeat(64), byteLength: "100", mediaType: "text/plain; charset=utf-8" }; + const preview = { prpEvent: { schema: "paperclip.prp.event.v1", schemaVersion: 1, eventType: "item.completed", payload: { outputBody: body } } }; + expect(redactEventPayload(preview)).toEqual(preview); + const chunk = { schema: "paperclip.output.body.chunk.v1", body, offset: "0", sha256: "b".repeat(64), text: "fragment of a registered credential" }; + const event = { prpEvent: { schema: "paperclip.prp.event.v1", schemaVersion: 1, eventType: "output.body.chunk", payload: chunk } }; + const sanitized = redactEventPayload(event); + expect(sanitized).toEqual({ prpEvent: { ...event.prpEvent, payload: { schema: chunk.schema, body, offset: "0", sha256: chunk.sha256 } } }); + expect(event.prpEvent.payload.text).toBe("fragment of a registered credential"); + expect(JSON.stringify(sanitized)).not.toContain("fragment"); + }); it("keeps the discriminator allowlist in exact PRP v1 schema parity", () => { const schema = JSON.parse( readFileSync( diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index 2616bd1a678..ab36d46cce0 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -219,6 +219,14 @@ vi.mock("@paperclipai/db", () => ({ reconcilePendingMigrationHistory: vi.fn(async () => ({ repairedMigrations: [] })), formatDatabaseBackupResult: vi.fn(() => "ok"), runDatabaseBackup: vi.fn(), + heartbeatRunEvents: { + id: {}, + eventEpoch: {}, + seq: {}, + runId: {}, + payload: {}, + message: {}, + }, authUsers: {}, companies: {}, companyMemberships: {}, diff --git a/server/src/__tests__/storage-local-provider.test.ts b/server/src/__tests__/storage-local-provider.test.ts index b72cdbaded8..c1f65bd3106 100644 --- a/server/src/__tests__/storage-local-provider.test.ts +++ b/server/src/__tests__/storage-local-provider.test.ts @@ -2,6 +2,8 @@ import { afterEach, describe, expect, it } from "vitest"; import os from "node:os"; import path from "node:path"; import { promises as fs } from "node:fs"; +import { Readable } from "node:stream"; +import { createHash } from "node:crypto"; import { createLocalDiskStorageProvider } from "../storage/local-disk-provider.js"; import { createStorageService } from "../storage/service.js"; @@ -21,6 +23,20 @@ describe("local disk storage provider", () => { tempRoots.length = 0; }); + it("streams and syncs exact content, preserving the prior object on incomplete or corrupt upload", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-storage-")); + tempRoots.push(root); + const provider = createLocalDiskStorageProvider(root); + const value = Buffer.from("streamed bytes"); + const input = { objectKey: "company/chunk", contentType: "application/octet-stream", contentLength: value.length, sha256: createHash("sha256").update(value).digest("hex") }; + await provider.putObject({ ...input, body: Readable.from([value.subarray(0, 4), value.subarray(4)]) }); + for (const bad of [Buffer.from("too short"), Buffer.alloc(value.length, 0x78), Buffer.alloc(value.length + 1)]) { + await expect(provider.putObject({ ...input, body: Readable.from([bad]) })).rejects.toThrow("storage_object_"); + expect(await readStreamToBuffer((await provider.getObject({ objectKey: input.objectKey })).stream)).toEqual(value); + } + expect(await fs.readdir(path.join(root, "company"))).toEqual(["chunk"]); + }); + it("round-trips bytes through storage service", async () => { const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-storage-")); tempRoots.push(root); diff --git a/server/src/redaction.ts b/server/src/redaction.ts index 34ed09ca27e..b07a50a3b95 100644 --- a/server/src/redaction.ts +++ b/server/src/redaction.ts @@ -101,6 +101,8 @@ export const PAPERCLIP_PUBLIC_SCHEMA_IDS = new Set([ "paperclip.native-model-envelope.v2", "paperclip.native-session-scope.v2", "paperclip.native-session-supervisor.v1", + "paperclip.output.body.v1", + "paperclip.output.body.chunk.v1", "paperclip.plan.updated.v1", "paperclip.provider.native.v1", "paperclip.provider.notice.v1", @@ -259,6 +261,7 @@ export const PRP_V1_EVENT_TYPES = new Set([ "turn.failed", "turn.interrupted", "turn.cancelled", + "output.body.chunk", "item.started", "item.delta", "item.completed", @@ -879,6 +882,15 @@ function isPaperclipSchemaDiscriminator( export function sanitizeRecord( record: Record, ): Record { + if (record.schema === "paperclip.output.body.chunk.v1") { + // Transport fragments are not independently redactable: a registered + // secret may cross their boundary. Public event views expose the reference + // only; the authorized download verifies/reassembles/redacts the full frame. + const body = isPlainObject(record.body) ? record.body : {}; + return { schema: record.schema, body: sanitizeRecord({ schema: body.schema, + bodyId: body.bodyId, sha256: body.sha256, byteLength: body.byteLength, mediaType: body.mediaType }), + offset: sanitizeValue(record.offset), sha256: sanitizeValue(record.sha256) }; + } const redacted: Record = {}; for (const [key, value] of Object.entries(record)) { if (COMMAND_ARGS_PAYLOAD_KEY_RE.test(key) && Array.isArray(value)) { diff --git a/server/src/routes/agents.ts b/server/src/routes/agents.ts index 2efcc764571..3830086348b 100644 --- a/server/src/routes/agents.ts +++ b/server/src/routes/agents.ts @@ -1,3 +1,5 @@ +import { parseRunEventCursor } from "../services/run-event-history.js"; +import { readRunOutputBody } from "../services/run-output-body.js"; import { resolveAgentAppearance, agentAvatarUrl } from "@paperclipai/shared"; import { listOpenRouterModels } from "../services/openrouter-models.js"; import { prepareManagedAiRuntime, assertManagedAiProjectAuth, stripAiAuthBindings } from "../services/ai-connection-runtime.js"; @@ -6999,7 +7001,7 @@ export function agentRoutes( } throw error; } - queued = queueRunnerPrpRuntimeRequestResolution({ + queued = await queueRunnerPrpRuntimeRequestResolution({ companyId: existing.companyId, runId, pendingRequest: currentPendingRequest, @@ -7266,9 +7268,12 @@ export function agentRoutes( if (!run) return; if (!(await assertRunTelemetryReadAllowed(req, res, run.companyId))) return; - const afterSeq = Number(req.query.afterSeq ?? 0); + const afterSeq = typeof req.query.afterSeq === "string" ? req.query.afterSeq : "0"; + if (afterSeq !== "tail") { + try { parseRunEventCursor(afterSeq); } catch { throw badRequest("Invalid run event cursor"); } + } const limit = Number(req.query.limit ?? 200); - const events = await heartbeat.listEvents(runId, Number.isFinite(afterSeq) ? afterSeq : 0, Number.isFinite(limit) ? limit : 200); + const events = await heartbeat.listEvents(runId, afterSeq, Number.isFinite(limit) ? limit : 200); const currentUserRedactionOptions = await getCurrentUserRedactionOptions(); const redactedEvents = events.map((event) => redactCurrentUserValue({ @@ -7279,6 +7284,21 @@ export function agentRoutes( res.json(await runRedactions.redactForRun(run.companyId, run.id, redactedEvents)); }); + router.get("/heartbeat-runs/:runId/output-bodies/:bodyId", async (req, res) => { + const runId = readHeartbeatRunId(req); + const run = await getAccessibleResource(req, res, heartbeat.getRunLogAccess(runId), "Heartbeat run not found"); + if (!run) return; + if (!(await assertRunTelemetryReadAllowed(req, res, run.companyId))) return; + const bodyId = String(req.params.bodyId); + if (!/^[a-f0-9]{64}$/.test(bodyId)) throw notFound("Output not found"); + const text = await readRunOutputBody(db, run.companyId, run.id, bodyId); + const redacted = await runRedactions.redactForRun(run.companyId, run.id, + redactCurrentUserValue(redactEventPayload({ text })!, await getCurrentUserRedactionOptions())); + res.set("Cache-Control", "no-cache, no-store"); + res.set("Content-Disposition", `attachment; filename="output-${bodyId.slice(0, 12)}.txt"`); + res.type("text/plain").send(redacted.text); + }); + router.get("/heartbeat-runs/:runId/log", async (req, res) => { const runId = readHeartbeatRunId(req); const run = await getAccessibleResource(req, res, heartbeat.getRunLogAccess(runId), "Heartbeat run not found"); @@ -7287,8 +7307,13 @@ export function agentRoutes( const offset = Number(req.query.offset ?? 0); const limitBytes = readRunLogLimitBytes(req.query.limitBytes); + const cursor = req.query.cursor; + if (cursor !== undefined && (typeof cursor !== "string" || (cursor !== "tail" && !/^(0|[1-9][0-9]{0,1023})$/.test(cursor)))) { + throw badRequest("Invalid run log cursor"); + } const result = await heartbeat.readLog(run, { offset: Number.isFinite(offset) ? offset : 0, + ...(typeof cursor === "string" ? { cursor } : {}), limitBytes, }); diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.ts index e9f6ab89060..754916dbcd6 100644 --- a/server/src/routes/issues.ts +++ b/server/src/routes/issues.ts @@ -14372,7 +14372,7 @@ export function issueRoutes( goalProjection?.goal?.status === "active" && goalProjection.workingNow ) { - const steer = queueLiveRunnerPrpCommand({ + const steer = await queueLiveRunnerPrpCommand({ companyId: issue.companyId, issueId: issue.id, agentId: issue.assigneeAgentId, @@ -17863,7 +17863,7 @@ export function issueRoutes( goalProjection?.goal?.status === "active" && goalProjection.workingNow ) { - const steer = queueLiveRunnerPrpCommand({ + const steer = await queueLiveRunnerPrpCommand({ companyId: currentIssue.companyId, issueId: currentIssue.id, agentId: currentIssue.assigneeAgentId, diff --git a/server/src/routes/openapi.ts b/server/src/routes/openapi.ts index 439262a6fce..8e3f478c7d6 100644 --- a/server/src/routes/openapi.ts +++ b/server/src/routes/openapi.ts @@ -7073,8 +7073,49 @@ registry.registerPath({ path: "/api/heartbeat-runs/{runId}/events", tags: ["runs"], summary: "Get events for a heartbeat run", - request: { params: z.object({ runId: heartbeatRunIdParamSchema }) }, - responses: { 200: r.ok(), 400: r.badRequest, 401: r.unauthorized }, + description: + "Returns a bounded page of redacted events. Preserve each returned cursor as text; " + + "use it as afterSeq for the next page. The last row's historyAfter flag indicates " + + "more output, including when the byte budget returns fewer rows than limit. " + + "A tail page can mark omitted earlier history with historyBefore on its first row.", + request: { + params: z.object({ runId: heartbeatRunIdParamSchema }), + query: z.object({ + afterSeq: z.string().optional().describe("Returned event cursor, legacy decimal sequence, or tail for the latest page. Defaults to 0."), + limit: z.coerce.number().int().min(1).max(1000).optional().describe("Maximum events, also subject to a byte budget. Defaults to 200."), + }), + }, + responses: { 200: r.ok(), 400: r.badRequest, 401: r.unauthorized, 403: r.forbidden, 404: r.notFound }, +}); + +registry.registerPath({ + method: "get", + path: "/api/heartbeat-runs/{runId}/output-bodies/{bodyId}", + tags: ["runs"], + summary: "Download a heartbeat run output body", + description: + "Returns one complete output frame as a redacted UTF-8 text attachment, after " + + "checking run access and output-read permission. The body is bounded to 4 MiB; " + + "this is not a limit on accumulated run output. The body ID identifies the " + + "original content; redaction may change the downloaded bytes. Incomplete or " + + "unverifiable stored output returns 409.", + request: { + params: z.object({ + runId: heartbeatRunIdParamSchema, + bodyId: z.string().regex(/^[a-f0-9]{64}$/), + }), + }, + responses: { + 200: { + description: "Redacted output text; Content-Disposition: attachment; Cache-Control: no-cache, no-store", + content: { "text/plain": { schema: { type: "string" } } }, + }, + 400: r.badRequest, + 401: r.unauthorized, + 403: r.forbidden, + 404: r.notFound, + 409: r.conflict, + }, }); registry.registerPath({ @@ -7082,8 +7123,20 @@ registry.registerPath({ path: "/api/heartbeat-runs/{runId}/log", tags: ["runs"], summary: "Get log for a heartbeat run", - request: { params: z.object({ runId: heartbeatRunIdParamSchema }) }, - responses: { 200: r.ok(), 400: r.badRequest, 401: r.unauthorized }, + description: + "Returns a bounded page of redacted log data. Preserve byte cursors as text " + + "to avoid rounding large history positions. Use the returned cursor for the " + + "next page while hasMore is true; legacy responses use nextOffset. A page " + + "may be shorter than limitBytes.", + request: { + params: z.object({ runId: heartbeatRunIdParamSchema }), + query: z.object({ + cursor: z.string().regex(/^(tail|0|[1-9][0-9]{0,1023})$/).optional().describe("Exact decimal byte cursor, or tail for the latest log data."), + offset: z.coerce.number().int().min(0).optional().describe("Legacy numeric offset; prefer the exact cursor when available."), + limitBytes: z.coerce.number().int().min(1).max(1024 * 1024).optional().describe("Maximum page bytes. Defaults to 256000."), + }), + }, + responses: { 200: r.ok(), 400: r.badRequest, 401: r.unauthorized, 403: r.forbidden, 404: r.notFound }, }); registry.registerPath({ diff --git a/server/src/services/activity.ts b/server/src/services/activity.ts index 2a6bbc5e34f..2c32843cb95 100644 --- a/server/src/services/activity.ts +++ b/server/src/services/activity.ts @@ -1,3 +1,4 @@ +import { heartbeatRunEventHeads } from "@paperclipai/db"; import { executionProjectionsForRuns } from "./execution-projection.js"; import { and, asc, desc, eq, inArray, isNull, or, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; @@ -455,15 +456,17 @@ export function activityService(db: Db) { runId: heartbeatRunEvents.runId, message: heartbeatRunEvents.message, }) - .from(heartbeatRunEvents) + .from(heartbeatRunEventHeads) + .innerJoin(heartbeatRunEvents, eq(heartbeatRunEventHeads.eventId, heartbeatRunEvents.id)) .where( and( - inArray(heartbeatRunEvents.runId, runIds), + inArray(heartbeatRunEventHeads.runId, runIds), + eq(heartbeatRunEventHeads.lane, "exhaustion"), eq(heartbeatRunEvents.eventType, "lifecycle"), sql`${heartbeatRunEvents.message} like 'Bounded retry exhausted%'`, ), ) - .orderBy(asc(heartbeatRunEvents.runId), desc(heartbeatRunEvents.id)); + ; const retryExhaustedReasonByRunId = new Map(); for (const row of exhaustionRows) { diff --git a/server/src/services/attention-exhausted-runs.ts b/server/src/services/attention-exhausted-runs.ts index 29f3978dd22..f89150bb33c 100644 --- a/server/src/services/attention-exhausted-runs.ts +++ b/server/src/services/attention-exhausted-runs.ts @@ -1,3 +1,4 @@ +import { heartbeatRunEventHeads } from "@paperclipai/db"; import { and, asc, desc, eq, inArray, notInArray, sql } from "drizzle-orm"; import { agents, heartbeatRunEvents, heartbeatRuns, type Db } from "@paperclipai/db"; @@ -5,18 +6,19 @@ export function listAttentionExhaustedRuns(db: Db, companyId: string) { // Recovery can revisit an exhausted run. Deduplicate its historical events // before joining run data so duplicate events never multiply the wire payload. const latestExhaustion = db - .selectDistinctOn([heartbeatRunEvents.runId], { + .select({ runId: heartbeatRunEvents.runId, eventId: heartbeatRunEvents.id, message: heartbeatRunEvents.message, }) - .from(heartbeatRunEvents) + .from(heartbeatRunEventHeads) + .innerJoin(heartbeatRunEvents, eq(heartbeatRunEventHeads.eventId, heartbeatRunEvents.id)) .where(and( + eq(heartbeatRunEventHeads.lane, "exhaustion"), eq(heartbeatRunEvents.companyId, companyId), eq(heartbeatRunEvents.eventType, "lifecycle"), sql`${heartbeatRunEvents.message} like 'Bounded retry exhausted%'`, )) - .orderBy(asc(heartbeatRunEvents.runId), desc(heartbeatRunEvents.id)) .as("latest_exhaustion"); return db diff --git a/server/src/services/chat-channels.ts b/server/src/services/chat-channels.ts index 52db9448b1e..e3df8c5ab06 100644 --- a/server/src/services/chat-channels.ts +++ b/server/src/services/chat-channels.ts @@ -12270,6 +12270,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { if (retiredOwner) { if ( !(await nativePreProviderRetryAfterCleanupStateIsSafe({ + db: tx as Db, failedExecution: run.runnerProfileJson?.nativeExecutionInput, retiredExecution: retiredOwner.predecessor.runnerProfileJson?.nativeExecutionInput, @@ -12300,6 +12301,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { (typeof checkpoint.providerSessionId !== "string" || checkpoint.providerSessionId.trim().length === 0)) || !(await nativeFailedRunRetryStateIsSafe({ + db: tx as Db, execution: run.runnerProfileJson?.nativeExecutionInput, companyId, issueId: source.issueId, diff --git a/server/src/services/chat-publication-reconciliation.ts b/server/src/services/chat-publication-reconciliation.ts index 8770d4a31c7..16b0f3530a7 100644 --- a/server/src/services/chat-publication-reconciliation.ts +++ b/server/src/services/chat-publication-reconciliation.ts @@ -36,6 +36,9 @@ export function publishChatPublicationCommitSignal(input: { agentId: string; eventType: string; seq?: number; + id?: number | string; + eventEpoch?: string; + cursor?: string; }): boolean { if (!isChatPublicationCommitEventType(input.eventType)) return false; try { @@ -47,6 +50,10 @@ export function publishChatPublicationCommitSignal(input: { agentId: input.agentId, issueId: input.issueId, ...(input.seq === undefined ? {} : { seq: input.seq }), + ...(input.id === undefined ? {} : { id: input.id }), + ...(input.eventEpoch === undefined ? {} : { eventEpoch: input.eventEpoch }), + ...(input.cursor === undefined ? {} : { cursor: input.cursor }), + commitHint: true, eventType: input.eventType, }, }); diff --git a/server/src/services/chat-run-publications.ts b/server/src/services/chat-run-publications.ts index 97a55497295..cb431173359 100644 --- a/server/src/services/chat-run-publications.ts +++ b/server/src/services/chat-run-publications.ts @@ -1,3 +1,5 @@ +import { heartbeatRunEventHeads } from "@paperclipai/db"; +import { runEventCursor } from "./run-event-history.js"; import { and, asc, @@ -109,7 +111,7 @@ type ChatRunMilestoneCandidate = { }; type SafeNativeChatProgressCandidate = { - eventId: number; + eventId: number | string; eventSeq: number; eventType: string; eventCreatedAt: Date; @@ -187,10 +189,6 @@ async function enqueueSafeNativeChatProgress( chatPublications, "safe_native_progress_base_publication", ); - const laterEvent = alias( - heartbeatRunEvents, - "safe_native_progress_later_event", - ); const issueIdFromContext = sql`${heartbeatRuns.contextSnapshot} ->> 'issueId'`; let inserted = 0; let cursor: { @@ -268,21 +266,10 @@ async function enqueueSafeNativeChatProgress( ]), gte(heartbeatRunEvents.createdAt, input.since), sql`${heartbeatRunEvents.createdAt} >= ${basePublication.createdAt} + interval '20 seconds'`, - notExists( - db - .select({ id: laterEvent.id }) - .from(laterEvent) - .where( - and( - eq(laterEvent.companyId, heartbeatRunEvents.companyId), - eq(laterEvent.runId, heartbeatRunEvents.runId), - gt(laterEvent.seq, heartbeatRunEvents.seq), - inArray(laterEvent.eventType, [ - ...SAFE_NATIVE_CHAT_PROGRESS_EVENT_TYPES, - ]), - ), - ), - ), + sql`${heartbeatRunEvents.id} = (select ${heartbeatRunEventHeads.eventId} from ${heartbeatRunEventHeads} + where ${heartbeatRunEventHeads.companyId} = ${heartbeatRunEvents.companyId} + and ${heartbeatRunEventHeads.runId} = ${heartbeatRunEvents.runId} + and ${heartbeatRunEventHeads.lane} = 'safe-native-progress')`, pageCursor ? or( gt(heartbeatRunEvents.createdAt, pageCursor.eventCreatedAt), @@ -381,6 +368,7 @@ async function enqueueSafeNativeChatProgress( const [currentEvent] = await tx .select({ id: heartbeatRunEvents.id, + eventEpoch: heartbeatRunEvents.eventEpoch, seq: heartbeatRunEvents.seq, eventType: heartbeatRunEvents.eventType, createdAt: heartbeatRunEvents.createdAt, @@ -390,12 +378,10 @@ async function enqueueSafeNativeChatProgress( and( eq(heartbeatRunEvents.companyId, row.companyId), eq(heartbeatRunEvents.runId, row.runId), - inArray(heartbeatRunEvents.eventType, [ - ...SAFE_NATIVE_CHAT_PROGRESS_EVENT_TYPES, - ]), + sql`${heartbeatRunEvents.id} = (select ${heartbeatRunEventHeads.eventId} from ${heartbeatRunEventHeads} + where ${heartbeatRunEventHeads.runId} = ${row.runId} and ${heartbeatRunEventHeads.lane} = 'safe-native-progress')`, ), ) - .orderBy(desc(heartbeatRunEvents.seq), desc(heartbeatRunEvents.id)) .limit(1); if ( !currentEvent || @@ -529,7 +515,7 @@ async function enqueueSafeNativeChatProgress( endpointId: row.endpointId, conversationId: row.conversationId, issueId: row.issueId, - idempotencyKey: `${progressKeyPrefix}${progress.phase}:${currentEvent.seq}`, + idempotencyKey: `${progressKeyPrefix}${progress.phase}:${runEventCursor(currentEvent)}`, payload: projectSafeChatPublication({ classification: "external", source: "safe_milestone", diff --git a/server/src/services/conversation-continuation.ts b/server/src/services/conversation-continuation.ts index 3dd737357f6..b9d4c6eeb64 100644 --- a/server/src/services/conversation-continuation.ts +++ b/server/src/services/conversation-continuation.ts @@ -1,6 +1,7 @@ import { and, desc, eq, inArray, isNotNull, or, sql } from "drizzle-orm"; import { environmentLeases, heartbeatRunEvents, heartbeatRuns, issueRecoveryActions, type Db } from "@paperclipai/db"; import { readProcessStartedAt } from "./hot-restart.js"; +import { readRunEventLane, runEventLane } from "./run-event-history.js"; // These adapters accept a conversation turn. Retrying a process or webhook can // replay the action itself, so those adapters retain their recovery contract. @@ -43,10 +44,8 @@ function conversationRunPredicate() { export async function historicalAdapterType(db: Db, run: typeof heartbeatRuns.$inferSelect): Promise { const selected = claimedAdapterType(run); if (selected) return selected; - const [invocation] = await db.select({ payload: heartbeatRunEvents.payload }).from(heartbeatRunEvents) - .where(and(eq(heartbeatRunEvents.companyId, run.companyId), eq(heartbeatRunEvents.runId, run.id), - eq(heartbeatRunEvents.eventType, "adapter.invoke"))) - .orderBy(desc(heartbeatRunEvents.seq)).limit(1); + const [invocation] = await readRunEventLane(db, run.id, runEventLane("type", "adapter.invoke"), 1); + if (!invocation || invocation.companyId !== run.companyId || invocation.eventType !== "adapter.invoke") return null; const adapterType = invocation?.payload?.adapterType; return typeof adapterType === "string" ? adapterType : null; } diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index cb484cc5ba2..d5ccabf31a1 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -2039,7 +2039,7 @@ function createSandboxEnvironmentDriver( if (!providerLease) { if ( input.adapterType === "paperclip_runner" && - !verifyNativeHarnessBackupStamp( + !await verifyNativeHarnessBackupStamp( reusableLease.metadata?.nativeHarnessBackup, reusableLease.providerLeaseId, ) @@ -3906,7 +3906,7 @@ export function environmentRuntimeService( "paperclip_runner" && leaseSnapshot.metadata?.sandboxLeaseAcquisition && (!leaseSnapshot.providerLeaseId || - !verifyNativeHarnessBackupStamp( + !await verifyNativeHarnessBackupStamp( leaseSnapshot.metadata.nativeHarnessBackup, leaseSnapshot.providerLeaseId, )) diff --git a/server/src/services/feedback.ts b/server/src/services/feedback.ts index 018a82546f4..14a47915564 100644 --- a/server/src/services/feedback.ts +++ b/server/src/services/feedback.ts @@ -47,6 +47,7 @@ import { sha256Digest, } from "./feedback-redaction.js"; import { getRunLogStore } from "./run-log-store.js"; +import { readRunEventExcerpt } from "./run-event-excerpt.js"; import { getOperatorSettingDefaults } from "./setting-defaults.js"; const FEEDBACK_SCHEMA_VERSION = "paperclip-feedback-envelope-v2"; @@ -314,27 +315,34 @@ async function findMatchingFile( return search(rootDir, 0); } -async function readFullRunLog(run: { +async function readRunLogExcerpt(run: { logStore: string | null; logRef: string | null; }) { - if (run.logStore !== "local_file" || !run.logRef) return null; + if (!["local_file", "local_segments"].includes(run.logStore ?? "") || !run.logRef) return { text: null, truncated: false }; const store = getRunLogStore(); let offset = 0; + let cursor: string | undefined; let combined = ""; while (true) { - const result = await store.read({ store: "local_file", logRef: run.logRef }, { + const result = await store.read({ store: run.logStore as "local_file" | "local_segments", logRef: run.logRef }, { offset, + cursor, limitBytes: 512_000, }).catch(() => null); - if (!result) return combined || null; + if (!result) return { text: combined || null, truncated: combined.length > 0 }; combined += result.content; - if (result.nextOffset == null) break; - offset = result.nextOffset; + if (!(result.hasMore ?? result.nextOffset != null)) break; + // Feedback already limits trace text to MAX_TRACE_FILE_CHARS. Apply that + // envelope budget while reading, rather than materializing days of output + // and only then truncating it. The persisted source history stays intact. + if (combined.length >= MAX_TRACE_FILE_CHARS) return { text: combined, truncated: true }; + cursor = result.cursor; + offset = result.nextOffset ?? offset; } - return combined || null; + return { text: combined || null, truncated: false }; } function parseRunLogEntries(logText: string | null) { @@ -1503,6 +1511,7 @@ async function buildFeedbackTraceBundleFromRow( logRef: heartbeatRuns.logRef, logBytes: heartbeatRuns.logBytes, logSha256: heartbeatRuns.logSha256, + nextEventSeq: heartbeatRuns.nextEventSeq, agentName: agents.name, agentRole: agents.role, agentTitle: agents.title, @@ -1517,12 +1526,12 @@ async function buildFeedbackTraceBundleFromRow( appendNote(notes, "source_run_unavailable"); } else { adapterType = run.adapterType; - const events = await db - .select() - .from(heartbeatRunEvents) - .where(eq(heartbeatRunEvents.runId, run.id)) - .orderBy(asc(heartbeatRunEvents.seq)); - const logText = await readFullRunLog(run); + const eventExcerpt = await readRunEventExcerpt(db, run.companyId, run.id); + const events = eventExcerpt.events; + if (eventExcerpt.truncated) appendNote(notes, "run_events_excerpt_truncated"); + const logExcerpt = await readRunLogExcerpt(run); + if (logExcerpt.truncated) appendNote(notes, "run_log_excerpt_truncated"); + const logText = logExcerpt.text; const logEntries = parseRunLogEntries(logText); const stdoutText = logEntries .filter((entry) => entry.stream === "stdout") @@ -1556,7 +1565,9 @@ async function buildFeedbackTraceBundleFromRow( logRef: run.logRef, logBytes: run.logBytes, logSha256: run.logSha256, - eventCount: events.length, + eventCount: run.nextEventSeq - 1, + capturedEventCount: events.length, + eventsTruncated: eventExcerpt.truncated, }, state, "bundle.paperclipRun", diff --git a/server/src/services/heartbeat-run-events.ts b/server/src/services/heartbeat-run-events.ts index e90f5c4e7d0..9602d0d6de4 100644 --- a/server/src/services/heartbeat-run-events.ts +++ b/server/src/services/heartbeat-run-events.ts @@ -1,7 +1,11 @@ -import { and, asc, eq, or, sql } from "drizzle-orm"; +import { allocateRunEventPosition, runEventExhaustionLane } from "./run-event-history.js"; +import { and, asc, desc, eq, gt, or, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; -import { heartbeatRunEvents, heartbeatRuns } from "@paperclipai/db"; +import { heartbeatRunEvents, heartbeatRuns, nativeSourceCursors, nativeSourceEpochs, nativeOutputBodyChunks, heartbeatRunEventHeads } from "@paperclipai/db"; +import { isEventEpochTransition, type EventEpochTransition } from "../vendor/paperclip-runner/index.js"; import { nativeSha256 } from "./native-runtime/canonical.js"; +import { recordNativeProcessEvidence } from "./native-process-evidence.js"; +import { prepareRunOutputChunkPayload } from "./run-output-body.js"; export interface AppendHeartbeatRunEventInput { companyId: string; @@ -23,8 +27,12 @@ export interface AppendHeartbeatRunEventInput { sourceInstanceId: string; sourceEventId: string; sourceSeq: number; + sourceEpoch?: string; + sourceEpochTransition?: EventEpochTransition; protocolSchemaVersion: number; canonicalPayload: Record; + /** Retained hidden-coordinator rows use tagged digests. Preserve exact replay bytes. */ + hashEncoding?: "sha256"; }; } @@ -32,6 +40,7 @@ export interface AppendHeartbeatRunEventResult { row: typeof heartbeatRunEvents.$inferSelect; disposition: "committed" | "duplicate"; highestContiguousSourceSeq: number; + highestContiguousSourceEpoch?: string; } export class HeartbeatRunEventConflictError extends Error { @@ -42,33 +51,25 @@ export class HeartbeatRunEventConflictError extends Error { } } -/** - * Atomically reserve the next event sequence on the run row. Both native PRP - * ingestion and legacy/direct-adapter writers use this allocator so the - * database uniqueness invariant cannot turn a concurrent log/cancel/recovery - * race into a failed run. - */ -export async function allocateHeartbeatRunEventSeq( - db: Db, - runId: string, -): Promise { - const [updated] = await db - .update(heartbeatRuns) - .set({ - nextEventSeq: sql`${heartbeatRuns.nextEventSeq} + 1`, - updatedAt: new Date(), - }) - .where(eq(heartbeatRuns.id, runId)) - .returning({ nextEventSeq: heartbeatRuns.nextEventSeq }); - if (!updated) throw new Error("heartbeat_run_event_binding_mismatch"); - return Number(updated.nextEventSeq) - 1; -} - export async function appendHeartbeatRunEvent( db: Db, input: AppendHeartbeatRunEventInput, ): Promise { - return db.transaction(async (tx) => { + if (input.eventType === "output.body.chunk") { + const [run] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.id, input.runId), eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.agentId, input.agentId), + )); + if (!run) throw new Error("heartbeat_run_event_binding_mismatch"); + } + const payload = await prepareRunOutputChunkPayload(input); + return db.transaction((tx) => appendHeartbeatRunEventInTransaction(tx, input, payload)); +} + +export async function appendHeartbeatRunEventInTransaction( + tx: Parameters[0]>[0], + input: AppendHeartbeatRunEventInput, + preparedPayload?: Record | null, +): Promise { const run = await tx .select() .from(heartbeatRuns) @@ -79,6 +80,7 @@ export async function appendHeartbeatRunEvent( if (!run || run.companyId !== input.companyId || run.agentId !== input.agentId) { throw new Error("heartbeat_run_event_binding_mismatch"); } + const storedPayload = preparedPayload === undefined ? await prepareRunOutputChunkPayload(input) : preparedPayload; if (input.retryExhaustion && !input.nativeSource) { // The run lock also serializes concurrent recovery checks across server @@ -91,6 +93,8 @@ export async function appendHeartbeatRunEvent( eq(heartbeatRunEvents.companyId, input.companyId), eq(heartbeatRunEvents.runId, input.runId), eq(heartbeatRunEvents.agentId, input.agentId), + sql`${heartbeatRunEvents.id} = (select ${heartbeatRunEventHeads.eventId} from ${heartbeatRunEventHeads} + where ${heartbeatRunEventHeads.runId} = ${input.runId} and ${heartbeatRunEventHeads.lane} = ${runEventExhaustionLane(input.retryExhaustion)})`, eq(heartbeatRunEvents.eventType, "lifecycle"), sql`${heartbeatRunEvents.message} like 'Bounded retry exhausted%'`, sql`${heartbeatRunEvents.payload} @> ${JSON.stringify(input.retryExhaustion)}::jsonb`, @@ -107,7 +111,7 @@ export async function appendHeartbeatRunEvent( } const sourceHash = input.nativeSource - ? nativeSha256(input.nativeSource.canonicalPayload) + ? `${input.nativeSource.hashEncoding === "sha256" ? "sha256:" : ""}${nativeSha256(input.nativeSource.canonicalPayload)}` : null; if (input.nativeSource) { const existing = await tx @@ -119,6 +123,7 @@ export async function appendHeartbeatRunEvent( eq(heartbeatRunEvents.sourceEventId, input.nativeSource.sourceEventId), and( eq(heartbeatRunEvents.sourceInstanceId, input.nativeSource.sourceInstanceId), + eq(heartbeatRunEvents.sourceEpoch, input.nativeSource.sourceEpoch ?? ""), eq(heartbeatRunEvents.sourceSeq, input.nativeSource.sourceSeq), ), ), @@ -132,16 +137,18 @@ export async function appendHeartbeatRunEvent( return { row: existing, disposition: "duplicate" as const, - highestContiguousSourceSeq: await contiguousCursor( + ...await contiguousCursor( tx as unknown as Db, input.runId, input.nativeSource.sourceInstanceId, + input.companyId, ), }; } } - const seq = await allocateHeartbeatRunEventSeq( + if (input.nativeSource) await acceptSourceEpoch(tx as unknown as Db, input); + const position = await allocateRunEventPosition( tx as unknown as Db, input.runId, ); @@ -149,47 +156,116 @@ export async function appendHeartbeatRunEvent( companyId: input.companyId, runId: input.runId, agentId: input.agentId, - seq, + ...position, eventType: input.eventType, stream: input.stream ?? null, level: input.level ?? null, color: input.color ?? null, message: input.message ?? null, - payload: input.payload ?? null, + payload: storedPayload, sourceInstanceId: input.nativeSource?.sourceInstanceId ?? null, sourceEventId: input.nativeSource?.sourceEventId ?? null, sourceSeq: input.nativeSource?.sourceSeq ?? null, + sourceEpoch: input.nativeSource?.sourceEpoch ?? "", sourcePayloadSha256: sourceHash, protocolSchemaVersion: input.nativeSource?.protocolSchemaVersion ?? null, }).returning(); if (!row) throw new Error("heartbeat_run_event_not_persisted"); + await recordNativeProcessEvidence(tx, row); + if (input.eventType === "output.body.chunk") { + const event = input.nativeSource?.canonicalPayload; + const chunk = event?.payload as { body?: { bodyId?: unknown } } | undefined; + const bodyId = chunk?.body?.bodyId; + if (typeof bodyId !== "string" || !/^[a-f0-9]{64}$/.test(bodyId) || !input.nativeSource) { + throw new Error("native_output_body_binding_invalid"); + } + await indexRunOutputChunk(tx as unknown as Db, { companyId: input.companyId, runId: input.runId, + bodyId, sourceSeq: input.nativeSource.sourceSeq, eventId: row.id, payload: event!.payload }); + } return { row, disposition: "committed" as const, - highestContiguousSourceSeq: input.nativeSource + ...(input.nativeSource ? await contiguousCursor( tx as unknown as Db, input.runId, input.nativeSource.sourceInstanceId, + input.companyId, ) - : 0, + : { highestContiguousSourceSeq: 0 }), }; - }); } -async function contiguousCursor(db: Db, runId: string, sourceInstanceId: string): Promise { - const rows = await db - .select({ sourceSeq: heartbeatRunEvents.sourceSeq }) - .from(heartbeatRunEvents) - .where(and( - eq(heartbeatRunEvents.runId, runId), - eq(heartbeatRunEvents.sourceInstanceId, sourceInstanceId), - )) - .orderBy(asc(heartbeatRunEvents.sourceSeq)); - let cursor = 0; - for (const row of rows) { - if (row.sourceSeq === cursor + 1) cursor += 1; - else if (row.sourceSeq !== null && row.sourceSeq > cursor + 1) break; +async function contiguousCursor(db: Db, runId: string, sourceInstanceId: string, companyId: string): Promise<{highestContiguousSourceSeq: number; highestContiguousSourceEpoch?: string}> { + // The caller holds the run row lock, serializing insertion and cursor updates. + const scope = and(eq(nativeSourceCursors.runId, runId), eq(nativeSourceCursors.sourceInstanceId, sourceInstanceId), eq(nativeSourceCursors.companyId, companyId)); + let stored = await db.select().from(nativeSourceCursors).where(scope).limit(1).then((rows) => rows[0]); + if (!stored) { + // Existing runs pay one migration scan, never a scan on each append. + // New runs normally have only their first event here. + const result = await db.execute(sql`select coalesce(min(case when source_seq <> ordinal then ordinal - 1 end), max(source_seq), 0)::bigint as cursor from (select source_seq, row_number() over(order by source_seq) as ordinal from heartbeat_run_events where run_id = ${runId} and source_instance_id = ${sourceInstanceId} and source_epoch = '') numbered`); + const cursor = Number(result[0]?.cursor ?? 0); + [stored] = await db.insert(nativeSourceCursors).values({ companyId, runId, sourceInstanceId, cursor }).returning(); + } + let cursor = stored!.cursor; + let gap = false; + do { + const rows = await db.select({ sourceSeq: heartbeatRunEvents.sourceSeq }).from(heartbeatRunEvents) + .where(and(eq(heartbeatRunEvents.runId, runId), eq(heartbeatRunEvents.sourceInstanceId, sourceInstanceId), eq(heartbeatRunEvents.sourceEpoch, stored!.sourceEpoch), gt(heartbeatRunEvents.sourceSeq, cursor))) + .orderBy(asc(heartbeatRunEvents.sourceSeq)).limit(128); + if (!rows.length) break; + for (const row of rows) { + if (row.sourceSeq === cursor + 1) cursor++; + else { gap = true; break; } + } + if (rows.length < 128) break; + } while (!gap); + if (cursor !== stored!.cursor) await db.update(nativeSourceCursors).set({ cursor }).where(scope); + return { highestContiguousSourceSeq: cursor, ...(stored!.sourceEpoch ? { highestContiguousSourceEpoch: stored!.sourceEpoch } : {}) }; +} + +/** Called under the run lock, before inserting a new event. Duplicates take the + * exact immutable event receipt path and cannot move this head backwards. */ +async function acceptSourceEpoch(db: Db, input: AppendHeartbeatRunEventInput): Promise { + const source = input.nativeSource!; + if (!Number.isSafeInteger(source.sourceSeq) || source.sourceSeq < 1) throw new HeartbeatRunEventConflictError(); + const scope = and(eq(nativeSourceCursors.runId, input.runId), eq(nativeSourceCursors.sourceInstanceId, source.sourceInstanceId), eq(nativeSourceCursors.companyId, input.companyId)); + await contiguousCursor(db, input.runId, source.sourceInstanceId, input.companyId); + const [head] = await db.select().from(nativeSourceCursors).where(scope).limit(1); + const sourceEpoch = source.sourceEpoch ?? ""; + if (head!.sourceEpoch === sourceEpoch) { + if (source.sourceEpochTransition) throw new HeartbeatRunEventConflictError(); + return; + } + const t = source.sourceEpochTransition; + if (!isEventEpochTransition(t) || t.runId !== input.runId || t.fromEpoch !== (head!.sourceEpoch || null) + || t.nextEpoch !== sourceEpoch || source.sourceSeq !== 1 || t.finalOrdinal !== head!.cursor) throw new HeartbeatRunEventConflictError(); + const [tail] = await db.select({ seq: heartbeatRunEvents.sourceSeq }).from(heartbeatRunEvents) + .where(and(eq(heartbeatRunEvents.runId, input.runId), eq(heartbeatRunEvents.sourceInstanceId, source.sourceInstanceId), eq(heartbeatRunEvents.sourceEpoch, head!.sourceEpoch))) + .orderBy(desc(heartbeatRunEvents.sourceSeq)).limit(1); + if (tail?.seq !== t.finalOrdinal) throw new HeartbeatRunEventConflictError(); + const [reused] = await db.select({ id: nativeSourceEpochs.transitionId }).from(nativeSourceEpochs).where(and( + eq(nativeSourceEpochs.runId, input.runId), eq(nativeSourceEpochs.sourceInstanceId, source.sourceInstanceId), + or(eq(nativeSourceEpochs.fromEpoch, sourceEpoch), eq(nativeSourceEpochs.nextEpoch, sourceEpoch), eq(nativeSourceEpochs.transitionId, t.transitionId)), + )).limit(1); + if (reused) throw new HeartbeatRunEventConflictError(); + await db.insert(nativeSourceEpochs).values({ companyId: input.companyId, runId: input.runId, sourceInstanceId: source.sourceInstanceId, + fromEpoch: head!.sourceEpoch, nextEpoch: sourceEpoch, transitionId: t.transitionId, finalOrdinal: t.finalOrdinal, transition: { ...t } }); + await db.update(nativeSourceCursors).set({ sourceEpoch, cursor: 0 }).where(scope); +} + +/** Body-local offsets are bounded by one provider frame and do not reset when + * the source event namespace rotates. Repeated identical bodies share chunks. */ +export async function indexRunOutputChunk(db: Db, input: { companyId: string; runId: string; bodyId: string; sourceSeq: number; eventId: number | string; payload: unknown }): Promise { + const chunk = input.payload as { offset?: unknown; sha256?: unknown }; + if (!chunk || typeof chunk.offset !== "string" || !/^(0|[1-9][0-9]{0,6})$/.test(chunk.offset) + || Number(chunk.offset) >= 4194304 || typeof chunk.sha256 !== "string" || !/^[a-f0-9]{64}$/.test(chunk.sha256)) throw new Error("native_output_body_binding_invalid"); + const { payload: _, ...row } = input; + const chunkOffset = Number(chunk.offset), chunkSha256 = chunk.sha256; + const [inserted] = await db.insert(nativeOutputBodyChunks).values({ ...row, chunkOffset, chunkSha256 }).onConflictDoNothing().returning(); + if (!inserted) { + const [existing] = await db.select().from(nativeOutputBodyChunks).where(and(eq(nativeOutputBodyChunks.companyId, input.companyId), eq(nativeOutputBodyChunks.runId, input.runId), + eq(nativeOutputBodyChunks.bodyId, input.bodyId), eq(nativeOutputBodyChunks.chunkOffset, chunkOffset))).limit(1); + if (!existing || existing.chunkSha256 !== chunkSha256) throw new HeartbeatRunEventConflictError(); } - return cursor; } diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index d20bbd95a27..b9efec0fdd4 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -1,3 +1,4 @@ +import { allocateRunEventPosition, readRunEventPage, readRunEventLane, runEventCursor, runEventLane } from "./run-event-history.js"; import { applyWorkspaceRestoreFailure } from "@paperclipai/adapter-utils/workspace-restore-result"; import { hasWorkspaceRestoreFailure } from "@paperclipai/shared"; import { externalConversationStateSql, nonIdleSlackIssueCondition } from "./slack-conversation-state.js"; @@ -185,7 +186,6 @@ import { export { scrubGitCredentialText }; import { publishLiveEvent } from "./live-events.js"; import { - allocateHeartbeatRunEventSeq, appendHeartbeatRunEvent, type AppendHeartbeatRunEventInput, } from "./heartbeat-run-events.js"; @@ -255,7 +255,7 @@ import { type NativeSessionBackend, } from "../vendor/paperclip-runner/index.js"; import { normalizeResponsibleUserDenialCode } from "./responsible-user-denial-run-outcomes.js"; -import { getRunLogStore, type RunLogHandle } from "./run-log-store.js"; +import { getRunLogStore, type RunLogHandle, type RunLogReadOptions, type RunLogFinalizeSummary } from "./run-log-store.js"; import { providerTraceStore, PROVIDER_TRACE_MAX_BYTES, @@ -13766,6 +13766,9 @@ export function heartbeatService( runId: run.id, agentId: run.agentId, issueId, + id: persistedEvent.row.id, + eventEpoch: persistedEvent.row.eventEpoch, + cursor: runEventCursor(persistedEvent.row), seq, eventType: event.eventType, stream: event.stream ?? null, @@ -13856,7 +13859,7 @@ export function heartbeatService( issueId: string, resultJson?: Record | null, ) { - const comments = await db + const query = db .select({ id: issueComments.id, body: issueComments.body, @@ -13867,9 +13870,13 @@ export function heartbeatService( eq(issueComments.companyId, companyId), eq(issueComments.issueId, issueId), eq(issueComments.createdByRunId, runId), + eq(issueComments.nativeToolGenerated, false), ), ) .orderBy(desc(issueComments.createdAt), desc(issueComments.id)); + // Legacy checkpoints keep their original compatibility behavior; migrated + // and fresh runs exclude generated comments in the indexed query itself. + const comments = resultJson?.semanticToolReceipts ? await query : await query.limit(1); return findHeartbeatRunCompletionComment(comments, resultJson); } @@ -13877,58 +13884,22 @@ export function heartbeatService( runId: string, companyId: string, ) { - const rows = await db - .select({ - seq: heartbeatRunEvents.seq, - payload: heartbeatRunEvents.payload, - }) - .from(heartbeatRunEvents) - .where( - and( - eq(heartbeatRunEvents.companyId, companyId), - eq(heartbeatRunEvents.runId, runId), - eq(heartbeatRunEvents.eventType, "item.completed"), - ), - ) - .orderBy(desc(heartbeatRunEvents.seq)) - .limit(200); - const candidates: Array<{ - seq: number; - text: string; - sourceEventId: string | null; - channel: "final" | "unknown"; - }> = []; - for (const row of rows) { - const prpEvent = parseObject(parseObject(row.payload).prpEvent); - const candidate = readCompletedAssistantMessageCandidate({ - seq: row.seq, - prpEvent, - }); + // This lane includes both completed items and semantic-recovery boundaries. + // Relative ranks exist only within this bounded projection, never as durable + // identities or counters spanning epochs. + const rows = (await readRunEventLane(db, runId, "presentation", 400)).reverse(); + const candidates: Array<{ seq: number; text: string; sourceEventId: string | null; channel: "final" | "unknown" }> = []; + let recoveryBoundary: number | null = null; + for (const [index, row] of rows.entries()) { + if (row.companyId !== companyId) throw new Error("heartbeat_run_event_binding_mismatch"); + const rank = index + 1; + if (row.eventType === "lifecycle" && parseObject(row.payload).retryReasonCode === "semantic_result_missing") { + recoveryBoundary = rank; + continue; + } + const candidate = readCompletedAssistantMessageCandidate({ seq: rank, prpEvent: parseObject(parseObject(row.payload).prpEvent) }); if (candidate) candidates.push(candidate); } - const recoveryBoundary = await db - .select({ - seq: heartbeatRunEvents.seq, - payload: heartbeatRunEvents.payload, - }) - .from(heartbeatRunEvents) - .where( - and( - eq(heartbeatRunEvents.companyId, companyId), - eq(heartbeatRunEvents.runId, runId), - eq(heartbeatRunEvents.eventType, "lifecycle"), - ), - ) - .orderBy(heartbeatRunEvents.seq) - .limit(200) - .then( - (lifecycleRows) => - lifecycleRows.find( - (row) => - parseObject(row.payload).retryReasonCode === - "semantic_result_missing", - )?.seq ?? null, - ); return selectHeartbeatRunFinalAgentMessage({ candidates, semanticResultRecoveryAfterSeq: recoveryBoundary, @@ -22713,10 +22684,10 @@ export function heartbeatService( at: Date; seq: number; stream: "stdout" | "stderr"; - bytes: number; + bytes: number | null; } | null; } = { pending: null }; - let persistedLogBytes = Number(run.logBytes ?? 0); + let persistedLogBytes: number | null = run.logBytes === null ? null : Number(run.logBytes); const flushOutputProgress = async (opts?: { force?: boolean }) => { const pendingOutputProgress = outputProgressState.pending; if (!pendingOutputProgress) return; @@ -22843,17 +22814,23 @@ export function heartbeatService( stderrExcerpt = appendExcerpt(stderrExcerpt, sanitizedChunk); const ts = new Date().toISOString(); - outputSeq += 1; + // Retained progress counters are diagnostic only. The committed byte + // position, carried by both transports, is the exact chunk identity. + outputSeq = Math.min(Number.MAX_SAFE_INTEGER, outputSeq + 1); const chunkSeq = outputSeq; - let appendedBytes = 0; + let chunkCursor: string | undefined; if (handle) { - appendedBytes = await runLogStore.append(handle, { - stream, - chunk: sanitizedChunk, - ts, - seq: chunkSeq, - }); - persistedLogBytes += appendedBytes; + const logEvent = { stream, chunk: sanitizedChunk, ts, seq: chunkSeq }; + if (runLogStore.appendPositioned) { + const receipt = await runLogStore.appendPositioned(handle, logEvent); + chunkCursor = receipt.cursor; + const exact = BigInt(receipt.nextCursor); + persistedLogBytes = exact <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(exact) : null; + } else { + const bytes = await runLogStore.append(handle, logEvent); + const total = (persistedLogBytes ?? 0) + bytes; + persistedLogBytes = Number.isSafeInteger(total) ? total : null; + } } outputProgressState.pending = { at: new Date(ts), @@ -22902,6 +22879,7 @@ export function heartbeatService( issueId, ts, seq: chunkSeq, + ...(chunkCursor === undefined ? {} : { cursor: chunkCursor }), stream, chunk: payloadChunk, truncated: payloadChunk.length !== sanitizedChunk.length, @@ -24787,11 +24765,7 @@ export function heartbeatService( "adapter_failed") : null; - let logSummary: { - bytes: number; - sha256?: string; - compressed: boolean; - } | null = null; + let logSummary: RunLogFinalizeSummary | null = null; if (handle) { logSummary = await runLogStore.finalize(handle); } @@ -25496,11 +25470,7 @@ export function heartbeatService( "adapter_failed"; logger.error({ err, runId }, "heartbeat execution failed"); - let logSummary: { - bytes: number; - sha256?: string; - compressed: boolean; - } | null = null; + let logSummary: RunLogFinalizeSummary | null = null; if (handle) { try { logSummary = await runLogStore.finalize(handle); @@ -27305,7 +27275,7 @@ export function heartbeatService( ); } - const eventSeq = await allocateHeartbeatRunEventSeq( + const eventPosition = await allocateRunEventPosition( tx as unknown as Db, cancelled.id, ); @@ -27314,7 +27284,7 @@ export function heartbeatService( companyId: cancelled.companyId, runId: cancelled.id, agentId: cancelled.agentId, - seq: eventSeq, + ...eventPosition, eventType: "lifecycle", stream: "system", level: "warn", @@ -27333,10 +27303,6 @@ export function heartbeatService( currentAssigneeAgentId: issue.assigneeAgentId, }, }); - await tx - .update(heartbeatRuns) - .set({ nextEventSeq: eventSeq + 1, updatedAt: now }) - .where(eq(heartbeatRuns.id, cancelled.id)); cancelledRunsToEmit.push(cancelled); @@ -29460,35 +29426,10 @@ export function heartbeatService( }; }, - listEvents: (runId: string, afterSeq = 0, limit = 200) => - db - .select() - .from(heartbeatRunEvents) - .where( - and( - eq(heartbeatRunEvents.runId, runId), - gt(heartbeatRunEvents.seq, afterSeq), - ), - ) - .orderBy(asc(heartbeatRunEvents.seq)) - .limit(Math.max(1, Math.min(limit, 1000))), + listEvents: (runId: string, afterSeq: number | string = 0, limit = 200) => readRunEventPage(db, runId, afterSeq, limit), getRetryExhaustedReason: async (runId: string) => { - const row = await db - .select({ - message: heartbeatRunEvents.message, - }) - .from(heartbeatRunEvents) - .where( - and( - eq(heartbeatRunEvents.runId, runId), - eq(heartbeatRunEvents.eventType, "lifecycle"), - sql`${heartbeatRunEvents.message} like 'Bounded retry exhausted%'`, - ), - ) - .orderBy(desc(heartbeatRunEvents.id)) - .limit(1) - .then((rows) => rows[0] ?? null); + const [row] = await readRunEventLane(db, runId, "exhaustion", 1); return row?.message ?? null; }, @@ -29501,7 +29442,7 @@ export function heartbeatService( logStore: string | null; logRef: string | null; }, - opts?: { offset?: number; limitBytes?: number }, + opts?: RunLogReadOptions, ) => { const run = typeof runOrLookup === "string" @@ -29514,7 +29455,7 @@ export function heartbeatService( const result = await runLogStore.read( { - store: run.logStore as "local_file", + store: run.logStore as RunLogHandle["store"], logRef: run.logRef, }, opts, diff --git a/server/src/services/run-event-excerpt.ts b/server/src/services/run-event-excerpt.ts new file mode 100644 index 00000000000..9047f88ddc0 --- /dev/null +++ b/server/src/services/run-event-excerpt.ts @@ -0,0 +1,30 @@ +import { readRunEventLaneIds } from "./run-event-history.js"; +import { and, asc, desc, eq, inArray, sql } from "drizzle-orm"; +import { heartbeatRunEvents, type Db } from "@paperclipai/db"; +import { redactEventPayload } from "../redaction.js"; + +/** Feedback is a bounded recent excerpt. Select sizes before fetching bodies so + * neither a long run nor a handful of large events becomes a history-sized + * allocation. The immutable run log remains the source for complete history. */ +export async function readRunEventExcerpt(db: Db, companyId: string, runId: string, + limits = { events: 200, bytes: 10_000_000 }) { + if (!Number.isInteger(limits.events) || limits.events < 1 || limits.events > 1000 || + !Number.isInteger(limits.bytes) || limits.bytes < 1 || limits.bytes > 10_000_000) throw new Error("Invalid run-event excerpt capacity"); + const scope = and(eq(heartbeatRunEvents.companyId, companyId), eq(heartbeatRunEvents.runId, runId)); + const recentIds = await readRunEventLaneIds(db, runId, "all", limits.events + 1); + const metadataRows = recentIds.length ? await db.select({ id: heartbeatRunEvents.id, + bytes: sql`octet_length(row_to_json(${heartbeatRunEvents})::text)` }) + .from(heartbeatRunEvents).where(and(scope, inArray(heartbeatRunEvents.id, recentIds))) : []; + const metadataById = new Map(metadataRows.map(row => [String(row.id), row])); + const metadata = recentIds.map(id => metadataById.get(id)).filter(row => row !== undefined); + const ids: (number | string)[] = []; + let bytes = 2; + for (const row of metadata) { + if (ids.length >= limits.events || bytes + row.bytes + 1 > limits.bytes) break; + ids.push(row.id); bytes += row.bytes + 1; + } + const rows = ids.length ? await db.select().from(heartbeatRunEvents) + .where(and(scope, inArray(heartbeatRunEvents.id, ids))) : []; + const byId = new Map(rows.map(row => [String(row.id), row])); + return { events: ids.reverse().map(id => byId.get(String(id))!).map(row => ({ ...row, payload: redactEventPayload(row.payload) })), truncated: ids.length < metadata.length }; +} diff --git a/server/src/services/run-event-history.test.ts b/server/src/services/run-event-history.test.ts new file mode 100644 index 00000000000..e918f1c65bf --- /dev/null +++ b/server/src/services/run-event-history.test.ts @@ -0,0 +1,90 @@ +import { randomUUID } from "node:crypto"; +import { and, eq, sql } from "drizzle-orm"; +import { expect, it } from "vitest"; +import { agents, companies, createDb, heartbeatRunEvents, heartbeatRuns, heartbeatRunEventEpochs, heartbeatRunEventHeads, heartbeatRunEventLinks, type Db } from "@paperclipai/db"; +import { startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; +import { allocateRunEventPosition, parseRunEventCursor, readRunEventLane, readRunEventPage, runEventCursor, runEventLane } from "./run-event-history.js"; +import { appendHeartbeatRunEvent } from "./heartbeat-run-events.js"; + +it("keeps exact cursors and semantic heads through repeated rollover, concurrency, rollback and reopen", async () => { + const temporary = await startEmbeddedPostgresTestDatabase("paperclip-public-epochs-"); + let db = createDb(temporary.connectionString); + const companyId = randomUUID(), agentId = randomUUID(), runId = randomUUID(), requestId = randomUUID(); + const events: (typeof heartbeatRunEvents.$inferSelect)[] = []; + try { + await db.insert(companies).values({ id: companyId, name: "Public epochs", issuePrefix: "PEP" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Epoch agent" }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running" }); + const append = (eventType: string, payload: Record = {}) => db.transaction(async tx => { + const position = await allocateRunEventPosition(tx as unknown as Db, runId, 4); + const [row] = await tx.insert(heartbeatRunEvents).values({ companyId, agentId, runId, ...position, eventType, payload }).returning(); + return row!; + }); + events.push(await append("runtime_request.created", { prpEvent: { payload: { request: { requestId } } } })); + for (let n = 0; n < 8; n++) events.push(await append(n % 2 ? "item.completed" : "output", { text: `old\0${n}` })); + expect(new Set(events.map(row => row.id)).size).toBe(9); + expect(events.map(row => row.seq)).toEqual([1, 2, 3, 4, 1, 2, 3, 4, 1]); + expect(events.every(row => typeof row.id === "string")).toBe(true); + expect((await readRunEventLane(db, runId, "all", 3)).map(row => row.id)).toEqual(events.slice(-3).reverse().map(row => row.id)); + expect((await readRunEventLane(db, runId, runEventLane("type", "item.completed"), 10)).map(row => row.id)).toEqual(events.filter(row => row.eventType === "item.completed").reverse().map(row => row.id)); + expect((await readRunEventLane(db, runId, runEventLane("request", requestId), 1))[0]?.id).toBe(events[0].id); + const tail = await readRunEventPage(db, runId, "tail", 3); + expect(tail.map(row => row.id)).toEqual(events.slice(-3).map(row => row.id)); + expect(tail[0]).toMatchObject({ historyBefore: true }); + expect((await readRunEventPage(db, runId, runEventCursor(events[3]), 5)).map(row => row.id)).toEqual(events.slice(4).map(row => row.id)); + db = createDb(temporary.connectionString); + let cursor: string | number = 0; const replayed: (string | number)[] = []; + for (;;) { + const page = await readRunEventPage(db, runId, cursor, 2); + if (!page.length) break; + replayed.push(...page.map(row => row.id)); cursor = page.at(-1)!.cursor; + } + expect(replayed).toEqual(events.map(row => row.id)); + expect((await readRunEventPage(db, runId, "0", 2))[1].payload).toEqual({ text: "old\0" + "0" }); + const before = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + await expect(db.transaction(async tx => { + await allocateRunEventPosition(tx as unknown as Db, runId, 2); + throw new Error("storage failure"); + })).rejects.toThrow("storage failure"); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId))).toEqual(before); + expect(await db.select().from(heartbeatRunEventEpochs)).toHaveLength(2); + events.push(await append("runtime_request.resolved", { prpEvent: { payload: { requestId } } })); + expect((await readRunEventLane(db, runId, runEventLane("request", requestId), 2)).map(row => row.id)).toEqual([events.at(-1)!.id, events[0].id]); + const concurrent = await Promise.all(Array.from({ length: 20 }, (_, n) => append("output", { n }))); + const durable = await readRunEventLane(db, runId, "all", 100); + expect(durable).toHaveLength(30); + expect(new Set(durable.map(runEventCursor)).size).toBe(30); + expect(new Set(concurrent.map(row => row.id)).size).toBe(20); + expect(await db.select().from(heartbeatRunEventEpochs)).toHaveLength(7); + await expect(db.insert(heartbeatRunEvents).values({ companyId: randomUUID(), agentId, runId, seq: 99, eventType: "wrong-company" })).rejects.toThrow(); + expect((await db.select().from(heartbeatRunEventHeads).where(and(eq(heartbeatRunEventHeads.runId, runId), eq(heartbeatRunEventHeads.lane, "all"))))[0]?.eventId).toBe(durable[0].id); + expect(await db.select().from(heartbeatRunEventLinks).where(and(eq(heartbeatRunEventLinks.runId, runId), eq(heartbeatRunEventLinks.lane, "all")))).toHaveLength(30); + await db.execute(sql`select setval(pg_get_serial_sequence('heartbeat_run_events', 'id'), 9223372036854775807)`); + expect(typeof (await appendHeartbeatRunEvent(db, { companyId, agentId, runId, eventType: "after-old-global-limit" })).row.id).toBe("string"); + const [largeStart] = await readRunEventLane(db, runId, "all", 1); + const large = []; + for (let n = 0; n < 3; n++) large.push(await append("output", { text: "x".repeat(2 * 1024 * 1024), n })); + // Highly compressible values must be budgeted by their uncompressed JSON, + // not by PostgreSQL's compact on-disk representation. Short pages still + // advertise their successor and work through a namespace boundary. + const largeIds: (string | number)[] = []; + let largeCursor = runEventCursor(largeStart); + for (let n = 0; n < 3; n++) { + const page = await readRunEventPage(db, runId, largeCursor, 100); + expect(page).toHaveLength(1); + expect("historyAfter" in page[0] && page[0].historyAfter).toBe(n < 2 ? true : false); + expect(page[0].payload).toEqual({ text: "x".repeat(2 * 1024 * 1024), n }); + largeIds.push(page[0].id); largeCursor = page[0].cursor; + } + expect(largeIds).toEqual(large.map(row => row.id)); + const largeTail = await readRunEventPage(db, runId, "tail", 100); + expect(largeTail).toHaveLength(1); + expect(largeTail[0]).toMatchObject({ id: large[2].id, historyBefore: true }); + } finally { await temporary.cleanup(); } +}, 60_000); + +it("rejects lossy cursors and retains opaque namespaces", () => { + for (const value of [-1, 1.5, Number.MAX_SAFE_INTEGER + 1, "NaN", "1e2", "01", "e:wrong:1", `e:${randomUUID()}:9007199254740992`]) expect(() => parseRunEventCursor(value)).toThrow("invalid_run_event_cursor"); + const epoch = randomUUID(); + expect(parseRunEventCursor(`e:${epoch}:1`)).toEqual({ eventEpoch: epoch, seq: 1 }); +}); diff --git a/server/src/services/run-event-history.ts b/server/src/services/run-event-history.ts new file mode 100644 index 00000000000..8a87d2adeb2 --- /dev/null +++ b/server/src/services/run-event-history.ts @@ -0,0 +1,155 @@ +import { createHash, randomUUID } from "node:crypto"; +import { and, asc, eq, gt, inArray, sql } from "drizzle-orm"; +import { heartbeatRunEvents, heartbeatRuns, heartbeatRunEventEpochs, heartbeatRunEventHeads, heartbeatRunEventLinks, type Db } from "@paperclipai/db"; + +export const RUN_EVENT_EPOCH_LIMIT = 1_000_000; +const epochPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; +export type RunEventPosition = { eventEpoch: string; seq: number }; +export function runEventCursor(row: RunEventPosition): string { + return row.eventEpoch ? `e:${row.eventEpoch}:${row.seq}` : String(row.seq); +} +export function parseRunEventCursor(value: number | string): RunEventPosition { + const encoded = String(value); + const match = /^e:([^:]+):(0|[1-9][0-9]*)$/.exec(encoded); + const ordinal = match?.[2] ?? encoded; + if ((!match && !/^(0|[1-9][0-9]*)$/.test(encoded)) || (match && !epochPattern.test(match[1]))) throw new Error("invalid_run_event_cursor"); + const seq = Number(ordinal); + if (!Number.isSafeInteger(seq) || seq < 0) throw new Error("invalid_run_event_cursor"); + return { eventEpoch: match?.[1] ?? "", seq }; +} +export function runEventLane(kind: "type" | "request" | "provider-session", key: string): string { + return `${kind}:${createHash("sha256").update(key).digest("hex")}`; +} + +export function runEventSettlementLane(sourceInstanceId: string, turnId?: string): string { + const key = `${Buffer.byteLength(sourceInstanceId)}:${sourceInstanceId}:${turnId ?? ""}`; + return `settlement:${createHash("sha256").update(key).digest("hex")}`; +} + +export function runEventExhaustionLane(input: { retryReason: string; scheduledRetryAttempt: number; maxAttempts: number }): string { + const key = `${Buffer.byteLength(input.retryReason)}:${input.retryReason}:${input.scheduledRetryAttempt}:${input.maxAttempts}`; + return `exhaustion-receipt:${createHash("sha256").update(key).digest("hex")}`; +} + +/** Caller must append in this transaction. The run lock serializes allocation, + * rotation, index maintenance and every native/legacy writer. */ +export async function allocateRunEventPosition(db: Db, runId: string, epochLimit = RUN_EVENT_EPOCH_LIMIT): Promise { + if (!Number.isSafeInteger(epochLimit) || epochLimit < 2 || epochLimit > RUN_EVENT_EPOCH_LIMIT) throw new Error("invalid_run_event_epoch_limit"); + const [run] = await db.select({ eventEpoch: heartbeatRuns.eventEpoch, seq: heartbeatRuns.nextEventSeq, companyId: heartbeatRuns.companyId }) + .from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)).for("update"); + if (!run) throw new Error("heartbeat_run_event_binding_mismatch"); + if (!Number.isSafeInteger(run.seq) || run.seq < 1) throw new Error("invalid_run_event_head"); + let position: RunEventPosition = run; + if (run.seq > epochLimit) { + let nextEpoch: string | undefined; + for (let attempt = 0; attempt < 32; attempt++) { + const candidate = randomUUID(); + if (candidate === run.eventEpoch) continue; + const [used] = await db.select({ epoch: heartbeatRunEventEpochs.epoch }).from(heartbeatRunEventEpochs) + .where(and(eq(heartbeatRunEventEpochs.runId, runId), eq(heartbeatRunEventEpochs.epoch, candidate))).limit(1); + if (!used) { nextEpoch = candidate; break; } + } + if (!nextEpoch) throw new Error("run_event_epoch_identity_unavailable"); + await db.insert(heartbeatRunEventEpochs).values({ companyId: run.companyId, runId, epoch: run.eventEpoch, nextEpoch, finalSeq: run.seq - 1 }); + position = { eventEpoch: nextEpoch, seq: 1 }; + } + await db.update(heartbeatRuns).set({ eventEpoch: position.eventEpoch, nextEventSeq: position.seq + 1, updatedAt: new Date() }).where(eq(heartbeatRuns.id, runId)); + return { eventEpoch: position.eventEpoch, seq: position.seq }; +} + +/** Newest first, bounded in both index probes and materialized rows. The CTE + * follows exact predecessor keys; epoch count and unrelated output do not enter + * the query plan. Hydrate through Drizzle to preserve encoded NUL payloads. */ +export async function readRunEventLaneIds(db: Db, runId: string, lane: string, limit = 200, beforeId?: string | number): Promise { + const count = Math.max(1, Math.min(1001, Math.floor(limit))); + const anchor = beforeId === undefined + ? sql`select ${heartbeatRunEventHeads.eventId}, 1 from ${heartbeatRunEventHeads} + where ${heartbeatRunEventHeads.runId} = ${runId} and ${heartbeatRunEventHeads.lane} = ${lane}` + : sql`select ${heartbeatRunEventLinks.previousId}, 1 from ${heartbeatRunEventLinks} + where ${heartbeatRunEventLinks.runId} = ${runId} and ${heartbeatRunEventLinks.lane} = ${lane} + and ${heartbeatRunEventLinks.eventId} = ${String(beforeId)} and ${heartbeatRunEventLinks.previousId} is not null`; + const records = await db.execute(sql` + with recursive history_window(event_id, depth) as ( + ${anchor} + union all + select link.previous_id, history_window.depth + 1 from history_window + join ${heartbeatRunEventLinks} link on link.run_id = ${runId} and link.lane = ${lane} and link.event_id = history_window.event_id + where history_window.depth < ${count} and link.previous_id is not null + ) select event_id from history_window order by depth asc`); + return Array.from(records as unknown as Iterable<{ event_id: string }>).map(row => row.event_id); +} +export async function readRunEventLane(db: Db, runId: string, lane: string, limit = 200, beforeId?: string | number) { + const ids = await readRunEventLaneIds(db, runId, lane, limit, beforeId); + if (!ids.length) return []; + const rows = await db.select().from(heartbeatRunEvents).where(and(eq(heartbeatRunEvents.runId, runId), inArray(heartbeatRunEvents.id, ids))); + const byId = new Map(rows.map(row => [String(row.id), row])); + return ids.map(id => { const row = byId.get(id); if (!row) throw new Error("run_event_index_incomplete"); return row; }); +} + +const EVENT_PAGE_BYTES = 4 * 1024 * 1024; +const pageMetadata = { + id: heartbeatRunEvents.id, + eventEpoch: heartbeatRunEvents.eventEpoch, + seq: heartbeatRunEvents.seq, + // Measure uncompressed JSON before moving payloads into the server heap. + bytes: sql`coalesce(octet_length(${heartbeatRunEvents.payload}::text), 0) + + coalesce(octet_length(${heartbeatRunEvents.message}), 0) + 2048`.mapWith(Number), +}; +type PageMetadata = RunEventPosition & { id: string | number; bytes: number }; +function budgetedRows(rows: PageMetadata[], limit: number): PageMetadata[] { + let bytes = 0; + const selected: PageMetadata[] = []; + for (const row of rows) { + if (selected.length >= limit || (selected.length && bytes + row.bytes > EVENT_PAGE_BYTES)) break; + // One individually bounded protocol frame can exceed the page target. + // Keep it intact so a page can always advance through retained legacy data. + selected.push(row); bytes += row.bytes; + } + return selected; +} +async function hydratePage(db: Db, runId: string, selected: PageMetadata[]) { + if (!selected.length) return []; + const rows = await db.select().from(heartbeatRunEvents).where(and(eq(heartbeatRunEvents.runId, runId), inArray(heartbeatRunEvents.id, selected.map(row => row.id)))); + const byId = new Map(rows.map(row => [String(row.id), row])); + return selected.map(metadata => { + const row = byId.get(String(metadata.id)); + if (!row) throw new Error("run_event_index_incomplete"); + return { ...row, cursor: runEventCursor(row) }; + }); +} + +/** Forward pages never compare ordinals belonging to different epochs. One + * bounded page may cross several small test epochs, retaining arrival order. */ +export async function readRunEventPage(db: Db, runId: string, after: number | string = 0, limit = 200) { + const count = Math.max(1, Math.min(1000, Math.floor(limit))); + if (after === "tail") { + const ids = await readRunEventLaneIds(db, runId, "all", count + 1); + if (!ids.length) return []; + const metadata = await db.select(pageMetadata).from(heartbeatRunEvents).where(and(eq(heartbeatRunEvents.runId, runId), inArray(heartbeatRunEvents.id, ids))); + const byId = new Map(metadata.map(row => [String(row.id), row])); + const newest = ids.map(id => { const row = byId.get(id); if (!row) throw new Error("run_event_index_incomplete"); return row; }); + const selected = budgetedRows(newest, count); + const rows = await hydratePage(db, runId, selected.reverse()); + return rows.map((row, index) => ({ ...row, ...(index === 0 && newest.length > selected.length ? { historyBefore: true } : {}) })); + } + let position = parseRunEventCursor(after); + const result: PageMetadata[] = []; + // Each committed epoch contains at least one event. This is page-sized work, + // never an unbounded walk from the first namespace to the current head. + for (let boundaries = 0; boundaries <= count + 1; boundaries++) { + const rows = await db.select(pageMetadata).from(heartbeatRunEvents).where(and(eq(heartbeatRunEvents.runId, runId), eq(heartbeatRunEvents.eventEpoch, position.eventEpoch), gt(heartbeatRunEvents.seq, position.seq))) + .orderBy(asc(heartbeatRunEvents.seq)).limit(count + 1 - result.length); + result.push(...rows); + if (result.length === count + 1) break; + const last = rows.at(-1); + if (last) position = last; + const [closed] = await db.select().from(heartbeatRunEventEpochs).where(and(eq(heartbeatRunEventEpochs.runId, runId), eq(heartbeatRunEventEpochs.epoch, position.eventEpoch))).limit(1); + if (!closed) break; + if (position.seq > closed.finalSeq) throw new Error("invalid_run_event_cursor"); + if (position.seq < closed.finalSeq) throw new Error("run_event_epoch_incomplete"); + position = { eventEpoch: closed.nextEpoch, seq: 0 }; + } + const selected = budgetedRows(result, count); + const rows = await hydratePage(db, runId, selected); + return rows.map((row, index) => ({ ...row, ...(index === rows.length - 1 && result.length > selected.length ? { historyAfter: true } : {}) })); +} diff --git a/server/src/services/run-log-store.test.ts b/server/src/services/run-log-store.test.ts index ab12ae17b34..02466482c61 100644 --- a/server/src/services/run-log-store.test.ts +++ b/server/src/services/run-log-store.test.ts @@ -67,6 +67,32 @@ afterEach(async () => { const begin = { companyId: "co1", agentId: "ag1", runId: "run1" }; describe("createDurableRunLogStore", () => { + it("returns durable exact UTF-8 byte positions for concurrent legacy appends and resumes after reopen", async () => { + let store = createDurableRunLogStore({ basePath: baseDir }); + const handle = await store.begin(begin); + const events = ["🙂 café", "漢字", "e\u0301", "tail"].map(chunk => ({ stream: "stdout" as const, chunk, ts: "t" })); + const receipts = await Promise.all(events.map(event => store.appendPositioned!(handle, event))); + const ordered = receipts.map((receipt, index) => ({ receipt, event: events[index]! })).sort((a, b) => Number(BigInt(a.receipt.cursor) - BigInt(b.receipt.cursor))); + let expectedCursor = 0n; + for (const { receipt, event } of ordered) { + expect(receipt.cursor).toBe(String(expectedCursor)); + const encoded = Buffer.from(`${JSON.stringify({ ...event, cursor: receipt.cursor })}\n`); + expect(receipt.bytes).toBe(encoded.length); + expectedCursor += BigInt(encoded.length); + expect(receipt.nextCursor).toBe(String(expectedCursor)); + } + const content = await fs.readFile(path.join(baseDir, handle.logRef)); + expect(content.length).toBe(Number(expectedCursor)); + for (const { event } of ordered) expect(content.toString()).toContain(event.chunk); + + store = createDurableRunLogStore({ basePath: baseDir }); + const reopened = await store.begin(begin); + const next = { stream: "stderr" as const, chunk: "reopened 🦀", ts: "t2" }; + const receipt = await store.appendPositioned!(reopened, next); + expect(receipt.cursor).toBe(String(expectedCursor)); + expect(receipt.nextCursor).toBe(String(expectedCursor + BigInt(receipt.bytes))); + }); + it("keeps store id 'local_file' so downstream coupling (feedback, casts) is unchanged", async () => { const { provider } = createMemoryProvider(); const store = createDurableRunLogStore({ basePath: baseDir, s3: { provider } }); diff --git a/server/src/services/run-log-store.ts b/server/src/services/run-log-store.ts index f07ede9abdf..b1b8e6be6e9 100644 --- a/server/src/services/run-log-store.ts +++ b/server/src/services/run-log-store.ts @@ -1,12 +1,13 @@ -import { createReadStream, promises as fs } from "node:fs"; +import { constants, createReadStream, promises as fs } from "node:fs"; import path from "node:path"; import { createHash } from "node:crypto"; import { notFound } from "../errors.js"; import { resolvePaperclipInstanceRoot } from "../home-paths.js"; import { createS3StorageProvider } from "../storage/s3-provider.js"; import type { StorageProvider } from "../storage/types.js"; +import { createSegmentedRunLogStore } from "./segmented-run-log-store.js"; -export type RunLogStoreType = "local_file"; +export type RunLogStoreType = "local_file" | "local_segments"; export interface RunLogHandle { store: RunLogStoreType; @@ -15,26 +16,36 @@ export interface RunLogHandle { export interface RunLogReadOptions { offset?: number; + /** Exact decimal byte offset, or `tail` for a bounded recent page. */ + cursor?: string; limitBytes?: number; } export interface RunLogReadResult { content: string; nextOffset?: number; + cursor?: string; + hasMore?: boolean; } export interface RunLogFinalizeSummary { - bytes: number; + bytes: number | null; + bytesExact?: string; sha256?: string; compressed: boolean; } +export interface RunLogEvent { stream: "stdout" | "stderr" | "system"; chunk: string; ts: string; seq?: number } +export interface RunLogAppendReceipt { bytes: number; cursor: string; nextCursor: string } + export interface RunLogStore { begin(input: { companyId: string; agentId: string; runId: string }): Promise; append( handle: RunLogHandle, - event: { stream: "stdout" | "stderr" | "system"; chunk: string; ts: string; seq?: number }, + event: RunLogEvent, ): Promise; + /** Durable byte position is the chunk identity; no lifetime event counter. */ + appendPositioned?(handle: RunLogHandle, event: RunLogEvent): Promise; finalize(handle: RunLogHandle): Promise; read(handle: RunLogHandle, opts?: RunLogReadOptions): Promise; // Optional so existing fakes/fixtures keep compiling: uploads every dirty @@ -63,6 +74,8 @@ function normalizeKeyPrefix(prefix: string | undefined): string { export interface DurableRunLogStoreOptions { basePath: string; + /** Fresh-format selection only; retained segmented logs remain readable. */ + segmented?: boolean; // When provided, completed logs are mirrored to object storage on finalize and // served from there on read whenever the local file is missing (e.g. the pod // rolled and wiped the emptyDir). When omitted, the store is local-only (the @@ -98,6 +111,7 @@ export interface DurableRunLogStoreOptions { export function createDurableRunLogStore(options: DurableRunLogStoreOptions): RunLogStore { const { basePath } = options; const s3 = options.s3; + const segmented = createSegmentedRunLogStore(options); const s3Prefix = normalizeKeyPrefix(s3?.keyPrefix); const inflightMirrorMs = s3?.inflightMirrorMs && s3.inflightMirrorMs > 0 ? s3.inflightMirrorMs : 0; @@ -203,17 +217,18 @@ export function createDurableRunLogStore(options: DurableRunLogStoreOptions): Ru async function readLocalRange( filePath: string, - offset: number, + offset: number | "tail", limitBytes: number, ): Promise { const stat = await fs.stat(filePath).catch(() => null); if (!stat) return null; - const start = Math.max(0, Math.min(offset, stat.size)); + const start = offset === "tail" ? Math.max(0, stat.size - limitBytes) : Math.max(0, Math.min(offset, stat.size)); // No lower clamp to `start`: when the reader is fully caught up // (offset === size) that clamp made end === start and produced a // 1-byte-past-EOF range instead of an empty read. const end = Math.min(start + limitBytes - 1, stat.size - 1); - if (start > end) return { content: "", nextOffset: start < stat.size ? start : undefined }; + if (start > end) return { content: "", nextOffset: start < stat.size ? start : undefined, + ...(offset === "tail" ? { cursor: String(start), hasMore: false } : {}) }; const chunks: Buffer[] = []; try { @@ -232,12 +247,12 @@ export function createDurableRunLogStore(options: DurableRunLogStoreOptions): Ru } const content = Buffer.concat(chunks).toString("utf8"); const nextOffset = end + 1 < stat.size ? end + 1 : undefined; - return { content, nextOffset }; + return { content, nextOffset, ...(offset === "tail" ? { cursor: String(end + 1), hasMore: end + 1 < stat.size } : {}) }; } async function readS3Range( logRef: string, - offset: number, + offset: number | "tail", limitBytes: number, ): Promise { if (!s3) throw notFound("Run log not found"); @@ -245,13 +260,14 @@ export function createDurableRunLogStore(options: DurableRunLogStoreOptions): Ru const head = await s3.provider.headObject({ objectKey: key }); if (!head.exists) throw notFound("Run log not found"); const total = head.contentLength ?? 0; - const start = Math.max(0, Math.min(offset, total)); + const start = offset === "tail" ? Math.max(0, total - limitBytes) : Math.max(0, Math.min(offset, total)); // Unlike local file streams, S3 rejects a range that starts at or past // EOF with 416 InvalidRange, so a caught-up reader (offset === total) // must short-circuit to an empty read instead of clamping end up to // start and requesting `bytes=total-total`. const end = Math.min(start + limitBytes - 1, total - 1); - if (total === 0 || start > end) return { content: "", nextOffset: start < total ? start : undefined }; + if (total === 0 || start > end) return { content: "", nextOffset: start < total ? start : undefined, + ...(offset === "tail" ? { cursor: String(start), hasMore: false } : {}) }; const result = await s3.provider.getObject({ objectKey: key, range: { start, end } }); const chunks: Buffer[] = []; @@ -262,7 +278,7 @@ export function createDurableRunLogStore(options: DurableRunLogStoreOptions): Ru }); const content = Buffer.concat(chunks).toString("utf8"); const nextOffset = end + 1 < total ? end + 1 : undefined; - return { content, nextOffset }; + return { content, nextOffset, ...(offset === "tail" ? { cursor: String(end + 1), hasMore: end + 1 < total } : {}) }; } async function sha256File(filePath: string): Promise { @@ -275,12 +291,48 @@ export function createDurableRunLogStore(options: DurableRunLogStoreOptions): Ru }); } + const appendChains = new Map>(); + function appendLegacy(handle: RunLogHandle, event: RunLogEvent, positioned: boolean): Promise { + const before = appendChains.get(handle.logRef) ?? Promise.resolve(); + const pending = before.then(async () => { + if (handle.store !== "local_file") throw new Error("run_log_store_unsupported"); + const fd = await fs.open(resolveWithin(basePath, handle.logRef), constants.O_WRONLY | constants.O_APPEND | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + const stat = await fd.stat({ bigint: true }); + if (!stat.isFile()) throw new Error("run_log_file_unsafe"); + const cursor = String(stat.size); + const bytes = Buffer.from(`${JSON.stringify({ ts: event.ts, stream: event.stream, chunk: event.chunk, + ...(typeof event.seq === "number" && Number.isSafeInteger(event.seq) ? { seq: event.seq } : {}), + ...(positioned ? { cursor } : {}) })}\n`); + await fd.writeFile(bytes); + if (positioned) await fd.sync(); + noteInflightAppend(handle.logRef); + return { bytes: bytes.length, cursor, nextCursor: String(stat.size + BigInt(bytes.length)) }; + } finally { await fd.close(); } + }); + appendChains.set(handle.logRef, pending); + // Retain a failed chain: a partial legacy append has no recoverable head + // and must not be followed by another acknowledged record in this owner. + void pending.then(() => { if (appendChains.get(handle.logRef) === pending) appendChains.delete(handle.logRef); }, () => {}); + return pending; + } + return { async begin(input) { const [companyId, agentId] = safeSegments(input.companyId, input.agentId); const runId = safeSegments(input.runId)[0]!; const relDir = path.join(companyId, agentId); const relPath = path.join(relDir, `${runId}.ndjson`); + const segmentedRef = `${input.companyId}/${input.agentId}/${input.runId}.segments`; + if (await fs.lstat(resolveWithin(basePath, `${segmentedRef}/head.json`)).catch(() => null)) return segmented.begin(input); + const existing = await fs.lstat(resolveWithin(basePath, relPath)).catch(() => null); + if (existing) { + if (!existing.isFile() || existing.isSymbolicLink()) throw new Error("run_log_file_unsafe"); + await appendChains.get(relPath); + return { store: "local_file", logRef: relPath }; + } + if (s3 && (await s3.provider.headObject({ objectKey: s3Key(`${segmentedRef}/head.json`) })).exists) return segmented.begin(input); + if (options.segmented) return segmented.begin(input); await ensureDir(relDir); const absPath = resolveWithin(basePath, relPath); await fs.writeFile(absPath, "", "utf8"); @@ -289,25 +341,18 @@ export function createDurableRunLogStore(options: DurableRunLogStoreOptions): Ru }, async append(handle, event) { - if (handle.store !== "local_file") return 0; - const absPath = resolveWithin(basePath, handle.logRef); - const line = JSON.stringify({ - ts: event.ts, - stream: event.stream, - chunk: event.chunk, - // Monotonic per-run sequence so readers can dedupe and order records - // even when several identical chunks share the same millisecond ts - // (common for ACP-style token deltas). - ...(typeof event.seq === "number" && Number.isFinite(event.seq) ? { seq: event.seq } : {}), - }); - const persisted = `${line}\n`; - await fs.appendFile(absPath, persisted, "utf8"); - noteInflightAppend(handle.logRef); - return Buffer.byteLength(persisted, "utf8"); + if (handle.store === "local_segments") return segmented.append(handle, event); + return (await appendLegacy(handle, event, false)).bytes; + }, + async appendPositioned(handle, event) { + if (handle.store === "local_segments") return segmented.appendPositioned!(handle, event); + return appendLegacy(handle, event, true); }, async finalize(handle) { + if (handle.store === "local_segments") return segmented.finalize(handle); if (handle.store !== "local_file") return { bytes: 0, compressed: false }; + await appendChains.get(handle.logRef); await retireInflightMirror(handle.logRef); const absPath = resolveWithin(basePath, handle.logRef); const stat = await fs.stat(absPath).catch(() => null); @@ -344,10 +389,13 @@ export function createDurableRunLogStore(options: DurableRunLogStoreOptions): Ru }, async read(handle, opts) { + if (handle.store === "local_segments") return segmented.read(handle, opts); if (handle.store !== "local_file") throw notFound("Run log not found"); const absPath = resolveWithin(basePath, handle.logRef); - const offset = opts?.offset ?? 0; + const offset = opts?.cursor === "tail" ? "tail" : opts?.cursor !== undefined ? Number(opts.cursor) : opts?.offset ?? 0; + if (offset !== "tail" && (!Number.isSafeInteger(offset) || offset < 0)) throw new Error("run_log_cursor_invalid"); const limitBytes = opts?.limitBytes ?? 256_000; + if (!Number.isSafeInteger(limitBytes) || limitBytes < 4 || limitBytes > 4 * 1024 * 1024) throw new Error("run_log_page_size_invalid"); const local = await readLocalRange(absPath, offset, limitBytes); if (local) return local; // Local file gone (pod rolled) -> serve from the S3 mirror if configured. @@ -355,6 +403,7 @@ export function createDurableRunLogStore(options: DurableRunLogStoreOptions): Ru }, async flushInflightMirrors() { + await segmented.flushInflightMirrors?.(); if (!s3 || inflightMirrorMs <= 0) return; const flushEntry = async (logRef: string, entry: InflightMirrorEntry) => { // Loop until the entry is clean: an append that lands while an @@ -423,7 +472,7 @@ let cachedStore: RunLogStore | null = null; export function getRunLogStore() { if (cachedStore) return cachedStore; const basePath = process.env.RUN_LOG_BASE_PATH ?? path.resolve(resolvePaperclipInstanceRoot(), "data", "run-logs"); - cachedStore = createDurableRunLogStore({ basePath, s3: resolveRunLogS3() }); + cachedStore = createDurableRunLogStore({ basePath, s3: resolveRunLogS3(), segmented: process.env.PAPERCLIP_NATIVE_INDEXED_STATE === "1" }); return cachedStore; } diff --git a/server/src/services/run-output-body.test.ts b/server/src/services/run-output-body.test.ts new file mode 100644 index 00000000000..491271f4360 --- /dev/null +++ b/server/src/services/run-output-body.test.ts @@ -0,0 +1,39 @@ +import { createHash } from "node:crypto"; +import { expect, it } from "vitest"; +import { assembleRunOutputBody } from "./run-output-body.js"; +import { redactRegisteredSecretValues } from "./run-secret-redaction.js"; +const hash = (s: string) => createHash("sha256").update(s).digest("hex"); +function fixture(text: string) { + const bodyId = hash(text), body = { schema: "paperclip.output.body.v1", bodyId, sha256: bodyId, byteLength: String(Buffer.byteLength(text)), mediaType: "text/plain; charset=utf-8" }; + const chunks: Record[] = []; + for (let offset = 0; offset < text.length; offset += 8192) { + const part = text.slice(offset, offset + 8192); + chunks.push({ schema: "paperclip.output.body.chunk.v1", body, offset: String(offset), sha256: hash(part), text: part }); + } + return { bodyId, chunks }; +} +async function* stream(values: unknown[]) { yield* values; } +it("reconstructs and verifies one bounded body across pages, including a secret split across transport chunks", async () => { + const secret = "registered-provider-key"; + const text = "a".repeat(8190) + secret + "z".repeat(2 * 1024 * 1024); + const f = fixture(text); + const actual = await assembleRunOutputBody(f.bodyId, stream(f.chunks)); + expect(actual).toBe(text); + expect(redactRegisteredSecretValues(actual, [secret])).not.toContain(secret); +}); +it("rejects missing, reordered, corrupt, wrong-body and oversized data", async () => { + const f = fixture("a".repeat(20_000)); + await expect(assembleRunOutputBody(f.bodyId, stream([]))).rejects.toThrow("not found"); + await expect(assembleRunOutputBody(f.bodyId, stream(f.chunks.slice(0, 1)))).rejects.toThrow("not complete"); + for (const values of [ + [...f.chunks].reverse(), + [{ ...f.chunks[0], text: "corrupt" }, ...f.chunks.slice(1)], + [{ ...f.chunks[0], body: { ...(f.chunks[0]!.body as object), byteLength: "9999999" } }], + ]) await expect(assembleRunOutputBody(f.bodyId, stream(values))).rejects.toThrow("verification"); + await expect(assembleRunOutputBody("0".repeat(64), stream(f.chunks))).rejects.toThrow("verification"); +}); +it("stops after the first verified content-addressed occurrence without loading later history", async () => { + const f = fixture("a".repeat(20_000)); + async function* source() { yield* f.chunks; throw new Error("must not read later history"); } + expect(await assembleRunOutputBody(f.bodyId, source())).toHaveLength(20_000); +}); diff --git a/server/src/services/run-output-body.ts b/server/src/services/run-output-body.ts new file mode 100644 index 00000000000..b1a5a36c55d --- /dev/null +++ b/server/src/services/run-output-body.ts @@ -0,0 +1,90 @@ +import { createHash } from "node:crypto"; +import { and, asc, eq, gt } from "drizzle-orm"; +import { heartbeatRunEvents, nativeOutputBodyChunks, type Db } from "@paperclipai/db"; +import { conflict, notFound } from "../errors.js"; +import { createHistoryPayloadStore, type HistoryPayloadStore } from "./native-runtime/history-payload-store.js"; +import type { AppendHeartbeatRunEventInput } from "./heartbeat-run-events.js"; + +// This bounds ONE admitted provider frame, never a run or its accumulated output. +export const MAX_RUN_OUTPUT_BODY_BYTES = 4 * 1024 * 1024; +const MAX_CHUNK_BYTES = 32 * 1024 - 2; +const hex = /^[a-f0-9]{64}$/; +const object = (v: unknown): Record => v !== null && typeof v === "object" && !Array.isArray(v) ? v as Record : {}; +const digest = (text: string) => createHash("sha256").update(text).digest("hex"); + +/** Provider output is durable before its event transaction can acknowledge it. + * Keep the original canonical payload digest for exact replay; the public run + * log and the body catalog carry only this scoped immutable object reference. */ +export async function prepareRunOutputChunkPayload(input: AppendHeartbeatRunEventInput, store?: HistoryPayloadStore): Promise | null> { + if (input.eventType !== "output.body.chunk") return input.payload ?? null; + const event = object(input.payload?.prpEvent), chunk = object(event.payload); + const canonical = object(input.nativeSource?.canonicalPayload.payload); + if (typeof chunk.text !== "string" || !chunk.text.length || Buffer.byteLength(chunk.text) > MAX_CHUNK_BYTES || + chunk.schema !== "paperclip.output.body.chunk.v1" || digest(chunk.text) !== chunk.sha256 || + canonical.text !== chunk.text || canonical.sha256 !== chunk.sha256) throw new Error("native_output_body_binding_invalid"); + const { text, ...metadata } = chunk; + const textRef = await (store ?? createHistoryPayloadStore()).put({ companyId: input.companyId, runId: input.runId }, Buffer.from(text), "text/plain; charset=utf-8"); + return { ...input.payload, prpEvent: { ...event, payload: { ...metadata, textRef } } }; +} + +export async function readRunOutputChunk(scope: { companyId: string; runId: string }, value: unknown, store?: HistoryPayloadStore): Promise> { + const chunk = object(value); + if (chunk.textRef === undefined) return chunk; // Retained inline events. + if (chunk.text !== undefined || object(chunk.textRef).sha256 !== chunk.sha256 || object(chunk.textRef).mediaType !== "text/plain; charset=utf-8") throw conflict("Stored output reference failed verification"); + const text = (await (store ?? createHistoryPayloadStore()).read(scope, chunk.textRef, MAX_CHUNK_BYTES)).toString("utf8"); + return { ...chunk, text }; +} + +/** Read only one immutable body. Pages and the one-frame allocation are bounded. + * Never serve partial output as complete, including a crash before the last chunk. + * Run-specific redaction must be applied to the complete frame by the caller so + * a registered secret split across transport chunks cannot escape that policy. */ +export async function assembleRunOutputBody(bodyId: string, chunks: AsyncIterable): Promise { + if (!hex.test(bodyId)) throw notFound("Output not found"); + let bytes = 0, size = 0; + const parts: string[] = []; + for await (const value of chunks) { + const chunk = object(value), body = object(chunk.body); + const text = chunk.text; + const offset = typeof chunk.offset === "string" && /^(0|[1-9][0-9]{0,6})$/.test(chunk.offset) ? Number(chunk.offset) : -1; + const declared = typeof body.byteLength === "string" && /^[1-9][0-9]{0,6}$/.test(body.byteLength) ? Number(body.byteLength) : 0; + if (chunk.schema !== "paperclip.output.body.chunk.v1" || body.schema !== "paperclip.output.body.v1" || + body.bodyId !== bodyId || body.sha256 !== bodyId || body.mediaType !== "text/plain; charset=utf-8" || + typeof text !== "string" || offset !== bytes || declared < 1 || declared > MAX_RUN_OUTPUT_BODY_BYTES || + (bytes > 0 && declared !== size) || Buffer.byteLength(text) > MAX_CHUNK_BYTES || text.length === 0 || + digest(text) !== chunk.sha256) throw conflict("Stored output failed verification"); + size = declared; bytes += Buffer.byteLength(text); + if (bytes > size) throw conflict("Stored output exceeds its declared length"); + parts.push(text); + if (bytes === size) { + const result = parts.join(""); + if (digest(result) !== bodyId) throw conflict("Stored output failed verification"); + return result; + } + } + if (!parts.length) throw notFound("Output not found"); + throw conflict("Output is not complete yet"); +} + +export async function readRunOutputBody(db: Db, companyId: string, runId: string, bodyId: string): Promise { + async function* pages() { + let after = -1; + for (;;) { + const rows = await db.select({ payload: heartbeatRunEvents.payload, chunkOffset: nativeOutputBodyChunks.chunkOffset }) + .from(nativeOutputBodyChunks).innerJoin(heartbeatRunEvents, and( + eq(heartbeatRunEvents.id, nativeOutputBodyChunks.eventId), + eq(heartbeatRunEvents.companyId, nativeOutputBodyChunks.companyId), + eq(heartbeatRunEvents.runId, nativeOutputBodyChunks.runId), + )).where(and( + eq(nativeOutputBodyChunks.companyId, companyId), eq(nativeOutputBodyChunks.runId, runId), + eq(nativeOutputBodyChunks.bodyId, bodyId), gt(nativeOutputBodyChunks.chunkOffset, after), + )).orderBy(asc(nativeOutputBodyChunks.chunkOffset)).limit(128); + if (!rows.length) return; + for (const row of rows) { + after = row.chunkOffset; + yield await readRunOutputChunk({ companyId, runId }, object(object(row.payload).prpEvent).payload); + } + } + } + return assembleRunOutputBody(bodyId, pages()); +} diff --git a/server/src/services/runner-goals.test.ts b/server/src/services/runner-goals.test.ts index dfec76c01c1..fb2e28f9d3a 100644 --- a/server/src/services/runner-goals.test.ts +++ b/server/src/services/runner-goals.test.ts @@ -1,6 +1,10 @@ import { randomUUID } from "node:crypto"; import { eq } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { + runnerGoalActionRequestSchema, + runnerGoalRevisionSchema, +} from "@paperclipai/shared"; import { agentSessionGoalActions, agentTaskSessions, @@ -118,13 +122,84 @@ describeEmbeddedPostgres("runner goal service", () => { ...binding, adapterType: "paperclip_runner", }, requestId, "provider_start_failed"); expect(failed?.pendingAction).toBeNull(); - expect(failed?.revision).toBe(accepted.projection.revision + 1); + expect(failed?.revision).not.toBe(accepted.projection.revision); + expect(failed?.revision).toMatch(/^r:[0-9a-f-]{36}$/i); expect(await failRunnerGoalAction(db, { ...binding, adapterType: "paperclip_runner", }, requestId, "duplicate_failure")).toBeNull(); expect((await service.projection(binding.companyId, binding.issueId))?.revision).toBe(failed?.revision); }); + it("activates an opaque revision from the legacy integer maximum and replays the accepted result", async () => { + const binding = await seed(); + await db.insert(agentTaskSessions).values({ + ...binding, + adapterType: "paperclip_runner", + taskKey: binding.issueId, + goalRevision: 2_147_483_647, + }); + const service = runnerGoalService(db, { + dispatchLiveControl: () => ({ runId: "run-live", completion: Promise.resolve() }), + queueLiveCommand: () => null, + }); + expect((await service.projection(binding.companyId, binding.issueId))?.revision).toBe(2_147_483_647); + const request = { + requestId: randomUUID(), agentId: binding.agentId, expectedRevision: 2_147_483_647, + action: "create" as const, objective: "Activate the opaque revision", + }; + const accepted = await service.act(binding.companyId, binding.issueId, request); + expect(accepted.projection.revision).toMatch(/^r:[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i); + const [sessionAfterFirstMutation] = await db.select().from(agentTaskSessions); + expect(sessionAfterFirstMutation).toMatchObject({ goalRevision: 2_147_483_647, goalRevisionToken: accepted.projection.revision }); + + await expect(service.act(binding.companyId, binding.issueId, { + ...request, requestId: randomUUID(), action: "pause", objective: undefined, + })).rejects.toMatchObject({ code: "stale_revision" }); + const replay = await service.act(binding.companyId, binding.issueId, request); + expect(replay.projection).toEqual(accepted.projection); + expect(replay.status).toBe("pending"); + const [sessionAfterReplay] = await db.select().from(agentTaskSessions); + expect(sessionAfterReplay?.goalRevisionToken).toBe(accepted.projection.revision); + }); + + it("accepts only safe legacy numbers or exact opaque UUIDv4 revisions", () => { + expect(runnerGoalRevisionSchema.safeParse(Number.MAX_SAFE_INTEGER).success).toBe(true); + expect(runnerGoalRevisionSchema.safeParse(Number.MAX_SAFE_INTEGER + 1).success).toBe(false); + expect(runnerGoalRevisionSchema.safeParse("r:123e4567-e89b-42d3-a456-426614174000").success).toBe(true); + expect(runnerGoalRevisionSchema.safeParse("r:123e4567-e89b-12d3-a456-426614174000").success).toBe(false); + expect(runnerGoalRevisionSchema.safeParse("r:not-a-uuid").success).toBe(false); + expect(runnerGoalActionRequestSchema.safeParse({ + requestId: "request", agentId: randomUUID(), expectedRevision: -1, action: "clear", + }).success).toBe(false); + }); + + it("serializes concurrent compare-and-set actions against one revision", async () => { + const binding = await seed(); + const service = runnerGoalService(db, { enqueueOfflineControl: async () => {} }); + const [first, second] = await Promise.allSettled([ + service.act(binding.companyId, binding.issueId, { + requestId: randomUUID(), agentId: binding.agentId, expectedRevision: 0, + action: "create", objective: "First concurrent goal", + }), + service.act(binding.companyId, binding.issueId, { + requestId: randomUUID(), agentId: binding.agentId, expectedRevision: 0, + action: "create", objective: "Second concurrent goal", + }), + ]); + expect([first.status, second.status].filter((status) => status === "fulfilled")).toHaveLength(1); + const rejected = [first, second].find((result) => result.status === "rejected"); + expect(rejected).toMatchObject({ status: "rejected", reason: { code: "stale_revision" } }); + }); + + it("fails closed when a persisted revision token is malformed", async () => { + const binding = await seed(); + await db.insert(agentTaskSessions).values({ + companyId: binding.companyId, agentId: binding.agentId, adapterType: "paperclip_runner", + taskKey: binding.issueId, goalRevisionToken: "r:broken", + }); + await expect(runnerGoalService(db).projection(binding.companyId, binding.issueId)).rejects.toThrow(/revision token is malformed/); + }); + it("enforces revisions, correlates acknowledgements, and fences cleared goals", async () => { const binding = await seed(); const service = runnerGoalService(db, { @@ -166,7 +241,7 @@ describeEmbeddedPostgres("runner goal service", () => { tokenBudget: 2_000, })).rejects.toMatchObject({ code: "idempotency_key_conflict", - projection: { revision: 1 }, + projection: { revision: accepted.projection.revision }, }); await expect(service.act(binding.companyId, binding.issueId, { @@ -330,7 +405,7 @@ describeEmbeddedPostgres("runner goal service", () => { binding.companyId, binding.issueId, binding.agentId, - )).resolves.toMatchObject({ revision: 1, goal: { status: "complete" } }); + )).resolves.toMatchObject({ revision: expect.stringMatching(/^r:/), goal: { status: "complete" } }); const successorEvent = { eventType: "session.goal.updated", @@ -383,7 +458,7 @@ describeEmbeddedPostgres("runner goal service", () => { // The predecessor cannot erase the successor's active goal. await expect(applyRunnerGoalPrpEvent(db, eventBinding, delayed("session.goal.cleared", 100))).resolves.toBeNull(); await expect(runnerGoalService(db).projection(binding.companyId, binding.issueId)).resolves.toMatchObject({ - revision: 2, goal: { objective: "Successor heartbeat objective" }, + revision: expect.stringMatching(/^r:/), goal: { objective: "Successor heartbeat objective" }, }); await applyRunnerGoalPrpEvent(db, eventBinding, { eventType: "session.goal.cleared", sourceInstanceId: runnerId, sourceRunId: nextRunId, sourceSeq: 2, payload: { goal: null }, @@ -394,7 +469,7 @@ describeEmbeddedPostgres("runner goal service", () => { await expect(applyRunnerGoalPrpEvent(db, eventBinding, { eventType: "session.goal.updated", sourceSeq: 102, payload: delayed("", 0).payload, })).resolves.toBeNull(); - await expect(runnerGoalService(db).projection(binding.companyId, binding.issueId)).resolves.toMatchObject({ revision: 3, goal: null }); + await expect(runnerGoalService(db).projection(binding.companyId, binding.issueId)).resolves.toMatchObject({ revision: expect.stringMatching(/^r:/), goal: null }); const [session] = await db.select().from(agentTaskSessions); expect(session).toMatchObject({ goalSourceId: `${runnerId}:${nextRunId}`, goalSourceCursor: 2 }); }); @@ -419,7 +494,7 @@ describeEmbeddedPostgres("runner goal service", () => { payload: { goal: null }, }); expect(blocked).toMatchObject({ - revision: 2, + revision: expect.stringMatching(/^r:/), goal: { objective: "Recover the durable provider goal", status: "blocked", @@ -434,12 +509,14 @@ describeEmbeddedPostgres("runner goal service", () => { })).resolves.toBeNull(); const [session] = await db.select({ revision: agentTaskSessions.goalRevision, + revisionToken: agentTaskSessions.goalRevisionToken, sourceCursor: agentTaskSessions.goalSourceCursor, desiredState: agentTaskSessions.goalDesiredState, goal: agentTaskSessions.goalJson, }).from(agentTaskSessions); expect(session).toMatchObject({ - revision: 2, + revision: 0, + revisionToken: expect.stringMatching(/^r:/), sourceCursor: 3, desiredState: "paused", goal: { @@ -496,7 +573,7 @@ describeEmbeddedPostgres("runner goal service", () => { const capability = await applyRunnerGoalPrpEvent(db, eventBinding, capabilityEvent); expect(capability).toMatchObject({ capability: { availability: "available", verified: true }, - revision: 1, + revision: expect.stringMatching(/^r:/), }); await expect( applyRunnerGoalPrpEvent(db, eventBinding, capabilityEvent), @@ -509,7 +586,7 @@ describeEmbeddedPostgres("runner goal service", () => { status: "active", workingNow: true, }, - revision: 2, + revision: expect.stringMatching(/^r:/), }); await expect( applyRunnerGoalPrpEvent(db, eventBinding, goalEvent), @@ -523,12 +600,14 @@ describeEmbeddedPostgres("runner goal service", () => { const [session] = await db.select({ revision: agentTaskSessions.goalRevision, + revisionToken: agentTaskSessions.goalRevisionToken, sourceCursor: agentTaskSessions.goalSourceCursor, capability: agentTaskSessions.goalCapabilityJson, goal: agentTaskSessions.goalJson, }).from(agentTaskSessions); expect(session).toMatchObject({ - revision: 2, + revision: 0, + revisionToken: expect.stringMatching(/^r:/), sourceCursor: 3, capability: { availability: "available" }, goal: { diff --git a/server/src/services/runner-goals.ts b/server/src/services/runner-goals.ts index 4b7953647fa..3fce423b3e3 100644 --- a/server/src/services/runner-goals.ts +++ b/server/src/services/runner-goals.ts @@ -7,6 +7,7 @@ import { agentSessionGoalActions, agentTaskSessions, heartbeatRuns, + nativeSourceCursors, issues, } from "@paperclipai/db"; import type { @@ -15,8 +16,10 @@ import type { RunnerGoalCapability, RunnerGoalPendingAction, RunnerGoalProjection, + RunnerGoalRevision, RunnerGoalSnapshot, } from "@paperclipai/shared"; +import { runnerGoalRevisionTokenSchema } from "@paperclipai/shared"; import { queueLiveRunnerPrpCommand } from "../realtime/runner-prp-ws.js"; import { dispatchLiveRunnerGoalControl } from "./runner-goal-control-broker.js"; import { publishLiveEvent } from "./live-events.js"; @@ -27,6 +30,17 @@ const OPEN_ACTION_STATUSES = ["pending", "delivering", "delivered"] as const; type AgentBinding = Pick; type TaskSession = typeof agentTaskSessions.$inferSelect; +function sessionGoalRevision(session: TaskSession): RunnerGoalRevision { + if (session.goalRevisionToken === null) return session.goalRevision; + const parsed = runnerGoalRevisionTokenSchema.safeParse(session.goalRevisionToken); + if (!parsed.success) throw new Error("Stored runner goal revision token is malformed."); + return parsed.data; +} + +function nextGoalRevisionToken(): string { + return `r:${randomUUID()}`; +} + export class RunnerGoalConflictError extends Error { constructor( readonly code: "stale_revision" | "replacement_required" | "idempotency_key_conflict", @@ -237,7 +251,7 @@ function commandSequence(request: RunnerGoalActionRequest) { export function runnerGoalService( db: Db, options: { - queueLiveCommand?: typeof queueLiveRunnerPrpCommand; + queueLiveCommand?: (...args: Parameters) => ReturnType | Awaited>; dispatchLiveControl?: typeof dispatchLiveRunnerGoalControl; enqueueOfflineControl?: (input: { companyId: string; @@ -343,7 +357,7 @@ export function runnerGoalService( workingNow: goal?.workingNow ?? false, activeRunId: activeRun?.id ?? null, pendingAction: projectedPendingAction, - revision: session?.goalRevision ?? 0, + revision: session ? sessionGoalRevision(session) : 0, observedAt: session?.goalObservedAt?.toISOString() ?? null, }; } @@ -391,7 +405,7 @@ export function runnerGoalService( capability: storedCapability(session, fallbackCapability), goal: currentGoal, workingNow: currentGoal?.workingNow ?? false, - revision: session.goalRevision, + revision: sessionGoalRevision(session), observedAt: session.goalObservedAt?.toISOString() ?? null, }; @@ -410,7 +424,8 @@ export function runnerGoalService( result: asRecord(existingAction.resultJson), }; } - if (session.goalRevision !== request.expectedRevision) { + const currentRevision = sessionGoalRevision(session); + if (currentRevision !== request.expectedRevision) { throw new RunnerGoalConflictError("stale_revision", currentProjection); } const capability = storedCapability(session, fallbackCapability); @@ -440,7 +455,7 @@ export function runnerGoalService( throw new RunnerGoalActionError("goal_not_found", "There is no current session goal."); } - const nextRevision = session.goalRevision + 1; + const nextRevision = nextGoalRevisionToken(); const desiredState = request.action === "pause" ? "paused" : request.action === "clear" @@ -448,7 +463,7 @@ export function runnerGoalService( : "active"; await tx.update(agentTaskSessions).set({ goalDesiredState: desiredState, - goalRevision: nextRevision, + goalRevisionToken: nextRevision, updatedAt: new Date(), }).where(eq(agentTaskSessions.id, session.id)); const pendingProjection: RunnerGoalProjection = { @@ -466,7 +481,7 @@ export function runnerGoalService( }).returning(); return { repeated: false, - session: { ...session, goalRevision: nextRevision }, + session: { ...session, goalRevisionToken: nextRevision }, status: action!.status, result: { projection: pendingProjection }, }; @@ -509,14 +524,14 @@ export function runnerGoalService( live = true; liveCompletions.push(adapterControl.completion); } else if (hasActiveRun) { - const queuedCommands = commands.map((command, index) => queueLiveCommand({ + const queuedCommands = await Promise.all(commands.map((command, index) => queueLiveCommand({ companyId, issueId, agentId: request.agentId, type: command.type, payload: command.payload, commandId: `goal_${request.requestId}_${index + 1}`, - })); + }))); live = queuedCommands.every((queued) => queued !== null); for (const queued of queuedCommands) { if (queued) liveCompletions.push(queued.completion); @@ -642,6 +657,7 @@ export async function applyRunnerGoalPrpEvent( sourceInstanceId?: string; sourceRunId?: string; sourceSeq: number; + sourceEpoch?: string; payload: unknown; }, ): Promise { @@ -717,11 +733,17 @@ export async function applyRunnerGoalPrpEvent( } if ( !session || - (sameSource && session.goalSourceCursor !== null && event.sourceSeq <= session.goalSourceCursor) + (sameSource && (session.goalSourceEpoch ?? undefined) === event.sourceEpoch && session.goalSourceCursor !== null && event.sourceSeq <= session.goalSourceCursor) ) { return false; } + if (sameSource && (session.goalSourceEpoch ?? undefined) !== event.sourceEpoch) { + if (!event.sourceRunId || !event.sourceInstanceId) return false; + const [head] = await tx.select().from(nativeSourceCursors).where(and(eq(nativeSourceCursors.companyId, binding.companyId), + eq(nativeSourceCursors.runId, event.sourceRunId), eq(nativeSourceCursors.sourceInstanceId, event.sourceInstanceId))).limit(1); + if (!head || (head.sourceEpoch || undefined) !== event.sourceEpoch || head.cursor < event.sourceSeq) return false; + } const capability = asRecord(payload.sessionGoals); const providerError = typeof payload.error === "string" && payload.error.length > 0; const turnLifecycleEvent = event.eventType.startsWith("turn."); @@ -802,6 +824,7 @@ export async function applyRunnerGoalPrpEvent( const observedAt = new Date(); const update: Partial = { goalSourceCursor: event.sourceSeq, + goalSourceEpoch: event.sourceEpoch ?? null, ...(sourceId ? { goalSourceId: sourceId } : {}), goalObservedAt: observedAt, updatedAt: observedAt, @@ -810,7 +833,7 @@ export async function applyRunnerGoalPrpEvent( await tx.update(agentTaskSessions).set(update).where(eq(agentTaskSessions.id, session.id)); return false; } - update.goalRevision = session.goalRevision + 1; + update.goalRevisionToken = nextGoalRevisionToken(); if (capability) { update.goalCapabilityJson = capability; } @@ -876,7 +899,7 @@ export async function failRunnerGoalAction( // Pending-action transitions are part of the same revisioned projection as // provider snapshots. Clients must not discard a failed start as a replay. await tx.update(agentTaskSessions).set({ - goalRevision: session.goalRevision + 1, + goalRevisionToken: nextGoalRevisionToken(), goalObservedAt: now, updatedAt: now, }).where(eq(agentTaskSessions.id, session.id)); @@ -923,7 +946,7 @@ export async function blockRunnerGoalRecovery( }, goalStatus: "blocked", goalDesiredState: "paused", - goalRevision: session.goalRevision + 1, + goalRevisionToken: nextGoalRevisionToken(), goalObservedAt: now, updatedAt: now, }).where(eq(agentTaskSessions.id, session.id)); @@ -975,7 +998,7 @@ export async function settleLiveRunnerGoalBeforeInterrupt( const adapterControl = dispatchLiveRunnerGoalControl(binding, { requestId, action }); const nativeControl = adapterControl ? null - : queueLiveRunnerPrpCommand({ + : await queueLiveRunnerPrpCommand({ ...binding, type: action === "pause" ? "session.goal.set" : "session.goal.clear", payload: action === "pause" diff --git a/server/src/services/segmented-run-log-scale.qualification.test.ts b/server/src/services/segmented-run-log-scale.qualification.test.ts new file mode 100644 index 00000000000..22e394fcac4 --- /dev/null +++ b/server/src/services/segmented-run-log-scale.qualification.test.ts @@ -0,0 +1,60 @@ +import { createHash, randomBytes } from "node:crypto"; +import { promises as fs } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it } from "vitest"; +import { createSegmentedRunLogStore } from "./segmented-run-log-store.js"; + +it.skipIf(process.env.PAPERCLIP_SEGMENTED_LOG_SCALE !== "1")("streams 256 MiB through append, verifies every byte, and resumes without old segments", async () => { + const basePath = await fs.mkdtemp(join(tmpdir(), "paperclip-log-growth-")); + const reportPath = process.env.PAPERCLIP_SEGMENTED_LOG_REPORT; + try { + let store = createSegmentedRunLogStore({ basePath }); + const handle = await store.begin({ companyId: "qualification", agentId: "agent", runId: "scale" }); + const expected = createHash("sha256"); + let total = 0; + const began = performance.now(); + while (total < 256 * 1024 * 1024) { + const event = { stream: "stdout" as const, ts: new Date().toISOString(), chunk: randomBytes(3 * 1024 * 1024).toString("base64") }; + const line = Buffer.from(JSON.stringify(event) + "\n"); + total += await store.append(handle, event); + expected.update(line); + } + const appendMs = performance.now() - began; + const actual = createHash("sha256"); + let cursor = "0"; + for (;;) { + const page = await store.read(handle, { cursor, limitBytes: 1024 * 1024 }); + actual.update(page.content); + cursor = page.cursor!; + if (!page.hasMore) break; + } + expect(cursor).toBe(String(total)); + expect(actual.digest("hex")).toBe(expected.digest("hex")); + const directory = join(basePath, handle.logRef); + const headBytes = (await fs.stat(join(directory, "head.json"))).size; + let allocatedBytes = 0, segments = 0; + for (const group of await fs.readdir(join(directory, "segments"))) { + for (const file of await fs.readdir(join(directory, "segments", group))) { + if (!file.endsWith(".ndjson")) continue; + const stat = await fs.stat(join(directory, "segments", group, file)); + allocatedBytes += stat.blocks * 512; segments++; + expect(stat.size).toBeLessThanOrEqual(32 * 1024 * 1024); + } + } + expect(allocatedBytes).toBeGreaterThanOrEqual(total); + await fs.rename(join(directory, "segments/00/00.ndjson"), join(directory, "unavailable-old-segment")); + store = createSegmentedRunLogStore({ basePath }); + const resume = performance.now(); + await store.begin({ companyId: "qualification", agentId: "agent", runId: "scale" }); + const reopenMs = performance.now() - resume; + await store.append(handle, { stream: "stdout", ts: "resumed", chunk: "continued after 256 MiB" }); + const finalize = performance.now(); + const summary = await store.finalize(handle); + const finalizeMs = performance.now() - finalize; + expect((await store.read(handle, { cursor })).content).toContain("continued after 256 MiB"); + const report = { schema: "paperclip.segmented-log-growth.v1", status: "passed", totalBytes: total, allocatedBytes, segments, headBytes, appendMs, reopenMs, finalizeMs, finalizedBytes: summary.bytesExact }; + if (reportPath) await fs.writeFile(reportPath, JSON.stringify(report, null, 2) + "\n"); + console.log(JSON.stringify(report)); + } finally { await fs.rm(basePath, { recursive: true, force: true }); } +}, 300_000); diff --git a/server/src/services/segmented-run-log-store.test.ts b/server/src/services/segmented-run-log-store.test.ts new file mode 100644 index 00000000000..9d89960dc0e --- /dev/null +++ b/server/src/services/segmented-run-log-store.test.ts @@ -0,0 +1,643 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { promises as fs } from "node:fs"; +import { createHash } from "node:crypto"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Readable } from "node:stream"; +import type { StorageProvider } from "../storage/types.js"; +import { createSegmentedRunLogStore } from "./segmented-run-log-store.js"; +import { createDurableRunLogStore, type RunLogHandle, type RunLogStore } from "./run-log-store.js"; + +const roots: string[] = []; +afterEach(async () => { vi.restoreAllMocks(); await Promise.all(roots.splice(0).map(path => fs.rm(path, { recursive: true, force: true }))); }); +async function temporary() { const root = await fs.mkdtemp(join(tmpdir(), "paperclip-log-segments-")); roots.push(root); return root; } +const binding = { companyId: "company", agentId: "agent", runId: "run" }; +const event = (chunk: string) => ({ stream: "stdout" as const, ts: "2026-09-28T00:00:00Z", chunk }); +function storage() { + const objects = new Map(); + const writes: Array<{ key: string; bytes: number }> = []; + const etag = (bytes: Buffer) => `"${createHash("sha256").update(bytes).digest("hex")}"`; + const provider: StorageProvider = { + id: "s3", + async putObject(input) { + const chunks: Buffer[] = []; + if (Buffer.isBuffer(input.body)) chunks.push(input.body); + else for await (const chunk of input.body) chunks.push(Buffer.from(chunk)); + const bytes = Buffer.concat(chunks); + expect(bytes.length).toBe(input.contentLength); + expect(createHash("sha256").update(bytes).digest("hex")).toBe(input.sha256); + writes.push({ key: input.objectKey, bytes: bytes.length }); + objects.set(input.objectKey, bytes); + }, + async putObjectConditional(input, expected) { + const previous = objects.get(input.objectKey); + if (expected === null ? previous !== undefined : previous === undefined || etag(previous) !== expected) { + throw Object.assign(new Error("precondition failed"), { name: "PreconditionFailed" }); + } + // Tests have one atomic in-memory storage operation. Consuming a stream + // may yield; check the condition again at actual publication below. + const chunks: Buffer[] = []; + if (Buffer.isBuffer(input.body)) chunks.push(input.body); + else for await (const chunk of input.body) chunks.push(Buffer.from(chunk)); + const current = objects.get(input.objectKey); + if (expected === null ? current !== undefined : current === undefined || etag(current) !== expected) throw Object.assign(new Error("precondition failed"), { name: "PreconditionFailed" }); + const bytes = Buffer.concat(chunks); + expect(bytes.length).toBe(input.contentLength); + expect(createHash("sha256").update(bytes).digest("hex")).toBe(input.sha256); + writes.push({ key: input.objectKey, bytes: bytes.length }); objects.set(input.objectKey, bytes); + return { etag: etag(bytes) }; + }, + async getObject(input) { + const bytes = objects.get(input.objectKey); + if (!bytes) throw Object.assign(new Error("object_missing"), { status: 404 }); + const selected = input.range ? bytes.subarray(input.range.start, input.range.end + 1) : bytes; + return { stream: Readable.from(selected), contentLength: selected.length, etag: etag(bytes) }; + }, + async headObject(input) { const value = objects.get(input.objectKey); return value ? { exists: true, contentLength: value.length, etag: etag(value) } : { exists: false }; }, + async deleteObject(input) { objects.delete(input.objectKey); }, + }; + return { provider, objects, writes }; +} + +it.each([false, true])("queues a task page's concurrent old log reads with bounded I/O (read failure: %s)", async failOne => { + const basePath = await temporary(), { provider } = storage(); + const writer = createSegmentedRunLogStore({ basePath, segmentBytes: 1024, s3: { provider } }); + const handles: RunLogHandle[] = []; + for (let index = 0; index < 12; index++) { + const handle = await writer.begin({ ...binding, runId: `read-${index}` }); + await writer.append(handle, event(`history ${index}`)); handles.push(handle); + } + await writer.flushInflightMirrors!(); + const original = provider.getObject.bind(provider); + let active = 0, peak = 0, reads = 0; + vi.spyOn(provider, "getObject").mockImplementation(async input => { + if (!input.objectKey.includes("/tails/")) return original(input); + active++; peak = Math.max(peak, active); reads++; + try { + // Hold each body request across an event-loop turn so all twelve HTTP + // equivalents contend for admission rather than completing serially. + await new Promise(resolve => setTimeout(resolve, 10)); + if (failOne && input.objectKey.startsWith(handles[0]!.logRef + "/")) throw new Error("qualified body read failure"); + return await original(input); + } finally { active--; } + }); + const reader = createSegmentedRunLogStore({ basePath: await temporary(), s3: { provider } }); + const pages = await Promise.allSettled(handles.map(handle => reader.read(handle))); + expect(peak).toBe(2); + expect(reads).toBe(12); + for (const [index, page] of pages.entries()) { + if (failOne && index === 0) expect(page).toMatchObject({ status: "rejected", reason: { message: "qualified body read failure" } }); + else expect(page).toMatchObject({ status: "fulfilled", value: { content: JSON.stringify(event(`history ${index}`)) + "\n" } }); + } + expect(active).toBe(0); +}); + +it("bounds the log-read waiting queue and reports HTTP backpressure rather than an internal error", async () => { + const basePath = await temporary(), { provider } = storage(); + const writer = createSegmentedRunLogStore({ basePath, segmentBytes: 1024, s3: { provider } }); + const handles: RunLogHandle[] = []; + for (let index = 0; index < 70; index++) { + const handle = await writer.begin({ ...binding, runId: `queue-${index}` }); + await writer.append(handle, event(`history ${index}`)); handles.push(handle); + } + await writer.flushInflightMirrors!(); + const original = provider.getObject.bind(provider); + let release!: () => void, entered = 0; + const blocked = new Promise(resolve => { release = resolve; }); + vi.spyOn(provider, "getObject").mockImplementation(async input => { + if (input.objectKey.includes("/tails/")) { entered++; await blocked; } + return original(input); + }); + const reader = createSegmentedRunLogStore({ basePath: await temporary(), s3: { provider } }); + const rejected: unknown[] = []; + const pending = Promise.allSettled(handles.map(handle => reader.read(handle).catch(error => { rejected.push(error); throw error; }))); + try { + await vi.waitFor(() => { expect(entered).toBe(2); expect(rejected).toHaveLength(4); }); + expect(rejected.every(error => (error as { status?: number }).status === 429)).toBe(true); + release(); + expect((await pending).filter(result => result.status === "fulfilled")).toHaveLength(66); + // Credits return after completion; the rejected requests can retry. + expect((await reader.read(handles[69]!)).content).toContain("history 69"); + } finally { release(); await pending; } +}); + +it("refreshes two retired tails while newer requests are queued without waiting on its own credit", async () => { + const basePath = await temporary(), { provider, objects } = storage(); + const writer = createSegmentedRunLogStore({ basePath, segmentBytes: 4096, s3: { provider } }); + const handles = await Promise.all([0, 1].map(index => writer.begin({ ...binding, runId: `retired-${index}` }))); + const first = event("old checkpoint"), next = event("next checkpoint"); + for (const handle of handles) await writer.append(handle, first); + await writer.flushInflightMirrors!(); + const oldKeys = new Set(handles.map(handle => `${handle.logRef}/${JSON.parse(objects.get(`${handle.logRef}/head.json`)!.toString()).tailObject}`)); + const original = provider.getObject.bind(provider); + let release!: () => void, entered = 0; + const blocked = new Promise(resolve => { release = resolve; }); + vi.spyOn(provider, "getObject").mockImplementation(async input => { + if (oldKeys.has(input.objectKey)) { entered++; await blocked; } + return original(input); + }); + const reader = createSegmentedRunLogStore({ basePath: await temporary(), s3: { provider } }); + const oldReads = Promise.all(handles.map(handle => reader.read(handle))); + void oldReads.catch(() => {}); + let newReads: Promise | undefined; + try { + await vi.waitFor(() => expect(entered).toBe(2)); + for (const handle of handles) await writer.append(handle, next); + await writer.flushInflightMirrors!(); + expect([...oldKeys].every(key => !objects.has(key))).toBe(true); + newReads = Promise.all(handles.map(handle => reader.read(handle))); + void newReads.catch(() => {}); + // Both newer requests reach the queue before the old snapshots refresh. + await new Promise(resolve => setTimeout(resolve, 30)); + release(); + for (const page of await oldReads) expect(page.content).toBe(JSON.stringify(first) + "\n"); + for (const page of await newReads) expect(page).toMatchObject({ content: [first, next].map(value => JSON.stringify(value) + "\n").join("") }); + } finally { release(); await Promise.allSettled([oldReads, ...(newReads ? [newReads] : [])]); } +}); + +async function all(store: RunLogStore, handle: RunLogHandle, limitBytes = 83) { + let cursor = "0", content = ""; + for (;;) { + const page = await store.read(handle, { cursor, limitBytes }); + content += page.content; + if (!page.hasMore) return content; + expect(BigInt(page.cursor!)).toBeGreaterThan(BigInt(cursor)); + cursor = page.cursor!; + } +} + +it("rejects a late old tail publication after a replacement claims the remote head", async () => { + const { provider } = storage(), oldPath = await temporary(), nextPath = await temporary(); + let reached!: () => void, release!: () => void, armed = false; + const blocked = new Promise(resolve => { reached = resolve; }); + const resumed = new Promise(resolve => { release = resolve; }); + const old = createSegmentedRunLogStore({ basePath: oldPath, segmentBytes: 1024, s3: { provider }, + onBoundary: async boundary => { if (armed && boundary === "remote-tail") { armed = false; reached(); await resumed; } } }); + const handle = await old.begin(binding), first = event("original"), stale = event("stale"), next = event("new owner"); + await old.append(handle, first); await old.flushInflightMirrors!(); + await old.append(handle, stale); armed = true; + const pending = old.flushInflightMirrors!(); void pending.catch(() => {}); + try { + await blocked; + const replacement = createSegmentedRunLogStore({ basePath: nextPath, s3: { provider } }); + await replacement.begin(binding); await replacement.append(handle, next); await replacement.flushInflightMirrors!(); + release(); await expect(pending).rejects.toThrow("precondition failed"); + const reader = createSegmentedRunLogStore({ basePath: await temporary(), s3: { provider } }); + expect(await all(reader, handle)).toBe([first, next].map(value => JSON.stringify(value) + "\n").join("")); + // The old local tail cannot be adopted as an unmirrored successor of the + // replacement's journal merely because its byte count is greater. + await expect(createSegmentedRunLogStore({ basePath: oldPath, s3: { provider } }).begin(binding)).rejects.toThrow("remote_owner_changed"); + } finally { release(); await pending.catch(() => {}); } +}); + +it("replaces an uncommitted remote segment reference without overwriting immutable content", async () => { + const { provider } = storage(), basePath = await temporary(); + let armed = false; + let store = createSegmentedRunLogStore({ basePath, segmentBytes: 256, s3: { provider }, + onBoundary: async boundary => { if (armed && boundary === "segment") { armed = false; throw new Error("crashed before local publication"); } } }); + const handle = await store.begin(binding), first = event("first"), next = event("continued " + "n".repeat(700)); + await store.append(handle, first); await store.flushInflightMirrors!(); armed = true; + await expect(store.append(handle, event("x".repeat(700)))).rejects.toThrow("crashed"); + store = createSegmentedRunLogStore({ basePath, s3: { provider } }); + await store.begin(binding); await store.append(handle, next); await store.flushInflightMirrors!(); + const reader = createSegmentedRunLogStore({ basePath: await temporary(), s3: { provider } }); + expect(await all(reader, handle)).toBe([first, next].map(value => JSON.stringify(value) + "\n").join("")); +}); + +it("fences a paused segment writer after another owner publishes different content", async () => { + const { provider } = storage(), oldPath = await temporary(); + let reached!: () => void, release!: () => void, armed = false; + const blocked = new Promise(resolve => { reached = resolve; }); + const resumed = new Promise(resolve => { release = resolve; }); + const old = createSegmentedRunLogStore({ basePath: oldPath, segmentBytes: 256, s3: { provider }, + onBoundary: async boundary => { if (armed && boundary === "segment") { armed = false; reached(); await resumed; } } }); + const handle = await old.begin(binding), first = event("first"), next = event("n".repeat(900)); + await old.append(handle, first); await old.flushInflightMirrors!(); armed = true; + const pending = old.append(handle, event("o".repeat(900))); void pending.catch(() => {}); + try { + await blocked; + const replacement = createSegmentedRunLogStore({ basePath: await temporary(), s3: { provider } }); + await replacement.begin(binding); await replacement.append(handle, next); await replacement.flushInflightMirrors!(); + release(); await expect(pending).rejects.toThrow("remote_owner_changed"); + const reader = createSegmentedRunLogStore({ basePath: await temporary(), s3: { provider } }); + expect(await all(reader, handle)).toBe([first, next].map(value => JSON.stringify(value) + "\n").join("")); + } finally { release(); await pending.catch(() => {}); } +}); + +it("changes reference identity even for identical bytes so a late orphan update cannot win an ETag ABA", async () => { + const { provider } = storage(), oldPath = await temporary(); + const first = event("first"), segment = event("x".repeat(400)); + let armed = false; + const seed = createSegmentedRunLogStore({ basePath: oldPath, segmentBytes: 256, s3: { provider }, + onBoundary: async boundary => { if (armed && boundary === "segment") { armed = false; throw new Error("orphan"); } } }); + const handle = await seed.begin(binding); await seed.append(handle, first); await seed.flushInflightMirrors!(); armed = true; + await expect(seed.append(handle, segment)).rejects.toThrow("orphan"); + const old = createSegmentedRunLogStore({ basePath: oldPath, s3: { provider } }); await old.begin(binding); + let reached!: () => void, release!: () => void; + const blocked = new Promise(resolve => { reached = resolve; }); + const resumed = new Promise(resolve => { release = resolve; }); + const original = provider.putObjectConditional!.bind(provider); + let pause = true; + vi.spyOn(provider, "putObjectConditional").mockImplementation(async (input, expected) => { + if (pause && input.objectKey.endsWith("/00.ndjson.json")) { + pause = false; expect(expected).not.toBeNull(); reached(); await resumed; + } + return original(input, expected); + }); + const pending = old.append(handle, segment); void pending.catch(() => {}); + try { + await blocked; + const replacement = createSegmentedRunLogStore({ basePath: await temporary(), s3: { provider } }); + await replacement.begin(binding); await replacement.append(handle, segment); await replacement.flushInflightMirrors!(); + release(); await expect(pending).rejects.toThrow("precondition failed"); + const reader = createSegmentedRunLogStore({ basePath: await temporary(), s3: { provider } }); + expect(await all(reader, handle)).toBe([first, segment].map(value => JSON.stringify(value) + "\n").join("")); + } finally { release(); await pending.catch(() => {}); } +}); + +it.each(["append", "replace"] as const)("reclaims old mirrored tails while a reader holds its prior head (%s)", async action => { + const { provider, objects } = storage(), basePath = await temporary(); + const writer = createSegmentedRunLogStore({ basePath, segmentBytes: 4096, s3: { provider } }); + const handle = await writer.begin(binding), first = event("first"); + await writer.append(handle, first); await writer.flushInflightMirrors!(); + const head = JSON.parse(objects.get(`${handle.logRef}/head.json`)!.toString()); + let reached!: () => void, release!: () => void; + const blocked = new Promise(resolve => { reached = resolve; }); + const resumed = new Promise(resolve => { release = resolve; }); + const original = provider.getObject.bind(provider); + let pause = true; + vi.spyOn(provider, "getObject").mockImplementation(async input => { + if (pause && input.objectKey === `${handle.logRef}/${head.tailObject}`) { + pause = false; reached(); await resumed; + } + return original(input); + }); + const reader = createSegmentedRunLogStore({ basePath: await temporary(), s3: { provider } }); + const pending = reader.read(handle); void pending.catch(() => {}); + try { + await blocked; + if (action === "replace") { + const replacement = createSegmentedRunLogStore({ basePath: await temporary(), s3: { provider } }); + await replacement.begin(binding); + } else { + for (let index = 0; index < 12; index++) { + await writer.append(handle, event(`next ${index}`)); await writer.flushInflightMirrors!(); + } + } + expect(objects.has(`${handle.logRef}/${head.tailObject}`)).toBe(false); + expect([...objects.keys()].filter(key => key.includes("/tails/"))).toHaveLength(1); + release(); expect((await pending).content).toBe(JSON.stringify(first) + "\n"); + } finally { release(); await pending.catch(() => {}); } +}); + +it("retains one pending tail deletion through cleanup failure and resumes it on reopen", async () => { + const { provider, objects } = storage(), basePath = await temporary(); + let writer = createSegmentedRunLogStore({ basePath, segmentBytes: 1024, s3: { provider } }); + const handle = await writer.begin(binding); + await writer.append(handle, event("first")); await writer.flushInflightMirrors!(); + await writer.append(handle, event("next")); + const remove = vi.spyOn(provider, "deleteObject").mockRejectedValueOnce(new Error("deletion offline")); + await expect(writer.flushInflightMirrors!()).rejects.toThrow("deletion offline"); + const head = JSON.parse(objects.get(`${handle.logRef}/head.json`)!.toString()); + expect(head.retiredTail).toBeDefined(); + expect(objects.has(`${handle.logRef}/${head.retiredTail}`)).toBe(true); + await expect(writer.append(handle, event("cannot accumulate more garbage"))).rejects.toThrow("writer_fenced"); + remove.mockRestore(); + writer = createSegmentedRunLogStore({ basePath, s3: { provider } }); await writer.begin(binding); + expect([...objects.keys()].filter(key => key.includes("/tails/"))).toHaveLength(1); + expect(await all(writer, handle)).toContain("next"); +}); + +it("reads retained v1 remote segments and explicitly publishes v2 when a writer takes ownership", async () => { + const { provider, objects } = storage(), basePath = await temporary(); + let writer = createSegmentedRunLogStore({ basePath, segmentBytes: 256, s3: { provider } }); + const handle = await writer.begin(binding); + await writer.append(handle, event("old format " + "x".repeat(600))); await writer.flushInflightMirrors!(); + const expected = await all(writer, handle); + // Reconstruct the original v1 shape, including fixed object names. + const head = JSON.parse(objects.get(`${handle.logRef}/head.json`)!.toString()); + objects.set(`${handle.logRef}/tails/${head.revision}.ndjson`, objects.get(`${handle.logRef}/${head.tailObject}`)!); + head.schema = "paperclip.run-log.segments.v1"; delete head.writerId; delete head.tailObject; delete head.retiredTail; + objects.set(`${handle.logRef}/head.json`, Buffer.from(JSON.stringify(head))); + for (const [key, bytes] of [...objects]) if (key.includes("/segments/") && key.endsWith(".json")) { + const reference = JSON.parse(bytes.toString()); + objects.set(key.slice(0, -5), objects.get(`${handle.logRef}/${reference.object}`)!); + reference.schema = "paperclip.run-log.segment.v1"; delete reference.object; delete reference.writerId; + objects.set(key, Buffer.from(JSON.stringify(reference))); + } + await fs.rm(basePath, { recursive: true }); + writer = createSegmentedRunLogStore({ basePath, s3: { provider } }); + expect(await all(writer, handle)).toBe(expected); + await writer.begin(binding); + expect(JSON.parse(objects.get(`${handle.logRef}/head.json`)!.toString()).schema).toBe("paperclip.run-log.segments.v2"); + const next = event("new format " + "n".repeat(500)); + await writer.append(handle, next); await writer.flushInflightMirrors!(); + expect(await all(writer, handle)).toBe(expected + JSON.stringify(next) + "\n"); +}); + +it("rotates real bytes, survives reopen, and reads UTF-8 across segment/page boundaries", async () => { + const basePath = await temporary(); + let store = createSegmentedRunLogStore({ basePath, segmentBytes: 128 }); + const handle = await store.begin(binding); + let expected = ""; + for (let i = 0; i < 20; i++) { + const value = event(`${i}: 🦀 café 漢字 ${"x".repeat(61)}`); + await store.append(handle, value); expected += `${JSON.stringify(value)}\n`; + } + store = createSegmentedRunLogStore({ basePath, segmentBytes: 256 }); + expect(await store.begin(binding)).toEqual(handle); // Persisted segment size wins. + const resumed = event("resumed"); + await store.append(handle, resumed); expected += `${JSON.stringify(resumed)}\n`; + expect(await all(store, handle, 67)).toBe(expected); + expect(await store.finalize(handle)).toEqual({ bytes: Buffer.byteLength(expected), bytesExact: String(Buffer.byteLength(expected)), compressed: false }); + expect(await store.read(handle, { cursor: String(Buffer.byteLength(expected)) })).toMatchObject({ content: "", hasMore: false, cursor: String(Buffer.byteLength(expected)) }); + const head = await fs.readFile(join(basePath, handle.logRef, "head.json")); + expect(head.length).toBeLessThan(512); +}); + +it("positions appends across segment boundaries and resumes exact byte cursors after reopen", async () => { + const basePath = await temporary(); + let store = createSegmentedRunLogStore({ basePath, segmentBytes: 128 }); + const handle = await store.begin(binding); + const first = event("é🦀" + "a".repeat(150)); + const firstReceipt = await store.appendPositioned!(handle, first); + expect(firstReceipt.cursor).toBe("0"); + expect(firstReceipt.bytes).toBe(Buffer.byteLength(`${JSON.stringify({ ...first, cursor: "0" })}\n`)); + const second = event("second 漢字"); + const secondReceipt = await store.appendPositioned!(handle, second); + expect(secondReceipt.cursor).toBe(firstReceipt.nextCursor); + expect(secondReceipt.nextCursor).toBe(String(BigInt(secondReceipt.cursor) + BigInt(secondReceipt.bytes))); + + store = createSegmentedRunLogStore({ basePath, segmentBytes: 256 }); + expect(await store.begin(binding)).toEqual(handle); + const third = event("after reopen 🦀"); + const thirdReceipt = await store.appendPositioned!(handle, third); + expect(thirdReceipt.cursor).toBe(secondReceipt.nextCursor); + const whole = [first, second, third].map((value, index) => { + const cursor = [firstReceipt, secondReceipt, thirdReceipt][index]!.cursor; + return `${JSON.stringify({ ...value, cursor })}\n`; + }).join(""); + expect(await all(store, handle)).toBe(whole); +}); + +it("reads a bounded tail page and continues with the exact decimal cursor", async () => { + const basePath = await temporary(); + const store = createSegmentedRunLogStore({ basePath, segmentBytes: 128 }); + const handle = await store.begin(binding); + for (let i = 0; i < 12; i++) await store.append(handle, event(`${i}: ${"x".repeat(30)}`)); + const total = BigInt((await fs.readFile(join(basePath, handle.logRef, "head.json"), "utf8")).match(/"bytes":"(\d+)"/)![1]!); + let page = await store.read(handle, { cursor: "tail", limitBytes: 80 }); + expect(Buffer.byteLength(page.content)).toBeLessThanOrEqual(80); + expect(page.content).toContain('"chunk":"11: '); + while (page.hasMore) page = await store.read(handle, { cursor: page.cursor, limitBytes: 80 }); + expect(BigInt(page.cursor!)).toBe(total); + const appended = event("arrived after tail page"); + await store.append(handle, appended); + const resumed = await store.read(handle, { cursor: page.cursor, limitBytes: 256 }); + expect(resumed.content).toBe(`${JSON.stringify(appended)}\n`); + expect(resumed.hasMore).toBe(false); +}); + +it("routes a synthetic committed segment and tail cursor above MAX_SAFE_INTEGER exactly", async () => { + const basePath = await temporary(); + const store = createSegmentedRunLogStore({ basePath, segmentBytes: 128 }); + const handle = await store.begin(binding); + const tail = Buffer.from(`${JSON.stringify(event("high-offset-" + "x".repeat(40)))}\n`); + const segmentBytes = 128n; + const index = (BigInt(Number.MAX_SAFE_INTEGER) / segmentBytes) + 2n; + const total = index * segmentBytes + BigInt(tail.length); + const hex = index.toString(16).padStart(4, "0"); + const relative = `segments/${hex.slice(0, -2)}/${hex.slice(-2)}.ndjson`; + const segmentPath = join(basePath, handle.logRef, relative); + await fs.mkdir(join(segmentPath, ".."), { recursive: true, mode: 0o700 }); + await fs.writeFile(segmentPath, tail, { mode: 0o600 }); + await fs.chmod(segmentPath, 0o600); + await fs.writeFile(join(basePath, handle.logRef, "head.json"), JSON.stringify({ + schema: "paperclip.run-log.segments.v2", logRef: handle.logRef, segmentBytes: 128, + bytes: String(total), lastRecordCursor: String(index * segmentBytes), revision: "2", tailSha256: createHash("sha256").update(tail).digest("hex"), finalized: false, + }), { mode: 0o600 }); + await fs.chmod(join(basePath, handle.logRef, "head.json"), 0o600); + + const reader = createSegmentedRunLogStore({ basePath }); + let page = await reader.read(handle, { cursor: "tail", limitBytes: 48 }); + expect(BigInt(page.cursor!)).toBeGreaterThan(BigInt(Number.MAX_SAFE_INTEGER)); + let content = page.content; + while (page.hasMore) { page = await reader.read(handle, { cursor: page.cursor, limitBytes: 48 }); content += page.content; } + expect(page.cursor).toBe(String(total)); + expect(content).toBe(tail.toString()); + expect(page.hasMore).toBe(false); +}); + +it.each([false, true])("pages a final record larger than the tail window after reopen (retained head: %s)", async retained => { + const basePath = await temporary(); + const store = createSegmentedRunLogStore({ basePath, segmentBytes: 64 * 1024 }); + const handle = await store.begin(binding); + await store.append(handle, event("earlier output")); + const final = event("🙂大".repeat(100_000)); + const receipt = await store.appendPositioned!(handle, final); + await store.finalize(handle); + const path = join(basePath, handle.logRef, "head.json"); + if (retained) { + const head = JSON.parse(await fs.readFile(path, "utf8")); + delete head.lastRecordCursor; + await fs.writeFile(path, JSON.stringify(head)); + } + const reader = createSegmentedRunLogStore({ basePath }); + let page = await reader.read(handle, { cursor: "tail", limitBytes: 256_000 }); + expect(page.hasMore).toBe(true); + let encoded = page.content, requests = 1; + while (page.hasMore) { + expect(Buffer.byteLength(page.content)).toBeLessThanOrEqual(256_000); + const previous = page.cursor; + page = await reader.read(handle, { cursor: previous, limitBytes: 256_000 }); + expect(BigInt(page.cursor!)).toBeGreaterThan(BigInt(previous!)); + encoded += page.content; + expect(++requests).toBeLessThan(10); + } + expect(JSON.parse(encoded)).toEqual({ ...final, cursor: receipt.cursor }); + expect(page.cursor).toBe(receipt.nextCursor); +}); + +it.each(["local", "s3"] as const)("rejects corrupt historical segment bytes before returning a %s page", async backend => { + const basePath = await temporary(), { provider, objects } = storage(); + const options = { basePath, segmentBytes: 256, ...(backend === "s3" ? { s3: { provider } } : {}) }; + const writer = createSegmentedRunLogStore(options), handle = await writer.begin(binding); + await writer.append(handle, event("x".repeat(800))); + await writer.flushInflightMirrors!(); + if (backend === "local") { + const path = join(basePath, handle.logRef, "segments/00/00.ndjson"); + const bytes = await fs.readFile(path); bytes[200] ^= 1; await fs.writeFile(path, bytes); + } else { + const ref = JSON.parse(objects.get(`${handle.logRef}/segments/00/00.ndjson.json`)!.toString()); + const key = `${handle.logRef}/${ref.object}`; + const bytes = Buffer.from(objects.get(key)!); bytes[200] ^= 1; objects.set(key, bytes); + await fs.rm(basePath, { recursive: true }); + } + const reader = createSegmentedRunLogStore(options); + // The damaged byte is outside this small requested page. The fixed-size + // segment must still verify before any of its content is displayed. + await expect(reader.read(handle, { limitBytes: 64 })).rejects.toThrow("segment_digest_mismatch"); +}); + +it("rejects a changed remote segment binding and verifies each cached body once", async () => { + const basePath = await temporary(), { provider, objects } = storage(); + const writer = createSegmentedRunLogStore({ basePath, segmentBytes: 1024, s3: { provider } }); + const handle = await writer.begin(binding); + await writer.append(handle, event("x".repeat(1600))); await writer.flushInflightMirrors!(); + await fs.rm(basePath, { recursive: true }); + const reads = vi.spyOn(provider, "getObject"); + const reader = createSegmentedRunLogStore({ basePath, s3: { provider } }); + await reader.read(handle, { cursor: "0", limitBytes: 64 }); + await reader.read(handle, { cursor: "64", limitBytes: 64 }); + expect(reads.mock.calls.filter(([input]) => /\/00\.ndjson\.[a-f0-9]{64}$/.test(input.objectKey))).toHaveLength(1); + const key = `${handle.logRef}/segments/00/00.ndjson.json`; + const reference = JSON.parse(objects.get(key)!.toString()); reference.logRef = "another/agent/run.segments"; + objects.set(key, Buffer.from(JSON.stringify(reference))); + await expect(createSegmentedRunLogStore({ basePath, s3: { provider } }).read(handle)).rejects.toThrow("segment_reference_invalid"); +}); + +it("reopen, append and finalize do not require any completed historical segments", async () => { + const basePath = await temporary(); + let store = createSegmentedRunLogStore({ basePath, segmentBytes: 128 }); + const handle = await store.begin(binding); + await store.append(handle, event("a".repeat(4096))); + await fs.rm(join(basePath, handle.logRef, "segments/00/00.ndjson")); + store = createSegmentedRunLogStore({ basePath }); + const old = JSON.parse(await fs.readFile(join(basePath, handle.logRef, "head.json"), "utf8")); + await store.begin(binding); + await store.append(handle, event("next")); + expect((await store.read(handle, { cursor: old.bytes })).content).toContain("next"); + await store.finalize(handle); + await expect(store.read(handle)).rejects.toThrow(); +}); + +it.each(["segment", "head"] as const)("recovers interruption at %s without publishing uncommitted bytes", async boundary => { + const basePath = await temporary(); + let armed = false; + let store = createSegmentedRunLogStore({ basePath, segmentBytes: 128, onBoundary: async point => { if (armed && point === boundary) { armed = false; throw new Error("interrupted"); } } }); + const handle = await store.begin(binding); + const first = event("first"); + await store.append(handle, first); + const second = event("b".repeat(300)); + armed = true; + await expect(store.append(handle, second)).rejects.toThrow("interrupted"); + await expect(store.append(handle, event("unsafe retry"))).rejects.toThrow("writer_fenced"); + store = createSegmentedRunLogStore({ basePath }); + await store.begin(binding); + const third = event("third"); + await store.append(handle, third); + expect(await all(store, handle)).toBe([first, ...(boundary === "head" ? [second] : []), third].map(value => `${JSON.stringify(value)}\n`).join("")); +}); + +it("uploads closed segments once, mirrors only the current tail, and restores after a pod loss", async () => { + const basePath = await temporary(); + const { provider, writes } = storage(); + let store = createSegmentedRunLogStore({ basePath, segmentBytes: 256, s3: { provider } }); + const handle = await store.begin(binding); + const first = event("f".repeat(3000)); + await store.append(handle, first); + await store.flushInflightMirrors!(); + const count = writes.length; + await store.append(handle, event("tail")); + await store.flushInflightMirrors!(); + expect(writes.slice(count).every(write => write.bytes < 1024)).toBe(true); + const completed = writes.filter(write => write.key.includes("/segments/")); + expect(new Set(completed.map(write => write.key)).size).toBe(completed.length); + const before = await all(store, handle); + await fs.rm(basePath, { recursive: true }); + store = createSegmentedRunLogStore({ basePath, s3: { provider } }); + expect(await all(store, handle)).toBe(before); + await store.begin(binding); // Restore only the head and current tail. + const next = event("continued after pod loss"); + await store.append(handle, next); + expect(await all(store, handle)).toBe(before + JSON.stringify(next) + "\n"); + const finalized = await store.finalize(handle); + await fs.rm(basePath, { recursive: true }); + expect(await all(store, handle)).toBe(before + JSON.stringify(next) + "\n"); + expect(finalized.bytes).toBe(Buffer.byteLength(before + JSON.stringify(next) + "\n")); +}); + +it.each(["remote-tail", "remote-head"] as const)("an interruption at %s exposes only a complete remote generation", async boundary => { + const basePath = await temporary(); + const { provider } = storage(); + let armed = false; + const store = createSegmentedRunLogStore({ basePath, segmentBytes: 256, s3: { provider }, onBoundary: async point => { if (armed && point === boundary) { armed = false; throw new Error("interrupted"); } } }); + const handle = await store.begin(binding); + const first = event("first"); await store.append(handle, first); await store.flushInflightMirrors!(); + const second = event("second"); await store.append(handle, second); + armed = true; + await expect(store.flushInflightMirrors!()).rejects.toThrow("interrupted"); + await fs.rm(basePath, { recursive: true }); + const reader = createSegmentedRunLogStore({ basePath, s3: { provider } }); + expect(await all(reader, handle)).toBe([first, ...(boundary === "remote-head" ? [second] : [])].map(value => JSON.stringify(value) + "\n").join("")); +}); + +it("retains format across rollout changes and never truncates an existing log on begin", async () => { + const basePath = await temporary(); + let store = createDurableRunLogStore({ basePath, segmented: true }); + const handle = await store.begin(binding); + await store.append(handle, event("retained")); + store = createDurableRunLogStore({ basePath, segmented: false }); + expect(await store.begin(binding)).toEqual(handle); + expect((await store.read(handle)).content).toContain("retained"); + const old = await store.begin({ ...binding, runId: "legacy" }); + await store.append(old, event("legacy retained")); + store = createDurableRunLogStore({ basePath, segmented: true }); + expect(await store.begin({ ...binding, runId: "legacy" })).toEqual(old); + expect((await store.read(old)).content).toContain("legacy retained"); +}); + +it("creates a private segmented run beneath existing legacy company/agent directories", async () => { + const basePath = await temporary(); + const legacy = createDurableRunLogStore({ basePath }); + const old = await legacy.begin(binding); + await legacy.append(old, event("old")); + const indexed = createDurableRunLogStore({ basePath, segmented: true }); + const current = await indexed.begin({ ...binding, runId: "next" }); + await indexed.append(current, event("new")); + expect((await indexed.read(current)).content).toContain("new"); + expect((await indexed.read(old)).content).toContain("old"); +}); + +it("serializes concurrent first appends after reopen without losing bytes", async () => { + const basePath = await temporary(); + let store = createSegmentedRunLogStore({ basePath, segmentBytes: 128 }); + const handle = await store.begin(binding); + store = createSegmentedRunLogStore({ basePath }); + await Promise.all(Array.from({ length: 8 }, (_, i) => store.append(handle, event(String(i))))); + expect((await all(store, handle)).trim().split("\n").map(line => JSON.parse(line).chunk)).toEqual(["0", "1", "2", "3", "4", "5", "6", "7"]); +}); + +it("fences an upload failure and recovers only when begin reopens its durable head", async () => { + const basePath = await temporary(); + const { provider } = storage(); + const store = createSegmentedRunLogStore({ basePath, segmentBytes: 128, s3: { provider } }); + const handle = await store.begin(binding); + const first = event("first"); + await store.append(handle, first); + const put = vi.spyOn(provider, "putObjectConditional").mockRejectedValueOnce(new Error("storage offline")); + await expect(store.append(handle, event("x".repeat(200)))).rejects.toThrow("storage offline"); + await expect(store.append(handle, event("unsafe"))).rejects.toThrow("writer_fenced"); + put.mockRestore(); + await store.begin(binding); + const next = event("recovered"); + await store.append(handle, next); + expect(await all(store, handle)).toBe([first, next].map(value => JSON.stringify(value) + "\n").join("")); +}); + +it("rejects corrupt tails and symlinked segment directories", async () => { + const basePath = await temporary(); + let store = createSegmentedRunLogStore({ basePath, segmentBytes: 128 }); + const handle = await store.begin(binding); + await store.append(handle, event("first")); + const file = join(basePath, handle.logRef, "segments/00/00.ndjson"); + const original = await fs.readFile(file); + await fs.writeFile(file, Buffer.alloc(original.length, 0x78)); + store = createSegmentedRunLogStore({ basePath }); + await expect(store.begin(binding)).rejects.toThrow("digest_mismatch"); + await fs.writeFile(file, original); + const directory = join(basePath, handle.logRef, "segments/00"); + await fs.rename(directory, `${directory}-old`); + await fs.symlink(`${directory}-old`, directory); + await expect(store.read(handle)).rejects.toThrow("directory_unsafe"); +}); diff --git a/server/src/services/segmented-run-log-store.ts b/server/src/services/segmented-run-log-store.ts new file mode 100644 index 00000000000..37e067345f5 --- /dev/null +++ b/server/src/services/segmented-run-log-store.ts @@ -0,0 +1,603 @@ +import { constants, createReadStream, promises as fs } from "node:fs"; +import { createHash, randomUUID, type Hash } from "node:crypto"; +import { dirname, join, resolve, sep } from "node:path"; +import type { Readable } from "node:stream"; +import { badRequest, notFound, tooManyRequests } from "../errors.js"; +import type { StorageProvider } from "../storage/types.js"; +import type { RunLogHandle, RunLogReadOptions, RunLogReadResult, RunLogStore, RunLogEvent, RunLogAppendReceipt } from "./run-log-store.js"; + +const SCHEMA = "paperclip.run-log.segments.v2"; +const LEGACY_SCHEMA = "paperclip.run-log.segments.v1"; +const MAX_APPEND = 32 * 1024 * 1024; +const MAX_READ = 1024 * 1024; +const EMPTY_HASH = createHash("sha256").digest("hex"); + +interface Head { + schema: typeof SCHEMA | typeof LEGACY_SCHEMA; + logRef: string; + segmentBytes: number; + bytes: string; + revision: string; + tailSha256: string; + finalized: boolean; + writerId?: string; + tailObject?: string; + retiredTail?: string; + lastRecordCursor?: string; +} +interface Writer { + head: Head; + tailHash: Hash; + chain: Promise; + queued: number; + failed: boolean; + timer?: NodeJS.Timeout; + dirty: boolean; + remoteEtag: string | null; +} +export interface SegmentedRunLogOptions { + basePath: string; + /** Smaller sizes are useful for deterministic rotation/fault tests. */ + segmentBytes?: number; + s3?: { provider: StorageProvider; keyPrefix?: string; inflightMirrorMs?: number }; + onBoundary?: (boundary: "segment" | "head" | "remote-tail" | "remote-head") => Promise; +} + +function integer(value: string): bigint { + if (!/^(0|[1-9][0-9]*)$/.test(value)) throw badRequest("Invalid run log cursor"); + return BigInt(value); +} +function safeRef(value: string): string { + if (!/^[a-zA-Z0-9_-]+\/[a-zA-Z0-9_-]+\/[a-zA-Z0-9_-]+\.segments$/.test(value)) throw badRequest("Invalid run log reference"); + return value; +} +function segmentPath(index: bigint): string { + // Direct arithmetic routing: no growing manifest or directory enumeration. + const value = index.toString(16).padStart(4, "0"); + return `segments/${value.slice(0, -2)}/${value.slice(-2)}.ndjson`; +} +function parseHead(bytes: Buffer, logRef: string): Head { + if (bytes.length > 4096) throw new Error("run_log_head_invalid"); + const head = JSON.parse(bytes.toString()) as Head; + if ((head.schema !== SCHEMA && head.schema !== LEGACY_SCHEMA) || head.logRef !== logRef || typeof head.bytes !== "string" || typeof head.revision !== "string" || + !Number.isSafeInteger(head.segmentBytes) || head.segmentBytes < 64 || head.segmentBytes > MAX_APPEND || + !/^[0-9a-f]{64}$/.test(head.tailSha256) || typeof head.finalized !== "boolean") throw new Error("run_log_head_invalid"); + if ((head.writerId !== undefined && !/^[0-9a-f-]{36}$/.test(head.writerId)) || + (head.tailObject !== undefined && !/^tails\/[0-9a-f-]{36}\/(0|[1-9][0-9]*)\.ndjson$/.test(head.tailObject))) throw new Error("run_log_head_invalid"); + if (head.retiredTail !== undefined && (!/^tails\/(?:[0-9a-f-]{36}\/)?(0|[1-9][0-9]*)\.ndjson$/.test(head.retiredTail) || head.retiredTail === head.tailObject)) throw new Error("run_log_head_invalid"); + integer(head.bytes); integer(head.revision); + if (head.lastRecordCursor !== undefined && (typeof head.lastRecordCursor !== "string" || + integer(head.lastRecordCursor) >= integer(head.bytes) || integer(head.bytes) - integer(head.lastRecordCursor) > BigInt(MAX_APPEND))) { + throw new Error("run_log_head_invalid"); + } + return head; +} +async function syncDirectory(path: string) { + if (process.platform === "win32") return; + const fd = await fs.open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + try { await fd.sync(); } finally { await fd.close(); } +} +async function privateRead(path: string, maximum: number): Promise { + const fd = await fs.open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + const stat = await fd.stat(); + if (!stat.isFile() || stat.size > maximum || (process.platform !== "win32" && ((stat.mode & 0o077) !== 0 || stat.uid !== process.getuid?.()))) throw new Error("run_log_file_unsafe"); + const bytes = Buffer.alloc(stat.size); + let offset = 0; + while (offset < bytes.length) { const read = await fd.read(bytes, offset, bytes.length - offset, offset); if (!read.bytesRead) throw new Error("run_log_file_changed"); offset += read.bytesRead; } + const after = await fd.stat(); + if (after.size !== stat.size || after.mtimeMs !== stat.mtimeMs || after.ctimeMs !== stat.ctimeMs) throw new Error("run_log_file_changed"); + return bytes; + } finally { await fd.close(); } +} +async function durableWrite(path: string, bytes: Buffer, createOnly = false) { + const temp = `${path}.${randomUUID()}.tmp`; + const fd = await fs.open(temp, "wx", 0o600); + try { await fd.writeFile(bytes); await fd.sync(); } + finally { await fd.close(); } + try { + if (createOnly) await fs.link(temp, path).catch(error => { if (error.code !== "EEXIST") throw error; }); + else await fs.rename(temp, path); + await syncDirectory(dirname(path)); + } + finally { await fs.rm(temp, { force: true }); } +} +async function collect(stream: Readable, maximum: number): Promise { + const chunks: Buffer[] = []; let length = 0; + for await (const value of stream) { + const chunk = Buffer.isBuffer(value) ? value : Buffer.from(value); + length += chunk.length; + if (length > maximum) { stream.destroy(); throw new Error("run_log_read_oversize"); } + chunks.push(chunk); + } + return Buffer.concat(chunks); +} + +/** Fixed-size segments with a bounded current tail. The caller's run lease + * owns the sole writer. A failed/indeterminate write fences that writer until + * it is reopened under a freshly verified lease. Completed segments are never + * read or rewritten by append, reopen, mirror, or finalize. */ +export function createSegmentedRunLogStore(options: SegmentedRunLogOptions): RunLogStore { + const base = resolve(options.basePath); + const segmentBytes = options.segmentBytes ?? MAX_APPEND; + if (!Number.isSafeInteger(segmentBytes) || segmentBytes < 64 || segmentBytes > MAX_APPEND) throw new Error("run_log_segment_size_invalid"); + const writers = new Map(); + const opening = new Map>(); + // Verification is per fixed-size segment, never per complete log. Cache at + // most two verified segments and run at most two distinct reads at once. + // A task page opens several prior run logs concurrently. Queue their small + // descriptors instead of treating its third read as an internal error. + const verified = new Map(); + const verifying = new Map>(); + let activeReads = 0; + const waitingReads: Array<() => void> = []; + async function acquireRead(): Promise<() => void> { + if (activeReads < 2) activeReads++; + else { + if (waitingReads.length >= 64) throw tooManyRequests("Run log reads are busy. Try again shortly."); + await new Promise((resolve, reject) => { + const ready = () => { clearTimeout(timer); resolve(); }; + const timer = setTimeout(() => { + const index = waitingReads.indexOf(ready); + if (index !== -1) waitingReads.splice(index, 1); + reject(tooManyRequests("Run log reads are busy. Try again shortly.")); + }, 30_000); + timer.unref(); + waitingReads.push(ready); + }); + } + return () => { + const next = waitingReads.shift(); + if (next) next(); // Transfer this credit; a newcomer cannot steal it. + else activeReads--; + }; + } + const prefix = options.s3?.keyPrefix?.replace(/^\/+|\/+$/g, ""); + const key = (ref: string, relative: string) => [prefix, safeRef(ref), relative].filter(Boolean).join("/"); + const local = (ref: string, relative: string) => join(base, safeRef(ref), relative); + + async function ensureDirectory(path: string) { + await fs.mkdir(base, { recursive: true, mode: 0o700 }); + const baseStat = await fs.lstat(base); + if (!baseStat.isDirectory() || baseStat.isSymbolicLink()) throw new Error("run_log_directory_unsafe"); + const relative = path.slice(base.length + 1); + if (!path.startsWith(base + sep) || relative.split(sep).some(part => part === "..")) throw new Error("run_log_directory_unsafe"); + let current = base; + let depth = 0; + for (const part of relative.split(sep)) { + depth++; + current = join(current, part); + let created = true; + await fs.mkdir(current, { mode: 0o700 }).catch(error => { if (error.code !== "EEXIST") throw error; created = false; }); + const stat = await fs.lstat(current); + // Company/agent directories may predate this format. Only the private + // run directory and its children require 0700; every ancestor must still + // be a real directory owned by this service. + if (!stat.isDirectory() || stat.isSymbolicLink() || (process.platform !== "win32" && ((depth >= 3 && (stat.mode & 0o077) !== 0) || stat.uid !== process.getuid?.()))) throw new Error("run_log_directory_unsafe"); + if (created) await syncDirectory(dirname(current)); + } + } + async function assertDirectory(path: string) { + if (!path.startsWith(base + sep)) throw new Error("run_log_directory_unsafe"); + let current = base; + const root = await fs.lstat(current); + if (!root.isDirectory() || root.isSymbolicLink()) throw new Error("run_log_directory_unsafe"); + let depth = 0; + for (const part of path.slice(base.length + 1).split(sep)) { + depth++; + current = join(current, part); + const stat = await fs.lstat(current); + if (!stat.isDirectory() || stat.isSymbolicLink() || (process.platform !== "win32" && ((depth >= 3 && (stat.mode & 0o077) !== 0) || stat.uid !== process.getuid?.()))) throw new Error("run_log_directory_unsafe"); + } + } + async function put(ref: string, relative: string, body: Buffer | ReturnType, length: number, sha256: string) { + if (!options.s3) return; + try { + await options.s3.provider.putObjectConditional!({ objectKey: key(ref, relative), body, contentLength: length, contentType: relative.endsWith(".json") ? "application/json" : "application/x-ndjson", sha256 }, null); + } catch (error) { + if (!Buffer.isBuffer(body)) body.destroy(); + const code = error as { name?: string; $metadata?: { httpStatusCode?: number } }; + if (code.name !== "PreconditionFailed" && code.$metadata?.httpStatusCode !== 412) throw error; + // A replay may reuse identical immutable bytes. A competing writer's + // different segment can never be overwritten, even before its head CAS. + const prior = await options.s3.provider.getObject({ objectKey: key(ref, relative) }); + const bytes = await collect(prior.stream, length); + if (bytes.length !== length || createHash("sha256").update(bytes).digest("hex") !== sha256) throw new Error("run_log_immutable_object_conflict"); + } + } + const remoteTail = (head: Head) => head.tailObject ?? `tails/${head.revision}.ndjson`; + async function remoteHead(ref: string): Promise<{ head: Head; etag: string } | null> { + const provider = options.s3!.provider, objectKey = key(ref, "head.json"); + if (!(await provider.headObject({ objectKey })).exists) return null; + const result = await provider.getObject({ objectKey }); + if (!result.etag) { result.stream.destroy(); throw new Error("run_log_remote_head_etag_missing"); } + return { head: parseHead(await collect(result.stream, 4096), ref), etag: result.etag }; + } + async function readHead(ref: string): Promise<{ head: Head; remote: boolean }> { + try { + await assertDirectory(local(ref, "")); + return { head: parseHead(await privateRead(local(ref, "head.json"), 4096), ref), remote: false }; + } + catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } + if (!options.s3) throw notFound("Run log not found"); + const result = await options.s3.provider.getObject({ objectKey: key(ref, "head.json") }); + return { head: parseHead(await collect(result.stream, 4096), ref), remote: true }; + } + async function segmentReference(ref: string, relative: string, index: bigint, bytes: number) { + let encoded: Buffer; + try { + await assertDirectory(dirname(local(ref, relative))); + encoded = await privateRead(local(ref, `${relative}.json`), 4096); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT" || !options.s3) throw error; + encoded = await collect((await options.s3.provider.getObject({ objectKey: key(ref, `${relative}.json`) })).stream, 4096); + } + const value = JSON.parse(encoded.toString()) as Record; + if (!["paperclip.run-log.segment.v1", "paperclip.run-log.segment.v2"].includes(String(value.schema)) || value.logRef !== ref || value.segment !== String(index) || value.bytes !== bytes || + typeof value.sha256 !== "string" || !/^[a-f0-9]{64}$/.test(value.sha256)) throw new Error("run_log_segment_reference_invalid"); + if (value.object !== undefined && (value.object !== `${relative}.${value.sha256}` || typeof value.writerId !== "string" || !/^[0-9a-f-]{36}$/.test(value.writerId))) throw new Error("run_log_segment_reference_invalid"); + if (value.schema === "paperclip.run-log.segment.v2" && value.object === undefined) throw new Error("run_log_segment_reference_invalid"); + return { sha256: value.sha256, object: typeof value.object === "string" ? value.object : relative }; + } + async function verifiedSegment(head: Head, index: bigint, remote: boolean, refreshTail = true, admitted = false): Promise { + const ref = head.logRef, relative = segmentPath(index); + const tail = index === integer(head.bytes) / BigInt(head.segmentBytes); + const length = tail ? Number(integer(head.bytes) % BigInt(head.segmentBytes)) : head.segmentBytes; + const reference = tail ? { sha256: head.tailSha256, object: remoteTail(head) } : await segmentReference(ref, relative, index, length); + const sha256 = reference.sha256; + const identity = `${ref}:${index}:${length}:${sha256}`; + const cached = verified.get(identity); + if (cached) { verified.delete(identity); verified.set(identity, cached); return cached; } + // A retired-tail retry already owns a read credit. It must not join a + // queued request for the newer head which is waiting for that same credit. + const existing = admitted ? undefined : verifying.get(identity); + if (existing) return existing; + const pending = (async () => { + const release = admitted ? () => {} : await acquireRead(); + try { + // Another credited read may have populated this while we were queued. + const cached = verified.get(identity); + if (cached) return cached; + let bytes: Buffer | undefined; + if (!remote) { + let fd; + try { + await assertDirectory(dirname(local(ref, relative))); + fd = await fs.open(local(ref, relative), constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + const stat = await fd.stat(); + if (!stat.isFile() || stat.size < length || stat.size > head.segmentBytes || (process.platform !== "win32" && ((stat.mode & 0o077) !== 0 || stat.uid !== process.getuid?.()))) throw new Error("run_log_segment_unsafe"); + bytes = Buffer.alloc(length); + let offset = 0; + while (offset < length) { const result = await fd.read(bytes, offset, length - offset, offset); if (!result.bytesRead) throw new Error("run_log_segment_missing"); offset += result.bytesRead; } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT" || !options.s3 || tail) throw error; + } finally { await fd?.close(); } + } + if (!bytes) { + try { + const result = await options.s3!.provider.getObject({ objectKey: key(ref, reference.object) }); + bytes = await collect(result.stream, length); + } catch (error) { + if (!tail || !refreshTail || (error as { status?: number }).status !== 404) throw error; + const newer = await remoteHead(ref); + if (!newer || newer.head.segmentBytes !== head.segmentBytes || integer(newer.head.bytes) < integer(head.bytes) || remoteTail(newer.head) === reference.object) throw error; + // Mirroring can retire a tail after this reader read its old head. + // Fetch at most one newer segment and verify its exact old prefix + // against the original head before returning this snapshot's page. + if (newer.head.bytes === head.bytes && newer.head.tailSha256 === head.tailSha256) { + // An ownership claim can change only the object's location. Its + // cache identity is identical: joining our own in-flight promise + // would deadlock. Read that new object directly instead. + bytes = await collect((await options.s3!.provider.getObject({ objectKey: key(ref, remoteTail(newer.head)) })).stream, length); + } else { + const current = await verifiedSegment(newer.head, index, true, false, true); + bytes = Buffer.from(current.subarray(0, length)); + } + } + } + if (bytes.length !== length || createHash("sha256").update(bytes).digest("hex") !== sha256) throw new Error("run_log_segment_digest_mismatch"); + verified.set(identity, bytes); + while (verified.size > 2) verified.delete(verified.keys().next().value!); + return bytes; + } finally { release(); } + })().finally(() => { if (!admitted) verifying.delete(identity); }); + if (!admitted) verifying.set(identity, pending); + return pending; + } + async function restoreWriter(ref: string): Promise { + if (options.s3 && !options.s3.provider.putObjectConditional) throw new Error("run_log_conditional_storage_required"); + const existing = writers.get(ref); + if (existing) return existing; + const { head, remote } = await readHead(ref); + const published = options.s3 ? await remoteHead(ref) : null; + if (published) { + const prior = published.head; + const sameBytes = prior.bytes === head.bytes && prior.revision === head.revision && prior.tailSha256 === head.tailSha256; + if (prior.segmentBytes !== head.segmentBytes || (prior.finalized && !head.finalized) || + integer(prior.bytes) > integer(head.bytes) || integer(prior.revision) > integer(head.revision) || + (!sameBytes && prior.writerId !== head.writerId)) throw new Error("run_log_remote_owner_changed"); + } + const tailHash = createHash("sha256"); + const tailBytes = Number(integer(head.bytes) % BigInt(head.segmentBytes)); + if (remote) { + if (tailBytes) { + const bytes = await verifiedSegment(head, integer(head.bytes) / BigInt(head.segmentBytes), true); + const path = local(ref, segmentPath(integer(head.bytes) / BigInt(head.segmentBytes))); + await ensureDirectory(dirname(path)); + await durableWrite(path, bytes); + } + await ensureDirectory(local(ref, "segments")); + await durableWrite(local(ref, "head.json"), Buffer.from(JSON.stringify(head))); + } + if (tailBytes) { + // Only the current tail is needed after a controller restart. A partial + // uncommitted append is discarded using the durable head's exact length. + const path = local(ref, segmentPath(integer(head.bytes) / BigInt(head.segmentBytes))); + await assertDirectory(dirname(path)); + const fd = await fs.open(path, constants.O_RDWR | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + const stat = await fd.stat(); + if (!stat.isFile() || stat.size < tailBytes || stat.size > head.segmentBytes || + (process.platform !== "win32" && ((stat.mode & 0o077) !== 0 || stat.uid !== process.getuid?.()))) throw new Error("run_log_tail_invalid"); + const bytes = Buffer.alloc(tailBytes); + let offset = 0; + while (offset < bytes.length) { const read = await fd.read(bytes, offset, bytes.length - offset, offset); if (!read.bytesRead) throw new Error("run_log_tail_missing"); offset += read.bytesRead; } + tailHash.update(bytes); + if (tailHash.copy().digest("hex") !== head.tailSha256) throw new Error("run_log_tail_digest_mismatch"); + if (stat.size !== tailBytes) { await fd.truncate(tailBytes); await fd.sync(); } + } finally { await fd.close(); } + } else if (head.tailSha256 !== EMPTY_HASH) throw new Error("run_log_tail_digest_mismatch"); + const writer: Writer = { head: { ...head, schema: SCHEMA, ...(options.s3 ? { writerId: randomUUID() } : {}) }, tailHash, chain: Promise.resolve(), queued: 0, failed: false, dirty: !head.finalized, remoteEtag: published?.etag ?? null }; + if (options.s3 && !head.finalized) { + // Claim this writer at the remote authority before begin returns. A late + // upload by an older owner holds a different ETag and cannot roll it back. + await mirror(writer); + } + writers.set(ref, writer); + return writer; + } + async function openWriter(ref: string): Promise { + const existing = writers.get(ref); + if (existing) return existing; + let pending = opening.get(ref); + if (!pending) { + pending = restoreWriter(ref).finally(() => { opening.delete(ref); }); + opening.set(ref, pending); + } + return pending; + } + async function serialize(ref: string, action: (writer: Writer) => Promise): Promise { + const writer = await openWriter(ref); + if (writer.failed) throw new Error("run_log_writer_fenced"); + if (writer.queued >= 8) throw new Error("run_log_storage_pressure"); + writer.queued++; + const pending = writer.chain.then(async () => { + if (writer.failed) throw new Error("run_log_writer_fenced"); + try { return await action(writer); } + catch (error) { writer.failed = true; throw error; } + finally { writer.queued--; } + }); + writer.chain = pending.catch(() => undefined); + return pending; + } + async function mirror(writer: Writer) { + if (!options.s3 || !writer.dirty) return; + const head = { ...writer.head }; + if (head.retiredTail) { + await options.s3.provider.deleteObject({ objectKey: key(head.logRef, head.retiredTail) }); + delete head.retiredTail; + } + const tailBytes = Number(integer(head.bytes) % BigInt(head.segmentBytes)); + const previousTail = writer.head.tailObject; + if (tailBytes) { + const path = local(head.logRef, segmentPath(integer(head.bytes) / BigInt(head.segmentBytes))); + // Each mirrored tail has its own identity; a late old upload cannot + // change bytes referenced by a newer published head. + head.tailObject = `tails/${head.writerId}/${head.revision}.ndjson`; + await put(head.logRef, head.tailObject, createReadStream(path, { start: 0, end: tailBytes - 1 }), tailBytes, head.tailSha256); + await options.onBoundary?.("remote-tail"); + } else delete head.tailObject; + if (previousTail && previousTail !== head.tailObject) head.retiredTail = previousTail; + const bytes = Buffer.from(JSON.stringify(head)); + const published = await options.s3.provider.putObjectConditional!({ objectKey: key(head.logRef, "head.json"), body: bytes, + contentLength: bytes.length, contentType: "application/json", sha256: createHash("sha256").update(bytes).digest("hex") }, writer.remoteEtag); + writer.remoteEtag = published.etag; + await options.onBoundary?.("remote-head"); + await durableWrite(local(head.logRef, "head.json"), bytes); + writer.head = head; + // One durable pending deletion, never a lifetime-sized tail catalog. + // Missing-object readers retry against a newer head and verify the old + // prefix. Unknown prepublication failures still need orphan maintenance. + if (head.retiredTail) await options.s3.provider.deleteObject({ objectKey: key(head.logRef, head.retiredTail) }); + writer.dirty = false; + } + async function publishSegment(writer: Writer, index: bigint, relative: string, path: string, hash: string, reference: Buffer) { + if (!options.s3) return; + const provider = options.s3.provider, ref = writer.head.logRef; + // Content-addressed bodies are immutable. An interrupted append may have + // published a different, unreferenced body for this ordinal; it cannot + // corrupt the next owner's committed segment by overwriting that object. + await put(ref, `${relative}.${hash}`, createReadStream(path), writer.head.segmentBytes, hash); + const objectKey = key(ref, `${relative}.json`); + const previous = await provider.headObject({ objectKey }); + if (previous.exists && !previous.etag) throw new Error("run_log_remote_segment_etag_missing"); + // Capture the reference ETag BEFORE checking ownership. A later owner + // publishes its own UUID in the reference, preventing an ETag ABA even + // when it uses identical content. The old writer cannot replace it. + const current = await remoteHead(ref); + if (!current || current.etag !== writer.remoteEtag || current.head.writerId !== writer.head.writerId) throw new Error("run_log_remote_owner_changed"); + if (index < integer(current.head.bytes) / BigInt(current.head.segmentBytes)) throw new Error("run_log_segment_already_published"); + await provider.putObjectConditional!({ objectKey, body: reference, contentLength: reference.length, + contentType: "application/json", sha256: createHash("sha256").update(reference).digest("hex") }, previous.etag ?? null); + } + function schedule(writer: Writer) { + if (!options.s3 || !options.s3.inflightMirrorMs || writer.timer || writer.head.finalized) return; + writer.timer = setTimeout(() => { + writer.timer = undefined; + void serialize(writer.head.logRef, mirror).catch(() => { /* The fenced writer reports the resource failure on its next operation. */ }); + }, options.s3.inflightMirrorMs); + writer.timer.unref(); + } + async function appendRecord(handle: RunLogHandle, event: RunLogEvent, positioned: boolean): Promise { + // One append is bounded live work; its size never depends on history. + if (Buffer.byteLength(event.chunk) > MAX_APPEND / 2) throw new Error("run_log_append_too_large"); + return serialize(handle.logRef, async writer => { + if (writer.head.finalized) throw new Error("run_log_finalized"); + let total = integer(writer.head.bytes); + const cursor = String(total); + const bytes = Buffer.from(`${JSON.stringify(positioned ? { ...event, cursor } : event)}\n`); + if (bytes.length > MAX_APPEND) throw new Error("run_log_append_too_large"); + const size = BigInt(writer.head.segmentBytes); + let consumed = 0; + while (consumed < bytes.length) { + const index = total / size; + const offset = Number(total % size); + const count = Math.min(bytes.length - consumed, Number(size) - offset); + const relative = segmentPath(index); + const path = local(handle.logRef, relative); + await ensureDirectory(dirname(path)); + const fd = await fs.open(path, constants.O_RDWR | constants.O_CREAT | constants.O_NOFOLLOW | constants.O_NONBLOCK, 0o600); + try { + const stat = await fd.stat(); + if (!stat.isFile() || stat.size < offset || (process.platform !== "win32" && ((stat.mode & 0o077) !== 0 || stat.uid !== process.getuid?.()))) throw new Error("run_log_segment_unsafe"); + await fd.truncate(offset); + let written = 0; + while (written < count) { const result = await fd.write(bytes, consumed + written, count - written, offset + written); if (!result.bytesWritten) throw new Error("run_log_short_write"); written += result.bytesWritten; } + await fd.sync(); + } finally { await fd.close(); } + writer.tailHash.update(bytes.subarray(consumed, consumed + count)); + consumed += count; total += BigInt(count); + if (total % size === 0n) { + const hash = writer.tailHash.digest("hex"); + const reference = Buffer.from(JSON.stringify({ schema: options.s3 ? "paperclip.run-log.segment.v2" : "paperclip.run-log.segment.v1", logRef: handle.logRef, segment: String(index), bytes: Number(size), sha256: hash, + ...(options.s3 ? { object: `${relative}.${hash}`, writerId: writer.head.writerId } : {}) })); + await durableWrite(`${path}.json`, reference); + // Completed segments reach configured object storage before the + // local head can acknowledge them. Failed uploads hold admission. + await publishSegment(writer, index, relative, path, hash, reference); + writer.tailHash = createHash("sha256"); + await options.onBoundary?.("segment"); + } + } + const head = { ...writer.head, bytes: String(total), lastRecordCursor: cursor, + revision: String(integer(writer.head.revision) + 1n), tailSha256: writer.tailHash.copy().digest("hex") }; + if (options.s3) { + const current = await remoteHead(handle.logRef); + if (!current || current.etag !== writer.remoteEtag || current.head.writerId !== writer.head.writerId) throw new Error("run_log_remote_owner_changed"); + } + await durableWrite(local(handle.logRef, "head.json"), Buffer.from(JSON.stringify(head))); + await options.onBoundary?.("head"); + writer.head = head; writer.dirty = true; + schedule(writer); + return { bytes: bytes.length, cursor, nextCursor: String(total) }; + }); + } + return { + async begin(input) { + if (options.s3 && !options.s3.provider.putObjectConditional) throw new Error("run_log_conditional_storage_required"); + const ref = safeRef(`${input.companyId}/${input.agentId}/${input.runId}.segments`); + const failed = writers.get(ref); + if (failed?.failed) { + // begin is admitted by the caller's fresh run lease. Join all work + // owned by the failed writer before reconstructing its durable head. + if (failed.timer) clearTimeout(failed.timer); + await failed.chain; + if (writers.get(ref) === failed) writers.delete(ref); + } + await ensureDirectory(local(ref, "segments")); + try { await privateRead(local(ref, "head.json"), 4096); } + catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + if (!options.s3 || !(await options.s3.provider.headObject({ objectKey: key(ref, "head.json") })).exists) { + const head: Head = { schema: SCHEMA, logRef: ref, segmentBytes, bytes: "0", revision: "0", tailSha256: EMPTY_HASH, finalized: false }; + await durableWrite(local(ref, "head.json"), Buffer.from(JSON.stringify(head)), true); + } + } + schedule(await openWriter(ref)); + return { store: "local_segments", logRef: ref }; + }, + async append(handle, event) { return (await appendRecord(handle, event, false)).bytes; }, + appendPositioned: (handle, event) => appendRecord(handle, event, true), + async finalize(handle) { + return serialize(handle.logRef, async writer => { + if (writer.timer) { clearTimeout(writer.timer); writer.timer = undefined; } + writer.head = { ...writer.head, finalized: true }; + await durableWrite(local(handle.logRef, "head.json"), Buffer.from(JSON.stringify(writer.head))); + writer.dirty = true; + await mirror(writer); + const total = integer(writer.head.bytes); + writers.delete(handle.logRef); + // A whole-history SHA would require a history-sized finalization scan. + // Segment hashes are the integrity contract for this format. + return { bytes: total <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(total) : null, bytesExact: String(total), compressed: false }; + }); + }, + async read(handle: RunLogHandle, opts?: RunLogReadOptions): Promise { + const { head, remote } = await readHead(handle.logRef); + const total = integer(head.bytes); + const limit = opts?.limitBytes ?? 256_000; + if (!Number.isSafeInteger(limit) || limit < 4 || limit > MAX_READ) throw badRequest("Invalid run log page size"); + let start = opts?.cursor === "tail" ? (total > BigInt(limit) ? total - BigInt(limit) : 0n) + : opts?.cursor !== undefined ? integer(opts.cursor) : BigInt(opts?.offset ?? 0); + if (start < 0n || start > total || (!opts?.cursor && !Number.isSafeInteger(opts?.offset ?? 0))) throw badRequest("Invalid run log cursor"); + if (opts?.cursor === "tail" && start > 0n) { + let lastRecord = head.lastRecordCursor === undefined ? undefined : integer(head.lastRecordCursor); + if (lastRecord === undefined) { + // Retained heads predate this index. Search at most one admitted + // record, backwards through verified segments, without collecting + // the record or scanning lifetime history. Exclude its final LF. + const floor = total > BigInt(MAX_APPEND + 1) ? total - BigInt(MAX_APPEND + 1) : 0n; + let end = total - 1n; + while (end > floor) { + const size = BigInt(head.segmentBytes), index = (end - 1n) / size; + const base = index * size, from = floor > base ? floor : base; + const bytes = await verifiedSegment(head, index, remote); + const newline = bytes.subarray(Number(from - base), Number(end - base)).lastIndexOf(10); + if (newline >= 0) { lastRecord = from + BigInt(newline) + 1n; break; } + end = from; + } + if (lastRecord === undefined) { + if (floor !== 0n) throw new Error("run_log_tail_record_invalid"); + lastRecord = 0n; + } + } + // A large final row needs several ordinary pages. Starting inside it + // would discard the only row and incorrectly report EOF to the UI. + if (lastRecord < start) start = lastRecord; + } + const wanted = Number(total - start > BigInt(limit) ? BigInt(limit) : total - start); + const chunks: Buffer[] = []; + let position = start; + while (position < start + BigInt(wanted)) { + const index = position / BigInt(head.segmentBytes); + const offset = Number(position % BigInt(head.segmentBytes)); + const length = Math.min(head.segmentBytes - offset, wanted - Number(position - start)); + const bytes = await verifiedSegment(head, index, remote); + // Copy only the requested page; a slice must not pin evicted 32 MiB + // backing buffers while a page crosses many small historical segments. + chunks.push(Buffer.from(bytes.subarray(offset, offset + length))); + position += BigInt(length); + } + const bytes = Buffer.concat(chunks); + // Leave a trailing partial UTF-8 character for the next page. Byte + // cursors refer to persisted bytes, independent of rendered text length. + let used = bytes.length; + if (position < total && used) { + let lead = used - 1; + while (lead > 0 && (bytes[lead]! & 0xc0) === 0x80) lead--; + const first = bytes[lead]!; + const width = first < 0x80 ? 1 : first < 0xe0 ? 2 : first < 0xf0 ? 3 : 4; + if (used - lead < width) used = lead; + } + const cursor = start + BigInt(used); + return { content: bytes.subarray(0, used).toString("utf8"), cursor: String(cursor), hasMore: cursor < total, + ...(cursor < total && cursor <= BigInt(Number.MAX_SAFE_INTEGER) ? { nextOffset: Number(cursor) } : {}) }; + }, + async flushInflightMirrors() { + for (const [ref, writer] of writers) { + if (writer.timer) { clearTimeout(writer.timer); writer.timer = undefined; } + await serialize(ref, mirror); + } + }, + }; +} diff --git a/server/src/services/work-products.ts b/server/src/services/work-products.ts index 6e6c40a1a9e..9ae6bddd125 100644 --- a/server/src/services/work-products.ts +++ b/server/src/services/work-products.ts @@ -1,6 +1,6 @@ import { and, desc, eq, inArray } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; -import { executionWorkspaces, heartbeatRunEvents, issueWorkProducts, workspaceRuntimeServices } from "@paperclipai/db"; +import { executionWorkspaces, issueWorkProducts, workspaceRuntimeServices } from "@paperclipai/db"; import type { IssueWorkProduct } from "@paperclipai/shared"; import { unprocessable } from "../errors.js"; import { insertRowsInChunks } from "./batch-insert.js"; @@ -15,6 +15,7 @@ import { type GitHubCommitDiffDetailsResolver, } from "./github-commit-details.js"; import type { ImportIssueWorkProductRow } from "./import-write-types.js"; +import { readRunEventLane } from "./run-event-history.js"; type IssueWorkProductRow = typeof issueWorkProducts.$inferSelect; type WorkProductTransaction = Parameters[0]>[0]; @@ -228,15 +229,7 @@ export function workProductService( }, latestRunDiffSummary: async (runId: string): Promise => { - const rows = await db - .select({ payload: heartbeatRunEvents.payload }) - .from(heartbeatRunEvents) - .where(and( - eq(heartbeatRunEvents.runId, runId), - inArray(heartbeatRunEvents.eventType, ["workspace.change.updated", "workspace.diff.recorded"]), - )) - .orderBy(desc(heartbeatRunEvents.seq)) - .limit(20); + const rows = await readRunEventLane(db, runId, "workspace-diff", 20); for (const row of rows) { const summary = workProductDiffSummaryFromEventPayload(row.payload); if (summary) return summary; diff --git a/server/src/storage/local-disk-provider.ts b/server/src/storage/local-disk-provider.ts index 56b35495452..30dff4c4e73 100644 --- a/server/src/storage/local-disk-provider.ts +++ b/server/src/storage/local-disk-provider.ts @@ -1,5 +1,6 @@ -import { createReadStream, promises as fs } from "node:fs"; +import { constants, createReadStream, promises as fs } from "node:fs"; import path from "node:path"; +import { createHash, randomUUID } from "node:crypto"; import type { StorageProvider, GetObjectResult, HeadObjectResult } from "./types.js"; import { notFound, badRequest } from "../errors.js"; @@ -44,11 +45,42 @@ export function createLocalDiskStorageProvider(baseDir: string): StorageProvider async putObject(input) { const targetPath = resolveWithin(root, input.objectKey); const dir = path.dirname(targetPath); - await fs.mkdir(dir, { recursive: true }); + const firstCreated = await fs.mkdir(dir, { recursive: true, mode: 0o700 }); - const tempPath = `${targetPath}.tmp-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; - await fs.writeFile(tempPath, input.body); - await fs.rename(tempPath, targetPath); + const tempPath = `${targetPath}.tmp-${randomUUID()}`; + const file = await fs.open(tempPath, "wx", 0o600); + try { + const hash = createHash("sha256"); + let length = 0; + const chunks = Buffer.isBuffer(input.body) ? [input.body] : input.body; + for await (const value of chunks) { + const chunk = Buffer.isBuffer(value) ? value : Buffer.from(value); + length += chunk.length; + if (length > input.contentLength) throw new Error("storage_object_length_mismatch"); + hash.update(chunk); + await file.writeFile(chunk); + } + if (length !== input.contentLength || (input.sha256 && input.sha256 !== hash.digest("hex"))) throw new Error("storage_object_integrity_mismatch"); + await file.sync(); + await file.close(); + await fs.rename(tempPath, targetPath); + if (process.platform !== "win32") { + // A leaf fsync alone does not make newly created ancestor directory + // entries durable. Flush through the first pre-existing parent before + // allowing a database reference to these bytes to commit. + const stop = firstCreated ? path.dirname(firstCreated) : dir; + let current = dir; + for (;;) { + const directory = await fs.open(current, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW); + try { await directory.sync(); } finally { await directory.close(); } + if (current === stop) break; + current = path.dirname(current); + } + } + } finally { + await file.close(); + await fs.rm(tempPath, { force: true }); + } }, async getObject(input): Promise { diff --git a/server/src/storage/s3-provider.ts b/server/src/storage/s3-provider.ts index 48c305bbfbd..1112e809ae5 100644 --- a/server/src/storage/s3-provider.ts +++ b/server/src/storage/s3-provider.ts @@ -88,10 +88,28 @@ export function createS3StorageProvider(config: S3ProviderConfig): StorageProvid Body: input.body, ContentType: input.contentType, ContentLength: input.contentLength, + ChecksumSHA256: input.sha256 ? Buffer.from(input.sha256, "hex").toString("base64") : undefined, }), ); }, + async putObjectConditional(input, expectedEtag) { + const output = await client.send(new PutObjectCommand({ + Bucket: bucket, + Key: buildKey(prefix, input.objectKey), + Body: input.body, + ContentType: input.contentType, + ContentLength: input.contentLength, + ChecksumSHA256: input.sha256 ? Buffer.from(input.sha256, "hex").toString("base64") : undefined, + IfMatch: expectedEtag ?? undefined, + IfNoneMatch: expectedEtag === null ? "*" : undefined, + })); + // An unknown commit result fences the caller; never retry it as a blind + // overwrite. A fresh owner must inspect the actual published object. + if (!output.ETag) throw new Error("storage_conditional_publication_indeterminate"); + return { etag: output.ETag }; + }, + async getObject(input): Promise { const key = buildKey(prefix, input.objectKey); try { diff --git a/server/src/storage/types.ts b/server/src/storage/types.ts index dab23fa4c06..96094cd0b06 100644 --- a/server/src/storage/types.ts +++ b/server/src/storage/types.ts @@ -8,6 +8,8 @@ export interface PutObjectInput { body: Buffer | Readable; contentType: string; contentLength: number; + /** Optional exact digest required by durable history chunks. */ + sha256?: string; } export interface GetObjectInput { @@ -37,6 +39,9 @@ export interface HeadObjectResult { export interface StorageProvider { id: StorageProviderId; putObject(input: PutObjectInput): Promise; + /** Atomic publication, not a separate HEAD/PUT check. null creates only; + * otherwise the backend must match the exact opaque prior ETag. */ + putObjectConditional?(input: PutObjectInput, expectedEtag: string | null): Promise<{ etag: string }>; getObject(input: GetObjectInput): Promise; headObject(input: GetObjectInput): Promise; deleteObject(input: GetObjectInput): Promise; diff --git a/ui/src/adapters/transcript.ts b/ui/src/adapters/transcript.ts index 008360edbf8..08894f64715 100644 --- a/ui/src/adapters/transcript.ts +++ b/ui/src/adapters/transcript.ts @@ -1,7 +1,7 @@ import { redactHomePathUserSegments, redactTranscriptEntryPaths } from "@paperclipai/adapter-utils"; import type { TranscriptEntry, StdoutLineParser, TranscriptParserSource } from "./types"; -export type RunLogChunk = { ts: string; stream: "stdout" | "stderr" | "system"; chunk: string; seq?: number }; +export type RunLogChunk = { ts: string; stream: "stdout" | "stderr" | "system"; chunk: string; seq?: number; cursor?: string }; type TranscriptBuildOptions = { censorUsernameInLogs?: boolean }; type RedactionOptions = { enabled: boolean }; diff --git a/ui/src/api/heartbeats.test.ts b/ui/src/api/heartbeats.test.ts index 8d65eeb9ab3..64fd90afd51 100644 --- a/ui/src/api/heartbeats.test.ts +++ b/ui/src/api/heartbeats.test.ts @@ -38,6 +38,27 @@ describe("heartbeatsApi.list", () => { }); }); +describe("heartbeatsApi.events", () => { + beforeEach(() => { + mockApi.get.mockReset(); + mockApi.get.mockResolvedValue([]); + }); + + it("passes opaque epoch cursors and the tail sentinel without numeric coercion", async () => { + await heartbeatsApi.events("run-1", "e:123e4567-e89b-42d3-a456-426614174000:8", 100); + expect(mockApi.get).toHaveBeenCalledWith( + "/heartbeat-runs/run-1/events?afterSeq=e%3A123e4567-e89b-42d3-a456-426614174000%3A8&limit=100", + undefined, + ); + mockApi.get.mockClear(); + await heartbeatsApi.events("run-1", "tail", 1_000); + expect(mockApi.get).toHaveBeenCalledWith( + "/heartbeat-runs/run-1/events?afterSeq=tail&limit=1000", + undefined, + ); + }); +}); + describe("heartbeatsApi.liveRunsForCompany", () => { beforeEach(() => { mockApi.get.mockReset(); diff --git a/ui/src/api/heartbeats.ts b/ui/src/api/heartbeats.ts index cf913ef0054..03917883ba1 100644 --- a/ui/src/api/heartbeats.ts +++ b/ui/src/api/heartbeats.ts @@ -3,6 +3,7 @@ import type { IssueRecoveryAction } from "@paperclipai/shared"; import type { HeartbeatRun, HeartbeatRunEvent, + HeartbeatRunLogPage, WorkspaceOperation, ProviderTraceFrame, ProviderTraceMetadata, @@ -37,6 +38,7 @@ export interface ActiveRunForIssue { adapterType: string; logBytes?: number | null; lastOutputBytes?: number | null; + logStore?: HeartbeatRun["logStore"]; issueId?: string | null; livenessState?: RunLivenessFields["livenessState"]; livenessReason?: string | null; @@ -70,6 +72,7 @@ export interface LiveRunForIssue { adapterType: string; logBytes?: number | null; lastOutputBytes?: number | null; + logStore?: HeartbeatRun["logStore"]; issueId?: string | null; livenessState?: RunLivenessFields["livenessState"]; livenessReason?: string | null; @@ -133,20 +136,14 @@ export const heartbeatsApi = { ); }, get: (runId: string) => api.get(`/heartbeat-runs/${runId}`), - events: (runId: string, afterSeq = 0, limit = 200, options?: RequestOptions) => + events: (runId: string, afterSeq: number | string = 0, limit = 200, options?: RequestOptions) => api.get( `/heartbeat-runs/${runId}/events?afterSeq=${encodeURIComponent(String(afterSeq))}&limit=${encodeURIComponent(String(limit))}`, options, ), - log: (runId: string, offset = 0, limitBytes = 256000, options?: RequestOptions) => - api.get<{ - runId: string; - store: string; - logRef: string; - content: string; - nextOffset?: number; - }>( - `/heartbeat-runs/${runId}/log?offset=${encodeURIComponent(String(offset))}&limitBytes=${encodeURIComponent(String(limitBytes))}`, + log: (runId: string, offset: number | string = 0, limitBytes = 256000, options?: RequestOptions) => + api.get( + `/heartbeat-runs/${runId}/log?${typeof offset === "string" ? "cursor" : "offset"}=${encodeURIComponent(String(offset))}&limitBytes=${encodeURIComponent(String(limitBytes))}`, options, ), workspaceOperations: (runId: string) => diff --git a/ui/src/components/LiveRunWidget.tsx b/ui/src/components/LiveRunWidget.tsx index 62852ff3b63..5534832434c 100644 --- a/ui/src/components/LiveRunWidget.tsx +++ b/ui/src/components/LiveRunWidget.tsx @@ -67,6 +67,7 @@ export function LiveRunWidget({ issueId, companyId }: LiveRunWidgetProps) { adapterType: activeRun.adapterType, logBytes: activeRun.logBytes, lastOutputBytes: activeRun.lastOutputBytes, + logStore: activeRun.logStore, issueId, }); } diff --git a/ui/src/components/RunOutputDownload.tsx b/ui/src/components/RunOutputDownload.tsx new file mode 100644 index 00000000000..45fb63ee2c9 --- /dev/null +++ b/ui/src/components/RunOutputDownload.tsx @@ -0,0 +1,12 @@ +import type { RunOutputBodyReference } from "@paperclipai/shared"; + +export function RunOutputDownload({ runId, payload }: { runId: string; payload: unknown }) { + if (!payload || typeof payload !== "object") return null; + const event = (payload as Record).prpEvent; + if (!event || typeof event !== "object") return null; + const content = (event as Record).payload; + if (!content || typeof content !== "object") return null; + const ref = (content as Record).outputBody as RunOutputBodyReference | undefined; + if (ref?.schema !== "paperclip.output.body.v1" || !/^[a-f0-9]{64}$/.test(ref.bodyId)) return null; + return Download full output; +} diff --git a/ui/src/components/RunnerInspector.test.tsx b/ui/src/components/RunnerInspector.test.tsx index dc1ed5546f4..a1e13b7e385 100644 --- a/ui/src/components/RunnerInspector.test.tsx +++ b/ui/src/components/RunnerInspector.test.tsx @@ -169,6 +169,34 @@ describe("RunnerInspector", () => { expect(rerun).toHaveBeenCalledOnce(); }); + it("loads additional event pages by opaque cursor without comparing epoch-local sequences", async () => { + const firstPage = Array.from({ length: 1_000 }, (_, index) => ({ + id: `event-a-${index + 1}`, companyId: "company-1", runId: "run-1", agentId: "agent-1", + seq: index + 1, eventEpoch: "epoch-a", cursor: `e:epoch-a:${index + 1}`, + eventType: "session.notice", stream: null, level: null, color: null, message: null, + payload: null, createdAt: new Date(0), + })); + const secondPage = [{ + ...firstPage[0]!, id: "event-b-1", seq: 1, eventEpoch: "epoch-b", cursor: "e:epoch-b:1", + }]; + eventsMock.mockResolvedValueOnce(firstPage).mockResolvedValueOnce(secondPage); + flushSync(() => root.render( + , + )); + await flush(); + expect(eventsMock).toHaveBeenCalledTimes(1); + expect(eventsMock).toHaveBeenNthCalledWith(1, "run-1", 0, 1_000); + flushSync(() => { + Array.from(container.querySelectorAll("button")) + .find((button) => button.textContent?.includes("Load more events")) + ?.click(); + }); + await flush(); + expect(eventsMock).toHaveBeenCalledTimes(2); + expect(eventsMock).toHaveBeenNthCalledWith(2, "run-1", "e:epoch-a:1000", 1_000); + expect(container.textContent).toContain("seq 1"); + }); + it("shows redacted frames by default and warns before exact reveal", async () => { traceMock.mockResolvedValue({ trace: { diff --git a/ui/src/components/RunnerInspector.tsx b/ui/src/components/RunnerInspector.tsx index 23ed03d8299..d8f8236d159 100644 --- a/ui/src/components/RunnerInspector.tsx +++ b/ui/src/components/RunnerInspector.tsx @@ -30,6 +30,7 @@ import { Trash2, } from "lucide-react"; import { heartbeatsApi, type ProviderTraceInspection } from "@/api/heartbeats"; +import { mergeRunEvents, nextRunEventCursor, runEventPageHasMore, type RunEventCursor } from "@/lib/run-event-pagination"; import { accessApi } from "@/api/access"; import { parsePaperclipRunnerStdoutLine } from "@/adapters/paperclip-runner"; import { TaskChatProtocolCard } from "@/components/task-chat/TaskChatProtocolCard"; @@ -305,14 +306,14 @@ function buildOperations( const root = find(Number(frame.frameId)); groupedFrames.set(root, [...(groupedFrames.get(root) ?? []), frame]); } - const claimedEvents = new Set(); + const claimedEvents = new Set(); const operations: TraceOperation[] = []; for (const groupFrames of groupedFrames.values()) { const frameIds = new Set(groupFrames.map((frame) => Number(frame.frameId))); const stages = interpretations.filter((entry) => frameIds.has(Number(entry.frameId))); const emittedIds = new Set(stages.flatMap(interpretationEventIds)); const groupEvents = events.filter((event) => emittedIds.has(eventSourceId(event))); - groupEvents.forEach((event) => claimedEvents.add(event.id)); + groupEvents.forEach((event) => claimedEvents.add(String(event.id))); const methods = unique(groupFrames.map(frameMethod)); const itemTypes = unique(groupFrames.map(frameItemType)); const title = itemTypes.at(-1) || methods.at(-1) || text(groupFrames[0]?.direction) || "Provider frame"; @@ -338,7 +339,7 @@ function buildOperations( }); } for (const event of events) { - if (claimedEvents.has(event.id)) continue; + if (claimedEvents.has(String(event.id))) continue; const decision = visibilityDecision(event); operations.push({ key: `event:${event.id}`, @@ -359,17 +360,10 @@ function buildOperations( return operations.sort((left, right) => left.timestamp - right.timestamp); } -async function loadAllRunEvents(runId: string) { - const events: HeartbeatRunEvent[] = []; - let afterSeq = 0; - for (;;) { - const page = await heartbeatsApi.events(runId, afterSeq, 1_000); - events.push(...page); - if (page.length < 1_000) return events; - const nextSeq = page.at(-1)?.seq ?? afterSeq; - if (nextSeq <= afterSeq) return events; - afterSeq = nextSeq; - } +async function loadRunEventPage(runId: string, after: RunEventCursor) { + const page = await heartbeatsApi.events(runId, after, 1_000); + const next = nextRunEventCursor(after, page); + return { events: page, cursor: next, hasMore: runEventPageHasMore(page, 1_000) && next !== after }; } function jsonMatches(value: unknown, query: string): boolean { @@ -656,6 +650,9 @@ export function RunnerInspector({ }) { const [inspection, setInspection] = useState(null); const [events, setEvents] = useState([]); + const [eventsCursor, setEventsCursor] = useState(0); + const [hasMoreEvents, setHasMoreEvents] = useState(false); + const [loadingMoreEvents, setLoadingMoreEvents] = useState(false); const [loading, setLoading] = useState(false); const [error, setError] = useState(null); const [view, setView] = useState("pipeline"); @@ -680,6 +677,9 @@ export function RunnerInspector({ setRawTraceAccess(null); setInspection(null); setEvents([]); + setEventsCursor(0); + setHasMoreEvents(false); + setLoadingMoreEvents(false); setLoading(open); setError(null); setSelectedKey(null); @@ -727,8 +727,8 @@ export function RunnerInspector({ const loadEpoch = rawTraceAccessEpochRef.current; setLoading(true); setError(null); - Promise.all([accessApi.getCurrentBoardAccess(), loadAllRunEvents(runId)]) - .then(async ([boardAccess, nextEvents]) => { + Promise.all([accessApi.getCurrentBoardAccess(), loadRunEventPage(runId, 0)]) + .then(async ([boardAccess, eventPage]) => { if (!active || rawTraceAccessEpochRef.current !== loadEpoch) return; const canRaw = boardAccess.source === "local_implicit" || boardAccess.isInstanceAdmin; const access: RawTraceAccess = { @@ -753,7 +753,9 @@ export function RunnerInspector({ return; } setInspection(trace); - setEvents(nextEvents); + setEvents(eventPage.events); + setEventsCursor(eventPage.cursor); + setHasMoreEvents(eventPage.hasMore); }) .catch((cause) => { if (!active || rawTraceAccessEpochRef.current !== loadEpoch) return; @@ -769,6 +771,21 @@ export function RunnerInspector({ }; }, [open, runId]); + const loadMoreEvents = useCallback(async () => { + if (loadingMoreEvents || !hasMoreEvents) return; + setLoadingMoreEvents(true); + try { + const page = await loadRunEventPage(runId, eventsCursor); + setEvents((previous) => mergeRunEvents(previous, page.events)); + setEventsCursor(page.cursor); + setHasMoreEvents(page.hasMore); + } catch (cause) { + setError(cause instanceof Error ? cause.message : "Run events could not be loaded"); + } finally { + setLoadingMoreEvents(false); + } + }, [eventsCursor, hasMoreEvents, loadingMoreEvents, runId]); + useEffect(() => { if (!open) return; let active = true; @@ -1129,6 +1146,7 @@ export function RunnerInspector({ {view === "pipeline" ? ( <> + {hasMoreEvents ?
: null}
setQuery(event.target.value)} placeholder="Search operations, fields, and events" />
diff --git a/ui/src/components/TaskChatThread.test.tsx b/ui/src/components/TaskChatThread.test.tsx index 315a106eab1..64890dc5519 100644 --- a/ui/src/components/TaskChatThread.test.tsx +++ b/ui/src/components/TaskChatThread.test.tsx @@ -26,6 +26,7 @@ const nativeTranscriptState = vi.hoisted(() => ({ transcriptByRun: new Map(), errorsByRun: new Map(), hydratedRunIds: undefined as Set | undefined, + historyCollapsedRunIds: new Set(), })); const transcriptHookRuns = vi.hoisted(() => ({ legacy: [] as unknown[][], @@ -60,6 +61,7 @@ vi.mock("@/components/transcript/useNativeRunTranscripts", () => ({ transcriptByRun: new Map(nativeTranscriptState.transcriptByRun), errorsByRun: new Map(nativeTranscriptState.errorsByRun), hydratedRunIds: nativeTranscriptState.hydratedRunIds, + historyCollapsedRunIds: new Set(nativeTranscriptState.historyCollapsedRunIds), }; }, })); @@ -114,6 +116,7 @@ beforeEach(() => { nativeTranscriptState.transcriptByRun.clear(); nativeTranscriptState.errorsByRun.clear(); nativeTranscriptState.hydratedRunIds = undefined; + nativeTranscriptState.historyCollapsedRunIds.clear(); transcriptHookRuns.legacy.length = 0; transcriptHookRuns.native.length = 0; sidebarState.isMobile = false; @@ -565,6 +568,34 @@ describe("TaskChatThread draft pass-through", () => { }); describe("TaskChatThread runtime transcript selection", () => { + it("shows the durable-history banner for a collapsed native run", () => { + nativeTranscriptState.historyCollapsedRunIds.add("collapsed-native-history"); + render( + {}} + linkedRuns={[{ + runId: "collapsed-native-history", + runtimeMode: "native", + status: "succeeded", + agentId: "agent-1", + agentName: "Runner", + adapterType: "paperclip_runner", + createdAt: "2026-08-25T18:00:00.000Z", + startedAt: "2026-08-25T18:00:00.000Z", + finishedAt: "2026-08-25T18:00:02.000Z", + }]} + />, + ); + + const marker = container.querySelector('[data-testid="task-chat-collapsible-marker"]'); + expect(marker?.textContent).toContain("Earlier run activity collapsed"); + const toggle = marker?.querySelector('button[aria-expanded="false"]'); + expect(toggle).not.toBeNull(); + flushSync(() => toggle!.click()); + expect(marker?.textContent).toContain("durable inspector"); + }); + it("selects persisted runtime facts while retaining the log parser as native fallback", () => { render( = LONG_THREAD_BLOCKER_REPEAT_COUNT; } +function collapsedNativeHistoryMarker( + runId: string, + agentId: string | null | undefined, + agentUrlKey: string | undefined, +): TaskChatMarkerItem { + return { + id: `${runId}:history-collapsed`, + kind: "marker", + variant: "turn_boundary", + tone: "neutral", + collapsible: true, + label: "Earlier run activity collapsed", + detail: "This transcript keeps the newest 1,000 run events or about 2 MiB in memory. Older events remain available in the run’s durable inspector.", + runId, + ...(agentId ? { runHref: `/agents/${encodeURIComponent(agentUrlKey ?? agentId)}/runs/${encodeURIComponent(runId)}` } : {}), + }; +} + function isNativePaperclipRunnerRun( run: | { @@ -846,6 +865,7 @@ export function TaskChatThread(props: TaskChatThreadProps) { runtimeMode: r.runtimeMode, hasStoredOutput: r.hasStoredOutput, logBytes: r.logBytes, + logStore: r.logStore, }); } for (const r of liveRuns ?? []) { @@ -857,6 +877,7 @@ export function TaskChatThread(props: TaskChatThreadProps) { hasStoredOutput: map.get(r.id)?.hasStoredOutput, logBytes: r.logBytes, lastOutputBytes: r.lastOutputBytes, + logStore: r.logStore, }); } if (activeRun) { @@ -867,6 +888,7 @@ export function TaskChatThread(props: TaskChatThreadProps) { runtimeMode: activeRun.runtimeMode, logBytes: activeRun.logBytes, lastOutputBytes: activeRun.lastOutputBytes, + logStore: activeRun.logStore, }); } return [...map.values()]; @@ -894,6 +916,7 @@ export function TaskChatThread(props: TaskChatThreadProps) { errorsByRun: nativeTranscriptErrorsByRun, isInitialHydrating: nativeEventsAreInitiallyHydrating, hydratedRunIds: hydratedNativeRunIds, + historyCollapsedRunIds: collapsedNativeHistoryRunIds, retry: retryNativeEvents, } = useNativeRunTranscripts(nativeRuns); const fallbackByRunRef = useRef( @@ -1439,6 +1462,19 @@ export function TaskChatThread(props: TaskChatThreadProps) { if (liveRun && source.id === liveRun.id) continue; const entries = transcriptByRun.get(source.id) ?? []; const meta = linkedRunMetaById.get(source.id); + if (source.runtimeMode === "native" && collapsedNativeHistoryRunIds.has(source.id)) { + const id = `${source.id}:history-collapsed`; + entriesWithFailures.push({ + ms: toMs(meta?.startedAt ?? meta?.createdAt ?? entries[0]?.ts), + order: 2, + id, + item: collapsedNativeHistoryMarker( + source.id, + meta?.agentId, + meta?.agentId ? agentMap?.get(meta.agentId)?.urlKey : undefined, + ), + }); + } // A workspace admission attempt never started provider work. Its live // successor owns the waiting indicator; retain this attempt in the run log. if (source.status === "cancelled" && meta?.errorCode === "workspace_busy") { @@ -2057,6 +2093,7 @@ export function TaskChatThread(props: TaskChatThreadProps) { runs, liveRun, transcriptByRun, + collapsedNativeHistoryRunIds, linkedRunMetaById, lastCommentIdByRun, comments, @@ -2259,6 +2296,7 @@ export function TaskChatThread(props: TaskChatThreadProps) { linkedRunMetaById.get(tailRunId ?? "")?.agentId ?? issueAssigneeAgentId; const tailAgent = tailAgentId ? agentMap?.get(tailAgentId) : undefined; + const tailHistoryCollapsed = Boolean(tailRunId && collapsedNativeHistoryRunIds.has(tailRunId)); const visibleTailAgentName = tailAgentName ?? tailAgent?.name ?? null; const visibleTailAgentIcon = tailAgent?.icon ?? null; const tailItems = useMemo( @@ -2269,9 +2307,12 @@ export function TaskChatThread(props: TaskChatThreadProps) { agentName: tailAgentName, running: tailStreaming, }); - return tailPlanItem + const visibleItems = tailPlanItem ? embedPlanDocumentAtWriteBoundary(parsed, tailPlanItem) : parsed; + return tailHistoryCollapsed + ? [collapsedNativeHistoryMarker(tailRunId, tailAgentId, tailAgent?.urlKey), ...visibleItems] + : visibleItems; }, // tailEntries is a fresh array each render; tailContentKey tracks its content. // eslint-disable-next-line react-hooks/exhaustive-deps @@ -2281,6 +2322,9 @@ export function TaskChatThread(props: TaskChatThreadProps) { tailStreaming, tailAgentName, tailPlanContentKey, + tailHistoryCollapsed, + tailAgentId, + tailAgent?.urlKey, ], ); const tailTurnStatus = useMemo( diff --git a/ui/src/components/task-chat/RunnerGoalWidget.test.tsx b/ui/src/components/task-chat/RunnerGoalWidget.test.tsx index a7a5b9f699c..526b972ecf1 100644 --- a/ui/src/components/task-chat/RunnerGoalWidget.test.tsx +++ b/ui/src/components/task-chat/RunnerGoalWidget.test.tsx @@ -5,6 +5,7 @@ import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { RunnerGoalProjection } from "@paperclipai/shared"; import { issuesApi } from "@/api/issues"; +import { useCompanyLiveEvent } from "@/context/LiveUpdatesProvider"; import { queryKeys } from "@/lib/queryKeys"; import { RunnerGoalWidget, useRunnerGoalControl } from "./RunnerGoalWidget"; @@ -18,7 +19,7 @@ const projection: RunnerGoalProjection = { goal: { objective: "Original objective", status: "active", tokenBudget: null, tokensUsed: 20, elapsedSeconds: 2, iterations: 1, lastReason: null, createdAt: "2026-09-08T00:00:00Z", updatedAt: "2026-09-08T00:00:02Z", completedAt: null, workingNow: false }, - workingNow: false, activeRunId: null, pendingAction: null, revision: 7, observedAt: "2026-09-08T00:00:02Z", + workingNow: false, activeRunId: null, pendingAction: null, revision: "r:123e4567-e89b-42d3-a456-426614174000", observedAt: "2026-09-08T00:00:02Z", }; function Harness({ agentId = "agent-goal" }: { agentId?: string }) { @@ -35,6 +36,7 @@ function Harness({ agentId = "agent-goal" }: { agentId?: string }) { let root: Root; let container: HTMLDivElement; let client: QueryClient; +let liveEventHandler: ((event: unknown) => void) | null = null; function button(name: string) { const element = [...document.querySelectorAll("button")].find((node) => (node.getAttribute("aria-label") ?? node.textContent?.trim()) === name); @@ -57,6 +59,10 @@ async function render(agentId = "agent-goal") { beforeEach(async () => { vi.clearAllMocks(); + liveEventHandler = null; + vi.mocked(useCompanyLiveEvent).mockImplementation(((handler: (event: unknown) => void) => { + liveEventHandler = handler; + }) as never); client = new QueryClient({ defaultOptions: { queries: { retry: false, staleTime: Infinity }, mutations: { retry: false } } }); client.setQueryData(queryKeys.issues.runnerGoal("issue-goal", "agent-goal"), projection); vi.mocked(issuesApi.getRunnerGoal).mockResolvedValue(projection); @@ -76,7 +82,7 @@ describe("session goal dialogs", () => { it("hides the widget after an expanded goal is cleared", async () => { await click("Focus goal"); vi.mocked(issuesApi.actOnRunnerGoal).mockResolvedValue({ - accepted: true, projection: { ...projection, goal: null, revision: 8 }, + accepted: true, projection: { ...projection, goal: null, revision: "r:123e4567-e89b-42d3-a456-426614174001" }, } as never); await click("Clear goal"); await vi.waitFor(() => expect(document.querySelector('[data-testid="runner-goal-widget"]')).toBeNull()); @@ -119,11 +125,27 @@ describe("session goal dialogs", () => { await objective("Revised objective"); await click("Save goal"); expect(issuesApi.actOnRunnerGoal).toHaveBeenCalledWith("issue-goal", expect.objectContaining({ - action: "edit", objective: "Revised objective", expectedRevision: 7, agentId: "agent-goal", + action: "edit", objective: "Revised objective", expectedRevision: projection.revision, agentId: "agent-goal", })); await vi.waitFor(() => expect(document.querySelector('[role="dialog"]')).toBeNull()); }); + it("refetches instead of ordering unequal opaque live revisions", async () => { + const currentRevision = "r:123e4567-e89b-42d3-a456-426614174000"; + await act(async () => { + client.setQueryData(queryKeys.issues.runnerGoal("issue-goal", "agent-goal"), { ...projection, revision: currentRevision }); + }); + vi.mocked(issuesApi.getRunnerGoal).mockResolvedValue({ ...projection, revision: currentRevision }); + vi.mocked(issuesApi.getRunnerGoal).mockClear(); + const handler = liveEventHandler; + expect(handler).toBeTypeOf("function"); + await act(async () => { + handler!({ type: "agent.session.goal.changed", payload: { ...projection, revision: "r:123e4567-e89b-42d3-a456-426614174001" } }); + }); + await vi.waitFor(() => expect(issuesApi.getRunnerGoal).toHaveBeenCalledTimes(1)); + expect(client.getQueryData(queryKeys.issues.runnerGoal("issue-goal", "agent-goal"))?.revision).toBe(currentRevision); + }); + it("requires explicit replacement confirmation and lets cancellation preserve the goal", async () => { await click("Request replacement"); expect(document.querySelector('[role="dialog"]')?.textContent).toContain("Replace session goal?"); @@ -134,7 +156,7 @@ describe("session goal dialogs", () => { await click("Request replacement"); await click("Replace goal"); expect(issuesApi.actOnRunnerGoal).toHaveBeenCalledWith("issue-goal", expect.objectContaining({ - action: "replace", objective: "Replacement objective", confirmReplace: true, expectedRevision: 7, + action: "replace", objective: "Replacement objective", confirmReplace: true, expectedRevision: projection.revision, })); }); diff --git a/ui/src/components/task-chat/RunnerGoalWidget.tsx b/ui/src/components/task-chat/RunnerGoalWidget.tsx index 256305c3779..3eb696608d1 100644 --- a/ui/src/components/task-chat/RunnerGoalWidget.tsx +++ b/ui/src/components/task-chat/RunnerGoalWidget.tsx @@ -5,6 +5,7 @@ import type { RunnerGoalAction, RunnerGoalActionRequest, RunnerGoalProjection, + RunnerGoalRevision, } from "@paperclipai/shared"; import { issuesApi } from "@/api/issues"; import { useCompanyLiveEvent } from "@/context/LiveUpdatesProvider"; @@ -49,7 +50,7 @@ export function useRunnerGoalControl(issueId: string | null, agentId: string | n const [dialog, setDialog] = useState<{ action: "edit" | "replace"; objective: string; - revision: number; + revision: RunnerGoalRevision; } | null>(null); const [actionError, setActionError] = useState(null); useEffect(() => { @@ -79,18 +80,27 @@ export function useRunnerGoalControl(issueId: string | null, agentId: string | n const next = event.payload as unknown as RunnerGoalProjection; if (next.issueId !== issueId || (agentId && next.agentId !== agentId)) return; const current = queryClient.getQueryData(key); - if (current && next.revision > current.revision + 1) { + if (!current) { + queryClient.setQueryData(key, next); + return; + } + if (typeof current.revision === "string" || typeof next.revision === "string") { + if (next.revision !== current.revision) void query.refetch(); + else queryClient.setQueryData(key, next); + return; + } + if (next.revision > current.revision + 1) { void query.refetch(); return; } - if (!current || next.revision >= current.revision) queryClient.setQueryData(key, next); + if (next.revision >= current.revision) queryClient.setQueryData(key, next); }); const executeAction = useCallback(async ( action: RunnerGoalAction, objective?: string, confirmReplace = false, - expectedRevision?: number, + expectedRevision?: RunnerGoalRevision, ) => { setActionError(null); try { diff --git a/ui/src/components/task-chat/task-protocol-surfaces.ts b/ui/src/components/task-chat/task-protocol-surfaces.ts index 138aa08588d..38f5bb0abdf 100644 --- a/ui/src/components/task-chat/task-protocol-surfaces.ts +++ b/ui/src/components/task-chat/task-protocol-surfaces.ts @@ -107,6 +107,7 @@ export const TASK_PROTOCOL_EVENT_SURFACE_REGISTRY: Readonly [eventType, folded("turn", "Routine lifecycle transitions are summarized by the run turn and composer state.")])), ...Object.fromEntries(diagnosticEvents.map((eventType) => [eventType, folded("system_notice", "Actionable failure text is folded into the run or system notice.")])), ...Object.fromEntries(itemEvents.map((eventType) => [eventType, folded("conversation_or_tool", "Item payloads normalize into messages, reasoning, tools, diffs, or usage." )])), + "output.body.chunk": debugOnly("run_debug", "Body chunks carry immutable output bytes; messages and tools render their normalized content."), "sandbox.metric": debugOnly("run_debug", "Sandbox telemetry is available in run details, not the primary task thread."), ...Object.fromEntries(mcpLifecycleEvents.map((eventType) => [eventType, debugOnly("run_debug", "MCP application transport lifecycle remains in run details unless it produces an actionable failure.")])), ...Object.fromEntries(governanceEvents.map((eventType) => [eventType, folded("governance", "Authoritative task status, attention, and interaction records own this state." )])), diff --git a/ui/src/components/transcript/native-run-events.test.ts b/ui/src/components/transcript/native-run-events.test.ts index 754bd43348a..cd0bcacd043 100644 --- a/ui/src/components/transcript/native-run-events.test.ts +++ b/ui/src/components/transcript/native-run-events.test.ts @@ -164,6 +164,7 @@ describe("nativeRunEventsToTranscript", () => { "terminal_other_session", "terminal_other_turn", "terminal_earlier", + "terminal_different_epoch", "terminal_failed", "terminal_interrupted", "terminal_other_disposition", @@ -179,7 +180,9 @@ describe("nativeRunEventsToTranscript", () => { string, unknown >; - const events = [accepted, terminal]; + const events = mode === "terminal_earlier" + ? [terminal, accepted] + : [accepted, terminal]; if (mode === "provider_acceptance") acceptedEnvelope.sourceKind = "runner"; if (mode === "provider_terminal") terminalEnvelope.sourceKind = "runner"; @@ -215,7 +218,11 @@ describe("nativeRunEventsToTranscript", () => { terminalEnvelope.normalizedSessionId = "other-session"; if (mode === "terminal_other_turn") terminalEnvelope.turnId = "other-turn"; - if (mode === "terminal_earlier") terminal.seq = 1; + if (mode === "terminal_different_epoch") { + accepted.eventEpoch = "epoch-a"; + terminal.eventEpoch = "epoch-b"; + terminal.seq = 4; + } if (mode === "terminal_failed") terminalPayload.runTerminalState = "failed"; if (mode === "terminal_interrupted") @@ -225,6 +232,7 @@ describe("nativeRunEventsToTranscript", () => { if (mode === "missing_terminal") events.pop(); if (mode === "conflicting_acceptance") { const other = structuredClone(accepted); + other.id = 4; other.seq = 4; (other.payload!.prpEvent as Record).sourceEventId = "other-accepted-result"; @@ -284,7 +292,6 @@ describe("nativeRunEventsToTranscript", () => { it("projects provider-neutral messages, tools, usage, and the final reply", () => { const transcript = nativeRunEventsToTranscript([ - event(6, "run.result.proposed", runResult("Done safely.")), event(1, "item.delta", { itemId: "message-1", kind: "agentMessage", text: "Done " }), event(2, "item.delta", { itemId: "message-1", kind: "agentMessage", text: "safely." }), event(3, "item.completed", { itemId: "message-1", kind: "agentMessage", text: "Done safely." }), @@ -312,6 +319,7 @@ describe("nativeRunEventsToTranscript", () => { outputTruncated: false, outputDigest: null, }), + event(6, "run.result.proposed", runResult("Done safely.")), event(7, "usage.reported", { runDeltaAvailable: true, runDelta: { diff --git a/ui/src/components/transcript/native-run-events.ts b/ui/src/components/transcript/native-run-events.ts index 2f39dbdbc0d..515bb3704fa 100644 --- a/ui/src/components/transcript/native-run-events.ts +++ b/ui/src/components/transcript/native-run-events.ts @@ -596,15 +596,20 @@ export function nativeRunEventsToTranscript(events: readonly HeartbeatRunEvent[] costUsd: number; } | null = null; let runResultFallback: { ts: string; text: string } | null = null; + const seenEventIds = new Set(); + const seenCursors = new Set(); const seenSourceEventIds = new Set(); - const orderedEvents = [...events] - .sort((a, b) => a.seq - b.seq) - .filter((event) => { + const orderedEvents = events.filter((event) => { + const id = String(event.id); + if (seenEventIds.has(id) || (event.cursor !== undefined && seenCursors.has(event.cursor))) return false; + seenEventIds.add(id); + if (event.cursor !== undefined) seenCursors.add(event.cursor); const envelope = record(event.payload?.prpEvent); const sourceEventId = text(envelope?.sourceEventId); if (!sourceEventId) return true; - if (seenSourceEventIds.has(sourceEventId)) return false; - seenSourceEventIds.add(sourceEventId); + const sourceKey = `${event.eventEpoch ?? "legacy"}:${sourceEventId}`; + if (seenSourceEventIds.has(sourceKey)) return false; + seenSourceEventIds.add(sourceKey); return true; }); const hasAcceptedResult = orderedEvents.some( @@ -622,14 +627,16 @@ export function nativeRunEventsToTranscript(events: readonly HeartbeatRunEvent[] sourceInstanceId: string; normalizedSessionId: string; turnId: string | null; - seq: number; + eventEpoch: string | undefined; + arrivalIndex: number; } | null = null; const acceptedResultCounts = new Map(); const terminalsByRun = new Map< string, { entry: Extract; - seq: number; + eventEpoch: string | undefined; + arrivalIndex: number; envelope: Record; } >(); @@ -686,7 +693,7 @@ export function nativeRunEventsToTranscript(events: readonly HeartbeatRunEvent[] } const itemIdentityById = new Map(); - for (const event of orderedEvents) { + for (const [arrivalIndex, event] of orderedEvents.entries()) { const envelope = record(event.payload?.prpEvent); if ( !envelope @@ -940,7 +947,8 @@ export function nativeRunEventsToTranscript(events: readonly HeartbeatRunEvent[] entries.push(terminal); terminalsByRun.set(event.runId, { entry: terminal, - seq: event.seq, + eventEpoch: event.eventEpoch, + arrivalIndex, envelope, }); } @@ -993,7 +1001,8 @@ export function nativeRunEventsToTranscript(events: readonly HeartbeatRunEvent[] sourceInstanceId, normalizedSessionId, turnId: text(envelope.turnId), - seq: event.seq, + eventEpoch: event.eventEpoch, + arrivalIndex, }; } hasRunResult = true; @@ -1012,7 +1021,8 @@ export function nativeRunEventsToTranscript(events: readonly HeartbeatRunEvent[] if ( acceptedResultCounts.get(candidate.runId) === 1 && terminal && - terminal.seq > candidate.seq && + terminal.eventEpoch === candidate.eventEpoch && + terminal.arrivalIndex > candidate.arrivalIndex && terminal.envelope.sourceKind === "control_plane" && text(terminal.envelope.sourceEventId)?.trim() && terminal.envelope.sourceInstanceId === candidate.sourceInstanceId && diff --git a/ui/src/components/transcript/useLiveRunTranscripts.test.tsx b/ui/src/components/transcript/useLiveRunTranscripts.test.tsx index b54d92b3728..6650b41b8ac 100644 --- a/ui/src/components/transcript/useLiveRunTranscripts.test.tsx +++ b/ui/src/components/transcript/useLiveRunTranscripts.test.tsx @@ -7,9 +7,11 @@ import { ApiError } from "../../api/client"; import { useLiveRunTranscripts } from "./useLiveRunTranscripts"; import { TRANSCRIPT_REQUEST_TIMEOUT_MS } from "./read-transcript-request"; +type MockLogResult = { runId: string; store: string; logRef: string; content: string; nextOffset?: number; cursor?: string; hasMore?: boolean }; +type MockLogOptions = { signal?: AbortSignal }; const { useQueryMock, logMock, buildTranscriptMock } = vi.hoisted(() => ({ useQueryMock: vi.fn(() => ({ data: { censorUsernameInLogs: false } })), - logMock: vi.fn(async () => ({ runId: "run-1", store: "memory", logRef: "log-1", content: "", nextOffset: 0 })), + logMock: vi.fn(async (_runId?: string, _offset?: number | string, _limit?: number, _options?: { signal?: AbortSignal }): Promise => ({ runId: "run-1", store: "memory", logRef: "log-1", content: "", nextOffset: 0 })), buildTranscriptMock: vi.fn((chunks: unknown[]) => chunks), })); @@ -398,6 +400,155 @@ describe("useLiveRunTranscripts", () => { container.remove(); }); + it("starts segmented logs at tail and preserves exact byte cursors across pages", async () => { + vi.useFakeTimers(); + const endCursor = "900719925474099312345"; + logMock + .mockResolvedValueOnce({ runId: "run-1", store: "local_segments", logRef: "log-1", content: "", cursor: endCursor, hasMore: true }) + .mockResolvedValue({ runId: "run-1", store: "local_segments", logRef: "log-1", content: "", cursor: endCursor, hasMore: false }); + function Harness() { + useLiveRunTranscripts({ + companyId: "company-1", + runs: [{ id: "run-1", status: "running", adapterType: "codex_local", logStore: "local_segments" }], + enableRealtimeUpdates: false, + logPollIntervalMs: 2_000, + }); + return null; + } + const container = document.createElement("div"); + const root = createRoot(container); + try { + await act(async () => { root.render(); await Promise.resolve(); }); + expect(logMock).toHaveBeenNthCalledWith(1, "run-1", "tail", 256_000, expect.anything()); + await act(async () => { await vi.advanceTimersByTimeAsync(2_000); }); + expect(logMock).toHaveBeenLastCalledWith("run-1", endCursor, 256_000, expect.anything()); + } finally { + await act(async () => root.unmount()); + vi.useRealTimers(); + } + }); + + it("finishes an oversized terminal segmented tail row over exact UTF-8 byte pages", async () => { + vi.useFakeTimers(); + const limit = 256_000; + let padding = 255_000; + let bytes: Buffer; + let emojiStart: number; + const makeRecord = () => Buffer.from(`${JSON.stringify({ + ts: "2026-09-29T15:00:00.000Z", + stream: "stdout", + chunk: `${"x".repeat(padding)}🐙${"y".repeat(2_000)}`, + seq: 7, + cursor: "1200", + })}\n`); + while (true) { + bytes = makeRecord(); + emojiStart = bytes.indexOf(Buffer.from("🐙")); + if (emojiStart === limit - 2) break; + padding += limit - 2 - emojiStart; + } + const tailPrefix = bytes.subarray(0, emojiStart).toString("utf8"); + const tailRemainder = bytes.subarray(emojiStart).toString("utf8"); + logMock + .mockResolvedValueOnce({ runId: "run-1", store: "local_segments", logRef: "log-1", content: tailPrefix, cursor: String(emojiStart), hasMore: true }) + .mockResolvedValueOnce({ runId: "run-1", store: "local_segments", logRef: "log-1", content: tailRemainder, cursor: String(bytes.length), hasMore: false }); + const captured: { value: ReturnType | null } = { value: null }; + function Harness() { + captured.value = useLiveRunTranscripts({ + companyId: "company-1", + runs: [{ id: "run-1", status: "succeeded", adapterType: "codex_local", logStore: "local_segments" }], + }); + return null; + } + const container = document.createElement("div"); + const root = createRoot(container); + try { + await act(async () => { root.render(); await Promise.resolve(); await Promise.resolve(); }); + expect(logMock.mock.calls.map(([, cursor]) => cursor)).toEqual(["tail", String(emojiStart)]); + const rows = captured.value?.transcriptByRun.get("run-1") as Array<{ chunk: string }> | undefined; + expect(rows).toHaveLength(1); + expect(rows?.[0]?.chunk).toContain("🐙"); + expect(rows?.[0]?.chunk).toContain("y".repeat(2_000)); + await act(async () => { await vi.advanceTimersByTimeAsync(60_000); }); + expect(logMock).toHaveBeenCalledTimes(2); + } finally { + await act(async () => root.unmount()); + vi.useRealTimers(); + } + }); + + it("stops terminal segmented continuation when the exact cursor stalls", async () => { + logMock + .mockResolvedValueOnce({ runId: "run-1", store: "local_segments", logRef: "log-1", content: "partial row", cursor: "500", hasMore: true }) + .mockResolvedValueOnce({ runId: "run-1", store: "local_segments", logRef: "log-1", content: "", cursor: "500", hasMore: true }); + const captured: { value: ReturnType | null } = { value: null }; + function Harness() { + captured.value = useLiveRunTranscripts({ + companyId: "company-1", + runs: [{ id: "run-1", status: "succeeded", adapterType: "codex_local", logStore: "local_segments" }], + }); + return null; + } + const root = createRoot(document.createElement("div")); + try { + await act(async () => { root.render(); await Promise.resolve(); await Promise.resolve(); }); + expect(logMock.mock.calls.map(([, cursor]) => cursor)).toEqual(["tail", "500"]); + expect(captured.value?.errorsByRun.get("run-1")?.message).toContain("cursor stopped advancing"); + } finally { + await act(async () => root.unmount()); + } + }); + + it("rejects a terminal segmented continuation that returns a backwards cursor before merging it", async () => { + const row = (chunk: string, cursor: string) => `${JSON.stringify({ + ts: "2026-09-29T15:00:00.000Z", stream: "stdout", chunk, cursor, + })}\n`; + logMock + .mockResolvedValueOnce({ runId: "run-1", store: "local_segments", logRef: "log-1", content: row("retained", "400"), cursor: "500", hasMore: true }) + .mockResolvedValueOnce({ runId: "run-1", store: "local_segments", logRef: "log-1", content: row("stale page must not merge", "250"), cursor: "499", hasMore: true }); + const captured: { value: ReturnType | null } = { value: null }; + function Harness() { + captured.value = useLiveRunTranscripts({ + companyId: "company-1", + runs: [{ id: "run-1", status: "succeeded", adapterType: "codex_local", logStore: "local_segments" }], + }); + return null; + } + const root = createRoot(document.createElement("div")); + try { + await act(async () => { root.render(); await Promise.resolve(); await Promise.resolve(); }); + expect(logMock.mock.calls.map(([, cursor]) => cursor)).toEqual(["tail", "500"]); + const rows = captured.value?.transcriptByRun.get("run-1") as Array<{ chunk: string }> | undefined; + expect(rows?.map((entry) => entry.chunk)).toEqual(["retained"]); + expect(captured.value?.errorsByRun.get("run-1")?.message).toContain("cursor moved backwards"); + } finally { + await act(async () => root.unmount()); + } + }); + + it("aborts an in-flight terminal segmented continuation when the view unmounts", async () => { + let continuationSignal: AbortSignal | undefined; + logMock + .mockResolvedValueOnce({ runId: "run-1", store: "local_segments", logRef: "log-1", content: "partial row", cursor: "500", hasMore: true }) + .mockImplementationOnce((_runId?: string, _cursor?: number | string, _limit?: number, options?: MockLogOptions) => { + continuationSignal = options?.signal; + return new Promise(() => {}); + }); + function Harness() { + useLiveRunTranscripts({ + companyId: "company-1", + runs: [{ id: "run-1", status: "succeeded", adapterType: "codex_local", logStore: "local_segments" }], + }); + return null; + } + const root = createRoot(document.createElement("div")); + await act(async () => { root.render(); await Promise.resolve(); await Promise.resolve(); }); + expect(logMock).toHaveBeenCalledTimes(2); + expect(continuationSignal?.aborted).toBe(false); + await act(async () => root.unmount()); + expect(continuationSignal?.aborted).toBe(true); + }); + it("keeps identical same-timestamp log records that carry distinct seq values", async () => { const ts = "2026-04-20T00:00:00.000Z"; const tokenRow = (seq: number) => @@ -628,6 +779,58 @@ describe("useLiveRunTranscripts", () => { container.remove(); }); + it("ignores event commit hints and deduplicates websocket rows by opaque event identity", async () => { + function Harness() { + useLiveRunTranscripts({ + companyId: "company-1", + runs: [{ id: "run-1", status: "running", adapterType: "codex_local" }], + }); + return null; + } + + const container = document.createElement("div"); + document.body.appendChild(container); + const root = createRoot(container); + try { + await act(async () => { + root.render(); + await Promise.resolve(); + }); + const socket = FakeWebSocket.instances[0]!; + buildTranscriptMock.mockClear(); + const sendEvent = (payload: Record) => socket.onmessage?.( + new MessageEvent("message", { + data: JSON.stringify({ + companyId: "company-1", + type: "heartbeat.run.event", + createdAt: "2026-09-29T15:00:00.000Z", + payload: { runId: "run-1", eventType: "run.progress", ...payload }, + }), + }), + ); + + await act(async () => { + sendEvent({ id: "hint-id", eventEpoch: "epoch-a", cursor: "e:epoch-a:1", seq: 1, commitHint: true, message: "wake only" }); + await Promise.resolve(); + }); + expect(buildTranscriptMock).not.toHaveBeenCalled(); + + await act(async () => { + sendEvent({ id: "event-a", eventEpoch: "epoch-a", cursor: "e:epoch-a:1", seq: 1, message: "first epoch" }); + sendEvent({ id: "event-a", eventEpoch: "epoch-a", cursor: "e:epoch-a:1", seq: 1, message: "duplicate" }); + sendEvent({ id: "event-b", eventEpoch: "epoch-b", cursor: "e:epoch-b:1", seq: 1, message: "second epoch" }); + await Promise.resolve(); + }); + + const rows = buildTranscriptMock.mock.calls.at(-1)?.[0] as Array<{ chunk: string }>; + expect(rows.map((row) => row.chunk)).toEqual(["first epoch", "second epoch"]); + expect(rows.some((row) => row.chunk.includes("wake only"))).toBe(false); + } finally { + await act(async () => root.unmount()); + container.remove(); + } + }); + it("retains an accumulated buffer through a transient empty poll (PAP-462 B3)", async () => { const ts = "2026-08-08T00:00:00.000Z"; const row = JSON.stringify({ diff --git a/ui/src/components/transcript/useLiveRunTranscripts.ts b/ui/src/components/transcript/useLiveRunTranscripts.ts index fa6a7f0695c..610b41f4d37 100644 --- a/ui/src/components/transcript/useLiveRunTranscripts.ts +++ b/ui/src/components/transcript/useLiveRunTranscripts.ts @@ -1,6 +1,7 @@ import { usePageVisibility } from "../../lib/page-visibility"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { readTranscriptRequest } from "./read-transcript-request"; +import { nextRunLogPosition, runLogHasMore } from "../../lib/run-log-position"; import { useQuery } from "@tanstack/react-query"; import type { LiveEvent } from "@paperclipai/shared"; import { ApiError } from "../../api/client"; @@ -13,6 +14,7 @@ import { tryCreateWebSocket } from "../../lib/websocket"; import { mergeRunLogChunks, parsePersistedLogContent, + readChunkCursor, readChunkSeq, type ChunkRetentionBudget, } from "../../lib/run-log-chunks"; @@ -25,6 +27,11 @@ const SOCKET_CONNECTING = 0; const SOCKET_OPEN = 1; const LOG_POLL_INTERVAL_MS = 2000; const LOG_READ_LIMIT_BYTES = 256_000; +// Segmented-log append records are capped at 32 MiB. A terminal tail that +// starts at an oversized final record may need a bounded sequence of pages to +// finish that one row before the NDJSON parser can emit it. +const MAX_SEGMENTED_LOG_RECORD_BYTES = 32 * 1024 * 1024; +const MAX_SEGMENTED_TAIL_CONTINUATION_PAGES = 1_024; // When realtime websocket updates are enabled, the frequent log poll is // redundant with the live stream; keep only a slow safety-net poll to cover // gaps and reconnects instead of polling every couple of seconds. @@ -53,6 +60,7 @@ export interface RunTranscriptSource { hasStoredOutput?: boolean; logBytes?: number | null; lastOutputBytes?: number | null; + logStore?: string | null; } interface UseLiveRunTranscriptsOptions { @@ -74,6 +82,24 @@ function readString(value: unknown): string | null { return typeof value === "string" && value.trim().length > 0 ? value : null; } +function readRunEventDedupeKey(runId: string, payload: Record, fallback: string): string { + const id = payload["id"]; + if ((typeof id === "string" && id.length > 0) || (typeof id === "number" && Number.isSafeInteger(id))) { + return `socket:event:${runId}:id:${String(id)}`; + } + const cursor = readString(payload["cursor"]); + if (cursor) return `socket:event:${runId}:cursor:${cursor}`; + const eventEpoch = readString(payload["eventEpoch"]); + const seq = payload["seq"]; + if (eventEpoch && typeof seq === "number" && Number.isSafeInteger(seq) && seq >= 0) { + return `socket:event:${runId}:epoch:${eventEpoch}:${seq}`; + } + if (typeof seq === "number" && Number.isSafeInteger(seq) && seq >= 0) { + return `socket:event:${runId}:legacy-seq:${seq}`; + } + return `socket:event:${runId}:fallback:${fallback}`; +} + function isTerminalStatus(status: string): boolean { return status === "failed" || status === "timed_out" || status === "cancelled" || status === "interrupted" || status === "succeeded"; } @@ -89,6 +115,13 @@ function runKnownLogBytes(run: RunTranscriptSource): number | null { return typeof bytes === "number" && Number.isFinite(bytes) && bytes > 0 ? bytes : null; } +function exactLogPosition(value: number | string): bigint | null { + if (typeof value === "number") { + return Number.isSafeInteger(value) && value >= 0 ? BigInt(value) : null; + } + return /^(?:0|[1-9]\d*)$/.test(value) ? BigInt(value) : null; +} + export function resolveInitialLogOffset(run: RunTranscriptSource, limitBytes: number): number { const knownBytes = runKnownLogBytes(run); if (knownBytes === null) return 0; @@ -120,7 +153,7 @@ export function useLiveRunTranscripts({ .map((run) => { const logBytes = typeof run.logBytes === "number" ? run.logBytes : ""; const lastOutputBytes = typeof run.lastOutputBytes === "number" ? run.lastOutputBytes : ""; - return `${run.id}:${run.status}:${run.adapterType}:${run.hasStoredOutput === true ? "1" : "0"}:${logBytes}:${lastOutputBytes}`; + return `${run.id}:${run.status}:${run.adapterType}:${run.hasStoredOutput === true ? "1" : "0"}:${logBytes}:${lastOutputBytes}:${run.logStore ?? ""}`; }) .sort((a, b) => a.localeCompare(b)) .join(","), @@ -140,8 +173,10 @@ export function useLiveRunTranscripts({ // records re-delivered by the other transport are dropped instead of being // re-inserted ahead of newer output. const trimmedSeqFloorByRunRef = useRef(new Map()); + const trimmedCursorFloorByRunRef = useRef(new Map()); const pendingLogRowsByRunRef = useRef(new Map()); - const logOffsetByRunRef = useRef(new Map()); + const logOffsetByRunRef = useRef(new Map()); + const segmentedTailContinuationRunIdsRef = useRef(new Set()); const missingTerminalLogRunIdsRef = useRef(new Set()); // PAP-462 B3: buffered runs that dropped out of the `runs` list, mapped to the // wall-clock deadline (ms) after which their buffer may be pruned. A run still @@ -192,6 +227,7 @@ export function useLiveRunTranscripts({ chunks, { seenChunkKeys: seenChunkKeysRef.current, + trimmedCursorFloorByRun: trimmedCursorFloorByRunRef.current, trimmedSeqFloorByRun: trimmedSeqFloorByRunRef.current, }, retentionBudget, @@ -270,11 +306,17 @@ export function useLiveRunTranscripts({ logOffsetByRunRef.current.delete(runId); } } + for (const runId of segmentedTailContinuationRunIdsRef.current) { + if (!retainedRunIds.has(runId)) segmentedTailContinuationRunIdsRef.current.delete(runId); + } for (const runId of trimmedSeqFloorByRunRef.current.keys()) { if (!retainedRunIds.has(runId)) { trimmedSeqFloorByRunRef.current.delete(runId); } } + for (const runId of trimmedCursorFloorByRunRef.current.keys()) { + if (!retainedRunIds.has(runId)) trimmedCursorFloorByRunRef.current.delete(runId); + } for (const runId of missingTerminalLogRunIdsRef.current.keys()) { if (!retainedRunIds.has(runId)) { missingTerminalLogRunIdsRef.current.delete(runId); @@ -311,28 +353,72 @@ export function useLiveRunTranscripts({ return; } inFlightRunIds.add(run.id); - const offset = logOffsetByRunRef.current.get(run.id) ?? resolveInitialLogOffset(run, logReadLimitBytes); + const offset = logOffsetByRunRef.current.get(run.id) + ?? (run.logStore === "local_segments" ? "tail" : resolveInitialLogOffset(run, logReadLimitBytes)); try { - const result = await readTranscriptRequest( - (signal) => heartbeatsApi.log(run.id, offset, logReadLimitBytes, { signal }), + const canContinueSegmentedTail = run.logStore === "local_segments" && isTerminalStatus(run.status); + const pageLimit = Math.max(1, logReadLimitBytes); + const maxPages = Math.min( + MAX_SEGMENTED_TAIL_CONTINUATION_PAGES, + Math.ceil(MAX_SEGMENTED_LOG_RECORD_BYTES / pageLimit) + 2, + ); + let pageOffset = offset; + let pagesRead = 0; + let result = await readTranscriptRequest( + (signal) => heartbeatsApi.log(run.id, pageOffset, logReadLimitBytes, { signal }), controller.signal, ); - if (cancelled) return; + while (true) { + if (cancelled) return; + const nextOffset = nextRunLogPosition(pageOffset, result); + const previousPosition = exactLogPosition(pageOffset); + const nextPosition = exactLogPosition(nextOffset); + const stalled = nextPosition !== null && previousPosition !== null && nextPosition === previousPosition; + const regressed = nextPosition !== null && previousPosition !== null && nextPosition < previousPosition; + if (canContinueSegmentedTail && (regressed || (stalled && runLogHasMore(result)))) { + setErrorsByRun((previous) => new Map(previous).set( + run.id, + new Error(regressed + ? "Run history cursor moved backwards before the final output record was complete." + : "Run history cursor stopped advancing before the final output record was complete."), + )); + break; + } + setErrorsByRun((previous) => { + if (!previous.has(run.id)) return previous; + const next = new Map(previous); + next.delete(run.id); + return next; + }); + appendChunks(run.id, parsePersistedLogContent(run.id, result.content, pendingLogRowsByRunRef.current)); - setErrorsByRun((previous) => { - if (!previous.has(run.id)) return previous; - const next = new Map(previous); - next.delete(run.id); - return next; - }); - appendChunks(run.id, parsePersistedLogContent(run.id, result.content, pendingLogRowsByRunRef.current)); + logOffsetByRunRef.current.set(run.id, nextOffset); + if (run.logStore === "local_segments") { + if ( + runLogHasMore(result) && + (pageOffset === "tail" || segmentedTailContinuationRunIdsRef.current.has(run.id)) + ) { + segmentedTailContinuationRunIdsRef.current.add(run.id); + } else if (!runLogHasMore(result)) { + segmentedTailContinuationRunIdsRef.current.delete(run.id); + } + } - if (result.nextOffset !== undefined) { - logOffsetByRunRef.current.set(run.id, result.nextOffset); - return; - } - if (result.content.length > 0) { - logOffsetByRunRef.current.set(run.id, offset + result.content.length); + if (!canContinueSegmentedTail || !segmentedTailContinuationRunIdsRef.current.has(run.id)) break; + if (pagesRead >= maxPages) { + setErrorsByRun((previous) => new Map(previous).set( + run.id, + new Error("The final output record exceeded the bounded page limit. Retry to continue loading it."), + )); + break; + } + + pageOffset = nextOffset; + pagesRead += 1; + result = await readTranscriptRequest( + (signal) => heartbeatsApi.log(run.id, pageOffset, logReadLimitBytes, { signal }), + controller.signal, + ); } } catch (error) { if (cancelled) return; @@ -466,20 +552,25 @@ export function useLiveRunTranscripts({ stream, chunk, seq: readChunkSeq(payload["seq"]), + cursor: readChunkCursor(payload["cursor"]), dedupeKey: `log:${runId}:${ts}:${stream}:${chunk}`, }]); return; } if (event.type === "heartbeat.run.event") { - const seq = typeof payload["seq"] === "number" ? payload["seq"] : null; + if (payload["commitHint"] === true) return; const eventType = readString(payload["eventType"]) ?? "event"; const messageText = readString(payload["message"]) ?? eventType; appendChunks(runId, [{ ts: event.createdAt, stream: eventType === "error" ? "stderr" : "system", chunk: messageText, - dedupeKey: `socket:event:${runId}:${seq ?? `${eventType}:${messageText}:${event.createdAt}`}`, + dedupeKey: readRunEventDedupeKey( + runId, + payload, + `${eventType}:${messageText}:${event.createdAt}`, + ), }]); return; } diff --git a/ui/src/components/transcript/useNativeRunTranscripts.test.tsx b/ui/src/components/transcript/useNativeRunTranscripts.test.tsx index 4666e242cf3..6b105994c2e 100644 --- a/ui/src/components/transcript/useNativeRunTranscripts.test.tsx +++ b/ui/src/components/transcript/useNativeRunTranscripts.test.tsx @@ -163,7 +163,61 @@ describe("native history readiness and stable projection", () => { expect(latest.hydratedRunIds.has("slow")).toBe(false); expect(latest.transcriptByRun.has("one")).toBe(true); await act(async () => { root.render(); }); - expect(eventsMock.mock.calls.filter(([id]) => id === "one").map(([, cursor]) => cursor)).toEqual([0, 7]); + expect(eventsMock.mock.calls.filter(([id]) => id === "one").map(([, cursor]) => cursor)).toEqual(["tail", 7]); expect(latest.isInitialHydrating).toBe(false); }); + + it("marks older durable history when the tail page reports earlier events", async () => { + eventsMock.mockResolvedValueOnce([{ + id: "event-tail", companyId: "company", runId: "one", agentId: "agent", seq: 4, + eventEpoch: "epoch-b", cursor: "e:epoch-b:4", historyBefore: true, + eventType: "turn.started", stream: null, level: null, color: null, + message: null, payload: null, createdAt: new Date(0), + }]); + await act(async () => { root.render(); }); + expect(eventsMock).toHaveBeenCalledWith("one", "tail", 1_000, expect.any(Object)); + expect(latest.historyCollapsedRunIds.has("one")).toBe(true); + }); + + it("continues a short page when its last row reports more history after the byte budget", async () => { + const event = (id: string, cursor: string) => ({ + id, companyId: "company", runId: "one", agentId: "agent", seq: 1, + eventEpoch: "epoch", cursor, eventType: "turn.started", stream: null, + level: null, color: null, message: null, payload: null, createdAt: new Date(0), + }); + eventsMock + .mockResolvedValueOnce([{ ...event("first", "cursor-1"), historyAfter: true }]) + .mockResolvedValueOnce([event("second", "cursor-2")]); + await act(async () => { + root.render(); + }); + expect(eventsMock.mock.calls.map(([, cursor]) => cursor)).toEqual(["tail", "cursor-1"]); + expect(latest.transcriptByRun.has("one")).toBe(true); + }); + + it("caps automatic backlog catch-up and jumps to the latest tail", async () => { + const fullPage = (page: number) => Array.from({ length: 1_000 }, (_, index) => ({ + id: `event-${page}-${index}`, + companyId: "company", runId: "one", agentId: "agent", seq: page * 1_000 + index, + eventEpoch: "epoch", cursor: `cursor-${page}-${index}`, + eventType: "unknown.event", stream: null, level: null, color: null, + message: null, payload: null, createdAt: new Date(0), + })); + eventsMock + .mockResolvedValueOnce(fullPage(1)) + .mockResolvedValueOnce(fullPage(2)) + .mockResolvedValueOnce(fullPage(3)) + .mockResolvedValueOnce(fullPage(4)) + .mockResolvedValueOnce([{ ...fullPage(5)[999], historyBefore: true }]); + + await act(async () => { + root.render(); + }); + + expect(eventsMock.mock.calls.map(([, cursor]) => cursor)).toEqual([ + "tail", "cursor-1-999", "cursor-2-999", "cursor-3-999", "tail", + ]); + expect(latest.historyCollapsedRunIds.has("one")).toBe(true); + expect(latest.hydratedRunIds.has("one")).toBe(true); + }); }); diff --git a/ui/src/components/transcript/useNativeRunTranscripts.ts b/ui/src/components/transcript/useNativeRunTranscripts.ts index a8dd3553e54..31608e4021c 100644 --- a/ui/src/components/transcript/useNativeRunTranscripts.ts +++ b/ui/src/components/transcript/useNativeRunTranscripts.ts @@ -2,10 +2,14 @@ import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import type { HeartbeatRunEvent } from "@paperclipai/shared"; import type { TranscriptEntry } from "@/adapters"; import { heartbeatsApi } from "@/api/heartbeats"; +import { mergeRunEvents, nextRunEventCursor, runEventPageHasMore, type RunEventCursor } from "@/lib/run-event-pagination"; import { nativeRunEventsToTranscript } from "./native-run-events"; import { readTranscriptRequest } from "./read-transcript-request"; const EVENT_PAGE_SIZE = 1_000; +const MAX_AUTO_CATCHUP_PAGES = 4; +const MAX_RETAINED_EVENTS = 1_000; +const MAX_RETAINED_EVENT_BYTES = 2 * 1024 * 1024; const EVENT_POLL_INTERVAL_MS = 2_000; export interface NativeRunTranscriptSource { @@ -23,6 +27,17 @@ function isLive(status: string): boolean { return status === "queued" || status === "running"; } +function retainEventTail(events: HeartbeatRunEvent[]) { + let bytes = 0; + let start = Math.max(0, events.length - MAX_RETAINED_EVENTS); + for (const event of events.slice(start)) bytes += JSON.stringify(event).length; + while (start < events.length - 1 && bytes > MAX_RETAINED_EVENT_BYTES) { + bytes -= JSON.stringify(events[start]!).length; + start += 1; + } + return { events: start ? events.slice(start) : events, collapsed: start > 0 }; +} + export function useNativeRunTranscripts(runs: readonly NativeRunTranscriptSource[]) { const nativeRunsKey = runs .filter((run) => run.runtimeMode === "native") @@ -36,12 +51,15 @@ export function useNativeRunTranscripts(runs: readonly NativeRunTranscriptSource [nativeRunsKey], ); const [eventsByRun, setEventsByRun] = useState>(new Map()); + const eventsByRunRef = useRef(eventsByRun); + eventsByRunRef.current = eventsByRun; const [errorsByRun, setErrorsByRun] = useState>(new Map()); const [hydratedRunIds, setHydratedRunIds] = useState>(new Set()); + const [historyCollapsedRunIds, setHistoryCollapsedRunIds] = useState>(new Set()); const [retryGeneration, setRetryGeneration] = useState(0); const retry = useCallback(() => setRetryGeneration((value) => value + 1), []); const projectionCacheRef = useRef(new Map()); - const cursorByRunRef = useRef(new Map()); + const cursorByRunRef = useRef(new Map()); useEffect(() => { let cancelled = false; @@ -58,6 +76,10 @@ export function useNativeRunTranscripts(runs: readonly NativeRunTranscriptSource const next = new Set([...previous].filter((id) => retainedIds.has(id))); return next.size === previous.size ? previous : next; }); + setHistoryCollapsedRunIds((previous) => { + const next = new Set([...previous].filter((id) => retainedIds.has(id))); + return next.size === previous.size ? previous : next; + }); for (const id of cursorByRunRef.current.keys()) { if (!retainedIds.has(id)) cursorByRunRef.current.delete(id); } @@ -65,31 +87,67 @@ export function useNativeRunTranscripts(runs: readonly NativeRunTranscriptSource const refreshRun = async (run: NativeRunTranscriptSource) => { let failed = false; try { - let cursor = cursorByRunRef.current.get(run.id) ?? 0; + let cursor = cursorByRunRef.current.get(run.id) ?? "tail"; const incoming: HeartbeatRunEvent[] = []; + let historyBefore = false; + let incomingWasTrimmed = false; + let pagesFetched = 0; for (;;) { const page = await readTranscriptRequest( (signal) => heartbeatsApi.events(run.id, cursor, EVENT_PAGE_SIZE, { signal }), controller.signal, ); if (cancelled) return; - const last = page.at(-1); - const nextCursor = last ? Math.max(cursor, last.seq) : cursor; - incoming.push(...page.filter((event) => event.seq > cursor)); - if (page.length < EVENT_PAGE_SIZE || nextCursor === cursor) { + pagesFetched += 1; + historyBefore ||= page[0]?.historyBefore === true; + const boundedIncoming = retainEventTail([...incoming, ...page]); + incoming.splice(0, incoming.length, ...boundedIncoming.events); + incomingWasTrimmed ||= boundedIncoming.collapsed; + const nextCursor = nextRunEventCursor(cursor, page); + if (!runEventPageHasMore(page, EVENT_PAGE_SIZE) || nextCursor === cursor) { cursor = nextCursor; break; } cursor = nextCursor; + if (pagesFetched >= MAX_AUTO_CATCHUP_PAGES) { + // Catch-up must not walk an unbounded backlog. Jump to the latest + // durable page so terminal runs also hydrate their current tail. + const tail = await readTranscriptRequest( + (signal) => heartbeatsApi.events(run.id, "tail", EVENT_PAGE_SIZE, { signal }), + controller.signal, + ); + if (cancelled) return; + const boundedTail = retainEventTail([...incoming, ...tail]); + incoming.splice(0, incoming.length, ...boundedTail.events); + incomingWasTrimmed ||= boundedTail.collapsed || incoming.length > 0; + historyBefore = true; + if (tail.length > 0) cursor = nextRunEventCursor("tail", tail); + break; + } } // Commit this run's cursor with its rows. A slow sibling must neither // hold its readiness hostage nor stall live polling for this run. cursorByRunRef.current.set(run.id, cursor); - if (incoming.length > 0) setEventsByRun((previous) => { - const next = new Map(previous); - next.set(run.id, [...(previous.get(run.id) ?? []), ...incoming]); - return next; - }); + if (incoming.length > 0) { + const merged = mergeRunEvents(eventsByRunRef.current.get(run.id) ?? [], incoming); + const retained = retainEventTail(merged); + if (historyBefore || incomingWasTrimmed || retained.collapsed) { + setHistoryCollapsedRunIds((previous) => previous.has(run.id) + ? previous + : new Set([...previous, run.id])); + } + eventsByRunRef.current = new Map(eventsByRunRef.current).set(run.id, retained.events); + setEventsByRun((previous) => { + const next = new Map(previous); + next.set(run.id, retained.events); + return next; + }); + } + if (incoming.length === 0 && historyBefore) { + setHistoryCollapsedRunIds((previous) => previous.has(run.id) + ? previous + : new Set([...previous, run.id])); + } setErrorsByRun((previous) => { if (!previous.has(run.id)) return previous; const next = new Map(previous); @@ -146,7 +204,7 @@ export function useNativeRunTranscripts(runs: readonly NativeRunTranscriptSource }, [eventsByRun, nativeRuns]); return { - transcriptByRun, errorsByRun, hydratedRunIds, retry, + transcriptByRun, errorsByRun, hydratedRunIds, historyCollapsedRunIds, retry, isInitialHydrating: nativeRuns.some((run) => !hydratedRunIds.has(run.id)), }; } diff --git a/ui/src/components/useSummaryDraftStream.test.tsx b/ui/src/components/useSummaryDraftStream.test.tsx index dc6c4ffde1d..8a9ac26ea15 100644 --- a/ui/src/components/useSummaryDraftStream.test.tsx +++ b/ui/src/components/useSummaryDraftStream.test.tsx @@ -254,7 +254,7 @@ describe("useSummaryDraftStream", () => { it("rehydrates the draft from the persisted run log after a refresh", async () => { // Simulate a page refresh mid-generation: no live event has run id yet, so // the hook resolves it from the active-run endpoint and reads the log. - mockHeartbeatsApi.activeRunForIssue.mockResolvedValue({ id: "run-1", adapterType: "claude-local" }); + mockHeartbeatsApi.activeRunForIssue.mockResolvedValue({ id: "run-1", adapterType: "claude-local", logStore: "local_segments" }); const persistedRows = [ "STATUS: writing the summary…", @@ -268,6 +268,7 @@ describe("useSummaryDraftStream", () => { ts: `t${index}`, stream: "stdout", seq: index + 1, + cursor: String(900719925474099312300n + BigInt(index)), chunk: JSON.stringify({ type: "acpx.text_delta", text: `${line}\n`, channel: "output" }), }), ) @@ -277,7 +278,7 @@ describe("useSummaryDraftStream", () => { store: "s", logRef: "r", content: persistedRows, - nextOffset: persistedRows.length, + cursor: "900719925474099312399", }); const subscribers = new Set(); @@ -287,5 +288,6 @@ describe("useSummaryDraftStream", () => { expect(captured.current?.runId).toBe("run-1"); expect(captured.current?.draft).toBe("## Needs you\nRecovered after refresh."); expect(captured.current?.draftClosed).toBe(true); + expect(mockHeartbeatsApi.log).toHaveBeenCalledWith("run-1", "tail", 256_000); }); }); diff --git a/ui/src/components/useSummaryDraftStream.ts b/ui/src/components/useSummaryDraftStream.ts index 4cd052dd63c..55016401a33 100644 --- a/ui/src/components/useSummaryDraftStream.ts +++ b/ui/src/components/useSummaryDraftStream.ts @@ -5,12 +5,14 @@ import type { LiveEvent, SummarySlotIssueRef } from "@paperclipai/shared"; import type { RunLogChunk } from "@/adapters"; import { heartbeatsApi } from "@/api/heartbeats"; +import { nextRunLogPosition } from "@/lib/run-log-position"; import { useCompanyLiveEvent } from "@/context/LiveUpdatesProvider"; import { queryKeys } from "@/lib/queryKeys"; import { mergeRunLogChunks, parsePersistedLogContent, readChunkSeq, + readChunkCursor, type ChunkMergeRefs, type IncomingRunLogChunk, } from "@/lib/run-log-chunks"; @@ -38,7 +40,11 @@ export interface SummaryDraftStream { } function freshMergeRefs(): ChunkMergeRefs { - return { seenChunkKeys: new Set(), trimmedSeqFloorByRun: new Map() }; + return { + seenChunkKeys: new Set(), + trimmedSeqFloorByRun: new Map(), + trimmedCursorFloorByRun: new Map(), + }; } function readPayloadString(value: unknown): string | null { @@ -70,7 +76,7 @@ export function useSummaryDraftStream( const mergeRefs = useRef(freshMergeRefs()); const pendingLogRowsRef = useRef(new Map()); - const logOffsetRef = useRef(0); + const logOffsetRef = useRef(0); // Reset all stream state whenever the tracked generation changes — including // a superseded generation (new issue id) or generation ending (null). @@ -121,9 +127,9 @@ export function useSummaryDraftStream( }, [fallbackRunId]); useEffect(() => { - logOffsetRef.current = 0; + logOffsetRef.current = activeRunQuery.data?.logStore === "local_segments" ? "tail" : 0; pendingLogRowsRef.current = new Map(); - }, [runId]); + }, [runId, activeRunQuery.data?.logStore]); // Live token deltas over the shared company-events socket. useCompanyLiveEvent((event: LiveEvent) => { @@ -137,7 +143,7 @@ export function useSummaryDraftStream( const stream = payload.stream === "stderr" ? "stderr" : payload.stream === "system" ? "system" : "stdout"; appendChunks([ - { ts, stream, chunk, seq: readChunkSeq(payload.seq), dedupeKey: `log:${runId}:${ts}:${stream}:${chunk}` }, + { ts, stream, chunk, seq: readChunkSeq(payload.seq), cursor: readChunkCursor(payload.cursor), dedupeKey: `log:${runId}:${ts}:${stream}:${chunk}` }, ]); }); @@ -153,11 +159,7 @@ export function useSummaryDraftStream( const result = await heartbeatsApi.log(runId, logOffsetRef.current, LOG_READ_LIMIT_BYTES); if (cancelled) return; appendChunks(parsePersistedLogContent(runId, result.content, pendingLogRowsRef.current)); - if (result.nextOffset !== undefined) { - logOffsetRef.current = result.nextOffset; - } else if (result.content.length > 0) { - logOffsetRef.current += result.content.length; - } + logOffsetRef.current = nextRunLogPosition(logOffsetRef.current, result); } catch { // Ignore transient/404 reads (log not yet flushed, run just started). } finally { diff --git a/ui/src/lib/issue-chat-messages.ts b/ui/src/lib/issue-chat-messages.ts index bc6e7dfef1d..af653b87c76 100644 --- a/ui/src/lib/issue-chat-messages.ts +++ b/ui/src/lib/issue-chat-messages.ts @@ -58,6 +58,7 @@ export interface IssueChatLinkedRun { finishedAt?: Date | string | null; hasStoredOutput?: boolean; logBytes?: number | null; + logStore?: string | null; errorCode?: string | null; scheduledRetryAt?: string | null; nextAction?: string | null; diff --git a/ui/src/lib/issueChatTranscriptRuns.ts b/ui/src/lib/issueChatTranscriptRuns.ts index 3b67fc286a9..c70b863ead6 100644 --- a/ui/src/lib/issueChatTranscriptRuns.ts +++ b/ui/src/lib/issueChatTranscriptRuns.ts @@ -36,6 +36,7 @@ export function resolveIssueChatTranscriptRuns(args: { adapterType: run.adapterType, logBytes: run.logBytes, lastOutputBytes: run.lastOutputBytes, + logStore: run.logStore, }); } @@ -46,6 +47,7 @@ export function resolveIssueChatTranscriptRuns(args: { adapterType: activeRun.adapterType, logBytes: activeRun.logBytes, lastOutputBytes: activeRun.lastOutputBytes, + logStore: activeRun.logStore, }); } @@ -65,6 +67,7 @@ export function resolveIssueChatTranscriptRuns(args: { adapterType, hasStoredOutput: run.hasStoredOutput, logBytes: run.logBytes, + logStore: run.logStore, }); } diff --git a/ui/src/lib/run-event-pagination.test.ts b/ui/src/lib/run-event-pagination.test.ts new file mode 100644 index 00000000000..3b29dc7ef3a --- /dev/null +++ b/ui/src/lib/run-event-pagination.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from "vitest"; +import type { HeartbeatRunEvent } from "@paperclipai/shared"; +import { mergeRunEvents, nextRunEventCursor, retainRunEventTail, runEventPageHasMore } from "./run-event-pagination"; + +function event( + id: number | string, + seq: number, + eventEpoch: string, + cursor = `e:${eventEpoch}:${seq}`, +): HeartbeatRunEvent { + return { + id, companyId: "company", runId: "run", agentId: "agent", seq, + eventEpoch, cursor, eventType: "test", stream: null, level: null, + color: null, message: null, payload: null, createdAt: new Date(0), + }; +} + +describe("run event pagination", () => { + it("keeps repeated epoch-local sequences in server arrival order", () => { + const first = event("legacy-1", 1, "epoch-a"); + const rollover = event("event-b", 1, "epoch-b"); + const later = event("event-c", 2, "epoch-b"); + expect(mergeRunEvents([first], [rollover, later])).toEqual([first, rollover, later]); + }); + + it("deduplicates by stringified id or exact cursor without sorting the page", () => { + const first = event(1, 7, "epoch-a"); + const duplicateId = event("1", 1, "epoch-b", "e:epoch-b:1"); + const duplicateCursor = event("other-id", 8, "epoch-a", first.cursor); + const second = event("event-2", 1, "epoch-b"); + expect(mergeRunEvents([first], [duplicateId, duplicateCursor, second])).toEqual([first, second]); + }); + + it("advances with the opaque cursor and stops when the cursor repeats", () => { + const current = "e:epoch-a:99"; + expect(nextRunEventCursor(current, [event("id", 1, "epoch-b")])).toBe("e:epoch-b:1"); + expect(nextRunEventCursor(current, [event("id", 99, "epoch-a", current)])).toBe(current); + expect(nextRunEventCursor(0, [event("id", 4, "epoch-a", "")])).toBe(""); + expect(nextRunEventCursor(0, [{ ...event("legacy", 4, "legacy"), cursor: undefined }])).toBe(4); + }); + + it("uses the last-row byte-budget marker when deciding whether to load another page", () => { + expect(runEventPageHasMore([{ historyAfter: true }], 1_000)).toBe(true); + expect(runEventPageHasMore([{ historyAfter: false }], 1)).toBe(false); + expect(runEventPageHasMore([{}], 1_000)).toBe(false); + expect(runEventPageHasMore([{}, {}], 2)).toBe(true); + }); + + it("bounds retained events by serialized bytes and reports omitted history", () => { + const events = [1, 2, 3].map((id) => ({ + ...event(`event-${id}`, id, "epoch"), + payload: { body: "x".repeat(800_000) }, + })); + const retained = retainRunEventTail(events); + expect(retained.collapsed).toBe(true); + expect(retained.events.map((row) => row.id)).toEqual(["event-2", "event-3"]); + }); +}); diff --git a/ui/src/lib/run-event-pagination.ts b/ui/src/lib/run-event-pagination.ts new file mode 100644 index 00000000000..5838f23cbb2 --- /dev/null +++ b/ui/src/lib/run-event-pagination.ts @@ -0,0 +1,58 @@ +import type { HeartbeatRunEvent } from "@paperclipai/shared"; + +export type RunEventCursor = number | string; +export const MAX_RETAINED_RUN_EVENT_COUNT = 2_000; +export const MAX_RETAINED_RUN_EVENT_BYTES = 2 * 1024 * 1024; + +/** The last row may indicate that a byte budget shortened an otherwise full page. */ +export function runEventPageHasMore( + page: readonly { historyAfter?: boolean }[], + pageSize: number, +): boolean { + const last = page.at(-1); + return last?.historyAfter ?? page.length >= pageSize; +} + +export function retainRunEventTail(events: HeartbeatRunEvent[]) { + let start = Math.max(0, events.length - MAX_RETAINED_RUN_EVENT_COUNT); + let bytes = events.slice(start).reduce((total, event) => total + JSON.stringify(event).length, 0); + while (start < events.length - 1 && bytes > MAX_RETAINED_RUN_EVENT_BYTES) { + bytes -= JSON.stringify(events[start]!).length; + start += 1; + } + return { events: start ? events.slice(start) : events, collapsed: start > 0 }; +} + +/** Advance with the exact server cursor, falling back to the legacy local seq. */ +export function nextRunEventCursor( + current: RunEventCursor, + page: readonly HeartbeatRunEvent[], +): RunEventCursor { + const last = page.at(-1); + if (!last) return current; + const next = last.cursor ?? last.seq; + return next === current ? current : next; +} + +/** Merge in arrival order; seq is epoch-local and is never an identity. */ +export function mergeRunEvents( + previous: HeartbeatRunEvent[], + incoming: readonly HeartbeatRunEvent[], + limit = Number.POSITIVE_INFINITY, +): HeartbeatRunEvent[] { + if (incoming.length === 0) return previous as HeartbeatRunEvent[]; + const ids = new Set(previous.map((event) => String(event.id))); + const cursors = new Set(previous.flatMap((event) => event.cursor ? [event.cursor] : [])); + const next = [...previous]; + let added = false; + for (const event of incoming) { + const id = String(event.id); + if (ids.has(id) || (event.cursor !== undefined && cursors.has(event.cursor))) continue; + ids.add(id); + if (event.cursor !== undefined) cursors.add(event.cursor); + next.push(event); + added = true; + } + if (!added) return previous; + return next.length > limit ? next.slice(next.length - limit) : next; +} diff --git a/ui/src/lib/run-log-chunks.test.ts b/ui/src/lib/run-log-chunks.test.ts index 0aa075b78f0..8f12cb45b3c 100644 --- a/ui/src/lib/run-log-chunks.test.ts +++ b/ui/src/lib/run-log-chunks.test.ts @@ -6,6 +6,7 @@ import { isTrimmedOutputMarkerChunk, mergeRunLogChunks, parsePersistedLogContent, + readChunkCursor, readChunkSeq, TRIMMED_OUTPUT_MARKER_TEXT, type ChunkMergeRefs, @@ -13,13 +14,21 @@ import { } from "./run-log-chunks"; function freshRefs(): ChunkMergeRefs { - return { seenChunkKeys: new Set(), trimmedSeqFloorByRun: new Map() }; + return { + seenChunkKeys: new Set(), + trimmedSeqFloorByRun: new Map(), + trimmedCursorFloorByRun: new Map(), + }; } function seqChunk(seq: number, chunk: string): IncomingRunLogChunk { return { ts: `t${seq}`, stream: "stdout", chunk, seq, dedupeKey: `k${seq}` }; } +function cursorChunk(cursor: string, seq: number, chunk: string): IncomingRunLogChunk { + return { ts: `t${cursor}`, stream: "stdout", chunk, cursor, seq, dedupeKey: `cursor:${cursor}` }; +} + describe("readChunkSeq", () => { it("accepts finite numbers only", () => { expect(readChunkSeq(3)).toBe(3); @@ -30,6 +39,16 @@ describe("readChunkSeq", () => { }); }); +describe("readChunkCursor", () => { + it("accepts exact canonical decimal byte positions without number conversion", () => { + expect(readChunkCursor("900719925474099312345")).toBe("900719925474099312345"); + expect(readChunkCursor("0")).toBe("0"); + expect(readChunkCursor("01")).toBeUndefined(); + expect(readChunkCursor(1)).toBeUndefined(); + expect(readChunkCursor("1.5")).toBeUndefined(); + }); +}); + describe("isStructuredStreamingTextDelta", () => { it("matches acpx.text_delta and text records", () => { expect(isStructuredStreamingTextDelta('{"type":"acpx.text_delta","text":"x"}')).toBe(true); @@ -44,18 +63,20 @@ describe("parsePersistedLogContent", () => { const pending = new Map(); const first = parsePersistedLogContent( "run-1", - '{"ts":"a","stream":"stdout","chunk":"one","seq":1}\n{"ts":"b","stream":"std', + '{"ts":"a","stream":"stdout","chunk":"one","seq":1,"cursor":"42"}\n{"ts":"b","stream":"std', pending, ); expect(first).toHaveLength(1); expect(first[0]!.chunk).toBe("one"); expect(first[0]!.seq).toBe(1); + expect(first[0]!.cursor).toBe("42"); // Second read completes the partial row from the first. - const second = parsePersistedLogContent("run-1", 'out","chunk":"two","seq":2}\n', pending); + const second = parsePersistedLogContent("run-1", 'out","chunk":"two","seq":2,"cursor":"99"}\n', pending); expect(second).toHaveLength(1); expect(second[0]!.chunk).toBe("two"); expect(second[0]!.seq).toBe(2); + expect(second[0]!.cursor).toBe("99"); }); it("skips blank and malformed rows", () => { @@ -69,6 +90,41 @@ describe("parsePersistedLogContent", () => { }); describe("mergeRunLogChunks", () => { + it("orders and deduplicates by exact large byte cursor, preferring persisted full chunks", () => { + const refs = freshRefs(); + let state: RunLogChunk[] = []; + ({ chunks: state } = mergeRunLogChunks("r", state, [ + cursorChunk("900719925474099300010", 1, "next"), + ], refs, 100)); + ({ chunks: state } = mergeRunLogChunks("r", state, [ + cursorChunk("900719925474099300000", 2, "first"), + cursorChunk("900719925474099300010", 3, "next persisted full payload"), + ], refs, 100)); + expect(state.map((chunk) => [chunk.cursor, chunk.chunk])).toEqual([ + ["900719925474099300000", "first"], + ["900719925474099300010", "next persisted full payload"], + ]); + }); + + it("does not use seq identity or ordering when byte cursors are present", () => { + const refs = freshRefs(); + let state: RunLogChunk[] = []; + ({ chunks: state } = mergeRunLogChunks("r", state, [ + cursorChunk("100", 7, "later cursor"), + cursorChunk("90", 7, "earlier cursor"), + ], refs, 100)); + expect(state.map((chunk) => chunk.chunk)).toEqual(["earlier cursor", "later cursor"]); + }); + + it("drops cursor replays at or below the per-run trimmed byte floor", () => { + const refs = freshRefs(); + const rows = [cursorChunk("10", 1, "a"), cursorChunk("20", 2, "b"), cursorChunk("30", 3, "c")]; + const trimmed = mergeRunLogChunks("r", [], rows, refs, 2); + expect(trimmed.chunks.map((chunk) => chunk.cursor)).toEqual(["20", "30"]); + expect(refs.trimmedCursorFloorByRun.get("r")).toBe("10"); + expect(mergeRunLogChunks("r", trimmed.chunks, [cursorChunk("10", 99, "replay")], refs, 2).changed).toBe(false); + }); + it("orders sequenced chunks by seq regardless of arrival order", () => { const refs = freshRefs(); let state: RunLogChunk[] = []; diff --git a/ui/src/lib/run-log-chunks.ts b/ui/src/lib/run-log-chunks.ts index 9072816c0e1..b2e410921b2 100644 --- a/ui/src/lib/run-log-chunks.ts +++ b/ui/src/lib/run-log-chunks.ts @@ -21,6 +21,8 @@ export type IncomingRunLogChunk = RunLogChunk & { dedupeKey: string }; */ export interface ChunkMergeRefs { seenChunkKeys: Set; + trimmedCursorFloorByRun: Map; + /** Legacy-only ceiling for records that do not carry byte cursors. */ trimmedSeqFloorByRun: Map; } @@ -59,7 +61,8 @@ export function isTrimmedOutputMarkerChunk(chunk: RunLogChunk): boolean { return ( chunk.stream === "system" && chunk.chunk === TRIMMED_OUTPUT_MARKER_TEXT && - chunk.seq === undefined + chunk.seq === undefined && + chunk.cursor === undefined ); } @@ -85,7 +88,7 @@ function normalizeRetentionBudget(budget: number | ChunkRetentionBudget): ChunkR export function applyRetentionBudget( chunks: RunLogChunk[], budget: ChunkRetentionBudget, -): { chunks: RunLogChunk[]; trimmedSeq: number | null } { +): { chunks: RunLogChunk[]; trimmedSeq: number | null; trimmedCursor: string | null } { const { maxChunks, maxBytes, collapseTrimmed } = budget; // Peel off any existing marker so it is never counted or duplicated; a single @@ -113,31 +116,40 @@ export function applyRetentionBudget( // Nothing new to trim. Preserve an existing marker so an earlier collapse // keeps its trace; otherwise return the marker-stripped array only if we // actually stripped one (we never had a real trim to justify keeping it). - if (hadMarker) return { chunks, trimmedSeq: null }; - return { chunks: real, trimmedSeq: null }; + if (hadMarker) return { chunks, trimmedSeq: null, trimmedCursor: null }; + return { chunks: real, trimmedSeq: null, trimmedCursor: null }; } const removed = real.slice(0, removeCount); const kept = real.slice(removeCount); let trimmedSeq: number | null = null; + let trimmedCursor: string | null = null; for (const item of removed) { - if (typeof item.seq === "number" && (trimmedSeq === null || item.seq > trimmedSeq)) { + if (item.cursor === undefined && typeof item.seq === "number" && (trimmedSeq === null || item.seq > trimmedSeq)) { trimmedSeq = item.seq; } + if (item.cursor && (trimmedCursor === null || BigInt(item.cursor) > BigInt(trimmedCursor))) { + trimmedCursor = item.cursor; + } } if (collapseTrimmed || hadMarker) { const markerTs = kept[0]?.ts ?? removed[removed.length - 1]!.ts; - return { chunks: [makeTrimmedOutputMarkerChunk(markerTs), ...kept], trimmedSeq }; + return { chunks: [makeTrimmedOutputMarkerChunk(markerTs), ...kept], trimmedSeq, trimmedCursor }; } - return { chunks: kept, trimmedSeq }; + return { chunks: kept, trimmedSeq, trimmedCursor }; } export function readChunkSeq(value: unknown): number | undefined { return typeof value === "number" && Number.isFinite(value) ? value : undefined; } +/** Byte-position cursors are canonical non-negative decimal strings. */ +export function readChunkCursor(value: unknown): string | undefined { + return typeof value === "string" && /^(?:0|[1-9]\d*)$/.test(value) ? value : undefined; +} + export function isStructuredStreamingTextDelta(chunk: string): boolean { return /"type"\s*:\s*"(?:acpx\.text_delta|text)"/.test(chunk); } @@ -164,7 +176,7 @@ export function parsePersistedLogContent( const trimmed = line.trim(); if (!trimmed) continue; try { - const raw = JSON.parse(trimmed) as { ts?: unknown; stream?: unknown; chunk?: unknown; seq?: unknown }; + const raw = JSON.parse(trimmed) as { ts?: unknown; stream?: unknown; chunk?: unknown; seq?: unknown; cursor?: unknown }; const stream = raw.stream === "stderr" || raw.stream === "system" ? raw.stream : "stdout"; const chunk = typeof raw.chunk === "string" ? raw.chunk : ""; const ts = typeof raw.ts === "string" ? raw.ts : new Date().toISOString(); @@ -174,6 +186,7 @@ export function parsePersistedLogContent( stream, chunk, seq: readChunkSeq(raw.seq), + cursor: readChunkCursor(raw.cursor), dedupeKey: `log:${runId}:${ts}:${stream}:${chunk}`, }); } catch { @@ -191,10 +204,10 @@ export function parsePersistedLogContent( * * Ordering rules (unchanged from the original `useLiveRunTranscripts` * implementation): - * - Sequenced chunks dedupe/order by the server-assigned monotonic `seq`. When - * the same `seq` arrives from both transports the longer payload wins (the - * websocket copy may be tail-truncated). Records at or below the trimmed - * floor are dropped. + * - Cursor-bearing chunks use exact decimal byte positions for identity and + * ordering. When a cursor arrives from both transports the longer payload + * wins (the websocket copy may be tail-truncated). Records at or below the + * trimmed cursor floor are dropped. Legacy records retain seq behavior. * - Unsequenced chunks dedupe by content key (skipping structured streaming * text deltas, which legitimately repeat) and act as an ordering barrier for * subsequent sequenced inserts. @@ -212,10 +225,33 @@ export function mergeRunLogChunks( let changed = false; for (const chunk of incoming) { + if (chunk.cursor !== undefined) { + const cursor = readChunkCursor(chunk.cursor); + if (cursor === undefined) continue; + const cursorFloor = refs.trimmedCursorFloorByRun.get(runId); + if (cursorFloor !== undefined && BigInt(cursor) <= BigInt(cursorFloor)) continue; + const duplicateAt = existing.findIndex((item) => item.cursor === cursor); + if (duplicateAt !== -1) { + if (chunk.chunk.length > existing[duplicateAt]!.chunk.length) { + existing[duplicateAt] = { ts: chunk.ts, stream: chunk.stream, chunk: chunk.chunk, seq: chunk.seq, cursor }; + changed = true; + } + continue; + } + let insertAt = existing.length; + while (insertAt > 0) { + const prior = existing[insertAt - 1]!; + if (prior.cursor === undefined || BigInt(prior.cursor) < BigInt(cursor)) break; + insertAt -= 1; + } + existing.splice(insertAt, 0, { ts: chunk.ts, stream: chunk.stream, chunk: chunk.chunk, seq: chunk.seq, cursor }); + changed = true; + continue; + } if (typeof chunk.seq === "number") { const seqFloor = refs.trimmedSeqFloorByRun.get(runId) ?? 0; if (chunk.seq <= seqFloor) continue; - const duplicateAt = existing.findIndex((item) => item.seq === chunk.seq); + const duplicateAt = existing.findIndex((item) => item.cursor === undefined && item.seq === chunk.seq); if (duplicateAt !== -1) { // Same record arrived via the other delivery path. Prefer the longer // payload: websocket chunks may be tail-truncated while the persisted @@ -253,11 +289,17 @@ export function mergeRunLogChunks( refs.seenChunkKeys.clear(); } - const { chunks: retained, trimmedSeq } = applyRetentionBudget(existing, normalizeRetentionBudget(budget)); + const { chunks: retained, trimmedSeq, trimmedCursor } = applyRetentionBudget(existing, normalizeRetentionBudget(budget)); if (trimmedSeq !== null) { const seqFloor = refs.trimmedSeqFloorByRun.get(runId) ?? 0; if (trimmedSeq > seqFloor) refs.trimmedSeqFloorByRun.set(runId, trimmedSeq); } + if (trimmedCursor !== null) { + const cursorFloor = refs.trimmedCursorFloorByRun.get(runId); + if (cursorFloor === undefined || BigInt(trimmedCursor) > BigInt(cursorFloor)) { + refs.trimmedCursorFloorByRun.set(runId, trimmedCursor); + } + } return { chunks: retained, changed: true }; } diff --git a/ui/src/lib/run-log-position.test.ts b/ui/src/lib/run-log-position.test.ts new file mode 100644 index 00000000000..25665d2c370 --- /dev/null +++ b/ui/src/lib/run-log-position.test.ts @@ -0,0 +1,12 @@ +import { expect, it } from "vitest"; +import { nextRunLogPosition, runLogHasMore } from "./run-log-position"; + +it("preserves an exact large cursor and knows a caught-up cursor is not another page", () => { + expect(nextRunLogPosition("9007199254740992", { content: "🐙", cursor: "9007199254740996", hasMore: false })).toBe("9007199254740996"); + expect(runLogHasMore({ content: "", cursor: "9007199254740996", hasMore: false })).toBe(false); + expect(runLogHasMore({ content: "x", cursor: "9007199254740996", hasMore: true })).toBe(true); +}); +it("counts UTF-8 bytes only when an old server omits its cursor", () => { + expect(nextRunLogPosition(4, { content: "🐙" })).toBe(8); + expect(nextRunLogPosition(4, { content: "🐙", nextOffset: 10 })).toBe(10); +}); diff --git a/ui/src/lib/run-log-position.ts b/ui/src/lib/run-log-position.ts new file mode 100644 index 00000000000..1c0124c7a60 --- /dev/null +++ b/ui/src/lib/run-log-position.ts @@ -0,0 +1,12 @@ +import type { HeartbeatRunLogPage } from "@paperclipai/shared"; + +type Page = Pick; +export function nextRunLogPosition(previous: number | string, page: Page): number | string { + if (page.cursor !== undefined) return page.cursor; + if (page.nextOffset !== undefined) return page.nextOffset; + const length = new TextEncoder().encode(page.content).length; + return typeof previous === "string" ? String(BigInt(previous) + BigInt(length)) : previous + length; +} +export function runLogHasMore(page: Page): boolean { + return page.hasMore ?? page.nextOffset !== undefined; +} diff --git a/ui/src/pages/AgentDetail.log-visibility.test.tsx b/ui/src/pages/AgentDetail.log-visibility.test.tsx index d0a96b68a60..6dc6e17a49a 100644 --- a/ui/src/pages/AgentDetail.log-visibility.test.tsx +++ b/ui/src/pages/AgentDetail.log-visibility.test.tsx @@ -23,6 +23,41 @@ vi.mock("../components/transcript/RunTranscriptView", () => ({ (globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true; afterEach(() => { vi.restoreAllMocks(); vi.unstubAllGlobals(); vi.useRealTimers(); log.mockReset(); events.mockClear(); }); +it.each([LogViewer, ProductionLogViewer])("keeps an exact segmented-log cursor across visibility recovery (%#)", async Viewer => { + const visibility = vi.spyOn(document, "visibilityState", "get").mockReturnValue("visible"); + const cursor = "9007199254740999"; + log.mockResolvedValue({ content: JSON.stringify({ ts: "t", stream: "stdout", chunk: "🐙 retained" }) + "\n", cursor, hasMore: false }); + const run = { id: "run-cursor", companyId: "company", agentId: "agent", status: "succeeded", logRef: "log" } as HeartbeatRun; + const container = document.createElement("div"); + const root = createRoot(container); + try { + await act(async () => root.render()); + await act(async () => { visibility.mockReturnValue("hidden"); document.dispatchEvent(new Event("visibilitychange")); }); + log.mockResolvedValue({ content: "", cursor, hasMore: false }); + await act(async () => { visibility.mockReturnValue("visible"); document.dispatchEvent(new Event("visibilitychange")); }); + expect(log).toHaveBeenLastCalledWith("run-cursor", cursor, expect.any(Number)); + expect(container.textContent).toContain("🐙 retained"); + } finally { await act(async () => root.unmount()); } +}); + +it.each([LogViewer, ProductionLogViewer])("marks a segmented tail that begins after earlier output (%#)", async Viewer => { + vi.spyOn(document, "visibilityState", "get").mockReturnValue("visible"); + log.mockResolvedValue({ + content: JSON.stringify({ ts: "t", stream: "stdout", chunk: "recent output", cursor: "120" }) + "\n", + cursor: "160", + hasMore: false, + }); + const run = { id: "run-tail", companyId: "company", agentId: "agent", status: "succeeded", logRef: "log", logStore: "local_segments" } as HeartbeatRun; + const container = document.createElement("div"); + const root = createRoot(container); + try { + await act(async () => root.render()); + expect(log).toHaveBeenCalledWith("run-tail", "tail", expect.any(Number)); + expect(container.textContent).toContain("recent output"); + expect(container.textContent).toContain("Showing recent output; older output is available."); + } finally { await act(async () => root.unmount()); } +}); + it.each([LogViewer, ProductionLogViewer])("retains legacy history and reads only the next offset on visibility recovery (%#)", async (Viewer) => { const visibility = vi.spyOn(document, "visibilityState", "get").mockReturnValue("visible"); const row = (seq: number, chunk: string) => JSON.stringify({ seq, ts: `2026-09-10T12:00:0${seq}Z`, stream: "stdout", chunk }) + "\n"; @@ -92,3 +127,45 @@ it.each([LogViewer, ProductionLogViewer])("polls logs when WebSocket constructio } expect(sockets[0].close).toHaveBeenCalled(); }); + +it.each([LogViewer, ProductionLogViewer])("renders durable events after ignoring websocket commit hints (%#)", async (Viewer) => { + vi.useFakeTimers(); + vi.spyOn(document, "visibilityState", "get").mockReturnValue("visible"); + const sockets: Array<{ + onopen: (() => void) | null; + onmessage: ((message: { data: string }) => void) | null; + close: () => void; + }> = []; + vi.stubGlobal("WebSocket", class { + onopen: (() => void) | null = null; + onmessage: ((message: { data: string }) => void) | null = null; + onclose: (() => void) | null = null; + close() { this.onclose?.(); } + constructor() { sockets.push(this); } + }); + const durableEvent = { + id: "durable-event", companyId: "company-1", runId: "run-1", agentId: "agent-1", + seq: 3, eventEpoch: "epoch", cursor: "e:epoch:3", eventType: "custom.event", + stream: null, level: null, color: null, message: "full durable row", payload: null, + createdAt: new Date(0), + }; + events.mockResolvedValue([durableEvent] as never[]); + log.mockResolvedValue({ content: "", nextOffset: 0 }); + const run = { id: "run-1", companyId: "company-1", agentId: "agent-1", status: "running" } as HeartbeatRun; + const container = document.createElement("div"); + const root = createRoot(container); + try { + await act(async () => root.render()); + await act(async () => sockets[0]?.onopen?.()); + await act(async () => sockets[0]?.onmessage?.({ data: JSON.stringify({ + type: "heartbeat.run.event", companyId: run.companyId, createdAt: new Date(0).toISOString(), + payload: { runId: run.id, id: "durable-event", seq: 3, eventType: "custom.event", message: "commit hint only", commitHint: true }, + }) })); + expect(container.textContent).not.toContain("commit hint only"); + await act(async () => vi.advanceTimersByTimeAsync(2_000)); + expect(events).toHaveBeenCalledWith("run-1", 0, 100); + expect(container.textContent).toContain("full durable row"); + } finally { + await act(async () => root.unmount()); + } +}); diff --git a/ui/src/pages/AgentDetail.production.tsx b/ui/src/pages/AgentDetail.production.tsx index 91efc76433d..e9437a7f185 100644 --- a/ui/src/pages/AgentDetail.production.tsx +++ b/ui/src/pages/AgentDetail.production.tsx @@ -1,6 +1,9 @@ +import { RunOutputDownload } from "@/components/RunOutputDownload"; +import { nextRunLogPosition, runLogHasMore } from "@/lib/run-log-position"; import { AgentCharacter } from "../components/AgentCharacter"; import { characterStateForAgent } from "@paperclipai/shared"; -import { mergeRunLogChunks, readChunkSeq } from "../lib/run-log-chunks"; +import { mergeRunLogChunks, readChunkCursor, readChunkSeq } from "../lib/run-log-chunks"; +import { mergeRunEvents, nextRunEventCursor, retainRunEventTail, type RunEventCursor } from "../lib/run-event-pagination"; import { getPageVisibility, usePageVisibility } from "../lib/page-visibility"; import { useCallback, useEffect, useMemo, useState, useRef } from "react"; import { useParams, useNavigate, Link, Navigate, useBeforeUnload, type NavigateFunction } from "@/lib/router"; @@ -52,6 +55,7 @@ import { Identity } from "../components/Identity"; import { AuditFeed } from "./audit/AuditFeed.production"; import { PageSkeleton } from "../components/PageSkeleton"; import { AgentActionButtons } from "../components/AgentActionButtons"; +import { RunnerInspector } from "../components/RunnerInspector"; import { InlineBanner } from "../components/InlineBanner"; import { BuiltInBundlePanel } from "../components/BuiltInBundlePanel"; import { ConfigureBuiltInAgentModal } from "../components/ConfigureBuiltInAgentModal"; @@ -104,6 +108,7 @@ import { MAX_LIVE_EVENTS, MAX_LIVE_LOG_LINES, } from "../lib/live-log-buffer"; + import { isUuidLike, type Agent, @@ -360,6 +365,8 @@ function setsEqual(left: Set, right: Set) { return true; } +const MAX_RETAINED_RUN_LOG_BYTES = 2 * 1024 * 1024; + function runMetrics(run: HeartbeatRun) { const usage = (run.usageJson ?? null) as Record | null; const result = (run.resultJson ?? null) as Record | null; @@ -388,6 +395,7 @@ function runMetrics(run: HeartbeatRun) { export type RunLogChunk = { seq?: number; + cursor?: string; ts: string; stream: "stdout" | "stderr" | "system"; chunk: string; @@ -529,13 +537,13 @@ function parseStoredLogContent(content: string): RunLogChunk[] { const trimmed = line.trim(); if (!trimmed) continue; try { - const raw = JSON.parse(trimmed) as { ts?: unknown; stream?: unknown; chunk?: unknown }; + const raw = JSON.parse(trimmed) as { ts?: unknown; stream?: unknown; chunk?: unknown; seq?: unknown; cursor?: unknown }; const stream = raw.stream === "stderr" || raw.stream === "system" ? raw.stream : "stdout"; const chunk = typeof raw.chunk === "string" ? raw.chunk : ""; const ts = typeof raw.ts === "string" ? raw.ts : new Date().toISOString(); if (!chunk) continue; - parsed.push({ ts, stream, chunk }); + parsed.push({ ts, stream, chunk, seq: readChunkSeq(raw.seq), cursor: readChunkCursor(raw.cursor) }); } catch { // Ignore malformed log lines. } @@ -3262,6 +3270,7 @@ function RunDetail({ run: initialRun, agentRouteId, adapterType, adapterConfig } }, [run.responsibleUserId, userDirectory]); const responsibleDenialCode = isResponsibleUserDenialCode(run.errorCode) ? run.errorCode : null; const [sessionOpen, setSessionOpen] = useState(false); + const [inspectorOpen, setInspectorOpen] = useState(false); const [claudeLoginResult, setClaudeLoginResult] = useState(null); useEffect(() => { @@ -3710,29 +3719,33 @@ function RunDetail({ run: initialRun, agentRouteId, adapterType, adapterConfig } })()} {/* Log viewer */} - + setInspectorOpen(true)} /> +
); } /* ---- Log Viewer ---- */ -export function LogViewer({ run, adapterType }: { run: HeartbeatRun; adapterType: string }) { +export function LogViewer({ run, adapterType, onViewHistory }: { run: HeartbeatRun; adapterType: string; onViewHistory?: () => void }) { const { visible } = usePageVisibility(); const [events, setEvents] = useState([]); + const [eventHistoryCollapsed, setEventHistoryCollapsed] = useState(false); const [logLines, setLogLines] = useState([]); const [loading, setLoading] = useState(true); const [logLoading, setLogLoading] = useState(!!run.logRef); const [logError, setLogError] = useState(null); - const [logOffset, setLogOffsetState] = useState(0); - const logOffsetRef = useRef(0); - const setLogOffset = useCallback((next: number | ((previous: number) => number)) => { - logOffsetRef.current = typeof next === "function" ? next(logOffsetRef.current) : next; + const initialLogOffset = run.logStore === "local_segments" ? "tail" : 0; + const [logOffset, setLogOffsetState] = useState(initialLogOffset); + const logOffsetRef = useRef(initialLogOffset); + const setLogOffset = useCallback((next: number | string) => { + logOffsetRef.current = next; setLogOffsetState(logOffsetRef.current); }, []); - const logMergeRefs = useRef({ seenChunkKeys: new Set(), trimmedSeqFloorByRun: new Map() }); + const logMergeRefs = useRef({ seenChunkKeys: new Set(), trimmedSeqFloorByRun: new Map(), trimmedCursorFloorByRun: new Map() }); const [hasMoreLog, setHasMoreLog] = useState(false); + const [hasEarlierOutput, setHasEarlierOutput] = useState(false); const [loadingMoreLog, setLoadingMoreLog] = useState(false); const [isFollowing, setIsFollowing] = useState(false); const [isStreamingConnected, setIsStreamingConnected] = useState(false); @@ -3761,7 +3774,11 @@ export function LogViewer({ run, adapterType }: { run: HeartbeatRun; adapterType function appendLogLines(incoming: RunLogChunk[]) { setLogLines((previous) => mergeRunLogChunks(run.id, previous, incoming.map((line) => ({ ...line, dedupeKey: `log:${run.id}:${line.ts}:${line.stream}:${line.chunk}`, - })), logMergeRefs.current, isLive ? MAX_LIVE_LOG_LINES : Number.POSITIVE_INFINITY).chunks); + })), logMergeRefs.current, { + maxChunks: MAX_LIVE_LOG_LINES, + maxBytes: MAX_RETAINED_RUN_LOG_BYTES, + collapseTrimmed: true, + }).chunks); } function appendLogContent(content: string, finalize = false) { @@ -3779,21 +3796,20 @@ export function LogViewer({ run, adapterType }: { run: HeartbeatRun; adapterType const trimmed = line.trim(); if (!trimmed) continue; try { - const raw = JSON.parse(trimmed) as { ts?: unknown; stream?: unknown; chunk?: unknown; seq?: unknown }; + const raw = JSON.parse(trimmed) as { ts?: unknown; stream?: unknown; chunk?: unknown; seq?: unknown; cursor?: unknown }; const stream = raw.stream === "stderr" || raw.stream === "system" ? raw.stream : "stdout"; const chunk = typeof raw.chunk === "string" ? raw.chunk : ""; const ts = typeof raw.ts === "string" ? raw.ts : new Date().toISOString(); if (!chunk) continue; - parsed.push({ ts, stream, chunk, seq: readChunkSeq(raw.seq) }); + parsed.push({ ts, stream, chunk, seq: readChunkSeq(raw.seq), cursor: readChunkCursor(raw.cursor) }); } catch { // ignore malformed lines } } if (parsed.length > 0) { - // Live runs stream forever, so cap the retained tail. Terminated runs are - // paginated by the user via "Load more log" and keep their full history. + if (parsed[0]?.cursor && BigInt(parsed[0].cursor) > 0n) setHasEarlierOutput(true); appendLogLines(parsed); } } @@ -3801,12 +3817,24 @@ export function LogViewer({ run, adapterType }: { run: HeartbeatRun; adapterType // Fetch events const { data: initialEvents } = useQuery({ queryKey: ["run-events", run.id], - queryFn: () => heartbeatsApi.events(run.id, 0, 200), + queryFn: () => heartbeatsApi.events(run.id, "tail", 200), }); + const durableEventCursorRef = useRef(0); + + useEffect(() => { + durableEventCursorRef.current = 0; + setEvents([]); + setEventHistoryCollapsed(false); + }, [run.id]); + useEffect(() => { if (initialEvents) { - setEvents(initialEvents); + const last = initialEvents.at(-1); + if (last) durableEventCursorRef.current = last.cursor ?? last.seq; + const retained = retainRunEventTail(initialEvents); + if (initialEvents[0]?.historyBefore || retained.collapsed) setEventHistoryCollapsed(true); + setEvents((previous) => retainRunEventTail(mergeRunEvents(previous, retained.events, MAX_LIVE_EVENTS)).events); setLoading(false); } }, [initialEvents]); @@ -3895,14 +3923,15 @@ export function LogViewer({ run, adapterType }: { run: HeartbeatRun; adapterType // Reset only when the log source changes, never when visibility changes. useEffect(() => { pendingLogLineRef.current = ""; - logMergeRefs.current = { seenChunkKeys: new Set(), trimmedSeqFloorByRun: new Map() }; + logMergeRefs.current = { seenChunkKeys: new Set(), trimmedSeqFloorByRun: new Map(), trimmedCursorFloorByRun: new Map() }; seenProgressLogLineKeysRef.current = new Set(); setLogLines([]); - setLogOffset(0); + setHasEarlierOutput(false); + setLogOffset(run.logStore === "local_segments" ? "tail" : 0); setHasMoreLog(false); setLoadingMoreLog(false); setLogError(null); - }, [run.id, run.logRef, setLogOffset]); + }, [run.id, run.logRef, run.logStore, setLogOffset]); // Fetch persisted shell log, retaining partial rows and offsets across hides. useEffect(() => { @@ -3921,10 +3950,10 @@ export function LogViewer({ run, adapterType }: { run: HeartbeatRun; adapterType try { const result = await heartbeatsApi.log(run.id, offset, RUN_LOG_PAGE_BYTES); if (cancelled) return; - appendLogContent(result.content, result.nextOffset === undefined); - const next = result.nextOffset ?? offset + result.content.length; + appendLogContent(result.content, !runLogHasMore(result)); + const next = nextRunLogPosition(offset, result); setLogOffset(next); - setHasMoreLog(!shouldPollShellLog && result.nextOffset !== undefined); + setHasMoreLog(!shouldPollShellLog && runLogHasMore(result)); } catch (err) { if (!cancelled) { if (shouldPollShellLog && isRunLogUnavailable(err)) { @@ -3942,7 +3971,7 @@ export function LogViewer({ run, adapterType }: { run: HeartbeatRun; adapterType return () => { cancelled = true; }; - }, [visible, run.id, run.logRef, run.logBytes, shouldPollShellLog]); + }, [visible, run.id, run.logRef, run.logBytes, run.logStore, shouldPollShellLog]); async function loadMorePersistedLog() { if (loadingMoreLog || !hasMoreLog) return; @@ -3950,10 +3979,10 @@ export function LogViewer({ run, adapterType }: { run: HeartbeatRun; adapterType setLogError(null); try { const result = await heartbeatsApi.log(run.id, logOffset, RUN_LOG_PAGE_BYTES); - appendLogContent(result.content, result.nextOffset === undefined); - const next = result.nextOffset ?? logOffset + result.content.length; + appendLogContent(result.content, !runLogHasMore(result)); + const next = nextRunLogPosition(logOffset, result); setLogOffset(next); - setHasMoreLog(result.nextOffset !== undefined); + setHasMoreLog(runLogHasMore(result)); } catch (err) { setLogError(err instanceof Error ? err.message : "Failed to load more run log"); } finally { @@ -3963,18 +3992,24 @@ export function LogViewer({ run, adapterType }: { run: HeartbeatRun; adapterType // Poll for live updates useEffect(() => { - if (!visible || !isLive || isStreamingConnected) return; + if (!visible || !isLive) return; let pending = false; let cancelled = false; const interval = setInterval(async () => { if (pending || cancelled || !getPageVisibility().visible) return; pending = true; - const maxSeq = events.length > 0 ? Math.max(...events.map((e) => e.seq)) : 0; try { - const newEvents = await heartbeatsApi.events(run.id, maxSeq, 100); + const cursor = durableEventCursorRef.current; + const newEvents = await heartbeatsApi.events(run.id, cursor, 100); if (cancelled) return; if (newEvents.length > 0) { - setEvents((prev) => appendCapped(prev, newEvents, MAX_LIVE_EVENTS)); + const nextCursor = nextRunEventCursor(cursor, newEvents); + if (nextCursor !== cursor) durableEventCursorRef.current = nextCursor; + setEvents((prev) => { + const retained = retainRunEventTail(mergeRunEvents(prev, newEvents, MAX_LIVE_EVENTS)); + if (newEvents[0]?.historyBefore || retained.collapsed) setEventHistoryCollapsed(true); + return retained.events; + }); } } catch { // ignore polling errors @@ -3986,7 +4021,7 @@ export function LogViewer({ run, adapterType }: { run: HeartbeatRun; adapterType cancelled = true; clearInterval(interval); }; - }, [visible, run.id, isLive, isStreamingConnected, events]); + }, [visible, run.id, isLive]); // Poll shell log for running runs useEffect(() => { @@ -4000,13 +4035,9 @@ export function LogViewer({ run, adapterType }: { run: HeartbeatRun; adapterType const result = await heartbeatsApi.log(run.id, logOffset, 256_000); if (cancelled) return; if (result.content) { - appendLogContent(result.content, result.nextOffset === undefined); - } - if (result.nextOffset !== undefined) { - setLogOffset(result.nextOffset); - } else if (result.content.length > 0) { - setLogOffset((prev) => prev + result.content.length); + appendLogContent(result.content, !runLogHasMore(result)); } + setLogOffset(nextRunLogPosition(logOffset, result)); } catch (err) { if (isRunLogUnavailable(err)) return; // ignore polling errors @@ -4070,7 +4101,7 @@ export function LogViewer({ run, adapterType }: { run: HeartbeatRun; adapterType const streamRaw = asNonEmptyString(payload.stream); const stream = streamRaw === "stderr" || streamRaw === "system" ? streamRaw : "stdout"; const ts = asNonEmptyString((payload as Record).ts) ?? event.createdAt; - appendLogLines([{ ts, stream, chunk, seq: readChunkSeq(payload.seq) }]); + appendLogLines([{ ts, stream, chunk, seq: readChunkSeq(payload.seq), cursor: readChunkCursor(payload.cursor) }]); return; } @@ -4084,41 +4115,7 @@ export function LogViewer({ run, adapterType }: { run: HeartbeatRun; adapterType return; } - if (event.type !== "heartbeat.run.event") return; - - const seq = typeof payload.seq === "number" ? payload.seq : null; - if (seq === null || !Number.isFinite(seq)) return; - - const streamRaw = asNonEmptyString(payload.stream); - const stream = - streamRaw === "stdout" || streamRaw === "stderr" || streamRaw === "system" - ? streamRaw - : null; - const levelRaw = asNonEmptyString(payload.level); - const level = - levelRaw === "info" || levelRaw === "warn" || levelRaw === "error" - ? levelRaw - : null; - - const liveEvent: HeartbeatRunEvent = { - id: seq, - companyId: run.companyId, - runId: run.id, - agentId: run.agentId, - seq, - eventType: asNonEmptyString(payload.eventType) ?? "event", - stream, - level, - color: asNonEmptyString(payload.color), - message: asNonEmptyString(payload.message), - payload: asRecord(payload.payload), - createdAt: new Date(event.createdAt), - }; - - setEvents((prev) => { - if (prev.some((existing) => existing.seq === seq)) return prev; - return appendCapped(prev, [liveEvent], MAX_LIVE_EVENTS); - }); + if (event.type === "heartbeat.run.event") return; }; socket.onerror = () => { @@ -4271,6 +4268,9 @@ export function LogViewer({ run, adapterType }: { run: HeartbeatRun; adapterType limit={isLive ? LIVE_TRANSCRIPT_RENDER_LIMIT : undefined} emptyMessage={run.logRef ? "Waiting for transcript..." : "No persisted transcript for this run."} /> + {hasEarlierOutput && ( +

Showing recent output; older output is available.

+ )} {hasMoreLog && (
+
+ )}
Events ({events.length})
{events.map((evt) => { @@ -4359,6 +4365,7 @@ export function LogViewer({ run, adapterType }: { run: HeartbeatRun; adapterType ? JSON.stringify(redactPathValue(evt.payload, censorUsernameInLogs)) : ""} +
); })} diff --git a/ui/src/pages/AgentDetail.tsx b/ui/src/pages/AgentDetail.tsx index b6896c6f729..9b32b063121 100644 --- a/ui/src/pages/AgentDetail.tsx +++ b/ui/src/pages/AgentDetail.tsx @@ -1,6 +1,9 @@ +import { RunOutputDownload } from "@/components/RunOutputDownload"; +import { nextRunLogPosition, runLogHasMore } from "@/lib/run-log-position"; import { AgentCharacter } from "../components/AgentCharacter"; import { characterStateForAgent } from "@paperclipai/shared"; -import { mergeRunLogChunks, readChunkSeq } from "../lib/run-log-chunks"; +import { mergeRunLogChunks, readChunkCursor, readChunkSeq } from "../lib/run-log-chunks"; +import { mergeRunEvents, nextRunEventCursor, retainRunEventTail, type RunEventCursor } from "../lib/run-event-pagination"; import { getPageVisibility, usePageVisibility } from "../lib/page-visibility"; import { useCallback, useEffect, useMemo, useState, useRef } from "react"; import { useParams, useNavigate, Link, Navigate, useBeforeUnload, type NavigateFunction } from "@/lib/router"; @@ -101,6 +104,7 @@ import { MAX_LIVE_EVENTS, MAX_LIVE_LOG_LINES, } from "../lib/live-log-buffer"; + import { isUuidLike, type Agent, @@ -360,6 +364,8 @@ function setsEqual(left: Set, right: Set) { return true; } +const MAX_RETAINED_RUN_LOG_BYTES = 2 * 1024 * 1024; + function runMetrics(run: HeartbeatRun) { const usage = (run.usageJson ?? null) as Record | null; const result = (run.resultJson ?? null) as Record | null; @@ -388,6 +394,7 @@ function runMetrics(run: HeartbeatRun) { export type RunLogChunk = { seq?: number; + cursor?: string; ts: string; stream: "stdout" | "stderr" | "system"; chunk: string; @@ -529,13 +536,13 @@ function parseStoredLogContent(content: string): RunLogChunk[] { const trimmed = line.trim(); if (!trimmed) continue; try { - const raw = JSON.parse(trimmed) as { ts?: unknown; stream?: unknown; chunk?: unknown }; + const raw = JSON.parse(trimmed) as { ts?: unknown; stream?: unknown; chunk?: unknown; seq?: unknown; cursor?: unknown }; const stream = raw.stream === "stderr" || raw.stream === "system" ? raw.stream : "stdout"; const chunk = typeof raw.chunk === "string" ? raw.chunk : ""; const ts = typeof raw.ts === "string" ? raw.ts : new Date().toISOString(); if (!chunk) continue; - parsed.push({ ts, stream, chunk }); + parsed.push({ ts, stream, chunk, seq: readChunkSeq(raw.seq), cursor: readChunkCursor(raw.cursor) }); } catch { // Ignore malformed log lines. } @@ -3781,7 +3788,7 @@ function RunDetail({ run: initialRun, agentRouteId, adapterType, adapterConfig } })()} {/* Log viewer */} - + setInspectorOpen(true)} /> void }) { const { visible } = usePageVisibility(); const [events, setEvents] = useState([]); + const [eventHistoryCollapsed, setEventHistoryCollapsed] = useState(false); const [logLines, setLogLines] = useState([]); const [loading, setLoading] = useState(true); const [logLoading, setLogLoading] = useState(!!run.logRef); const [logError, setLogError] = useState(null); - const [logOffset, setLogOffsetState] = useState(0); - const logOffsetRef = useRef(0); - const setLogOffset = useCallback((next: number | ((previous: number) => number)) => { - logOffsetRef.current = typeof next === "function" ? next(logOffsetRef.current) : next; + const initialLogOffset = run.logStore === "local_segments" ? "tail" : 0; + const [logOffset, setLogOffsetState] = useState(initialLogOffset); + const logOffsetRef = useRef(initialLogOffset); + const setLogOffset = useCallback((next: number | string) => { + logOffsetRef.current = next; setLogOffsetState(logOffsetRef.current); }, []); - const logMergeRefs = useRef({ seenChunkKeys: new Set(), trimmedSeqFloorByRun: new Map() }); + const logMergeRefs = useRef({ seenChunkKeys: new Set(), trimmedSeqFloorByRun: new Map(), trimmedCursorFloorByRun: new Map() }); const [hasMoreLog, setHasMoreLog] = useState(false); + const [hasEarlierOutput, setHasEarlierOutput] = useState(false); const [loadingMoreLog, setLoadingMoreLog] = useState(false); const [isFollowing, setIsFollowing] = useState(false); const [isStreamingConnected, setIsStreamingConnected] = useState(false); @@ -3843,7 +3853,11 @@ export function LogViewer({ run, adapterType }: { run: HeartbeatRun; adapterType function appendLogLines(incoming: RunLogChunk[]) { setLogLines((previous) => mergeRunLogChunks(run.id, previous, incoming.map((line) => ({ ...line, dedupeKey: `log:${run.id}:${line.ts}:${line.stream}:${line.chunk}`, - })), logMergeRefs.current, isLive ? MAX_LIVE_LOG_LINES : Number.POSITIVE_INFINITY).chunks); + })), logMergeRefs.current, { + maxChunks: MAX_LIVE_LOG_LINES, + maxBytes: MAX_RETAINED_RUN_LOG_BYTES, + collapseTrimmed: true, + }).chunks); } function appendLogContent(content: string, finalize = false) { @@ -3861,21 +3875,20 @@ export function LogViewer({ run, adapterType }: { run: HeartbeatRun; adapterType const trimmed = line.trim(); if (!trimmed) continue; try { - const raw = JSON.parse(trimmed) as { ts?: unknown; stream?: unknown; chunk?: unknown; seq?: unknown }; + const raw = JSON.parse(trimmed) as { ts?: unknown; stream?: unknown; chunk?: unknown; seq?: unknown; cursor?: unknown }; const stream = raw.stream === "stderr" || raw.stream === "system" ? raw.stream : "stdout"; const chunk = typeof raw.chunk === "string" ? raw.chunk : ""; const ts = typeof raw.ts === "string" ? raw.ts : new Date().toISOString(); if (!chunk) continue; - parsed.push({ ts, stream, chunk, seq: readChunkSeq(raw.seq) }); + parsed.push({ ts, stream, chunk, seq: readChunkSeq(raw.seq), cursor: readChunkCursor(raw.cursor) }); } catch { // ignore malformed lines } } if (parsed.length > 0) { - // Live runs stream forever, so cap the retained tail. Terminated runs are - // paginated by the user via "Load more log" and keep their full history. + if (parsed[0]?.cursor && BigInt(parsed[0].cursor) > 0n) setHasEarlierOutput(true); appendLogLines(parsed); } } @@ -3883,12 +3896,24 @@ export function LogViewer({ run, adapterType }: { run: HeartbeatRun; adapterType // Fetch events const { data: initialEvents } = useQuery({ queryKey: ["run-events", run.id], - queryFn: () => heartbeatsApi.events(run.id, 0, 200), + queryFn: () => heartbeatsApi.events(run.id, "tail", 200), }); + const durableEventCursorRef = useRef(0); + + useEffect(() => { + durableEventCursorRef.current = 0; + setEvents([]); + setEventHistoryCollapsed(false); + }, [run.id]); + useEffect(() => { if (initialEvents) { - setEvents(initialEvents); + const last = initialEvents.at(-1); + if (last) durableEventCursorRef.current = last.cursor ?? last.seq; + const retained = retainRunEventTail(initialEvents); + if (initialEvents[0]?.historyBefore || retained.collapsed) setEventHistoryCollapsed(true); + setEvents((previous) => retainRunEventTail(mergeRunEvents(previous, retained.events, MAX_LIVE_EVENTS)).events); setLoading(false); } }, [initialEvents]); @@ -3977,14 +4002,15 @@ export function LogViewer({ run, adapterType }: { run: HeartbeatRun; adapterType // Reset only when the log source changes, never when visibility changes. useEffect(() => { pendingLogLineRef.current = ""; - logMergeRefs.current = { seenChunkKeys: new Set(), trimmedSeqFloorByRun: new Map() }; + logMergeRefs.current = { seenChunkKeys: new Set(), trimmedSeqFloorByRun: new Map(), trimmedCursorFloorByRun: new Map() }; seenProgressLogLineKeysRef.current = new Set(); setLogLines([]); - setLogOffset(0); + setHasEarlierOutput(false); + setLogOffset(run.logStore === "local_segments" ? "tail" : 0); setHasMoreLog(false); setLoadingMoreLog(false); setLogError(null); - }, [run.id, run.logRef, setLogOffset]); + }, [run.id, run.logRef, run.logStore, setLogOffset]); // Fetch persisted shell log, retaining partial rows and offsets across hides. useEffect(() => { @@ -4003,10 +4029,10 @@ export function LogViewer({ run, adapterType }: { run: HeartbeatRun; adapterType try { const result = await heartbeatsApi.log(run.id, offset, RUN_LOG_PAGE_BYTES); if (cancelled) return; - appendLogContent(result.content, result.nextOffset === undefined); - const next = result.nextOffset ?? offset + result.content.length; + appendLogContent(result.content, !runLogHasMore(result)); + const next = nextRunLogPosition(offset, result); setLogOffset(next); - setHasMoreLog(!shouldPollShellLog && result.nextOffset !== undefined); + setHasMoreLog(!shouldPollShellLog && runLogHasMore(result)); } catch (err) { if (!cancelled) { if (shouldPollShellLog && isRunLogUnavailable(err)) { @@ -4024,7 +4050,7 @@ export function LogViewer({ run, adapterType }: { run: HeartbeatRun; adapterType return () => { cancelled = true; }; - }, [visible, run.id, run.logRef, run.logBytes, shouldPollShellLog]); + }, [visible, run.id, run.logRef, run.logBytes, run.logStore, shouldPollShellLog]); async function loadMorePersistedLog() { if (loadingMoreLog || !hasMoreLog) return; @@ -4032,10 +4058,10 @@ export function LogViewer({ run, adapterType }: { run: HeartbeatRun; adapterType setLogError(null); try { const result = await heartbeatsApi.log(run.id, logOffset, RUN_LOG_PAGE_BYTES); - appendLogContent(result.content, result.nextOffset === undefined); - const next = result.nextOffset ?? logOffset + result.content.length; + appendLogContent(result.content, !runLogHasMore(result)); + const next = nextRunLogPosition(logOffset, result); setLogOffset(next); - setHasMoreLog(result.nextOffset !== undefined); + setHasMoreLog(runLogHasMore(result)); } catch (err) { setLogError(err instanceof Error ? err.message : "Failed to load more run log"); } finally { @@ -4045,18 +4071,24 @@ export function LogViewer({ run, adapterType }: { run: HeartbeatRun; adapterType // Poll for live updates useEffect(() => { - if (!visible || !isLive || isStreamingConnected) return; + if (!visible || !isLive) return; let pending = false; let cancelled = false; const interval = setInterval(async () => { if (pending || cancelled || !getPageVisibility().visible) return; pending = true; - const maxSeq = events.length > 0 ? Math.max(...events.map((e) => e.seq)) : 0; try { - const newEvents = await heartbeatsApi.events(run.id, maxSeq, 100); + const cursor = durableEventCursorRef.current; + const newEvents = await heartbeatsApi.events(run.id, cursor, 100); if (cancelled) return; if (newEvents.length > 0) { - setEvents((prev) => appendCapped(prev, newEvents, MAX_LIVE_EVENTS)); + const nextCursor = nextRunEventCursor(cursor, newEvents); + if (nextCursor !== cursor) durableEventCursorRef.current = nextCursor; + setEvents((prev) => { + const retained = retainRunEventTail(mergeRunEvents(prev, newEvents, MAX_LIVE_EVENTS)); + if (newEvents[0]?.historyBefore || retained.collapsed) setEventHistoryCollapsed(true); + return retained.events; + }); } } catch { // ignore polling errors @@ -4068,7 +4100,7 @@ export function LogViewer({ run, adapterType }: { run: HeartbeatRun; adapterType cancelled = true; clearInterval(interval); }; - }, [visible, run.id, isLive, isStreamingConnected, events]); + }, [visible, run.id, isLive]); // Poll shell log for running runs useEffect(() => { @@ -4082,13 +4114,9 @@ export function LogViewer({ run, adapterType }: { run: HeartbeatRun; adapterType const result = await heartbeatsApi.log(run.id, logOffset, 256_000); if (cancelled) return; if (result.content) { - appendLogContent(result.content, result.nextOffset === undefined); - } - if (result.nextOffset !== undefined) { - setLogOffset(result.nextOffset); - } else if (result.content.length > 0) { - setLogOffset((prev) => prev + result.content.length); + appendLogContent(result.content, !runLogHasMore(result)); } + setLogOffset(nextRunLogPosition(logOffset, result)); } catch (err) { if (isRunLogUnavailable(err)) return; // ignore polling errors @@ -4152,7 +4180,7 @@ export function LogViewer({ run, adapterType }: { run: HeartbeatRun; adapterType const streamRaw = asNonEmptyString(payload.stream); const stream = streamRaw === "stderr" || streamRaw === "system" ? streamRaw : "stdout"; const ts = asNonEmptyString((payload as Record).ts) ?? event.createdAt; - appendLogLines([{ ts, stream, chunk, seq: readChunkSeq(payload.seq) }]); + appendLogLines([{ ts, stream, chunk, seq: readChunkSeq(payload.seq), cursor: readChunkCursor(payload.cursor) }]); return; } @@ -4166,41 +4194,7 @@ export function LogViewer({ run, adapterType }: { run: HeartbeatRun; adapterType return; } - if (event.type !== "heartbeat.run.event") return; - - const seq = typeof payload.seq === "number" ? payload.seq : null; - if (seq === null || !Number.isFinite(seq)) return; - - const streamRaw = asNonEmptyString(payload.stream); - const stream = - streamRaw === "stdout" || streamRaw === "stderr" || streamRaw === "system" - ? streamRaw - : null; - const levelRaw = asNonEmptyString(payload.level); - const level = - levelRaw === "info" || levelRaw === "warn" || levelRaw === "error" - ? levelRaw - : null; - - const liveEvent: HeartbeatRunEvent = { - id: seq, - companyId: run.companyId, - runId: run.id, - agentId: run.agentId, - seq, - eventType: asNonEmptyString(payload.eventType) ?? "event", - stream, - level, - color: asNonEmptyString(payload.color), - message: asNonEmptyString(payload.message), - payload: asRecord(payload.payload), - createdAt: new Date(event.createdAt), - }; - - setEvents((prev) => { - if (prev.some((existing) => existing.seq === seq)) return prev; - return appendCapped(prev, [liveEvent], MAX_LIVE_EVENTS); - }); + if (event.type === "heartbeat.run.event") return; }; socket.onerror = () => { @@ -4353,6 +4347,9 @@ export function LogViewer({ run, adapterType }: { run: HeartbeatRun; adapterType limit={isLive ? LIVE_TRANSCRIPT_RENDER_LIMIT : undefined} emptyMessage={run.logRef ? "Waiting for transcript..." : "No persisted transcript for this run."} /> + {hasEarlierOutput && ( +

Showing recent output; older output is available.

+ )} {hasMoreLog && (
+
+ )}
Events ({events.length})
{events.map((evt) => { @@ -4441,6 +4444,7 @@ export function LogViewer({ run, adapterType }: { run: HeartbeatRun; adapterType ? JSON.stringify(redactPathValue(evt.payload, censorUsernameInLogs)) : ""} +
); })}