diff --git a/doc/run-log-events.md b/doc/run-log-events.md index 5615b6851a4..be2c7c8d1e5 100644 --- a/doc/run-log-events.md +++ b/doc/run-log-events.md @@ -1024,6 +1024,29 @@ comments one run read, with each read's time and the comment revision it saw, whatever the run's outcome. A failure to write the event leaves the comment owed in the same way. +When a run reads a finished child task, the server writes one +`child_task.read` event (`stream: "system"`, level `info`) to that run's log. +The reads that count are `get_task_context` on the parent task, and a +successful agent request with a run token for `GET /api/issues/:id`, +`GET /api/issues/:id/documents` or `GET /api/issues/:id/documents/:key` of a +child in `done`. The route writes the event only after the requested content +has loaded, so a 400 or 404 records nothing. The payload has `parentIssueId` +and `children`, which maps each finished child ID to the revision the run +received: its `status`, `updatedAt` and `statusDecisionId`. It contains no +titles, summaries or document bodies. The event creates no Telemetry or +OpenTelemetry export. + +`hasPendingNativeChildCompletion` reads this event. A child-completion wake no +longer keeps the parent's done claim open when the finishing run received that +child in `done` at the completion the wake reports: the received +`statusDecisionId` is the wake's completion decision, or the received +`updatedAt` is at or after that decision was saved. The event's own write time +does not count, so a delayed write of an older revision cannot deliver a newer +completion. The wake is then cancelled at release instead of starting another +parent turn. A read while the child was still open, a read of an older +revision, a read by another run, or a failure to write the event leaves the +child result owed. + ## Managed Agent File Save Receipts The server writes `instruction_save` after managed file collection or a warm diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index ecb8058cfb0..b3b03482d04 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -265,6 +265,7 @@ import { } from "../services/native-runtime/native-restart-recovery.ts"; import { claimNativeSessionResumptions } from "../services/native-runtime/native-finalization-reconciler.ts"; import { PaperclipControlPlanePort } from "../services/native-runtime/paperclip-control-plane-port.js"; +import { PaperclipRunnerToolAuthority } from "../services/native-runtime/paperclip-runner-tool-authority.js"; import { finalizeNativeRun } from "../services/native-runtime/native-run-finalizer.js"; import { createWakeQueue } from "../modules/wake-queue/index.js"; import { recordNativeAttentionAssessment } from "../services/native-runtime/work-assessments.js"; @@ -15479,11 +15480,15 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { }); it.each([ - ["before", "done", "completion_claim_policy_accepted"], - ["after", "in_progress", "native_child_completion_pending"], - ] as const)("treats a child result saved %s the parent run started as read when the run finishes the task", async ( - savedRelativeToStart, expectedStatus, expectedReason, - ) => { + { saved: "before", read: "none", delivered: true }, + { saved: "after", read: "none", delivered: false }, + { saved: "after", read: "after the child finished", delivered: true }, + { saved: "after", read: "while the child was open", delivered: false }, + ] as const)("counts a child result as read by the finishing parent run (saved $saved the run started, read: $read)", async ({ + saved: savedRelativeToStart, read, delivered, + }) => { + const expectedStatus = delivered ? "done" : "in_progress"; + const expectedReason = delivered ? "completion_claim_policy_accepted" : "native_child_completion_pending"; const fixture = await seedStrandedIssueFixture({ status: "in_progress", runStatus: "succeeded", livenessState: "advanced" }); const { companyId, agentId, issueId, runId } = fixture; const childId = randomUUID(); @@ -15516,6 +15521,17 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { startedAt: runStartedAt, finishedAt: new Date(), }).where(eq(heartbeatRuns.id, runId)); await db.update(issues).set({ executionRunId: runId, executionLockedAt: runStartedAt }).where(eq(issues.id, issueId)); + if (read !== "none") { + if (read === "while the child was open") await db.update(issues).set({ status: "in_progress" }).where(eq(issues.id, childId)); + await db.update(heartbeatRuns).set({ status: "running", finishedAt: null }).where(eq(heartbeatRuns.id, runId)); + const reader = new PaperclipRunnerToolAuthority(db, { companyId, agentId, issueId, runId }); + try { + const context = await reader.execute({ tool: "get_task_context", callId: `context-${runId}`, arguments: {} }); + expect((context as { childTasks: Array<{ id: string }> }).childTasks.map(task => task.id)).toEqual([childId]); + } finally { reader.close(); } + await db.update(issues).set({ status: "done" }).where(eq(issues.id, childId)); + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, runId)); + } await db.insert(issueComments).values({ companyId, issueId, authorType: "agent", authorAgentId: agentId, createdByRunId: runId, body: "Both children are done; merged their ledgers. Task complete." }); await new PaperclipControlPlanePort(db, { @@ -15550,7 +15566,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { .toEqual([expect.objectContaining({ status: expectedStatus })]); const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId)); const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, childResult!.id)); - if (savedRelativeToStart === "before") { + if (delivered) { expect(wake).toMatchObject({ status: "cancelled", runId: null }); expect(runs).toHaveLength(1); } else { diff --git a/server/src/__tests__/native-child-task-read-routes.test.ts b/server/src/__tests__/native-child-task-read-routes.test.ts new file mode 100644 index 00000000000..36fde389b0a --- /dev/null +++ b/server/src/__tests__/native-child-task-read-routes.test.ts @@ -0,0 +1,127 @@ +import { randomUUID } from "node:crypto"; +import express from "express"; +import request from "supertest"; +import { and, eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { agentWakeupRequests, agents, companies, createDb, heartbeatRunEvents, heartbeatRuns, issues } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { issueRoutes } from "../routes/issues.js"; +import { errorHandler } from "../middleware/index.js"; +import { documentService } from "../services/documents.js"; +import { CHILD_TASK_READ_EVENT, hasPendingNativeChildCompletion } from "../services/native-runtime/native-child-completion-delivery.js"; + +const support = await getEmbeddedPostgresTestSupport(); +const describePostgres = support.supported ? describe : describe.skip; + +describePostgres("a parent run reading a finished child through the issue routes", () => { + let db: ReturnType; + let database: Awaited>; + const companyId = randomUUID(); + const agentId = randomUUID(); + const otherCompanyId = randomUUID(); + const otherAgentId = randomUUID(); + const otherRunId = randomUUID(); + + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("paperclip-native-child-read-routes-"); + db = createDb(database.connectionString); + await db.insert(companies).values({ id: companyId, name: "Paperclip", issuePrefix: "CHR", defaultResponsibleUserId: "board-user", + requireBoardApprovalForNewAgents: false }); + await db.insert(agents).values({ id: agentId, companyId, name: "Coder", role: "engineer", adapterType: "codex_local", status: "active" }); + await db.insert(companies).values({ id: otherCompanyId, name: "Other", issuePrefix: "OTH", defaultResponsibleUserId: "board-user", + requireBoardApprovalForNewAgents: false }); + await db.insert(agents).values({ id: otherAgentId, companyId: otherCompanyId, name: "Outsider", role: "engineer", + adapterType: "codex_local", status: "active" }); + await db.insert(heartbeatRuns).values({ id: otherRunId, companyId: otherCompanyId, agentId: otherAgentId, status: "running", + runtimeMode: "native", startedAt: new Date(), contextSnapshot: {} }); + }, 30_000); + afterAll(async () => { await database?.cleanup(); }); + + function app(actor: Express.Request["actor"]) { + const instance = express(); + instance.use(express.json()); + instance.use((req, _res, next) => { req.actor = actor; next(); }); + instance.use("/api", issueRoutes(db, {} as never)); + instance.use(errorHandler); + return instance; + } + const agentActor = (runId: string) => ({ type: "agent", companyId, agentId, runId, source: "agent_jwt", + onBehalfOfUserId: null, onBehalfOfMemberships: [] }) as Express.Request["actor"]; + const boardActor = { type: "board", userId: "board-user", source: "local_implicit", isInstanceAdmin: true } as Express.Request["actor"]; + + async function seed(childStatus: "done" | "in_progress", issueNumber: number, options: { childRevisionAgeMs?: number; wakeAgeMs?: number } = {}) { + const runId = randomUUID(); + const childId = randomUUID(); + const parentId = randomUUID(); + await db.insert(issues).values({ id: parentId, companyId, title: "Merge the ledgers", identifier: `CHR-${issueNumber + 100}`, + issueNumber: issueNumber + 100, status: "in_progress", assigneeAgentId: agentId }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running", runtimeMode: "native", + nativeIssueId: parentId, startedAt: new Date(Date.now() - 60_000), contextSnapshot: { issueId: parentId } }); + await db.insert(issues).values({ id: childId, companyId, parentId, title: "Child ledger", identifier: `CHR-${issueNumber}`, + issueNumber, status: childStatus, assigneeAgentId: agentId, + ...(options.childRevisionAgeMs ? { updatedAt: new Date(Date.now() - options.childRevisionAgeMs) } : {}) }); + await documentService(db).upsertIssueDocument({ issueId: childId, key: "ledger", format: "markdown", body: "ROUND 1" }); + if (options.childRevisionAgeMs) await db.update(issues).set({ updatedAt: new Date(Date.now() - options.childRevisionAgeMs) }) + .where(eq(issues.id, childId)); + const [wake] = await db.insert(agentWakeupRequests).values({ companyId, agentId, source: "automation", triggerDetail: "system", + reason: "issue_execution_deferred", requestedByActorType: "system", requestedByActorId: `native-status-wake-dispatch:${randomUUID()}`, + status: "deferred_issue_execution", createdAt: new Date(Date.now() - (options.wakeAgeMs ?? 30_000)), + payload: { issueId: parentId, _paperclipWakeContext: { issueId: parentId, wakeReason: "issue_children_completed", + nativeChildCompletionDecisionId: randomUUID(), completedChildIssueId: childId } } }).returning(); + return { runId, childId, parentId, wakeId: wake!.id }; + } + const reads = (runId: string) => db.select().from(heartbeatRunEvents) + .where(and(eq(heartbeatRunEvents.runId, runId), eq(heartbeatRunEvents.eventType, CHILD_TASK_READ_EVENT))); + const pending = (f: { runId: string; parentId: string }) => + hasPendingNativeChildCompletion(db, { companyId, issueId: f.parentId, agentId, runId: f.runId, sourceIntentId: null }); + + it.each(["", "/documents", "/documents/ledger"])("records the parent run's read of a finished child (GET /issues/:id%s)", async (suffix) => { + const f = await seed("done", 10 + suffix.length); + expect(await pending(f)).toBe(true); + expect((await request(app(agentActor(f.runId))).get(`/api/issues/${f.childId}${suffix}`)).status).toBe(200); + expect(await reads(f.runId)).toEqual([expect.objectContaining({ payload: { + parentIssueId: f.parentId, children: { [f.childId]: { status: "done", updatedAt: expect.any(String), statusDecisionId: null } } } })]); + expect(await pending(f)).toBe(false); + }); + + it("does not count a read of a child that is still open, or a board read", async () => { + const open = await seed("in_progress", 40); + expect((await request(app(agentActor(open.runId))).get(`/api/issues/${open.childId}`)).status).toBe(200); + expect(await reads(open.runId)).toEqual([]); + expect(await pending(open)).toBe(true); + + const board = await seed("done", 41); + expect((await request(app(boardActor)).get(`/api/issues/${board.childId}`)).status).toBe(200); + expect(await reads(board.runId)).toEqual([]); + expect(await pending(board)).toBe(true); + }); + + it("does not record a read by another company's agent", async () => { + const f = await seed("done", 42); + const outsider = { type: "agent", companyId: otherCompanyId, agentId: otherAgentId, runId: otherRunId, source: "agent_jwt", + onBehalfOfUserId: null, onBehalfOfMemberships: [] } as Express.Request["actor"]; + for (const suffix of ["", "/documents", "/documents/ledger"]) { + expect((await request(app(outsider)).get(`/api/issues/${f.childId}${suffix}`)).status).not.toBe(200); + } + expect(await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.eventType, CHILD_TASK_READ_EVENT)) + .then(rows => rows.filter(row => row.runId === otherRunId || row.companyId === otherCompanyId))).toEqual([]); + expect(await pending(f)).toBe(true); + }); + + it.each([ + ["an invalid document key", "/documents/Not A Key!", 400], + ["a missing document", "/documents/missing", 404], + ])("does not count a failed request for %s as a read", async (_shape, suffix, status) => { + const f = await seed("done", 43 + suffix.length); + expect((await request(app(agentActor(f.runId))).get(`/api/issues/${f.childId}${encodeURI(suffix)}`)).status).toBe(status); + expect(await reads(f.runId)).toEqual([]); + expect(await pending(f)).toBe(true); + }); + + it("does not let a read of an older finished revision deliver a newer completion", async () => { + const f = await seed("done", 70, { childRevisionAgeMs: 60_000, wakeAgeMs: 10_000 }); + expect((await request(app(agentActor(f.runId))).get(`/api/issues/${f.childId}`)).status).toBe(200); + expect(await reads(f.runId)).toHaveLength(1); + expect(await pending(f)).toBe(true); + }); +}); diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.ts index ce82b68095b..f4c944e3d5b 100644 --- a/server/src/routes/issues.ts +++ b/server/src/routes/issues.ts @@ -24,6 +24,7 @@ import { listIssueStatusBlockerAttention } from "../services/issue-status-presen import { assertNoCaseHandoffMutation } from "../services/heartbeat/case-handoff-lifecycle.js"; import { assertHandedOffRunTaskOwnership } from "../run-task-ownership.js"; import { recordNativeHumanSelfHandoff } from "../services/native-runtime/native-human-self-handoff.js"; +import { recordChildTaskRead } from "../services/native-runtime/native-child-completion-delivery.js"; import { runRetainsIssueIdentity } from "../services/task-repository-acquisition.js"; import { withConversationReadState } from "../services/agent-chat-read-state.js"; import { markIssueReadSchema } from "@paperclipai/shared"; @@ -3611,6 +3612,13 @@ function logIssueListRequest(input: { }); } +async function recordAgentChildTaskRead(db: Db, req: Request, issue: { id: string; companyId: string; parentId: string | null; status: string; updatedAt: Date; lastStatusDecisionId?: string | null }) { + if (req.actor.type !== "agent" || !req.actor.agentId || !req.actor.runId || !issue.parentId || issue.status !== "done") return; + await recordChildTaskRead(db, { companyId: issue.companyId, agentId: req.actor.agentId, runId: req.actor.runId, + parentIssueId: issue.parentId, children: [issue] }) + .catch(error => logger.warn({ err: error, runId: req.actor.runId }, "could not record the child task a run read")); +} + export function issueRoutes( db: Db, storage: StorageService, @@ -10079,6 +10087,7 @@ export function issueRoutes( // and related-task enrichment. This never refreshes external providers. if (req.query.view === "core") { const core = await readIssueCore(issue, timing.time); + await recordAgentChildTaskRead(db, req, issue); res.setHeader("Server-Timing", timing.header()); res.json(core); return; @@ -10160,6 +10169,7 @@ export function issueRoutes( const visibleMentionedProjects = mentionedProjects.filter((_, index) => mentionedProjectVisibility[index]); // Recovery revalidation may change the blocker; read it afterwards. const executionBlocker = await timing.time("execution_blocker", () => getExecutionBlocker(db, issue.companyId, issue.id)); + await recordAgentChildTaskRead(db, req, issue); res.setHeader("Server-Timing", timing.header()); res.json({ ...issue, @@ -11366,6 +11376,7 @@ export function issueRoutes( const docs = await documentsSvc.listIssueDocuments(issue.id, { includeSystem: req.query.includeSystem === "true", }); + await recordAgentChildTaskRead(db, req, issue); res.json(docs); }); @@ -11399,6 +11410,7 @@ export function issueRoutes( res.status(404).json({ error: "Document not found" }); return; } + await recordAgentChildTaskRead(db, req, issue); if (!shouldIncludeDocumentAnnotations(req)) { res.json(doc); return; diff --git a/server/src/services/native-runtime/native-child-completion-delivery.ts b/server/src/services/native-runtime/native-child-completion-delivery.ts index 2d0b79f1160..0ae2377fd5c 100644 --- a/server/src/services/native-runtime/native-child-completion-delivery.ts +++ b/server/src/services/native-runtime/native-child-completion-delivery.ts @@ -1,5 +1,6 @@ import { and, eq, inArray, isNull, ne, or, sql } from "drizzle-orm"; -import { agentWakeupRequests, heartbeatRuns, type Db } from "@paperclipai/db"; +import { agentWakeupRequests, heartbeatRunEvents, heartbeatRuns, statusDecisions, type Db } from "@paperclipai/db"; +import { appendHeartbeatRunEvent } from "../heartbeat-run-events.js"; export interface NativeChildCompletionRecipient { companyId: string; @@ -9,8 +10,25 @@ export interface NativeChildCompletionRecipient { sourceIntentId: string | null; } +export const CHILD_TASK_READ_EVENT = "child_task.read"; + +export async function recordChildTaskRead(db: Db, input: { + companyId: string; agentId: string; runId: string; parentIssueId: string; + children: ReadonlyArray<{ id: string; parentId: string | null; status: string; updatedAt: Date; lastStatusDecisionId?: string | null }>; +}): Promise { + const completed = input.children.filter(child => child.parentId === input.parentIssueId && child.status === "done"); + if (completed.length === 0) return; + await appendHeartbeatRunEvent(db, { companyId: input.companyId, runId: input.runId, agentId: input.agentId, + eventType: CHILD_TASK_READ_EVENT, stream: "system", level: "info", + payload: { parentIssueId: input.parentIssueId, + children: Object.fromEntries(completed.map(child => [child.id, { status: child.status, updatedAt: child.updatedAt.toISOString(), + statusDecisionId: child.lastStatusDecisionId ?? null }])) } }); +} + /** New child results belong to a fresh parent turn, including across a Done claim. */ export async function hasPendingNativeChildCompletion(db: Db, input: NativeChildCompletionRecipient) { + const completedChildId = sql`coalesce(${agentWakeupRequests.payload}->>'completedChildIssueId', + ${agentWakeupRequests.payload}->'_paperclipWakeContext'->>'completedChildIssueId')`; const [pending] = await db.select({ id: agentWakeupRequests.id }).from(agentWakeupRequests).where(and( eq(agentWakeupRequests.companyId, input.companyId), eq(agentWakeupRequests.agentId, input.agentId), @@ -26,6 +44,18 @@ export async function hasPendingNativeChildCompletion(db: Db, input: NativeChild or(isNull(agentWakeupRequests.runId), ne(agentWakeupRequests.runId, input.runId)), sql`${agentWakeupRequests.createdAt} >= coalesce((select ${heartbeatRuns.startedAt} from ${heartbeatRuns} where ${heartbeatRuns.id} = ${input.runId} and ${heartbeatRuns.companyId} = ${input.companyId}), '-infinity'::timestamptz)`, + sql`not exists (select 1 from ${heartbeatRunEvents} child_read + where child_read.company_id = ${input.companyId} and child_read.run_id = ${input.runId} + and child_read.event_type = ${CHILD_TASK_READ_EVENT} + and child_read.payload->>'parentIssueId' = ${input.issueId} + and child_read.payload->'children'->(${completedChildId})->>'status' = 'done' + and (child_read.payload->'children'->(${completedChildId})->>'statusDecisionId' + = ${agentWakeupRequests.payload}->'_paperclipWakeContext'->>'nativeChildCompletionDecisionId' + or (child_read.payload->'children'->(${completedChildId})->>'updatedAt')::timestamptz + >= date_trunc('milliseconds', coalesce((select ${statusDecisions.createdAt} from ${statusDecisions} + where ${statusDecisions.companyId} = ${input.companyId} + and ${statusDecisions.id}::text = ${agentWakeupRequests.payload}->'_paperclipWakeContext'->>'nativeChildCompletionDecisionId'), + ${agentWakeupRequests.createdAt}))))`, input.sourceIntentId ? ne(agentWakeupRequests.id, input.sourceIntentId) : undefined, )).limit(1); return Boolean(pending); diff --git a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts index 96cbf77c25f..9e6495fdc06 100644 --- a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts +++ b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts @@ -139,6 +139,7 @@ import { type NativeChatAttachmentReadScope, } from "./chat-attachment-read.js"; import { recordTaskHistoryRead } from "../queued-comment-delivery.js"; +import { recordChildTaskRead } from "./native-child-completion-delivery.js"; import { logger } from "../../middleware/logger.js"; const EXTERNAL_WORKSPACE_OPERATIONS = new Set(["get_workspace", "list_workspaces", "select_workspace", "prepare_repository", "register_deliverable"]); @@ -947,6 +948,9 @@ export class PaperclipRunnerToolAuthority { const readableChildren = this.binding.museRuntime ? await this.#readableMuseTaskIds(context.run, childTasks.map(task => task.id)) : null; const visibleChildTasks = readableChildren ? childTasks.filter(task => readableChildren.has(task.id)) : childTasks; + await recordChildTaskRead(this.db, { companyId: this.binding.companyId, agentId: this.binding.agentId, + runId: this.binding.runId, parentIssueId: this.binding.issueId, children: visibleChildTasks.slice(0, 100) }).catch(error => + logger.warn({ err: error, runId: this.binding.runId }, "could not record the child tasks a run read")); return { company: { id: this.binding.companyId }, actor: redactedActor(context.actor),