From 98b0c0a728935c976dcf6d35f685570f7d7d76a4 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 11 Oct 2026 06:23:19 -0500 Subject: [PATCH] refactor(heartbeat): extract task context preparation Co-Authored-By: Paperclip --- doc/DEVELOPING.md | 13 +- .../__tests__/heartbeat-task-context.test.ts | 177 +++++++++ server/src/services/heartbeat.ts | 343 +--------------- server/src/services/heartbeat/task-context.ts | 376 ++++++++++++++++++ tests/e2e/workspace-bootstrap/README.md | 4 +- tests/e2e/workspace-bootstrap/worker.mjs | 10 + 6 files changed, 587 insertions(+), 336 deletions(-) create mode 100644 server/src/__tests__/heartbeat-task-context.test.ts create mode 100644 server/src/services/heartbeat/task-context.ts diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 3425955fed7..5a44fc58328 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -1250,9 +1250,20 @@ When a workspace service runs Paperclip for browser OAuth QA, configure its `exp Keep relevant heartbeat extractions and their focused tests in `server/src/services/heartbeat/`. Task assignment Markdown is rendered by `server/src/services/heartbeat/task-markdown.ts`. -`heartbeat.ts` calls the renderer and re-exports it for existing callers. Keep prompt +`heartbeat.ts` re-exports it for existing callers. Keep prompt formatting changes in the renderer and its tests, separate from run orchestration. +Task context assembly is in `server/src/services/heartbeat/task-context.ts`. +`prepareHeartbeatTaskContext` rebuilds prompt fields on the current run context: +continuation summaries, wake comments, task plans, communication guidance, and +full/compact assignment Markdown. It preserves quarantine and secret redaction +before computing turn-context revisions. The executor supplies the already-loaded +issue, session, and trust policy, and retains admission, resource ownership, and +cleanup. Public-path characterization tests are in +`server/src/__tests__/heartbeat-task-context.test.ts`; the workspace-bootstrap +browser fixture also checks the persisted prompt variants and description revision +from a real process adapter before completing its task. + Run-log formatting is in `server/src/services/heartbeat/run-log.ts`. It bounds stored event payloads, redacts and shortens log chunks, and caps stdout/stderr excerpts. `heartbeat.ts` keeps event writes, current-user redaction, and live diff --git a/server/src/__tests__/heartbeat-task-context.test.ts b/server/src/__tests__/heartbeat-task-context.test.ts new file mode 100644 index 00000000000..bca00d2a403 --- /dev/null +++ b/server/src/__tests__/heartbeat-task-context.test.ts @@ -0,0 +1,177 @@ +import { createHash, randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { agents, companies, createDb, heartbeatRuns, issueComments, issues } from "@paperclipai/db"; +import { FAST_RESPONSE_AGENT_GUIDANCE } from "@paperclipai/shared"; +import { registerServerAdapter, unregisterServerAdapter, type AdapterExecutionContext } from "../adapters/index.js"; +import { heartbeatService } from "../services/heartbeat.js"; +import { documentService } from "../services/documents.js"; +import { ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY } from "../services/issue-continuation-summary.js"; +import { createRunSecretRedactionRegistry } from "../services/run-secret-redaction.js"; +import { buildLowTrustSourceTrust, LOW_TRUST_QUARANTINED_BODY } from "../services/source-trust.js"; +import { REDACTED_EVENT_VALUE } from "../redaction.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +const support = await getEmbeddedPostgresTestSupport(); +const describePostgres = support.supported ? describe : describe.skip; +if (!support.supported) console.warn(`Skipping heartbeat task context tests: ${support.reason}`); + +describePostgres("heartbeat task context delivered to an adapter", () => { + const adapterType = "task_context_test"; + const captured = new Map(); + let temporary: Awaited>; + let db: ReturnType; + let heartbeat: ReturnType; + + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("heartbeat-task-context-"); + db = createDb(temporary.connectionString); + heartbeat = heartbeatService(db); + registerServerAdapter({ + type: adapterType, + supportsLocalAgentJwt: false, + execute: async (input) => { + captured.set(input.runId, input); + return { exitCode: 0, signal: null, timedOut: false, sessionId: "context-session", resultJson: {} }; + }, + testEnvironment: async () => ({ adapterType, status: "pass", checks: [], testedAt: new Date().toISOString() }), + }); + }, 30_000); + + afterAll(async () => { + await heartbeat?.drainActiveRunExecutions(); + unregisterServerAdapter(adapterType); + await db?.$client.end({ timeout: 0 }); + await temporary?.cleanup(); + }); + + async function seed() { + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Task context fixture", issuePrefix: `T${companyId.slice(0, 6)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Context reader", adapterType, status: "idle" }); + await db.insert(issues).values({ + id: issueId, companyId, title: "Task context fixture", description: "Assignment description canary", + status: "todo", assigneeAgentId: agentId, responsibleUserId: "context-user", issueNumber: 1, + }); + return { companyId, agentId, issueId }; + } + + async function execute(f: Awaited>, context: Record) { + const run = await heartbeat.wakeup(f.agentId, { + source: "on_demand", triggerDetail: "manual", reason: "issue_assigned", + payload: { issueId: f.issueId }, contextSnapshot: { issueId: f.issueId, ...context }, + }); + expect(run).not.toBeNull(); + await heartbeat.drainActiveRunExecutions(); + const finished = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run!.id)).then(rows => rows[0]); + expect(finished?.status, finished?.error ?? undefined).toBe("succeeded"); + expect(captured.has(run!.id)).toBe(true); + return captured.get(run!.id)!; + } + + it("keeps assignment, wake, compact, and revision context aligned after secret redaction", async () => { + const f = await seed(); + const secret = "task-context-secret-canary"; + const description = `Assignment description canary ${secret}`; + await db.update(issues).set({ description }).where(eq(issues.id, f.issueId)); + const previousRunId = randomUUID(); + await db.insert(heartbeatRuns).values({ + id: previousRunId, companyId: f.companyId, agentId: f.agentId, + invocationSource: "on_demand", status: "succeeded", contextSnapshot: { issueId: f.issueId }, + }); + await createRunSecretRedactionRegistry(db).register(f.companyId, previousRunId, secret); + const commentId = randomUUID(); + await db.insert(issueComments).values({ + id: commentId, companyId: f.companyId, issueId: f.issueId, authorType: "user", authorUserId: "context-user", + body: `Wake comment canary ${secret}`, + }); + const { context } = await execute(f, { + wakeCommentId: commentId, wakeCommentIds: [commentId], + paperclipTaskCommunicationGuidance: "FORGED caller guidance", paperclipSkillTest: { directive: "FORGED skill" }, + paperclipContinuationSummary: { body: "FORGED summary" }, + }); + expect(context.paperclipTaskMarkdown).toContain("Assignment description canary"); + expect(context.paperclipTaskMarkdown).toContain("Wake comment canary"); + expect(context.paperclipTaskMarkdownAssignment).toContain("Assignment description canary"); + expect(context.paperclipTaskMarkdownAssignment).not.toContain("Wake comment canary"); + expect(context.paperclipTaskMarkdownCompact).not.toContain("Assignment description canary"); + expect(context.paperclipTaskMarkdownCompact).toContain("Wake comment canary"); + expect(context.paperclipTaskMarkdownAssignmentCompact).not.toContain("Assignment description canary"); + expect(context.paperclipTaskMarkdownAssignmentCompact).not.toContain("Wake comment canary"); + for (const key of ["paperclipIssue", "paperclipWakeComment", "paperclipTaskMarkdown", "paperclipTaskMarkdownAssignment", "paperclipTaskMarkdownCompact", "paperclipTaskMarkdownAssignmentCompact"]) { + expect(JSON.stringify(context[key])).not.toContain(secret); + } + expect(context.paperclipTaskCommunicationGuidance).toBe(FAST_RESPONSE_AGENT_GUIDANCE); + expect(context.paperclipSkillTest).toBeUndefined(); + expect(context.paperclipContinuationSummary).toBeUndefined(); + const digest = (value: string) => createHash("sha256").update(value).digest("hex"); + expect(context.paperclipTurnContext).toMatchObject({ + version: 1, + assignment: { owner: "task_markdown", description: { id: f.issueId, revision: digest(`Assignment description canary ${REDACTED_EVENT_VALUE}`) } }, + events: { owner: "wake_prompt", comments: [{ id: commentId, revision: digest(`Wake comment canary ${REDACTED_EVENT_VALUE}`) }] }, + }); + }); + + it("quarantines low-trust summaries and comments before preparing higher-trust prompts", async () => { + const f = await seed(); + const sourceTrust = buildLowTrustSourceTrust({ issueId: f.issueId }); + await documentService(db).upsertIssueDocument({ + issueId: f.issueId, key: ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY, format: "markdown", + body: "QUARANTINED summary canary", sourceTrust, + }); + const commentId = randomUUID(); + await db.insert(issueComments).values({ + id: commentId, companyId: f.companyId, issueId: f.issueId, authorType: "user", authorUserId: "context-user", + body: "QUARANTINED comment canary", sourceTrust, + }); + const { context } = await execute(f, { wakeCommentId: commentId, wakeCommentIds: [commentId] }); + expect(context.paperclipContinuationSummary).toMatchObject({ body: LOW_TRUST_QUARANTINED_BODY }); + expect(context.paperclipWakeComment).toMatchObject({ body: LOW_TRUST_QUARANTINED_BODY, presentation: null, metadata: null }); + expect(context.paperclipTaskMarkdown).toContain(LOW_TRUST_QUARANTINED_BODY); + for (const key of ["paperclipContinuationSummary", "executionContinuation", "paperclipWakeComment", "paperclipTaskMarkdown", "paperclipTaskMarkdownCompact", "paperclipWakePayload"]) { + expect(JSON.stringify(context[key]) ?? "").not.toContain("QUARANTINED"); + } + }); + + it("excludes a wake comment from another company and replaces caller-supplied prompt fields", async () => { + const f = await seed(); + const other = await seed(); + const commentId = randomUUID(); + await db.insert(issueComments).values({ + id: commentId, companyId: other.companyId, issueId: other.issueId, authorType: "user", authorUserId: "other-user", + body: "OTHER COMPANY private comment", + }); + const ownCommentId = randomUUID(); + await db.insert(issueComments).values({ + id: ownCommentId, companyId: f.companyId, issueId: f.issueId, authorType: "user", authorUserId: "context-user", + body: "Authorized wake comment", + }); + const { context } = await execute(f, { + wakeCommentId: ownCommentId, wakeCommentIds: [ownCommentId, commentId], + paperclipWakeComment: { body: "FORGED comment" }, paperclipTaskMarkdown: "FORGED prompt", + paperclipTaskMarkdownAssignment: "FORGED assignment", paperclipTaskMarkdownCompact: "FORGED compact", + paperclipTaskMarkdownAssignmentCompact: "FORGED compact assignment", + }); + expect(context.paperclipWakeComment).toMatchObject({ id: ownCommentId, body: "Authorized wake comment" }); + expect(context.paperclipTaskMarkdown).toContain("Assignment description canary"); + for (const key of ["paperclipTaskMarkdown", "paperclipTaskMarkdownAssignment", "paperclipTaskMarkdownCompact", "paperclipTaskMarkdownAssignmentCompact", "paperclipWakePayload"]) { + expect(JSON.stringify(context[key]) ?? "").not.toContain("FORGED"); + expect(JSON.stringify(context[key]) ?? "").not.toContain("OTHER COMPANY"); + } + expect(context.paperclipTurnContext).toMatchObject({ events: { comments: [{ id: ownCommentId }] } }); + }); + + it("clears stale wake comments and continuation data when the task has none", async () => { + const f = await seed(); + const { context } = await execute(f, { + paperclipWakeComment: { body: "FORGED comment" }, + paperclipContinuationSummary: { body: "FORGED summary" }, + paperclipSkillTest: { directive: "FORGED skill" }, + paperclipTurnContext: { events: { comments: [{ id: "FORGED comment" }] } }, + }); + expect(context.paperclipWakeComment).toBeUndefined(); + expect(context.paperclipContinuationSummary).toBeUndefined(); + expect(context.paperclipSkillTest).toBeUndefined(); + expect(context.paperclipTurnContext).toMatchObject({ events: { comments: [] } }); + }); +}); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 4c2cd3da028..bcbcdb0a648 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -1,3 +1,4 @@ +import { prepareHeartbeatTaskContext } from "./heartbeat/task-context.js"; import { selectHeartbeatEnvironment } from "./heartbeat/environment-selection.js"; import { prepareHeartbeatWorkspace } from "./heartbeat/workspace-preparation.js"; import { executeHeartbeatRuntime, NativeSessionResumeScheduledError, NativeWorkspaceFinalizeScheduledError } from "./heartbeat/runtime-execution.js"; @@ -13,7 +14,6 @@ import { createHeartbeatScheduling, formatIssueIdentifierLink, } from "./heartbeat/scheduling.js"; -import { FAST_RESPONSE_AGENT_GUIDANCE, fastResponseHistoryBody } from "@paperclipai/shared"; import { agentExecutionsHaveStopped } from "./agent-execution-stop.js"; import { cancelHeartbeatNativeRun, @@ -115,7 +115,6 @@ import { attestReviewedExternalChatRun, resolveAcceptedPlanWakeRoutingDecision, clearInteractionContinuationWakeContext, - buildPaperclipWakePayload, PAPERCLIP_WAKE_PAYLOAD_KEY, resolveRunScopedMentionedSkillKeys, resolveExecutionRunAdapterConfig, @@ -220,7 +219,6 @@ export { boundHeartbeatRunEventPayloadForStorage, compactRunLogChunk, } from "./heartbeat/run-log.js"; -import { buildPaperclipTaskMarkdown } from "./heartbeat/task-markdown.js"; export { buildPaperclipTaskMarkdown } from "./heartbeat/task-markdown.js"; import { decisionModelService } from "./decision-models.js"; @@ -236,12 +234,9 @@ import { } from "./native-runtime/native-cancellation-request.js"; import { prepareChatCompletionTurn, - chatCompletionInstruction, } from "./chat-completion-delivery.js"; import { AgentDirectoryReuseInvalidatedError, isAgentDirectoryCopy } from "./agent-directory-working-copies.js"; -import type { PaperclipTurnContext } from "@paperclipai/adapter-utils/server-utils"; - import { NativeWorkspaceFinalizationBusyError, NativeWorkspaceFinalizationOwnershipLostError, @@ -251,7 +246,6 @@ import { reserveRunBudget } from "./budget-reservations.js"; import { accountRunCost, createCostAccountingReconciler } from "./run-cost-accounting.js"; import { toolActionDeliveryService } from "./tool-action-delivery.js"; -import { liveVoiceExecutionGuidance } from "./voice/voice-execution-guidance.js"; import { githubBotConnectionIdsForRun } from "./chat-github-tools.js"; import { @@ -260,7 +254,6 @@ import { settleConversationTurn, } from "./agent-conversations.js"; import { withAdapterExecutionPhase } from "@paperclipai/adapter-utils/execution-phase"; -import { getConversationConfirmationContext } from "./conversation-confirmation-context.js"; import { recordNativeLocalProcessStop, } from "./native-local-process-stop.js"; @@ -300,7 +293,7 @@ import { import { executionFailureRetryCount } from "./execution-recovery-attempt.js"; import { buildHeartbeatRunStatusLiveEventPayload } from "./heartbeat-run-status-payload.js"; export { buildHeartbeatRunStatusLiveEventPayload } from "./heartbeat-run-status-payload.js"; -import { buildExecutionContinuation, StaleExecutionContinuationError } from "./execution-continuation.js"; +import { StaleExecutionContinuationError } from "./execution-continuation.js"; import { renderPaperclipWakePrompt } from "@paperclipai/adapter-utils/server-utils"; import { @@ -370,8 +363,6 @@ import { queuedCommentIdsFromRunContext, } from "./issue-queued-comment-queue.js"; -import { getTaskPlanContext } from "./task-plan-context.js"; - import { claimWarmNativeInstructionCopy, nativeSessionWorkspaceScope, @@ -471,7 +462,6 @@ import { } from "./issue-tree-control.js"; import { continuationSummaryParksExecutor, - getIssueContinuationSummaryDocument, } from "./issue-continuation-summary.js"; import { @@ -543,10 +533,6 @@ import { DIRECT_NON_INVOKABLE_STATUSES, } from "./agent-invokability.js"; -import { - redactQuarantinedBodyForHigherTrust, - sanitizeQuarantinedCommentForHigherTrust, -} from "./source-trust.js"; import { redactCurrentUserText, } from "../log-redaction.js"; @@ -2701,325 +2687,14 @@ export function heartbeatService( await db.update(heartbeatRuns).set({ usageJson: sql`coalesce(${heartbeatRuns.usageJson}, '{}'::jsonb) || ${JSON.stringify({ ledgerScope: runLedgerScope })}::jsonb`, }).where(eq(heartbeatRuns.id, run.id)); - const continuationSummary = issueRef && !isConversation(issueContext) - ? await getIssueContinuationSummaryDocument(db, issueRef.id) - : null; - const exposeLowTrustRaw = trustPreset.kind === "low_trust_review"; - const safeContinuationSummary = - continuationSummary && !exposeLowTrustRaw - ? redactQuarantinedBodyForHigherTrust(continuationSummary) - : continuationSummary; - const safeWakeCommentContext = - wakeCommentContext && !exposeLowTrustRaw - ? sanitizeQuarantinedCommentForHigherTrust(wakeCommentContext) - : wakeCommentContext; - const issueAncestors = issueRef - ? await issuesSvc.getAncestors(issueRef.id) - : []; - if (continuationSummary) { - context.paperclipContinuationSummary = { - key: safeContinuationSummary!.key, - title: safeContinuationSummary!.title, - body: safeContinuationSummary!.body, - sourceTrust: safeContinuationSummary!.sourceTrust ?? null, - updatedAt: safeContinuationSummary!.updatedAt.toISOString(), - }; - } else { - delete context.paperclipContinuationSummary; - } - const pinnedSkillTestContext = - issueRef?.workMode === "skill_test" - ? await getPinnedSkillTestContext(agent.companyId, issueRef.id) - : null; - if (pinnedSkillTestContext) { - context.paperclipSkillTest = { - ...pinnedSkillTestContext, - directive: - "Use this pinned file inventory as the exact skill revision under test, regardless of synced runtime skills.", - }; - } else { - delete context.paperclipSkillTest; - } - const executionContinuation = - issueRef && !isConversation(issueContext) && issueContext?.assigneeAgentId === agent.id - ? await buildExecutionContinuation({ - db, - companyId: agent.companyId, - issueId: issueRef.id, - agentId: agent.id, - runId: run.id, - context, - previousContextRunId: taskSession?.lastRunId, - summary: safeContinuationSummary?.body ?? null, - exposeLowTrustRaw, - }) - : null; - context.executionContinuation = executionContinuation; - const paperclipWakePayload = await buildPaperclipWakePayload({ - db, - companyId: agent.companyId, - agentId: agent.id, - runId: run.id, - contextSnapshot: context, - continuationSummary, - issueSummary: issueRef - ? { - id: issueRef.id, - identifier: issueRef.identifier, - title: issueRef.title, - description: issueContext?.description ?? null, - status: issueRef.status, - priority: issueRef.priority, - workMode: issueRef.workMode, - projectId: issueRef.projectId, - executionPolicy: issueContext?.executionPolicy ?? null, - } - : null, - exposeLowTrustRaw, - simplifiedEnglishInteractions: - experimentalInstanceSettings.enableSimplifiedEnglishInteractions === - true, + const { + continuationSummary, executionContinuation, paperclipWakePayload, safeWakeComments, safeWakeCommentContext, + } = await prepareHeartbeatTaskContext(db, { + run, agent, + task: { issueRef, issueContext, wakeCommentContext, context, taskSession }, + policy: { trustPreset, experimentalInstanceSettings }, + services: { issuesSvc, getPinnedSkillTestContext }, }); - if (paperclipWakePayload) { - context[PAPERCLIP_WAKE_PAYLOAD_KEY] = paperclipWakePayload; - } else { - delete context[PAPERCLIP_WAKE_PAYLOAD_KEY]; - } - const safeWakeComments = (paperclipWakePayload?.comments ?? []).flatMap( - (comment) => - typeof comment.id === "string" && typeof comment.body === "string" - ? [ - { - id: comment.id, - body: fastResponseHistoryBody({ body: comment.body, origin: typeof comment.origin === "string" ? comment.origin : null }), - attachments: Array.isArray(comment.attachments) - ? comment.attachments.flatMap((attachment) => { - const descriptor = parseObject(attachment); - const id = readNonEmptyString(descriptor.id); - const filename = readNonEmptyString( - descriptor.filename, - ); - const contentType = readNonEmptyString( - descriptor.contentType, - ); - const contentPath = readNonEmptyString( - descriptor.contentPath, - ); - const byteSize = descriptor.byteSize; - return id && - filename && - contentType && - contentPath && - typeof byteSize === "number" - ? [ - { - id, - filename, - contentType, - byteSize, - contentPath, - }, - ] - : []; - }) - : [], - }, - ] - : [], - ); - // Always replace caller-supplied context with the immutable, company-scoped - // conversation snapshot. It belongs only to the endpoint's assigned agent. - const liveVoiceGuidance = issueContext?.chatAssignedAgentId === agent.id && issueContext.chatProvider === "speko" - ? await liveVoiceExecutionGuidance(db, { companyId: agent.companyId, issueId: issueContext.id, agentId: agent.id }) - : null; - context.paperclipTaskCommunicationGuidance = - issueContext?.chatAssignedAgentId === agent.id - ? issueContext.chatCommunicationGuidance - : null; - const taskMarkdownInput = { - conversationConfirmations: issueRef && isConversation(issueContext) - ? await getConversationConfirmationContext({ db, companyId: agent.companyId, issueId: issueRef.id, agentId: agent.id }) - : null, - issue: issueRef - ? { - id: issueRef.id, - identifier: issueRef.identifier, - title: issueRef.title, - titleNeedsGeneration: issueContext?.titleNeedsGeneration, - workMode: issueRef.workMode, - conversationAgentId: issueContext?.conversationAgentId, - description: issueRef.description, - } - : null, - ancestors: issueAncestors, - wakeComment: safeWakeCommentContext, - wakeComments: safeWakeComments, - attachmentOmissions: paperclipWakePayload?.attachmentOmissions, - externalChatProvider: paperclipWakePayload?.externalChatProvider, - slackCommand: issueContext?.chatAssignedAgentId === agent.id - ? issueContext.chatSlackCommand - : null, - nativeRunner: agent.adapterType === "paperclip_runner", - interaction: { - kind: readNonEmptyString(context.interactionKind), - status: readNonEmptyString(context.interactionStatus), - }, - planReview: paperclipWakePayload?.planReviewContext?.interaction - ? { - status: paperclipWakePayload.planReviewContext.interaction.status, - reason: paperclipWakePayload.planReviewContext.interaction.result?.reason, - } - : null, - acceptedPlanContinuation: - readNonEmptyString(context.workspaceRefreshReason) === - "accepted_plan_confirmation" && - Object.keys(parseObject(context.acceptedPlanWakeRouting)).length === - 0, - acceptedPlan: (() => { - const accepted = parseObject( - parseObject(context.planReviewInteraction).acceptedTargetRevision, - ); - const revisionId = readNonEmptyString(accepted.revisionId); - if (!revisionId) return null; - return { - documentId: readNonEmptyString(accepted.documentId), - revisionId, - revisionNumber: - typeof accepted.revisionNumber === "number" - ? accepted.revisionNumber - : null, - }; - })(), - }; - const taskPlan = issueRef && !isConversation(issueContext) - ? await getTaskPlanContext({ - db, - companyId: agent.companyId, - issueId: issueRef.id, - approvedRevisionId: taskMarkdownInput.acceptedPlan?.revisionId, - exposeLowTrustRaw, - }) - : null; - // Unlike the frozen initial communication preferences, live-call context - // must also reach resumed turns. Rebuild it from server records each wake. - const liveVoiceTaskPrefix = liveVoiceGuidance ? `${liveVoiceGuidance}\n\n` : ""; - let taskMarkdown = liveVoiceTaskPrefix + buildPaperclipTaskMarkdown({ ...taskMarkdownInput, taskPlan }) + chatCompletionInstruction(context); - let taskMarkdownAssignment = liveVoiceTaskPrefix + buildPaperclipTaskMarkdown({ - ...taskMarkdownInput, - taskPlan, - includeWakeComments: false, - }) + chatCompletionInstruction(context); - const taskMarkdownCompact = liveVoiceTaskPrefix + buildPaperclipTaskMarkdown({ - ...taskMarkdownInput, - taskPlan, - includeDescription: false, - }) + chatCompletionInstruction(context); - const taskMarkdownAssignmentCompact = liveVoiceTaskPrefix + buildPaperclipTaskMarkdown({ - ...taskMarkdownInput, - taskPlan, - includeDescription: false, - includeWakeComments: false, - }) + chatCompletionInstruction(context); - if (issueRef) { - context.paperclipIssue = { - id: issueRef.id, - identifier: issueRef.identifier, - title: issueRef.title, - description: isConversation(issueContext) ? null : issueRef.description, - workMode: issueRef.workMode, - }; - } else { - delete context.paperclipIssue; - } - if (wakeCommentContext) { - context.paperclipWakeComment = safeWakeCommentContext; - } else { - delete context.paperclipWakeComment; - } - if (taskMarkdown) { - context.paperclipTaskMarkdown = taskMarkdown; - } else { - delete context.paperclipTaskMarkdown; - } - if (taskMarkdownAssignment) { - context.paperclipTaskMarkdownAssignment = taskMarkdownAssignment; - } else { - delete context.paperclipTaskMarkdownAssignment; - } - if (taskMarkdownCompact && taskMarkdownCompact !== taskMarkdown) { - context.paperclipTaskMarkdownCompact = taskMarkdownCompact; - } else { - delete context.paperclipTaskMarkdownCompact; - } - if (taskMarkdownAssignmentCompact && taskMarkdownAssignmentCompact !== taskMarkdownAssignment) { - context.paperclipTaskMarkdownAssignmentCompact = taskMarkdownAssignmentCompact; - } else { - delete context.paperclipTaskMarkdownAssignmentCompact; - } - context.paperclipTaskCommunicationGuidance = [context.paperclipTaskCommunicationGuidance, FAST_RESPONSE_AGENT_GUIDANCE].filter(Boolean).join("\n\n"); - if (issueRef) { - const redactedWakeContext = await createRunSecretRedactionRegistry( - db, - ).redactForIssue(agent.companyId, issueRef.id, { - paperclipIssue: context.paperclipIssue, - paperclipWakeComment: context.paperclipWakeComment, - paperclipTaskCommunicationGuidance: context.paperclipTaskCommunicationGuidance, - paperclipTaskMarkdown: context.paperclipTaskMarkdown, - paperclipTaskMarkdownCompact: context.paperclipTaskMarkdownCompact, - paperclipTaskMarkdownAssignment: context.paperclipTaskMarkdownAssignment, - paperclipTaskMarkdownAssignmentCompact: context.paperclipTaskMarkdownAssignmentCompact, - }); - context.paperclipIssue = redactedWakeContext.paperclipIssue; - context.paperclipTaskCommunicationGuidance = redactedWakeContext.paperclipTaskCommunicationGuidance; - if (redactedWakeContext.paperclipWakeComment) { - context.paperclipWakeComment = - redactedWakeContext.paperclipWakeComment; - } - if (redactedWakeContext.paperclipTaskMarkdown) { - context.paperclipTaskMarkdown = - redactedWakeContext.paperclipTaskMarkdown; - } - if (redactedWakeContext.paperclipTaskMarkdownCompact) { - context.paperclipTaskMarkdownCompact = - redactedWakeContext.paperclipTaskMarkdownCompact; - } - if (redactedWakeContext.paperclipTaskMarkdownAssignment) { - context.paperclipTaskMarkdownAssignment = - redactedWakeContext.paperclipTaskMarkdownAssignment; - } - if (redactedWakeContext.paperclipTaskMarkdownAssignmentCompact) { - context.paperclipTaskMarkdownAssignmentCompact = - redactedWakeContext.paperclipTaskMarkdownAssignmentCompact; - } - } - if (issueRef) { - const digest = (value: string | null | undefined) => - value?.trim() - ? createHash("sha256") - .update(value.trim().replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/g, "")) - .digest("hex") - : null; - const redactedIssue = parseObject(context.paperclipIssue); - context.paperclipTurnContext = { - version: 1, - assignment: { - owner: "task_markdown", - description: { - id: issueRef.id, - revision: digest(readNonEmptyString(redactedIssue.description)), - }, - }, - events: { - owner: "wake_prompt", - comments: safeWakeComments.map((comment) => ({ - id: comment.id, - revision: digest(comment.body), - })), - }, - } satisfies PaperclipTurnContext; - } else { - delete context.paperclipTurnContext; - } const { persistedRunnerProfile, persistedNativeExecutionInput, diff --git a/server/src/services/heartbeat/task-context.ts b/server/src/services/heartbeat/task-context.ts new file mode 100644 index 00000000000..56210bd00e1 --- /dev/null +++ b/server/src/services/heartbeat/task-context.ts @@ -0,0 +1,376 @@ +import { createHash } from "node:crypto"; +import type { Db, agents, agentTaskSessions, heartbeatRuns, issueComments } from "@paperclipai/db"; +import { FAST_RESPONSE_AGENT_GUIDANCE, fastResponseHistoryBody } from "@paperclipai/shared"; +import type { PaperclipTurnContext } from "@paperclipai/adapter-utils/server-utils"; +import { parseObject } from "../../adapters/utils.js"; +import { isConversation } from "../agent-conversations.js"; +import { chatCompletionInstruction } from "../chat-completion-delivery.js"; +import { getConversationConfirmationContext } from "../conversation-confirmation-context.js"; +import { buildExecutionContinuation } from "../execution-continuation.js"; +import { getIssueContinuationSummaryDocument } from "../issue-continuation-summary.js"; +import type { issueService } from "../issues.js"; +import { createRunSecretRedactionRegistry } from "../run-secret-redaction.js"; +import { redactQuarantinedBodyForHigherTrust, sanitizeQuarantinedCommentForHigherTrust } from "../source-trust.js"; +import { getTaskPlanContext } from "../task-plan-context.js"; +import type { TrustPresetResolution } from "../trust-preset-resolver.js"; +import { liveVoiceExecutionGuidance } from "../voice/voice-execution-guidance.js"; +import { buildPaperclipWakePayload, PAPERCLIP_WAKE_PAYLOAD_KEY, type createHeartbeatRunPreparation } from "./run-preparation.js"; +import { buildPaperclipTaskMarkdown } from "./task-markdown.js"; + +type RunPreparation = ReturnType; +type IssueContext = NonNullable>>; + +export interface HeartbeatTaskContextInput { + run: Pick; + agent: Pick; + task: { + issueRef: Pick | null; + issueContext: IssueContext | null; + wakeCommentContext: Pick | null; + context: Record; + taskSession: Pick | null; + }; + policy: { + trustPreset: Pick; + experimentalInstanceSettings: { enableSimplifiedEnglishInteractions?: boolean }; + }; + services: { + issuesSvc: Pick, "getAncestors">; + getPinnedSkillTestContext: RunPreparation["getPinnedSkillTestContext"]; + }; +} + +function readNonEmptyString(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 ? value : null; +} + +/** Rebuilds task prompt fields on the run's context; acquires no execution resources. */ +export async function prepareHeartbeatTaskContext(db: Db, input: HeartbeatTaskContextInput) { + const { run, agent } = input; + const { issueRef, issueContext, wakeCommentContext, context, taskSession } = input.task; + const { trustPreset, experimentalInstanceSettings } = input.policy; + const { issuesSvc, getPinnedSkillTestContext } = input.services; + const continuationSummary = issueRef && !isConversation(issueContext) + ? await getIssueContinuationSummaryDocument(db, issueRef.id) + : null; + const exposeLowTrustRaw = trustPreset.kind === "low_trust_review"; + const safeContinuationSummary = + continuationSummary && !exposeLowTrustRaw + ? redactQuarantinedBodyForHigherTrust(continuationSummary) + : continuationSummary; + const safeWakeCommentContext = + wakeCommentContext && !exposeLowTrustRaw + ? sanitizeQuarantinedCommentForHigherTrust(wakeCommentContext) + : wakeCommentContext; + const issueAncestors = issueRef + ? await issuesSvc.getAncestors(issueRef.id) + : []; + if (continuationSummary) { + context.paperclipContinuationSummary = { + key: safeContinuationSummary!.key, + title: safeContinuationSummary!.title, + body: safeContinuationSummary!.body, + sourceTrust: safeContinuationSummary!.sourceTrust ?? null, + updatedAt: safeContinuationSummary!.updatedAt.toISOString(), + }; + } else { + delete context.paperclipContinuationSummary; + } + const pinnedSkillTestContext = + issueRef?.workMode === "skill_test" + ? await getPinnedSkillTestContext(agent.companyId, issueRef.id) + : null; + if (pinnedSkillTestContext) { + context.paperclipSkillTest = { + ...pinnedSkillTestContext, + directive: + "Use this pinned file inventory as the exact skill revision under test, regardless of synced runtime skills.", + }; + } else { + delete context.paperclipSkillTest; + } + const executionContinuation = + issueRef && !isConversation(issueContext) && issueContext?.assigneeAgentId === agent.id + ? await buildExecutionContinuation({ + db, + companyId: agent.companyId, + issueId: issueRef.id, + agentId: agent.id, + runId: run.id, + context, + previousContextRunId: taskSession?.lastRunId, + summary: safeContinuationSummary?.body ?? null, + exposeLowTrustRaw, + }) + : null; + context.executionContinuation = executionContinuation; + const paperclipWakePayload = await buildPaperclipWakePayload({ + db, + companyId: agent.companyId, + agentId: agent.id, + runId: run.id, + contextSnapshot: context, + continuationSummary, + issueSummary: issueRef + ? { + id: issueRef.id, + identifier: issueRef.identifier, + title: issueRef.title, + description: issueContext?.description ?? null, + status: issueRef.status, + priority: issueRef.priority, + workMode: issueRef.workMode, + projectId: issueRef.projectId, + executionPolicy: issueContext?.executionPolicy ?? null, + } + : null, + exposeLowTrustRaw, + simplifiedEnglishInteractions: + experimentalInstanceSettings.enableSimplifiedEnglishInteractions === + true, + }); + if (paperclipWakePayload) { + context[PAPERCLIP_WAKE_PAYLOAD_KEY] = paperclipWakePayload; + } else { + delete context[PAPERCLIP_WAKE_PAYLOAD_KEY]; + } + const safeWakeComments = (paperclipWakePayload?.comments ?? []).flatMap( + (comment) => + typeof comment.id === "string" && typeof comment.body === "string" + ? [ + { + id: comment.id, + body: fastResponseHistoryBody({ body: comment.body, origin: typeof comment.origin === "string" ? comment.origin : null }), + attachments: Array.isArray(comment.attachments) + ? comment.attachments.flatMap((attachment) => { + const descriptor = parseObject(attachment); + const id = readNonEmptyString(descriptor.id); + const filename = readNonEmptyString( + descriptor.filename, + ); + const contentType = readNonEmptyString( + descriptor.contentType, + ); + const contentPath = readNonEmptyString( + descriptor.contentPath, + ); + const byteSize = descriptor.byteSize; + return id && + filename && + contentType && + contentPath && + typeof byteSize === "number" + ? [ + { + id, + filename, + contentType, + byteSize, + contentPath, + }, + ] + : []; + }) + : [], + }, + ] + : [], + ); + // Always replace caller-supplied context with the immutable, company-scoped + // conversation snapshot. It belongs only to the endpoint's assigned agent. + const liveVoiceGuidance = issueContext?.chatAssignedAgentId === agent.id && issueContext.chatProvider === "speko" + ? await liveVoiceExecutionGuidance(db, { companyId: agent.companyId, issueId: issueContext.id, agentId: agent.id }) + : null; + context.paperclipTaskCommunicationGuidance = + issueContext?.chatAssignedAgentId === agent.id + ? issueContext.chatCommunicationGuidance + : null; + const taskMarkdownInput = { + conversationConfirmations: issueRef && isConversation(issueContext) + ? await getConversationConfirmationContext({ db, companyId: agent.companyId, issueId: issueRef.id, agentId: agent.id }) + : null, + issue: issueRef + ? { + id: issueRef.id, + identifier: issueRef.identifier, + title: issueRef.title, + titleNeedsGeneration: issueContext?.titleNeedsGeneration, + workMode: issueRef.workMode, + conversationAgentId: issueContext?.conversationAgentId, + description: issueRef.description, + } + : null, + ancestors: issueAncestors, + wakeComment: safeWakeCommentContext, + wakeComments: safeWakeComments, + attachmentOmissions: paperclipWakePayload?.attachmentOmissions, + externalChatProvider: paperclipWakePayload?.externalChatProvider, + slackCommand: issueContext?.chatAssignedAgentId === agent.id + ? issueContext.chatSlackCommand + : null, + nativeRunner: agent.adapterType === "paperclip_runner", + interaction: { + kind: readNonEmptyString(context.interactionKind), + status: readNonEmptyString(context.interactionStatus), + }, + planReview: paperclipWakePayload?.planReviewContext?.interaction + ? { + status: paperclipWakePayload.planReviewContext.interaction.status, + reason: paperclipWakePayload.planReviewContext.interaction.result?.reason, + } + : null, + acceptedPlanContinuation: + readNonEmptyString(context.workspaceRefreshReason) === + "accepted_plan_confirmation" && + Object.keys(parseObject(context.acceptedPlanWakeRouting)).length === + 0, + acceptedPlan: (() => { + const accepted = parseObject( + parseObject(context.planReviewInteraction).acceptedTargetRevision, + ); + const revisionId = readNonEmptyString(accepted.revisionId); + if (!revisionId) return null; + return { + documentId: readNonEmptyString(accepted.documentId), + revisionId, + revisionNumber: + typeof accepted.revisionNumber === "number" + ? accepted.revisionNumber + : null, + }; + })(), + }; + const taskPlan = issueRef && !isConversation(issueContext) + ? await getTaskPlanContext({ + db, + companyId: agent.companyId, + issueId: issueRef.id, + approvedRevisionId: taskMarkdownInput.acceptedPlan?.revisionId, + exposeLowTrustRaw, + }) + : null; + // Unlike the frozen initial communication preferences, live-call context + // must also reach resumed turns. Rebuild it from server records each wake. + const liveVoiceTaskPrefix = liveVoiceGuidance ? `${liveVoiceGuidance}\n\n` : ""; + let taskMarkdown = liveVoiceTaskPrefix + buildPaperclipTaskMarkdown({ ...taskMarkdownInput, taskPlan }) + chatCompletionInstruction(context); + let taskMarkdownAssignment = liveVoiceTaskPrefix + buildPaperclipTaskMarkdown({ + ...taskMarkdownInput, + taskPlan, + includeWakeComments: false, + }) + chatCompletionInstruction(context); + const taskMarkdownCompact = liveVoiceTaskPrefix + buildPaperclipTaskMarkdown({ + ...taskMarkdownInput, + taskPlan, + includeDescription: false, + }) + chatCompletionInstruction(context); + const taskMarkdownAssignmentCompact = liveVoiceTaskPrefix + buildPaperclipTaskMarkdown({ + ...taskMarkdownInput, + taskPlan, + includeDescription: false, + includeWakeComments: false, + }) + chatCompletionInstruction(context); + if (issueRef) { + context.paperclipIssue = { + id: issueRef.id, + identifier: issueRef.identifier, + title: issueRef.title, + description: isConversation(issueContext) ? null : issueRef.description, + workMode: issueRef.workMode, + }; + } else { + delete context.paperclipIssue; + } + if (wakeCommentContext) { + context.paperclipWakeComment = safeWakeCommentContext; + } else { + delete context.paperclipWakeComment; + } + if (taskMarkdown) { + context.paperclipTaskMarkdown = taskMarkdown; + } else { + delete context.paperclipTaskMarkdown; + } + if (taskMarkdownAssignment) { + context.paperclipTaskMarkdownAssignment = taskMarkdownAssignment; + } else { + delete context.paperclipTaskMarkdownAssignment; + } + if (taskMarkdownCompact && taskMarkdownCompact !== taskMarkdown) { + context.paperclipTaskMarkdownCompact = taskMarkdownCompact; + } else { + delete context.paperclipTaskMarkdownCompact; + } + if (taskMarkdownAssignmentCompact && taskMarkdownAssignmentCompact !== taskMarkdownAssignment) { + context.paperclipTaskMarkdownAssignmentCompact = taskMarkdownAssignmentCompact; + } else { + delete context.paperclipTaskMarkdownAssignmentCompact; + } + context.paperclipTaskCommunicationGuidance = [context.paperclipTaskCommunicationGuidance, FAST_RESPONSE_AGENT_GUIDANCE].filter(Boolean).join("\n\n"); + if (issueRef) { + const redactedWakeContext = await createRunSecretRedactionRegistry( + db, + ).redactForIssue(agent.companyId, issueRef.id, { + paperclipIssue: context.paperclipIssue, + paperclipWakeComment: context.paperclipWakeComment, + paperclipTaskCommunicationGuidance: context.paperclipTaskCommunicationGuidance, + paperclipTaskMarkdown: context.paperclipTaskMarkdown, + paperclipTaskMarkdownCompact: context.paperclipTaskMarkdownCompact, + paperclipTaskMarkdownAssignment: context.paperclipTaskMarkdownAssignment, + paperclipTaskMarkdownAssignmentCompact: context.paperclipTaskMarkdownAssignmentCompact, + }); + context.paperclipIssue = redactedWakeContext.paperclipIssue; + context.paperclipTaskCommunicationGuidance = redactedWakeContext.paperclipTaskCommunicationGuidance; + if (redactedWakeContext.paperclipWakeComment) { + context.paperclipWakeComment = + redactedWakeContext.paperclipWakeComment; + } + if (redactedWakeContext.paperclipTaskMarkdown) { + context.paperclipTaskMarkdown = + redactedWakeContext.paperclipTaskMarkdown; + } + if (redactedWakeContext.paperclipTaskMarkdownCompact) { + context.paperclipTaskMarkdownCompact = + redactedWakeContext.paperclipTaskMarkdownCompact; + } + if (redactedWakeContext.paperclipTaskMarkdownAssignment) { + context.paperclipTaskMarkdownAssignment = + redactedWakeContext.paperclipTaskMarkdownAssignment; + } + if (redactedWakeContext.paperclipTaskMarkdownAssignmentCompact) { + context.paperclipTaskMarkdownAssignmentCompact = + redactedWakeContext.paperclipTaskMarkdownAssignmentCompact; + } + } + if (issueRef) { + const digest = (value: string | null | undefined) => + value?.trim() + ? createHash("sha256") + .update(value.trim().replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/g, "")) + .digest("hex") + : null; + const redactedIssue = parseObject(context.paperclipIssue); + context.paperclipTurnContext = { + version: 1, + assignment: { + owner: "task_markdown", + description: { + id: issueRef.id, + revision: digest(readNonEmptyString(redactedIssue.description)), + }, + }, + events: { + owner: "wake_prompt", + comments: safeWakeComments.map((comment) => ({ + id: comment.id, + revision: digest(comment.body), + })), + }, + } satisfies PaperclipTurnContext; + } else { + delete context.paperclipTurnContext; + } + return { continuationSummary, executionContinuation, paperclipWakePayload, safeWakeComments, safeWakeCommentContext }; +} diff --git a/tests/e2e/workspace-bootstrap/README.md b/tests/e2e/workspace-bootstrap/README.md index 3bf25b641e7..2d16539cf5e 100644 --- a/tests/e2e/workspace-bootstrap/README.md +++ b/tests/e2e/workspace-bootstrap/README.md @@ -31,5 +31,7 @@ pnpm exec playwright test --config tests/e2e/workspace-bootstrap/playwright.conf The tests prove that one timeout schedules a retry and completes without manual Retry, that persistent timeouts stop after three total runs, and that the final state survives reload. Source edits remain intact and ignored private files are -not copied. Screenshots and failure traces go to `test-results/workspace-bootstrap`. +not copied. The worker also verifies that persisted task context contains the +assignment, the full and compact prompt variants, and the description revision. +Screenshots and failure traces go to `test-results/workspace-bootstrap`. Stop test-drive with Ctrl-C. Its disposable data directory remains for inspection. diff --git a/tests/e2e/workspace-bootstrap/worker.mjs b/tests/e2e/workspace-bootstrap/worker.mjs index 07222bfb2bc..f0b8635d32c 100644 --- a/tests/e2e/workspace-bootstrap/worker.mjs +++ b/tests/e2e/workspace-bootstrap/worker.mjs @@ -3,6 +3,7 @@ import { readFile, readdir, writeFile, access } from "node:fs/promises"; import path from "node:path"; import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; const env = process.env; const headers = { Authorization: `Bearer ${env.PAPERCLIP_API_KEY}`, "X-Paperclip-Run-Id": env.PAPERCLIP_RUN_ID, "Content-Type": "application/json" }; @@ -14,6 +15,15 @@ const api = async (route, method = "GET", body) => { const run = await api(`/heartbeat-runs/${env.PAPERCLIP_RUN_ID}`); assert.equal(run.contextSnapshot.paperclipEnvironment.driver, "local"); const issueId = run.contextSnapshot.issueId; +const issue = await api(`/issues/${issueId}`); +assert.equal(run.contextSnapshot.paperclipIssue.id, issue.id); +assert.equal(run.contextSnapshot.paperclipIssue.description, issue.description); +assert(run.contextSnapshot.paperclipTaskMarkdown.includes(issue.description)); +assert(run.contextSnapshot.paperclipTaskMarkdownAssignment.includes(issue.description)); +assert(!run.contextSnapshot.paperclipTaskMarkdownCompact.includes(issue.description)); +assert(!run.contextSnapshot.paperclipTaskMarkdownAssignmentCompact.includes(issue.description)); +assert.equal(run.contextSnapshot.paperclipTurnContext.assignment.description.revision, + createHash("sha256").update(issue.description.trim()).digest("hex")); const repositories = await readdir(path.join(process.cwd(), ".paperclip-repositories")); const repo = path.join(process.cwd(), ".paperclip-repositories", repositories.find((name) => !name.includes(".clone-"))); assert.equal(await readFile(path.join(repo, "README.md"), "utf8"), "Existing uncommitted work\n");