diff --git a/doc/DATABASE.md b/doc/DATABASE.md index 1dc73cbf9d5..28950218145 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -49,6 +49,12 @@ and leaves all existing cases passive. Source identities are not grants. ### Task query performance +The issue-lock adapter's release read omits `result_json.stdout` and +`result_json.stderr` before transferring a run to the controller. Every other result key, context field, and +stop, retry, lease, and workspace-recovery check stays intact. Full output remains +stored for execution history and detail reads. Terminal telemetry loads only its +run identity, timestamps, usage, and typed task identifier. + Recovery's latest-run queries return the context fields used by recovery decisions. Prompts, execution continuation bodies, and workspace setup payloads remain in the stored run and its full detail reads. These queries also return diff --git a/server/src/__tests__/agent-task-run-telemetry.test.ts b/server/src/__tests__/agent-task-run-telemetry.test.ts index ef036b73e46..87dbf7a26a3 100644 --- a/server/src/__tests__/agent-task-run-telemetry.test.ts +++ b/server/src/__tests__/agent-task-run-telemetry.test.ts @@ -1,6 +1,7 @@ -import { randomUUID } from "node:crypto"; +import { randomBytes, randomUUID } from "node:crypto"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; -import { agents, companies, createDb, heartbeatRuns } from "@paperclipai/db"; +import { agents, companies, createDb, heartbeatRuns, type Db } from "@paperclipai/db"; +import { eq } from "drizzle-orm"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, @@ -23,7 +24,7 @@ vi.mock("@paperclipai/shared/telemetry", async () => { }; }); -import { emitAgentTaskRun } from "../services/agent-task-run-telemetry.ts"; +import { emitAgentTaskRun, emitAgentTaskRunById } from "../services/agent-task-run-telemetry.ts"; const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; @@ -104,6 +105,55 @@ describeEmbeddedPostgres("emitAgentTaskRun", () => { } }); + it("emits the same event from a small scoped run read without loading execution payloads", async () => { + await seedCompanyAndAgent(); + const payload = randomBytes(128 * 1024).toString("base64"); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "succeeded", + startedAt: new Date("2020-01-01T00:00:00Z"), finishedAt: new Date("2020-01-01T00:00:07Z"), + contextSnapshot: { issueId: randomUUID(), prompt: payload }, runnerProfileJson: { instructions: payload }, + stdoutExcerpt: payload, resultJson: { stdout: payload }, usageJson: { model: "fixture", inputTokens: 12, outputTokens: 34, cachedInputTokens: 5 }, + }).returning(); + await emitAgentTaskRun(db, run); + const expected = mockTrackAgentTaskRun.mock.calls[0]; + mockTrackAgentTaskRun.mockClear(); + const reads: unknown[][] = []; + const recorded = new Proxy(db, { get(target, key) { + if (key !== "select") return Reflect.get(target, key); + return (...fields: Parameters) => { + const builder = target.select(...fields), from = builder.from.bind(builder); + builder.from = ((...args: Parameters) => { + const query = from(...args); + if (args[0] === heartbeatRuns) { + const execute = query.execute.bind(query); + query.execute = async (...executeArgs: Parameters) => { + const rows = await execute(...executeArgs); reads.push(rows); return rows; + }; + } + return query; + }) as typeof builder.from; + return builder; + }; + }}); + await emitAgentTaskRunById(recorded, { companyId, runId: run.id }); + expect(mockTrackAgentTaskRun).toHaveBeenCalledExactlyOnceWith(...expected); + expect(reads).toHaveLength(1); + expect(Buffer.byteLength(JSON.stringify(reads))).toBeLessThan(1024); + await emitAgentTaskRunById(recorded, { companyId: randomUUID(), runId: run.id }); + expect(mockTrackAgentTaskRun).toHaveBeenCalledTimes(1); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0]).toEqual(run); + }); + + it.each([null, [], 42, { issueId: 42 }, { issueId: " fixture-task " }, { issueId: "" }].map(context => ({ context })))("preserves telemetry task identity for $context", async ({ context }) => { + await seedCompanyAndAgent(); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "failed", + contextSnapshot: context as Record | null }).returning(); + await emitAgentTaskRun(db, run); + const expected = mockTrackAgentTaskRun.mock.calls[0]; + mockTrackAgentTaskRun.mockClear(); + await emitAgentTaskRunById(db, { companyId, runId: run.id }); + expect(mockTrackAgentTaskRun).toHaveBeenCalledExactlyOnceWith(...expected); + }); + it("omits durationSeconds when the run has no startedAt", async () => { await seedCompanyAndAgent(); const run = { diff --git a/server/src/modules/wake-queue/adapters/postgres.test.ts b/server/src/modules/wake-queue/adapters/postgres.test.ts index 24230989937..16479253a2f 100644 --- a/server/src/modules/wake-queue/adapters/postgres.test.ts +++ b/server/src/modules/wake-queue/adapters/postgres.test.ts @@ -1,5 +1,5 @@ import { instanceSettingsService } from "../../../services/instance-settings.js"; -import { randomUUID } from "node:crypto"; +import { randomBytes, randomUUID } from "node:crypto"; import { eq } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; import type { Db } from "@paperclipai/db"; @@ -177,6 +177,79 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => { return id; } + function recordReleaseRunReads() { + const rows: unknown[][] = []; + function recordSelects>(executor: T): T { + return new Proxy(executor, { get(target, key) { + if (key !== "select") return Reflect.get(target, key); + return (...fields: Parameters) => { + const builder = target.select(...fields), from = builder.from.bind(builder); + builder.from = ((...args: Parameters) => { + const query = from(...args); + if (args[0] === heartbeatRuns) { + const execute = query.execute.bind(query); + query.execute = async (...executeArgs: Parameters) => { + const result = await execute(...executeArgs); rows.push(result); return result; + }; + } + return query; + }) as typeof builder.from; + return builder; + }; + }}); + } + const transaction: Db["transaction"] = (callback, config) => + db.transaction(tx => callback(recordSelects(tx)), config); + const recorded = new Proxy(db, { get(target, key) { + return key === "transaction" ? transaction : Reflect.get(target, key); + }}); + return { recorded, rows }; + } + + it("keeps in-flight work and every recovery receipt while omitting output from release reads", async () => { + const companyId = await seedCompany(); + const agentId = await seedAgent({ companyId }); + const issueId = await seedIssue({ companyId, assigneeAgentId: agentId }); + const runId = await seedRun({ companyId, agentId, status: "running", contextSnapshot: { issueId } }); + const wakeId = await seedDeferredWake({ companyId, agentId, issueId }); + const payload = randomBytes(512 * 1024).toString("base64"); + const evidence = { cancellation: { source: "queued_message", expected: true, initiator: { type: "system" }, reason: "Fixture", recordedAt: "2020-01-01T00:00:00Z" }, + workspaceRestoreFailure: "restore_unsafe_archive", workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [randomUUID()] }, + nativeCancellation: { schema: "paperclip.native-cancellation.v1", runId, companyId, issueId }, + executionCancellation: { state: "acknowledged" }, executionRecovery: { kind: "interrupted", providerStopped: true }, + configurationIncomplete: { reason: "workspace_unavailable" }, unknownFutureReceipt: { retained: true } }; + await db.update(heartbeatRuns).set({ resultJson: { ...evidence, stdout: payload.repeat(3), stderr: payload } }).where(eq(heartbeatRuns.id, runId)); + await db.update(issues).set({ executionRunId: runId, checkoutRunId: runId }).where(eq(issues.id, issueId)); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + const [issue] = await db.select().from(issues).where(eq(issues.id, issueId)); + const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeId)); + const { recorded, rows } = recordReleaseRunReads(); + const result = await createPostgresWakeQueueAdapter(recorded, stubDeps).withIssueExecutionLock( + { companyId, runId, now: new Date(), resumeStrandedQueue: true }, + async () => { throw new Error("in-flight work must not drain"); }, + ); + expect(result.outcome.kind).toBe("released"); + expect(rows).toHaveLength(1); + expect(rows[0]).toEqual([{ ...run, resultJson: evidence }]); + expect(Buffer.byteLength(JSON.stringify(rows))).toBeLessThan(8 * 1024); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)))[0]).toEqual(run); + expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0]).toEqual(issue); + expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeId)))[0]).toEqual(wake); + }); + + it.each([null, ["stdout", { stderr: "array evidence" }], "scalar", 42].map(result => ({ result })))("preserves non-object result evidence $result during release", async ({ result }) => { + const companyId = await seedCompany(), agentId = await seedAgent({ companyId }); + const runId = await seedRun({ companyId, agentId, status: "running" }); + await db.update(heartbeatRuns).set({ resultJson: result as Record | null }).where(eq(heartbeatRuns.id, runId)); + const { recorded, rows } = recordReleaseRunReads(); + await createPostgresWakeQueueAdapter(recorded, stubDeps).withIssueExecutionLock( + { companyId, runId, now: new Date(), resumeStrandedQueue: true }, + async () => { throw new Error("in-flight work must not drain"); }, + ); + expect(rows).toHaveLength(1); + expect((rows[0][0] as typeof heartbeatRuns.$inferSelect).resultJson).toEqual(result); + }); + it.each(["assignee", "mentioned"])("preserves a paused recipient's accepted message until resume (%s)", async kind => { const companyId = await seedCompany(); const ownerId = await seedAgent({ companyId }); diff --git a/server/src/modules/wake-queue/adapters/postgres.ts b/server/src/modules/wake-queue/adapters/postgres.ts index c4929f482ba..6db54a82725 100644 --- a/server/src/modules/wake-queue/adapters/postgres.ts +++ b/server/src/modules/wake-queue/adapters/postgres.ts @@ -5,7 +5,7 @@ import { isCompletedOnboardingHandoffWake } from "../../../services/chat-complet import { instanceSettingsService } from "../../../services/instance-settings.js"; import { currentConversationCommentCondition } from "../../../services/agent-conversations.js"; import { getExecutionBlocker, hasPendingRunCleanup } from "../../../services/execution-blocker.js"; -import { and, asc, eq, inArray, isNull, notInArray, or, sql } from "drizzle-orm"; +import { and, asc, eq, getTableColumns, inArray, isNull, notInArray, or, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { extractIssueReferenceIdentifiers, isUuidLike } from "@paperclipai/shared"; import { @@ -74,6 +74,14 @@ const EXECUTION_PATH_HEARTBEAT_RUN_STATUSES = ["queued", "running", "scheduled_r type HeartbeatRunRow = typeof heartbeatRuns.$inferSelect; type IssueRow = typeof issues.$inferSelect; +const releaseRunFields = { + ...getTableColumns(heartbeatRuns), + resultJson: sql | null>`case + when jsonb_typeof(${heartbeatRuns.resultJson}) = 'object' + then ${heartbeatRuns.resultJson} - 'stdout' - 'stderr' + else ${heartbeatRuns.resultJson} end`, +}; + function normalizeAgentNameKey(value: string | null | undefined): string | null { if (typeof value !== "string") return null; const normalized = value.trim().toLowerCase(); @@ -1051,7 +1059,7 @@ export function createPostgresWakeQueueAdapter(db: Db, deps: WakeQueuePostgresAd return db.transaction(async (rawTx) => { const tx = rawTx as unknown as Db; const run = await tx - .select() + .select(releaseRunFields) .from(heartbeatRuns) .where(and(eq(heartbeatRuns.id, input.runId), eq(heartbeatRuns.companyId, input.companyId))) .then((rows) => rows[0] ?? null); diff --git a/server/src/services/agent-task-run-telemetry.ts b/server/src/services/agent-task-run-telemetry.ts index c7fb031276a..d9df8afeeef 100644 --- a/server/src/services/agent-task-run-telemetry.ts +++ b/server/src/services/agent-task-run-telemetry.ts @@ -1,4 +1,4 @@ -import { and, eq } from "drizzle-orm"; +import { and, eq, sql } from "drizzle-orm"; import { agents, heartbeatRuns, type Db } from "@paperclipai/db"; import { trackAgentTaskRun } from "@paperclipai/shared/telemetry"; import { parseObject } from "../adapters/utils.js"; @@ -6,6 +6,8 @@ import { logger } from "../middleware/logger.js"; import { getTelemetryClient } from "../telemetry.js"; type HeartbeatRun = typeof heartbeatRuns.$inferSelect; +type TelemetryRun = Pick; function readNonEmptyString(value: unknown): string | null { return typeof value === "string" && value.trim().length > 0 @@ -27,7 +29,7 @@ function readUsageToken( } function resolveDurationSeconds( - run: HeartbeatRun, + run: Pick, now: Date, ): number | undefined { if (!run.startedAt) return undefined; @@ -52,7 +54,7 @@ function resolveDurationSeconds( * the caller's return on the slower of the two promises, so it does not * remove the delay. */ -export async function emitAgentTaskRun(db: Db, run: HeartbeatRun): Promise { +export async function emitAgentTaskRun(db: Db, run: TelemetryRun): Promise { try { const client = getTelemetryClient(); if (!client) return; @@ -130,7 +132,15 @@ export async function emitAgentTaskRunById( ): Promise { try { const run = await db - .select() + .select({ + id: heartbeatRuns.id, + agentId: heartbeatRuns.agentId, + status: heartbeatRuns.status, + startedAt: heartbeatRuns.startedAt, + finishedAt: heartbeatRuns.finishedAt, + usageJson: heartbeatRuns.usageJson, + contextSnapshot: sql>`jsonb_build_object('issueId', ${heartbeatRuns.contextSnapshot}->'issueId')`, + }) .from(heartbeatRuns) .where(and(eq(heartbeatRuns.id, input.runId), eq(heartbeatRuns.companyId, input.companyId))) .then((rows) => rows[0] ?? null); diff --git a/server/src/services/recovery/run-result-read.ts b/server/src/services/recovery/run-result-read.ts index 3f2b0ddecdd..93c519a3273 100644 --- a/server/src/services/recovery/run-result-read.ts +++ b/server/src/services/recovery/run-result-read.ts @@ -7,9 +7,6 @@ const recoveryResultKeys = [ "providerQuotaRetryNotBefore", "finalizationReasonCode", ] as const; -/** These three classifiers inspect arbitrary provider error text. Other - * recovery decisions consume the structured fields below, not model output. - * Keep the full stored result and all execution/detail reads unchanged. */ export const recoveryRunResultColumn = sql | null>` case when ${heartbeatRuns.resultJson} is null then null