From 7f454eb377bc3d8d0406f3d4e638580cbdec783e Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 2 Nov 2026 11:47:02 -0600 Subject: [PATCH 01/66] feat(server): add domain-owned retained-history sleep proofs Keep finished delivery and resource history while preserving its downstream work, accounting, cleanup and transport barriers. Share the narrow waiting conversation proof across ordinary and scheduled inspection. Retain local runtime resources through owner reconstruction and process-local inspection. Co-Authored-By: Paperclip --- doc/idle-sleep-safety.md | 50 ++ .../__tests__/idle-completed-history.test.ts | 563 ++++++++++++++++++ .../external-agents/idle-history-proof.ts | 89 +++ server/src/services/browser-use-idle-proof.ts | 49 ++ .../github/idle-history-proof.ts | 9 + .../chat-channels/idle-history-proof.ts | 79 +++ .../chat-channels/slack/idle-history-proof.ts | 16 + .../telegram/idle-history-proof.ts | 12 + .../src/services/conversation-idle-proof.ts | 45 ++ server/src/services/decision-idle-proof.ts | 134 +++++ server/src/services/idle-issue-schedule.ts | 7 +- server/src/services/idle-sleep-safety.ts | 39 +- .../services/public-mcp/idle-history-proof.ts | 37 ++ server/src/services/tool-action-idle-proof.ts | 28 + .../services/workspace-history-idle-proof.ts | 37 ++ server/src/services/workspace-runtime.ts | 32 + 16 files changed, 1205 insertions(+), 21 deletions(-) create mode 100644 server/src/__tests__/idle-completed-history.test.ts create mode 100644 server/src/modules/external-agents/idle-history-proof.ts create mode 100644 server/src/services/browser-use-idle-proof.ts create mode 100644 server/src/services/chat-channels/github/idle-history-proof.ts create mode 100644 server/src/services/chat-channels/idle-history-proof.ts create mode 100644 server/src/services/chat-channels/slack/idle-history-proof.ts create mode 100644 server/src/services/chat-channels/telegram/idle-history-proof.ts create mode 100644 server/src/services/conversation-idle-proof.ts create mode 100644 server/src/services/decision-idle-proof.ts create mode 100644 server/src/services/public-mcp/idle-history-proof.ts create mode 100644 server/src/services/tool-action-idle-proof.ts create mode 100644 server/src/services/workspace-history-idle-proof.ts diff --git a/doc/idle-sleep-safety.md b/doc/idle-sleep-safety.md index 12f8dada5c4..1caf1cbceca 100644 --- a/doc/idle-sleep-safety.md +++ b/doc/idle-sleep-safety.md @@ -81,6 +81,56 @@ Independent SQL writers and additional processes need an explicit wake protocol; process-local notifications do not provide cross-replica delivery. Worker quiet alone is not evidence that the host or database actually suspended. +## Retained completion history + +The retained chat, tool-review, browser, decision, transfer, public-MCP and +external-agent families use their owning domain's completion receipts. Inspection +does not delete history, expire credentials, cancel work or shorten retention. +A terminal label alone is insufficient. Missing, contradictory, malformed or +foreign company/task/run/binding references keep the instance awake. + +Chat input can remain work after processing because its exact wake still owns +the conversation head. Publications retain provider ambiguity, endpoint leases +and file-transfer obligations. Tool reviews retain invocation completion, +interaction projection and continuation delivery. Browser runs require drained +events and settled/released accounting; a stopped browser must belong to the +same closed session. Decision success requires the selected option's full effect +manifest, matching ordinal/type/target receipts and audit records. The owning +dismissal, explicit recovery stop and reviewed archive paths have separate proofs. +Their continuation and archive-notification outboxes remain independent barriers. +Failed company transfers remain resumable. Completed/cancelled transfers still +block while their registered local spool contains cleanup work. + +Waiting agent conversations can be passive containers under both ordinary and +scheduled inspection. The same narrow proof requires the exact conversation +agent/company/user, waiting state, no checkout, active run, accounting, monitor +or pending delivery, and a finished wake for every accepted user-comment outbox +entry. An idle presentation state does not exempt an ordinary runnable task. + +Workspace runtime inspection also reads the owner's process-local resource +state. Startup reconstruction must succeed before that probe becomes quiet. +Adopted/live services, provisioning/control/replacement work, live allocation +claims and quarantined exposure ports remain barriers alongside durable rows. +Reset, failed reconstruction and unresolved desired-state restart cannot reuse an +earlier quiet proof. The probe performs no SQL, network request or cleanup and +adds no timer. A stopped row cannot hide a surviving registered process. + +MCP completion receipts do not exempt their downstream task, subscription or +upload cleanup. Active grants/subscriptions and Dot/Muse bindings/credentials +retain their transport/deadline blockers. Revoked or expired credential history +does not grant authority on the next request. A fenced external assignment is +not worker termination proof, and Muse transport notification is not input +consumption. Released Muse holds require their actual settlement/attestation +path. Dot stop holds remain unsupported because that owner has no completed +worker-stop receipt. Grok Bot shares Dot tables but has independent routine and +dispatch/recovery obligations; even revoked Grok bindings retain their blocker. + +Action/provider variants without an established receipt proof remain blocked. +This includes Photon notice lineage and unqualified legacy action results, +adapter-managed runtime resources, and active persistent transports. These +limits are not evidence of deployed wake support. Existing hosting admission, +generation, accounting, finalization and resource-cleanup checks still apply. + ## Native reply recovery Committed native replies use the shared delivery coordinator and its persisted diff --git a/server/src/__tests__/idle-completed-history.test.ts b/server/src/__tests__/idle-completed-history.test.ts new file mode 100644 index 00000000000..5c3bc152f52 --- /dev/null +++ b/server/src/__tests__/idle-completed-history.test.ts @@ -0,0 +1,563 @@ +import { randomBytes, randomUUID } from "node:crypto"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import net from "node:net"; +import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; +import { eq, sql } from "drizzle-orm"; +import * as tables from "@paperclipai/db"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { completedHistoryChecks, readIdleSleepSafety } from "../services/idle-sleep-safety.js"; +import { readIdleIssueSchedule } from "../services/idle-issue-schedule.js"; +import { companyTransferRunService } from "../services/company-transfer-runs.js"; +import { inspectIdleSpool } from "../services/idle-local-work.js"; + +describe("retained completed history", () => { + let database: Awaited> | undefined; + let db: ReturnType; + const now = new Date(); + const ownerId = randomUUID(); + const hold = () => ({ ownerId, draining: true, startedAt: new Date(now.getTime() - 1000), + expiresAt: new Date(Date.now() + 60_000), activeRuns: 0, pendingWakes: 0 }); + const read = () => readIdleSleepSafety(db, hold, Date.now, ownerId, async () => "none"); + const blocked = async (reason: string) => { + const check = completedHistoryChecks.find(check => check.reason === reason); + if (!check) throw new Error(`Missing family ${reason}`); + const [row] = await db.execute<{ blocked: boolean }>(sql`SELECT EXISTS (${sql.raw(check.query)}) AS blocked`); + return row.blocked; + }; + beforeAll(async () => { + const external = resolveExternalTestDatabaseUrl("PAPERCLIP_HISTORY_TEST_DATABASE_URL"); + if (!external) database = await startEmbeddedPostgresTestDatabase("idle-history-"); + db = tables.createDb(external ?? database!.connectionString, { maxConnections: 4 }); + }, 90_000); + afterEach(async () => { await db.transaction(async tx => { + await tx.execute(sql`SET LOCAL client_min_messages = warning`); + await tx.execute(sql`TRUNCATE companies CASCADE`); + }); }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }); + + async function seed() { + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID(); + await db.insert(tables.companies).values({ id: companyId, name: "Retained history", issuePrefix: randomUUID().slice(0, 8) }); + await db.insert(tables.agents).values({ id: agentId, companyId, name: "Idle agent", status: "idle", adapterType: "process" }); + await db.insert(tables.issues).values({ id: issueId, companyId, title: "Finished work", status: "done", assigneeAgentId: agentId }); + await db.insert(tables.heartbeatRuns).values({ id: runId, companyId, agentId, status: "succeeded", + invocationSource: "on_demand", finishedAt: now, contextSnapshot: { issueId } }); + return { companyId, agentId, issueId, runId }; + } + + async function connection(f: Awaited>) { + const [application] = await db.insert(tables.toolApplications).values({ companyId: f.companyId, + applicationKey: randomUUID(), name: "History fixture", type: "rest_api" }).returning(); + const [connection] = await db.insert(tables.toolConnections).values({ companyId: f.companyId, + applicationId: application.id, name: "Disabled draft", uid: randomUUID(), transport: "rest_api", status: "draft", enabled: false }).returning(); + const [grant] = await db.insert(tables.connectionGrants).values({ companyId: f.companyId, connectionId: connection.id, + kind: "organization", status: "revoked", revokedAt: now }).returning(); + return { connectionId: connection.id, grantId: grant.id }; + } + + async function mcp(f: Awaited>, agent = false) { + const userId = randomUUID(), clientId = randomUUID(); + await db.insert(tables.authUsers).values({ id: userId, name: "History user", email: `${userId}@example.test`, createdAt: now, updatedAt: now }); + await db.insert(tables.mcpOauthClients).values({ id: clientId, name: "History client", redirectUris: ["https://example.test/callback"] }); + const [grant] = await db.insert(tables.mcpOauthGrants).values({ companyId: f.companyId, userId, clientId, + resource: "https://example.test/mcp", purpose: agent ? "agent" : "personal", agentId: agent ? f.agentId : null, + scopes: agent ? ["paperclip:agent", "offline_access"] : ["paperclip:read"], revokedAt: now }).returning(); + return { userId, grantId: grant.id }; + } + + async function decisionFixture() { + const { decisionService } = await import("../services/decisions.js"); + const f = await seed(), p = await mcp(f); + await db.insert(tables.companyMemberships).values({ companyId: f.companyId, principalType: "user", principalId: p.userId, + status: "active", membershipRole: "member" }); + await db.update(tables.heartbeatRuns).set({ status: "running", responsibleUserId: p.userId }).where(eq(tables.heartbeatRuns.id, f.runId)); + await db.update(tables.issues).set({ status: "in_progress", responsibleUserId: p.userId }).where(eq(tables.issues.id, f.issueId)); + const actor = { type: "agent" as const, companyId: f.companyId, agentId: f.agentId, runId: f.runId, source: "agent_jwt" as const, + onBehalfOfUserId: p.userId, onBehalfOfMemberships: [{ companyId: f.companyId, membershipRole: "member", status: "active" }] }; + const userActor = { type: "board" as const, userId: p.userId, companyIds: [f.companyId], source: "session" as const, + memberships: [{ companyId: f.companyId, membershipRole: "member", status: "active" }] }; + const service = decisionService(db, { wakeOriginAgent: async () => null }); + const finish = () => db.update(tables.heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(tables.heartbeatRuns.id, f.runId)); + return { ...f, ...p, actor, userActor, service, finish }; + } + + async function connectedMcp(f: Awaited>) { + const { createPublicMcpOAuth, hashMcpSecret } = await import("../services/public-mcp/oauth.js"); + const { createMcpApiDispatch } = await import("../services/public-mcp/capabilities.js"); + const { issueRoutes } = await import("../routes/issues.js"); + const { createStorageService } = await import("../storage/service.js"); + const { createLocalDiskStorageProvider } = await import("../storage/local-disk-provider.js"); + const { instanceSettingsService } = await import("../services/instance-settings.js"); + const { Router } = await import("express"); + await instanceSettingsService(db).updateExperimental({ enablePublicMcp: true }); + const p = await mcp(f), origin = "https://history.example", resource = `${origin}/mcp/paperclip`; + await db.insert(tables.companyMemberships).values({ companyId: f.companyId, principalType: "user", principalId: p.userId, + membershipRole: "owner", status: "active" }); + await db.update(tables.mcpOauthGrants).set({ revokedAt: null, resource, scopes: ["paperclip:read", "paperclip:write"] }) + .where(eq(tables.mcpOauthGrants.id, p.grantId)); + const token = randomBytes(32).toString("base64url"); + await db.insert(tables.mcpOauthTokens).values({ grantId: p.grantId, kind: "access", tokenHash: hashMcpSecret(token), + expiresAt: new Date(Date.now() + 600_000) }); + const directory = await mkdtemp(path.join(tmpdir(), "history-mcp-storage-")); + const router = Router(); + router.use(issueRoutes(db, createStorageService(createLocalDiskStorageProvider(directory)))); + const oauth = createPublicMcpOAuth(db, { origin, resource }), api = createMcpApiDispatch(router); + return { ...p, token, oauth, api, cleanup: () => rm(directory, { recursive: true, force: true }) }; + } + + it("checks all 32 families on the real migrated schema and keeps an empty instance eligible", async () => { + expect(new Set(completedHistoryChecks.map(c => c.reason)).size).toBe(32); + for (const check of completedHistoryChecks) expect(await blocked(check.reason), check.reason).toBe(false); + expect(await read()).toEqual({ version: 1, backgroundWork: "none" }); + }); + + it.each(["completed", "cancelled"] as const)("retains a %s transfer without hiding its actual spool", async status => { + const f = await seed(); + const result = await companyTransferRunService.resumeOrCreate(db, { companyId: f.companyId, direction: "import", + actorKey: "board:history", idempotencyKey: randomUUID(), containerRef: { kind: "upload" } }); + expect(await blocked("company_transfer_runs")).toBe(true); + if (status === "completed") await companyTransferRunService.complete(db, result.run.id); + else await companyTransferRunService.cancel(db, result.run.id); + expect(await blocked("company_transfer_runs")).toBe(false); + const directory = await mkdtemp(path.join(tmpdir(), "history-spool-")); + try { + await writeFile(path.join(directory, result.run.id), "unfinished cleanup"); + const local = () => inspectIdleSpool(directory); + expect(await readIdleSleepSafety(db, hold, Date.now, ownerId, local)).toEqual({ version: 1, backgroundWork: "present" }); + await rm(path.join(directory, result.run.id)); + expect(await readIdleSleepSafety(db, hold, Date.now, ownerId, local)).toEqual({ version: 1, backgroundWork: "none" }); + expect(await companyTransferRunService.getRun(db, result.run.id)).toMatchObject({ status }); + const next = await companyTransferRunService.resumeOrCreate(db, { companyId: f.companyId, direction: "import", + actorKey: "board:history", idempotencyKey: randomUUID(), containerRef: { kind: "upload" } }); + expect(await blocked("company_transfer_runs")).toBe(true); + await companyTransferRunService.fail(db, next.run.id, "retry remains available"); + expect(await blocked("company_transfer_runs")).toBe(true); + } finally { await rm(directory, { recursive: true, force: true }); } + }); + + it("accepts MCP completion receipts and cleanup while preserving unknown writes and pending uploads", async () => { + const f = await seed(), p = await mcp(f); + const [receipt] = await db.insert(tables.mcpMutationReceipts).values({ ...p, companyId: f.companyId, + operation: "paperclip_create_task", requestId: randomUUID(), argumentsHash: "a", status: "completed", result: { id: f.issueId } }).returning(); + const [upload] = await db.insert(tables.mcpAttachmentUploads).values({ ...p, companyId: f.companyId, taskId: f.issueId, + requestId: randomUUID(), argumentsHash: "a", originalFilename: "history.txt", contentType: "text/plain", byteSize: 1, + sha256: "a".repeat(64), objectKey: "history/abandoned", storageProvider: "local_disk", expiresAt: now, cleanedAt: now }).returning(); + expect(await blocked("mcp_mutation_receipts")).toBe(false); + expect(await blocked("mcp_attachment_uploads")).toBe(false); + expect(await read()).toEqual({ version: 1, backgroundWork: "none" }); + await db.update(tables.mcpMutationReceipts).set({ status: "unknown" }).where(eq(tables.mcpMutationReceipts.id, receipt.id)); + expect(await blocked("mcp_mutation_receipts")).toBe(true); + await db.update(tables.mcpMutationReceipts).set({ status: "completed", result: null }).where(eq(tables.mcpMutationReceipts.id, receipt.id)); + expect(await blocked("mcp_mutation_receipts")).toBe(true); + await db.update(tables.mcpAttachmentUploads).set({ cleanedAt: null }).where(eq(tables.mcpAttachmentUploads.id, upload.id)); + expect(await blocked("mcp_attachment_uploads")).toBe(true); + }); + + it("retains the real MCP mutation receipt and still blocks the actual task it creates", async () => { + const { createPublicMcpExecutor } = await import("../services/public-mcp/capabilities.js"); + const f = await seed(), p = await connectedMcp(f); + try { + // Pausing is the product's ordinary admission control. No provider turn + // is run merely to exercise a mutation receipt or task creation. + await db.update(tables.agents).set({ status: "paused" }).where(eq(tables.agents.id, f.agentId)); + const execute = createPublicMcpExecutor(db, p.oauth, p.api); + const input = { companyId: f.companyId, title: "Actual MCP producer", description: "Retained receipt", + requestId: randomUUID(), assigneeAgentId: f.agentId }; + const result = await execute(p.token, "paperclip_create_task", input); + expect(result).toHaveProperty("task.status", "todo"); + expect(await blocked("mcp_mutation_receipts")).toBe(false); + expect(await execute(p.token, "paperclip_create_task", input)).toEqual(result); + expect(await db.select().from(tables.mcpMutationReceipts)).toHaveLength(1); + expect(await read()).toEqual({ version: 1, backgroundWork: "present" }); + const receipt = (await db.select().from(tables.mcpMutationReceipts))[0]; + await db.update(tables.mcpMutationReceipts).set({ status: "reserved" }).where(eq(tables.mcpMutationReceipts.id, receipt.id)); + expect(await blocked("mcp_mutation_receipts")).toBe(true); + } finally { await p.cleanup(); } + }, 60_000); + + it.each([200, 410])("uses actual MCP subscription and callback delivery receipts for HTTP %s", async status => { + const { createPublicMcpEvents } = await import("../services/public-mcp/events.js"); + const { logActivity } = await import("../services/activity-log.js"); + const f = await seed(), p = await connectedMcp(f); + try { + let received = 0; + const events = createPublicMcpEvents(db, p.oauth, p.api, { fetch: async (_url, init) => { + const body = JSON.parse(String(init.body)); + if (body.type === "verification") return Response.json({ challenge: body.challenge }); + received += 1; + return new Response(null, { status }); + } }); + const principal = await p.oauth.authenticate(p.token); + await events.subscribe(principal, { name: "paperclip.task.status_changed", arguments: { companyId: f.companyId, taskId: f.issueId }, + delivery: { mode: "webhook", url: "https://receiver.example/events", secret: `whsec_${randomBytes(32).toString("base64")}` } }); + await logActivity(db, { companyId: f.companyId, actorType: "system", actorId: "history", + action: "issue.updated", entityType: "issue", entityId: f.issueId, details: { status: "done" } }); + await events.tick(); + expect(received).toBeGreaterThan(0); + expect(await blocked("mcp_event_deliveries")).toBe(false); + expect((await db.select().from(tables.mcpEventDeliveries)).every(row => row.finishedAt !== null)).toBe(true); + expect(await blocked("mcp_event_subscriptions")).toBe(status !== 410); + expect(await blocked("mcp_oauth_grants")).toBe(true); + } finally { await p.cleanup(); } + }, 60_000); + + it.each(["delivered", "filtered", "inactive", "delivery_exhausted", "http_400", "http_503"])("retires finished MCP event outcome %s only with its source", async outcome => { + const f = await seed(), p = await mcp(f); + const subscriptionId = randomUUID(); + await db.insert(tables.mcpEventSubscriptions).values({ id: subscriptionId, companyId: f.companyId, grantId: p.grantId, + name: "paperclip.task.updated", taskId: f.issueId, arguments: { taskId: f.issueId }, deliveryMaterial: {}, verifiedAt: now, + expiresAt: now, stoppedAt: now }); + const [activity] = await db.insert(tables.activityLog).values({ companyId: f.companyId, actorType: "system", actorId: "history", + action: "issue.updated", entityType: "issue", entityId: f.issueId }).returning(); + const [delivery] = await db.insert(tables.mcpEventDeliveries).values({ subscriptionId, activityId: activity.id, + event: {}, finishedAt: now, outcome, attempts: 6 }).returning(); + expect(await blocked("mcp_event_subscriptions")).toBe(false); + expect(await blocked("mcp_event_deliveries")).toBe(false); + await db.update(tables.mcpEventDeliveries).set({ finishedAt: null }).where(eq(tables.mcpEventDeliveries.id, delivery.id)); + expect(await blocked("mcp_event_deliveries")).toBe(true); + }); + + it("keeps browser accounting, run ownership and provider resource cleanup independent", async () => { + const f = await seed(), c = await connection(f), invocationId = randomUUID(); + const [session] = await db.insert(tables.browserUseSessions).values({ ...f, ...c, status: "closed", providerSessionId: randomUUID() }).returning(); + const [run] = await db.insert(tables.browserUseRuns).values({ companyId: f.companyId, sessionId: session.id, heartbeatRunId: f.runId, + invocationId, providerRunId: randomUUID(), status: "completed", eventsDrained: 1, + result: { paperclipBudgetHold: { invocationId, state: "settled", accountedCentsExact: "0.0000000" } } }).returning(); + const [browser] = await db.insert(tables.browserUseBrowsers).values({ companyId: f.companyId, sessionId: session.id, + providerBrowserId: randomUUID(), status: "stopped" }).returning(); + for (const reason of ["browser_use_sessions", "browser_use_runs", "browser_use_browsers"]) expect(await blocked(reason)).toBe(false); + await db.update(tables.browserUseRuns).set({ eventsDrained: 0 }).where(eq(tables.browserUseRuns.id, run.id)); + expect(await blocked("browser_use_sessions")).toBe(true); + expect(await blocked("browser_use_runs")).toBe(true); + await db.update(tables.browserUseRuns).set({ eventsDrained: 1 }).where(eq(tables.browserUseRuns.id, run.id)); + await db.update(tables.browserUseBrowsers).set({ status: "active" }).where(eq(tables.browserUseBrowsers.id, browser.id)); + expect(await blocked("browser_use_sessions")).toBe(true); + expect(await blocked("browser_use_browsers")).toBe(true); + }); + + it("uses one conversation proof for ordinary and scheduled reports and detects the comment-to-wake crash window", async () => { + const { issueService } = await import("../services/issues.js"); + const { settleConversationTurn } = await import("../services/agent-conversations.js"); + const { instanceSettingsService } = await import("../services/instance-settings.js"); + await instanceSettingsService(db).updateExperimental({ enableAgentChat: true }); + const f = await seed(); + const service = issueService(db); + const conversation = await service.create(f.companyId, { title: "History chat", assigneeAgentId: f.agentId, + conversationAgentId: f.agentId, conversationUserId: "local-board", conversationState: "waiting", status: "in_review" }); + expect(await read()).toEqual({ version: 1, backgroundWork: "none" }); + expect((await readIdleIssueSchedule(db, ownerId, Date.now())).blocked).toBe(false); + const comment = await service.addComment(conversation.id, "A new question", { userId: "local-board" }, + { clientRequestId: randomUUID() }); + expect(await read()).toEqual({ version: 1, backgroundWork: "present" }); + // Preserve the original comment but reproduce the crash before enqueue. + await db.update(tables.issues).set({ conversationState: "waiting" }).where(eq(tables.issues.id, conversation.id)); + expect((await readIdleIssueSchedule(db, ownerId, Date.now())).blocked).toBe(true); + const runId = randomUUID(); + const [run] = await db.insert(tables.heartbeatRuns).values({ id: runId, companyId: f.companyId, agentId: f.agentId, + invocationSource: "on_demand", status: "succeeded", finishedAt: now, + contextSnapshot: { issueId: conversation.id, commentId: comment.id, conversationSessionGeneration: 0 } }).returning(); + await db.insert(tables.agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agentId, + idempotencyKey: `conversation-comment:${comment.id}`, source: "on_demand", triggerDetail: "manual", reason: "issue_commented", + status: "completed", finishedAt: now, runId, payload: { issueId: conversation.id } }); + await db.insert(tables.issueComments).values({ companyId: f.companyId, issueId: conversation.id, + authorAgentId: f.agentId, createdByRunId: runId, body: "The question is answered." }); + await db.update(tables.issues).set({ conversationState: "active", status: "in_progress" }).where(eq(tables.issues.id, conversation.id)); + expect(await settleConversationTurn(db, run)).toBe(true); + expect(await read()).toEqual({ version: 1, backgroundWork: "none" }); + expect((await readIdleIssueSchedule(db, ownerId, Date.now())).blocked).toBe(false); + await db.update(tables.heartbeatRuns).set({ costAccountingPending: true }).where(eq(tables.heartbeatRuns.id, runId)); + expect(await read()).toEqual({ version: 1, backgroundWork: "present" }); + }, 60_000); + + it.each(["rejected", "withdrawn", "expired"] as const)("accepts actual %s secret-proposal scrubbing and refuses retained ciphertext", async status => { + const { createSecretProposalsService } = await import("../services/secret-proposals.js"); + const f = await seed(); + const [proposal] = await db.insert(tables.companySecretProposals).values({ companyId: f.companyId, kind: "secret", + proposedName: "history/token", proposedKey: "TOKEN", justification: "History fixture", proposedByAgentId: f.agentId, + originRunId: f.runId, originIssueId: f.issueId, expiresAt: now, valueCiphertext: { pending: true } }).returning(); + expect(await blocked("company_secret_proposals")).toBe(true); + const result = await createSecretProposalsService(db).transition(f.companyId, proposal.id, status, { proposerAgentId: f.agentId }); + expect(result).toMatchObject({ status, valueCiphertext: null }); + expect(await blocked("company_secret_proposals")).toBe(false); + await db.update(tables.companySecretProposals).set({ valueCiphertext: { unfinished: true } }).where(eq(tables.companySecretProposals.id, proposal.id)); + expect(await blocked("company_secret_proposals")).toBe(true); + }, 60_000); + + it("retains actual decided effects, and reblocks when one effect lacks its completion receipt", async () => { + const { decisionService } = await import("../services/decisions.js"); + const previous = process.env.PAPERCLIP_DECISION_SIGNING_SECRET; + process.env.PAPERCLIP_DECISION_SIGNING_SECRET = "0123456789abcdef0123456789abcdef"; + try { + const f = await seed(), p = await mcp(f); + await db.insert(tables.companyMemberships).values({ companyId: f.companyId, principalType: "user", principalId: p.userId, + status: "active", membershipRole: "member" }); + await db.update(tables.heartbeatRuns).set({ status: "running", responsibleUserId: p.userId }).where(eq(tables.heartbeatRuns.id, f.runId)); + await db.update(tables.issues).set({ status: "in_progress", responsibleUserId: p.userId }).where(eq(tables.issues.id, f.issueId)); + const actor = { type: "agent" as const, companyId: f.companyId, agentId: f.agentId, runId: f.runId, source: "agent_jwt" as const, + onBehalfOfUserId: p.userId, onBehalfOfMemberships: [{ companyId: f.companyId, membershipRole: "member", status: "active" }] }; + const userActor = { type: "board" as const, userId: p.userId, companyIds: [f.companyId], source: "session" as const, + memberships: [{ companyId: f.companyId, membershipRole: "member", status: "active" }] }; + const service = decisionService(db, { wakeOriginAgent: async () => null }); + const decision = await service.create({ companyId: f.companyId, actor, agentId: f.agentId, runId: f.runId, + title: "Save this comment?", body: "Completion proof", continuationPolicy: "none", + options: [{ id: "yes", label: "Yes", effects: [{ type: "comment_on_issue", targetIssueId: f.issueId, + staleness: "lenient", bodyMarkdown: "Saved by the decision owner." }] }] }); + expect(await blocked("decisions")).toBe(true); + await service.decide({ id: decision.id, optionId: "yes", decidedByUserId: p.userId, userActor }); + await db.update(tables.heartbeatRuns).set({ status: "succeeded", finishedAt: now }).where(eq(tables.heartbeatRuns.id, f.runId)); + expect(await blocked("decisions")).toBe(false); + expect(await blocked("decision_effect_executions")).toBe(false); + await db.update(tables.decisions).set({ chosenOptionId: "another-option" }).where(eq(tables.decisions.id, decision.id)); + expect(await blocked("decisions")).toBe(true); + await db.update(tables.decisions).set({ chosenOptionId: "yes" }).where(eq(tables.decisions.id, decision.id)); + const [effect] = await db.select().from(tables.decisionEffectExecutions).where(eq(tables.decisionEffectExecutions.decisionId, decision.id)); + await db.delete(tables.decisionEffectExecutions).where(eq(tables.decisionEffectExecutions.id, effect.id)); + expect(await blocked("decisions")).toBe(true); + await db.insert(tables.decisionEffectExecutions).values(effect); + await db.update(tables.decisionEffectExecutions).set({ executedAt: null }).where(eq(tables.decisionEffectExecutions.decisionId, decision.id)); + expect(await blocked("decisions")).toBe(true); + expect(await blocked("decision_effect_executions")).toBe(true); + } finally { + if (previous === undefined) delete process.env.PAPERCLIP_DECISION_SIGNING_SECRET; + else process.env.PAPERCLIP_DECISION_SIGNING_SECRET = previous; + } + }, 60_000); + + it.each(["zero_effect", "dismissed", "cancelled", "expired", "partial", "failed"] as const)("proves the actual decision owner branch %s", async branch => { + const previous = process.env.PAPERCLIP_DECISION_SIGNING_SECRET; + process.env.PAPERCLIP_DECISION_SIGNING_SECRET = "0123456789abcdef0123456789abcdef"; + try { + const f = await decisionFixture(); + const effects = branch === "zero_effect" ? [] : [ + { type: "comment_on_issue" as const, targetIssueId: f.issueId, staleness: "lenient" as const, bodyMarkdown: "First effect" }, + ...(branch === "partial" ? [{ type: "comment_on_issue" as const, targetIssueId: f.issueId, + staleness: "strict" as const, bodyMarkdown: "Snapshot is now stale" }] : []), + ]; + const decision = await f.service.create({ companyId: f.companyId, actor: f.actor, agentId: f.agentId, runId: f.runId, + title: "Decision branch", body: branch, continuationPolicy: "none", options: [{ id: "yes", label: "Yes", effects }] }); + if (branch === "dismissed") await f.service.dismiss(decision.id, f.userId, f.userActor); + else if (branch === "cancelled") await f.service.cancel(decision.id, { actorType: "agent", actorId: f.agentId, runId: f.runId }); + else if (branch === "expired") { + await db.update(tables.decisions).set({ expiresAt: new Date(0) }).where(eq(tables.decisions.id, decision.id)); + await f.service.sweepTtlExpiry(); + } else await f.service.decide({ id: decision.id, optionId: "yes", decidedByUserId: f.userId, + userActor: branch === "failed" ? { type: "none", source: "none" } : f.userActor }); + await f.finish(); + const saved = await f.service.get(decision.id); + expect(saved?.executionStatus).toBe(branch === "partial" ? "partial" : branch === "failed" ? "failed" + : branch === "cancelled" || branch === "expired" ? null : "succeeded"); + expect(await blocked("decisions")).toBe(false); + expect(await blocked("decision_effect_executions")).toBe(false); + } finally { + if (previous === undefined) delete process.env.PAPERCLIP_DECISION_SIGNING_SECRET; else process.env.PAPERCLIP_DECISION_SIGNING_SECRET = previous; + } + }, 60_000); + + it.each(["archive", "keep", "manifest_failure"] as const)("checks the separate reviewed retention manifest for %s", async branch => { + const { hashAttentionArchiveManifest } = await import("../services/decision-retention.js"); + const previous = process.env.PAPERCLIP_DECISION_SIGNING_SECRET; + process.env.PAPERCLIP_DECISION_SIGNING_SECRET = "0123456789abcdef0123456789abcdef"; + try { + const f = await decisionFixture(), activityAt = new Date("2026-04-01T00:00:00.000Z"); + await db.insert(tables.decisionRetention).values({ companyId: f.companyId, sourceKind: "review", sourceId: f.issueId, + sourceActivityAt: activityAt, version: 1 }); + const manifest = [{ companyId: f.companyId, sourceKind: "review" as const, sourceId: f.issueId, + expectedVersion: 1, activityAt: activityAt.toISOString(), reason: "Reviewed history" }]; + const decision = await f.service.create({ companyId: f.companyId, actor: f.actor, agentId: f.agentId, runId: f.runId, + title: "Archive reviewed set", body: "Only the selected source", continuationPolicy: "none", + options: [{ id: "archive", label: "Archive", style: "destructive", effects: [] }, { id: "keep", label: "Keep", effects: [] }], + metadata: { kind: "attention_archive_proposal", manifestHash: branch === "manifest_failure" ? "invalid" : hashAttentionArchiveManifest(manifest) }, + additionalTargetSnapshots: { [`attention:review:${f.issueId}`]: { status: "attention", assigneeAgentId: null, + assigneeUserId: null, updatedAt: activityAt.toISOString(), attentionArchive: manifest[0] } } }); + const outcome = await f.service.decide({ id: decision.id, optionId: branch === "keep" ? "keep" : "archive", + decidedByUserId: f.userId, userActor: f.userActor }); + expect(outcome.executionStatus).toBe(branch === "manifest_failure" ? "failed" : "succeeded"); + await f.finish(); + expect(await blocked("decisions")).toBe(false); + if (branch === "archive") { + await db.update(tables.decisions).set({ targetSnapshots: {} }).where(eq(tables.decisions.id, decision.id)); + expect(await blocked("decisions")).toBe(true); + } + } finally { + if (previous === undefined) delete process.env.PAPERCLIP_DECISION_SIGNING_SECRET; else process.env.PAPERCLIP_DECISION_SIGNING_SECRET = previous; + } + }, 60_000); + + it("settles retained Muse assignments through the real pair, runner receipt and terminal reconciliation paths", async () => { + const { instanceSettingsService } = await import("../services/instance-settings.js"); + const { museIdentity, museCredentialHash } = await import("../services/muse-identity.js"); + const { museRunnerBroker } = await import("../services/muse-runner-broker.js"); + const { createHeartbeatLifecycle } = await import("../services/heartbeat/run-lifecycle.js"); + const f = await seed(), p = await mcp(f); + await instanceSettingsService(db).updateExperimental({ enableNativeRunner: true, enableMuse: true }); + await db.insert(tables.companyMemberships).values([ + { companyId: f.companyId, principalType: "user", principalId: p.userId, membershipRole: "owner", status: "active" }, + { companyId: f.companyId, principalType: "agent", principalId: f.agentId, membershipRole: "member", status: "active" }, + ]); + const adapterConfig = { provider: "muse", allowUnmeteredProvider: true, lifecycle: "per_turn" }; + await db.update(tables.agents).set({ status: "active", adapterType: "paperclip_runner", adapterConfig }).where(eq(tables.agents.id, f.agentId)); + const ticket = randomBytes(32).toString("base64url"); + const [binding] = await db.insert(tables.museAgentBindings).values({ companyId: f.companyId, agentId: f.agentId, operatorId: p.userId, + ticketHash: museCredentialHash(ticket), ticketExpiresAt: new Date(Date.now() + 600_000) }).returning(); + await db.update(tables.agents).set({ adapterConfig: { ...adapterConfig, museBindingId: binding.id } }).where(eq(tables.agents.id, f.agentId)); + const identity = museIdentity(db), credentials = await identity.pair(ticket, "2"); + const broker = museRunnerBroker(db, { publicOrigin: "https://paperclip.example", + heartbeat: { wakeup: async () => null, cancelRun: async () => null } }); + await db.update(tables.museAgentBindings).set({ status: "ready", receiverContactAt: new Date(), verifiedReplyAt: new Date() }) + .where(eq(tables.museAgentBindings.id, binding.id)); + const subject = await identity.authenticate(credentials.accessToken), sessionId = randomUUID(), turnId = `turn_${randomUUID()}`; + await db.update(tables.issues).set({ status: "in_progress", responsibleUserId: p.userId, + checkoutRunId: f.runId, executionRunId: f.runId }).where(eq(tables.issues.id, f.issueId)); + await db.update(tables.heartbeatRuns).set({ status: "running", runtimeMode: "native", driverKind: "muse_external", + nativeIssueId: f.issueId, nativeSessionId: sessionId, nativePhase: "turn_running", responsibleUserId: p.userId, finishedAt: null }) + .where(eq(tables.heartbeatRuns.id, f.runId)); + await db.insert(tables.nativeRunFinalizations).values({ runId: f.runId, companyId: f.companyId, issueId: f.issueId, + phase: "running", controllerGeneration: 1, leaseExpiresAt: new Date(Date.now() + 600_000) }); + const execution = { binding: { companyId: f.companyId, agentId: f.agentId, runId: f.runId }, + provider: { binding: await broker.snapshot(f.companyId, f.agentId, binding.id) } }; + const port = broker.port(execution); + const ref = { ...execution.provider.binding, runId: f.runId, normalizedSessionId: sessionId, turnId, assignmentRevision: 1 }; + const detach = await port.attach(async command => { + // Named external runner boundary: persist the real broker's receipt for + // the exact command it emitted, without invoking paid native execution. + await port.settle({ sourceEventId: `result_${command.requestId}`, payload: { binding: ref, requestId: command.requestId, + outcome: { status: command.action === "accept" ? "accepted" : "completed" } } }); + }); + try { + await port.dispatch({ sourceEventId: `offer_${randomUUID()}`, payload: { kind: "assignment", binding: ref, tools: [], + acceptByUnixMs: Date.now() + 600_000, expiresAtUnixMs: Date.now() + 3_600_000 } }); + const [assignment] = await db.select().from(tables.museRunnerAssignments).where(eq(tables.museRunnerAssignments.runId, f.runId)); + expect(await blocked("muse_runner_assignments")).toBe(true); + await broker.act(subject, { version: 1, command: "accept", assignmentId: assignment.id, requestId: randomUUID() }); + await broker.act(subject, { version: 1, command: "finish", assignmentId: assignment.id, requestId: randomUUID(), result: {} }); + expect(await blocked("muse_runner_assignments")).toBe(true); + const lifecycle = createHeartbeatLifecycle(db, { emitTerminalAgentTaskRun: () => {} } as unknown as + import("../services/heartbeat/run-lifecycle.js").HeartbeatLifecycleDependencies); + await lifecycle.setRunStatus(f.runId, "succeeded", { finishedAt: new Date() }); + await broker.reconcileTerminalAssignments(f.companyId, f.agentId, binding.id); + for (const family of ["muse_runner_assignments", "muse_runner_operations", "muse_mailbox_items", "external_agent_holds"]) { + expect(await blocked(family), family).toBe(false); + } + await db.update(tables.museRunnerOperations).set({ status: "unknown" }).where(eq(tables.museRunnerOperations.assignmentId, assignment.id)); + expect(await blocked("muse_runner_assignments")).toBe(true); + expect(await blocked("muse_runner_operations")).toBe(true); + } finally { await detach(); } + }, 60_000); + + it("uses the actual Dot revocation writer but preserves Grok routine and recovery blockers", async () => { + const { dotRunnerBroker } = await import("../services/dot-runner-broker.js"); + const f = await seed(), p = await mcp(f, true); + await db.update(tables.mcpOauthGrants).set({ revokedAt: null }).where(eq(tables.mcpOauthGrants.id, p.grantId)); + const [binding] = await db.insert(tables.dotAgentBindings).values({ companyId: f.companyId, agentId: f.agentId, + operatorId: p.userId, grantId: p.grantId, status: "ready", providerKind: "openai_dot" }).returning(); + expect(await blocked("dot_agent_bindings")).toBe(true); + await dotRunnerBroker(db).revoke(f.companyId, f.agentId, p.userId); + expect(await blocked("dot_agent_bindings")).toBe(false); + expect(await blocked("mcp_oauth_grants")).toBe(false); + const [secret] = await db.insert(tables.companySecrets).values({ companyId: f.companyId, key: "GROK_ROUTINE", + name: "Grok routine material", provider: "local_encrypted" }).returning(); + const [grok] = await db.insert(tables.dotAgentBindings).values({ companyId: f.companyId, agentId: f.agentId, + operatorId: p.userId, status: "revoked", providerKind: "grok_bot", revokedAt: now }).returning(); + await db.insert(tables.grokBotRoutines).values({ companyId: f.companyId, agentId: f.agentId, bindingId: grok.id, + bindingGeneration: grok.generation, botName: "History routine", secretId: secret.id, secretVersion: 1, + recoveryRequiredAt: new Date(), recoveryReason: "work_session" }); + expect(await blocked("dot_agent_bindings")).toBe(true); + expect(await read()).toEqual({ version: 1, backgroundWork: "present" }); + expect(await db.select().from(tables.grokBotRoutines)).toHaveLength(1); + expect((await db.select().from(tables.dotAgentBindings).where(eq(tables.dotAgentBindings.id, binding.id)))[0].status).toBe("revoked"); + }, 60_000); + + it("retains a stale runtime exclusivity lease only after its exact owner and resources finish", async () => { + const { workspaceRuntimeLeaseService } = await import("../services/workspace-runtime-leases.js"); + const f = await seed(); + const [workspace] = await db.insert(tables.executionWorkspaces).values({ companyId: f.companyId, name: "History workspace", + mode: "shared_workspace", strategyType: "project_primary", providerType: "local_fs" }).returning(); + const lease = await workspaceRuntimeLeaseService(db).claim({ companyId: f.companyId, executionWorkspaceId: workspace.id, + action: "start", owner: { actorType: "agent", agentId: f.agentId, issueId: f.issueId, runId: f.runId } }); + expect(lease.outcome).toBe("created"); + expect(await blocked("execution_workspace_runtime_leases")).toBe(false); + const [runtime] = await db.insert(tables.workspaceRuntimeServices).values({ id: randomUUID(), companyId: f.companyId, + executionWorkspaceId: workspace.id, issueId: f.issueId, serviceName: "Unstarted history", scopeType: "execution_workspace", + scopeId: workspace.id, status: "stopped", lifecycle: "shared", provider: "local_process", stoppedAt: now }).returning(); + expect(await blocked("workspace_runtime_services")).toBe(false); + await db.update(tables.workspaceRuntimeServices).set({ status: "running", stoppedAt: null }).where(eq(tables.workspaceRuntimeServices.id, runtime.id)); + expect(await blocked("workspace_runtime_services")).toBe(true); + expect(await blocked("execution_workspace_runtime_leases")).toBe(true); + await db.update(tables.workspaceRuntimeServices).set({ status: "stopped", stoppedAt: now, exposureHandle: "uncleaned" }) + .where(eq(tables.workspaceRuntimeServices.id, runtime.id)); + expect(await blocked("workspace_runtime_services")).toBe(true); + await db.update(tables.executionWorkspaceRuntimeLeases).set({ ownerKey: `issue:${randomUUID()}` }) + .where(eq(tables.executionWorkspaceRuntimeLeases.id, lease.lease!.id)); + expect(await blocked("execution_workspace_runtime_leases")).toBe(true); + }); + + it("inspects an actual owned runtime process across stopped-row fallback, adoption, termination and reconstruction failure", async () => { + const runtime = await import("../services/workspace-runtime.js"); + const supervisor = await import("../services/local-service-supervisor.js"); + let ownedProcess: Awaited> = null; + const directory = await mkdtemp(path.join(tmpdir(), "history-owned-runtime-")); + const previousHome = process.env.PAPERCLIP_HOME, previousInstance = process.env.PAPERCLIP_INSTANCE_ID; + process.env.PAPERCLIP_HOME = directory; + process.env.PAPERCLIP_INSTANCE_ID = "history-owned-runtime"; + const f = await seed(); + const [workspace] = await db.insert(tables.executionWorkspaces).values({ companyId: f.companyId, name: "Owned local runtime", + mode: "shared_workspace", strategyType: "project_primary", providerType: "local_fs", cwd: directory }).returning(); + try { + await runtime.resetRuntimeServicesForTests({ terminateProcesses: true }); + expect(runtime.readWorkspaceRuntimeIdleWork()).toBe("unknown"); + await runtime.reconcilePersistedRuntimeServicesOnStartup(db); + expect(runtime.readWorkspaceRuntimeIdleWork()).toBe("none"); + const server = net.createServer(); + await new Promise((resolve, reject) => { server.once("error", reject); server.listen(0, "127.0.0.1", resolve); }); + const port = (server.address() as net.AddressInfo).port; + await new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())); + const command = "node -e \"require('node:http').createServer((req,res)=>res.end('history')).listen(Number(process.env.PORT),'127.0.0.1')\""; + const refs = await runtime.startRuntimeServicesForWorkspaceControl({ db, + actor: { id: null, name: "History operator", companyId: f.companyId }, issue: null, + executionWorkspaceId: workspace.id, adapterEnv: {}, + workspace: { baseCwd: directory, source: "project_primary", projectId: null, workspaceId: null, + repoUrl: null, repoRef: null, strategy: "project_primary", cwd: directory, branchName: null, + worktreePath: null, warnings: [], created: false, branchCreatedByRuntime: false }, + config: { desiredState: "running", workspaceRuntime: { services: [{ name: "history-http", command, port, + expose: { urlTemplate: "http://127.0.0.1:{{port}}" }, + readiness: { type: "http", urlTemplate: "http://127.0.0.1:{{port}}", timeoutSec: 10, intervalMs: 50 }, + lifecycle: "shared", reuseScope: "execution_workspace", stopPolicy: { type: "manual" } }] } } }); + expect(refs).toHaveLength(1); + ownedProcess = await supervisor.findLocalServiceRegistryRecordByRuntimeServiceId({ runtimeServiceId: refs[0].id, profileKind: "workspace-runtime" }); + expect(ownedProcess?.pid).toBeGreaterThan(0); + expect(runtime.readWorkspaceRuntimeIdleWork()).toBe("present"); + await db.update(tables.workspaceRuntimeServices).set({ status: "stopped", stoppedAt: new Date(), url: null }) + .where(eq(tables.workspaceRuntimeServices.id, refs[0].id)); + expect(await blocked("workspace_runtime_services")).toBe(false); + expect(await readIdleSleepSafety(db, hold, Date.now, ownerId, async () => runtime.readWorkspaceRuntimeIdleWork())) + .toEqual({ version: 1, backgroundWork: "present" }); + await runtime.resetRuntimeServicesForTests({ simulateSupervisorExit: true }); + expect(runtime.readWorkspaceRuntimeIdleWork()).toBe("unknown"); + const adopted = await runtime.reconcilePersistedRuntimeServicesOnStartup(db); + expect(adopted.adopted).toBe(1); + expect(runtime.readWorkspaceRuntimeIdleWork()).toBe("present"); + await runtime.stopRuntimeServicesForExecutionWorkspace({ db, executionWorkspaceId: workspace.id, workspaceCwd: directory }); + ownedProcess = null; + expect(runtime.readWorkspaceRuntimeIdleWork()).toBe("none"); + expect(await blocked("workspace_runtime_services")).toBe(false); + const unavailable = Object.create(db) as typeof db; + unavailable.select = () => { throw new Error("controlled startup database failure"); }; + await expect(runtime.reconcilePersistedRuntimeServicesOnStartup(unavailable)).rejects.toThrow("controlled startup"); + expect(runtime.readWorkspaceRuntimeIdleWork()).toBe("unknown"); + await runtime.reconcilePersistedRuntimeServicesOnStartup(db); + expect(runtime.readWorkspaceRuntimeIdleWork()).toBe("none"); + await runtime.resetRuntimeServicesForTests(); + expect(runtime.readWorkspaceRuntimeIdleWork()).toBe("unknown"); + } finally { + await runtime.resetRuntimeServicesForTests({ terminateProcesses: true }); + if (ownedProcess) await supervisor.terminateLocalService({ pid: ownedProcess.pid, + processGroupId: ownedProcess.processGroupId ?? ownedProcess.pid, port: ownedProcess.port }); + if (previousHome === undefined) delete process.env.PAPERCLIP_HOME; else process.env.PAPERCLIP_HOME = previousHome; + if (previousInstance === undefined) delete process.env.PAPERCLIP_INSTANCE_ID; else process.env.PAPERCLIP_INSTANCE_ID = previousInstance; + await rm(directory, { recursive: true, force: true }); + } + }, 60_000); +}); diff --git a/server/src/modules/external-agents/idle-history-proof.ts b/server/src/modules/external-agents/idle-history-proof.ts new file mode 100644 index 00000000000..cbed05b1f77 --- /dev/null +++ b/server/src/modules/external-agents/idle-history-proof.ts @@ -0,0 +1,89 @@ +import { finishedIdleRun } from "../../services/idle-work-proof.js"; + +function completedAssignment(alias: string, provider: "dot" | "muse"): string { + const muse = provider === "muse"; + const identity = muse ? 'muse' : 'openai_dot'; + return `EXISTS (SELECT 1 FROM ${provider}_agent_bindings binding JOIN heartbeat_runs run + ON run.id = ${alias}.run_id AND run.company_id = binding.company_id AND run.agent_id = binding.agent_id + WHERE binding.id = ${alias}.binding_id AND binding.company_id = ${alias}.company_id + ${muse ? '' : "AND binding.provider_kind = 'openai_dot'"} + AND binding.agent_id = ${alias}.agent_id AND ${alias}.binding_generation > 0 + AND ${alias}.binding_generation <= binding.generation AND ${finishedIdleRun('run')} + AND ((${alias}.status = 'settled' AND ${muse + ? `run.status = 'succeeded' AND ${alias}.accepted_result_at IS NOT NULL AND ${alias}.finalized_at = run.finished_at` + : `${alias}.settled_at IS NOT NULL`}) + OR (${alias}.status = 'fenced' AND (${alias}.${muse ? 'claimed_at' : 'accepted_at'} IS NULL + OR EXISTS (SELECT 1 FROM external_agent_holds h WHERE h.provider = '${identity}' + AND ${muse ? 'true' : 'false'} + AND h.assignment_id = ${alias}.id AND h.company_id = ${alias}.company_id + AND h.agent_id = ${alias}.agent_id AND h.run_id = ${alias}.run_id + AND h.binding_id = ${alias}.binding_id AND h.binding_generation = ${alias}.binding_generation + AND h.released_at IS NOT NULL AND NOT h.worker_unknown AND NOT h.native_effects_unknown)))) + AND NOT EXISTS (SELECT 1 FROM external_agent_holds h WHERE h.provider = '${identity}' + AND h.assignment_id = ${alias}.id AND (h.company_id <> ${alias}.company_id + OR h.run_id <> ${alias}.run_id OR h.binding_id <> ${alias}.binding_id + OR h.binding_generation <> ${alias}.binding_generation OR h.agent_id <> ${alias}.agent_id + OR h.released_at IS NULL OR h.worker_unknown OR h.native_effects_unknown)))`; +} + +function completedOperation(alias: string, provider: "dot" | "muse"): string { + return `${alias}.status IN (${provider === 'dot' ? "'completed', 'rejected'" : "'settled', 'rejected'"}) + AND jsonb_typeof(${alias}.outcome) = 'object' + AND EXISTS (SELECT 1 FROM ${provider}_runner_assignments a WHERE a.id = ${alias}.assignment_id + AND a.company_id = ${alias}.company_id AND (${completedAssignment('a', provider)}))`; +} + +function assignmentCheck(provider: "dot" | "muse") { + return { reason: `${provider}_runner_assignments`, query: `SELECT 1 FROM ${provider}_runner_assignments a WHERE ( + (${completedAssignment('a', provider)}) AND NOT EXISTS (SELECT 1 FROM ${provider}_runner_operations o + WHERE o.assignment_id = a.id AND (o.company_id <> a.company_id OR (${completedOperation('o', provider)}) IS NOT TRUE))) IS NOT TRUE` }; +} + +function mailboxCheck(provider: "dot" | "muse") { + return { reason: `${provider}_mailbox_items`, query: `SELECT 1 FROM ${provider}_mailbox_items m WHERE ( + EXISTS (SELECT 1 FROM ${provider}_agent_bindings b WHERE b.id = m.binding_id AND b.company_id = m.company_id + AND m.binding_generation > 0 AND m.binding_generation <= b.generation) + AND ((m.assignment_id IS NOT NULL AND m.kind IN ('assignment', 'operation_result', 'follow_up', 'authority_revoked'${provider === 'muse' ? ", 'input_available'" : ''}) + AND EXISTS (SELECT 1 FROM ${provider}_runner_assignments a WHERE a.id = m.assignment_id + AND a.company_id = m.company_id AND a.binding_id = m.binding_id AND a.binding_generation = m.binding_generation + AND (${completedAssignment('a', provider)}))) + OR (m.assignment_id IS NULL AND m.kind = 'readiness_challenge' AND EXISTS ( + SELECT 1 FROM ${provider}_agent_bindings b WHERE b.id = m.binding_id AND b.company_id = m.company_id + AND b.status = 'revoked' AND b.revoked_at IS NOT NULL)))) IS NOT TRUE` }; +} + +export const idleExternalAgentHistoryChecks = [ + { reason: "dot_agent_bindings", query: `SELECT 1 FROM dot_agent_bindings b WHERE ( + b.provider_kind = 'openai_dot' AND b.status = 'revoked' AND b.revoked_at IS NOT NULL AND b.pairing_code_hash IS NULL + AND b.challenge_hash IS NULL AND EXISTS (SELECT 1 FROM agents a WHERE a.id = b.agent_id AND a.company_id = b.company_id) + AND (b.grant_id IS NULL OR EXISTS (SELECT 1 FROM mcp_oauth_grants g + WHERE g.id = b.grant_id AND g.company_id = b.company_id AND g.user_id = b.operator_id + AND g.agent_id = b.agent_id AND g.purpose = 'agent' AND g.revoked_at IS NOT NULL))) IS NOT TRUE` }, + { reason: "muse_agent_bindings", query: `SELECT 1 FROM muse_agent_bindings b WHERE ( + b.status = 'revoked' AND b.revoked_at IS NOT NULL AND b.ticket_hash IS NULL AND b.challenge_hash IS NULL + AND ((b.qualification_id IS NULL AND b.detector_removal_requested_at IS NULL) + OR (b.detector_removed_at IS NOT NULL AND b.detector_removed_at >= b.detector_removal_requested_at)) + AND EXISTS (SELECT 1 FROM agents a WHERE a.id = b.agent_id AND a.company_id = b.company_id) + AND (b.grant_id IS NULL OR EXISTS (SELECT 1 FROM mcp_oauth_grants g + WHERE g.id = b.grant_id AND g.company_id = b.company_id AND g.user_id = b.operator_id + AND g.agent_id = b.agent_id AND g.purpose = 'agent' AND g.revoked_at IS NOT NULL))) IS NOT TRUE` }, + { reason: "muse_credentials", query: `SELECT 1 FROM muse_credentials c WHERE ( + c.kind IN ('access', 'refresh', 'signal', 'cleanup', 'detector_cleanup') + AND (c.revoked_at IS NOT NULL OR c.consumed_at IS NOT NULL OR c.expires_at <= now()) + AND EXISTS (SELECT 1 FROM muse_agent_bindings b WHERE b.id = c.binding_id AND b.company_id = c.company_id + AND c.binding_generation > 0 AND c.binding_generation <= b.generation)) IS NOT TRUE` }, + assignmentCheck('dot'), assignmentCheck('muse'), + { reason: "dot_runner_operations", query: `SELECT 1 FROM dot_runner_operations o WHERE (${completedOperation('o', 'dot')}) IS NOT TRUE` }, + { reason: "muse_runner_operations", query: `SELECT 1 FROM muse_runner_operations o WHERE (${completedOperation('o', 'muse')}) IS NOT TRUE` }, + mailboxCheck('dot'), mailboxCheck('muse'), + { reason: "muse_input_deliveries", query: `SELECT 1 FROM muse_input_deliveries i WHERE ( + EXISTS (SELECT 1 FROM muse_runner_assignments a WHERE a.id = i.assignment_id AND a.company_id = i.company_id + AND a.turn_id = i.turn_id AND (${completedAssignment('a', 'muse')}))) IS NOT TRUE` }, + { reason: "external_agent_holds", query: `SELECT 1 FROM external_agent_holds h WHERE ( + h.provider = 'muse' AND h.released_at IS NOT NULL AND NOT h.worker_unknown AND NOT h.native_effects_unknown + AND EXISTS (SELECT 1 FROM muse_runner_assignments a WHERE a.id = h.assignment_id + AND a.company_id = h.company_id AND a.agent_id = h.agent_id AND a.run_id = h.run_id + AND a.binding_id = h.binding_id AND a.binding_generation = h.binding_generation + AND (h.operator_attested_at IS NOT NULL OR (a.status = 'settled' AND a.accepted_result_at IS NOT NULL)) + AND (${completedAssignment('a', 'muse')}))) IS NOT TRUE` }, +] as const; diff --git a/server/src/services/browser-use-idle-proof.ts b/server/src/services/browser-use-idle-proof.ts new file mode 100644 index 00000000000..c61baf1914f --- /dev/null +++ b/server/src/services/browser-use-idle-proof.ts @@ -0,0 +1,49 @@ +import { finishedIdleRun } from "./idle-work-proof.js"; + +/** Mirrors sweep/account: terminal provider status cannot settle usage. */ +export function completedBrowserRun(run: string): string { + const hold = `${run}.result->'paperclipBudgetHold'`; + return `${run}.status IN ('completed', 'failed', 'cancelled') + AND ${run}.events_drained = 1 AND ${run}.recovery_cursor IS NULL + AND jsonb_typeof(${hold}) = 'object' AND ${hold}->>'invocationId' = ${run}.invocation_id::text + AND ((${run}.provider_run_id IS NOT NULL AND ${hold}->>'state' = 'settled' + AND ${hold}->>'accountedCentsExact' ~ '^[0-9]+([.][0-9]+)?$') + OR (${run}.provider_run_id IS NULL AND ${run}.status = 'failed' AND ${hold}->>'state' = 'released')) + -- account() replaces held authority with the authoritative terminal + -- summary and ledger charge. It retains prior diagnostic flags in JSON. + AND (${hold}->>'state' = 'settled' OR COALESCE(${hold}->>'billingUnknown', 'false') = 'false') + AND EXISTS (SELECT 1 FROM browser_use_sessions browser_parent_session JOIN heartbeat_runs browser_parent_run + ON browser_parent_run.id = ${run}.heartbeat_run_id AND browser_parent_run.company_id = browser_parent_session.company_id + AND browser_parent_run.agent_id = browser_parent_session.agent_id + WHERE browser_parent_session.id = ${run}.session_id AND browser_parent_session.company_id = ${run}.company_id + AND (browser_parent_run.native_issue_id = browser_parent_session.issue_id + OR browser_parent_run.context_snapshot->>'issueId' = browser_parent_session.issue_id::text + OR browser_parent_run.context_snapshot->>'taskId' = browser_parent_session.issue_id::text) + AND (browser_parent_run.native_issue_id IS NULL OR browser_parent_run.native_issue_id = browser_parent_session.issue_id) + AND (browser_parent_run.context_snapshot->>'issueId' IS NULL + OR browser_parent_run.context_snapshot->>'issueId' = browser_parent_session.issue_id::text) + AND (browser_parent_run.context_snapshot->>'taskId' IS NULL + OR browser_parent_run.context_snapshot->>'taskId' = browser_parent_session.issue_id::text) + AND ${finishedIdleRun('browser_parent_run')})`; +} + +export const idleBrowserHistoryChecks = [ + { reason: "browser_use_sessions", query: `SELECT 1 FROM browser_use_sessions s WHERE ( + (s.status = 'closed' OR (s.status = 'failed' AND s.provider_session_id IS NULL)) + AND s.lease_token IS NULL AND s.lease_until IS NULL AND s.stop_requested IS NULL + AND EXISTS (SELECT 1 FROM issues i JOIN agents a ON a.id = s.agent_id AND a.company_id = i.company_id + JOIN tool_connections c ON c.id = s.connection_id AND c.company_id = i.company_id + JOIN connection_grants g ON g.id = s.grant_id AND g.company_id = i.company_id AND g.connection_id = c.id + WHERE i.id = s.issue_id AND i.company_id = s.company_id) + AND NOT EXISTS (SELECT 1 FROM browser_use_runs r WHERE r.session_id = s.id + AND (r.company_id <> s.company_id OR (${completedBrowserRun('r')}) IS NOT TRUE)) + AND NOT EXISTS (SELECT 1 FROM browser_use_browsers b WHERE b.session_id = s.id + AND (b.company_id <> s.company_id OR b.status <> 'stopped'))) IS NOT TRUE` }, + { reason: "browser_use_runs", query: `SELECT 1 FROM browser_use_runs r WHERE (${completedBrowserRun('r')}) IS NOT TRUE` }, + // stopped is written only after client.stop followed by client.browser + // confirms stopped. Parent/session/run checks remain independent barriers. + { reason: "browser_use_browsers", query: `SELECT 1 FROM browser_use_browsers b WHERE ( + b.status = 'stopped' AND EXISTS (SELECT 1 FROM browser_use_sessions s + WHERE s.id = b.session_id AND s.company_id = b.company_id AND s.status = 'closed' + AND s.stop_requested IS NULL AND s.lease_token IS NULL AND s.lease_until IS NULL)) IS NOT TRUE` }, +] as const; diff --git a/server/src/services/chat-channels/github/idle-history-proof.ts b/server/src/services/chat-channels/github/idle-history-proof.ts new file mode 100644 index 00000000000..3dade037416 --- /dev/null +++ b/server/src/services/chat-channels/github/idle-history-proof.ts @@ -0,0 +1,9 @@ +export function completedGithubAction(a: string): string { + return `(${a}.kind = 'github_response_comment' AND ${a}.status = 'processed' AND ${a}.result->>'final' = 'true') + OR (${a}.kind = 'github_webhook_ingress' AND ( + (${a}.status = 'processed' AND ${a}.result->>'httpStatus' = '202' AND ${a}.result->>'retryable' = 'false') + OR (${a}.status = 'failed' AND ${a}.result->>'code' = 'github_webhook_ingress_payload_invalid' + AND ${a}.result->>'retryable' = 'false'))) + OR (${a}.kind = 'github_review_check' AND ${a}.status = 'processed' + AND ${a}.result->>'retryable' = 'false')`; +} diff --git a/server/src/services/chat-channels/idle-history-proof.ts b/server/src/services/chat-channels/idle-history-proof.ts new file mode 100644 index 00000000000..778779d2b3d --- /dev/null +++ b/server/src/services/chat-channels/idle-history-proof.ts @@ -0,0 +1,79 @@ +import { finishedIdleRun, finishedIdleWake } from "../idle-work-proof.js"; +import { completedGithubAction } from "./github/idle-history-proof.js"; +import { completedSlackAction } from "./slack/idle-history-proof.js"; +import { completedTelegramAction } from "./telegram/idle-history-proof.js"; + +/** A processed inbound receipt can still own an unconsumed wake. */ +function completedAction(a: string): string { + const wake = `EXISTS (SELECT 1 FROM agent_wakeup_requests w WHERE w.company_id = ${a}.company_id + AND w.id = ${a}.id AND w.agent_id::text = ${a}.payload->>'agentId' + AND w.payload->>'issueId' = ${a}.payload->>'issueId' AND ${finishedIdleWake('w')})`; + return `jsonb_typeof(${a}.payload) = 'object' AND jsonb_typeof(${a}.result) = 'object' + AND EXISTS (SELECT 1 FROM chat_endpoints e WHERE e.id = ${a}.endpoint_id AND e.company_id = ${a}.company_id) + AND (${a}.delivery_id IS NULL OR EXISTS (SELECT 1 FROM chat_deliveries d + WHERE d.id = ${a}.delivery_id AND d.company_id = ${a}.company_id AND d.endpoint_id = ${a}.endpoint_id)) + AND CASE + WHEN ${a}.kind = 'inbound_wakeup' THEN ${a}.status IN ('processed', 'failed') AND ${wake} + WHEN ${a}.kind = 'interaction_wakeup' THEN ${a}.status = 'processed' AND + (${a}.result->>'code' = 'interaction_wakeup_no_longer_applicable' OR EXISTS ( + SELECT 1 FROM agent_wakeup_requests w WHERE w.company_id = ${a}.company_id + AND w.idempotency_key = 'interaction:' || (${a}.payload->>'interactionId') || ':' || (${a}.payload->>'interactionStatus') + AND ${finishedIdleWake('w')})) + WHEN ${a}.kind = 'failed_run_retry' THEN ${a}.status = 'processed' + AND ${a}.result->>'code' = 'failed_chat_retry_durable' AND ${wake} + WHEN ${a}.kind = 'provider_effect' THEN + (${a}.status = 'processed' AND length(${a}.result->>'providerMessageId') > 0 AND length(${a}.result->>'threadId') > 0) + OR (${a}.status = 'cancelled' AND ${a}.result->>'code' = 'provider_effect_no_longer_authorized') + OR (${a}.status = 'failed' AND ${a}.result->>'code' = 'provider_effect_failed' AND ${a}.result->>'retryable' = 'false') + WHEN ${a}.kind = 'receipt_reaction' THEN + (${a}.status = 'processed' AND jsonb_typeof(${a}.result->'attempts') = 'number' AND (${a}.result->>'attempts')::numeric > 0) + OR (${a}.status = 'cancelled' AND ${a}.result->>'code' IN ('receipt_reaction_superseded_by_terminal_publication', 'receipt_reaction_runtime_superseded')) + WHEN ${a}.kind = 'confirmation_response' THEN ${a}.status = 'processed' + AND ${a}.result->>'code' IN ('confirmation_resolution_committed_by_provider', 'confirmation_resolution_reconciled') + AND EXISTS (SELECT 1 FROM issue_thread_interactions interaction WHERE interaction.id::text = ${a}.result->>'interactionId' + AND interaction.company_id = ${a}.company_id AND interaction.status = ${a}.result->>'interactionStatus' + AND interaction.resolved_at IS NOT NULL) + WHEN ${a}.kind = 'task_control_authorization' THEN ${a}.status IN ('processed', 'cancelled') + AND EXISTS (SELECT 1 FROM chat_publications p WHERE p.company_id = ${a}.company_id AND p.endpoint_id = ${a}.endpoint_id + AND p.conversation_id = ${a}.conversation_id AND ${a}.provider_action_id = 'task-control-authorization:' || p.id::text + AND ((p.state = 'published' AND p.published_at IS NOT NULL) OR p.state = 'cancelled')) + ELSE (${[completedGithubAction, completedSlackAction, completedTelegramAction] + .map(proof => `(${proof(a)})`).join(' OR ')}) END`; +} + +export const idleChatHistoryChecks = [ + { reason: "chat_deliveries", query: `SELECT 1 FROM chat_deliveries d WHERE ( + EXISTS (SELECT 1 FROM chat_endpoints e WHERE e.id = d.endpoint_id AND e.company_id = d.company_id) + AND d.next_attempt_at IS NULL AND (d.state = 'failed' OR (d.state IN ('processed', 'filtered') AND d.processed_at IS NOT NULL)) + AND (d.conversation_id IS NULL OR EXISTS (SELECT 1 FROM chat_conversations c WHERE c.id = d.conversation_id + AND c.company_id = d.company_id AND c.endpoint_id = d.endpoint_id)) + AND NOT EXISTS (SELECT 1 FROM chat_actions a WHERE a.delivery_id = d.id + AND (a.company_id <> d.company_id OR a.endpoint_id <> d.endpoint_id OR (${completedAction('a')}) IS NOT TRUE))) IS NOT TRUE` }, + { reason: "chat_publications", query: `SELECT 1 FROM chat_publications p WHERE ( + (p.state = 'published' AND p.published_at IS NOT NULL AND p.redacted_error IS NULL + OR (p.state IN ('failed', 'cancelled') AND p.next_attempt_at IS NULL)) + AND EXISTS (SELECT 1 FROM chat_conversations c JOIN chat_endpoints e + ON e.id = c.endpoint_id AND e.company_id = c.company_id + JOIN issues i ON i.id = p.issue_id AND i.company_id = c.company_id + WHERE c.id = p.conversation_id AND c.company_id = p.company_id AND c.endpoint_id = p.endpoint_id) + AND NOT EXISTS (SELECT 1 FROM chat_endpoint_leases l WHERE l.company_id = p.company_id + AND l.endpoint_id = p.endpoint_id AND l.lease_key LIKE 'publication:' || p.id::text || ':%' AND l.expires_at > now()) + AND NOT EXISTS (SELECT 1 FROM chat_teams_file_transfers f WHERE f.publication_id = p.id + AND (f.company_id <> p.company_id OR f.phase NOT IN ('delivered', 'declined', 'expired', 'cancelled') + OR f.attempt_id IS NOT NULL OR f.attempt_expires_at IS NOT NULL))) IS NOT TRUE` }, + { reason: "chat_actions", query: `SELECT 1 FROM chat_actions a WHERE (${completedAction('a')}) IS NOT TRUE` }, + { reason: "chat_completion_deliveries", query: `SELECT 1 FROM chat_completion_deliveries d WHERE ( + d.status IN ('delivered', 'superseded', 'exhausted') + AND EXISTS (SELECT 1 FROM chat_task_handoffs h JOIN issues task ON task.id = h.task_id AND task.company_id = h.company_id + JOIN issues source ON source.id = h.conversation_id AND source.company_id = h.company_id + WHERE h.task_id = d.task_id AND h.company_id = d.company_id + AND source.conversation_agent_id = h.agent_id + AND (d.status <> 'delivered' OR (d.status_version = task.status_version AND task.status = 'done' + AND EXISTS (SELECT 1 FROM issue_comments c WHERE c.id = d.response_comment_id + AND c.company_id = d.company_id AND c.issue_id = h.conversation_id AND c.author_agent_id = h.agent_id)))) + AND (d.target_run_id IS NULL AND d.status <> 'delivered' OR EXISTS ( + SELECT 1 FROM heartbeat_runs r JOIN chat_task_handoffs h ON h.task_id = d.task_id + WHERE r.id = d.target_run_id AND r.company_id = d.company_id AND r.agent_id = h.agent_id + AND (r.native_issue_id = h.conversation_id OR r.context_snapshot->>'issueId' = h.conversation_id::text) + AND ${finishedIdleRun('r')}))) IS NOT TRUE` }, +] as const; diff --git a/server/src/services/chat-channels/slack/idle-history-proof.ts b/server/src/services/chat-channels/slack/idle-history-proof.ts new file mode 100644 index 00000000000..f57e9950f12 --- /dev/null +++ b/server/src/services/chat-channels/slack/idle-history-proof.ts @@ -0,0 +1,16 @@ +export function completedSlackAction(a: string): string { + return `(${a}.kind = 'slack_session_sync' AND ( + (${a}.status = 'processed' AND ${a}.result->>'outcome' IN ('updated', 'unsupported') + AND (${a}.result->>'outcome' <> 'updated' OR ${a}.result->>'sessionStatus' <> 'processing')) + OR (${a}.status = 'cancelled' AND ${a}.result->>'code' IN ('slack_session_no_published_state', 'slack_session_destination_changed')) + OR (${a}.status = 'failed' AND ${a}.result->>'code' = 'slack_session_sync_rejected'))) + OR (${a}.kind = 'slash_task_start' AND ${a}.status = 'processed' + AND length(${a}.result->>'providerMessageId') > 0 AND length(${a}.result->>'threadId') > 0) + OR (${a}.kind = 'slack_file_upload_receipt' AND ${a}.status = 'processed' + AND ${a}.result->>'code' = 'slack_file_upload_identity_confirmed') + OR (${a}.kind = 'slack_session_stop' AND ${a}.status IN ('processed', 'cancelled') + AND ${a}.result->>'code' IN ('slack_session_stop_cancelled', 'slack_session_stop_target_superseded', 'slack_session_stop_no_current_work') + AND NOT EXISTS (SELECT 1 FROM heartbeat_runs r WHERE r.id::text = ${a}.result->>'runId' + AND (r.company_id <> ${a}.company_id OR r.status NOT IN ('succeeded', 'failed', 'cancelled', 'timed_out', 'interrupted') + OR r.finished_at IS NULL OR r.cost_accounting_pending)))`; +} diff --git a/server/src/services/chat-channels/telegram/idle-history-proof.ts b/server/src/services/chat-channels/telegram/idle-history-proof.ts new file mode 100644 index 00000000000..bd37617d627 --- /dev/null +++ b/server/src/services/chat-channels/telegram/idle-history-proof.ts @@ -0,0 +1,12 @@ +export function completedTelegramAction(a: string): string { + return `(${a}.kind = 'telegram_maintenance' AND ( + (${a}.status = 'processed' AND (${a}.result->>'providerConfirmed' = 'true' + OR ${a}.result->>'code' = 'telegram_stop_subscription_confirmed')) + OR (${a}.status = 'cancelled' AND ${a}.result->>'code' = 'telegram_maintenance_credentials_superseded') + OR (${a}.status = 'failed' AND ${a}.result->>'retryable' = 'false' + AND ${a}.result->>'code' = 'telegram_maintenance_failed'))) + OR (${a}.kind = 'telegram_draft' AND ( + (${a}.status = 'processed' AND ${a}.result->>'phase' = 'published' AND length(${a}.result->>'providerMessageId') > 0) + OR (${a}.status = 'cancelled' AND (${a}.result->>'phase' = 'stopped' + OR ${a}.result->>'code' = 'telegram_draft_presentation_stopped'))))`; +} diff --git a/server/src/services/conversation-idle-proof.ts b/server/src/services/conversation-idle-proof.ts new file mode 100644 index 00000000000..2ce2cf48d0e --- /dev/null +++ b/server/src/services/conversation-idle-proof.ts @@ -0,0 +1,45 @@ +import { finishedIdleRun, finishedIdleWake } from "./idle-work-proof.js"; + +/** The same positive container proof is used by ordinary and scheduled sleep. + * waiting/in_review is set by settleConversationTurn, but is not sufficient: + * a user comment may commit before the wake is enqueued. */ +export function passiveIdleConversation(i: "i"): string { + return `${i}.conversation_agent_id IS NOT NULL AND length(${i}.conversation_user_id) > 0 + AND ${i}.conversation_state = 'waiting' AND ${i}.status = 'in_review' + AND ${i}.assignee_agent_id = ${i}.conversation_agent_id AND ${i}.assignee_user_id IS NULL + AND (${i}.conversation_user_id = 'local-board' OR EXISTS (SELECT 1 FROM company_memberships membership + WHERE membership.company_id = ${i}.company_id AND membership.principal_type = 'user' + AND membership.principal_id = ${i}.conversation_user_id AND membership.status = 'active')) + AND ${i}.conversation_session_generation >= 0 + AND ${i}.checkout_run_id IS NULL AND ${i}.execution_run_id IS NULL + AND ${i}.monitor_next_check_at IS NULL AND ${i}.monitor_wake_requested_at IS NULL + AND (${i}.execution_policy IS NULL OR (jsonb_typeof(${i}.execution_policy) = 'object' + AND COALESCE(${i}.execution_policy->'monitor', 'null'::jsonb) = 'null'::jsonb)) + AND (${i}.execution_state IS NULL OR (jsonb_typeof(${i}.execution_state) = 'object' + AND ${i}.execution_state->>'status' = 'idle' + AND COALESCE(${i}.execution_state->'monitor', 'null'::jsonb) = 'null'::jsonb + AND COALESCE(${i}.execution_state->'currentParticipant', 'null'::jsonb) = 'null'::jsonb + AND COALESCE(${i}.execution_state->'currentStageId', 'null'::jsonb) = 'null'::jsonb + AND COALESCE(${i}.execution_state->'reviewRequest', 'null'::jsonb) = 'null'::jsonb)) + AND EXISTS (SELECT 1 FROM agents a JOIN companies c ON c.id = a.company_id + WHERE a.id = ${i}.conversation_agent_id AND a.company_id = ${i}.company_id + AND c.status = 'active' AND a.status IN ('idle', 'active', 'paused', 'terminated')) + AND NOT EXISTS (SELECT 1 FROM heartbeat_runs r WHERE + (r.native_issue_id = ${i}.id OR r.context_snapshot->>'issueId' = ${i}.id::text OR r.context_snapshot->>'taskId' = ${i}.id::text) + AND (r.company_id <> ${i}.company_id OR r.agent_id <> ${i}.conversation_agent_id OR (${finishedIdleRun('r')}) IS NOT TRUE)) + AND NOT EXISTS (SELECT 1 FROM agent_wakeup_requests w WHERE + (w.payload->>'issueId' = ${i}.id::text OR w.payload->>'taskId' = ${i}.id::text) + AND (w.company_id <> ${i}.company_id OR w.agent_id <> ${i}.conversation_agent_id + OR (${finishedIdleWake('w')}) IS NOT TRUE)) + AND NOT EXISTS (SELECT 1 FROM issue_comments comment WHERE comment.issue_id = ${i}.id + AND comment.deleted_at IS NULL AND comment.author_user_id IS NOT NULL AND comment.author_agent_id IS NULL + AND comment.origin = 'comment' AND comment.client_request_id IS NOT NULL + AND NOT EXISTS (SELECT 1 FROM agent_wakeup_requests w WHERE w.company_id = ${i}.company_id + AND w.agent_id = ${i}.conversation_agent_id AND w.idempotency_key = 'conversation-comment:' || comment.id::text + AND ${finishedIdleWake('w')})) + AND NOT EXISTS (SELECT 1 FROM issue_thread_interactions interaction WHERE interaction.issue_id = ${i}.id + AND (interaction.company_id <> ${i}.company_id OR interaction.status = 'pending')) + AND NOT EXISTS (SELECT 1 FROM chat_completion_deliveries d JOIN chat_task_handoffs h ON h.task_id = d.task_id + WHERE h.conversation_id = ${i}.id AND (h.company_id <> ${i}.company_id OR d.company_id <> ${i}.company_id + OR d.status NOT IN ('delivered', 'superseded', 'exhausted')))`; +} diff --git a/server/src/services/decision-idle-proof.ts b/server/src/services/decision-idle-proof.ts new file mode 100644 index 00000000000..d95f476bbd1 --- /dev/null +++ b/server/src/services/decision-idle-proof.ts @@ -0,0 +1,134 @@ +import { finishedIdleRun } from "./idle-work-proof.js"; + +/** executeDecisionEffects visits every selected ordinal, including failures. + * Its final aggregate cannot prove a missing row or another option's effect. */ +function completedEffectManifest(d: string): string { + const options = `jsonb_array_elements(CASE WHEN jsonb_typeof(${d}.options) = 'array' THEN ${d}.options ELSE '[]'::jsonb END)`; + return `EXISTS (SELECT 1 FROM ${options} option_row WHERE option_row->>'id' = ${d}.chosen_option_id + AND jsonb_typeof(option_row->'effects') = 'array' + AND EXISTS (SELECT 1 FROM activity_log selection WHERE selection.company_id = ${d}.company_id + AND selection.entity_type = 'decision' AND selection.entity_id = ${d}.id::text + AND selection.action IN ('decision.decided', 'decision.dismissed') + AND selection.details->>'chosenOptionId' = ${d}.chosen_option_id + AND selection.details->>'decidedByUserId' = ${d}.decided_by_user_id) + AND (SELECT count(*) FROM ${options} duplicate_option WHERE duplicate_option->>'id' = ${d}.chosen_option_id) = 1 + AND NOT EXISTS (SELECT 1 FROM jsonb_array_elements(option_row->'effects') WITH ORDINALITY expected(effect, ordinal) + WHERE NOT EXISTS (SELECT 1 FROM decision_effect_executions execution JOIN issues target + ON target.id = execution.target_issue_id AND target.company_id = ${d}.company_id + JOIN activity_log audit ON audit.id = execution.activity_log_id AND audit.company_id = ${d}.company_id + AND audit.entity_type = 'decision' AND audit.entity_id = ${d}.id::text + AND audit.details->>'chosenOptionId' = ${d}.chosen_option_id + AND audit.details->>'executionId' = execution.id::text + WHERE execution.decision_id = ${d}.id AND execution.effect_index = expected.ordinal - 1 + AND execution.effect_type = expected.effect->>'type' AND execution.target_issue_id::text = expected.effect->>'targetIssueId' + AND execution.status IN ('executed', 'failed', 'skipped') AND execution.executed_at IS NOT NULL + AND audit.action = 'decision.effect_' || execution.status)) + AND NOT EXISTS (SELECT 1 FROM decision_effect_executions execution WHERE execution.decision_id = ${d}.id + AND NOT EXISTS (SELECT 1 FROM jsonb_array_elements(option_row->'effects') WITH ORDINALITY expected(effect, ordinal) + WHERE execution.effect_index = expected.ordinal - 1 AND execution.effect_type = expected.effect->>'type' + AND execution.target_issue_id::text = expected.effect->>'targetIssueId')) + AND CASE WHEN ${d}.execution_status = 'succeeded' THEN NOT EXISTS (SELECT 1 FROM decision_effect_executions e + WHERE e.decision_id = ${d}.id AND e.status <> 'executed') + WHEN ${d}.execution_status = 'partial' THEN EXISTS (SELECT 1 FROM decision_effect_executions e + WHERE e.decision_id = ${d}.id AND e.status = 'executed') AND EXISTS (SELECT 1 FROM decision_effect_executions e + WHERE e.decision_id = ${d}.id AND e.status IN ('failed', 'skipped')) + ELSE NOT EXISTS (SELECT 1 FROM decision_effect_executions e WHERE e.decision_id = ${d}.id AND e.status = 'executed') + AND EXISTS (SELECT 1 FROM decision_effect_executions e WHERE e.decision_id = ${d}.id AND e.status IN ('failed', 'skipped')) END)`; +} + +function completedDecisionOutcome(d: string): string { + return `(${d}.status IN ('expired', 'cancelled') AND ${d}.execution_status IS NULL AND ${d}.chosen_option_id IS NULL + AND NOT EXISTS (SELECT 1 FROM decision_effect_executions e WHERE e.decision_id = ${d}.id) + AND EXISTS (SELECT 1 FROM activity_log audit WHERE audit.company_id = ${d}.company_id + AND audit.entity_type = 'decision' AND audit.entity_id = ${d}.id::text AND audit.action = 'decision.' || ${d}.status)) + OR (${d}.status = 'decided' AND ${d}.decided_at IS NOT NULL AND ${d}.execution_status IN ('succeeded', 'partial', 'failed') + AND ((COALESCE(${d}.metadata->>'kind', '') <> 'attention_archive_proposal' AND (${completedEffectManifest(d)})) + OR (${completedArchiveOutcome(d)}) + OR (${d}.chosen_option_id = 'dismissed' AND ${d}.execution_status = 'succeeded' AND ${d}.metadata->>'dismissed' = 'true' + AND NOT EXISTS (SELECT 1 FROM decision_effect_executions e WHERE e.decision_id = ${d}.id) + AND EXISTS (SELECT 1 FROM activity_log audit WHERE audit.company_id = ${d}.company_id + AND audit.entity_type = 'decision' AND audit.entity_id = ${d}.id::text AND audit.action = 'decision.dismissed' + AND audit.details->>'chosenOptionId' = 'dismissed' AND audit.details->>'decidedByUserId' = ${d}.decided_by_user_id)) + OR (${d}.metadata->>'effectRecoveryError' IN ('missing_decider', 'missing_option') + AND ${d}.execution_status IN ('failed', 'partial') + AND EXISTS (SELECT 1 FROM activity_log audit WHERE audit.company_id = ${d}.company_id + AND audit.entity_type = 'decision' AND audit.entity_id = ${d}.id::text AND audit.action = 'decision.recovery_failed' + AND audit.details->>'reason' = ${d}.metadata->>'effectRecoveryError' + AND audit.details->>'executionStatus' = ${d}.execution_status + AND audit.details->>'chosenOptionId' IS NOT DISTINCT FROM ${d}.chosen_option_id))))`; +} + +/** Archive applies the reviewed set in one transaction. Every catch is final + * (execution_status=failed); it neither replays the set nor leaves per-effect + * claims. Continuation and notification queues remain independent checks. */ +function completedArchiveOutcome(d: string): string { + return `${d}.metadata->>'kind' = 'attention_archive_proposal' + AND NOT EXISTS (SELECT 1 FROM decision_effect_executions e WHERE e.decision_id = ${d}.id) + AND EXISTS (SELECT 1 FROM jsonb_array_elements(CASE WHEN jsonb_typeof(${d}.options) = 'array' + THEN ${d}.options ELSE '[]'::jsonb END) selected WHERE selected->>'id' = ${d}.chosen_option_id + AND selected->'effects' = '[]'::jsonb + AND (SELECT count(*) FROM jsonb_array_elements(${d}.options) duplicate WHERE duplicate->>'id' = ${d}.chosen_option_id) = 1) + AND ((${d}.execution_status = 'failed' AND ${d}.chosen_option_id = 'archive' + AND jsonb_typeof(${d}.metadata->'archiveProposalError') = 'string' + AND length(trim(${d}.metadata->>'archiveProposalError')) > 0) + OR (${d}.execution_status = 'succeeded' AND ${d}.metadata->>'archiveProposalError' IS NULL + AND (${d}.chosen_option_id <> 'archive' OR ( + jsonb_typeof(${d}.target_snapshots) = 'object' AND ${d}.target_snapshots <> '{}'::jsonb + AND ${d}.metadata->>'manifestHash' ~ '^[a-f0-9]{64}$' + AND NOT EXISTS (SELECT 1 FROM jsonb_each(${d}.target_snapshots) snapshot WHERE ( + snapshot.key = 'attention:' || (snapshot.value->'attentionArchive'->>'sourceKind') || ':' || (snapshot.value->'attentionArchive'->>'sourceId') + AND snapshot.value->'attentionArchive'->>'companyId' = ${d}.company_id::text + AND jsonb_typeof(snapshot.value->'attentionArchive'->'expectedVersion') = 'number' + AND snapshot.value->'attentionArchive'->>'expectedVersion' ~ '^[1-9][0-9]*$' + AND EXISTS (SELECT 1 FROM decision_retention retention WHERE retention.company_id = ${d}.company_id + AND retention.source_kind = snapshot.value->'attentionArchive'->>'sourceKind' + AND retention.source_id = snapshot.value->'attentionArchive'->>'sourceId' + AND retention.version::numeric > (snapshot.value->'attentionArchive'->>'expectedVersion')::numeric) + AND EXISTS (SELECT 1 FROM activity_log audit WHERE audit.company_id = ${d}.company_id + AND audit.actor_id = 'decision-archive-proposal' AND audit.action = 'decision_retention.archived' + AND audit.entity_type = 'attention_source' AND audit.entity_id = snapshot.value->'attentionArchive'->>'sourceId' + AND audit.details->>'sourceKind' = snapshot.value->'attentionArchive'->>'sourceKind' + AND audit.created_at >= ${d}.decided_at AND audit.responsible_user_id = ${d}.decided_by_user_id)) IS NOT TRUE)))))`; +} + +function completedDecision(alias: string): string { + return `(${completedDecisionOutcome(alias)}) + AND ${alias}.continuation_policy IN ('none', 'wake_origin_agent') + AND jsonb_typeof(${alias}.metadata) = 'object' + AND COALESCE(${alias}.metadata->>'continuationPending', 'false') = 'false' + AND (${alias}.continuation_policy = 'none' OR ${alias}.metadata->>'continuationDeliveredAt' IS NOT NULL) + AND EXISTS (SELECT 1 FROM heartbeat_runs r JOIN issues i ON i.id = ${alias}.origin_issue_id + AND i.company_id = r.company_id WHERE r.id = ${alias}.origin_run_id + AND r.company_id = ${alias}.company_id AND r.agent_id = ${alias}.origin_agent_id AND ${finishedIdleRun('r')}) + AND NOT EXISTS (SELECT 1 FROM decision_effect_executions e WHERE e.decision_id = ${alias}.id + AND (e.status NOT IN ('executed', 'failed', 'skipped') OR e.executed_at IS NULL))`; +} + +export const idleDecisionHistoryChecks = [ + { reason: "decisions", query: `SELECT 1 FROM decisions d WHERE (${completedDecision('d')}) IS NOT TRUE` }, + { reason: "decision_effect_executions", query: `SELECT 1 FROM decision_effect_executions e WHERE ( + e.status IN ('executed', 'failed', 'skipped') AND e.executed_at IS NOT NULL + AND EXISTS (SELECT 1 FROM decisions d JOIN issues i ON i.id = e.target_issue_id AND i.company_id = d.company_id + WHERE d.id = e.decision_id AND (${completedDecision('d')}))) IS NOT TRUE` }, + { reason: "decision_archive_notification_outbox", query: `SELECT 1 FROM decision_archive_notification_outbox o WHERE ( + o.status = 'delivered' AND o.delivered_at IS NOT NULL + AND o.archive_version > 0 AND EXISTS (SELECT 1 FROM decision_retention retention + WHERE retention.company_id = o.company_id AND retention.source_kind = o.source_kind + AND retention.source_id = o.source_id AND retention.archive_version >= o.archive_version) + AND EXISTS (SELECT 1 FROM agents a JOIN issues i ON i.company_id = a.company_id + WHERE a.id = o.origin_agent_id AND i.id = o.origin_issue_id AND i.company_id = o.company_id)) IS NOT TRUE` }, + { reason: "company_secret_proposals", query: `SELECT 1 FROM company_secret_proposals p WHERE ( + p.kind IN ('secret', 'binding') AND p.status IN ('approved', 'rejected', 'withdrawn', 'expired') + AND p.resolved_at IS NOT NULL AND p.value_ciphertext IS NULL AND p.ciphertext_scrubbed_at IS NOT NULL + AND EXISTS (SELECT 1 FROM heartbeat_runs r WHERE r.id = p.origin_run_id + AND r.company_id = p.company_id AND r.agent_id = p.proposed_by_agent_id AND ${finishedIdleRun('r')}) + AND (p.origin_issue_id IS NULL OR EXISTS (SELECT 1 FROM issues i + WHERE i.id = p.origin_issue_id AND i.company_id = p.company_id)) + AND (p.status <> 'approved' OR (p.kind = 'secret' AND EXISTS (SELECT 1 FROM company_secrets s + WHERE s.id = p.created_secret_id AND s.company_id = p.company_id)) + OR (p.kind = 'binding' AND p.applied_binding_config_path = p.config_path AND EXISTS ( + SELECT 1 FROM agents a WHERE a.id = p.target_id AND a.company_id = p.company_id))) + AND (p.interaction_id IS NULL OR EXISTS (SELECT 1 FROM issue_thread_interactions i + WHERE i.id = p.interaction_id AND i.company_id = p.company_id AND i.issue_id = p.origin_issue_id + AND i.status IN ('accepted', 'rejected', 'cancelled', 'expired') AND i.resolved_at IS NOT NULL))) IS NOT TRUE` }, +] as const; diff --git a/server/src/services/idle-issue-schedule.ts b/server/src/services/idle-issue-schedule.ts index c234403e873..f2ed5efb01e 100644 --- a/server/src/services/idle-issue-schedule.ts +++ b/server/src/services/idle-issue-schedule.ts @@ -5,6 +5,7 @@ import { AGENT_STATUSES, issueExecutionMonitorPolicySchema, PROVIDER_QUOTA_MONIT import type { ScheduledWake } from "./idle-scheduled-work.js"; import { onboardingFirstTaskAwaitingUserSql } from "./recovery/onboarding-first-task-wait.js"; import { completedAgentUnblockRequest, responseDrivenIssueWait } from "./idle-work-proof.js"; +import { passiveIdleConversation } from "./conversation-idle-proof.js"; const processGeneration = randomUUID(); const limit = 10_000; @@ -30,6 +31,7 @@ export async function readIdleIssueSchedule(tx: Pick, ownerId: st untouched_onboarding: boolean; response_driven_wait: boolean; unblock_request_finished: boolean; response_wait_generation: unknown; + passive_conversation: boolean; }>(sql`SELECT i.id, i.company_id, i.status, i.updated_at, i.assignee_agent_id, i.assignee_user_id, i.checkout_run_id, i.execution_run_id, i.execution_policy, i.execution_state, i.unblock_descriptor, i.conversation_agent_id, i.monitor_next_check_at, i.monitor_wake_requested_at, i.monitor_attempt_count, @@ -38,6 +40,7 @@ export async function readIdleIssueSchedule(tx: Pick, ownerId: st (i.assignee_agent_id IS NULL OR a.id IS NOT NULL) AS scoped, (${sql.raw(responseDrivenIssueWait('i'))}) AS response_driven_wait, (${sql.raw(completedAgentUnblockRequest('i'))}) AS unblock_request_finished, + (${sql.raw(passiveIdleConversation('i'))}) AS passive_conversation, (SELECT jsonb_agg(jsonb_build_array(w.id, w.updated_at, w.kind, w.status, w.source_run_id, w.effective_resolver_policy, w.continuation_policy, w.addressee_agent_id, w.addressee_user_id) ORDER BY w.id) FROM issue_thread_interactions w WHERE w.issue_id = i.id AND w.status = 'pending') AS response_wait_generation, @@ -61,7 +64,7 @@ export async function readIdleIssueSchedule(tx: Pick, ownerId: st if (!statuses.has(row.status) || row.company_status !== "active" || row.scoped !== true || (row.assignee_agent_id && !agentStatuses.has(row.agent_status ?? "")) || row.unsafe_dependencies !== false || row.checkout_run_id || row.execution_run_id - || row.conversation_agent_id || row.monitor_wake_requested_at + || (row.conversation_agent_id && row.passive_conversation !== true) || row.monitor_wake_requested_at || (row.execution_state !== null && state?.status !== "idle") || (row.execution_policy !== null && !policy)) return { blocked: true }; if (row.status === "blocked" && row.dependency_ready) return { blocked: true }; @@ -81,7 +84,7 @@ export async function readIdleIssueSchedule(tx: Pick, ownerId: st } if (policy?.monitor != null) return { blocked: true }; const unblock = record(row.unblock_descriptor); - const passive = row.status === "backlog" + const passive = row.passive_conversation === true || row.status === "backlog" || (row.status === "todo" && !row.assignee_agent_id) || (row.status === "todo" && row.assignee_agent_id && row.agent_status === "idle" && row.untouched_onboarding === true) || (row.status === "blocked" && (row.unblock_descriptor === null || unblock?.owner === "board" diff --git a/server/src/services/idle-sleep-safety.ts b/server/src/services/idle-sleep-safety.ts index c0d58a64bdd..f93b158c7f1 100644 --- a/server/src/services/idle-sleep-safety.ts +++ b/server/src/services/idle-sleep-safety.ts @@ -9,6 +9,23 @@ import { verifiedResourceCleanup } from "./cleanup-verification-proof.js"; import { companyHoldMismatch } from "../modules/agent-lifecycle/index.js"; import { pendingBudgetRunCancellationPredicate } from "./heartbeat/budget-cancellation.js"; import { nativeResponsePresentationPendingQuery } from "./native-runtime/native-response-presentation-query.js"; +import { passiveIdleConversation } from "./conversation-idle-proof.js"; +import { idleChatHistoryChecks } from "./chat-channels/idle-history-proof.js"; +import { idleToolActionHistoryChecks } from "./tool-action-idle-proof.js"; +import { idleBrowserHistoryChecks } from "./browser-use-idle-proof.js"; +import { idleDecisionHistoryChecks } from "./decision-idle-proof.js"; +import { idleWorkspaceHistoryChecks } from "./workspace-history-idle-proof.js"; +import { idleMcpHistoryChecks } from "./public-mcp/idle-history-proof.js"; +import { idleExternalAgentHistoryChecks } from "../modules/external-agents/idle-history-proof.js"; + +export const completedHistoryChecks = [ + ...idleChatHistoryChecks, ...idleToolActionHistoryChecks, ...idleBrowserHistoryChecks, + ...idleDecisionHistoryChecks, ...idleWorkspaceHistoryChecks, ...idleMcpHistoryChecks, + ...idleExternalAgentHistoryChecks, +] as const; + +const ordinaryIssues = `SELECT 1 FROM issues i WHERE i.status NOT IN ('done', 'cancelled') + AND (${passiveIdleConversation('i')}) IS NOT TRUE`; export interface IdleSleepDrainStatus { ownerId?: string; @@ -74,7 +91,7 @@ const WORK_CHECKS = [ SELECT 1 FROM heartbeat_runs r WHERE r.id = w.run_id AND r.company_id = w.company_id AND r.agent_id = w.agent_id AND ${finishedRun("r")})))`, - `SELECT 1 FROM issues WHERE status NOT IN ('done', 'cancelled')`, + ordinaryIssues, // A completed issue can still need its first watchdog review, including a // retry after immediate evaluation failed before creating a review/run. `SELECT 1 FROM issue_watchdogs WHERE status <> 'disabled'`, @@ -259,23 +276,7 @@ const WORK_CHECKS = [ OR effect.delivery_state <> 'delivered' OR effect.delivered_at IS NULL OR effect.next_attempt_at IS NOT NULL)) ) IS NOT TRUE`, - // These less common work sources fail closed on any retained state. Their - // terminal-state exceptions can be added with tests for the owning service. - ...[ - "chat_deliveries", "chat_publications", "chat_actions", - "chat_completion_deliveries", "tool_action_requests", "tool_action_deliveries", - "workspace_runtime_services", - "execution_workspace_runtime_leases", - "company_transfer_runs", "decisions", "decision_effect_executions", - "decision_archive_notification_outbox", "browser_use_sessions", "browser_use_runs", - "browser_use_browsers", - "company_secret_proposals", - "mcp_oauth_grants", "mcp_mutation_receipts", "mcp_event_subscriptions", - "mcp_event_deliveries", "mcp_attachment_uploads", "dot_agent_bindings", - "dot_runner_assignments", "dot_runner_operations", "dot_mailbox_items", - "muse_agent_bindings", "muse_credentials", "muse_runner_assignments", - "muse_runner_operations", "muse_mailbox_items", "muse_input_deliveries", "external_agent_holds", - ].map((table) => `SELECT 1 FROM ${table}`), + ...completedHistoryChecks.map(check => check.query), ] as const; function sameQuietHold(before: IdleSleepDrainStatus, after: IdleSleepDrainStatus, now: number): boolean { @@ -320,7 +321,7 @@ export async function readIdleSleepSafety( await tx.execute(sql`SET LOCAL statement_timeout = '5s'`); const checks = WORK_CHECKS.filter(query => (!scheduledRoutines || query !== `SELECT 1 FROM routines WHERE status NOT IN ('paused', 'archived')`) - && (!scheduledIssueMonitors || query !== `SELECT 1 FROM issues WHERE status NOT IN ('done', 'cancelled')`)) + && (!scheduledIssueMonitors || query !== ordinaryIssues)) .map((query) => sql`EXISTS (${sql.raw(query)})`); checks.push(sql`EXISTS (SELECT 1 FROM agents JOIN companies ON companies.id = agents.company_id WHERE ${companyHoldMismatch()})`); diff --git a/server/src/services/public-mcp/idle-history-proof.ts b/server/src/services/public-mcp/idle-history-proof.ts new file mode 100644 index 00000000000..b3c7f78abd1 --- /dev/null +++ b/server/src/services/public-mcp/idle-history-proof.ts @@ -0,0 +1,37 @@ +/** Completion receipts are retained for idempotency. Fixed aliases only. */ +export const idleMcpHistoryChecks = [ + { reason: "mcp_oauth_grants", query: `SELECT 1 FROM mcp_oauth_grants g WHERE + (g.revoked_at IS NOT NULL AND EXISTS (SELECT 1 FROM companies c WHERE c.id = g.company_id)) IS NOT TRUE` }, + { reason: "mcp_mutation_receipts", query: `SELECT 1 FROM mcp_mutation_receipts m WHERE ( + m.status = 'completed' AND jsonb_typeof(m.result) = 'object' + AND EXISTS (SELECT 1 FROM mcp_oauth_grants g WHERE g.id = m.grant_id + AND g.company_id = m.company_id AND g.user_id = m.user_id)) IS NOT TRUE` }, + // Stopped subscriptions no longer scan for events. Expiry alone is not a + // handoff of responsibility for active subscriptions or unfinished delivery. + { reason: "mcp_event_subscriptions", query: `SELECT 1 FROM mcp_event_subscriptions s WHERE ( + s.stopped_at IS NOT NULL AND EXISTS (SELECT 1 FROM mcp_oauth_grants g + WHERE g.id = s.grant_id AND g.company_id = s.company_id)) IS NOT TRUE` }, + { reason: "mcp_event_deliveries", query: `SELECT 1 FROM mcp_event_deliveries d WHERE ( + d.finished_at IS NOT NULL AND (d.outcome IN ('delivered', 'filtered', 'inactive', 'delivery_exhausted') + OR (d.outcome ~ '^http_[1-5][0-9][0-9]$' AND d.outcome !~ '^http_2' + AND (d.attempts >= 6 OR (d.outcome NOT IN ('http_408', 'http_429') AND d.outcome !~ '^http_5')))) + AND EXISTS (SELECT 1 FROM mcp_event_subscriptions s WHERE s.id = d.subscription_id + AND EXISTS (SELECT 1 FROM mcp_oauth_grants g WHERE g.id = s.grant_id AND g.company_id = s.company_id) + AND ((d.activity_id IS NOT NULL AND d.mailbox_item_id IS NULL AND EXISTS ( + SELECT 1 FROM activity_log a WHERE a.id = d.activity_id AND a.company_id = s.company_id)) + OR (d.mailbox_item_id IS NOT NULL AND d.activity_id IS NULL AND EXISTS ( + SELECT 1 FROM dot_mailbox_items m WHERE m.id = d.mailbox_item_id + AND m.company_id = s.company_id AND m.binding_id = s.binding_id))))) IS NOT TRUE` }, + // The upload owner allocates object identity before storage writes and only + // stamps cleaned_at after deletion. A committed attachment is never deleted. + { reason: "mcp_attachment_uploads", query: `SELECT 1 FROM mcp_attachment_uploads u WHERE ( + EXISTS (SELECT 1 FROM issues i WHERE i.id = u.task_id AND i.company_id = u.company_id) + AND (u.grant_id IS NULL OR EXISTS (SELECT 1 FROM mcp_oauth_grants g WHERE g.id = u.grant_id + AND g.company_id = u.company_id AND g.user_id = u.user_id)) + AND ((u.attachment_id IS NULL AND u.cleaned_at IS NOT NULL) + OR (u.cleaned_at IS NULL AND u.attachment_id IS NOT NULL AND EXISTS ( + SELECT 1 FROM issue_attachments a JOIN assets stored ON stored.id = a.asset_id + WHERE a.id = u.attachment_id AND a.company_id = u.company_id AND a.issue_id = u.task_id + AND stored.company_id = u.company_id AND stored.object_key = u.object_key + AND stored.provider = u.storage_provider)))) IS NOT TRUE` }, +] as const; diff --git a/server/src/services/tool-action-idle-proof.ts b/server/src/services/tool-action-idle-proof.ts new file mode 100644 index 00000000000..a000039d235 --- /dev/null +++ b/server/src/services/tool-action-idle-proof.ts @@ -0,0 +1,28 @@ +import { finishedIdleRun } from "./idle-work-proof.js"; + +function completedRequest(a: string): string { + return `${a}.status IN ('executed', 'failed', 'rejected', 'expired', 'cancelled') AND ${a}.resolved_at IS NOT NULL + AND EXISTS (SELECT 1 FROM tool_invocations v WHERE v.id = ${a}.invocation_id AND v.company_id = ${a}.company_id + AND v.issue_id IS NOT DISTINCT FROM ${a}.issue_id AND v.completed_at IS NOT NULL + AND v.status IN ('succeeded', 'failed', 'denied', 'cancelled', 'timed_out', 'rate_limited') + AND (v.run_id IS NULL OR EXISTS (SELECT 1 FROM heartbeat_runs r WHERE r.id = v.run_id + AND r.company_id = v.company_id AND ${finishedIdleRun('r')}))) + AND (${a}.interaction_id IS NULL OR EXISTS (SELECT 1 FROM issue_thread_interactions i WHERE i.id = ${a}.interaction_id + AND i.company_id = ${a}.company_id AND i.issue_id = ${a}.issue_id AND i.resolved_at IS NOT NULL + AND i.payload->'toolAction'->>'actionRequestId' = ${a}.id::text + AND i.payload->'toolAction'->>'invocationId' = ${a}.invocation_id::text + AND i.status IN ('accepted', 'rejected', 'expired', 'cancelled') + AND (${a}.status = 'rejected' OR i.result->'toolAction'->>'status' = CASE + WHEN ${a}.status = 'cancelled' THEN 'expired' ELSE ${a}.status END)))`; +} + +export const idleToolActionHistoryChecks = [ + { reason: "tool_action_requests", query: `SELECT 1 FROM tool_action_requests a WHERE (${completedRequest('a')}) IS NOT TRUE` }, + // delivered_at is the domain's acknowledgment. It also retires outcomes + // after issue completion/reassignment. Accepted continuations stay covered by + // the global wake/run checks, including coalesced run accounting. + { reason: "tool_action_deliveries", query: `SELECT 1 FROM tool_action_deliveries d WHERE ( + d.delivered_at IS NOT NULL AND EXISTS (SELECT 1 FROM tool_action_requests a + WHERE a.id = d.action_request_id AND a.company_id = d.company_id + AND a.issue_id = d.issue_id AND a.interaction_id = d.interaction_id AND (${completedRequest('a')}))) IS NOT TRUE` }, +] as const; diff --git a/server/src/services/workspace-history-idle-proof.ts b/server/src/services/workspace-history-idle-proof.ts new file mode 100644 index 00000000000..534968b4ca3 --- /dev/null +++ b/server/src/services/workspace-history-idle-proof.ts @@ -0,0 +1,37 @@ +import { finishedIdleRun } from "./idle-work-proof.js"; + +export const idleWorkspaceHistoryChecks = [ + // Failed imports are intentionally resumable. Completed/cancelled rows can + // still owe spool deletion, independently inspected by idle-local-work. + { reason: "company_transfer_runs", query: `SELECT 1 FROM company_transfer_runs t WHERE ( + t.status IN ('completed', 'cancelled') AND t.finished_at IS NOT NULL + AND t.direction IN ('import', 'export') AND jsonb_typeof(t.container_ref) = 'object' + AND jsonb_typeof(t.completed_parts) = 'array' + AND (t.company_id IS NOT NULL OR (t.direction = 'import' AND t.status = 'cancelled')) + AND (t.company_id IS NULL OR EXISTS (SELECT 1 FROM companies c WHERE c.id = t.company_id))) IS NOT TRUE` }, + { reason: "workspace_runtime_services", query: `SELECT 1 FROM workspace_runtime_services s WHERE ( + s.status = 'stopped' AND s.stopped_at IS NOT NULL AND s.provider = 'local_process' + AND s.exposure_handle IS NULL + AND (s.exposure IS NULL OR (jsonb_typeof(s.exposure) = 'object' + AND s.exposure->>'state' = 'removed' AND s.url IS NULL + AND COALESCE(s.exposure->'publicUrl', 'null'::jsonb) = 'null'::jsonb)) + AND (s.execution_workspace_id IS NULL OR EXISTS (SELECT 1 FROM execution_workspaces w + WHERE w.id = s.execution_workspace_id AND w.company_id = s.company_id)) + AND (s.issue_id IS NULL OR EXISTS (SELECT 1 FROM issues i WHERE i.id = s.issue_id AND i.company_id = s.company_id)) + AND (s.started_by_run_id IS NULL OR EXISTS (SELECT 1 FROM heartbeat_runs r + WHERE r.id = s.started_by_run_id AND r.company_id = s.company_id AND ${finishedIdleRun('r')}))) IS NOT TRUE` }, + // This is an exclusivity row, not a provider lease. The domain only + // evaluates staleness on the next claim; no expiry job is owed by history. + { reason: "execution_workspace_runtime_leases", query: `SELECT 1 FROM execution_workspace_runtime_leases l WHERE ( + EXISTS (SELECT 1 FROM execution_workspaces w WHERE w.id = l.execution_workspace_id AND w.company_id = l.company_id) + AND ((l.owner_issue_id IS NOT NULL AND l.owner_key = 'issue:' || l.owner_issue_id::text + AND EXISTS (SELECT 1 FROM issues i WHERE i.id = l.owner_issue_id AND i.company_id = l.company_id + AND i.status IN ('done', 'cancelled'))) + OR (l.owner_issue_id IS NULL AND l.owner_run_id IS NOT NULL AND l.owner_key = 'run:' || l.owner_run_id::text + AND EXISTS (SELECT 1 FROM heartbeat_runs r WHERE r.id = l.owner_run_id AND r.company_id = l.company_id + AND r.agent_id = l.owner_agent_id AND ${finishedIdleRun('r')}))) + AND (l.owner_run_id IS NULL OR EXISTS (SELECT 1 FROM heartbeat_runs r WHERE r.id = l.owner_run_id + AND r.company_id = l.company_id AND r.agent_id = l.owner_agent_id AND ${finishedIdleRun('r')})) + AND NOT EXISTS (SELECT 1 FROM workspace_runtime_services s WHERE s.execution_workspace_id = l.execution_workspace_id + AND (s.company_id <> l.company_id OR s.status <> 'stopped' OR s.stopped_at IS NULL))) IS NOT TRUE` }, +] as const; diff --git a/server/src/services/workspace-runtime.ts b/server/src/services/workspace-runtime.ts index 96f0b39edff..f2b1d46ba0e 100644 --- a/server/src/services/workspace-runtime.ts +++ b/server/src/services/workspace-runtime.ts @@ -46,6 +46,7 @@ import { import { and, desc, eq, gte, inArray, isNull, lte, ne, or, sql } from "drizzle-orm"; import { asNumber, asString, parseObject, renderTemplate } from "../adapters/utils.js"; import { conflict, unprocessable } from "../errors.js"; +import { registerIdleLocalWorkProbe, type IdleLocalWork } from "./idle-local-work.js"; import { withExecutionWorkspaceLifecycleLock } from "./execution-workspace-lifecycle-lock.js"; import { resolveHomeAwarePath } from "../home-paths.js"; import { hasVerifiedWorktreeSeedManifest, isVerifiedWorktreeSeedManifest } from "../worktree-seed-manifest.js"; @@ -283,6 +284,22 @@ const quarantinedRuntimeExposurePorts = new Set(); * companion together or not at all. */ const exposurePortPairClaims = new ExposurePortPairClaims(); +let runtimeReconstructionComplete = false; +let runtimeReconstructionEpoch = 0; +let runtimeReconciliation: ReturnType | null = null; + +/** Read owner memory only. Durable stopped rows do not describe a surviving + * registry process, an allocation being published, or unfinished exposure cleanup. */ +export function readWorkspaceRuntimeIdleWork(): IdleLocalWork { + if (!runtimeReconstructionComplete || runtimeReconciliation !== null) return "unknown"; + return runtimeServicesById.size || runtimeProvisionByWorkspace.size || runtimeControlStartByOwner.size + || runtimeReplacementClaimsByReuseKey.size || quarantinedRuntimeExposurePorts.size + || exposurePortPairClaims.activePorts().size + || [...inFlightAllocatedPorts.values()].some(until => until > Date.now()) ? "present" : "none"; +} +// This registry and its probe share the application process lifetime, like the +// native-session owner. Stopping one resource must not unregister its siblings. +registerIdleLocalWorkProbe(readWorkspaceRuntimeIdleWork); /** * Execution-workspace statuses that still hold an exclusive lease. `archived` * is the only terminal state; everything else — including `idle` — is a lane an @@ -525,6 +542,8 @@ type ProcessOutputAccumulator = { export async function resetRuntimeServicesForTests( opts: { terminateProcesses?: boolean; simulateSupervisorExit?: boolean } = {}, ) { + runtimeReconstructionComplete = false; + runtimeReconstructionEpoch += 1; if (opts.terminateProcesses) { for (const serviceId of [...runtimeServicesById.keys()]) { await stopRuntimeService(serviceId).catch(() => undefined); @@ -8577,6 +8596,19 @@ export async function refreshPersistedRuntimeServiceHealth(input: { } export async function reconcilePersistedRuntimeServicesOnStartup(db: Db) { + if (runtimeReconciliation) return runtimeReconciliation; + runtimeReconstructionComplete = false; + const epoch = ++runtimeReconstructionEpoch; + runtimeReconciliation = reconstructRuntimeServices(db).then(result => { + if (epoch === runtimeReconstructionEpoch) { + runtimeReconstructionComplete = result.restartFailed === 0 && result.exposureReservationDrift.length === 0; + } + return result; + }).finally(() => { runtimeReconciliation = null; }); + return runtimeReconciliation; +} + +async function reconstructRuntimeServices(db: Db) { const rows = await db .select() .from(workspaceRuntimeServices) From 450fcd81fd3bf802d9a83d6e68e28a115fb36fc0 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 2 Nov 2026 12:13:43 -0600 Subject: [PATCH 02/66] test(server): qualify completed history through domain writers Bind retained transfers and chat completion receipts to exact owner evidence, qualify every retained-history family through normal producer paths, and fence runtime stop cleanup in memory. Co-Authored-By: Paperclip --- doc/idle-sleep-safety.md | 10 +- .../__tests__/idle-completed-history.test.ts | 382 +++++++++++++++++- .../chat-channels/idle-history-proof.ts | 15 +- .../services/workspace-history-idle-proof.ts | 15 + server/src/services/workspace-runtime.ts | 29 +- 5 files changed, 431 insertions(+), 20 deletions(-) diff --git a/doc/idle-sleep-safety.md b/doc/idle-sleep-safety.md index d9f129d2b91..075ace1d59f 100644 --- a/doc/idle-sleep-safety.md +++ b/doc/idle-sleep-safety.md @@ -114,8 +114,9 @@ same closed session. Decision success requires the selected option's full effect manifest, matching ordinal/type/target receipts and audit records. The owning dismissal, explicit recovery stop and reviewed archive paths have separate proofs. Their continuation and archive-notification outboxes remain independent barriers. -Failed company transfers remain resumable. Completed/cancelled transfers still -block while their registered local spool contains cleanup work. +Failed company transfers remain resumable. Completed transfers require their +recorded manifest, exact part receipts and matching counts. Completed/cancelled +transfers still block while their registered local spool contains cleanup work. Waiting agent conversations can be passive containers under both ordinary and scheduled inspection. The same narrow proof requires the exact conversation @@ -504,8 +505,9 @@ operation by itself. Multiple replicas require shared fencing. The protocol does not make existing in-memory-only debt survive an unplanned process crash. Related public PR #13413 adds runtime-service controller requirements and uses -the same `purpose: "idle"` / `ownerId` vocabulary. This report conservatively -blocks all retained runtime-service state instead of approving such services. +the same `purpose: "idle"` / `ownerId` vocabulary. Retained stopped local-process +rows use the domain and process-reconstruction proof described above; live, +uncertain and adapter-managed runtime resources remain blockers. ## Muse maintenance diff --git a/server/src/__tests__/idle-completed-history.test.ts b/server/src/__tests__/idle-completed-history.test.ts index 5c3bc152f52..11c45896ef9 100644 --- a/server/src/__tests__/idle-completed-history.test.ts +++ b/server/src/__tests__/idle-completed-history.test.ts @@ -1,4 +1,4 @@ -import { randomBytes, randomUUID } from "node:crypto"; +import { createHash, randomBytes, randomUUID } from "node:crypto"; import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; @@ -7,13 +7,16 @@ import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; import { eq, sql } from "drizzle-orm"; import * as tables from "@paperclipai/db"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; -import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { completedHistoryChecks, readIdleSleepSafety } from "../services/idle-sleep-safety.js"; import { readIdleIssueSchedule } from "../services/idle-issue-schedule.js"; import { companyTransferRunService } from "../services/company-transfer-runs.js"; import { inspectIdleSpool } from "../services/idle-local-work.js"; -describe("retained completed history", () => { +const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_HISTORY_TEST_DATABASE_URL"); +const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); + +(support.supported ? describe : describe.skip)("retained completed history", () => { let database: Awaited> | undefined; let db: ReturnType; const now = new Date(); @@ -28,9 +31,8 @@ describe("retained completed history", () => { return row.blocked; }; beforeAll(async () => { - const external = resolveExternalTestDatabaseUrl("PAPERCLIP_HISTORY_TEST_DATABASE_URL"); - if (!external) database = await startEmbeddedPostgresTestDatabase("idle-history-"); - db = tables.createDb(external ?? database!.connectionString, { maxConnections: 4 }); + if (!externalDatabaseUrl) database = await startEmbeddedPostgresTestDatabase("idle-history-"); + db = tables.createDb(externalDatabaseUrl ?? database!.connectionString, { maxConnections: 4 }); }, 90_000); afterEach(async () => { await db.transaction(async tx => { await tx.execute(sql`SET LOCAL client_min_messages = warning`); @@ -105,7 +107,7 @@ describe("retained completed history", () => { const router = Router(); router.use(issueRoutes(db, createStorageService(createLocalDiskStorageProvider(directory)))); const oauth = createPublicMcpOAuth(db, { origin, resource }), api = createMcpApiDispatch(router); - return { ...p, token, oauth, api, cleanup: () => rm(directory, { recursive: true, force: true }) }; + return { ...p, token, oauth, api, storage: createStorageService(createLocalDiskStorageProvider(directory)), cleanup: () => rm(directory, { recursive: true, force: true }) }; } it("checks all 32 families on the real migrated schema and keeps an empty instance eligible", async () => { @@ -119,7 +121,13 @@ describe("retained completed history", () => { const result = await companyTransferRunService.resumeOrCreate(db, { companyId: f.companyId, direction: "import", actorKey: "board:history", idempotencyKey: randomUUID(), containerRef: { kind: "upload" } }); expect(await blocked("company_transfer_runs")).toBe(true); - if (status === "completed") await companyTransferRunService.complete(db, result.run.id); + if (status === "completed") { + await companyTransferRunService.recordManifest(db, result.run.id, { chunks: [{ name: "tasks.jsonl" }], blobs: [] }, "a".repeat(64)); + await companyTransferRunService.start(db, result.run.id); + expect(await companyTransferRunService.completePart(db, result.run.id, "tasks.jsonl")).toBe(true); + expect(await companyTransferRunService.claimApply(db, result.run.id)).toBe(true); + await companyTransferRunService.complete(db, result.run.id); + } else await companyTransferRunService.cancel(db, result.run.id); expect(await blocked("company_transfer_runs")).toBe(false); const directory = await mkdtemp(path.join(tmpdir(), "history-spool-")); @@ -138,6 +146,56 @@ describe("retained completed history", () => { } finally { await rm(directory, { recursive: true, force: true }); } }); + it("uses the attachment owner to commit bytes and clean an abandoned upload", async () => { + const { createPublicMcpTransfers } = await import("../services/public-mcp/file-transfers.js"); + const { default: express } = await import("express"); + const { default: request } = await import("supertest"); + const f = await seed(), p = await connectedMcp(f); + try { + const transfers = createPublicMcpTransfers(db, p.oauth, p.api, p.storage); + const principal = await p.oauth.authenticate(p.token), bytes = Buffer.from("verified retained attachment"); + const args = { companyId: f.companyId, taskId: f.issueId, requestId: randomUUID(), filename: "result.txt", contentType: "text/plain", + byteSize: bytes.length, sha256: createHash("sha256").update(bytes).digest("hex") }; + const upload = await transfers.getUploadUrl(principal, args); + expect(await blocked("mcp_attachment_uploads")).toBe(true); + const app = express(); app.use(transfers.router); + const url = new URL(String(upload.url)); + const result = await request(app).put(url.pathname + url.search).set("Content-Type", "text/plain").send(bytes); + expect(result.status).toBe(200); + expect(await blocked("mcp_attachment_uploads")).toBe(false); + const abandoned = await transfers.getUploadUrl(principal, { ...args, requestId: randomUUID() }); + expect(await blocked("mcp_attachment_uploads")).toBe(true); + await db.update(tables.mcpAttachmentUploads).set({ expiresAt: new Date(0) }).where(eq(tables.mcpAttachmentUploads.id, String(abandoned.uploadId))); + await transfers.getDownloadUrl(principal, result.body.attachment.id); + expect(await blocked("mcp_attachment_uploads")).toBe(false); + await db.update(tables.mcpAttachmentUploads).set({ attachmentId: null }).where(eq(tables.mcpAttachmentUploads.id, String(upload.uploadId))); + expect(await blocked("mcp_attachment_uploads")).toBe(true); + } finally { await p.cleanup(); } + }); + + it("retains the actual rejected tool review and only its acknowledged delivery", async () => { + const { commitToolActionReview } = await import("../services/tool-action-review.js"); + const { toolActionDeliveryService } = await import("../services/tool-action-delivery.js"); + const f = await seed(), requestId = randomUUID(), invocationId = randomUUID(), interactionId = randomUUID(); + await db.update(tables.issues).set({ status: "in_progress" }).where(eq(tables.issues.id, f.issueId)); + await db.insert(tables.toolInvocations).values({ id: invocationId, companyId: f.companyId, issueId: f.issueId, runId: f.runId, + actorType: "agent", actorId: f.agentId, agentId: f.agentId, toolName: "fixture:write", status: "awaiting_approval" }); + await db.insert(tables.issueThreadInteractions).values({ id: interactionId, companyId: f.companyId, issueId: f.issueId, + kind: "request_confirmation", payload: { version: 1, prompt: "Approve fixture action?", toolAction: { actionRequestId: requestId, invocationId } } }); + await db.insert(tables.toolActionRequests).values({ id: requestId, companyId: f.companyId, issueId: f.issueId, + invocationId, interactionId, requestedByAgentId: f.agentId, canonicalArgumentsHash: "a".repeat(64), canonicalArgumentsSummary: { summary: "fixture" } }); + expect(await blocked("tool_action_requests")).toBe(true); + await commitToolActionReview(db, { companyId: f.companyId, actionRequestId: requestId, decision: "rejected", actor: { userId: "local-board" } }); + expect(await blocked("tool_action_requests")).toBe(false); + expect(await blocked("tool_action_deliveries")).toBe(true); + await db.update(tables.issues).set({ status: "done" }).where(eq(tables.issues.id, f.issueId)); + await toolActionDeliveryService(db, { wakeup: async () => { throw new Error("Closed issue must retire its receipt"); } }).deliver(requestId); + expect(await blocked("tool_action_deliveries")).toBe(false); + await db.update(tables.toolInvocations).set({ completedAt: null }).where(eq(tables.toolInvocations.id, invocationId)); + expect(await blocked("tool_action_requests")).toBe(true); + expect(await blocked("tool_action_deliveries")).toBe(true); + }); + it("accepts MCP completion receipts and cleanup while preserving unknown writes and pending uploads", async () => { const f = await seed(), p = await mcp(f); const [receipt] = await db.insert(tables.mcpMutationReceipts).values({ ...p, companyId: f.companyId, @@ -220,6 +278,130 @@ describe("retained completed history", () => { expect(await blocked("mcp_event_deliveries")).toBe(true); }); + it("uses actual chat ingress, lifecycle application and operator publication receipts", async () => { + const { githubIngressFixture } = await import("./helpers/github-webhook-ingress.js"); + const { chatCredentialMutationLease } = await import("../services/chat-credential-mutation-lease.js"); + const { createChatResourceReconciliation } = await import("../services/chat-channels/resource-reconciliation.js"); + const { chatChannelService } = await import("../services/chat-channels.js"); + const { projectSafeChatPublication } = await import("../services/chat-publication-projection.js"); + const f = await githubIngressFixture(db), action = await f.insertAction(); + expect(await blocked("chat_actions")).toBe(true); + await f.service.processGitHubWebhookIngress(action.id); + expect(f.calls).toHaveLength(1); + expect(await blocked("chat_actions")).toBe(false); + const [issue] = await db.insert(tables.issues).values({ companyId: f.companyId, title: "Chat delivery", status: "done" }).returning(); + const [conversation] = await db.insert(tables.chatConversations).values({ companyId: f.companyId, endpointId: f.endpoint.id, + issueId: issue.id, externalConversationId: "fixture", externalThreadId: "fixture", externalLabel: "History", state: "completed" }).returning(); + const effect = { kind: "resource" as const, provider: "github" as const, providerEventId: randomUUID(), providerResourceId: "repository:fixture", + resourceType: "repository", label: "Fixture", availability: "removed" as const }; + const [delivery] = await db.insert(tables.chatDeliveries).values({ companyId: f.companyId, endpointId: f.endpoint.id, + providerEventId: `lifecycle:${effect.providerEventId}`, deduplicationKey: randomUUID(), eventKind: "uninstallation", state: "processing", attempts: 1, + normalizedEvent: { providerEventId: `lifecycle:${effect.providerEventId}`, kind: "uninstallation", lifecycle: effect } }).returning(); + expect(await blocked("chat_deliveries")).toBe(true); + const lease = chatCredentialMutationLease(db); + const resources = createChatResourceReconciliation(db, { endpointRecord: async () => ({ endpoint: f.endpoint }), + withCredentialMutationLease: lease, runtimeGeneration: () => 1, maxErrorText: 500 }); + await lease(f.endpoint, guard => resources.applyProviderAvailability({ currentEndpoint: f.endpoint, effect, + credentialLease: guard, staleResourceEffect: false, claimed: delivery, candidate: { attempts: 0 } })); + expect(await blocked("chat_deliveries")).toBe(false); + await db.update(tables.chatDeliveries).set({ normalizedEvent: {} }).where(eq(tables.chatDeliveries.id, delivery.id)); + expect(await blocked("chat_deliveries")).toBe(true); + const [publication] = await db.insert(tables.chatPublications).values({ companyId: f.companyId, endpointId: f.endpoint.id, conversationId: conversation.id, + issueId: issue.id, idempotencyKey: randomUUID(), state: "delivery_unknown", + payload: projectSafeChatPublication({ classification: "external", source: "explicit_board_send", text: "Retained result" }) }).returning(); + expect(await blocked("chat_publications")).toBe(true); + const service = chatChannelService(db, { heartbeat: { wakeup: async () => { throw new Error("No turn expected"); } }, + fetch: async () => { throw new Error("No external network expected"); } }); + try { + await service.resolvePublication(f.endpoint.id, publication.id, "mark_delivered", "local-board"); + expect(await blocked("chat_publications")).toBe(false); + await db.update(tables.chatPublications).set({ publishedAt: null }).where(eq(tables.chatPublications.id, publication.id)); + expect(await blocked("chat_publications")).toBe(true); + } finally { await service.shutdown(); } + }); + + it("retains a real acknowledged chat completion across later task versions", async () => { + const { issueService } = await import("../services/issues.js"); + const { prepareChatCompletionTurn } = await import("../services/chat-completion-delivery.js"); + const { instanceSettingsService } = await import("../services/instance-settings.js"); + const f = await seed(), service = issueService(db); + await instanceSettingsService(db).updateExperimental({ enableAgentChat: true }); + await db.update(tables.issues).set({ status: "in_progress", conversationAgentId: f.agentId, + conversationUserId: "local-board", conversationState: "waiting" }).where(eq(tables.issues.id, f.issueId)); + await db.update(tables.heartbeatRuns).set({ contextSnapshot: { issueId: f.issueId, conversationSessionGeneration: 0 } }).where(eq(tables.heartbeatRuns.id, f.runId)); + const task = await service.create(f.companyId, { title: "Actual handoff", status: "todo", createdByAgentId: f.agentId, actorRunId: f.runId }); + await service.update(task.id, { status: "done" }); + expect(await blocked("chat_completion_deliveries")).toBe(true); + const [turn] = await db.insert(tables.heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, status: "running", + contextSnapshot: { issueId: f.issueId, wakeReason: "chat_task_completed", conversationSessionGeneration: 0 } }).returning(); + await prepareChatCompletionTurn(db, turn); + const reply = await service.addComment(f.issueId, "The result is ready.", { agentId: f.agentId, runId: turn.id }, { completionReply: true }); + expect(await blocked("chat_completion_deliveries")).toBe(true); // actual reply run still owns work + await db.update(tables.heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(tables.heartbeatRuns.id, turn.id)); + expect(await blocked("chat_completion_deliveries")).toBe(false); + await service.update(task.id, { status: "todo" }); + expect(await blocked("chat_completion_deliveries")).toBe(false); // retained delivered version remains history + await db.update(tables.issueComments).set({ createdByRunId: f.runId }).where(eq(tables.issueComments.id, reply.id)); + expect(await blocked("chat_completion_deliveries")).toBe(true); + }); + + it("uses the real browser accounting and verified provider stop writers", async () => { + const { toolAccessService } = await import("../services/tool-access.js"); + const { browserUseService } = await import("../services/browser-use.js"); + const f = await seed(), userId = "browser-history-reviewer", providerRun = randomUUID(), providerSession = randomUUID(), providerBrowser = randomUUID(); + const oldKey = process.env.PAPERCLIP_SECRETS_MASTER_KEY; + process.env.PAPERCLIP_SECRETS_MASTER_KEY = "a".repeat(64); + let providerStatus = "running", stopped = false, stopCalls = 0; + const request = async (url: string, init: RequestInit = {}) => { + const route = new URL(url).pathname.replace("/api/v4", ""); + if (route === "/profiles") return Response.json({ items: [], totalItems: 0 }); + if (route === "/runs" && init.method === "POST") return Response.json({ id: providerRun, sessionId: providerSession, status: providerStatus, model: "fixture" }); + if (route.endsWith("/status")) return Response.json({ status: providerStatus }); + if (route.endsWith("/events")) return Response.json({ events: [], nextAfter: 0, hasMore: false }); + if (route.endsWith("/cancel")) { providerStatus = "cancelled"; return Response.json({}); } + if (route === `/runs/${providerRun}`) return Response.json({ status: providerStatus, totalCostUsd: "0.15", model: "fixture", result: "Finished" }); + if (route === "/browsers") return Response.json({ items: [{ id: providerBrowser, status: stopped ? "stopped" : "active" }], totalItems: 1 }); + if (route === `/browsers/${providerBrowser}`) { + if (init.method === "PATCH") { stopped = true; stopCalls++; } + return Response.json({ id: providerBrowser, agentSessionId: providerSession, status: stopped ? "stopped" : "active" }); + } + throw new Error(`Unexpected browser provider route ${route}`); + }; + try { + await db.insert(tables.companyMemberships).values({ companyId: f.companyId, principalType: "user", principalId: userId, membershipRole: "admin", status: "active" }); + const access = toolAccessService(db, { remoteHttpRequest: request, remoteHttpEndpointLookup: async () => [{ address: "8.8.8.8", family: 4 }] }); + const actor = { actorType: "user" as const, actorId: userId }; + const connected = await access.connectGalleryApp(f.companyId, { galleryKey: "browser-use-cloud", connectionMethodKey: "cloud-v4", + credentialValues: { "credentials.apiKey": "bu_fixture_secret" } }, actor); + await access.finishGalleryAppConnection(f.companyId, connected.connectionId, { enabledCatalogEntryIds: connected.catalog.map(entry => entry.id), + askFirstCatalogEntryIds: [], access: { agentIds: [f.agentId] } }, actor); + const [grant] = await db.select().from(tables.connectionGrants).where(eq(tables.connectionGrants.connectionId, connected.connectionId)); + await db.update(tables.heartbeatRuns).set({ status: "running", finishedAt: null, responsibleUserId: userId }).where(eq(tables.heartbeatRuns.id, f.runId)); + await db.update(tables.issues).set({ status: "in_progress" }).where(eq(tables.issues.id, f.issueId)); + const service = browserUseService(db, request); + const started = await service.execute({ companyId: f.companyId, agentId: f.agentId, issueId: f.issueId, runId: f.runId }, grant, + randomUUID(), "browser_start", { task: "Read the retained fixture", maxCostUsd: 1 }) as { sessionId: string }; + for (const reason of ["browser_use_sessions", "browser_use_runs"]) expect(await blocked(reason)).toBe(true); + providerStatus = "completed"; + await db.update(tables.browserUseSessions).set({ nextPollAt: new Date(0) }).where(eq(tables.browserUseSessions.id, started.sessionId)); + await service.sweep(connected.connectionId); + expect(await blocked("browser_use_browsers")).toBe(true); + const [session] = await db.select().from(tables.browserUseSessions).where(eq(tables.browserUseSessions.id, started.sessionId)); + await service.control(session, "end"); + await db.update(tables.browserUseSessions).set({ nextPollAt: new Date(0) }).where(eq(tables.browserUseSessions.id, started.sessionId)); + await service.sweep(connected.connectionId); + expect(stopCalls).toBeGreaterThan(0); + await db.update(tables.heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(tables.heartbeatRuns.id, f.runId)); + for (const reason of ["browser_use_sessions", "browser_use_runs", "browser_use_browsers"]) expect(await blocked(reason), reason).toBe(false); + const [run] = await db.select().from(tables.browserUseRuns).where(eq(tables.browserUseRuns.sessionId, started.sessionId)); + expect(run.result?.paperclipBudgetHold).toMatchObject({ state: "settled", accountedCentsExact: "15.0000000" }); + await db.update(tables.browserUseRuns).set({ eventsDrained: 0 }).where(eq(tables.browserUseRuns.id, run.id)); + expect(await blocked("browser_use_runs")).toBe(true); + } finally { + if (oldKey === undefined) delete process.env.PAPERCLIP_SECRETS_MASTER_KEY; else process.env.PAPERCLIP_SECRETS_MASTER_KEY = oldKey; + } + }); + it("keeps browser accounting, run ownership and provider resource cleanup independent", async () => { const f = await seed(), c = await connection(f), invocationId = randomUUID(); const [session] = await db.insert(tables.browserUseSessions).values({ ...f, ...c, status: "closed", providerSessionId: randomUUID() }).returning(); @@ -272,6 +454,48 @@ describe("retained completed history", () => { expect(await read()).toEqual({ version: 1, backgroundWork: "present" }); }, 60_000); + it("uses actual secret and binding approval receipts, including ciphertext scrubbing", async () => { + const { createSecretProposalsService } = await import("../services/secret-proposals.js"); + const f = await seed(), p = await mcp(f), oldKey = process.env.PAPERCLIP_SECRETS_MASTER_KEY; + process.env.PAPERCLIP_SECRETS_MASTER_KEY = "a".repeat(64); + try { + await db.update(tables.heartbeatRuns).set({ status: "running", finishedAt: null }).where(eq(tables.heartbeatRuns.id, f.runId)); + await db.update(tables.issues).set({ status: "in_progress", executionRunId: f.runId }).where(eq(tables.issues.id, f.issueId)); + const service = createSecretProposalsService(db); + const context = { companyId: f.companyId, heartbeatRunId: f.runId, registerForRedaction: () => {} }; + const secret = await service.createSecret(context, { name: "history/token", key: "HISTORY_TOKEN", value: "synthetic-history-secret", justification: "Retained completed approval" }); + expect(await blocked("company_secret_proposals")).toBe(true); + const approved = await service.approve(f.companyId, secret.id, { resolvedByUserId: p.userId }); + const binding = await service.createBinding(context, { secretId: approved.createdSecretId!, configPath: "env.HISTORY_TOKEN", + bindingTargetPolicy: "self_and_reports", justification: "Use the approved credential" }); + await service.approve(f.companyId, binding.id, { resolvedByUserId: p.userId }); + await db.update(tables.heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(tables.heartbeatRuns.id, f.runId)); + expect(await blocked("company_secret_proposals")).toBe(false); + await db.update(tables.companySecretProposals).set({ appliedBindingConfigPath: null }).where(eq(tables.companySecretProposals.id, binding.id)); + expect(await blocked("company_secret_proposals")).toBe(true); + } finally { if (oldKey === undefined) delete process.env.PAPERCLIP_SECRETS_MASTER_KEY; else process.env.PAPERCLIP_SECRETS_MASTER_KEY = oldKey; } + }); + + it("uses actual retention archive notification dispatch and durable acknowledgment", async () => { + const { decisionRetentionService } = await import("../services/decision-retention.js"); + const { decisionArchiveDeliveryService } = await import("../services/decision-archive-delivery.js"); + const f = await seed(), activityAt = new Date("2026-01-01T00:00:00.000Z"); + await db.update(tables.agents).set({ status: "active" }).where(eq(tables.agents.id, f.agentId)); + await db.update(tables.issues).set({ createdByAgentId: f.agentId }).where(eq(tables.issues.id, f.issueId)); + await db.insert(tables.decisionRetention).values({ companyId: f.companyId, sourceKind: "review", sourceId: f.issueId, sourceActivityAt: activityAt, version: 1 }); + expect(await decisionRetentionService(db).autoArchive({ companyId: f.companyId, items: [{ sourceKind: "review", subject: { id: f.issueId }, + activityAt: activityAt.toISOString(), retentionVersion: 1, keep: false, archivedAt: null }], now })).toBe(1); + expect(await blocked("decision_archive_notification_outbox")).toBe(true); + let notifications = 0; + expect(await decisionArchiveDeliveryService(db, { notifyOriginAgent: async batch => { + expect(batch).toMatchObject({ companyId: f.companyId, agentId: f.agentId, items: [{ sourceId: f.issueId }] }); notifications++; + } }).deliver()).toMatchObject({ delivered: 1 }); + expect(notifications).toBe(1); + expect(await blocked("decision_archive_notification_outbox")).toBe(false); + await db.update(tables.decisionArchiveNotificationOutbox).set({ deliveredAt: null }); + expect(await blocked("decision_archive_notification_outbox")).toBe(true); + }); + it.each(["rejected", "withdrawn", "expired"] as const)("accepts actual %s secret-proposal scrubbing and refuses retained ciphertext", async status => { const { createSecretProposalsService } = await import("../services/secret-proposals.js"); const f = await seed(); @@ -356,6 +580,31 @@ describe("retained completed history", () => { } }, 60_000); + it.each(["missing_decider", "missing_option"] as const)("requires the actual %s recovery stop receipt after partial effects", async reason => { + const previous = process.env.PAPERCLIP_DECISION_SIGNING_SECRET; + process.env.PAPERCLIP_DECISION_SIGNING_SECRET = "0123456789abcdef0123456789abcdef"; + try { + const f = await decisionFixture(); + const decision = await f.service.create({ companyId: f.companyId, actor: f.actor, agentId: f.agentId, runId: f.runId, + title: "Recover interrupted effect ownership", body: reason, continuationPolicy: "none", options: [{ id: "yes", label: "Yes", + effects: [{ type: "comment_on_issue", targetIssueId: f.issueId, staleness: "lenient", bodyMarkdown: "Already persisted effect" }] }] }); + await f.service.decide({ id: decision.id, optionId: "yes", decidedByUserId: f.userId, userActor: f.userActor }); + // Model a stranded claim whose durable option/decider was lost after an + // already-written effect. Recovery must record its own final stop. + await db.update(tables.decisions).set({ executionStatus: "running", updatedAt: new Date(Date.now() - 120_000), + ...(reason === "missing_decider" ? { decidedByUserId: null } : { chosenOptionId: "missing" }) }).where(eq(tables.decisions.id, decision.id)); + expect(await blocked("decisions")).toBe(true); + expect(await f.service.recoverEffects()).toEqual({ resumed: 1 }); + await f.finish(); + expect(await f.service.get(decision.id)).toMatchObject({ executionStatus: "partial", metadata: { effectRecoveryError: reason } }); + expect(await blocked("decisions")).toBe(false); + await db.delete(tables.activityLog).where(sql`${tables.activityLog.entityId} = ${decision.id} AND ${tables.activityLog.action} = 'decision.recovery_failed'`); + expect(await blocked("decisions")).toBe(true); + } finally { + if (previous === undefined) delete process.env.PAPERCLIP_DECISION_SIGNING_SECRET; else process.env.PAPERCLIP_DECISION_SIGNING_SECRET = previous; + } + }); + it.each(["archive", "keep", "manifest_failure"] as const)("checks the separate reviewed retention manifest for %s", async branch => { const { hashAttentionArchiveManifest } = await import("../services/decision-retention.js"); const previous = process.env.PAPERCLIP_DECISION_SIGNING_SECRET; @@ -424,7 +673,8 @@ describe("retained completed history", () => { // Named external runner boundary: persist the real broker's receipt for // the exact command it emitted, without invoking paid native execution. await port.settle({ sourceEventId: `result_${command.requestId}`, payload: { binding: ref, requestId: command.requestId, - outcome: { status: command.action === "accept" ? "accepted" : "completed" } } }); + outcome: command.action === "consume_input" ? { status: "consumed", ...command.input } + : { status: command.action === "accept" ? "accepted" : "completed" } } }); }); try { await port.dispatch({ sourceEventId: `offer_${randomUUID()}`, payload: { kind: "assignment", binding: ref, tools: [], @@ -432,18 +682,95 @@ describe("retained completed history", () => { const [assignment] = await db.select().from(tables.museRunnerAssignments).where(eq(tables.museRunnerAssignments.runId, f.runId)); expect(await blocked("muse_runner_assignments")).toBe(true); await broker.act(subject, { version: 1, command: "accept", assignmentId: assignment.id, requestId: randomUUID() }); + const { digestPaperclipSemanticContent } = await import("../vendor/paperclip-runner/index.js"); + const nativeRequestId = "history_question", response = { schema: "paperclip.question_response.v1", answers: [] }; + const inputDigest = digestPaperclipSemanticContent({ requestId: nativeRequestId, turnId, response }); + await port.inputAvailable({ sourceEventId: randomUUID(), payload: { binding: ref, requestId: nativeRequestId, turnId, inputDigest, response } }); + expect(await blocked("muse_input_deliveries")).toBe(true); + await broker.act(subject, { version: 1, command: "consume_input", assignmentId: assignment.id, requestId: randomUUID(), + nativeRequestId, inputDigest, continuationReceiptId: randomUUID(), continuationPersisted: true }); await broker.act(subject, { version: 1, command: "finish", assignmentId: assignment.id, requestId: randomUUID(), result: {} }); expect(await blocked("muse_runner_assignments")).toBe(true); const lifecycle = createHeartbeatLifecycle(db, { emitTerminalAgentTaskRun: () => {} } as unknown as import("../services/heartbeat/run-lifecycle.js").HeartbeatLifecycleDependencies); await lifecycle.setRunStatus(f.runId, "succeeded", { finishedAt: new Date() }); await broker.reconcileTerminalAssignments(f.companyId, f.agentId, binding.id); - for (const family of ["muse_runner_assignments", "muse_runner_operations", "muse_mailbox_items", "external_agent_holds"]) { + for (const family of ["muse_runner_assignments", "muse_runner_operations", "muse_mailbox_items", "muse_input_deliveries", "external_agent_holds"]) { expect(await blocked(family), family).toBe(false); } await db.update(tables.museRunnerOperations).set({ status: "unknown" }).where(eq(tables.museRunnerOperations.assignmentId, assignment.id)); expect(await blocked("muse_runner_assignments")).toBe(true); expect(await blocked("muse_runner_operations")).toBe(true); + await db.update(tables.museRunnerOperations).set({ status: "settled" }).where(eq(tables.museRunnerOperations.assignmentId, assignment.id)); + expect(await blocked("muse_agent_bindings")).toBe(true); + expect(await blocked("muse_credentials")).toBe(true); + await broker.revoke(f.companyId, f.agentId, p.userId); + expect(await blocked("muse_agent_bindings")).toBe(false); + expect(await blocked("muse_credentials")).toBe(true); // cleanup authority still lives for its owner TTL + await db.update(tables.museCredentials).set({ expiresAt: new Date(0) }).where(eq(tables.museCredentials.bindingId, binding.id)); + expect(await blocked("muse_credentials")).toBe(false); + } finally { await detach(); } + }, 60_000); + + it("settles actual Dot runner assignments, operation receipts and mailbox history", async () => { + const { dotRunnerBroker } = await import("../services/dot-runner-broker.js"); + const { instanceSettingsService } = await import("../services/instance-settings.js"); + const { createHeartbeatLifecycle } = await import("../services/heartbeat/run-lifecycle.js"); + const f = await seed(), p = await mcp(f, true), sessionId = randomUUID(), turnId = `turn_${randomUUID()}`; + await instanceSettingsService(db).updateExperimental({ enablePublicMcp: true, enableOpenAiDot: true }); + await db.insert(tables.companyMemberships).values({ companyId: f.companyId, principalType: "user", principalId: p.userId, membershipRole: "owner", status: "active" }); + await db.update(tables.mcpOauthGrants).set({ revokedAt: null }).where(eq(tables.mcpOauthGrants.id, p.grantId)); + const [grant] = await db.select().from(tables.mcpOauthGrants).where(eq(tables.mcpOauthGrants.id, p.grantId)); + const [binding] = await db.insert(tables.dotAgentBindings).values({ companyId: f.companyId, agentId: f.agentId, operatorId: p.userId, + grantId: p.grantId, providerKind: "openai_dot", status: "connected" }).returning(); + await db.update(tables.agents).set({ status: "active", adapterType: "paperclip_runner", + adapterConfig: { provider: "openai_dot", dotBindingId: binding.id, allowUnmeteredProvider: true } }).where(eq(tables.agents.id, f.agentId)); + await db.update(tables.issues).set({ status: "in_progress", responsibleUserId: p.userId, checkoutRunId: f.runId, executionRunId: f.runId }).where(eq(tables.issues.id, f.issueId)); + await db.update(tables.heartbeatRuns).set({ status: "running", runtimeMode: "native", driverKind: "openai_dot_mcp", finishedAt: null, + nativeIssueId: f.issueId, nativeSessionId: sessionId, nativePhase: "turn_running", responsibleUserId: p.userId }).where(eq(tables.heartbeatRuns.id, f.runId)); + await db.insert(tables.nativeRunFinalizations).values({ runId: f.runId, companyId: f.companyId, issueId: f.issueId, + phase: "running", controllerGeneration: 1, leaseExpiresAt: new Date(Date.now() + 600_000) }); + const broker = dotRunnerBroker(db, { heartbeat: { wakeup: async () => null, cancelRun: async () => null } }); + const { createPublicMcpOAuth } = await import("../services/public-mcp/oauth.js"); + const { createPublicMcpEvents } = await import("../services/public-mcp/events.js"); + const principal = { grant, actor: { type: "agent" as const, agentId: f.agentId, companyId: f.companyId }, + company: { id: f.companyId, name: "History", issuePrefix: "H", status: "active" } }; + const events = createPublicMcpEvents(db, createPublicMcpOAuth(db, { origin: "https://example.test", resource: "https://example.test/mcp" }, { mode: "openai_dot_agent" }), + async () => { throw new Error("No personal API expected"); }, { enableDotRunner: true, fetch: async (_url, init) => { + const body = JSON.parse(String(init.body)); + return body.type === "verification" ? Response.json({ challenge: body.challenge }) : new Response(null, { status: 204 }); + } }); + await events.subscribe(principal, { name: "paperclip.dot.mailbox_updated", arguments: { companyId: f.companyId, bindingId: binding.id }, + delivery: { mode: "webhook", url: "https://receiver.example/dot", secret: `whsec_${randomBytes(32).toString("base64")}` } }); + await events.tick(); + const challenge = (await broker.mailbox(principal)).items.find(item => item.kind === "readiness_challenge")!; + await broker.confirmChallenge(principal, String(challenge.references.challenge)); + const execution = { binding: { companyId: f.companyId, agentId: f.agentId, runId: f.runId }, + provider: { kind: "openai_dot" as const, binding: await broker.snapshot(f.companyId, f.agentId, binding.id) } }; + const ref = { ...execution.provider.binding, runId: f.runId, normalizedSessionId: sessionId, turnId, assignmentRevision: 1 }; + const port = broker.port(execution); + const detach = await port.attach(async command => { + await port.settle({ sourceEventId: randomUUID(), payload: { binding: ref, requestId: command.requestId, + outcome: { status: command.action === "accept" ? "accepted" : "completed" } } }); + }); + try { + await port.dispatch({ sourceEventId: randomUUID(), payload: { kind: "assignment", binding: ref, tools: [], + acceptByUnixMs: Date.now() + 600_000, expiresAtUnixMs: Date.now() + 3_600_000 } }); + const [assignment] = await db.select().from(tables.dotRunnerAssignments).where(eq(tables.dotRunnerAssignments.runId, f.runId)); + expect(await blocked("dot_runner_assignments")).toBe(true); + expect(await blocked("dot_mailbox_items")).toBe(true); + await broker.operation(principal, assignment.id, randomUUID(), "accept", {}); + await broker.operation(principal, assignment.id, randomUUID(), "finish", { result: {} }); + const lifecycle = createHeartbeatLifecycle(db, { emitTerminalAgentTaskRun: () => {} } as unknown as + import("../services/heartbeat/run-lifecycle.js").HeartbeatLifecycleDependencies); + await lifecycle.setRunStatus(f.runId, "succeeded", { finishedAt: new Date() }); + for (const reason of ["dot_runner_assignments", "dot_runner_operations"]) expect(await blocked(reason), reason).toBe(false); + // The independent readiness mailbox is retained until its binding is revoked. + await broker.revoke(f.companyId, f.agentId, p.userId); + expect(await blocked("dot_mailbox_items")).toBe(false); + await db.update(tables.dotRunnerOperations).set({ status: "dispatched", outcome: null }).where(eq(tables.dotRunnerOperations.assignmentId, assignment.id)); + expect(await blocked("dot_runner_assignments")).toBe(true); + expect(await blocked("dot_runner_operations")).toBe(true); } finally { await detach(); } }, 60_000); @@ -539,8 +866,41 @@ describe("retained completed history", () => { const adopted = await runtime.reconcilePersistedRuntimeServicesOnStartup(db); expect(adopted.adopted).toBe(1); expect(runtime.readWorkspaceRuntimeIdleWork()).toBe("present"); - await runtime.stopRuntimeServicesForExecutionWorkspace({ db, executionWorkspaceId: workspace.id, workspaceCwd: directory }); + let enterCleanup!: () => void, releaseCleanup!: () => void; + const cleanupEntered = new Promise(resolve => { enterCleanup = resolve; }); + const cleanupGate = new Promise(resolve => { releaseCleanup = resolve; }); + const failedControl = Object.create(db) as typeof db; + failedControl.select = (() => ({ from: () => ({ where: async () => { + enterCleanup(); await cleanupGate; throw new Error("controlled stop cleanup failure"); + } }) })) as typeof db.select; + const stopping = runtime.stopRuntimeServicesForExecutionWorkspace({ db: failedControl, + executionWorkspaceId: workspace.id, workspaceCwd: directory }); + await cleanupEntered; + expect(runtime.readWorkspaceRuntimeIdleWork()).toBe("present"); + let releaseReconstruction!: () => void; + const reconstructionGate = new Promise(resolve => { releaseReconstruction = resolve; }); + const staleReconstruction = Object.create(db) as typeof db; + let firstSelect = true; + staleReconstruction.select = ((...fields: unknown[]) => { + const query = Reflect.apply(db.select, db, fields); + if (!firstSelect) return query; + firstSelect = false; + return { from: (...from: unknown[]) => { + const selected = Reflect.apply(query.from, query, from); + return { where: (...where: unknown[]) => reconstructionGate.then(() => Reflect.apply(selected.where, selected, where)) }; + } }; + }) as typeof db.select; + const earlierReconstruction = runtime.reconcilePersistedRuntimeServicesOnStartup(staleReconstruction); + releaseCleanup(); + await expect(stopping).rejects.toThrow("controlled stop cleanup failure"); + releaseReconstruction(); + await earlierReconstruction; + expect(runtime.readWorkspaceRuntimeIdleWork()).toBe("unknown"); // older successful flight cannot erase the failed control epoch + await expect(fetch(`http://127.0.0.1:${port}`, { signal: AbortSignal.timeout(1000) })).rejects.toThrow(); ownedProcess = null; + expect(runtime.readWorkspaceRuntimeIdleWork()).toBe("unknown"); + await runtime.reconcilePersistedRuntimeServicesOnStartup(db); + await runtime.stopRuntimeServicesForExecutionWorkspace({ db, executionWorkspaceId: workspace.id, workspaceCwd: directory }); expect(runtime.readWorkspaceRuntimeIdleWork()).toBe("none"); expect(await blocked("workspace_runtime_services")).toBe(false); const unavailable = Object.create(db) as typeof db; diff --git a/server/src/services/chat-channels/idle-history-proof.ts b/server/src/services/chat-channels/idle-history-proof.ts index 778779d2b3d..8f9642b3ac5 100644 --- a/server/src/services/chat-channels/idle-history-proof.ts +++ b/server/src/services/chat-channels/idle-history-proof.ts @@ -44,6 +44,15 @@ function completedAction(a: string): string { export const idleChatHistoryChecks = [ { reason: "chat_deliveries", query: `SELECT 1 FROM chat_deliveries d WHERE ( EXISTS (SELECT 1 FROM chat_endpoints e WHERE e.id = d.endpoint_id AND e.company_id = d.company_id) + AND jsonb_typeof(d.normalized_event) = 'object' + AND d.normalized_event->>'providerEventId' = d.provider_event_id + AND d.normalized_event->>'kind' = d.event_kind + AND d.event_kind IN ('mention', 'message', 'direct_message', 'message_updated', 'message_deleted', 'message_restored', + 'reaction_added', 'reaction_removed', 'action', 'modal_submitted', 'modal_closed', 'slash_command', 'file_shared', 'installation', 'uninstallation', 'unknown') + AND (d.state <> 'processed' OR d.event_kind NOT IN ('mention', 'message', 'direct_message') + OR EXISTS (SELECT 1 FROM chat_actions consumed WHERE consumed.delivery_id = d.id + AND consumed.company_id = d.company_id AND consumed.endpoint_id = d.endpoint_id + AND consumed.kind = 'inbound_wakeup' AND (${completedAction('consumed')}))) AND d.next_attempt_at IS NULL AND (d.state = 'failed' OR (d.state IN ('processed', 'filtered') AND d.processed_at IS NOT NULL)) AND (d.conversation_id IS NULL OR EXISTS (SELECT 1 FROM chat_conversations c WHERE c.id = d.conversation_id AND c.company_id = d.company_id AND c.endpoint_id = d.endpoint_id)) @@ -68,12 +77,14 @@ export const idleChatHistoryChecks = [ JOIN issues source ON source.id = h.conversation_id AND source.company_id = h.company_id WHERE h.task_id = d.task_id AND h.company_id = d.company_id AND source.conversation_agent_id = h.agent_id - AND (d.status <> 'delivered' OR (d.status_version = task.status_version AND task.status = 'done' + AND (d.status <> 'delivered' OR (d.status_version <= task.status_version AND EXISTS (SELECT 1 FROM issue_comments c WHERE c.id = d.response_comment_id - AND c.company_id = d.company_id AND c.issue_id = h.conversation_id AND c.author_agent_id = h.agent_id)))) + AND c.company_id = d.company_id AND c.issue_id = h.conversation_id AND c.author_agent_id = h.agent_id + AND c.created_by_run_id = d.target_run_id)))) AND (d.target_run_id IS NULL AND d.status <> 'delivered' OR EXISTS ( SELECT 1 FROM heartbeat_runs r JOIN chat_task_handoffs h ON h.task_id = d.task_id WHERE r.id = d.target_run_id AND r.company_id = d.company_id AND r.agent_id = h.agent_id + AND h.company_id = d.company_id AND r.context_snapshot->>'conversationSessionGeneration' = h.session_generation::text AND (r.native_issue_id = h.conversation_id OR r.context_snapshot->>'issueId' = h.conversation_id::text) AND ${finishedIdleRun('r')}))) IS NOT TRUE` }, ] as const; diff --git a/server/src/services/workspace-history-idle-proof.ts b/server/src/services/workspace-history-idle-proof.ts index 534968b4ca3..d6d3f3ce428 100644 --- a/server/src/services/workspace-history-idle-proof.ts +++ b/server/src/services/workspace-history-idle-proof.ts @@ -7,6 +7,21 @@ export const idleWorkspaceHistoryChecks = [ t.status IN ('completed', 'cancelled') AND t.finished_at IS NOT NULL AND t.direction IN ('import', 'export') AND jsonb_typeof(t.container_ref) = 'object' AND jsonb_typeof(t.completed_parts) = 'array' + AND (t.status = 'cancelled' OR (t.error IS NULL AND t.manifest_sha256 ~ '^[a-f0-9]{64}$' + AND CASE WHEN jsonb_typeof(t.manifest->'chunks') = 'array' AND jsonb_typeof(t.manifest->'blobs') = 'array' THEN + t.chunk_count = jsonb_array_length(t.manifest->'chunks') AND t.blob_count = jsonb_array_length(t.manifest->'blobs') + AND NOT EXISTS (SELECT 1 FROM jsonb_array_elements((t.manifest->'chunks') || (t.manifest->'blobs')) part + WHERE jsonb_typeof(part->'name') IS DISTINCT FROM 'string' OR NOT t.completed_parts ? (part->>'name')) + AND NOT EXISTS (SELECT 1 FROM jsonb_array_elements_text(t.completed_parts) done + WHERE NOT EXISTS (SELECT 1 FROM jsonb_array_elements((t.manifest->'chunks') || (t.manifest->'blobs')) part WHERE part->>'name' = done)) + WHEN jsonb_typeof(t.manifest->'parts') = 'array' THEN + t.chunk_count = jsonb_array_length(t.manifest->'parts') AND t.blob_count = 0 + AND NOT EXISTS (SELECT 1 FROM jsonb_array_elements(t.manifest->'parts') WITH ORDINALITY part(value, ordinal) + WHERE NOT t.completed_parts ? ('part-' || (ordinal - 1)::text)) + AND NOT EXISTS (SELECT 1 FROM jsonb_array_elements_text(t.completed_parts) done + WHERE NOT EXISTS (SELECT 1 FROM jsonb_array_elements(t.manifest->'parts') WITH ORDINALITY part(value, ordinal) + WHERE done = 'part-' || (ordinal - 1)::text)) + ELSE false END)) AND (t.company_id IS NOT NULL OR (t.direction = 'import' AND t.status = 'cancelled')) AND (t.company_id IS NULL OR EXISTS (SELECT 1 FROM companies c WHERE c.id = t.company_id))) IS NOT TRUE` }, { reason: "workspace_runtime_services", query: `SELECT 1 FROM workspace_runtime_services s WHERE ( diff --git a/server/src/services/workspace-runtime.ts b/server/src/services/workspace-runtime.ts index f2b1d46ba0e..8a6211968a6 100644 --- a/server/src/services/workspace-runtime.ts +++ b/server/src/services/workspace-runtime.ts @@ -285,6 +285,7 @@ const quarantinedRuntimeExposurePorts = new Set(); */ const exposurePortPairClaims = new ExposurePortPairClaims(); let runtimeReconstructionComplete = false; +let runtimeStopsInFlight = 0; let runtimeReconstructionEpoch = 0; let runtimeReconciliation: ReturnType | null = null; @@ -292,7 +293,7 @@ let runtimeReconciliation: ReturnType | null * registry process, an allocation being published, or unfinished exposure cleanup. */ export function readWorkspaceRuntimeIdleWork(): IdleLocalWork { if (!runtimeReconstructionComplete || runtimeReconciliation !== null) return "unknown"; - return runtimeServicesById.size || runtimeProvisionByWorkspace.size || runtimeControlStartByOwner.size + return runtimeStopsInFlight || runtimeServicesById.size || runtimeProvisionByWorkspace.size || runtimeControlStartByOwner.size || runtimeReplacementClaimsByReuseKey.size || quarantinedRuntimeExposurePorts.size || exposurePortPairClaims.activePorts().size || [...inFlightAllocatedPorts.values()].some(until => until > Date.now()) ? "present" : "none"; @@ -7174,7 +7175,21 @@ async function cleanupRecordExposure( await persistRuntimeServiceRecord(record.db, record).catch(() => undefined); } +async function withRuntimeStopWork(stop: () => Promise): Promise { + runtimeStopsInFlight += 1; + try { return await stop(); } + catch (error) { + runtimeReconstructionComplete = false; + runtimeReconstructionEpoch += 1; + throw error; + } finally { runtimeStopsInFlight -= 1; } +} + async function stopRuntimeService(serviceId: string) { + return withRuntimeStopWork(() => stopRuntimeServiceRecord(serviceId)); +} + +async function stopRuntimeServiceRecord(serviceId: string) { const record = runtimeServicesById.get(serviceId); if (!record) return; clearIdleTimer(record); @@ -8315,7 +8330,11 @@ export async function releaseRuntimeServicesForRun(runId: string) { } } -export async function stopRuntimeServicesForExecutionWorkspace(input: { +export async function stopRuntimeServicesForExecutionWorkspace(input: Parameters[0]) { + return withRuntimeStopWork(() => stopRuntimeServicesForExecutionWorkspaceOwned(input)); +} + +async function stopRuntimeServicesForExecutionWorkspaceOwned(input: { db?: Db; executionWorkspaceId: string; workspaceCwd?: string | null; @@ -8370,7 +8389,11 @@ export async function stopRuntimeServicesForExecutionWorkspace(input: { } } -export async function stopRuntimeServicesForProjectWorkspace(input: { +export async function stopRuntimeServicesForProjectWorkspace(input: Parameters[0]) { + return withRuntimeStopWork(() => stopRuntimeServicesForProjectWorkspaceOwned(input)); +} + +async function stopRuntimeServicesForProjectWorkspaceOwned(input: { db?: Db; projectWorkspaceId: string; runtimeServiceId?: string | null; From aaf55692c4def3ce572465fa8df3ff2961d9a687 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 2 Nov 2026 12:25:21 -0600 Subject: [PATCH 03/66] refactor(server): inspect chat action receipts once per report Aggregate exact action completion evidence per delivery to avoid repeated unindexed action scans. Keep pending siblings and missing consumed wakes blocked. Bound the real upload integration and preserve slow-call diagnostics without extra SQL. Co-Authored-By: Paperclip --- .../__tests__/idle-completed-history.test.ts | 20 ++++++++++++++++--- .../chat-channels/idle-history-proof.ts | 16 +++++++++------ 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/server/src/__tests__/idle-completed-history.test.ts b/server/src/__tests__/idle-completed-history.test.ts index 11c45896ef9..c4fe2b56843 100644 --- a/server/src/__tests__/idle-completed-history.test.ts +++ b/server/src/__tests__/idle-completed-history.test.ts @@ -27,7 +27,10 @@ const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPos const blocked = async (reason: string) => { const check = completedHistoryChecks.find(check => check.reason === reason); if (!check) throw new Error(`Missing family ${reason}`); + const started = performance.now(); const [row] = await db.execute<{ blocked: boolean }>(sql`SELECT EXISTS (${sql.raw(check.query)}) AS blocked`); + const elapsed = performance.now() - started; + if (elapsed > 1000) console.info(`History predicate ${reason}: ${Math.round(elapsed)}ms`); return row.blocked; }; beforeAll(async () => { @@ -160,7 +163,7 @@ const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPos expect(await blocked("mcp_attachment_uploads")).toBe(true); const app = express(); app.use(transfers.router); const url = new URL(String(upload.url)); - const result = await request(app).put(url.pathname + url.search).set("Content-Type", "text/plain").send(bytes); + const result = await request(app).put(url.pathname + url.search).set("Content-Type", "text/plain").send(bytes).timeout({ response: 5000, deadline: 10_000 }); expect(result.status).toBe(200); expect(await blocked("mcp_attachment_uploads")).toBe(false); const abandoned = await transfers.getUploadUrl(principal, { ...args, requestId: randomUUID() }); @@ -171,7 +174,7 @@ const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPos await db.update(tables.mcpAttachmentUploads).set({ attachmentId: null }).where(eq(tables.mcpAttachmentUploads.id, String(upload.uploadId))); expect(await blocked("mcp_attachment_uploads")).toBe(true); } finally { await p.cleanup(); } - }); + }, 60_000); it("retains the actual rejected tool review and only its acknowledged delivery", async () => { const { commitToolActionReview } = await import("../services/tool-action-review.js"); @@ -304,7 +307,18 @@ const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPos await lease(f.endpoint, guard => resources.applyProviderAvailability({ currentEndpoint: f.endpoint, effect, credentialLease: guard, staleResourceEffect: false, claimed: delivery, candidate: { attempts: 0 } })); expect(await blocked("chat_deliveries")).toBe(false); - await db.update(tables.chatDeliveries).set({ normalizedEvent: {} }).where(eq(tables.chatDeliveries.id, delivery.id)); + const firstAction = await f.insertAction({ deliveryId: delivery.id }); + expect(await blocked("chat_deliveries")).toBe(true); + await f.service.processGitHubWebhookIngress(firstAction.id); + expect(await blocked("chat_deliveries")).toBe(false); + const secondAction = await f.insertAction({ deliveryId: delivery.id }); + expect(await blocked("chat_deliveries")).toBe(true); // one completed sibling cannot hide another pending action + await f.service.processGitHubWebhookIngress(secondAction.id); + expect(await blocked("chat_deliveries")).toBe(false); + await db.update(tables.chatDeliveries).set({ eventKind: "message", + normalizedEvent: { providerEventId: delivery.providerEventId, kind: "message" } }).where(eq(tables.chatDeliveries.id, delivery.id)); + expect(await blocked("chat_deliveries")).toBe(true); // ordinary input requires its own consumed wake, not arbitrary completed actions + await db.update(tables.chatDeliveries).set({ eventKind: "uninstallation", normalizedEvent: {} }).where(eq(tables.chatDeliveries.id, delivery.id)); expect(await blocked("chat_deliveries")).toBe(true); const [publication] = await db.insert(tables.chatPublications).values({ companyId: f.companyId, endpointId: f.endpoint.id, conversationId: conversation.id, issueId: issue.id, idempotencyKey: randomUUID(), state: "delivery_unknown", diff --git a/server/src/services/chat-channels/idle-history-proof.ts b/server/src/services/chat-channels/idle-history-proof.ts index 8f9642b3ac5..434087d5e38 100644 --- a/server/src/services/chat-channels/idle-history-proof.ts +++ b/server/src/services/chat-channels/idle-history-proof.ts @@ -42,7 +42,14 @@ function completedAction(a: string): string { } export const idleChatHistoryChecks = [ - { reason: "chat_deliveries", query: `SELECT 1 FROM chat_deliveries d WHERE ( + { reason: "chat_deliveries", query: `WITH action_outcomes AS MATERIALIZED ( + SELECT a.delivery_id, a.kind, (${completedAction('a')}) IS TRUE AS completed + FROM chat_actions a WHERE a.delivery_id IS NOT NULL + ), action_receipts AS ( + SELECT delivery_id, bool_and(completed) AS completed, + bool_or(kind = 'inbound_wakeup' AND completed) AS consumed + FROM action_outcomes GROUP BY delivery_id + ) SELECT 1 FROM chat_deliveries d LEFT JOIN action_receipts receipt ON receipt.delivery_id = d.id WHERE ( EXISTS (SELECT 1 FROM chat_endpoints e WHERE e.id = d.endpoint_id AND e.company_id = d.company_id) AND jsonb_typeof(d.normalized_event) = 'object' AND d.normalized_event->>'providerEventId' = d.provider_event_id @@ -50,14 +57,11 @@ export const idleChatHistoryChecks = [ AND d.event_kind IN ('mention', 'message', 'direct_message', 'message_updated', 'message_deleted', 'message_restored', 'reaction_added', 'reaction_removed', 'action', 'modal_submitted', 'modal_closed', 'slash_command', 'file_shared', 'installation', 'uninstallation', 'unknown') AND (d.state <> 'processed' OR d.event_kind NOT IN ('mention', 'message', 'direct_message') - OR EXISTS (SELECT 1 FROM chat_actions consumed WHERE consumed.delivery_id = d.id - AND consumed.company_id = d.company_id AND consumed.endpoint_id = d.endpoint_id - AND consumed.kind = 'inbound_wakeup' AND (${completedAction('consumed')}))) + OR receipt.consumed IS TRUE) AND d.next_attempt_at IS NULL AND (d.state = 'failed' OR (d.state IN ('processed', 'filtered') AND d.processed_at IS NOT NULL)) AND (d.conversation_id IS NULL OR EXISTS (SELECT 1 FROM chat_conversations c WHERE c.id = d.conversation_id AND c.company_id = d.company_id AND c.endpoint_id = d.endpoint_id)) - AND NOT EXISTS (SELECT 1 FROM chat_actions a WHERE a.delivery_id = d.id - AND (a.company_id <> d.company_id OR a.endpoint_id <> d.endpoint_id OR (${completedAction('a')}) IS NOT TRUE))) IS NOT TRUE` }, + AND COALESCE(receipt.completed, true)) IS NOT TRUE` }, { reason: "chat_publications", query: `SELECT 1 FROM chat_publications p WHERE ( (p.state = 'published' AND p.published_at IS NOT NULL AND p.redacted_error IS NULL OR (p.state IN ('failed', 'cancelled') AND p.next_attempt_at IS NULL)) From 56acdfdffaa3d962dfc9a887e8942117e1681fcc Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 2 Nov 2026 14:12:41 -0600 Subject: [PATCH 04/66] Fix completed-history owner receipts and verified attachment cleanup Use truthful storage absence receipts for MCP deletion and preserve replay fences. Qualify retained chat controls and provider submissions through their actual owners, retain voice and registration obligations, and bound delivery receipt lookup. Co-Authored-By: Paperclip --- doc/idle-sleep-safety.md | 24 + .../migrations/0423_glossy_mariko_yashida.sql | 1 + ...{0418_snapshot.json => 0423_snapshot.json} | 328 ++++++- packages/db/src/migrations/meta/_journal.json | 7 + packages/db/src/schema/public_mcp.ts | 3 + .../chat-channels.integration.test.ts | 36 +- .../__tests__/helpers/chat-history-proof.ts | 11 + .../__tests__/idle-completed-history.test.ts | 837 +++++++++++++++++- .../__tests__/storage-local-provider.test.ts | 26 + .../voice-session-store.integration.test.ts | 43 +- server/src/modules/external-agents/index.ts | 2 + server/src/routes/issues.ts | 6 +- .../discord/idle-history-proof.ts | 25 + .../github/idle-history-proof.ts | 70 +- .../chat-channels/idle-history-proof.ts | 67 +- .../chat-channels/imessage-photon/events.ts | 41 +- .../imessage-photon/idle-history-proof.ts | 58 ++ .../imessage-photon/response-notices.ts | 27 + .../microsoft-teams/idle-history-proof.ts | 35 + .../chat-channels/slack/idle-history-proof.ts | 63 +- .../telegram/idle-history-proof.ts | 10 +- .../services/chat-question-form-idle-proof.ts | 54 ++ server/src/services/idle-sleep-safety.ts | 2 +- server/src/services/issues.ts | 8 +- .../native-session-executor.test.ts | 5 + .../src/services/public-mcp/file-transfers.ts | 22 +- .../services/public-mcp/idle-history-proof.ts | 13 +- .../src/services/public-mcp/upload-cleanup.ts | 38 + .../src/services/voice/idle-history-proof.ts | 71 ++ .../src/services/voice/voice-call-reports.ts | 3 +- server/src/storage/local-disk-provider.ts | 11 +- 31 files changed, 1849 insertions(+), 98 deletions(-) create mode 100644 packages/db/src/migrations/0423_glossy_mariko_yashida.sql rename packages/db/src/migrations/meta/{0418_snapshot.json => 0423_snapshot.json} (99%) create mode 100644 server/src/__tests__/helpers/chat-history-proof.ts create mode 100644 server/src/services/chat-channels/discord/idle-history-proof.ts create mode 100644 server/src/services/chat-channels/imessage-photon/idle-history-proof.ts create mode 100644 server/src/services/chat-channels/imessage-photon/response-notices.ts create mode 100644 server/src/services/chat-channels/microsoft-teams/idle-history-proof.ts create mode 100644 server/src/services/chat-question-form-idle-proof.ts create mode 100644 server/src/services/public-mcp/upload-cleanup.ts create mode 100644 server/src/services/voice/idle-history-proof.ts diff --git a/doc/idle-sleep-safety.md b/doc/idle-sleep-safety.md index 075ace1d59f..81e5c6c262f 100644 --- a/doc/idle-sleep-safety.md +++ b/doc/idle-sleep-safety.md @@ -118,6 +118,30 @@ Failed company transfers remain resumable. Completed transfers require their recorded manifest, exact part receipts and matching counts. Completed/cancelled transfers still block while their registered local spool contains cleanup work. +Chat control tokens can remain issued or become expired after another actor +resolves their canonical interaction. Their exact interaction, publication and +conversation receipts establish completion; a token status alone does not. +Provider submission receipts do not exempt pending source wakes or publications. +Photon attachment provenance remains immutable history after ordinary attachment +removal, but any surviving metadata must agree with its recorded identities and +hashes. This proof establishes completed registration, not storage cleanup. + +Public-MCP upload cleanup requires both the existing cleanup stamp and a new +verified-cleanup stamp. Ordinary attachment and issue removal write this proof +only after successful object deletion and an absence readback. Legacy cleanup +stamps are not backfilled: the existing bounded opportunistic sweep can verify +expired or orphaned uploads. Reissuing an uncommitted upload clears both stamps; +a committed/deleted request replay preserves them. An expired or cleaned upload +cannot accept a still-valid old transport ticket. + +Voice report refresh remains work for the owner's full 24-hour window after a +provider session ends, even when the report is available. Live session recovery, +unconfirmed inbound cleanup and ambiguous reply submission still block. A +completed provider submission does not prove spoken playback. Speko setup also +requires the existing successful whole-configuration receipt after its individual +tool and webhook writes. Discord command registration retains its own future +check; a completed command invocation does not retire that deadline. + Waiting agent conversations can be passive containers under both ordinary and scheduled inspection. The same narrow proof requires the exact conversation agent/company/user, waiting state, no checkout, active run, accounting, monitor diff --git a/packages/db/src/migrations/0423_glossy_mariko_yashida.sql b/packages/db/src/migrations/0423_glossy_mariko_yashida.sql new file mode 100644 index 00000000000..b90f13b2b22 --- /dev/null +++ b/packages/db/src/migrations/0423_glossy_mariko_yashida.sql @@ -0,0 +1 @@ +ALTER TABLE "mcp_attachment_uploads" ADD COLUMN IF NOT EXISTS "cleanup_verified_at" timestamp with time zone; \ No newline at end of file diff --git a/packages/db/src/migrations/meta/0418_snapshot.json b/packages/db/src/migrations/meta/0423_snapshot.json similarity index 99% rename from packages/db/src/migrations/meta/0418_snapshot.json rename to packages/db/src/migrations/meta/0423_snapshot.json index 23ede87cb5d..4014dff68d6 100644 --- a/packages/db/src/migrations/meta/0418_snapshot.json +++ b/packages/db/src/migrations/meta/0423_snapshot.json @@ -1,6 +1,6 @@ { - "id": "c777fa84-85a6-418f-bea1-0d60b56cf071", - "prevId": "c2c69a4a-8281-419c-9eeb-a7bcbc2beaae", + "id": "aef8cf7c-1939-4841-bb4a-b20203f10b69", + "prevId": "fe0b877c-d63e-46d9-8798-c7729fd13eb7", "version": "7", "dialect": "postgresql", "tables": { @@ -2233,6 +2233,74 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.agent_file_roots": { + "name": "agent_file_roots", + "schema": "", + "columns": { + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "generation": { + "name": "generation", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "entries": { + "name": "entries", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "agent_file_roots_company_id_agent_id_agents_company_id_id_fk": { + "name": "agent_file_roots_company_id_agent_id_agents_company_id_id_fk", + "tableFrom": "agent_file_roots", + "tableTo": "agents", + "columnsFrom": [ + "company_id", + "agent_id" + ], + "columnsTo": [ + "company_id", + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "agent_file_roots_company_id_agent_id_pk": { + "name": "agent_file_roots_company_id_agent_id_pk", + "columns": [ + "company_id", + "agent_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.agent_identity_keys": { "name": "agent_identity_keys", "schema": "", @@ -29705,6 +29773,241 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.execution_workspace_snapshots": { + "name": "execution_workspace_snapshots", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "execution_workspace_id": { + "name": "execution_workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_operation_id": { + "name": "source_operation_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_run_id": { + "name": "source_run_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "parent_snapshot_id": { + "name": "parent_snapshot_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "generation": { + "name": "generation", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "epoch": { + "name": "epoch", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "format": { + "name": "format", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'paperclip.remote-workspace-archive.v1'" + }, + "object_key": { + "name": "object_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sha256": { + "name": "sha256", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "byte_size": { + "name": "byte_size", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "key": { + "name": "key", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "manifest_digest": { + "name": "manifest_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "delete_after": { + "name": "delete_after", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "execution_workspace_snapshots_operation_uniq": { + "name": "execution_workspace_snapshots_operation_uniq", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "execution_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source_operation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "execution_workspace_snapshots_generation_uniq": { + "name": "execution_workspace_snapshots_generation_uniq", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "execution_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "generation", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "execution_workspace_snapshots_workspace_idx": { + "name": "execution_workspace_snapshots_workspace_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "execution_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "execution_workspace_snapshots_cleanup_idx": { + "name": "execution_workspace_snapshots_cleanup_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "delete_after", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.execution_workspaces": { "name": "execution_workspaces", "schema": "", @@ -37105,6 +37408,15 @@ "client_request_id" ] }, + "issue_comments_agent_client_request_uq": { + "name": "issue_comments_agent_client_request_uq", + "nullsNotDistinct": false, + "columns": [ + "issue_id", + "author_agent_id", + "client_request_id" + ] + }, "issue_comments_company_id_uq": { "name": "issue_comments_company_id_uq", "nullsNotDistinct": false, @@ -52770,6 +53082,12 @@ "primaryKey": false, "notNull": false }, + "next_eval_reason": { + "name": "next_eval_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, "next_eval_at": { "name": "next_eval_at", "type": "timestamp with time zone", @@ -62124,6 +62442,12 @@ "primaryKey": false, "notNull": false }, + "cleanup_verified_at": { + "name": "cleanup_verified_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, "created_at": { "name": "created_at", "type": "timestamp with time zone", diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index b3bc9221371..3d83909fe61 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -2948,6 +2948,13 @@ "when": 1793503729899, "tag": "0422_eminent_darwin", "breakpoints": true + }, + { + "idx": 423, + "version": "7", + "when": 1793645545010, + "tag": "0423_glossy_mariko_yashida", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/db/src/schema/public_mcp.ts b/packages/db/src/schema/public_mcp.ts index ec42c436739..7011a37aa0c 100644 --- a/packages/db/src/schema/public_mcp.ts +++ b/packages/db/src/schema/public_mcp.ts @@ -188,6 +188,9 @@ export const mcpAttachmentUploads = pgTable("mcp_attachment_uploads", { storageProvider: text("storage_provider").notNull(), expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(), cleanedAt: timestamp("cleaned_at", { withTimezone: true }), + // Written only after storage deletion and an authoritative absence readback. + // Legacy cleanedAt can predate truthful local filesystem error handling. + cleanupVerifiedAt: timestamp("cleanup_verified_at", { withTimezone: true }), createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), }, (t) => [ uniqueIndex("mcp_attachment_upload_request_uq").on(t.companyId, t.userId, t.requestId), diff --git a/server/src/__tests__/chat-channels.integration.test.ts b/server/src/__tests__/chat-channels.integration.test.ts index 08c61039a80..40783fae2cd 100644 --- a/server/src/__tests__/chat-channels.integration.test.ts +++ b/server/src/__tests__/chat-channels.integration.test.ts @@ -1,3 +1,5 @@ +import { chatHistoryActionBlocks } from "./helpers/chat-history-proof.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { resolveCommittedNativeRecoveryNotices, pendingNativeRecoveryNotices } from "../services/native-runtime/native-recovery-notices.js"; import { issueRecoveryActionService } from "../services/issue-recovery-actions.js"; import { projectCommittedRun } from "../services/native-runtime/native-run-projection.js"; @@ -284,7 +286,8 @@ import { startEmbeddedPostgresTestDatabase, } from "./helpers/embedded-postgres.js"; -const externalTestDatabaseUrl = process.env.PAPERCLIP_TEST_DATABASE_URL; +const historyTestDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_HISTORY_TEST_DATABASE_URL"); +const externalTestDatabaseUrl = historyTestDatabaseUrl ?? process.env.PAPERCLIP_TEST_DATABASE_URL; const embeddedPostgresSupport = externalTestDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); @@ -1122,6 +1125,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { // Do not expire or time out its connections on those virtual clock jumps. db = createDb(externalTestDatabaseUrl ?? tempDb!.connectionString, { idleTimeoutSeconds: 0, connectTimeoutSeconds: 0, maxLifetimeSeconds: 0, + ...(historyTestDatabaseUrl ? { maxConnections: 4 } : {}), }); }, 90_000); @@ -3006,6 +3010,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { const [review] = await readChatGitHubReviewsByEndpointId(f.endpoint.id); expect(review.assessment?.findings[0].category).toBe(categories[0]); expect((await f.action()).status).toBe("processed"); + expect(await chatHistoryActionBlocks(db, (await f.action()).id)).toBe(false); expect(f.responseComments.size).toBe(1); }); it("publishes one score heading when a newly configured bot submits a formatted summary", async () => { @@ -3069,6 +3074,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { const [response] = await db.select().from(chatActions).where(and(eq(chatActions.deliveryId, delivery.id), eq(chatActions.kind, "github_response_comment"))); expect(response).toMatchObject({ status: "cancelled", result: { final: true, code: "passive_response_wait" } }); + expect(await chatHistoryActionBlocks(db, response.id)).toBe(false); const writes = f.responseWrites.length; const updatedAt = response.updatedAt; await responses.processPending(); @@ -3314,6 +3320,9 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { const body = [...f.responseComments.values()][0].body; expect(body).not.toMatch(/Review queued|Working on this/); expect(body).toContain("paperclip-response-state:final"); + const responses = await db.select().from(chatActions).where(and(eq(chatActions.endpointId, f.endpoint.id), eq(chatActions.kind, "github_response_comment"))); + expect(responses.length).toBeGreaterThan(0); + for (const receipt of responses) expect(await chatHistoryActionBlocks(db, receipt.id)).toBe(false); }); it("retires a legacy coalesced GitHub placeholder before replacing its receipt", async () => { const f = await publicationRetryFixture(); @@ -3425,6 +3434,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { expect([...f.responseComments.values()].some(comment => comment.body.includes("Newer progress"))).toBe(true); const [settled] = await db.select().from(chatActions).where(eq(chatActions.id, first.id)); expect(settled.result?.final).toBe(false); + expect(await chatHistoryActionBlocks(db, settled.id)).toBe(true); }); it.each([false, true])("recovers an approved queued GitHub review without another approval (revoked: %s)", async revoked => { const f = await publicationRetryFixture(); @@ -3460,6 +3470,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { await db.update(chatActions).set({ result: { ...action.result, retryAt: new Date(0).toISOString() } }).where(eq(chatActions.id, action.id)); await f.service.processPending(); expect((await f.action()).status).toBe(revoked ? "cancelled" : "processed"); + expect(await chatHistoryActionBlocks(db, (await f.action()).id)).toBe(false); if (revoked) await expect(gateway.executeTool({ sessionToken: session.token, tool: tool.name, parameters: f.assessment })).rejects.toThrow(); else expect(await gateway.executeTool({ sessionToken: session.token, tool: tool.name, parameters: f.assessment })) .toMatchObject({ status: "replayed", result: { data: { status: "processed", score: 3 } } }); @@ -4830,6 +4841,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { expect(f.responseWrites.filter(write => write.method === "POST")).toHaveLength(2); const [deleted] = await db.select().from(chatActions).where(and(eq(chatActions.endpointId, f.endpoint.id), eq(chatActions.kind, "github_response_comment"), sql`${chatActions.result}->>'id' = ${String(firstId)}`)); expect(deleted).toMatchObject({ status: "cancelled", result: { final: true, code: "response_deleted" } }); + expect(await chatHistoryActionBlocks(db, deleted.id)).toBe(false); await githubResponseCommentService(db, f.providerFetch).processPending(); expect(f.responseComments.size).toBe(1); await expect(githubRunReplyState(db, { ...f.session, endpointId: f.endpoint.id })).resolves.toBe("confirmed"); @@ -9767,6 +9779,9 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { expect(f.provider.mock.calls.filter(([url]) => String(url).endsWith("apps.manifest.create"))).toHaveLength(0); expect((await f.service.get(f.endpoint.id)).status).not.toBe("revoked"); expect((await f.service.get(f.endpoint.id)).setup.slackRegistration).toMatchObject({ status: "configured", appId: f.appId }); + const [reconnectReceipt] = await db.select().from(chatActions).where(and(eq(chatActions.endpointId, f.endpoint.id), eq(chatActions.kind, "slack_managed_reconnect"))); + expect(reconnectReceipt).toMatchObject({ status: "processed", result: null }); + expect(await chatHistoryActionBlocks(db, reconnectReceipt.id)).toBe(false); await f.service.slackRegistration.managed.provision(f.endpoint.id, actor, reconnectInput); await f.provision(); expect(f.provider.mock.calls.filter(([url]) => String(url).endsWith("apps.managedInstall"))).toHaveLength(1); @@ -17019,12 +17034,13 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { context.endpoint.id, ); expect(context.posts).toEqual([]); - if (change === "edited comment") + if (change === "edited comment") { expect(await context.receipt()).toMatchObject({ status: "cancelled", result: { attempts: 0, code: "source_changed_or_unavailable" }, }); - else expect(await context.receipt()).toBeUndefined(); + expect(await chatHistoryActionBlocks(db, (await context.receipt())!.id)).toBe(false); + } else expect(await context.receipt()).toBeUndefined(); expect(context.wakeup).not.toHaveBeenCalled(); } finally { await retirePublicationFixture(context.service, context.endpoint.id); @@ -17705,6 +17721,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { result: { code: "github_webhook_ingress_runtime_superseded" }, }); expect((await context.ingress())!.payload).not.toHaveProperty("body"); + expect(await chatHistoryActionBlocks(db, (await context.ingress())!.id)).toBe(false); expect(sdkDispatch).not.toHaveBeenCalled(); expect( await db @@ -17756,6 +17773,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { result: { code: "github_webhook_recovery_source_changed" }, }); expect(cancelled!.payload).not.toHaveProperty("body"); + expect(await chatHistoryActionBlocks(db, cancelled!.id)).toBe(false); expect(cancelled!.result?.retryable).not.toBe(true); expect(context.requests).toEqual([]); expect( @@ -43530,6 +43548,11 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { expect(original?.state).toBe("published"); expect(original?.payload.card?.actions).toEqual(expect.arrayContaining([expect.objectContaining({ type: "callback" })])); if (kind === "connection_intent") expect(original?.payload.card?.actions).toContainEqual({ type: "link", label: "Review connection", url: expect.any(String) }); + const controls = (await db.select().from(chatActions).where(and(eq(chatActions.endpointId, endpoint.id), + eq(sql`${chatActions.payload}->>'interactionId'`, interaction.id)))) + .filter(row => ["confirmation_response", "question_form_open", "question_form_submit", "question_answer"].includes(row.kind)); + expect(controls.length).toBeGreaterThan(0); + for (const control of controls) expect(await chatHistoryActionBlocks(db, control.id)).toBe(true); if (kind === "connection_intent") await interactions.resolveConnectionIntent(issue, interaction.id, { version: 1, outcome: "declined" }, { userId: "owner-user" }); else if (kind === "request_item_verdicts") await interactions.submitItemVerdicts(issue, interaction.id, { verdicts: [{ id: "docs", verdict: "approve" }] }, { userId: "owner-user" }); else await interactions.rejectInteraction(issue, interaction.id, {}, { userId: "owner-user" }); @@ -43539,6 +43562,11 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { expect(terminal?.state).toBe("published"); expect(terminal?.providerMessageId).toBe(original.providerMessageId); expect(terminal?.payload.card?.actions).toBeUndefined(); + for (const control of controls) expect(await chatHistoryActionBlocks(db, control.id)).toBe(false); + const control = controls[0]!; + await db.update(chatActions).set({ payload: { ...control.payload, interactionId: randomUUID() } }).where(eq(chatActions.id, control.id)); + expect(await chatHistoryActionBlocks(db, control.id)).toBe(true); + await db.update(chatActions).set({ payload: control.payload }).where(eq(chatActions.id, control.id)); } finally { await service.shutdown(); if (previousPublicUrl === undefined) delete process.env.PAPERCLIP_PUBLIC_URL; @@ -80927,6 +80955,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { expect(context.consentCards).toHaveLength(1); expect(context.fileCards).toEqual([]); expect(context.uploadRequest).not.toHaveBeenCalled(); + expect(await chatHistoryActionBlocks(db, intent!.id)).toBe(true); const before = await context.service.getPublicationBatchStatus( context.endpoint.id, context.conversation.id, @@ -80955,6 +80984,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { body: context.bytes, }); expect(context.fileCards).toHaveLength(1); + expect(await chatHistoryActionBlocks(db, intent!.id)).toBe(false); const status = await context.service.getPublicationBatchStatus( context.endpoint.id, context.conversation.id, diff --git a/server/src/__tests__/helpers/chat-history-proof.ts b/server/src/__tests__/helpers/chat-history-proof.ts new file mode 100644 index 00000000000..60dcef89966 --- /dev/null +++ b/server/src/__tests__/helpers/chat-history-proof.ts @@ -0,0 +1,11 @@ +import { sql } from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; +import { idleChatHistoryChecks } from "../../services/chat-channels/idle-history-proof.js"; + +/** Scope the actual shared predicate to a real producer's receipt. Other pending + * fixture work remains independently observable through its owning gates. */ +export async function chatHistoryActionBlocks(db: Db, actionId: string): Promise { + const check = idleChatHistoryChecks.find(check => check.reason === "chat_actions")!; + const [row] = await db.execute<{ blocked: boolean }>(sql`SELECT EXISTS (${sql.raw(check.query)} AND a.id = ${actionId}::uuid) AS blocked`); + return row.blocked; +} diff --git a/server/src/__tests__/idle-completed-history.test.ts b/server/src/__tests__/idle-completed-history.test.ts index c4fe2b56843..4e86383c51b 100644 --- a/server/src/__tests__/idle-completed-history.test.ts +++ b/server/src/__tests__/idle-completed-history.test.ts @@ -1,5 +1,5 @@ -import { createHash, randomBytes, randomUUID } from "node:crypto"; -import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { createHash, generateKeyPairSync, randomBytes, randomUUID } from "node:crypto"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import net from "node:net"; @@ -33,6 +33,11 @@ const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPos if (elapsed > 1000) console.info(`History predicate ${reason}: ${Math.round(elapsed)}ms`); return row.blocked; }; + const blockedActionKind = async (kind: string) => { + const query = completedHistoryChecks.find(check => check.reason === "chat_actions")!.query; + const [row] = await db.execute<{ blocked: boolean }>(sql`SELECT EXISTS (${sql.raw(query)} AND a.kind = ${kind}) AS blocked`); + return row.blocked; + }; beforeAll(async () => { if (!externalDatabaseUrl) database = await startEmbeddedPostgresTestDatabase("idle-history-"); db = tables.createDb(externalDatabaseUrl ?? database!.connectionString, { maxConnections: 4 }); @@ -107,10 +112,28 @@ const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPos await db.insert(tables.mcpOauthTokens).values({ grantId: p.grantId, kind: "access", tokenHash: hashMcpSecret(token), expiresAt: new Date(Date.now() + 600_000) }); const directory = await mkdtemp(path.join(tmpdir(), "history-mcp-storage-")); - const router = Router(); - router.use(issueRoutes(db, createStorageService(createLocalDiskStorageProvider(directory)))); + const router = Router(), storage = createStorageService(createLocalDiskStorageProvider(directory)); + router.use(issueRoutes(db, storage)); const oauth = createPublicMcpOAuth(db, { origin, resource }), api = createMcpApiDispatch(router); - return { ...p, token, oauth, api, storage: createStorageService(createLocalDiskStorageProvider(directory)), cleanup: () => rm(directory, { recursive: true, force: true }) }; + return { ...p, token, oauth, api, storage, cleanup: () => rm(directory, { recursive: true, force: true }) }; + } + + async function slackFixture() { + const f = await seed(), c = await connection(f), p = await mcp(f); + await db.update(tables.toolConnections).set({ enabled: true, status: "active" }).where(eq(tables.toolConnections.id, c.connectionId)); + await db.insert(tables.companyMemberships).values({ companyId: f.companyId, principalType: "user", principalId: p.userId, + status: "active", membershipRole: "owner" }); + const [endpoint] = await db.insert(tables.chatEndpoints).values({ companyId: f.companyId, connectionId: c.connectionId, + assignedAgentId: f.agentId, publicId: randomUUID(), provider: "slack", status: "active", providerAccountId: "TWORKSPACE", + botExternalId: "UBOT", allowDirectMessages: true, publicationMode: "automatic" }).returning(); + const [conversation] = await db.insert(tables.chatConversations).values({ companyId: f.companyId, endpointId: endpoint.id, + issueId: f.issueId, externalConversationId: "DCHANNEL", externalThreadId: "slack:DCHANNEL:123.456", externalLabel: "Personal", + state: "active", isDirectMessage: true }).returning(); + const [principal] = await db.insert(tables.chatExternalPrincipals).values({ companyId: f.companyId, provider: "slack", + providerAccountId: "TWORKSPACE", externalId: "UREQUESTER", kind: "user", isBot: false }).returning(); + const authority = { endpoint, issueId: f.issueId, conversation, userId: p.userId, slackUserId: "UREQUESTER", principalId: principal.id, + deliveryId: null, identityContextId: null, revision: "fixture-revision", workMode: "standard", botToken: "fixture-bot-token" }; + return { ...f, ...p, ...c, endpoint, conversation, principal, authority }; } it("checks all 32 families on the real migrated schema and keeps an empty instance eligible", async () => { @@ -166,13 +189,117 @@ const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPos const result = await request(app).put(url.pathname + url.search).set("Content-Type", "text/plain").send(bytes).timeout({ response: 5000, deadline: 10_000 }); expect(result.status).toBe(200); expect(await blocked("mcp_attachment_uploads")).toBe(false); - const abandoned = await transfers.getUploadUrl(principal, { ...args, requestId: randomUUID() }); + await db.update(tables.mcpAttachmentUploads).set({ cleanedAt: now, cleanupVerifiedAt: now }).where(eq(tables.mcpAttachmentUploads.id, String(upload.uploadId))); + expect(await blocked("mcp_attachment_uploads")).toBe(true); + await db.update(tables.mcpAttachmentUploads).set({ cleanedAt: null, cleanupVerifiedAt: null }).where(eq(tables.mcpAttachmentUploads.id, String(upload.uploadId))); + const abandonedArgs = { ...args, requestId: randomUUID() }; + const abandoned = await transfers.getUploadUrl(principal, abandonedArgs); + expect(await blocked("mcp_attachment_uploads")).toBe(true); + const [old] = await db.select().from(tables.mcpAttachmentUploads).where(eq(tables.mcpAttachmentUploads.id, String(abandoned.uploadId))); + await p.storage.putFile({ companyId: f.companyId, namespace: "mcp-transfers", objectKey: old.objectKey, + originalFilename: args.filename, contentType: args.contentType, body: bytes }); + // Model the old owner's optimistic cleanup: its stamp exists while bytes + // survive. It must stay blocked until actual deletion and readback. + await db.update(tables.mcpAttachmentUploads).set({ expiresAt: new Date(0), cleanedAt: new Date() }).where(eq(tables.mcpAttachmentUploads.id, old.id)); + expect(await p.storage.headObject(f.companyId, old.objectKey)).toMatchObject({ exists: true }); expect(await blocked("mcp_attachment_uploads")).toBe(true); - await db.update(tables.mcpAttachmentUploads).set({ expiresAt: new Date(0) }).where(eq(tables.mcpAttachmentUploads.id, String(abandoned.uploadId))); await transfers.getDownloadUrl(principal, result.body.attachment.id); + expect(await p.storage.headObject(f.companyId, old.objectKey)).toEqual({ exists: false }); expect(await blocked("mcp_attachment_uploads")).toBe(false); - await db.update(tables.mcpAttachmentUploads).set({ attachmentId: null }).where(eq(tables.mcpAttachmentUploads.id, String(upload.uploadId))); + const verified = (await db.select().from(tables.mcpAttachmentUploads).where(eq(tables.mcpAttachmentUploads.id, old.id)))[0]; + const staleUrl = new URL(String(abandoned.url)); + expect((await request(app).put(staleUrl.pathname + staleUrl.search).set("Content-Type", "text/plain").send(bytes) + .timeout({ response: 5000, deadline: 10_000 })).status).toBe(403); + expect(await p.storage.headObject(f.companyId, old.objectKey)).toEqual({ exists: false }); + expect((await db.select().from(tables.mcpAttachmentUploads).where(eq(tables.mcpAttachmentUploads.id, old.id)))[0]).toEqual(verified); + const reissued = await transfers.getUploadUrl(principal, abandonedArgs); + expect((await db.select().from(tables.mcpAttachmentUploads).where(eq(tables.mcpAttachmentUploads.id, old.id)))[0]) + .toMatchObject({ cleanedAt: null, cleanupVerifiedAt: null }); expect(await blocked("mcp_attachment_uploads")).toBe(true); + const secondUrl = new URL(String(reissued.url)); + const second = await request(app).put(secondUrl.pathname + secondUrl.search).set("Content-Type", "text/plain").send(bytes) + .timeout({ response: 5000, deadline: 10_000 }); + expect(second.status).toBe(200); + for (const attachment of [result.body.attachment, second.body.attachment]) { + expect(await p.api(principal, "DELETE", `/attachments/${attachment.id}`)).toEqual({ ok: true }); + } + const [removed] = await db.select().from(tables.mcpAttachmentUploads).where(eq(tables.mcpAttachmentUploads.id, String(upload.uploadId))); + expect(removed).toMatchObject({ attachmentId: result.body.attachment.id, cleanedAt: expect.any(Date), cleanupVerifiedAt: expect.any(Date) }); + expect(await p.storage.headObject(f.companyId, removed.objectKey)).toEqual({ exists: false }); + expect(await blocked("mcp_attachment_uploads")).toBe(false); + await expect(transfers.getUploadUrl(principal, args)).rejects.toThrow(); + expect((await db.select().from(tables.mcpAttachmentUploads).where(eq(tables.mcpAttachmentUploads.id, removed.id)))[0]).toEqual(removed); + expect(await blocked("mcp_attachment_uploads")).toBe(false); + await db.update(tables.mcpAttachmentUploads).set({ cleanupVerifiedAt: null }).where(eq(tables.mcpAttachmentUploads.id, removed.id)); + expect(await blocked("mcp_attachment_uploads")).toBe(true); + } finally { await p.cleanup(); } + }, 60_000); + + it.each(["delete_failed", "object_survived", "head_failed"] as const)("retains MCP cleanup uncertainty after actual attachment removal: %s", async failure => { + const { createPublicMcpTransfers } = await import("../services/public-mcp/file-transfers.js"); + const { default: express } = await import("express"); + const { default: request } = await import("supertest"); + const f = await seed(), p = await connectedMcp(f); + try { + const principal = await p.oauth.authenticate(p.token), bytes = Buffer.from("retained uncertain attachment"); + const transfers = createPublicMcpTransfers(db, p.oauth, p.api, p.storage); + const upload = await transfers.getUploadUrl(principal, { companyId: f.companyId, taskId: f.issueId, requestId: randomUUID(), + filename: "uncertain.txt", contentType: "text/plain", byteSize: bytes.length, sha256: createHash("sha256").update(bytes).digest("hex") }); + const app = express(); app.use(transfers.router); + const url = new URL(String(upload.url)); + const result = await request(app).put(url.pathname + url.search).set("Content-Type", "text/plain").send(bytes) + .timeout({ response: 5000, deadline: 10_000 }); + expect(result.status).toBe(200); + const [before] = await db.select().from(tables.mcpAttachmentUploads).where(eq(tables.mcpAttachmentUploads.id, String(upload.uploadId))); + const originalDelete = p.storage.deleteObject, originalHead = p.storage.headObject; + if (failure === "delete_failed") p.storage.deleteObject = async () => { throw new Error("controlled storage removal failure"); }; + if (failure === "object_survived") p.storage.deleteObject = async () => {}; + if (failure === "head_failed") p.storage.headObject = async () => { throw new Error("controlled storage readback failure"); }; + try { expect(await p.api(principal, "DELETE", `/attachments/${result.body.attachment.id}`)).toEqual({ ok: true }); } + finally { p.storage.deleteObject = originalDelete; p.storage.headObject = originalHead; } + expect(await db.select().from(tables.issueAttachments).where(eq(tables.issueAttachments.id, result.body.attachment.id))).toHaveLength(0); + expect((await db.select().from(tables.mcpAttachmentUploads).where(eq(tables.mcpAttachmentUploads.id, before.id)))[0]) + .toMatchObject({ attachmentId: before.attachmentId, cleanedAt: null, cleanupVerifiedAt: null }); + expect((await p.storage.headObject(f.companyId, before.objectKey)).exists).toBe(failure !== "head_failed"); + expect(await blocked("mcp_attachment_uploads")).toBe(true); + // The existing bounded opportunistic owner can later retire this orphan + // only by a real successful delete and readback, preserving its identity. + await db.update(tables.mcpAttachmentUploads).set({ expiresAt: new Date(0) }).where(eq(tables.mcpAttachmentUploads.id, before.id)); + const pending = await transfers.getUploadUrl(principal, { companyId: f.companyId, taskId: f.issueId, requestId: randomUUID(), + filename: "new.txt", contentType: "text/plain", byteSize: 1, sha256: createHash("sha256").update("x").digest("hex") }); + expect((await db.select().from(tables.mcpAttachmentUploads).where(eq(tables.mcpAttachmentUploads.id, before.id)))[0]) + .toMatchObject({ attachmentId: before.attachmentId, cleanupVerifiedAt: expect.any(Date) }); + expect(await p.storage.headObject(f.companyId, before.objectKey)).toEqual({ exists: false }); + expect(await blocked("mcp_attachment_uploads")).toBe(true); // newly allocated upload remains pending + expect(pending.uploadId).toBeTruthy(); + } finally { await p.cleanup(); } + }, 60_000); + + it.each([false, true])("preserves verified MCP cleanup after ordinary issue deletion, storageFailure=%s", async storageFailure => { + const { createPublicMcpTransfers } = await import("../services/public-mcp/file-transfers.js"); + const { default: express } = await import("express"); + const { default: request } = await import("supertest"); + const f = await seed(), p = await connectedMcp(f); + try { + const principal = await p.oauth.authenticate(p.token), bytes = Buffer.from("removed with parent"); + const transfers = createPublicMcpTransfers(db, p.oauth, p.api, p.storage); + const upload = await transfers.getUploadUrl(principal, { companyId: f.companyId, taskId: f.issueId, requestId: randomUUID(), + filename: "parent.txt", contentType: "text/plain", byteSize: bytes.length, sha256: createHash("sha256").update(bytes).digest("hex") }); + const app = express(); app.use(transfers.router); + const url = new URL(String(upload.url)); + const result = await request(app).put(url.pathname + url.search).set("Content-Type", "text/plain").send(bytes) + .timeout({ response: 5000, deadline: 10_000 }); + expect(result.status).toBe(200); + const originalDelete = p.storage.deleteObject; + if (storageFailure) p.storage.deleteObject = async () => { throw new Error("controlled parent storage failure"); }; + try { await p.api(principal, "DELETE", `/issues/${f.issueId}`); } + finally { p.storage.deleteObject = originalDelete; } + expect(await db.select().from(tables.issues).where(eq(tables.issues.id, f.issueId))).toHaveLength(0); + const [removed] = await db.select().from(tables.mcpAttachmentUploads).where(eq(tables.mcpAttachmentUploads.id, String(upload.uploadId))); + expect(removed.attachmentId).toBe(result.body.attachment.id); + expect(removed.cleanupVerifiedAt !== null).toBe(!storageFailure); + expect((await p.storage.headObject(f.companyId, removed.objectKey)).exists).toBe(storageFailure); + expect(await blocked("mcp_attachment_uploads")).toBe(storageFailure); } finally { await p.cleanup(); } }, 60_000); @@ -205,7 +332,7 @@ const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPos operation: "paperclip_create_task", requestId: randomUUID(), argumentsHash: "a", status: "completed", result: { id: f.issueId } }).returning(); const [upload] = await db.insert(tables.mcpAttachmentUploads).values({ ...p, companyId: f.companyId, taskId: f.issueId, requestId: randomUUID(), argumentsHash: "a", originalFilename: "history.txt", contentType: "text/plain", byteSize: 1, - sha256: "a".repeat(64), objectKey: "history/abandoned", storageProvider: "local_disk", expiresAt: now, cleanedAt: now }).returning(); + sha256: "a".repeat(64), objectKey: "history/abandoned", storageProvider: "local_disk", expiresAt: now, cleanedAt: now, cleanupVerifiedAt: now }).returning(); expect(await blocked("mcp_mutation_receipts")).toBe(false); expect(await blocked("mcp_attachment_uploads")).toBe(false); expect(await read()).toEqual({ version: 1, backgroundWork: "none" }); @@ -281,6 +408,698 @@ const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPos expect(await blocked("mcp_event_deliveries")).toBe(true); }); + it.each([false, true])("requires actual Speko configure completion after tool receipts, defaultsFailure=%s", async defaultsFailure => { + const { chatChannelService } = await import("../services/chat-channels.js"); + const { instanceSettingsService } = await import("../services/instance-settings.js"); + const f = await seed(), p = await mcp(f), oldKey = process.env.PAPERCLIP_SECRETS_MASTER_KEY; + process.env.PAPERCLIP_SECRETS_MASTER_KEY = "a".repeat(64); + await db.insert(tables.companyMemberships).values({ companyId: f.companyId, principalType: "user", principalId: p.userId, + membershipRole: "owner", status: "active" }); + await instanceSettingsService(db).updateExperimental({ enableChatConnectors: true }); + await db.insert(tables.principalPermissionGrants).values({ companyId: f.companyId, principalType: "user", principalId: p.userId, + permissionKey: "agents:configure" }); + const tools: Record[] = [], hooks: Record[] = []; + let failDefaults = defaultsFailure; + const chat = chatChannelService(db, { heartbeat: { wakeup: async () => { throw new Error("No call expected"); } }, + publicBaseUrl: "https://history.example", webhookPublicBaseUrl: "https://history.example", fetch: async (input, init) => { + const pathname = new URL(String(input)).pathname, method = init?.method ?? "GET"; + if (pathname === "/v1/agents/agent_test") { + if (method === "PATCH" && failDefaults) return new Response("Defaults failed", { status: 503 }); + return Response.json({ id: "agent_test", organizationId: "org_test", name: "History voice" }); + } + if (pathname.startsWith("/v1/agents/agent_test/tools")) { + if (method === "GET") return Response.json(tools); + const tool = { ...JSON.parse(String(init?.body)), id: pathname.split("/").length > 5 ? pathname.split("/").at(-1)! : `tool_${tools.length}` }; + const index = tools.findIndex(old => old.id === tool.id); if (index < 0) tools.push(tool); else tools[index] = tool; + return Response.json({ id: tool.id }); + } + if (pathname.startsWith("/v1/webhooks")) { + if (method === "GET") return Response.json({ data: hooks }); + const hook = { ...JSON.parse(String(init?.body)), id: "hook_test", filterTags: {} }; hooks.splice(0, hooks.length, hook); + return Response.json({ id: hook.id }); + } + throw new Error(`Unexpected Speko provider boundary ${method} ${pathname}`); + } }); + try { + const endpoint = await chat.create(f.companyId, { provider: "speko", assignedAgentId: f.agentId }, p.userId); + let configureAttempts = 0; + const configure = () => chat.configure(endpoint.id, { action: configureAttempts++ === 0 ? "configure" : "reconnect", + credentials: { apiKey: "fixture-api-key", agentId: "agent_test", signingSecret: `whsec_${Buffer.alloc(32, 7).toString("base64")}` } }, p.userId); + if (defaultsFailure) { + await expect(configure()).rejects.toMatchObject({ status: 422 }); + const partial = (await db.select().from(tables.chatActions)).filter(a => a.endpointId === endpoint.id); + expect(partial).toHaveLength(4); + expect(partial.every(a => a.status === "completed")).toBe(true); + for (const receipt of partial) expect(await blockedActionKind(receipt.kind)).toBe(true); + failDefaults = false; + } + await configure(); + const receipts = (await db.select().from(tables.chatActions)).filter(a => a.endpointId === endpoint.id); + expect(receipts).toHaveLength(4); + expect(await blockedActionKind("speko_tool_setup")).toBe(false); + expect(await blockedActionKind("speko_lifecycle_setup")).toBe(false); + const [activity] = await db.select().from(tables.activityLog).where(eq(tables.activityLog.action, + defaultsFailure ? "chat_endpoint.reconnected" : "chat_endpoint.credentials_verified")); + expect(activity.details).toMatchObject({ endpointId: endpoint.id, provider: "speko" }); + } finally { + await chat.shutdown(); + if (oldKey === undefined) delete process.env.PAPERCLIP_SECRETS_MASTER_KEY; else process.env.PAPERCLIP_SECRETS_MASTER_KEY = oldKey; + } + }, 60_000); + + it("retires actual GitHub omission navigation only after its exact native response publication settles", async () => { + const { createGitHubPublicationPolicy } = await import("../services/chat-channels/github/publication-policy.js"); + const { ensureNativeCompletionContract } = await import("../services/native-runtime/completion-contracts.js"); + const { chatChannelService } = await import("../services/chat-channels.js"); + const f = await slackFixture(); + const [endpoint] = await db.update(tables.chatEndpoints).set({ provider: "github" }).where(eq(tables.chatEndpoints.id, f.endpoint.id)).returning(); + const contract = await ensureNativeCompletionContract({ db, companyId: f.companyId, issue: { id: f.issueId, title: "Omitted source", description: null }, actorId: "history" }); + const [comment] = await db.insert(tables.issueComments).values({ companyId: f.companyId, issueId: f.issueId, + authorAgentId: f.agentId, createdByRunId: f.runId, body: "Source unavailable; see the task" }).returning(); + await db.update(tables.heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: f.issueId, completionContractId: contract.row.id, + resultJson: { presentationDecision: { commentId: comment.id } } }).where(eq(tables.heartbeatRuns.id, f.runId)); + const [result] = await db.insert(tables.nativeRunResults).values({ companyId: f.companyId, issueId: f.issueId, runId: f.runId, + completionContractId: contract.row.id, schemaStatus: "accepted", serverFingerprint: randomUUID(), canonicalSha256: "a".repeat(64), resultJson: {} }).returning(); + await db.insert(tables.nativeRunFinalizations).values({ companyId: f.companyId, issueId: f.issueId, runId: f.runId, + phase: "committed", resultId: result.id }); + const [publication] = await db.insert(tables.chatPublications).values({ companyId: f.companyId, endpointId: endpoint.id, + conversationId: f.conversation.id, issueId: f.issueId, commentId: comment.id, idempotencyKey: randomUUID(), payload: { text: comment.body } }).returning(); + const unused = async (): Promise => { throw new Error("Unexpected GitHub publication boundary"); }; + const policy = createGitHubPublicationPolicy(db, { isExplicitOperatorPublication: () => false, getTaskBaseUrl: () => "https://history.example", + // Exact current native-input authority is the named consumer boundary; + // the actual policy validates canonical owner rows and writes the marker. + failedChatRetrySource: async () => ({ endpointId: endpoint.id, conversationId: f.conversation.id, + sources: [{ omissionReasons: { download_unavailable: 1 } }] }) as never, + isExternalActionAuthorizationChange: () => false, FailedChatRetryPublicationReadError: class extends Error {}, endpointRecord: unused, + runIdFromMilestonePublication: () => null, fetchImpl: unused, stageTerminalReceiptReactionRemovals: unused, + runtimeContextForRecord: unused, scheduleMessageProcessing: unused, processReceiptReaction: unused }); + const text = await db.transaction(tx => policy.githubCurrentOmissionPreparedText(tx, publication, endpoint)); + expect(text).toContain("Open this Paperclip task"); + const [receipt] = await db.select().from(tables.chatActions).where(eq(tables.chatActions.kind, "github_omission_navigation")); + expect(receipt).toMatchObject({ status: "processed", result: null }); + expect(await blockedActionKind("github_omission_navigation")).toBe(true); + await db.update(tables.chatPublications).set({ payload: { ...publication.payload, text: String(text) }, state: "delivery_unknown" }) + .where(eq(tables.chatPublications.id, publication.id)); + const chat = chatChannelService(db, { heartbeat: { wakeup: unused }, fetch: unused }); + try { await chat.resolvePublication(endpoint.id, publication.id, "mark_delivered", f.userId); } + finally { await chat.shutdown(); } + expect(await blockedActionKind("github_omission_navigation")).toBe(false); + await db.update(tables.chatActions).set({ payload: { ...receipt.payload, resultId: randomUUID() } }).where(eq(tables.chatActions.id, receipt.id)); + expect(await blockedActionKind("github_omission_navigation")).toBe(true); + }); + + it("retires an actual Discord status-command receipt while preserving registration deadline work", async () => { + const { reconcileStoredDiscordCommandRegistration } = await import("../services/chat-discord-command-registration-store.js"); + const { createDiscordCommands } = await import("../services/chat-channels/discord/commands.js"); + const { chatCredentialMutationLease } = await import("../services/chat-credential-mutation-lease.js"); + const f = await slackFixture(), applicationId = String(1_400_000_000_000_000_000n + BigInt(Date.now()) + BigInt(randomBytes(2).readUInt16BE())), guildId = "1457808928258658549", channelId = "333333333333333333"; + const [endpoint] = await db.update(tables.chatEndpoints).set({ provider: "discord", botExternalId: applicationId, + providerAccountId: guildId, capabilities: { slashCommands: true } }).where(eq(tables.chatEndpoints.id, f.endpoint.id)).returning(); + const [principal] = await db.update(tables.chatExternalPrincipals).set({ provider: "discord", externalId: "444444444444444444", + providerAccountId: guildId }).where(eq(tables.chatExternalPrincipals.id, f.principal.id)).returning(); + const context = { generation: 1, credentialFingerprint: "c".repeat(64), localEpoch: 0, version: "fixture", discordGatewayOwned: true }; + const scope = { companyId: f.companyId, endpointId: endpoint.id, applicationId, guildId }; + const remote: Record[] = []; + const registered = await reconcileStoredDiscordCommandRegistration(db, { scope, runtimeFence: { generation: context.generation, credentialFingerprint: context.credentialFingerprint }, botToken: "fixture-token", + authorize: async tx => { + expect((await tx.select().from(tables.chatEndpoints).where(eq(tables.chatEndpoints.id, endpoint.id)).for("update"))[0].status).toBe("active"); + }, fetch: async (_url, init) => { + if ((init?.method ?? "GET") === "GET") return Response.json(remote); + const command = { ...JSON.parse(String(init?.body)), id: "888888888888888888", application_id: applicationId, version: "999999999999999999" }; + remote.splice(0, remote.length, command); return Response.json(command); + } }); + expect(registered.kind).toBe("registered"); + const leaseKey = "discord_gateway", token = randomUUID(); + await db.insert(tables.chatEndpointLeases).values({ companyId: f.companyId, endpointId: endpoint.id, leaseKey, token, + expiresAt: new Date(Date.now() + 60_000) }); + const record = { endpoint, assignedAgentName: "Idle agent", connectionName: "Discord", applicationId: randomUUID(), credentialSecretRefs: [] }; + const unused = async (): Promise => { throw new Error("No Discord transport/control boundary expected"); }; + const service = createDiscordCommands(db, { endpointRecord: async () => record, runtimeContextForRecord: () => context, + runtimeGeneration: () => 1, credentialFingerprint: () => context.credentialFingerprint, resolveCredentialRefs: unused, + fetchImpl: unused, withCredentialMutationLease: chatCredentialMutationLease(db), runtimeCallbackRecord: async () => record, + ensurePrincipal: async () => ({ principal, userId: f.userId }), runtimeCallbackEndpoint: async () => endpoint, + nonDirectDestinationAllowed: () => true, lockCurrentPrincipalAuthorization: async () => ({ allowed: true, userId: f.userId }), + getDiscordGatewayOwnership: () => ({ leaseKey, token }), discordGatewayRuntimeIsCurrent: () => true, + stageAuthorizedTaskControlPublication: unused, scheduleMessageProcessing: unused, processPendingPublications: unused }); + const thread = `discord:${guildId}:${channelId}`; + const result = await service.handleDiscordNativeCommand({ endpointId: endpoint.id, provider: "discord", transport: "discord_gateway", + event: { command: "/paperclip status", text: "", channelId: thread, channel: { id: thread }, + user: { userId: principal.externalId, isBot: false, isMe: false }, raw: { id: "777777777777777777", application_id: applicationId, + type: 2, version: 1, channel_id: channelId, channel: { id: channelId, type: 0 }, guild_id: guildId, context: 0, + authorizing_integration_owners: { "0": guildId }, data: { id: "888888888888888888", type: 1, name: "paperclip", + options: [{ type: 1, name: "status" }] }, user: { id: principal.externalId, username: "operator", bot: false } } } } as never, context); + expect(result).toMatchObject({ kind: "accepted" }); + expect(await blockedActionKind("discord_native_command")).toBe(false); + expect(await blockedActionKind("discord_command_registration")).toBe(true); + const [receipt] = await db.select().from(tables.chatActions).where(eq(tables.chatActions.kind, "discord_native_command")); + await db.update(tables.chatActions).set({ result: { ...receipt.result, kind: "unknown" } }).where(eq(tables.chatActions.id, receipt.id)); + expect(await blockedActionKind("discord_native_command")).toBe(true); + }); + + it.each(["live_photo_video", "heif_jpeg_preview"] as const)("retains actual Photon %s provenance after source-bound storage registration", async relationKind => { + const { createInboundAttachments } = await import("../services/chat-channels/inbound-attachments.js"); + const { createInboundWakeups } = await import("../services/chat-channels/inbound/wakeups.js"); + const { downloadPhotonAttachment } = await import("../services/photon/attachments.js"); + const { issueService } = await import("../services/issues.js"); + const { createStorageService } = await import("../storage/service.js"); + const { createLocalDiskStorageProvider } = await import("../storage/local-disk-provider.js"); + const f = await slackFixture(), directory = await mkdtemp(path.join(tmpdir(), "history-photon-provenance-")); + const [endpoint] = await db.update(tables.chatEndpoints).set({ provider: "imessage-photon" }).where(eq(tables.chatEndpoints.id, f.endpoint.id)).returning(); + const [comment] = await db.insert(tables.issueComments).values({ companyId: f.companyId, issueId: f.issueId, body: "Source-bound photo" }).returning(); + const providerEventId = randomUUID(); + const [delivery] = await db.insert(tables.chatDeliveries).values({ companyId: f.companyId, endpointId: endpoint.id, + conversationId: f.conversation.id, principalId: f.principal.id, providerEventId, deduplicationKey: randomUUID(), + eventKind: "message", state: "processing", attempts: 1, normalizedEvent: { providerEventId, kind: "message" } }).returning(); + const unused = async (): Promise => { throw new Error("Unexpected Photon admission boundary"); }; + const wakes = createInboundWakeups(db, { authorizeInboundWakeup: unused, authorizeInboundWorkStart: unused, + attachmentOmissionDetail: () => null, isExternalActionAuthorizationChange: () => false, redactError: String, + fetchImpl: unused, heartbeat: { wakeup: unused } }); + try { + await db.transaction(tx => wakes.stageInboundWakeup(tx, { endpoint, deliveryId: delivery.id, conversation: f.conversation, + commentId: comment.id, principalId: f.principal.id, actorUserId: null })); + const storage = createStorageService(createLocalDiskStorageProvider(directory)); + const bytes = relationKind === "heif_jpeg_preview" + ? await readFile(new URL("./photon/fixtures/synthetic.heic", import.meta.url)) + : Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/6S8AAAAASUVORK5CYII=", "base64"); + const type = relationKind === "heif_jpeg_preview" ? "image/heic" : "image/png"; + let body = bytes, closed = false; + if (relationKind === "live_photo_video") { + const companion = Buffer.from("Actual bounded companion bytes"), locator = { kind: "photon_attachment" as const, + lineId: "line", chatGuid: "chat", messageGuid: "message", attachmentGuid: "attachment" }; + const info = { guid: "attachment", totalBytes: bytes.length, mimeType: type, fileName: "photo.png", isSticker: false, isHidden: false }; + const stream = { async *[Symbol.asyncIterator]() { + yield { type: "header", info, companionInfo: { kind: "live-photo-video", mimeType: "video/mp4", totalBytes: companion.length, fileName: "motion.mp4" } }; + yield { type: "primaryChunk", data: bytes }; yield { type: "companionChunk", data: companion }; + }, close: async () => { closed = true; } }; + body = await downloadPhotonAttachment({ messages: { get: async () => ({ guid: "message", chatGuids: ["chat"], content: { attachments: [info] } }) }, + attachments: { downloadStream: () => stream } } as never, "line", locator); + expect(closed).toBe(true); + } + const service = createInboundAttachments(db, { storage, authorizeInboundWakeup: async () => ({ delivery, conversation: f.conversation }), + lifecycleRuntimeFence: () => ({ generation: 1, credentialFingerprint: "c".repeat(64) }), sanitizeFilename: value => value ?? null, + issuesSvc: issueService(db), isExternalActionAuthorizationChange: () => false, redactError: String }); + const result = await service.ingestAttachments({ endpoint, endpointRuntime: {} as never, deliveryId: delivery.id, + issueId: f.issueId, issueCommentId: comment.id, actorUserId: null, + attachments: [{ type: "image", name: relationKind === "heif_jpeg_preview" ? "photo.heic" : "photo.png", mimeType: type, + size: bytes.length, fetchData: async () => body }] }); + expect(result.omissionReasons).toEqual({}); + expect(result.storedIds).toHaveLength(2); + const receipt = (await db.select().from(tables.chatActions)).find(a => a.kind === (relationKind === "heif_jpeg_preview" ? "attachment_derivative" : "attachment_companion"))!; + expect(receipt).toMatchObject({ status: "processed", payload: { kind: relationKind } }); + expect(await blockedActionKind(receipt.kind)).toBe(false); + expect(await blockedActionKind("inbound_wakeup")).toBe(true); + const assets = await db.select().from(tables.assets); + for (const asset of assets) expect((await storage.headObject(f.companyId, asset.objectKey)).exists).toBe(true); + const { chatChannelService } = await import("../services/chat-channels.js"); + const chat = chatChannelService(db, { heartbeat: { wakeup: unused }, fetch: unused }); + try { + await expect(chat.publishBoardMessage(endpoint.id, f.conversation.id, "Reuse an already bound attachment", randomUUID(), f.userId, + [result.storedIds[0]])).rejects.toMatchObject({ status: 409 }); + expect(await blockedActionKind("board_send_rejected")).toBe(false); + } finally { await chat.shutdown(); } + await db.update(tables.chatActions).set({ payload: { ...receipt.payload, derivativeSha256: "0".repeat(64) } }).where(eq(tables.chatActions.id, receipt.id)); + expect(await blockedActionKind(receipt.kind)).toBe(true); + await db.update(tables.chatActions).set({ payload: receipt.payload }).where(eq(tables.chatActions.id, receipt.id)); + const [otherIssue] = await db.insert(tables.issues).values({ companyId: f.companyId, title: "Wrong lineage" }).returning(); + await db.update(tables.issueAttachments).set({ issueId: otherIssue.id }).where(eq(tables.issueAttachments.id, String(receipt.payload.derivativeAttachmentId))); + expect(await blockedActionKind(receipt.kind)).toBe(true); + await db.update(tables.issueAttachments).set({ issueId: f.issueId }).where(eq(tables.issueAttachments.id, String(receipt.payload.derivativeAttachmentId))); + const { issueRoutes } = await import("../routes/issues.js"); + const { createMcpApiDispatch } = await import("../services/public-mcp/capabilities.js"); + const { Router } = await import("express"); + const router = Router(); router.use(issueRoutes(db, storage)); + const api = createMcpApiDispatch(router), actor = { type: "board", userId: f.userId, companyIds: [f.companyId], source: "session", + memberships: [{ companyId: f.companyId, membershipRole: "owner", status: "active" }] }; + for (const id of result.storedIds) expect(await api({ actor } as never, "DELETE", `/attachments/${id}`)).toEqual({ ok: true }); + expect(await db.select().from(tables.issueAttachments)).toHaveLength(0); + for (const asset of assets) expect((await storage.headObject(f.companyId, asset.objectKey)).exists).toBe(false); + expect(await blockedActionKind(receipt.kind)).toBe(false); + expect(await blockedActionKind("inbound_wakeup")).toBe(true); + } finally { wakes.unregisterDeferredWakeOwner(); await rm(directory, { recursive: true, force: true }); } + }, 60_000); + + it("retires actual Photon interaction, retry and response-notice history with their canonical owners", async () => { + const { issueThreadInteractionService } = await import("../services/issue-thread-interactions.js"); + const { recordPhotonPublicationRetry } = await import("../services/chat-channels/imessage-photon/publication-retry.js"); + const { stagePhotonResponseNotice } = await import("../services/chat-channels/imessage-photon/response-notices.js"); + const { chatChannelService } = await import("../services/chat-channels.js"); + const f = await seed(), c = await connection(f); + await db.update(tables.issues).set({ status: "in_progress" }).where(eq(tables.issues.id, f.issueId)); + const [endpoint] = await db.insert(tables.chatEndpoints).values({ companyId: f.companyId, connectionId: c.connectionId, + assignedAgentId: f.agentId, provider: "imessage-photon", publicId: randomUUID(), status: "active", publicationMode: "automatic" }).returning(); + const [conversation] = await db.insert(tables.chatConversations).values({ companyId: f.companyId, endpointId: endpoint.id, + issueId: f.issueId, externalConversationId: "photon-personal", externalThreadId: "photon-personal", externalLabel: "Personal", state: "active" }).returning(); + const interactions = issueThreadInteractionService(db); + const interaction = await interactions.create({ id: f.issueId, companyId: f.companyId }, { kind: "ask_user_questions", + continuationPolicy: "wake_assignee", payload: { version: 1, title: "Choose", submitLabel: "Continue", questions: [{ + id: "choice", prompt: "Choose", selectionMode: "single", required: true, allowOther: false, + options: [{ id: "yes", label: "Yes" }, { id: "no", label: "No" }] }] } }, { agentId: f.agentId }); + const [token] = await db.select().from(tables.chatActions); + expect(token).toMatchObject({ kind: "photon_interaction", status: "issued", conversationId: conversation.id }); + expect(await blockedActionKind("photon_interaction")).toBe(true); + await interactions.answerQuestions({ id: f.issueId, companyId: f.companyId }, interaction.id, + { answers: [{ questionId: "choice", optionIds: ["yes"] }] }, { userId: "local-board" }); + expect(await blockedActionKind("photon_interaction")).toBe(false); + const [previousSource] = await db.select().from(tables.chatPublications); + const [source] = await db.update(tables.chatPublications).set({ state: "delivery_unknown" }) + .where(eq(tables.chatPublications.id, previousSource.id)).returning(); + await db.transaction(async tx => { + await recordPhotonPublicationRetry(tx, endpoint.id, source, "local-board"); + await stagePhotonResponseNotice(tx, { publication: source, reference: String(token.payload.reference), questionIndex: 0, + key: `photon-response:${token.payload.reference}:1`, text: "Answer saved" }); + }); + expect(await blockedActionKind("photon_publication_retry")).toBe(true); + expect(await blockedActionKind("photon_response_notice")).toBe(true); + const unused = async (): Promise => { throw new Error("No Photon network expected"); }; + const service = chatChannelService(db, { heartbeat: { wakeup: unused }, fetch: unused }); + try { + for (const publication of await db.select().from(tables.chatPublications)) { + await db.update(tables.chatPublications).set({ state: "delivery_unknown" }).where(eq(tables.chatPublications.id, publication.id)); + await service.resolvePublication(endpoint.id, publication.id, "mark_delivered", "local-board"); + } + expect(await blocked("chat_actions")).toBe(false); + await db.update(tables.chatActions).set({ payload: { ...token.payload, interactionId: randomUUID() } }).where(eq(tables.chatActions.id, token.id)); + expect(await blockedActionKind("photon_interaction")).toBe(true); + } finally { await service.shutdown(); } + }); + + it("retains actual Slack connect discovery while its separate notice remains pending", async () => { + const { createSlackCommands, slackCommandForAgent } = await import("../services/chat-channels/slack/commands.js"); + const { createProviderEffects } = await import("../services/chat-channels/provider-effects.js"); + const { chatCredentialMutationLease } = await import("../services/chat-credential-mutation-lease.js"); + const f = await slackFixture(), unused = async (): Promise => { throw new Error("Unexpected discovery boundary"); }; + const effects = createProviderEffects(db, { endpointRecord: unused, runtimeCallbackEndpoint: async () => f.endpoint, + withCredentialMutationLease: chatCredentialMutationLease(db), runtimeFor: unused, invalidateRuntime: unused, + lockCurrentPrincipalAuthorization: unused, redactError: String, MAX_ERROR_TEXT: 500, + nonDirectDestinationAllowed: () => false, lockTelegramCallbackNoticeAuthority: unused }); + const command = slackCommandForAgent("Idle agent", f.endpoint.publicId), scheduled: string[] = []; + const service = createSlackCommands(db, { runtime: { get: () => undefined }, runtimeFor: unused, handleSdkMessage: unused, + stageProviderEffect: effects.stageProviderEffect, scheduleProviderEffect: id => { scheduled.push(id); }, + ensurePrincipal: async () => ({ principal: f.principal, userId: null, linkedDenied: false }), + runtimeCallbackEndpoint: async () => f.endpoint, getPublicBaseUrl: () => "https://history.example", + canonicalProviderResourceId: unused, nonDirectDestinationAllowed: () => false, sponsorAllowsGuest: unused, + latestSlackDmControlThreadId: unused, scheduleMessageProcessing: unused, processSlackTaskStart: unused }); + await service.handleSlackSlashCommand({ endpointId: f.endpoint.id, provider: "slack", transport: "slack_socket", + event: { command, text: "connect", triggerId: randomUUID(), channel: { id: "DCHANNEL" }, + user: { userId: "UREQUESTER", isBot: false, isMe: false }, raw: {} } } as never, + { generation: 1, credentialFingerprint: "c".repeat(64), localEpoch: 0, version: "fixture" }, + { endpoint: f.endpoint, assignedAgentName: "Idle agent", connectionName: "Slack", applicationId: randomUUID(), credentialSecretRefs: [] }, command, "connect"); + expect(scheduled).toHaveLength(1); + expect((await db.select().from(tables.chatIdentityLinks))[0]).toMatchObject({ status: "pending", confirmationTokenHash: expect.any(String) }); + expect(await blockedActionKind("slack_connect")).toBe(false); + expect(await blockedActionKind("provider_effect")).toBe(true); + expect(await blocked("chat_actions")).toBe(true); + }); + + it("retains the actual Telegram stop-subscription ownership receipt", async () => { + const { createTelegramMaintenance } = await import("../services/chat-channels/telegram/maintenance.js"); + const { chatCredentialMutationLease } = await import("../services/chat-credential-mutation-lease.js"); + const f = await seed(), c = await connection(f); + await db.update(tables.toolConnections).set({ enabled: true, status: "active" }).where(eq(tables.toolConnections.id, c.connectionId)); + const [endpoint] = await db.insert(tables.chatEndpoints).values({ companyId: f.companyId, connectionId: c.connectionId, + assignedAgentId: f.agentId, provider: "telegram", publicId: randomUUID(), status: "active", botExternalId: "1234567" }).returning(); + const unused = async (): Promise => { throw new Error("Unexpected Telegram provider boundary"); }; + const context = { generation: 3, credentialFingerprint: "c".repeat(64) }, withCredentialMutationLease = chatCredentialMutationLease(db); + const service = createTelegramMaintenance(db, { endpointRecord: unused, runtimeContextForRecord: () => context, + withCredentialMutationLease, resolveCredentials: unused, clearCredentials: unused, + runtimeCallbackEndpoint: async () => endpoint, getWebhookPublicBaseUrl: () => "https://history.example", + fetchImpl: unused, runtimeGeneration: () => 3, credentialFingerprint: () => context.credentialFingerprint, + redactError: String, credentialCheckTimeoutMs: 1000 }); + await withCredentialMutationLease(endpoint, lease => service.recordTelegramStopSubscription(endpoint, context, lease)); + expect(await service.hasTelegramStopSubscription(endpoint, context)).toBe(true); + expect(await blocked("chat_actions")).toBe(false); + const [receipt] = await db.select().from(tables.chatActions); + await db.update(tables.chatActions).set({ payload: { ...receipt.payload, webhookUrlSha256: "unknown" } }).where(eq(tables.chatActions.id, receipt.id)); + expect(await blocked("chat_actions")).toBe(true); + }); + + it("retains the actual Slack private-source boundary and governed completed write", async () => { + const { recordSlackReadBoundary } = await import("../services/connectors/slack-access.js"); + const { registerSlackTaskAuthority } = await import("../services/connectors/slack-authority.js"); + const { executeSlackWrite } = await import("../services/connectors/slack-writes.js"); + const f = await slackFixture(); + await recordSlackReadBoundary(db, f.authority, { id: "CPRIVATE", is_private: true }); + expect(await blocked("chat_actions")).toBe(false); + const invocationId = randomUUID(); + await db.insert(tables.toolInvocations).values({ id: invocationId, companyId: f.companyId, agentId: f.agentId, + issueId: f.issueId, runId: f.runId, connectionId: f.connectionId, actorType: "agent", actorId: f.agentId, + toolName: "slack_post_message", status: "executing" }); + const unregister = registerSlackTaskAuthority(db, async () => f.authority); + const methods: string[] = []; + try { + await executeSlackWrite(db, { companyId: f.companyId, agentId: f.agentId, issueId: f.issueId, runId: f.runId }, + "slack_post_message", { channel: "DCHANNEL", text: "Confirmed message", idempotencyKey: randomUUID() }, invocationId, + async input => { + const method = new URL(String(input)).pathname.split("/").at(-1)!; methods.push(method); + if (method === "conversations.info") return Response.json({ ok: true, channel: { id: "DCHANNEL", is_im: true, user: "UREQUESTER" } }); + if (method === "users.info") return Response.json({ ok: true, user: { id: "UREQUESTER", team_id: "TWORKSPACE", is_bot: false } }); + if (method === "chat.postMessage") return Response.json({ ok: true, channel: "DCHANNEL", ts: "123.456" }); + throw new Error(`Unexpected Slack boundary ${method}`); + }); + expect(methods).toContain("chat.postMessage"); + expect((await db.select().from(tables.chatActions)).find(a => a.kind === "slack_tool_write")) + .toMatchObject({ status: "processed", result: { channel: "DCHANNEL", ts: "123.456" } }); + expect(await blocked("chat_actions")).toBe(false); + await db.update(tables.chatActions).set({ status: "uncertain" }).where(eq(tables.chatActions.kind, "slack_tool_write")); + expect(await blocked("chat_actions")).toBe(true); + } finally { unregister(); } + }); + + it("retires actual Slack invitation, test-message and mirror receipts without hiding their publication", async () => { + const { slackTestMessageService } = await import("../services/connectors/slack/test-message.js"); + const { mirrorSlackBoardComment } = await import("../services/slack-board-messages.js"); + const { chatChannelService } = await import("../services/chat-channels.js"); + const { chatCredentialMutationLease } = await import("../services/chat-credential-mutation-lease.js"); + const { instanceSettingsService } = await import("../services/instance-settings.js"); + const f = await slackFixture(); + await instanceSettingsService(db).updateExperimental({ enableChatConnectors: true }); + const unused = async (): Promise => { throw new Error("Unexpected provider boundary"); }; + const credentialLease = chatCredentialMutationLease(db); + const send = slackTestMessageService(db, { withLock: (id, task) => credentialLease(f.endpoint, task), + runtimeBotToken: async () => "test-token", linkInstaller: async () => {}, ensureAccount: async () => f.principal.id, + createLinkIntent: async () => ({ confirmationUrl: "https://example.test/link", expiresAt: new Date(Date.now() + 600_000).toISOString() }), + fetch: async input => { + const method = new URL(String(input)).pathname.split("/").at(-1); + if (method === "users.info") return Response.json({ ok: true, user: { id: "UREQUESTER", team_id: "TWORKSPACE", is_bot: false } }); + if (method === "chat.postMessage") return Response.json({ ok: true, channel: "DCHANNEL", ts: "123.456" }); + throw new Error(`Unexpected setup boundary ${method}`); + } }); + expect(await send(f.endpoint.id, { userId: f.userId }, { requestId: randomUUID(), slackUserId: "UREQUESTER" })) + .toMatchObject({ status: "connection_pending" }); + expect(await blocked("chat_actions")).toBe(false); + await db.insert(tables.chatIdentityLinks).values({ companyId: f.companyId, endpointId: f.endpoint.id, + principalId: f.principal.id, paperclipUserId: f.userId, status: "linked", confirmedAt: now }); + expect(await send(f.endpoint.id, { userId: f.userId }, { requestId: randomUUID(), slackUserId: "UREQUESTER" })) + .toMatchObject({ status: "sent" }); + expect(await blocked("chat_actions")).toBe(false); + const [comment] = await db.insert(tables.issueComments).values({ companyId: f.companyId, issueId: f.issueId, + authorType: "user", authorUserId: f.userId, body: "Mirror this board comment" }).returning(); + await db.transaction(tx => mirrorSlackBoardComment(tx, comment)); + const [publication] = await db.select().from(tables.chatPublications); + expect(publication).toMatchObject({ state: "pending", commentId: comment.id }); + expect(await blocked("chat_actions")).toBe(true); + const service = chatChannelService(db, { heartbeat: { wakeup: unused }, fetch: unused }); + try { + await db.update(tables.chatPublications).set({ state: "delivery_unknown" }).where(eq(tables.chatPublications.id, publication.id)); + await service.resolvePublication(f.endpoint.id, publication.id, "mark_delivered", f.userId); + expect(await blocked("chat_actions")).toBe(false); + await db.update(tables.chatActions).set({ status: "uncertain" }).where(eq(tables.chatActions.kind, "slack_test_message")); + expect(await blocked("chat_actions")).toBe(true); + } finally { await service.shutdown(); } + }); + + it("retires the actual shared GitHub response target only after its request is final", async () => { + const { githubIngressFixture } = await import("./helpers/github-webhook-ingress.js"); + const { writeGitHubResponseComment } = await import("../services/chat-github-response-comments.js"); + const { stagePullResponse } = await import("../services/chat-channels/github/response-target.js"); + const { withGitHubPublicationLease } = await import("../services/chat-github-publication-lease.js"); + const f = await githubIngressFixture(db), providerEventId = randomUUID(); + const [issue] = await db.insert(tables.issues).values({ companyId: f.companyId, title: "GitHub reply", status: "done" }).returning(); + const [conversation] = await db.insert(tables.chatConversations).values({ companyId: f.companyId, endpointId: f.endpoint.id, + issueId: issue.id, externalConversationId: "repository:418", externalThreadId: "github:org/repo:7", externalLabel: "PR7" }).returning(); + const [delivery] = await db.insert(tables.chatDeliveries).values({ companyId: f.companyId, endpointId: f.endpoint.id, + conversationId: conversation.id, providerEventId, deduplicationKey: randomUUID(), eventKind: "action", normalizedEvent: { providerEventId, kind: "action" } }).returning(); + const source = { endpoint: f.endpoint, conversation, delivery, number: 7, replyId: null, repositoryId: "418" }; + await stagePullResponse(db, source); + expect(await blocked("chat_actions")).toBe(false); // initialized target owns no request yet + let posted: { id: number; html_url: string; body: string; user: { login: string | null } } | undefined; + const calls: string[] = []; + const api = { request: async (pathname: string, options?: { method?: string; body?: unknown }): Promise => { + calls.push(`${options?.method ?? "GET"} ${pathname}`); + if (!options?.method) return (pathname.includes("?per_page=") ? [] : posted) as T; + posted = { id: 123, html_url: "https://github.com/org/repo/pull/7#issuecomment-123", + body: (options.body as { body: string }).body, user: { login: f.endpoint.botUsername } }; + return posted as T; + } }; + const publish = (final: boolean) => withGitHubPublicationLease(db, { companyId: f.companyId, endpointId: f.endpoint.id, + repositoryId: "418", number: 7 }, async () => { throw new Error("No lease network expected"); }, lease => + writeGitHubResponseComment(db, source, api, lease, { body: final ? "Final result" : "Working", final }, async () => {})); + await publish(false); + expect(await blocked("chat_actions")).toBe(true); + await publish(true); + expect(calls).toContain("PATCH /issues/comments/123"); + expect(await blocked("chat_actions")).toBe(false); + const request = (await db.select().from(tables.chatActions)).find(a => a.kind === "github_response_comment")!; + expect(request).toMatchObject({ status: "processed", result: { final: true, id: "123" } }); + await db.update(tables.chatActions).set({ result: { ...request.result, final: false } }).where(eq(tables.chatActions.id, request.id)); + expect(await blocked("chat_actions")).toBe(true); + }); + + it("retires actual GitHub terminal review checks despite their retained retry hints", async () => { + const { githubIngressFixture } = await import("./helpers/github-webhook-ingress.js"); + const { githubReviewCheckService } = await import("../services/chat-github-checks.js"); + const { secretService } = await import("../services/secrets.js"); + const { defaultGitHubReviewPolicy } = await import("@paperclipai/shared"); + const oldKey = process.env.PAPERCLIP_SECRETS_MASTER_KEY; + process.env.PAPERCLIP_SECRETS_MASTER_KEY = "a".repeat(64); + try { + const f = await githubIngressFixture(db); + const privateKey = generateKeyPairSync("rsa", { modulusLength: 2048, + privateKeyEncoding: { type: "pkcs8", format: "pem" }, publicKeyEncoding: { type: "spki", format: "pem" } }).privateKey; + const refs = []; + for (const [key, value] of Object.entries({ appId: "12", installationId: "34", privateKey })) { + const secret = await secretService(db).create(f.companyId, { name: `GitHub ${key}`, key, provider: "local_encrypted", value }); + refs.push({ configPath: `credentials.${key}`, secretId: secret.id, versionSelector: "latest" as const }); + await secretService(db).createBinding({ companyId: f.companyId, secretId: secret.id, targetType: "tool_connection", + targetId: f.endpoint.connectionId, configPath: `credentials.${key}`, versionSelector: "latest" }); + } + await db.update(tables.toolConnections).set({ enabled: true, status: "active", credentialSecretRefs: refs }) + .where(eq(tables.toolConnections.id, f.endpoint.connectionId)); + await db.insert(tables.chatGitHubConfigurations).values({ companyId: f.companyId, endpointId: f.endpoint.id, updatedByUserId: "local-board", + configuration: { version: 1, toolsEnabled: true, responsibleUserId: "local-board", memberAccess: "all_linked", people: [], + repositories: {}, defaults: { ...defaultGitHubReviewPolicy(), ratingThreshold: 3 } } }); + await db.insert(tables.chatEndpointResources).values({ companyId: f.companyId, endpointId: f.endpoint.id, + providerResourceId: "org/repo", type: "repository", label: "org/repo", enabled: true, + availability: "available", metadata: { providerRepositoryId: "418" } }); + const event = { event: "opened" as const, deliveryId: randomUUID(), repositoryId: "418", repository: "org/repo", pullNumber: 7, + title: "Saved review", body: "", baseSha: "a".repeat(40), headSha: "b".repeat(40), baseBranch: "master", + author: { id: "42", login: "octocat", isBot: false }, sender: { id: "42", login: "octocat" }, draft: false, labels: [] }; + const calls: string[] = []; + let stale = false; + const service = githubReviewCheckService(db, async input => { + const pathname = new URL(String(input)).pathname; calls.push(pathname); + if (pathname === "/app/installations/34/access_tokens") return Response.json({ token: "test-token" }); + if (pathname === "/repos/org/repo/pulls/7") return Response.json({ head: { sha: stale ? "c".repeat(40) : event.headSha } }); + if (pathname.endsWith("/check-runs")) return Response.json(pathname.includes("/commits/") + ? { check_runs: [] } : { id: 123, html_url: "https://github.com/org/repo/runs/123" }); + throw new Error(`Unexpected GitHub boundary ${pathname}`); + }); + await service.enqueue(f.endpoint, event, false, "manual review required"); + const [pending] = await db.select().from(tables.chatActions).where(eq(tables.chatActions.kind, "github_review_check")); + expect(await blocked("chat_actions")).toBe(true); + await service.processPending(); + const [completed] = await db.select().from(tables.chatActions).where(eq(tables.chatActions.id, pending.id)); + expect(completed, JSON.stringify({ result: completed.result, calls })).toMatchObject({ status: "processed", result: { state: "manual_required", retryable: true } }); + expect(calls).toContain("/repos/org/repo/check-runs"); + expect(await blocked("chat_actions")).toBe(false); + stale = true; + await service.enqueue(f.endpoint, { ...event, deliveryId: randomUUID() }, false, "stale head"); + const next = (await db.select().from(tables.chatActions)).find(a => a.status === "received")!; + await service.processPending(); + expect((await db.select().from(tables.chatActions).where(eq(tables.chatActions.id, next.id)))[0]) + .toMatchObject({ status: "cancelled", result: { code: "stale_head" } }); + expect(await blocked("chat_actions")).toBe(false); + await service.enqueue(f.endpoint, { ...event, deliveryId: randomUUID() }, false, "connection removed"); + await db.update(tables.chatEndpointResources).set({ enabled: false }).where(eq(tables.chatEndpointResources.endpointId, f.endpoint.id)); + const last = (await db.select().from(tables.chatActions)).find(a => a.status === "received")!; + await service.processPending(); + expect((await db.select().from(tables.chatActions).where(eq(tables.chatActions.id, last.id)))[0]) + .toMatchObject({ status: "cancelled", result: { code: "connection_changed" } }); + expect(await blocked("chat_actions")).toBe(false); + await db.update(tables.chatActions).set({ result: { state: "queued", retryable: true } }).where(eq(tables.chatActions.id, completed.id)); + expect(await blocked("chat_actions")).toBe(true); + } finally { if (oldKey === undefined) delete process.env.PAPERCLIP_SECRETS_MASTER_KEY; else process.env.PAPERCLIP_SECRETS_MASTER_KEY = oldKey; } + }, 60_000); + + it("retires real question-form tokens only after their exact canonical answer commits", async () => { + const { issueThreadInteractionService } = await import("../services/issue-thread-interactions.js"); + const { claimChatQuestionFormSubmission, completeChatQuestionFormSubmission } = await import("../services/chat-question-forms.js"); + const f = await seed(), c = await connection(f); + await db.update(tables.issues).set({ status: "in_progress" }).where(eq(tables.issues.id, f.issueId)); + const [endpoint] = await db.insert(tables.chatEndpoints).values({ companyId: f.companyId, connectionId: c.connectionId, + assignedAgentId: f.agentId, publicId: randomUUID(), provider: "slack", status: "active", publicationMode: "automatic", + capabilities: { actions: true, modals: true } }).returning(); + const [conversation] = await db.insert(tables.chatConversations).values({ companyId: f.companyId, endpointId: endpoint.id, + issueId: f.issueId, externalConversationId: "question-channel", externalThreadId: "question-thread", externalLabel: "Question", state: "active" }).returning(); + const [principal] = await db.insert(tables.chatExternalPrincipals).values({ companyId: f.companyId, provider: "slack", + providerAccountId: "workspace", externalId: "answering-user" }).returning(); + const service = issueThreadInteractionService(db); + const interaction = await service.create({ id: f.issueId, companyId: f.companyId }, { + kind: "ask_user_questions", continuationPolicy: "wake_assignee", title: "Two questions", + payload: { version: 1, title: "Two questions", submitLabel: "Continue", questions: ["first", "second"].map(id => ({ + id, prompt: id, selectionMode: "single" as const, required: true, allowOther: false, + options: [{ id: "yes", label: "Yes" }, { id: "no", label: "No" }] })) }, + }, { agentId: f.agentId }); + const actions = await db.select().from(tables.chatActions); + const open = actions.find(a => a.kind === "question_form_open"), submit = actions.find(a => a.kind === "question_form_submit"); + expect(open).toMatchObject({ conversationId: conversation.id, status: "issued", result: null }); + expect(submit).toMatchObject({ status: "issued", result: null }); + expect(await blocked("chat_actions")).toBe(true); + await service.answerQuestions({ id: f.issueId, companyId: f.companyId }, interaction.id, + { answers: ["first", "second"].map(questionId => ({ questionId, optionIds: ["yes"] })) }, { userId: "local-board" }, { + afterResolveInTransaction: async (tx, resolved) => { + expect(await claimChatQuestionFormSubmission(tx, { actionRowId: submit!.id, principalId: principal.id })).toBe(true); + expect(await completeChatQuestionFormSubmission(tx, { actionRowId: submit!.id, interactionId: resolved.id })).toBe(true); + }, + }); + expect((await db.select().from(tables.issueThreadInteractions).where(eq(tables.issueThreadInteractions.id, interaction.id)))[0]) + .toMatchObject({ status: "answered", resolvedAt: expect.any(Date) }); + expect(await blocked("chat_actions")).toBe(false); + await db.update(tables.chatActions).set({ result: { code: "question_form_answered", interactionId: randomUUID() } }) + .where(eq(tables.chatActions.id, submit!.id)); + expect(await blocked("chat_actions")).toBe(true); + }); + + it("retires a staged inbound wake only after its actual delivery owner rejects it", async () => { + const { githubIngressFixture } = await import("./helpers/github-webhook-ingress.js"); + const { createInboundWakeups } = await import("../services/chat-channels/inbound/wakeups.js"); + const { chatChannelService } = await import("../services/chat-channels.js"); + const f = await githubIngressFixture(db); + const [issue] = await db.insert(tables.issues).values({ companyId: f.companyId, title: "Rejected input" }).returning(); + const [conversation] = await db.insert(tables.chatConversations).values({ companyId: f.companyId, endpointId: f.endpoint.id, + issueId: issue.id, externalConversationId: "rejected", externalThreadId: "rejected", externalLabel: "Rejected" }).returning(); + const [principal] = await db.insert(tables.chatExternalPrincipals).values({ companyId: f.companyId, provider: "github", + providerAccountId: "test", externalId: "42" }).returning(); + const [comment] = await db.insert(tables.issueComments).values({ companyId: f.companyId, issueId: issue.id, body: "Input" }).returning(); + const providerEventId = randomUUID(); + const [delivery] = await db.insert(tables.chatDeliveries).values({ companyId: f.companyId, endpointId: f.endpoint.id, + conversationId: conversation.id, providerEventId, deduplicationKey: randomUUID(), eventKind: "message", state: "received", + normalizedEvent: { providerEventId, kind: "message" } }).returning(); + const unused = async (): Promise => { throw new Error("No provider/admission boundary expected"); }; + const wakes = createInboundWakeups(db, { authorizeInboundWakeup: unused, authorizeInboundWorkStart: unused, + attachmentOmissionDetail: () => null, isExternalActionAuthorizationChange: () => false, redactError: String, + fetchImpl: unused, heartbeat: { wakeup: unused } }); + const service = chatChannelService(db, { heartbeat: { wakeup: unused }, fetch: unused }); + try { + await db.transaction(tx => wakes.stageInboundWakeup(tx, { endpoint: f.endpoint, deliveryId: delivery.id, + conversation, commentId: comment.id, principalId: principal.id, actorUserId: null })); + expect(await blocked("chat_actions")).toBe(true); + await wakes.settleRejectedInboundWakeups(delivery.id); + expect((await db.select().from(tables.chatActions))[0].status).toBe("preparing"); + await service.configure(f.endpoint.id, { action: "pause" }, "local-board"); + await wakes.settleRejectedInboundWakeups(delivery.id); + expect((await db.select().from(tables.chatActions))[0]).toMatchObject({ status: "failed", + result: { code: "inbound_wakeup_delivery_rejected" } }); + expect(await db.select().from(tables.agentWakeupRequests)).toHaveLength(0); + expect(await blocked("chat_actions")).toBe(false); + expect(await blocked("chat_deliveries")).toBe(false); + await db.update(tables.chatDeliveries).set({ nextAttemptAt: new Date(Date.now() + 60_000) }).where(eq(tables.chatDeliveries.id, delivery.id)); + expect(await blocked("chat_actions")).toBe(true); + } finally { wakes.unregisterDeferredWakeOwner(); await service.shutdown(); } + }); + + it("retires actual Teams consent and provider-effect receipts only after the transfer closes", async () => { + const { teamsFileTransferService } = await import("../services/chat-teams-file-transfers.js"); + const { projectTeamsFilePublication } = await import("../services/chat-teams-file-publication.js"); + const { installTeamsFileConsentHook, restoreTeamsFileConsentBinding } = await import("../services/chat-teams-file-consent.js"); + const oldKey = process.env.PAPERCLIP_SECRETS_MASTER_KEY; + process.env.PAPERCLIP_SECRETS_MASTER_KEY = "a".repeat(64); + try { + const f = await seed(), c = await connection(f), bytes = Buffer.from("Actual Teams transfer"); + const authority = { companyId: f.companyId, endpointId: randomUUID(), conversationId: randomUUID(), issueId: f.issueId, + publicationId: randomUUID(), commentId: randomUUID(), attachmentId: randomUUID(), principalId: randomUUID(), + authorizedUserId: "local-board", runtimeGeneration: 3, credentialFingerprint: "c".repeat(64), conversationGeneration: 2, + sourceDigest: "d".repeat(64), tenantId: randomUUID(), botAppId: randomUUID(), aadObjectId: randomUUID(), + providerConversationId: "a:exact-personal", providerUserId: "29:exact-recipient", + sha256: createHash("sha256").update(bytes).digest("hex"), byteSize: bytes.length, filename: "file.txt" }; + await db.insert(tables.chatEndpoints).values({ id: authority.endpointId, companyId: f.companyId, connectionId: c.connectionId, + assignedAgentId: f.agentId, publicId: randomUUID(), provider: "microsoft-teams", status: "active", + providerAccountId: authority.tenantId, botExternalId: authority.botAppId }); + await db.insert(tables.chatConversations).values({ id: authority.conversationId, companyId: f.companyId, + endpointId: authority.endpointId, issueId: f.issueId, externalConversationId: authority.providerConversationId, + externalThreadId: `teams:${Buffer.from(authority.providerConversationId).toString("base64url")}`, externalLabel: "Personal", + isDirectMessage: true, sessionGeneration: 2 }); + await db.insert(tables.chatExternalPrincipals).values({ id: authority.principalId, companyId: f.companyId, provider: "microsoft-teams", + providerAccountId: authority.tenantId, externalId: authority.aadObjectId }); + await db.insert(tables.issueComments).values({ id: authority.commentId, companyId: f.companyId, issueId: f.issueId, + authorUserId: "local-board", body: "Share this file" }); + const [asset] = await db.insert(tables.assets).values({ companyId: f.companyId, provider: "local_disk", objectKey: randomUUID(), + contentType: "text/plain", byteSize: bytes.length, sha256: authority.sha256, originalFilename: authority.filename }).returning(); + await db.insert(tables.issueAttachments).values({ id: authority.attachmentId, companyId: f.companyId, issueId: f.issueId, + assetId: asset.id, issueCommentId: authority.commentId }); + await db.insert(tables.chatPublications).values({ id: authority.publicationId, companyId: f.companyId, + endpointId: authority.endpointId, conversationId: authority.conversationId, issueId: f.issueId, commentId: authority.commentId, + idempotencyKey: randomUUID(), payload: { text: "Share file", attachmentIds: [authority.attachmentId] } }); + const service = teamsFileTransferService(db, { authorize: async () => ({ ...authority }), loadBytes: async () => bytes, + postConsent: async () => ({ id: "card-1" }), postFileInfo: async () => ({ id: "file-info-1" }), + uploadRequest: async () => Response.json({ id: "item-1", name: "file.txt", size: bytes.length }, { status: 201 }), + project: projectTeamsFilePublication }); + const created = await service.issue(authority, new Date(Date.now() + 60_000)); + await service.process(f.companyId, created.id); + const [pending] = await db.select().from(tables.chatTeamsFileTransfers); + expect(pending.phase).toBe("awaiting_consent"); + expect((await db.select().from(tables.chatActions)).map(a => a.kind)).toContain("teams_file_effect_intent"); + expect(await blocked("chat_actions")).toBe(true); + const binding = await restoreTeamsFileConsentBinding({ companyId: f.companyId, endpointId: authority.endpointId, + transferId: created.id, authorityDigest: pending.authorityDigest }, pending.privateState.binding as Record); + const handlers = new Map Promise<{ status: number }>>(); + installTeamsFileConsentHook({ on: (name, callback) => { handlers.set(name, callback); } }, { + companyId: f.companyId, endpointId: authority.endpointId, tenantId: authority.tenantId, botAppId: authority.botAppId, + onConsent: event => service.recordConsent(event) }); + expect((await handlers.get("file.consent.decline")!({ activity: { type: "invoke", name: "fileConsent/invoke", channelId: "msteams", + id: "decline-1", from: { id: authority.providerUserId, aadObjectId: authority.aadObjectId }, + recipient: { id: `28:${authority.botAppId}` }, conversation: { id: authority.providerConversationId, conversationType: "personal", tenantId: authority.tenantId }, + replyToId: "card-1", value: { action: "decline", context: { schema: binding.schema, token: binding.token, action: "decline" } } } })).status).toBe(200); + expect((await db.select().from(tables.chatTeamsFileTransfers))[0]).toMatchObject({ phase: "declined", attemptId: null, attemptExpiresAt: null }); + expect((await db.select().from(tables.chatActions)).map(a => a.kind)).toEqual(expect.arrayContaining(["teams_file_effect_intent", "teams_file_consent"])); + expect(await blocked("chat_actions")).toBe(false); + expect(await blocked("chat_publications")).toBe(false); + const [receipt] = await db.select().from(tables.chatActions).where(eq(tables.chatActions.kind, "teams_file_consent")); + await db.update(tables.chatActions).set({ payload: { ...receipt.payload, receiptDigest: "wrong" } }).where(eq(tables.chatActions.id, receipt.id)); + expect(await blocked("chat_actions")).toBe(true); + } finally { if (oldKey === undefined) delete process.env.PAPERCLIP_SECRETS_MASTER_KEY; else process.env.PAPERCLIP_SECRETS_MASTER_KEY = oldKey; } + }); + + it("measures both chat predicates against 10000 valid retained deliveries and 30000 completed actions", async () => { + const { githubIngressFixture } = await import("./helpers/github-webhook-ingress.js"); + const { chatChannelService } = await import("../services/chat-channels.js"); + const f = await githubIngressFixture(db, undefined, 256), providerEventId = randomUUID(); + const [delivery] = await db.insert(tables.chatDeliveries).values({ companyId: f.companyId, endpointId: f.endpoint.id, + providerEventId, deduplicationKey: randomUUID(), eventKind: "message", state: "received", + normalizedEvent: { providerEventId, kind: "message" } }).returning(); + const action = await f.insertAction({ deliveryId: delivery.id }); + await f.service.processGitHubWebhookIngress(action.id); + const unused = async (): Promise => { throw new Error("No external boundary expected"); }; + const service = chatChannelService(db, { heartbeat: { wakeup: unused }, fetch: unused }); + try { await service.configure(f.endpoint.id, { action: "pause" }, "local-board"); } + finally { await service.shutdown(); } + expect(await blocked("chat_deliveries")).toBe(false); + expect(await blocked("chat_actions")).toBe(false); + // Retention load clones a receipt produced above; it is measurement data, + // not a substitute for the producer regressions in the surrounding cases. + await db.execute(sql`INSERT INTO chat_deliveries (company_id, endpoint_id, provider_event_id, deduplication_key, event_kind, + state, normalized_event, processed_at) + SELECT ${f.companyId}::uuid, ${f.endpoint.id}::uuid, 'retained:' || n, 'retained:' || n, d.event_kind, d.state, + jsonb_set(d.normalized_event, '{providerEventId}', to_jsonb('retained:' || n)), d.processed_at + FROM chat_deliveries d CROSS JOIN generate_series(1, 9999) n WHERE d.id = ${delivery.id}::uuid`); + await db.execute(sql`INSERT INTO chat_actions (company_id, endpoint_id, delivery_id, kind, provider_action_id, payload, status, result) + SELECT a.company_id, a.endpoint_id, d.id, a.kind, 'retained:' || d.id || ':' || n, a.payload, a.status, a.result + FROM chat_actions a JOIN chat_deliveries d ON d.endpoint_id = a.endpoint_id CROSS JOIN generate_series(1, 3) n + WHERE a.id = ${action.id}::uuid AND d.id <> ${delivery.id}::uuid`); + await db.execute(sql`ANALYZE chat_deliveries`); + await db.execute(sql`ANALYZE chat_actions`); + const current = completedHistoryChecks.filter(c => ["chat_deliveries", "chat_actions"].includes(c.reason)); + const baselinePath = process.env.PAPERCLIP_HISTORY_QUERY_BASELINE_PATH; + const baseline: { reason: string; query: string }[] = baselinePath ? JSON.parse(await readFile(baselinePath, "utf8")) : []; + const measurements = []; + for (const [revision, checks] of [["published_aaf55692", baseline], ["repair", current]] as const) { + for (const check of checks) { + const started = performance.now(); + const plan = await db.transaction(async tx => { + await tx.execute(sql`SET LOCAL statement_timeout = '5s'`); + return tx.execute(sql`EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) SELECT EXISTS (${sql.raw(check.query)}) AS blocked`); + }); + measurements.push({ revision, reason: check.reason, wallMs: performance.now() - started, plan }); + } + } + const evidence = { deliveries: 10000, actions: 29998, measurements }; + if (process.env.PAPERCLIP_HISTORY_QUERY_PLANS_PATH) await writeFile(process.env.PAPERCLIP_HISTORY_QUERY_PLANS_PATH, + JSON.stringify(evidence, null, 2)); + console.info(`Retained chat history query plans ${JSON.stringify(evidence)}`); + expect(await blocked("chat_deliveries")).toBe(false); + expect(await blocked("chat_actions")).toBe(false); + }, 60_000); + it("uses actual chat ingress, lifecycle application and operator publication receipts", async () => { const { githubIngressFixture } = await import("./helpers/github-webhook-ingress.js"); const { chatCredentialMutationLease } = await import("../services/chat-credential-mutation-lease.js"); diff --git a/server/src/__tests__/storage-local-provider.test.ts b/server/src/__tests__/storage-local-provider.test.ts index e35500896fa..d25963d9fdf 100644 --- a/server/src/__tests__/storage-local-provider.test.ts +++ b/server/src/__tests__/storage-local-provider.test.ts @@ -119,6 +119,32 @@ describe("local disk storage provider", () => { await expect(service.getObject("company-1", stored.objectKey)).rejects.toMatchObject({ status: 404 }); }); + it.each(["EACCES", "EIO"])("retains storage uncertainty for %s instead of reporting absence", async code => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-storage-")); + tempRoots.push(root); + const service = createStorageService(createLocalDiskStorageProvider(root)); + const stored = await service.putFile({ companyId: "company-1", namespace: "issues/issue-1", originalFilename: "retained.txt", + contentType: "text/plain", body: Buffer.from("must not disappear") }); + const failure = Object.assign(new Error("controlled filesystem failure"), { code }); + const unlink = vi.spyOn(fs, "unlink").mockRejectedValueOnce(failure); + try { await expect(service.deleteObject("company-1", stored.objectKey)).rejects.toBe(failure); } + finally { unlink.mockRestore(); } + expect(await service.headObject("company-1", stored.objectKey)).toMatchObject({ exists: true }); + const stat = vi.spyOn(fs, "stat").mockRejectedValueOnce(failure); + try { await expect(service.headObject("company-1", stored.objectKey)).rejects.toBe(failure); } + finally { stat.mockRestore(); } + expect(await readStreamToBuffer((await service.getObject("company-1", stored.objectKey)).stream)).toEqual(Buffer.from("must not disappear")); + }); + + it("treats a non-directory object path as absent", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-storage-")); + tempRoots.push(root); + await fs.writeFile(path.join(root, "company-1"), "not a directory"); + const service = createStorageService(createLocalDiskStorageProvider(root)); + expect(await service.headObject("company-1", "company-1/object")).toEqual({ exists: false }); + await expect(service.deleteObject("company-1", "company-1/object")).resolves.toBeUndefined(); + }); + it.each(["headObject", "deleteObject"] as const)("cancels %s even if a provider stops responding", async method => { const provider = createLocalDiskStorageProvider(os.tmpdir()); const request = vi.spyOn(provider, method).mockImplementation(async () => new Promise(() => {})); diff --git a/server/src/__tests__/voice-session-store.integration.test.ts b/server/src/__tests__/voice-session-store.integration.test.ts index ed0b95194cc..de061793644 100644 --- a/server/src/__tests__/voice-session-store.integration.test.ts +++ b/server/src/__tests__/voice-session-store.integration.test.ts @@ -1,3 +1,5 @@ +import { chatHistoryActionBlocks } from "./helpers/chat-history-proof.js"; +import { voiceCallReports } from "../services/voice/voice-call-reports.js"; import { subscribeDatabaseWork } from "@paperclipai/db"; import * as voiceReplyNotifications from "../services/voice/voice-reply-notifications.js"; import { voiceReplySchedule } from "../services/voice/voice-reply-schedule.js"; @@ -40,15 +42,16 @@ import { buildPaperclipRuntimeMcpServers } from "../services/heartbeat.js"; import { createToolGatewayService } from "../services/tool-gateway.js"; import { initializeRunIdentity } from "../services/run-identity.js"; import { syncSpekoVoiceTools, spekoToolsForSession, executeSpekoVoiceTool } from "../services/voice/speko-agent-tools.js"; -const external = resolveExternalTestDatabaseUrl("PAPERCLIP_VOICE_TEST_DATABASE_URL"); +const historyExternal = resolveExternalTestDatabaseUrl("PAPERCLIP_HISTORY_TEST_DATABASE_URL"); +const external = historyExternal ?? resolveExternalTestDatabaseUrl("PAPERCLIP_VOICE_TEST_DATABASE_URL"); const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); (support.supported ? describe : describe.skip)("durable Speko voice sessions", () => { let db: ReturnType; let database: Awaited> | undefined; beforeAll(async () => { if (!external) database = await startEmbeddedPostgresTestDatabase("paperclip-speko-store-"); - if (external) await applyPendingMigrations(external); - db = createDb(external ?? database!.connectionString, { idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + if (external && !historyExternal) await applyPendingMigrations(external); + db = createDb(external ?? database!.connectionString, { idleTimeoutSeconds: 0, maxLifetimeSeconds: 0, ...(historyExternal ? { maxConnections: 4 } : {}) }); }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }); @@ -75,7 +78,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu return { companyId, endpointId, sessionId, token: reserved.token, envelope, webhookId: `msg_${toolId}`, fingerprint: createHash("sha256").update(JSON.stringify(envelope)).digest("hex"), accept }; }; async function publication(state: "streaming" | "published" = "published") { - const [row] = await db.insert(chatPublications).values({ companyId, endpointId, conversationId, issueId, idempotencyKey: randomUUID(), state, payload: { text: "Approved result" } }).returning(); + const [row] = await db.insert(chatPublications).values({ companyId, endpointId, conversationId, issueId, idempotencyKey: randomUUID(), state, publishedAt: state === "published" ? new Date() : null, payload: { text: "Approved result" } }).returning(); return row; } return { companyId, endpointId, agentId, connectionId, issueId, conversationId, callerId, sessionId, store, request, reserved, accept, toolInput, publication }; @@ -164,6 +167,13 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu const report = f.sign({ type: "call.report", session_id: f.call.providerSessionId, organization_id: f.event.organization_id }); await f.service.inbound.lifecycle(f.endpoint.publicId, report.body, report.headers); await settle(); expect(f.transport.callReport).toHaveBeenCalledTimes(2); expect(onError).not.toHaveBeenCalled(); + const receipts = (await db.select().from(chatActions).where(eq(chatActions.endpointId, f.endpointId))) + .filter(row => row.kind === "speko_lifecycle_receipt"); + expect(receipts.length).toBeGreaterThan(0); + for (const receipt of receipts) expect(await chatHistoryActionBlocks(db, receipt.id)).toBe(true); + await db.update(chatVoiceSessions).set({ endedAt: new Date(Date.now() - 2 * 86_400_000) }).where(eq(chatVoiceSessions.companyId, f.companyId)); + expect(await voiceCallReports(db, f.transport.callReport).load({ kind: "company", id: f.companyId })).toEqual([]); + for (const receipt of receipts) expect(await chatHistoryActionBlocks(db, receipt.id)).toBe(false); } finally { await coordinator.stop(); vi.useRealTimers(); const ended = f.sign({ type: "call.status", call_id: f.call.providerSessionId, status: "ended" }); @@ -1381,7 +1391,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu it("sends long approved phone answers as durable ordered parts and resumes after a rate limit", async () => { const f = await pushFixture(), text = "😀 answer ".repeat(4000) + "FINAL"; await db.update(chatPublications).set({payload: {text}}).where(eq(chatPublications.id, f.publication.id)); - const [later] = await db.insert(chatPublications).values({companyId: f.companyId, endpointId: f.endpointId, conversationId: f.conversationId, issueId: f.issueId, idempotencyKey: randomUUID(), state: "published", payload: {text: "Later answer B"}}).returning(); + const [later] = await db.insert(chatPublications).values({companyId: f.companyId, endpointId: f.endpointId, conversationId: f.conversationId, issueId: f.issueId, idempotencyKey: randomUUID(), state: "published", publishedAt: new Date(), payload: {text: "Later answer B"}}).returning(); await f.store.enqueuePublication(f.companyId, later.id); f.transport.sendCallMessage.mockResolvedValueOnce({messageId: "part-0"}).mockRejectedValueOnce(new SpekoProviderError("provider_unavailable", false, 429)); await f.push(); @@ -1405,6 +1415,13 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu for (const call of calls) {expect(call[1].length).toBeLessThanOrEqual(16_000); expect(/^[\uDC00-\uDFFF]|[\uD800-\uDBFF]$/.test(call[1])).toBe(false);} const [delivered] = await db.select().from(chatVoiceReplies).where(eq(chatVoiceReplies.id, reply.id)); expect(delivered.deliveredAt).toBeInstanceOf(Date); expect(delivered.spokenAt).toBeNull(); + const accepted = (await db.select().from(chatActions).where(eq(chatActions.endpointId, f.endpointId))) + .filter(row => row.kind === "speko_voice_reply_push"); + for (const receipt of accepted) expect(await chatHistoryActionBlocks(db, receipt.id)).toBe(true); + await f.service.end(f.companyId, f.session.id, f.input.caller); + await db.update(chatVoiceSessions).set({ endedAt: new Date(Date.now() - 2 * 86_400_000) }).where(eq(chatVoiceSessions.companyId, f.companyId)); + expect(await voiceCallReports(db, f.transport.callReport).load({ kind: "company", id: f.companyId })).toEqual([]); + for (const receipt of accepted) expect(await chatHistoryActionBlocks(db, receipt.id)).toBe(false); }); it("does not retry an uncertain long-answer part or send its suffix", async () => { const f = await pushFixture(); @@ -1416,6 +1433,13 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu expect(f.transport.sendCallMessage).toHaveBeenCalledTimes(2); const [reply] = await db.select().from(chatVoiceReplies).where(eq(chatVoiceReplies.publicationId, f.publication.id)); expect(reply.deliveredAt).toBeNull(); + await f.service.end(f.companyId, f.session.id, f.input.caller); + await db.update(chatVoiceSessions).set({ endedAt: new Date(Date.now() - 2 * 86_400_000) }).where(eq(chatVoiceSessions.companyId, f.companyId)); + expect(await voiceCallReports(db, f.transport.callReport).load({ kind: "company", id: f.companyId })).toEqual([]); + const uncertain = (await db.select().from(chatActions).where(eq(chatActions.endpointId, f.endpointId))) + .filter(row => row.kind === "speko_voice_reply_push" && row.status === "unknown"); + expect(uncertain).toHaveLength(1); + expect(await chatHistoryActionBlocks(db, uncertain[0]!.id)).toBe(true); }); it("pushes a published phone answer exactly once across concurrent dispatch and restart", async () => { const f = await pushFixture(); @@ -2027,6 +2051,15 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu expect(tools).toHaveLength(3); expect(f.transport.configureTool).toHaveBeenCalledTimes(7); expect(f.transport.configureTool.mock.calls.slice(-3).every((call) => Boolean(call[3]))).toBe(true); + const receipts = (await db.select().from(chatActions).where(eq(chatActions.endpointId, f.endpointId))) + .filter(row => ["speko_tool_setup", "speko_lifecycle_setup"].includes(row.kind)); + expect(receipts).toHaveLength(4); + for (const receipt of receipts) expect(await chatHistoryActionBlocks(db, receipt.id)).toBe(true); // real24h refresh obligation + await db.update(chatVoiceSessions).set({ endedAt: new Date(Date.now() - 2 * 86_400_000) }).where(eq(chatVoiceSessions.companyId, f.companyId)); + expect(await voiceCallReports(db, f.transport.callReport).load({ kind: "company", id: f.companyId })).toEqual([]); + // The resource helper alone has no post-configuration commit receipt; + // defaults/runtime completion belongs to the ordinary configure owner. + for (const receipt of receipts) expect(await chatHistoryActionBlocks(db, receipt.id)).toBe(true); }); it.each(["current", "legacy"])("moves only receipt-owned %s tools and hooks when callback origin and signing secret change", async proofVersion => { diff --git a/server/src/modules/external-agents/index.ts b/server/src/modules/external-agents/index.ts index 606d549b843..9d1b6e22c19 100644 --- a/server/src/modules/external-agents/index.ts +++ b/server/src/modules/external-agents/index.ts @@ -4,3 +4,5 @@ export { withExternalAdmissionGuard, withExternalExecutionAdmission, assertNoExt export { dotRunnerBroker, createDotRunnerMcpTools } from "./dot.js"; export { notifyExternalAgentSettlement, pendingMuseTerminalBindings, type ExternalAgentSettlementScope } from "./muse-terminal-settlement.js"; + +export { idleExternalAgentHistoryChecks } from "./idle-history-proof.js"; diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.ts index 3d6af2376f9..01bfe9f81d8 100644 --- a/server/src/routes/issues.ts +++ b/server/src/routes/issues.ts @@ -1,3 +1,4 @@ +import { confirmMcpAttachmentCleanup, recordMcpAttachmentCleanup, type McpAttachmentCleanup } from "../services/public-mcp/upload-cleanup.js"; import { notifyDeferredExecutionWork } from "../services/deferred-execution-notifications.js"; import { notifyNativeRecoveryNotices } from "../services/native-recovery-notice-notifications.js"; import { notifyNativeStatusReassessment } from "../services/native-status-reassessment-notifications.js"; @@ -16254,6 +16255,7 @@ export function issueRoutes( for (const attachment of attachments) { try { await storage.deleteObject(attachment.companyId, attachment.objectKey); + await recordMcpAttachmentCleanup(db, await confirmMcpAttachmentCleanup(db, storage, attachment)); } catch (err) { logger.warn( { err, issueId: id, attachmentId: attachment.id }, @@ -20138,8 +20140,10 @@ export function issueRoutes( if (!(await assertDeliverableMutationAllowedByRunContext(req, res, issue))) return; + let mcpCleanup: McpAttachmentCleanup | null = null; try { await storage.deleteObject(attachment.companyId, attachment.objectKey); + mcpCleanup = await confirmMcpAttachmentCleanup(db, storage, attachment); } catch (err) { logger.warn( { err, attachmentId }, @@ -20147,7 +20151,7 @@ export function issueRoutes( ); } - const removed = await svc.removeAttachment(attachmentId); + const removed = await svc.removeAttachment(attachmentId, { mcpCleanup }); if (!removed) { res.status(404).json({ error: "Attachment not found" }); return; diff --git a/server/src/services/chat-channels/discord/idle-history-proof.ts b/server/src/services/chat-channels/discord/idle-history-proof.ts new file mode 100644 index 00000000000..e90be06c0f6 --- /dev/null +++ b/server/src/services/chat-channels/discord/idle-history-proof.ts @@ -0,0 +1,25 @@ +/** Command acceptance is final history; a staged control publication remains an + * independent obligation and must settle before its command receipt is quiet. */ +export function completedDiscordAction(a: string): string { + return `${a}.kind = 'discord_native_command' AND ${a}.status = 'processed' + AND ${a}.payload->>'version' = '1' AND ${a}.payload->'invocation'->>'schema' = 'paperclip.discord.native-command.v1' + AND ${a}.payload->'invocation'->>'command' IN ('status', 'new', 'close') + AND ${a}.payload->'invocation'->>'digest' ~ '^[a-f0-9]{64}$' + AND ${a}.provider_action_id = 'discord-native-command:' || (${a}.payload->'invocation'->>'interactionId') + AND ${a}.result->>'kind' = 'discord_native_command_recorded' AND length(${a}.result->>'content') > 0 + AND EXISTS (SELECT 1 FROM chat_external_principals principal WHERE principal.id = ${a}.principal_id + AND principal.company_id = ${a}.company_id AND principal.provider = 'discord' + AND principal.external_id = ${a}.payload->'invocation'->>'actorExternalId') + AND ((${a}.conversation_id IS NULL AND ${a}.payload->'target' = 'null'::jsonb) + OR EXISTS (SELECT 1 FROM chat_conversations conversation WHERE conversation.id = ${a}.conversation_id + AND conversation.company_id = ${a}.company_id AND conversation.endpoint_id = ${a}.endpoint_id + AND conversation.id::text = ${a}.payload->'target'->>'conversationId' + AND conversation.issue_id::text = ${a}.payload->'target'->>'issueId' + AND conversation.session_generation::text = ${a}.payload->'target'->>'sessionGeneration')) + AND (${a}.result->'publicationId' = 'null'::jsonb OR EXISTS ( + SELECT 1 FROM chat_publications publication WHERE publication.id::text = ${a}.result->>'publicationId' + AND publication.company_id = ${a}.company_id AND publication.endpoint_id = ${a}.endpoint_id + AND publication.conversation_id = ${a}.conversation_id + AND ((publication.state = 'published' AND publication.published_at IS NOT NULL) + OR (publication.state IN ('failed', 'cancelled') AND publication.next_attempt_at IS NULL))))`; +} diff --git a/server/src/services/chat-channels/github/idle-history-proof.ts b/server/src/services/chat-channels/github/idle-history-proof.ts index 3dade037416..696460494f3 100644 --- a/server/src/services/chat-channels/github/idle-history-proof.ts +++ b/server/src/services/chat-channels/github/idle-history-proof.ts @@ -1,9 +1,65 @@ +function completedIngress(a: string): string { + return `(${a}.status = 'processed' AND ${a}.result->>'httpStatus' ~ '^2[0-9][0-9]$' AND ${a}.result->>'retryable' = 'false') + OR (${a}.status = 'failed' AND ${a}.result->>'retryable' = 'false' AND ( + ${a}.result->>'code' = 'github_webhook_ingress_payload_invalid' + OR ${a}.result->>'httpStatus' ~ '^4[0-9][0-9]$' + OR (jsonb_typeof(${a}.result->'attempts') = 'number' AND (${a}.result->>'attempts')::numeric >= 5))) + OR (${a}.status = 'cancelled' AND ${a}.result->>'retryable' = 'false' + AND ${a}.result->>'code' IN ('github_webhook_ingress_runtime_superseded', 'github_webhook_recovery_source_changed'))`; +} + export function completedGithubAction(a: string): string { - return `(${a}.kind = 'github_response_comment' AND ${a}.status = 'processed' AND ${a}.result->>'final' = 'true') - OR (${a}.kind = 'github_webhook_ingress' AND ( - (${a}.status = 'processed' AND ${a}.result->>'httpStatus' = '202' AND ${a}.result->>'retryable' = 'false') - OR (${a}.status = 'failed' AND ${a}.result->>'code' = 'github_webhook_ingress_payload_invalid' - AND ${a}.result->>'retryable' = 'false'))) - OR (${a}.kind = 'github_review_check' AND ${a}.status = 'processed' - AND ${a}.result->>'retryable' = 'false')`; + return `(${a}.kind = 'github_omission_navigation' AND ${a}.status = 'processed' AND ${a}.result IS NULL + AND ${a}.payload->>'version' = '1' AND ${a}.payload->>'preparedTextSha256' ~ '^[a-f0-9]{64}$' + AND ${a}.provider_action_id = 'github-omission-navigation:' || (${a}.payload->>'publicationId') + AND EXISTS (SELECT 1 FROM chat_publications publication JOIN heartbeat_runs run + ON run.id::text = ${a}.payload->>'runId' AND run.company_id = publication.company_id + AND run.native_issue_id = publication.issue_id AND run.runtime_mode = 'native' + JOIN native_run_finalizations finalization ON finalization.run_id = run.id AND finalization.company_id = run.company_id + AND finalization.issue_id = run.native_issue_id AND finalization.result_id::text = ${a}.payload->>'resultId' AND finalization.phase = 'committed' + WHERE publication.id::text = ${a}.payload->>'publicationId' AND publication.company_id = ${a}.company_id + AND publication.endpoint_id = ${a}.endpoint_id AND publication.conversation_id = ${a}.conversation_id + AND run.status = 'succeeded' AND run.finished_at IS NOT NULL AND run.cost_accounting_pending = false + AND ((publication.state = 'published' AND publication.published_at IS NOT NULL) + OR (publication.state IN ('failed', 'cancelled') AND publication.next_attempt_at IS NULL)))) + OR (${a}.kind = 'github_response_comment' AND ${a}.result->>'final' = 'true' + AND (${a}.status = 'processed' OR (${a}.status = 'cancelled' AND ${a}.result->>'code' + IN ('run_already_answered', 'passive_response_wait', 'response_deleted')))) + OR (${a}.kind = 'github_pr_response_comment' AND ${a}.status = 'processed' + AND (${a}.result IS NULL OR (${a}.result->>'final' = 'true' AND length(${a}.result->>'id') > 0 + AND length(${a}.result->>'url') > 0)) + AND EXISTS (SELECT 1 FROM chat_conversations conversation WHERE conversation.id = ${a}.conversation_id + AND conversation.company_id = ${a}.company_id AND conversation.endpoint_id = ${a}.endpoint_id) + AND NOT EXISTS (SELECT 1 FROM chat_actions request WHERE request.kind = 'github_response_comment' + AND request.company_id = ${a}.company_id AND request.endpoint_id = ${a}.endpoint_id + AND request.conversation_id = ${a}.conversation_id AND (request.result->>'final' = 'true') IS NOT TRUE)) + OR (${a}.kind = 'github_webhook_ingress' AND (${completedIngress(a)})) + OR (${a}.kind = 'github_review_check' AND ( + (${a}.status = 'processed' AND (${a}.result->>'state' IN ('manual_required', 'incomplete', 'error') + OR (${a}.result->>'code' = 'assessment_owns_check' AND EXISTS ( + SELECT 1 FROM chat_github_reviews review WHERE review.id::text = ${a}.result->>'reviewId' + AND review.company_id = ${a}.company_id AND review.endpoint_id = ${a}.endpoint_id AND review.assessment IS NOT NULL)))) + OR (${a}.status = 'cancelled' AND ${a}.result->>'code' IN ('connection_changed', 'stale_head') + AND (${a}.result->>'code' <> 'stale_head' OR NOT EXISTS ( + SELECT 1 FROM chat_github_reviews review JOIN chat_deliveries delivery + ON delivery.id::text = review.delivery_id AND delivery.endpoint_id = review.endpoint_id + WHERE review.company_id = ${a}.company_id AND review.endpoint_id = ${a}.endpoint_id + AND review.state IN ('queued', 'running') AND review.assessment IS NULL + AND delivery.normalized_event->'githubAutomatic'->'context'->>'deliveryId' = ${a}.payload->'event'->>'deliveryId'))))) + OR (${a}.kind = 'github_review_publication' AND ( + (${a}.status = 'processed' AND ( + (${a}.payload->>'operation' IN ('comment', 'update_comment', 'formal_review') + AND length(${a}.result->>'id') > 0 AND length(${a}.result->>'url') > 0) + OR (${a}.payload->>'operation' = 'assessment' AND EXISTS (SELECT 1 FROM chat_github_reviews review + WHERE review.id::text = ${a}.payload->>'reviewId' AND review.id::text = ${a}.result->>'reviewId' + AND review.company_id = ${a}.company_id AND review.endpoint_id = ${a}.endpoint_id + AND review.assessment IS NOT NULL AND review.check_url = ${a}.result->>'checkUrl')))) + OR (${a}.status = 'cancelled' AND ${a}.result->>'retryable' = 'false' + AND ${a}.result->>'code' IN ('publication_binding_unavailable', 'stale_head', 'authorization_changed')))) + OR (${a}.kind = 'github_webhook_recovery' AND ( + (${a}.status = 'cancelled' AND ${a}.result->>'code' IN ('not_eligible_for_recovery', 'source_changed_or_unavailable')) + OR (${a}.status = 'processed' AND EXISTS (SELECT 1 FROM chat_actions ingress + WHERE ingress.kind = 'github_webhook_ingress' AND ingress.company_id = ${a}.company_id + AND ingress.endpoint_id = ${a}.endpoint_id AND ingress.payload->>'deliveryId' = ${a}.payload->'original'->>'guid' + AND (${completedIngress('ingress')})))))`; } diff --git a/server/src/services/chat-channels/idle-history-proof.ts b/server/src/services/chat-channels/idle-history-proof.ts index 434087d5e38..992c71d8645 100644 --- a/server/src/services/chat-channels/idle-history-proof.ts +++ b/server/src/services/chat-channels/idle-history-proof.ts @@ -1,6 +1,11 @@ import { finishedIdleRun, finishedIdleWake } from "../idle-work-proof.js"; import { completedGithubAction } from "./github/idle-history-proof.js"; import { completedSlackAction } from "./slack/idle-history-proof.js"; +import { completedExpiredChatControl, completedQuestionFormAction } from "../chat-question-form-idle-proof.js"; +import { completedPhotonAction } from "./imessage-photon/idle-history-proof.js"; +import { completedDiscordAction } from "./discord/idle-history-proof.js"; +import { completedVoiceAction } from "../voice/idle-history-proof.js"; +import { completedTeamsAction } from "./microsoft-teams/idle-history-proof.js"; import { completedTelegramAction } from "./telegram/idle-history-proof.js"; /** A processed inbound receipt can still own an unconsumed wake. */ @@ -8,12 +13,18 @@ function completedAction(a: string): string { const wake = `EXISTS (SELECT 1 FROM agent_wakeup_requests w WHERE w.company_id = ${a}.company_id AND w.id = ${a}.id AND w.agent_id::text = ${a}.payload->>'agentId' AND w.payload->>'issueId' = ${a}.payload->>'issueId' AND ${finishedIdleWake('w')})`; - return `jsonb_typeof(${a}.payload) = 'object' AND jsonb_typeof(${a}.result) = 'object' + return `jsonb_typeof(${a}.payload) = 'object' AND EXISTS (SELECT 1 FROM chat_endpoints e WHERE e.id = ${a}.endpoint_id AND e.company_id = ${a}.company_id) AND (${a}.delivery_id IS NULL OR EXISTS (SELECT 1 FROM chat_deliveries d WHERE d.id = ${a}.delivery_id AND d.company_id = ${a}.company_id AND d.endpoint_id = ${a}.endpoint_id)) AND CASE - WHEN ${a}.kind = 'inbound_wakeup' THEN ${a}.status IN ('processed', 'failed') AND ${wake} + WHEN ${a}.kind = 'inbound_wakeup' THEN (${a}.status IN ('processed', 'failed') AND ${wake}) + OR (${a}.status = 'failed' AND ${a}.result->>'code' = 'inbound_wakeup_delivery_rejected' + AND EXISTS (SELECT 1 FROM chat_deliveries rejected WHERE rejected.id = ${a}.delivery_id + AND rejected.company_id = ${a}.company_id AND rejected.endpoint_id = ${a}.endpoint_id + AND rejected.state IN ('filtered', 'failed') AND rejected.next_attempt_at IS NULL)) + WHEN ${a}.kind IN ('question_form_open', 'question_form_submit', 'question_form_open_attempt', 'question_answer') + THEN (${completedQuestionFormAction(a)}) WHEN ${a}.kind = 'interaction_wakeup' THEN ${a}.status = 'processed' AND (${a}.result->>'code' = 'interaction_wakeup_no_longer_applicable' OR EXISTS ( SELECT 1 FROM agent_wakeup_requests w WHERE w.company_id = ${a}.company_id @@ -28,40 +39,64 @@ function completedAction(a: string): string { WHEN ${a}.kind = 'receipt_reaction' THEN (${a}.status = 'processed' AND jsonb_typeof(${a}.result->'attempts') = 'number' AND (${a}.result->>'attempts')::numeric > 0) OR (${a}.status = 'cancelled' AND ${a}.result->>'code' IN ('receipt_reaction_superseded_by_terminal_publication', 'receipt_reaction_runtime_superseded')) - WHEN ${a}.kind = 'confirmation_response' THEN ${a}.status = 'processed' + WHEN ${a}.kind = 'confirmation_response' THEN (${completedExpiredChatControl(a)}) OR (${a}.status = 'processed' AND ${a}.result->>'code' IN ('confirmation_resolution_committed_by_provider', 'confirmation_resolution_reconciled') AND EXISTS (SELECT 1 FROM issue_thread_interactions interaction WHERE interaction.id::text = ${a}.result->>'interactionId' AND interaction.company_id = ${a}.company_id AND interaction.status = ${a}.result->>'interactionStatus' - AND interaction.resolved_at IS NOT NULL) + AND interaction.resolved_at IS NOT NULL)) + WHEN ${a}.kind = 'board_send_rejected' THEN ${a}.status = 'processed' + AND ${a}.result->>'code' = 'chat_board_send_attachments_already_bound' + AND ${a}.payload->>'schemaVersion' = '1' AND length(${a}.payload->>'requestedByUserId') > 0 + AND jsonb_typeof(${a}.payload->'attachmentIds') = 'array' AND jsonb_array_length(CASE WHEN jsonb_typeof(${a}.payload->'attachmentIds') = 'array' THEN ${a}.payload->'attachmentIds' ELSE '[]'::jsonb END) BETWEEN 1 AND 20 + AND NOT EXISTS (SELECT 1 FROM jsonb_array_elements_text(CASE WHEN jsonb_typeof(${a}.payload->'attachmentIds') = 'array' THEN ${a}.payload->'attachmentIds' ELSE '[]'::jsonb END) rejected_id(value) + WHERE rejected_id.value !~ '^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$') + AND EXISTS (SELECT 1 FROM chat_conversations conversation WHERE conversation.id = ${a}.conversation_id + AND conversation.company_id = ${a}.company_id AND conversation.endpoint_id = ${a}.endpoint_id + AND conversation.issue_id::text = ${a}.payload->>'issueId' + AND ${a}.provider_action_id = 'board-send-rejected:' || conversation.id::text || ':' || (${a}.payload->>'idempotencyKey')) WHEN ${a}.kind = 'task_control_authorization' THEN ${a}.status IN ('processed', 'cancelled') AND EXISTS (SELECT 1 FROM chat_publications p WHERE p.company_id = ${a}.company_id AND p.endpoint_id = ${a}.endpoint_id AND p.conversation_id = ${a}.conversation_id AND ${a}.provider_action_id = 'task-control-authorization:' || p.id::text AND ((p.state = 'published' AND p.published_at IS NOT NULL) OR p.state = 'cancelled')) - ELSE (${[completedGithubAction, completedSlackAction, completedTelegramAction] + ELSE (${[completedGithubAction, completedSlackAction, completedTelegramAction, completedTeamsAction, completedPhotonAction, completedDiscordAction, completedVoiceAction] .map(proof => `(${proof(a)})`).join(' OR ')}) END`; } export const idleChatHistoryChecks = [ - { reason: "chat_deliveries", query: `WITH action_outcomes AS MATERIALIZED ( - SELECT a.delivery_id, a.kind, (${completedAction('a')}) IS TRUE AS completed - FROM chat_actions a WHERE a.delivery_id IS NOT NULL - ), action_receipts AS ( - SELECT delivery_id, bool_and(completed) AS completed, - bool_or(kind = 'inbound_wakeup' AND completed) AS consumed - FROM action_outcomes GROUP BY delivery_id - ) SELECT 1 FROM chat_deliveries d LEFT JOIN action_receipts receipt ON receipt.delivery_id = d.id WHERE ( + { reason: "chat_deliveries", query: `WITH candidate_deliveries AS MATERIALIZED ( + SELECT d.id, d.company_id, d.endpoint_id, d.state, d.event_kind FROM chat_deliveries d WHERE ( EXISTS (SELECT 1 FROM chat_endpoints e WHERE e.id = d.endpoint_id AND e.company_id = d.company_id) AND jsonb_typeof(d.normalized_event) = 'object' AND d.normalized_event->>'providerEventId' = d.provider_event_id AND d.normalized_event->>'kind' = d.event_kind AND d.event_kind IN ('mention', 'message', 'direct_message', 'message_updated', 'message_deleted', 'message_restored', 'reaction_added', 'reaction_removed', 'action', 'modal_submitted', 'modal_closed', 'slash_command', 'file_shared', 'installation', 'uninstallation', 'unknown') - AND (d.state <> 'processed' OR d.event_kind NOT IN ('mention', 'message', 'direct_message') - OR receipt.consumed IS TRUE) AND d.next_attempt_at IS NULL AND (d.state = 'failed' OR (d.state IN ('processed', 'filtered') AND d.processed_at IS NOT NULL)) AND (d.conversation_id IS NULL OR EXISTS (SELECT 1 FROM chat_conversations c WHERE c.id = d.conversation_id AND c.company_id = d.company_id AND c.endpoint_id = d.endpoint_id)) - AND COALESCE(receipt.completed, true)) IS NOT TRUE` }, +) IS TRUE + ), action_outcomes AS MATERIALIZED ( + SELECT a.delivery_id, a.kind, (${completedAction('a')}) IS TRUE AS completed + FROM chat_actions a JOIN candidate_deliveries candidate ON candidate.id = a.delivery_id + ), action_receipts AS MATERIALIZED ( + SELECT delivery_id, bool_and(completed) AS completed, + bool_or(kind = 'inbound_wakeup' AND completed) AS consumed + FROM action_outcomes GROUP BY delivery_id + ) SELECT 1 FROM chat_deliveries d WHERE ( + EXISTS (SELECT 1 FROM chat_endpoints e WHERE e.id = d.endpoint_id AND e.company_id = d.company_id) + AND jsonb_typeof(d.normalized_event) = 'object' + AND d.normalized_event->>'providerEventId' = d.provider_event_id + AND d.normalized_event->>'kind' = d.event_kind + AND d.event_kind IN ('mention', 'message', 'direct_message', 'message_updated', 'message_deleted', 'message_restored', + 'reaction_added', 'reaction_removed', 'action', 'modal_submitted', 'modal_closed', 'slash_command', 'file_shared', 'installation', 'uninstallation', 'unknown') + AND d.next_attempt_at IS NULL AND (d.state = 'failed' OR (d.state IN ('processed', 'filtered') AND d.processed_at IS NOT NULL)) + AND (d.conversation_id IS NULL OR EXISTS (SELECT 1 FROM chat_conversations c WHERE c.id = d.conversation_id + AND c.company_id = d.company_id AND c.endpoint_id = d.endpoint_id)) +) IS NOT TRUE + UNION ALL SELECT 1 FROM candidate_deliveries d + WHERE d.id IN (SELECT receipt.delivery_id FROM action_receipts receipt WHERE NOT receipt.completed) + OR (d.state = 'processed' AND d.event_kind IN ('mention', 'message', 'direct_message') + AND d.id NOT IN (SELECT receipt.delivery_id FROM action_receipts receipt WHERE receipt.consumed))` }, { reason: "chat_publications", query: `SELECT 1 FROM chat_publications p WHERE ( (p.state = 'published' AND p.published_at IS NOT NULL AND p.redacted_error IS NULL OR (p.state IN ('failed', 'cancelled') AND p.next_attempt_at IS NULL)) diff --git a/server/src/services/chat-channels/imessage-photon/events.ts b/server/src/services/chat-channels/imessage-photon/events.ts index 75f6d2cc92b..882a57e7a7a 100644 --- a/server/src/services/chat-channels/imessage-photon/events.ts +++ b/server/src/services/chat-channels/imessage-photon/events.ts @@ -1,11 +1,10 @@ +import { stagePhotonResponseNotice } from "./response-notices.js"; import { notifyChatReceiver } from "../runtime/receiver-work-notifications.js"; import type { EndpointRow, DbTransaction, RuntimeContext, LifecycleRuntimeFence } from "../../chat-channels/runtime-types.js"; -import { notifyChatPublicationWork } from "../../chat-work-notifications.js"; import { nativeSha256 } from "../../native-runtime/canonical.js"; -import { projectSafeChatPublicationText } from "../../chat-publication-projection.js"; import { PhotonAnswerValidationError, nativePhotonInteraction, photonResponseCommand, parsePhotonQuestionAnswer, type PhotonPromptReceipt, type PhotonInteractionBinding, type PhotonDraft } from "../../photon/interactions.js"; import { validateNativeQuestionResponseInput } from "../../native-runtime/native-question-bridge.js"; import type { AskUserQuestionsAnswer, AskUserQuestionsInteraction, IssueThreadInteraction } from "@paperclipai/shared"; @@ -368,41 +367,9 @@ export function createPhotonEvents(db: Db, options: PhotonEventsOptions) { } const enqueueResponse = async (key: string, text: string) => { await assertCurrent(); - await db.transaction(async (tx) => { - await tx - .insert(chatActions) - .values({ - companyId: endpoint.companyId, - endpointId: endpoint.id, - conversationId: conversation.id, - kind: "photon_response_notice", - providerActionId: `photon-notice:${key}`, - payload: { - reference, - questionIndex, - publicationId: binding.publicationId, - }, - status: "processed", - }) - .onConflictDoNothing(); - await notifyChatPublicationWork(tx); - await tx - .insert(chatPublications) - .values({ - companyId: endpoint.companyId, - endpointId: endpoint.id, - conversationId: conversation.id, - issueId: conversation.issueId, - idempotencyKey: key, - payload: { - ...publication.payload, - card: undefined, - text: projectSafeChatPublicationText(text), - }, - state: "pending", - }) - .onConflictDoNothing(); - }); + await db.transaction(tx => stagePhotonResponseNotice(tx, { + publication, reference, questionIndex, key, text, + })); scheduleMessageProcessing(() => processPendingPublications().then(() => undefined), ); diff --git a/server/src/services/chat-channels/imessage-photon/idle-history-proof.ts b/server/src/services/chat-channels/imessage-photon/idle-history-proof.ts new file mode 100644 index 00000000000..2e10387d722 --- /dev/null +++ b/server/src/services/chat-channels/imessage-photon/idle-history-proof.ts @@ -0,0 +1,58 @@ +/** Photon tokens and immutable provenance receipts do not own a turn once their + * exact canonical owner has settled. Receiver ownership remains a separate gate. */ +export function completedPhotonAction(a: string): string { + const publication = (id: string) => `EXISTS (SELECT 1 FROM chat_publications publication + WHERE publication.id::text = ${id} AND publication.company_id = ${a}.company_id + AND publication.endpoint_id = ${a}.endpoint_id AND publication.conversation_id = ${a}.conversation_id + AND ((publication.state = 'published' AND publication.published_at IS NOT NULL) + OR (publication.state IN ('failed', 'cancelled') AND publication.next_attempt_at IS NULL)))`; + return `(${a}.kind = 'photon_interaction' AND (${a}.status IN ('issued', 'processed') OR (${a}.status = 'expired' AND ${a}.result->>'code' = 'interaction_resolved_elsewhere')) + AND ${a}.payload->>'version' = '1' AND length(${a}.payload->>'reference') > 0 + AND ${a}.provider_action_id = 'photon:' || (${a}.payload->>'reference') + AND EXISTS (SELECT 1 FROM issue_thread_interactions interaction JOIN chat_publications publication + ON publication.id::text = ${a}.payload->>'publicationId' AND publication.company_id = interaction.company_id + AND publication.issue_id = interaction.issue_id AND publication.payload->>'interactionId' = interaction.id::text + JOIN chat_conversations conversation ON conversation.id = publication.conversation_id + AND conversation.company_id = interaction.company_id AND conversation.issue_id = interaction.issue_id + WHERE interaction.id::text = ${a}.payload->>'interactionId' AND interaction.company_id = ${a}.company_id + AND interaction.kind IN ('ask_user_questions', 'request_confirmation') + AND interaction.status IN ('answered', 'accepted', 'rejected', 'cancelled', 'expired') AND interaction.resolved_at IS NOT NULL + AND publication.endpoint_id = ${a}.endpoint_id AND conversation.id = ${a}.conversation_id + AND conversation.session_generation::text = ${a}.payload->>'sessionGeneration')) + OR (${a}.kind = 'photon_publication_retry' AND ${a}.status = 'processed' + AND length(${a}.result->>'authorizedByUserId') > 0 AND ${a}.payload->>'attempt' ~ '^[1-9][0-9]*$' + AND ${a}.provider_action_id = 'photon-retry:' || (${a}.payload->>'publicationId') || ':' || (${a}.payload->>'attempt') + AND ${publication(`${a}.payload->>'publicationId'`)}) + OR (${a}.kind = 'photon_response_notice' AND ${a}.status = 'processed' AND ${a}.result IS NULL + AND length(${a}.payload->>'reference') > 0 AND ${a}.payload->>'questionIndex' ~ '^[0-9]+$' + AND EXISTS (SELECT 1 FROM chat_publications notice WHERE notice.company_id = ${a}.company_id + AND notice.endpoint_id = ${a}.endpoint_id AND notice.conversation_id = ${a}.conversation_id + AND ${a}.provider_action_id = 'photon-notice:' || notice.idempotency_key + AND ((notice.state = 'published' AND notice.published_at IS NOT NULL) + OR (notice.state IN ('failed', 'cancelled') AND notice.next_attempt_at IS NULL))) + AND EXISTS (SELECT 1 FROM chat_publications source WHERE source.id::text = ${a}.payload->>'publicationId' + AND source.company_id = ${a}.company_id AND source.endpoint_id = ${a}.endpoint_id + AND source.conversation_id = ${a}.conversation_id)) + OR (${a}.kind IN ('attachment_derivative', 'attachment_companion') AND ${a}.status = 'processed' AND ${a}.result IS NULL + AND EXISTS (SELECT 1 FROM chat_endpoints endpoint WHERE endpoint.id = ${a}.endpoint_id + AND endpoint.company_id = ${a}.company_id AND endpoint.provider = 'imessage-photon') + AND ${a}.delivery_id IS NOT NULL + AND ((${a}.kind = 'attachment_derivative' AND ${a}.payload->>'kind' = 'heif_jpeg_preview') + OR (${a}.kind = 'attachment_companion' AND ${a}.payload->>'kind' = 'live_photo_video')) + AND ${a}.payload->>'originalAttachmentId' ~ '^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$' + AND ${a}.payload->>'derivativeAttachmentId' ~ '^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$' + AND ${a}.payload->>'originalAttachmentId' <> ${a}.payload->>'derivativeAttachmentId' + AND ${a}.payload->>'originalSha256' ~ '^[a-f0-9]{64}$' AND ${a}.payload->>'derivativeSha256' ~ '^[a-f0-9]{64}$' + AND ${a}.provider_action_id = 'photon-' || (${a}.payload->>'kind') || ':' || ${a}.delivery_id::text || ':' || (${a}.payload->>'originalAttachmentId') + AND NOT EXISTS (SELECT 1 FROM issue_attachments attachment WHERE attachment.id::text IN + (${a}.payload->>'originalAttachmentId', ${a}.payload->>'derivativeAttachmentId') + AND (attachment.company_id <> ${a}.company_id OR NOT EXISTS (SELECT 1 FROM assets asset + WHERE asset.id = attachment.asset_id AND asset.company_id = ${a}.company_id AND asset.sha256 = CASE + WHEN attachment.id::text = ${a}.payload->>'originalAttachmentId' THEN ${a}.payload->>'originalSha256' + ELSE ${a}.payload->>'derivativeSha256' END))) + AND NOT EXISTS (SELECT 1 FROM issue_attachments original JOIN issue_attachments derivative + ON derivative.id::text = ${a}.payload->>'derivativeAttachmentId' + WHERE original.id::text = ${a}.payload->>'originalAttachmentId' + AND (original.company_id <> derivative.company_id OR original.issue_id <> derivative.issue_id + OR original.issue_comment_id IS DISTINCT FROM derivative.issue_comment_id)))`; +} diff --git a/server/src/services/chat-channels/imessage-photon/response-notices.ts b/server/src/services/chat-channels/imessage-photon/response-notices.ts new file mode 100644 index 00000000000..18a7c65b3aa --- /dev/null +++ b/server/src/services/chat-channels/imessage-photon/response-notices.ts @@ -0,0 +1,27 @@ +import { chatActions, chatPublications, type Db } from "@paperclipai/db"; +import { notifyChatPublicationWork } from "../../chat-work-notifications.js"; +import { projectSafeChatPublicationText } from "../../chat-publication-projection.js"; + +type Tx = Parameters[0]>[0]; + +/** Keep the notice marker and its outbox row in the same owner transaction. */ +export async function stagePhotonResponseNotice(tx: Tx, input: { + publication: typeof chatPublications.$inferSelect; + reference: string; + questionIndex: number; + key: string; + text: string; +}) { + const { publication, reference, questionIndex, key, text } = input; + await tx.insert(chatActions).values({ + companyId: publication.companyId, endpointId: publication.endpointId, conversationId: publication.conversationId, + kind: "photon_response_notice", providerActionId: `photon-notice:${key}`, + payload: { reference, questionIndex, publicationId: publication.id }, status: "processed", + }).onConflictDoNothing(); + await notifyChatPublicationWork(tx); + await tx.insert(chatPublications).values({ + companyId: publication.companyId, endpointId: publication.endpointId, conversationId: publication.conversationId, + issueId: publication.issueId, idempotencyKey: key, + payload: { ...publication.payload, card: undefined, text: projectSafeChatPublicationText(text) }, state: "pending", + }).onConflictDoNothing(); +} diff --git a/server/src/services/chat-channels/microsoft-teams/idle-history-proof.ts b/server/src/services/chat-channels/microsoft-teams/idle-history-proof.ts new file mode 100644 index 00000000000..9d90d550623 --- /dev/null +++ b/server/src/services/chat-channels/microsoft-teams/idle-history-proof.ts @@ -0,0 +1,35 @@ +/** Teams keeps immutable provider-effect and recipient-intent receipts after + * delivery. Only the exact closed transfer/publication retires that authority. */ +export function completedTeamsAction(a: string): string { + return `${a}.status = 'processed' AND ${a}.kind IN ('teams_file_effect_intent', 'teams_file_consent', 'teams_board_file_intent') + AND EXISTS (SELECT 1 FROM chat_teams_file_transfers transfer JOIN chat_publications publication + ON publication.id = transfer.publication_id AND publication.company_id = transfer.company_id + WHERE transfer.company_id = ${a}.company_id AND transfer.endpoint_id = ${a}.endpoint_id + AND transfer.conversation_id = ${a}.conversation_id + AND publication.id::text = ${a}.payload->>'publicationId' + AND publication.endpoint_id = ${a}.endpoint_id AND publication.conversation_id = ${a}.conversation_id + AND transfer.phase IN ('delivered', 'declined', 'expired', 'cancelled') + AND transfer.attempt_id IS NULL AND transfer.attempt_expires_at IS NULL + AND ((publication.state = 'published' AND publication.published_at IS NOT NULL) + OR (publication.state IN ('failed', 'cancelled') AND publication.next_attempt_at IS NULL)) + AND CASE WHEN ${a}.kind = 'teams_file_effect_intent' THEN + ${a}.payload->>'schema' = 'paperclip.teams.file-effect-intent.v1' + AND ${a}.payload->>'transferId' = transfer.id::text + AND ${a}.provider_action_id = 'teams-file-effect:' || (${a}.payload->>'attemptId') + AND ${a}.payload->>'stage' IN ('consent', 'upload', 'file_info') + WHEN ${a}.kind = 'teams_file_consent' THEN + ${a}.payload->>'schema' = 'paperclip.teams.consent-receipt.v1' + AND ${a}.payload->>'transferId' = transfer.id::text + AND ${a}.payload->>'receiptDigest' = transfer.response_digest + AND ${a}.payload->>'action' IN ('accept', 'decline') + AND ${a}.principal_id IS NOT DISTINCT FROM transfer.principal_id + ELSE ${a}.payload->>'schema' = 'paperclip.teams.board-file-intent.v1' + AND ${a}.provider_action_id = 'teams_board_file_intent:' || publication.id::text + AND ${a}.payload->>'issueId' = transfer.issue_id::text + AND ${a}.payload->>'attachmentId' = transfer.attachment_id::text + AND ${a}.payload->>'commentId' = publication.comment_id::text + AND ${a}.payload->>'sourceScopeSha256' ~ '^[a-f0-9]{64}$' + AND ${a}.payload->>'recipientBindingSha256' ~ '^[a-f0-9]{64}$' + AND ${a}.payload->>'byteIdentitySha256' ~ '^[a-f0-9]{64}$' + AND ${a}.payload->>'commentBodySha256' ~ '^[a-f0-9]{64}$' END)`; +} diff --git a/server/src/services/chat-channels/slack/idle-history-proof.ts b/server/src/services/chat-channels/slack/idle-history-proof.ts index f57e9950f12..1db0169861a 100644 --- a/server/src/services/chat-channels/slack/idle-history-proof.ts +++ b/server/src/services/chat-channels/slack/idle-history-proof.ts @@ -1,3 +1,63 @@ +/** Connector rows below are immutable authority/confirmed-call receipts. Their + * own downstream publication, interaction and run rows keep independent gates. */ +function completedSlackConnectorAction(a: string): string { + const principal = `EXISTS (SELECT 1 FROM chat_external_principals principal WHERE principal.id = ${a}.principal_id + AND principal.company_id = ${a}.company_id AND principal.provider = 'slack')`; + const issue = `EXISTS (SELECT 1 FROM issues issue WHERE issue.id::text = ${a}.payload->>'issueId' + AND issue.company_id = ${a}.company_id AND (${a}.conversation_id IS NULL OR EXISTS ( + SELECT 1 FROM chat_conversations conversation WHERE conversation.id = ${a}.conversation_id + AND conversation.company_id = issue.company_id AND conversation.issue_id = issue.id + AND conversation.endpoint_id = ${a}.endpoint_id)))`; + return `(${a}.kind = 'slack_private_source' AND ${a}.status = 'processed' AND ${a}.result IS NULL + AND ${issue} AND ${principal} AND length(${a}.payload->>'channelId') > 0 + AND length(${a}.payload->>'requesterId') > 0 + AND ${a}.provider_action_id = 'slack-private:' || (${a}.payload->>'issueId') || ':' + || (${a}.payload->>'channelId') || ':' || (${a}.payload->>'requesterId')) + OR (${a}.kind = 'slack_connect' AND ${a}.status = 'processed' + AND ${a}.payload->>'version' = '1' AND ${a}.result->>'code' = 'slack_identity_discovered' + AND length(${a}.payload->>'channelId') > 0 AND ${principal} + AND EXISTS (SELECT 1 FROM chat_external_principals principal WHERE principal.id = ${a}.principal_id + AND principal.external_id = ${a}.payload->>'userId')) + OR (${a}.kind IN ('slack_test_message', 'slack_test_message_invitation') AND ${a}.status = 'processed' + AND length(${a}.payload->>'userId') > 0 AND ${a}.payload->>'slackUserId' ~ '^[UW][A-Z0-9]+$' + AND CASE WHEN ${a}.kind = 'slack_test_message' THEN ${a}.result->>'slackUrl' ~ '^https://app[.]slack[.]com/client/[^/]+/D[A-Z0-9]+$' + ELSE ${principal} AND ${a}.result->>'expiresAt' ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}T' END) + OR (${a}.kind = 'slack_managed_reconnect' AND ${a}.status = 'processed' AND ${a}.result IS NULL + AND ${a}.provider_action_id LIKE 'slack_managed_reconnect:%' + AND length(${a}.payload->>'registrationRequestId') > 0 + AND EXISTS (SELECT 1 FROM chat_slack_manager_grants grant_record WHERE grant_record.id::text = ${a}.payload->>'grantId' + AND grant_record.company_id = ${a}.company_id)) + OR (${a}.kind = 'slack_tool_write' AND ${a}.status = 'processed' AND jsonb_typeof(${a}.result) = 'object' + AND ${a}.payload->>'name' IN ('slack_open_dm', 'slack_post_message', 'slack_update_message', 'slack_delete_message', + 'slack_add_reaction', 'slack_remove_reaction', 'slack_add_pin', 'slack_remove_pin', 'slack_add_bookmark', + 'slack_remove_bookmark', 'slack_create_canvas', 'slack_edit_canvas', 'slack_replace_canvas_section', + 'slack_share_list', 'slack_edit_list', 'slack_create_list', 'slack_create_list_item', 'slack_update_list_item', + 'slack_invite', 'slack_create_channel', 'slack_upload_file') + AND ${a}.payload->>'hash' ~ '^[a-f0-9]{64}$' AND jsonb_typeof(${a}.payload->'args') = 'object' + AND length(${a}.payload->>'userId') > 0 AND length(${a}.payload->>'revision') > 0 + AND ${a}.provider_action_id = 'slack-tool:' || (${a}.payload->'binding'->>'issueId') || ':' || (${a}.payload->'args'->>'idempotencyKey') + AND EXISTS (SELECT 1 FROM tool_invocations invocation JOIN heartbeat_runs run ON run.id = invocation.run_id + AND run.company_id = invocation.company_id AND run.agent_id = invocation.agent_id + WHERE invocation.id::text = ${a}.payload->>'invocationId' AND invocation.company_id = ${a}.company_id + AND invocation.run_id::text = ${a}.payload->'binding'->>'runId' + AND invocation.agent_id::text = ${a}.payload->'binding'->>'agentId' + AND invocation.issue_id::text = ${a}.payload->'binding'->>'issueId' + AND ${a}.payload->'binding'->>'companyId' = ${a}.company_id::text)) + OR (${a}.kind = 'slack_board_message' AND ${issue} AND EXISTS ( + SELECT 1 FROM issue_comments comment JOIN chat_publications publication + ON publication.comment_id = comment.id AND publication.company_id = comment.company_id + WHERE comment.id::text = ${a}.payload->>'commentId' AND comment.company_id = ${a}.company_id + AND comment.issue_id::text = ${a}.payload->>'issueId' AND comment.author_user_id = ${a}.payload->>'userId' + AND publication.endpoint_id = ${a}.endpoint_id AND publication.conversation_id = ${a}.conversation_id + AND ((publication.state = 'published' AND publication.published_at IS NOT NULL) + OR (publication.state IN ('failed', 'cancelled') AND publication.next_attempt_at IS NULL))) + AND ((${a}.status = 'cancelled' AND ${a}.result->>'code' = 'slack_board_work_not_authorized') + OR (${a}.status = 'processed' AND (${a}.result IS NULL OR (${a}.result->>'resumeApplied' = 'true' + AND EXISTS (SELECT 1 FROM agent_wakeup_requests wake WHERE wake.company_id = ${a}.company_id + AND wake.idempotency_key = 'slack-board-comment:' || ${a}.id::text AND wake.finished_at IS NOT NULL + AND wake.status IN ('completed', 'failed', 'cancelled', 'skipped', 'timed_out')))))))`; +} + export function completedSlackAction(a: string): string { return `(${a}.kind = 'slack_session_sync' AND ( (${a}.status = 'processed' AND ${a}.result->>'outcome' IN ('updated', 'unsupported') @@ -12,5 +72,6 @@ export function completedSlackAction(a: string): string { AND ${a}.result->>'code' IN ('slack_session_stop_cancelled', 'slack_session_stop_target_superseded', 'slack_session_stop_no_current_work') AND NOT EXISTS (SELECT 1 FROM heartbeat_runs r WHERE r.id::text = ${a}.result->>'runId' AND (r.company_id <> ${a}.company_id OR r.status NOT IN ('succeeded', 'failed', 'cancelled', 'timed_out', 'interrupted') - OR r.finished_at IS NULL OR r.cost_accounting_pending)))`; + OR r.finished_at IS NULL OR r.cost_accounting_pending))) + OR (${completedSlackConnectorAction(a)})`; } diff --git a/server/src/services/chat-channels/telegram/idle-history-proof.ts b/server/src/services/chat-channels/telegram/idle-history-proof.ts index bd37617d627..87dea9045e1 100644 --- a/server/src/services/chat-channels/telegram/idle-history-proof.ts +++ b/server/src/services/chat-channels/telegram/idle-history-proof.ts @@ -1,5 +1,13 @@ export function completedTelegramAction(a: string): string { - return `(${a}.kind = 'telegram_maintenance' AND ( + return `(${a}.kind = 'telegram_stop_subscription' AND ${a}.status = 'processed' + AND ${a}.result->>'code' = 'telegram_stop_subscription_confirmed' AND ${a}.payload->>'version' = '1' + AND ${a}.payload->>'botUserId' ~ '^[1-9][0-9]{0,15}$' + AND ${a}.payload->>'credentialFingerprint' ~ '^[a-f0-9]{64}$' + AND ${a}.payload->>'webhookUrlSha256' ~ '^[a-f0-9]{64}$' + AND ${a}.payload->>'runtimeGeneration' ~ '^[0-9]+$' + AND ${a}.provider_action_id = 'telegram-stop-subscription:' || (${a}.payload->>'runtimeGeneration') || ':' + || (${a}.payload->>'credentialFingerprint') || ':' || (${a}.payload->>'webhookUrlSha256')) + OR (${a}.kind = 'telegram_maintenance' AND ( (${a}.status = 'processed' AND (${a}.result->>'providerConfirmed' = 'true' OR ${a}.result->>'code' = 'telegram_stop_subscription_confirmed')) OR (${a}.status = 'cancelled' AND ${a}.result->>'code' = 'telegram_maintenance_credentials_superseded') diff --git a/server/src/services/chat-question-form-idle-proof.ts b/server/src/services/chat-question-form-idle-proof.ts new file mode 100644 index 00000000000..b82cbb36c82 --- /dev/null +++ b/server/src/services/chat-question-form-idle-proof.ts @@ -0,0 +1,54 @@ +/** Form-open tokens stay issued: resolving the canonical question revokes their + * authority. A processed submission additionally needs its exact answer receipt. */ +export function completedQuestionFormAction(a: string): string { + const resolved = `EXISTS (SELECT 1 FROM issue_thread_interactions interaction + JOIN chat_conversations conversation ON conversation.issue_id = interaction.issue_id AND conversation.company_id = interaction.company_id + JOIN chat_publications publication ON publication.id::text = ${a}.payload->>'publicationId' + AND publication.company_id = interaction.company_id AND publication.issue_id = interaction.issue_id + AND publication.conversation_id = conversation.id AND publication.endpoint_id = conversation.endpoint_id + WHERE interaction.id::text = ${a}.payload->>'interactionId' AND interaction.company_id = ${a}.company_id + AND (interaction.kind = 'ask_user_questions' OR (interaction.kind = 'request_item_verdicts' AND EXISTS ( + SELECT 1 FROM chat_endpoints endpoint WHERE endpoint.id = ${a}.endpoint_id AND endpoint.company_id = ${a}.company_id AND endpoint.provider = 'slack'))) AND interaction.status IN ('answered', 'cancelled', 'expired') + AND interaction.resolved_at IS NOT NULL AND conversation.id = ${a}.conversation_id + AND conversation.endpoint_id = ${a}.endpoint_id AND publication.payload->>'interactionId' = interaction.id::text)`; + return `(${a}.kind IN ('question_form_open', 'question_form_submit') AND ${a}.payload->>'version' = '1' + AND ${resolved} AND ( + (${a}.kind = 'question_form_open' AND (${a}.status = 'issued' OR (${a}.status = 'expired' + AND ${a}.result->>'code' IN ('question_form_submitted', 'interaction_resolved_elsewhere'))) AND EXISTS ( + SELECT 1 FROM chat_actions submit WHERE submit.company_id = ${a}.company_id AND submit.endpoint_id = ${a}.endpoint_id + AND submit.conversation_id = ${a}.conversation_id AND submit.kind = 'question_form_submit' + AND submit.provider_action_id = ${a}.payload->>'formActionId' + AND submit.payload->>'formActionId' = submit.provider_action_id + AND submit.payload->>'interactionId' = ${a}.payload->>'interactionId' + AND submit.payload->>'publicationId' = ${a}.payload->>'publicationId' + AND submit.payload->>'expiresAt' = ${a}.payload->>'expiresAt' + AND (submit.status = 'issued' OR (submit.status = 'expired' AND submit.result->>'code' = 'interaction_resolved_elsewhere') OR (submit.status = 'processed' + AND submit.result->>'code' = 'question_form_answered' + AND submit.result->>'interactionId' = submit.payload->>'interactionId')))) + OR (${a}.kind = 'question_form_submit' AND ${a}.payload->>'formActionId' = ${a}.provider_action_id + AND (${a}.status = 'issued' OR (${a}.status = 'expired' AND ${a}.result->>'code' = 'interaction_resolved_elsewhere') OR (${a}.status = 'processed' AND ${a}.result->>'code' = 'question_form_answered' + AND ${a}.result->>'interactionId' = ${a}.payload->>'interactionId'))))) + OR (${a}.kind = 'question_answer' AND ${resolved} + AND (${a}.status = 'issued' OR (${a}.status = 'processed' + AND ${a}.result->>'interactionId' = ${a}.payload->>'interactionId') + OR (${a}.status = 'expired' AND ${a}.result->>'code' IN ('interaction_resolved_by_sibling_action', 'interaction_resolved_elsewhere')))) + OR (${a}.kind = 'question_form_open_attempt' AND ${a}.status = 'processed' + AND ${a}.result->>'code' = 'question_form_opened')`; +} + +/** The terminal interaction writer expires its unused callback tokens after a + * board/sibling resolution. Expiry alone never proves canonical completion. */ +export function completedExpiredChatControl(a: string): string { + return `${a}.kind = 'confirmation_response' AND ${a}.status = 'expired' + AND ${a}.result->>'code' = 'interaction_resolved_elsewhere' AND ${a}.payload->>'version' = '1' + AND ${a}.payload->>'decision' IN ('accept', 'reject') AND EXISTS ( + SELECT 1 FROM issue_thread_interactions interaction JOIN chat_publications publication + ON publication.id::text = ${a}.payload->>'publicationId' AND publication.company_id = interaction.company_id + AND publication.issue_id = interaction.issue_id AND publication.payload->>'interactionId' = interaction.id::text + JOIN chat_conversations conversation ON conversation.id = publication.conversation_id + AND conversation.company_id = interaction.company_id AND conversation.issue_id = interaction.issue_id + WHERE interaction.id::text = ${a}.payload->>'interactionId' AND interaction.company_id = ${a}.company_id + AND interaction.kind IN ('request_confirmation', 'request_checkbox_confirmation', 'suggest_tasks', 'connection_intent') + AND interaction.status IN ('accepted', 'rejected', 'cancelled', 'expired') AND interaction.resolved_at IS NOT NULL + AND publication.endpoint_id = ${a}.endpoint_id AND conversation.id = ${a}.conversation_id)`; +} diff --git a/server/src/services/idle-sleep-safety.ts b/server/src/services/idle-sleep-safety.ts index 73b360dad20..88f193cae55 100644 --- a/server/src/services/idle-sleep-safety.ts +++ b/server/src/services/idle-sleep-safety.ts @@ -17,7 +17,7 @@ import { idleBrowserHistoryChecks } from "./browser-use-idle-proof.js"; import { idleDecisionHistoryChecks } from "./decision-idle-proof.js"; import { idleWorkspaceHistoryChecks } from "./workspace-history-idle-proof.js"; import { idleMcpHistoryChecks } from "./public-mcp/idle-history-proof.js"; -import { idleExternalAgentHistoryChecks } from "../modules/external-agents/idle-history-proof.js"; +import { idleExternalAgentHistoryChecks } from "../modules/external-agents/index.js"; export const completedHistoryChecks = [ ...idleChatHistoryChecks, ...idleToolActionHistoryChecks, ...idleBrowserHistoryChecks, diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index ecdb45c43a4..e3f66a720b5 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -1,3 +1,4 @@ +import { recordMcpAttachmentCleanup, type McpAttachmentCleanup } from "./public-mcp/upload-cleanup.js"; import { notifyStrandedAgentStatus, notifyActiveRunSilence } from "./live-run-recovery-notifications.js"; import { finishedIdleWake } from "./idle-work-proof.js"; import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; @@ -13907,7 +13908,7 @@ export function issueService(db: Db) { .where(eq(issueAttachments.id, id)) .then((rows) => rows[0] ?? null), - removeAttachment: async (id: string) => + removeAttachment: async (id: string, options?: { mcpCleanup?: McpAttachmentCleanup | null }) => db.transaction(async (tx) => { const existing = await tx .select({ @@ -13937,6 +13938,11 @@ export function issueService(db: Db) { await notifyIssueRecovery(tx, { companyId: existing.companyId }); await tx.delete(issueAttachments).where(eq(issueAttachments.id, id)); await tx.delete(assets).where(eq(assets.id, existing.assetId)); + if (options?.mcpCleanup?.id === existing.id && options.mcpCleanup.companyId === existing.companyId + && options.mcpCleanup.issueId === existing.issueId && options.mcpCleanup.provider === existing.provider + && options.mcpCleanup.objectKey === existing.objectKey) { + await recordMcpAttachmentCleanup(tx, options.mcpCleanup); + } return existing; }), diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 67194b6a9f3..366242b6883 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -9897,6 +9897,11 @@ describe("native warm session supervision", () => { it("blocks idle sleep for a released local warm session until its close checkpoint finishes", async () => { await closeIdleWarmNativeSessionsForRestart(); + const { reconcilePersistedRuntimeServicesOnStartup, readWorkspaceRuntimeIdleWork } = await import("../workspace-runtime.js"); + // The real server reconstructs the workspace owner before startup can + // become quiet. This fixture has no runtime rows or desired services. + await reconcilePersistedRuntimeServicesOnStartup(leaseDb()); + expect(readWorkspaceRuntimeIdleWork()).toBe("none"); markIdleIngressTracked(); markIdleStartupComplete({ scheduledBackups: false }); let finishClose!: () => void; const checkpoint = new Promise(resolve => { finishClose = resolve; }); diff --git a/server/src/services/public-mcp/file-transfers.ts b/server/src/services/public-mcp/file-transfers.ts index eb268e76bc8..83837600769 100644 --- a/server/src/services/public-mcp/file-transfers.ts +++ b/server/src/services/public-mcp/file-transfers.ts @@ -80,13 +80,22 @@ export function createPublicMcpTransfers(db: Db, oauth: PublicMcpOAuth, api: Api } async function sweep() { await db.delete(mcpFileTickets).where(lt(mcpFileTickets.expiresAt, new Date())); - // Serialize cleanup against completion/reissue. Never remove a committed file. + // Serialize cleanup against completion/reissue. Requalify old optimistic + // cleanup and orphaned removed attachments, never a still-linked file. await db.transaction(async tx => { - const abandoned = await tx.select().from(mcpAttachmentUploads).where(and(isNull(mcpAttachmentUploads.attachmentId), isNull(mcpAttachmentUploads.cleanedAt), lt(mcpAttachmentUploads.expiresAt, new Date()))).limit(20).for("update", { skipLocked: true }); + const abandoned = await tx.select().from(mcpAttachmentUploads).where(and( + isNull(mcpAttachmentUploads.cleanupVerifiedAt), lt(mcpAttachmentUploads.expiresAt, new Date()), + sql`(${mcpAttachmentUploads.attachmentId} IS NULL OR NOT EXISTS ( + SELECT 1 FROM issue_attachments attachment WHERE attachment.id = ${mcpAttachmentUploads.attachmentId}))`, + )).limit(20).for("update", { skipLocked: true }); for (const upload of abandoned) { if (upload.storageProvider !== storage.provider) continue; await storage.deleteObject(upload.companyId, upload.objectKey); - await tx.update(mcpAttachmentUploads).set({ cleanedAt: new Date() }).where(eq(mcpAttachmentUploads.id, upload.id)); + if ((await storage.headObject(upload.companyId, upload.objectKey)).exists !== false) { + throw new Error("MCP upload storage cleanup could not be confirmed"); + } + const verifiedAt = new Date(); + await tx.update(mcpAttachmentUploads).set({ cleanedAt: verifiedAt, cleanupVerifiedAt: verifiedAt }).where(eq(mcpAttachmentUploads.id, upload.id)); } }); } @@ -133,6 +142,7 @@ export function createPublicMcpTransfers(db: Db, oauth: PublicMcpOAuth, api: Api const published = await db.transaction(async tx => { const [upload] = await tx.select().from(mcpAttachmentUploads).where(eq(mcpAttachmentUploads.id, row.uploadId!)).for("update"); if (!upload || upload.companyId !== principal.grant.companyId || upload.userId !== principal.grant.userId || upload.workSessionId !== row.workSessionId || upload.grantId && upload.grantId !== row.grantId || upload.storageProvider !== storage.provider) throw unavailable(); + if (upload.expiresAt <= new Date() || upload.cleanedAt || upload.cleanupVerifiedAt) throw unavailable(); if (upload.sha256 !== digest || upload.byteSize !== bytes.length || req.get("Content-Type")?.split(";")[0]?.trim().toLowerCase() !== upload.contentType) throw new McpCapabilityError("Upload bytes, size or Content-Type do not match the requested file."); await taskAccess(principal, upload.taskId, principal.grant.purpose !== "agent", tx as unknown as Db, row.workSessionId ?? undefined); const stored = upload.attachmentId ? null : await storage.putFile({ companyId: upload.companyId, namespace: "mcp-transfers", objectKey: upload.objectKey, originalFilename: upload.originalFilename, contentType: upload.contentType, body: bytes }); @@ -200,7 +210,11 @@ export function createPublicMcpTransfers(db: Db, oauth: PublicMcpOAuth, api: Api objectKey: `${p.grant.companyId}/mcp-transfers/${id}`, storageProvider: storage.provider, expiresAt: new Date(Date.now() + lifetime) }).returning(); } if (!found || found.argumentsHash !== hash) throw new McpCapabilityError("requestId was already used for a different upload."); - await tx.update(mcpAttachmentUploads).set({ expiresAt: new Date(Date.now() + lifetime), cleanedAt: null }).where(eq(mcpAttachmentUploads.id, found.id)); + // A committed identity is immutable, including after user deletion. + // A rejected replay must not erase verified cleanup or permit new bytes. + if (!found.attachmentId) await tx.update(mcpAttachmentUploads) + .set({ expiresAt: new Date(Date.now() + lifetime), cleanedAt: null, cleanupVerifiedAt: null }) + .where(eq(mcpAttachmentUploads.id, found.id)); return found; }); if (upload.attachmentId) { diff --git a/server/src/services/public-mcp/idle-history-proof.ts b/server/src/services/public-mcp/idle-history-proof.ts index b3c7f78abd1..b861eee3775 100644 --- a/server/src/services/public-mcp/idle-history-proof.ts +++ b/server/src/services/public-mcp/idle-history-proof.ts @@ -22,14 +22,17 @@ export const idleMcpHistoryChecks = [ OR (d.mailbox_item_id IS NOT NULL AND d.activity_id IS NULL AND EXISTS ( SELECT 1 FROM dot_mailbox_items m WHERE m.id = d.mailbox_item_id AND m.company_id = s.company_id AND m.binding_id = s.binding_id))))) IS NOT TRUE` }, - // The upload owner allocates object identity before storage writes and only - // stamps cleaned_at after deletion. A committed attachment is never deleted. + // Upload identity survives user deletion. Only the new owner-verified + // absence stamp can retire cleanup; legacy cleaned_at alone was optimistic. { reason: "mcp_attachment_uploads", query: `SELECT 1 FROM mcp_attachment_uploads u WHERE ( - EXISTS (SELECT 1 FROM issues i WHERE i.id = u.task_id AND i.company_id = u.company_id) + EXISTS (SELECT 1 FROM companies c WHERE c.id = u.company_id) + AND NOT EXISTS (SELECT 1 FROM issues contradiction WHERE contradiction.id = u.task_id AND contradiction.company_id <> u.company_id) AND (u.grant_id IS NULL OR EXISTS (SELECT 1 FROM mcp_oauth_grants g WHERE g.id = u.grant_id AND g.company_id = u.company_id AND g.user_id = u.user_id)) - AND ((u.attachment_id IS NULL AND u.cleaned_at IS NOT NULL) - OR (u.cleaned_at IS NULL AND u.attachment_id IS NOT NULL AND EXISTS ( + AND ((u.cleaned_at IS NOT NULL AND u.cleanup_verified_at IS NOT NULL AND u.cleanup_verified_at >= u.cleaned_at + AND (u.attachment_id IS NULL OR NOT EXISTS (SELECT 1 FROM issue_attachments live WHERE live.id = u.attachment_id))) + OR (u.cleaned_at IS NULL AND u.cleanup_verified_at IS NULL AND u.attachment_id IS NOT NULL + AND EXISTS (SELECT 1 FROM issues i WHERE i.id = u.task_id AND i.company_id = u.company_id) AND EXISTS ( SELECT 1 FROM issue_attachments a JOIN assets stored ON stored.id = a.asset_id WHERE a.id = u.attachment_id AND a.company_id = u.company_id AND a.issue_id = u.task_id AND stored.company_id = u.company_id AND stored.object_key = u.object_key diff --git a/server/src/services/public-mcp/upload-cleanup.ts b/server/src/services/public-mcp/upload-cleanup.ts new file mode 100644 index 00000000000..205b5107c8d --- /dev/null +++ b/server/src/services/public-mcp/upload-cleanup.ts @@ -0,0 +1,38 @@ +import { and, eq } from "drizzle-orm"; +import { mcpAttachmentUploads, type Db } from "@paperclipai/db"; +import type { StorageService } from "../../storage/types.js"; + +type AttachmentStorage = { + id: string; + companyId: string; + issueId: string; + provider: string; + objectKey: string; +}; + +export type McpAttachmentCleanup = AttachmentStorage & { verifiedAt: Date }; + +function uploadIdentity(attachment: AttachmentStorage) { + return and(eq(mcpAttachmentUploads.companyId, attachment.companyId), + eq(mcpAttachmentUploads.taskId, attachment.issueId), eq(mcpAttachmentUploads.attachmentId, attachment.id), + eq(mcpAttachmentUploads.storageProvider, attachment.provider), eq(mcpAttachmentUploads.objectKey, attachment.objectKey)); +} + +/** Called after deletion by the attachment owner. A resolved delete alone can + * be optimistic; only the exact upload's storage absence authorizes cleanup. */ +export async function confirmMcpAttachmentCleanup(db: Db, storage: StorageService, attachment: AttachmentStorage): Promise { + const [upload] = await db.select({ id: mcpAttachmentUploads.id }).from(mcpAttachmentUploads).where(uploadIdentity(attachment)).limit(1); + if (!upload) return null; + if (storage.provider !== attachment.provider || (await storage.headObject(attachment.companyId, attachment.objectKey)).exists !== false) { + throw new Error("MCP attachment storage cleanup could not be confirmed"); + } + return { ...attachment, verifiedAt: new Date() }; +} + +/** Keep attachmentId as immutable replay identity. For attachment removal this + * runs in the metadata deletion transaction; issue removal stamps each object + * after its existing post-transaction storage cleanup completes. */ +export async function recordMcpAttachmentCleanup(db: Pick, cleanup: McpAttachmentCleanup | null | undefined) { + if (!cleanup) return; + await db.update(mcpAttachmentUploads).set({ cleanedAt: cleanup.verifiedAt, cleanupVerifiedAt: cleanup.verifiedAt }).where(uploadIdentity(cleanup)); +} diff --git a/server/src/services/voice/idle-history-proof.ts b/server/src/services/voice/idle-history-proof.ts new file mode 100644 index 00000000000..2c76095f8e2 --- /dev/null +++ b/server/src/services/voice/idle-history-proof.ts @@ -0,0 +1,71 @@ +import { VOICE_CALL_REPORT_WINDOW_MS } from "./voice-call-reports.js"; + +/** Available reports still own refresh work for the owner's whole 24h window. + * The accepted push receipt proves submission, never unobservable playback. */ +export function completedVoiceAction(a: string): string { + const closedSession = (s: string) => `${s}.state IN ('ended', 'failed', 'expired') AND ${s}.ended_at IS NOT NULL + AND COALESCE(${s}.error_code, '') <> 'cleanup_pending' + AND (${s}.provider_session_id IS NULL OR ${s}.ended_at < now() - interval '${VOICE_CALL_REPORT_WINDOW_MS} milliseconds')`; + const submitted = (alias: string) => `(${alias}.status = 'completed' AND length(${alias}.result->>'messageId') > 0 + AND ${alias}.result->>'playback' = 'unknown' AND ${alias}.result->>'acceptedAt' ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}T') + OR (${alias}.status = 'call_ended' AND ${alias}.result->>'errorCode' = 'provider_unavailable') + OR (${alias}.status = 'failed' AND ${alias}.result->>'errorCode' IN + ('credentials_rejected', 'insufficient_credits', 'provider_unavailable', 'voice_authority_unavailable'))`; + // Nonnull company/endpoint keys let PostgreSQL hash each debt set once, + // rather than rescanning every old session for every retained voice marker. + const settled = `(${a}.company_id, ${a}.endpoint_id) NOT IN ( + SELECT session.company_id, session.endpoint_id FROM chat_voice_sessions session + WHERE (${closedSession('session')}) IS NOT TRUE) + AND (${a}.company_id, ${a}.endpoint_id) NOT IN ( + SELECT admission.company_id, admission.endpoint_id FROM chat_voice_inbound_calls admission + WHERE admission.state NOT IN ('denied', 'expired', 'ended') + OR admission.cleanup_confirmed_at IS NULL OR admission.cleanup_next_attempt_at IS NOT NULL) + AND (${a}.company_id, ${a}.endpoint_id) NOT IN ( + SELECT push.company_id, push.endpoint_id FROM chat_actions push + WHERE push.kind = 'speko_voice_reply_push' AND (${submitted('push')}) IS NOT TRUE)`; + // Configuration records this existing audit event only after tools, webhook, + // voice defaults and runtime setup all succeeded. A partial setup cannot use + // individual resource receipts to certify the rest of the owning operation. + const setupCommitted = `EXISTS (SELECT 1 FROM activity_log receipt JOIN chat_endpoints endpoint + ON endpoint.id = ${a}.endpoint_id AND endpoint.company_id = ${a}.company_id + WHERE receipt.company_id = ${a}.company_id AND receipt.entity_type = 'tool_connection' + AND receipt.entity_id = endpoint.connection_id::text AND receipt.details->>'endpointId' = endpoint.id::text + AND receipt.details->>'provider' = 'speko' AND receipt.action IN ('chat_endpoint.credentials_verified', 'chat_endpoint.reconnected') + AND receipt.created_at >= ${a}.updated_at)`; + return `EXISTS (SELECT 1 FROM chat_endpoints endpoint WHERE endpoint.id = ${a}.endpoint_id + AND endpoint.company_id = ${a}.company_id AND endpoint.provider = 'speko') AND ${settled} AND ( + (${a}.kind = 'speko_tool_setup' AND ${a}.status = 'completed' AND ${setupCommitted} + AND ${a}.payload->>'name' IN ('submit_request', 'get_updates', 'answer_question') + AND ${a}.payload->>'fingerprint' ~ '^[a-f0-9]{64}$' AND ${a}.payload->>'ownershipDefinitionHash' ~ '^[a-f0-9]{64}$' + AND length(${a}.payload->>'ownershipAgentId') > 0 AND length(${a}.result->>'toolId') > 0 + AND ${a}.provider_action_id = 'speko_tool:' || (${a}.payload->>'name') || ':' || (${a}.payload->>'fingerprint')) + OR (${a}.kind = 'speko_lifecycle_setup' AND ${a}.status = 'completed' AND ${setupCommitted} + AND ${a}.payload->>'fingerprint' ~ '^[a-f0-9]{64}$' AND length(${a}.payload->>'ownershipAgentId') > 0 + AND ${a}.payload->>'ownershipUrl' LIKE 'https://%/events' AND length(${a}.result->>'webhookId') > 0 + AND ${a}.provider_action_id = 'speko_lifecycle:' || (${a}.payload->>'fingerprint')) + OR (${a}.kind = 'speko_lifecycle_receipt' AND ${a}.status = 'completed' AND ${a}.result->>'accepted' = 'true' + AND ${a}.payload->>'type' IN ('call.status', 'call.report') AND ${a}.payload->>'fingerprint' ~ '^[a-f0-9]{64}$' + AND ${a}.provider_action_id LIKE 'speko_event:%' AND ( + EXISTS (SELECT 1 FROM chat_voice_sessions session WHERE session.id::text = ${a}.payload->>'sessionId' + AND session.company_id = ${a}.company_id AND session.endpoint_id = ${a}.endpoint_id AND ${closedSession('session')}) + OR (${a}.payload->'sessionId' = 'null'::jsonb AND EXISTS ( + SELECT 1 FROM chat_voice_inbound_calls admission WHERE admission.company_id = ${a}.company_id + AND admission.endpoint_id = ${a}.endpoint_id AND admission.state IN ('denied', 'expired', 'ended') + AND admission.cleanup_confirmed_at IS NOT NULL AND admission.cleanup_next_attempt_at IS NULL)))) + OR (${a}.kind = 'speko_voice_reply_push' AND (${submitted(a)}) + AND ${a}.payload->>'textDigest' ~ '^[a-f0-9]{64}$' + AND ${a}.payload->>'partIndex' ~ '^[0-9]+$' AND ${a}.payload->>'partCount' ~ '^[1-9][0-9]*$' + AND CASE WHEN ${a}.payload->>'partIndex' ~ '^[0-9]+$' AND ${a}.payload->>'partCount' ~ '^[1-9][0-9]*$' + THEN (${a}.payload->>'partIndex')::numeric < (${a}.payload->>'partCount')::numeric ELSE false END + AND EXISTS (SELECT 1 FROM chat_voice_sessions session JOIN chat_voice_replies reply + ON reply.session_id = session.id AND reply.company_id = session.company_id + JOIN chat_publications publication ON publication.id = reply.publication_id AND publication.company_id = reply.company_id + WHERE session.id::text = ${a}.payload->>'sessionId' AND session.company_id = ${a}.company_id + AND session.endpoint_id = ${a}.endpoint_id AND session.conversation_id = ${a}.conversation_id + AND session.provider_session_id = ${a}.payload->>'providerSessionId' AND session.generation::text = ${a}.payload->>'generation' + AND reply.id::text = ${a}.payload->>'replyId' AND publication.id::text = ${a}.payload->>'publicationId' + AND ${a}.provider_action_id = 'voice_reply_push:' || reply.id::text || CASE WHEN ${a}.payload->>'partIndex' = '0' + THEN '' ELSE ':part:' || (${a}.payload->>'partIndex') END + AND publication.endpoint_id = session.endpoint_id AND publication.conversation_id = session.conversation_id + AND publication.issue_id = session.issue_id AND publication.state = 'published' AND publication.published_at IS NOT NULL AND ${closedSession('session')})))`; +} diff --git a/server/src/services/voice/voice-call-reports.ts b/server/src/services/voice/voice-call-reports.ts index 8f597f9f16b..0290554133d 100644 --- a/server/src/services/voice/voice-call-reports.ts +++ b/server/src/services/voice/voice-call-reports.ts @@ -13,7 +13,8 @@ import { SpekoProviderError } from "./speko-provider.js"; type Entry = { id: string; companyId: string; endpointId: string; connectionId: string; dueAt: number }; type Report = SpekoCallReport; -const WINDOW_MS = 24 * 60 * 60 * 1000; +export const VOICE_CALL_REPORT_WINDOW_MS = 24 * 60 * 60 * 1000; +const WINDOW_MS = VOICE_CALL_REPORT_WINDOW_MS; const eligible = (now: number) => and(inArray(chatVoiceSessions.state, ["ended", "failed", "expired"]), isNotNull(chatVoiceSessions.providerSessionId), gte(chatVoiceSessions.endedAt, new Date(now - WINDOW_MS))); diff --git a/server/src/storage/local-disk-provider.ts b/server/src/storage/local-disk-provider.ts index 9ee950117f3..b92f1eea05f 100644 --- a/server/src/storage/local-disk-provider.ts +++ b/server/src/storage/local-disk-provider.ts @@ -30,8 +30,10 @@ function resolveWithin(baseDir: string, objectKey: string): string { async function statOrNull(filePath: string) { try { return await fs.stat(filePath); - } catch { - return null; + } catch (error) { + const code = (error as NodeJS.ErrnoException | null)?.code; + if (code === "ENOENT" || code === "ENOTDIR") return null; + throw error; } } @@ -89,8 +91,9 @@ export function createLocalDiskStorageProvider(baseDir: string): StorageProvider const filePath = resolveWithin(root, input.objectKey); try { await fs.unlink(filePath); - } catch { - // idempotent delete + } catch (error) { + const code = (error as NodeJS.ErrnoException | null)?.code; + if (code !== "ENOENT" && code !== "ENOTDIR") throw error; } }, }; From cb8c992af334bcee07efe855467f945048457297 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 2 Nov 2026 14:44:30 -0600 Subject: [PATCH 05/66] Fix expired live MCP publication repair links Renew only the authorized live attachment repair expiry, preserving immutable identity and verified cleanup. Cover actual repair PUT, replay, mismatched ownership and overlapping deletion. Carry the exact controller identity accessor and regression from upstream 026fd9e34520a29cf8df1e224afc050e19009914 (#18649); original CI two-second deadline cause remains undetermined. Co-Authored-By: Paperclip --- .../__tests__/idle-completed-history.test.ts | 116 +++++++++++++++++- server/src/server-info.ts | 5 + .../native-controller-identity.test.ts | 37 ++++++ .../native-runtime/native-restart-recovery.ts | 4 +- .../src/services/public-mcp/file-transfers.ts | 20 +++ 5 files changed, 179 insertions(+), 3 deletions(-) create mode 100644 server/src/services/native-runtime/native-controller-identity.test.ts diff --git a/server/src/__tests__/idle-completed-history.test.ts b/server/src/__tests__/idle-completed-history.test.ts index 4e86383c51b..d79057fa43e 100644 --- a/server/src/__tests__/idle-completed-history.test.ts +++ b/server/src/__tests__/idle-completed-history.test.ts @@ -3,7 +3,7 @@ import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import net from "node:net"; -import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { eq, sql } from "drizzle-orm"; import * as tables from "@paperclipai/db"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; @@ -235,6 +235,120 @@ const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPos } finally { await p.cleanup(); } }, 60_000); + it("renews only an expired live agent attachment's publication repair window", async () => { + const { createPublicMcpTransfers } = await import("../services/public-mcp/file-transfers.js"); + const { createPublicMcpOAuth } = await import("../services/public-mcp/oauth.js"); + const { instanceSettingsService } = await import("../services/instance-settings.js"); + const { default: express } = await import("express"); + const { default: request } = await import("supertest"); + const f = await seed(), p = await connectedMcp(f); + try { + const board = await p.oauth.authenticate(p.token), sessionId = randomUUID(); + const resource = `${p.oauth.config.origin}/mcp/runner`; + await instanceSettingsService(db).updateExperimental({ enableOpenAiDot: true }); + await db.update(tables.mcpOauthGrants).set({ purpose: "agent", agentId: f.agentId, + resource, scopes: ["paperclip:agent", "offline_access"] }).where(eq(tables.mcpOauthGrants.id, p.grantId)); + await db.update(tables.issues).set({ status: "in_progress" }).where(eq(tables.issues.id, f.issueId)); + await db.update(tables.heartbeatRuns).set({ status: "running", finishedAt: null, runtimeMode: "native", + driverKind: "grok_bot_mcp", nativeIssueId: f.issueId }).where(eq(tables.heartbeatRuns.id, f.runId)); + const oauth = createPublicMcpOAuth(db, { ...p.oauth.config, resource }); + const principal = await oauth.authenticate(p.token); + let ownerAvailable = true; + let ownerRunId = f.runId; + let pauseRepair = false, repairLocked!: () => void, releaseRepair!: () => void; + const locked = new Promise(resolve => { repairLocked = resolve; }); + const released = new Promise(resolve => { releaseRepair = resolve; }); + // Named work-session boundary; OAuth, transfer HTTP, storage, attachment + // registration and native publication receipt all use their real owners. + const transfers = createPublicMcpTransfers(db, oauth, p.api, p.storage, { agentSession: { + authorize: async (actor, session, taskId, _write, queryDb) => { + if (!ownerAvailable || actor.grant.id !== p.grantId || actor.grant.agentId !== f.agentId + || session !== sessionId || taskId !== f.issueId) throw new Error("work session unavailable"); + if (pauseRepair && queryDb) { pauseRepair = false; repairLocked(); await released; } + return { agentId: f.agentId, runId: ownerRunId }; + }, + } }); + const app = express(); app.use(transfers.router); + const bytes = Buffer.from("expired live publication repair"); + const args = { companyId: f.companyId, taskId: f.issueId, requestId: randomUUID(), filename: "repair.txt", + contentType: "text/plain", byteSize: bytes.length, sha256: createHash("sha256").update(bytes).digest("hex") }; + const upload = await transfers.getUploadUrl(principal, args, sessionId); + const put = async (value: unknown) => { + const url = new URL(String(value)); + return request(app).put(url.pathname + url.search).set("Content-Type", "text/plain").send(bytes) + .timeout({ response: 5000, deadline: 10_000 }); + }; + const initial = await put(upload.url); + expect(initial.status).toBe(200); + const uploadId = String(upload.uploadId), attachmentId = initial.body.attachment.id; + const current = async () => (await db.select().from(tables.mcpAttachmentUploads) + .where(eq(tables.mcpAttachmentUploads.id, uploadId)))[0]; + const loseReceipt = async () => { + const [run] = await db.select().from(tables.heartbeatRuns).where(eq(tables.heartbeatRuns.id, f.runId)); + const receipts = { ...run.resultJson?.semanticToolReceipts as Record }; + delete receipts[`external-upload:${uploadId}`]; + await db.update(tables.heartbeatRuns).set({ resultJson: { ...run.resultJson, semanticToolReceipts: receipts } }) + .where(eq(tables.heartbeatRuns.id, f.runId)); + await db.update(tables.mcpAttachmentUploads).set({ expiresAt: new Date(0) }) + .where(eq(tables.mcpAttachmentUploads.id, uploadId)); + }; + await loseReceipt(); + const expired = await current(); + expect((await put(upload.url)).status).toBe(403); + const repair = await transfers.getUploadUrl(principal, args, sessionId); + const renewed = await current(); + expect(renewed).toEqual({ ...expired, expiresAt: expect.any(Date) }); + expect(renewed.expiresAt.getTime()).toBeGreaterThan(Date.now()); + const repaired = await put(repair.url); + expect(repaired.status).toBe(200); + expect(repaired.body).toMatchObject({ attachment: { id: attachmentId }, publication: { + source: "authenticated_upload", evidenceRef: `deliverable:${attachmentId}` } }); + expect(await transfers.getUploadUrl(principal, args, sessionId)).toMatchObject({ status: "completed", + attachment: { id: attachmentId }, publication: repaired.body.publication }); + expect(await current()).toEqual(renewed); + expect(await db.select().from(tables.issueAttachments).where(eq(tables.issueAttachments.issueId, f.issueId))).toHaveLength(1); + await loseReceipt(); + const beforeDenied = await current(); + ownerAvailable = false; + await expect(transfers.getUploadUrl(principal, args, sessionId)).rejects.toThrow("work session unavailable"); + expect(await current()).toEqual(beforeDenied); + ownerAvailable = true; + ownerRunId = randomUUID(); + await expect(transfers.getUploadUrl(principal, args, sessionId)).rejects.toThrow(); + expect(await current()).toEqual(beforeDenied); + ownerRunId = f.runId; + await db.update(tables.mcpAttachmentUploads).set({ cleanedAt: new Date(), cleanupVerifiedAt: new Date() }) + .where(eq(tables.mcpAttachmentUploads.id, uploadId)); + const contradictory = await current(); + await expect(transfers.getUploadUrl(principal, args, sessionId)).rejects.toThrow(); + expect(await current()).toEqual(contradictory); + expect((await put(repair.url)).status).toBe(403); + await db.update(tables.mcpAttachmentUploads).set({ cleanedAt: null, cleanupVerifiedAt: null }) + .where(eq(tables.mcpAttachmentUploads.id, uploadId)); + // Deletion owns metadata/cleanup settlement. Overlap it with the locked + // repair to prove renewal cannot revive a pointer or recreate its bytes. + pauseRepair = true; + const repairing = transfers.getUploadUrl(principal, args, sessionId); + await locked; + const deleting = p.api(board, "DELETE", `/attachments/${attachmentId}`); + let overlappingRepair: Awaited; + try { + await vi.waitFor(async () => expect(await p.storage.headObject(f.companyId, renewed.objectKey)).toEqual({ exists: false }), + { timeout: 5000, interval: 20 }); + } finally { + releaseRepair(); + [overlappingRepair] = await Promise.all([repairing, deleting]); + } + const deleted = await current(); + expect(deleted).toMatchObject({ attachmentId, cleanedAt: expect.any(Date), cleanupVerifiedAt: expect.any(Date) }); + await expect(transfers.getUploadUrl(principal, args, sessionId)).rejects.toThrow(); + expect(await current()).toEqual(deleted); + expect((await put(repair.url)).status).toBe(403); + expect((await put(overlappingRepair.url)).status).toBe(403); + expect(await p.storage.headObject(f.companyId, deleted.objectKey)).toEqual({ exists: false }); + } finally { await p.cleanup(); } + }, 60_000); + it.each(["delete_failed", "object_survived", "head_failed"] as const)("retains MCP cleanup uncertainty after actual attachment removal: %s", async failure => { const { createPublicMcpTransfers } = await import("../services/public-mcp/file-transfers.js"); const { default: express } = await import("express"); diff --git a/server/src/server-info.ts b/server/src/server-info.ts index 455e622c870..6d3e861e6bb 100644 --- a/server/src/server-info.ts +++ b/server/src/server-info.ts @@ -172,6 +172,11 @@ const GIT_INFO_CACHE_TTL_MS = 3000; const processStartedAt = new Date().toISOString(); let gitInfoCache: { value: ServerGitInfo; expiresAt: number } | null = null; +// Lifecycle identity must not run the synchronous Git probes used by diagnostics. +export function getServerProcessStartedAt(): string { + return processStartedAt; +} + export function getServerInfoSnapshot( opts: { now?: number; diff --git a/server/src/services/native-runtime/native-controller-identity.test.ts b/server/src/services/native-runtime/native-controller-identity.test.ts new file mode 100644 index 00000000000..fd11591da9a --- /dev/null +++ b/server/src/services/native-runtime/native-controller-identity.test.ts @@ -0,0 +1,37 @@ +import { afterEach, expect, it, vi } from "vitest"; +import * as childProcess from "node:child_process"; +import * as hotRestart from "../hot-restart.js"; +import { getServerInfoSnapshot, resetServerInfoCacheForTests } from "../../server-info.js"; +import { currentNativeControllerIdentity } from "./native-restart-recovery.js"; + +vi.mock("node:child_process", async importOriginal => ({ + ...await importOriginal(), + execFileSync: vi.fn(() => { + throw new Error("Git metadata is unavailable; identity must still work"); + }), +})); + +afterEach(() => { + vi.restoreAllMocks(); + resetServerInfoCacheForTests(); +}); + +it.each([true, false])("keeps controller identity independent of Git commands (OS start available=%s)", async available => { + const observed = "2026-11-02T12:00:00.000Z"; + const fallback = getServerInfoSnapshot({ gitCommand: () => { throw new Error("no checkout"); } }).processStartedAt; + resetServerInfoCacheForTests(); + vi.spyOn(hotRestart, "readProcessStartedAt").mockImplementation(async () => { + if (!available) throw new Error("process start unavailable"); + return observed; + }); + const subprocess = vi.mocked(childProcess.execFileSync); + subprocess.mockClear(); + + const first = await currentNativeControllerIdentity(); + const second = await currentNativeControllerIdentity(); + + expect(first).toEqual(second); + expect(first.pid).toBe(process.pid); + expect(first.processStartedAt.toISOString()).toBe(available ? observed : fallback); + expect(subprocess).not.toHaveBeenCalled(); +}); diff --git a/server/src/services/native-runtime/native-restart-recovery.ts b/server/src/services/native-runtime/native-restart-recovery.ts index 96a4112e5a6..a27af96ac3b 100644 --- a/server/src/services/native-runtime/native-restart-recovery.ts +++ b/server/src/services/native-runtime/native-restart-recovery.ts @@ -31,7 +31,7 @@ import { } from "@paperclipai/db"; import { REMOTE_CONTROLLER_LEASE_MS } from "./native-runner-recovery-contract.js"; import { readProcessStartedAt } from "../hot-restart.js"; -import { getServerInfoSnapshot } from "../../server-info.js"; +import { getServerProcessStartedAt } from "../../server-info.js"; import { redactSensitiveText } from "../../redaction.js"; import { issueRecoveryActionService } from "../issue-recovery-actions.js"; import { isNewTerminalRunFailure, reportRunFailure } from "../run-failure-report.js"; @@ -157,7 +157,7 @@ async function observedProcessStart(pid: number): Promise { export async function currentNativeControllerIdentity(): Promise { const observed = await observedProcessStart(process.pid); - const fallback = new Date(getServerInfoSnapshot().processStartedAt); + const fallback = new Date(getServerProcessStartedAt()); return { bootId: controllerBootId, pid: process.pid, diff --git a/server/src/services/public-mcp/file-transfers.ts b/server/src/services/public-mcp/file-transfers.ts index 83837600769..d9e49eeaacb 100644 --- a/server/src/services/public-mcp/file-transfers.ts +++ b/server/src/services/public-mcp/file-transfers.ts @@ -228,6 +228,26 @@ export function createPublicMcpTransfers(db: Db, oauth: PublicMcpOAuth, api: Api ...(publication ? { publication } : {}) }; // An older upload can lack publication proof. Verify the same bytes // through another bounded PUT; never manufacture a receipt from metadata. + await db.transaction(async tx => { + const [current] = await tx.select().from(mcpAttachmentUploads) + .where(eq(mcpAttachmentUploads.id, upload.id)).for("update"); + if (!current || current.companyId !== p.grant.companyId || current.userId !== p.grant.userId + || current.argumentsHash !== hash || current.workSessionId !== sessionId + || current.grantId && current.grantId !== p.grant.id + || current.attachmentId !== attachment.id || current.storageProvider !== storage.provider + || current.cleanedAt || current.cleanupVerifiedAt) throw unavailable(); + const owner = await workSession(p, sessionId, current.taskId, true, tx as unknown as Db); + const live = await attachmentAccess(p, attachment.id, tx as unknown as Db, sessionId); + if (!owner || live.issueId !== current.taskId || live.provider !== current.storageProvider + || live.objectKey !== current.objectKey || live.originalFilename !== current.originalFilename + || live.byteSize !== current.byteSize || live.sha256 !== current.sha256 + || live.originatingRunId !== owner.runId || live.createdByAgentId !== owner.agentId) throw unavailable(); + // Keep the committed attachment and cleanup identity immutable. A + // concurrent deletion still removes that attachment/stamps this row; + // PUT rechecks both and never stores bytes for a committed upload. + await tx.update(mcpAttachmentUploads).set({ expiresAt: new Date(Date.now() + lifetime) }) + .where(eq(mcpAttachmentUploads.id, current.id)); + }); } return { ...await ticket(p, { uploadId: upload.id }, sessionId), method: "PUT", headers: { "Content-Type": a.contentType }, uploadId: upload.id, maxBytes: MAX_ATTACHMENT_BYTES, instructions: "Send the exact file bytes with the given Content-Type. Success saves the attachment and returns its ID. An agent upload also returns a verified publication receipt; cite its evidenceRef in your completion report. There is no separate upload completion tool. Do not share this temporary file credential. Reuse requestId after a lost response." }; From d2b86225f1706a37d5563861f52539afe5d65262 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 2 Nov 2026 15:24:55 -0600 Subject: [PATCH 06/66] Test complete retention settlement after recovery resolution Hold the second actual object deletion to prove first-object absence is a partial state, then observe exact storage, snapshot, deadline and worker settlement within the unchanged wait bound. Keep the original CI evidence. Use the validated explicit history fixture without embedded provisioning or migrations. Co-Authored-By: Paperclip --- ...emote-workspace-snapshot-retention.test.ts | 37 ++++++++++++++++--- 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/server/src/services/remote-workspace-snapshot-retention.test.ts b/server/src/services/remote-workspace-snapshot-retention.test.ts index 26528e6b024..d008d393d38 100644 --- a/server/src/services/remote-workspace-snapshot-retention.test.ts +++ b/server/src/services/remote-workspace-snapshot-retention.test.ts @@ -13,13 +13,15 @@ import { createStorageService } from "../storage/service.js"; import { createS3StorageProvider } from "../storage/s3-provider.js"; import type { StorageService } from "../storage/types.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "../__tests__/helpers/external-test-database.js"; import { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; import { idleWorkSnapshot } from "./task-admission.js"; import { issueRecoveryActionService } from "./issue-recovery-actions.js"; import { notifyRemoteWorkspaceSnapshots, registerRemoteWorkspaceSnapshotRetentionWork, remoteWorkspaceSnapshotRetentionService } from "./remote-workspace-snapshot-retention.js"; -const url = process.env.PAPERCLIP_TEST_DATABASE_URL; -if (url && !/^remote_workspace_[a-z_]*test$/.test(new URL(url).pathname.slice(1))) throw new Error("Disposable remote_workspace_*test database required"); +const historyUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_HISTORY_TEST_DATABASE_URL"); +const url = historyUrl ?? process.env.PAPERCLIP_TEST_DATABASE_URL; +if (!historyUrl && url && !/^remote_workspace_[a-z_]*test$/.test(new URL(url).pathname.slice(1))) throw new Error("Disposable remote_workspace_*test database required"); const support = url ? { supported: true } : await getEmbeddedPostgresTestSupport(); async function versionedS3Fixture() { @@ -95,7 +97,8 @@ describe.skipIf(!support.supported)("remote workspace snapshot retention", () => let temporary: Awaited> | undefined; const now = new Date("2026-10-30T12:00:00Z"); beforeAll(async () => { - if (url) { await applyPendingMigrations(url); db = createDb(url); } + if (historyUrl) db = createDb(historyUrl, { maxConnections: 4 }); + else if (url) { await applyPendingMigrations(url); db = createDb(url); } else { temporary = await startEmbeddedPostgresTestDatabase("workspace-snapshot-retention-"); db = createDb(temporary.connectionString); } }, 90_000); afterAll(async () => { if (url) await closeRegisteredClients(url, { scope: "database" }); else await temporary?.cleanup(); }); @@ -320,6 +323,13 @@ describe.skipIf(!support.supported)("remote workspace snapshot retention", () => await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: f.issueId, kind: "active_run_watchdog", ownerType: "board", cause: "workspace_recovery", fingerprint: randomUUID(), nextAction: "Recover saved files" }); const owner = service(), coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => { throw error; } }); + const remove = storage.deleteObject.bind(storage); + let secondDeleteHeld = false, releaseSecondDelete!: () => void; + const secondDeleteReleased = new Promise(resolve => { releaseSecondDelete = resolve; }); + storage.deleteObject = async (...args) => { + if (args[1] === f.keys[1]) { secondDeleteHeld = true; await secondDeleteReleased; } + await remove(...args); + }; const worker = registerRemoteWorkspaceSnapshotRetentionWork(coordinator, owner, () => {}); try { await worker.ready; @@ -328,8 +338,23 @@ describe.skipIf(!support.supported)("remote workspace snapshot retention", () => const resolved = await issueRecoveryActionService(db).resolveActiveForIssue({ companyId: f.companyId, sourceIssueId: f.issueId, status: "resolved", outcome: "restored", resolutionNote: "Verified saved files" }); expect(resolved?.status).toBe("resolved"); - await vi.waitFor(async () => expect((await storage.headObject(f.companyId, f.keys[0]!)).exists).toBe(false)); - expect(await owner.nextDeadlineAt()).toBeNull(); - } finally { await worker.stop(); await coordinator.stop(); } + try { + await vi.waitFor(() => expect(secondDeleteHeld).toBe(true)); + expect((await storage.headObject(f.companyId, f.keys[0]!)).exists).toBe(false); + expect((await storage.headObject(f.companyId, f.keys[1]!)).exists).toBe(true); + expect((await db.select().from(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.id, f.row.id)))[0]) + .toMatchObject({ id: f.row.id, status: "delete_requested", deleteAfter: now }); + expect(await owner.nextDeadlineAt()).toBe(now.getTime()); + expect(idleWorkSnapshot().active).toBeGreaterThan(0); + } finally { releaseSecondDelete(); } + await vi.waitFor(async () => { + for (const key of f.keys) expect((await storage.headObject(f.companyId, key)).exists).toBe(false); + expect((await db.select().from(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.id, f.row.id)))[0]) + .toMatchObject({ id: f.row.id, status: "deleted", deleteAfter: null }); + expect(await owner.nextDeadlineAt()).toBeNull(); + expect(idleWorkSnapshot().active).toBe(0); + expect(coordinator.nextWakeAt()).toBeNull(); + }); + } finally { releaseSecondDelete(); await worker.stop(); await coordinator.stop(); } }); }); From 00fea0da528d5abdd2229f9f731e5e21c71a6b76 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Mon, 2 Nov 2026 16:50:58 -0600 Subject: [PATCH 07/66] refactor(perf): reuse database test setup for profile benchmark (#18683) ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - Tool profiles govern the tools that agents can use. > - Profile listing and app attention shared nested selector scans that consumed the server CPU. > - #18681 and #18673 replaced those scans with indexes for each computation. > - Their performance benchmarks need consistent setup and reproducible per-service measurements. > - This pull request uses the shared helper entry point and retains the exact measurement harness and results. > - Reviewers can inspect both the CPU savings and the limits of the evidence. ## Linked Issues or Issue Description Refs #18681 and #18673. **What existing behavior does this improve?** The consistency of benchmark setup and the reproducibility of evidence for the two merged CPU fixes. **Current behavior** The profile-review benchmark imports the database starter directly. The matching benchmark uses the shared server test-helper entry point. On the current base, that helper re-exports the database starter, which already supports `PAPERCLIP_TEST_POSTGRES_ADMIN_URL`. The previous committed benchmark measures both services together; the per-service measurements and separate HTTP/Inspector runs lacked a committed harness. **Proposed behavior** Use the same test-helper entry point in both benchmarks. Retain the compile, per-service timing, Inspector, and HTTP-probe programs with their exact commands and raw samples. Preserve the existing embedded and admin-server database behavior. **Reason and benefit** A shared entry point keeps benchmark setup consistent. The committed harness lets reviewers reproduce and compare each real owning service before and after the merged fixes. The failure-path checks verify that result-write errors still release the database and probe resources. **Breaking changes** None. This change affects benchmark setup and documentation. ## What Changed - Import the database starter from the repository test helper in `benchmark-profile-review-projection.ts`. - Record separate CPU and elapsed measurements for profile listing and app attention. - Retain the actual compile, per-service measurement, Inspector capture, and separate-process HTTP probe harness. - Always tear down the test database, HTTP server, and probe even when result writing fails. - Retain all timing and responsiveness samples as sanitized JSON. - State the runtime, fixture, source revisions, and limits of the local comparison. ## Verification - 24 focused tests passed across selector matching, pending projection, review-service freshness, and effective assignments. - Four selected real database-backed service/route tests passed for summaries, review changes, auto-allow behavior, and company isolation. The other 494 tests in that file were not selected. - Injected an `EISDIR` result-write failure into both profiling programs on an owned local admin server. Each child exited with the expected error and removed its disposable database. The responsiveness child also terminated its probe/server handles. - The changed benchmark completed with a configured local PostgreSQL admin URL. Its full responses matched the baseline, and query/row counts matched. - The exact documented compile, benchmark, and responsiveness commands completed from the committed harness at `ca458baddc6461fb6c888984246e329e2bb4bdfe`. The JSON records that build revision and the four owner source hashes. - Five alternating samples per service and implementation used Node 24.22.2 with compiled ES modules. No runtime TypeScript loader ran in those measurements. - Median process CPU from the committed harness: profile listing 3,164 ms to 385 ms; app attention 2,466 ms to 221 ms. Both versions returned identical ordered results and made seven queries per service. - Separate Inspector profiles confirmed that the repeated scope and summary scans were removed. Three separate-process HTTP probe trials per service showed less event-loop blocking. These are local synthetic service results, not hosted page-load acceptance. - Initial local setup failed on missing embedded PostgreSQL library aliases and an unbuilt plugin SDK. Restoring local aliases and building the SDK resolved those setup failures. The original failed setup logs were retained; the 28 selected tests then passed. - The merged summary fix CI run [41535304968](https://github.com/paperclipai/paperclip/actions/runs/41535304968) hit two 15-second timeouts in the unchanged attention-retention history cases. The server shard passed unchanged on one failed-jobs-only retry. Original and retry logs are retained. The compared runs used different runners and synthetic merge trees, so the original timeout cause is not established. No timeout or assertion was changed. - Full repository typecheck, test matrix, and build passed in CI on final head `2f484ed4284f3fa632d8e73d9be084827d1c987e`: 54 successful check runs, three expected skips, one non-blocking Paperclip review, and a successful Snyk status. No failures or pending checks remain. Those broad commands were not rerun locally for this benchmark-tooling follow-up; the 28 selected local regressions and the actual profiling/failure-path programs ran locally. - Fresh Greptile is 5/5 on that exact head. Both review P2 threads are resolved. The Paperclip review's current-base documentation note is corrected in this description; its optional tooling suggestions remain outside this bounded change. ## Risks - The test helper creates a unique disposable database on the configured admin server and removes it afterward. The operator must use a database server where those test operations are appropriate. - Synthetic data and shared-host activity affect absolute timings. The measurements do not establish a production latency percentage or capacity limit. - Production behavior is covered by the two linked merged fixes. This follow-up changes only benchmark setup and documentation. ## Model Used OpenAI GPT-6 through Codex, with code execution and repository tools. The exact runtime model revision and context window were not exposed in this session. The original linked performance fixes record their own model details. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- .../2026-11-02-profile-projection.json | 532 ++++++++++++++++++ doc/mcp-discovery-performance.md | 83 +++ .../benchmark-profile-review-projection.ts | 3 +- .../profile-tool-projection/benchmark.mjs | 95 ++++ .../profile-tool-projection/compile.mjs | 56 ++ .../scripts/profile-tool-projection/probe.mjs | 20 + .../profile-tool-projection/register.mjs | 13 + .../responsiveness.mjs | 74 +++ 8 files changed, 875 insertions(+), 1 deletion(-) create mode 100644 doc/benchmarks/2026-11-02-profile-projection.json create mode 100644 server/scripts/profile-tool-projection/benchmark.mjs create mode 100644 server/scripts/profile-tool-projection/compile.mjs create mode 100644 server/scripts/profile-tool-projection/probe.mjs create mode 100644 server/scripts/profile-tool-projection/register.mjs create mode 100644 server/scripts/profile-tool-projection/responsiveness.mjs diff --git a/doc/benchmarks/2026-11-02-profile-projection.json b/doc/benchmarks/2026-11-02-profile-projection.json new file mode 100644 index 00000000000..428f514b0ae --- /dev/null +++ b/doc/benchmarks/2026-11-02-profile-projection.json @@ -0,0 +1,532 @@ +{ + "measurements": [ + { + "kind": "environment", + "node": "v24.22.2", + "platform": "darwin", + "arch": "arm64", + "build": "esbuild transpiled ES modules, node24 target; no runtime TypeScript loader", + "baseline": "1f8f2e195f4897cd1c237bdc068de95780ab0c3c", + "pending": "fd59c0183ce64c44696f79604516f4d51ca5662b", + "treatmentHead": "ca458baddc6461fb6c888984246e329e2bb4bdfe", + "sourceSha256": { + "profiles/projection": "fdb8b2cd220ce71a1ef322afef8d25e684beff88b73fedb1a5b73122b95e9a8d", + "profiles/service": "d56f83ec975bf4d2955cb9a31e9b4ebeb4f3872f28d930436bd37b6004ce1824", + "profiles/resolution": "cb621da4864ee143922c94644487b8bf53a185848a419e9b05945ed6b0237a14", + "catalog/attention": "c3fdc44e01f39b59cec047584c75d99df192f4e67809a6078ac75dfbb3c2b005" + }, + "forcedGc": true, + "fixture": { + "profiles": 200, + "activeTools": 1900, + "removedTools": 695, + "rules": 76000, + "agents": 12 + }, + "repeats": 5, + "caveat": "Local synthetic service workload, shared developer host; not hosted page-load acceptance" + }, + { + "kind": "correctness", + "owner": "profiles", + "completeOutputEquality": true + }, + { + "kind": "correctness", + "owner": "attention", + "completeOutputEquality": true + }, + { + "kind": "timing", + "round": 0, + "owner": "profiles", + "label": "baseline", + "wallMs": 3044.6725000000006, + "cpuMs": 3086.254, + "queries": 7, + "rows": 78114, + "resultCount": 200 + }, + { + "kind": "timing", + "round": 0, + "owner": "profiles", + "label": "pending", + "wallMs": 1331.632625000002, + "cpuMs": 1475.759, + "queries": 7, + "rows": 78114, + "resultCount": 200 + }, + { + "kind": "timing", + "round": 0, + "owner": "profiles", + "label": "combined", + "wallMs": 214.5939580000013, + "cpuMs": 384.783, + "queries": 7, + "rows": 78114, + "resultCount": 200 + }, + { + "kind": "timing", + "round": 0, + "owner": "attention", + "label": "baseline", + "wallMs": 2378.09475, + "cpuMs": 2466.076, + "queries": 7, + "rows": 78101 + }, + { + "kind": "timing", + "round": 0, + "owner": "attention", + "label": "pending", + "wallMs": 174.1545839999999, + "cpuMs": 319.323, + "queries": 7, + "rows": 78101 + }, + { + "kind": "timing", + "round": 0, + "owner": "attention", + "label": "combined", + "wallMs": 159.26837499999965, + "cpuMs": 219.179, + "queries": 7, + "rows": 78101 + }, + { + "kind": "timing", + "round": 1, + "owner": "profiles", + "label": "combined", + "wallMs": 181.5132080000003, + "cpuMs": 224.232, + "queries": 7, + "rows": 78114, + "resultCount": 200 + }, + { + "kind": "timing", + "round": 1, + "owner": "profiles", + "label": "pending", + "wallMs": 1997.141208000001, + "cpuMs": 1830.433, + "queries": 7, + "rows": 78114, + "resultCount": 200 + }, + { + "kind": "timing", + "round": 1, + "owner": "profiles", + "label": "baseline", + "wallMs": 4942.23775, + "cpuMs": 4553.281, + "queries": 7, + "rows": 78114, + "resultCount": 200 + }, + { + "kind": "timing", + "round": 1, + "owner": "attention", + "label": "combined", + "wallMs": 318.5823750000018, + "cpuMs": 466.636, + "queries": 7, + "rows": 78101 + }, + { + "kind": "timing", + "round": 1, + "owner": "attention", + "label": "pending", + "wallMs": 192.32737500000076, + "cpuMs": 254.286, + "queries": 7, + "rows": 78101 + }, + { + "kind": "timing", + "round": 1, + "owner": "attention", + "label": "baseline", + "wallMs": 2694.246791999998, + "cpuMs": 2761.568, + "queries": 7, + "rows": 78101 + }, + { + "kind": "timing", + "round": 2, + "owner": "profiles", + "label": "baseline", + "wallMs": 4083.2392089999994, + "cpuMs": 3864.952, + "queries": 7, + "rows": 78114, + "resultCount": 200 + }, + { + "kind": "timing", + "round": 2, + "owner": "profiles", + "label": "pending", + "wallMs": 1423.3007910000015, + "cpuMs": 1581.88, + "queries": 7, + "rows": 78114, + "resultCount": 200 + }, + { + "kind": "timing", + "round": 2, + "owner": "profiles", + "label": "combined", + "wallMs": 223.0350000000035, + "cpuMs": 414.014, + "queries": 7, + "rows": 78114, + "resultCount": 200 + }, + { + "kind": "timing", + "round": 2, + "owner": "attention", + "label": "baseline", + "wallMs": 1864.8400000000038, + "cpuMs": 1928.275, + "queries": 7, + "rows": 78101 + }, + { + "kind": "timing", + "round": 2, + "owner": "attention", + "label": "pending", + "wallMs": 175.5971669999999, + "cpuMs": 339.946, + "queries": 7, + "rows": 78101 + }, + { + "kind": "timing", + "round": 2, + "owner": "attention", + "label": "combined", + "wallMs": 147.17741699999897, + "cpuMs": 202.889, + "queries": 7, + "rows": 78101 + }, + { + "kind": "timing", + "round": 3, + "owner": "profiles", + "label": "combined", + "wallMs": 182.54275000000052, + "cpuMs": 218.628, + "queries": 7, + "rows": 78114, + "resultCount": 200 + }, + { + "kind": "timing", + "round": 3, + "owner": "profiles", + "label": "pending", + "wallMs": 1203.7079159999994, + "cpuMs": 1258.248, + "queries": 7, + "rows": 78114, + "resultCount": 200 + }, + { + "kind": "timing", + "round": 3, + "owner": "profiles", + "label": "baseline", + "wallMs": 2998.833458000001, + "cpuMs": 3164.455, + "queries": 7, + "rows": 78114, + "resultCount": 200 + }, + { + "kind": "timing", + "round": 3, + "owner": "attention", + "label": "combined", + "wallMs": 174.7018339999995, + "cpuMs": 353.529, + "queries": 7, + "rows": 78101 + }, + { + "kind": "timing", + "round": 3, + "owner": "attention", + "label": "pending", + "wallMs": 147.98641700000007, + "cpuMs": 204.736, + "queries": 7, + "rows": 78101 + }, + { + "kind": "timing", + "round": 3, + "owner": "attention", + "label": "baseline", + "wallMs": 2482.0454580000005, + "cpuMs": 2538.083, + "queries": 7, + "rows": 78101 + }, + { + "kind": "timing", + "round": 4, + "owner": "profiles", + "label": "baseline", + "wallMs": 2980.427125000002, + "cpuMs": 3091.659, + "queries": 7, + "rows": 78114, + "resultCount": 200 + }, + { + "kind": "timing", + "round": 4, + "owner": "profiles", + "label": "pending", + "wallMs": 1334.6162909999985, + "cpuMs": 1494.868, + "queries": 7, + "rows": 78114, + "resultCount": 200 + }, + { + "kind": "timing", + "round": 4, + "owner": "profiles", + "label": "combined", + "wallMs": 233.16475000000355, + "cpuMs": 439.851, + "queries": 7, + "rows": 78114, + "resultCount": 200 + }, + { + "kind": "timing", + "round": 4, + "owner": "attention", + "label": "baseline", + "wallMs": 2031.9052500000034, + "cpuMs": 2090.426, + "queries": 7, + "rows": 78101 + }, + { + "kind": "timing", + "round": 4, + "owner": "attention", + "label": "pending", + "wallMs": 173.21041600000171, + "cpuMs": 316.039, + "queries": 7, + "rows": 78101 + }, + { + "kind": "timing", + "round": 4, + "owner": "attention", + "label": "combined", + "wallMs": 158.0989999999947, + "cpuMs": 221.091, + "queries": 7, + "rows": 78101 + }, + { + "kind": "profile", + "owner": "profiles", + "label": "baseline", + "durationMs": 2994.209, + "samples": 2345 + }, + { + "kind": "profile", + "owner": "profiles", + "label": "pending", + "durationMs": 1304.667, + "samples": 999 + }, + { + "kind": "profile", + "owner": "profiles", + "label": "combined", + "durationMs": 271.542, + "samples": 173 + }, + { + "kind": "profile", + "owner": "attention", + "label": "baseline", + "durationMs": 2502.958, + "samples": 1958 + }, + { + "kind": "profile", + "owner": "attention", + "label": "pending", + "durationMs": 248.75, + "samples": 149 + }, + { + "kind": "profile", + "owner": "attention", + "label": "combined", + "durationMs": 216.375, + "samples": 129 + } + ], + "responsiveness": [ + { + "kind": "responsiveness", + "round": 0, + "owner": "profiles", + "label": "baseline", + "operationMs": 3020.712625, + "completedHealthRequests": 4, + "maxHealthMs": 2885.9768750000003, + "errors": [], + "scope": "Separate-process HTTP probe to a minimal liveness handler in the same Node event loop as the real service; not the full application health route" + }, + { + "kind": "responsiveness", + "round": 0, + "owner": "profiles", + "label": "combined", + "operationMs": 216.8745829999989, + "completedHealthRequests": 5, + "maxHealthMs": 78.3608750000003, + "errors": [], + "scope": "Separate-process HTTP probe to a minimal liveness handler in the same Node event loop as the real service; not the full application health route" + }, + { + "kind": "responsiveness", + "round": 0, + "owner": "attention", + "label": "baseline", + "operationMs": 2220.037542, + "completedHealthRequests": 4, + "maxHealthMs": 2103.2218329999996, + "errors": [], + "scope": "Separate-process HTTP probe to a minimal liveness handler in the same Node event loop as the real service; not the full application health route" + }, + { + "kind": "responsiveness", + "round": 0, + "owner": "attention", + "label": "combined", + "operationMs": 178.0747500000016, + "completedHealthRequests": 4, + "maxHealthMs": 55.81975000000057, + "errors": [], + "scope": "Separate-process HTTP probe to a minimal liveness handler in the same Node event loop as the real service; not the full application health route" + }, + { + "kind": "responsiveness", + "round": 1, + "owner": "profiles", + "label": "combined", + "operationMs": 180.5625, + "completedHealthRequests": 4, + "maxHealthMs": 63.572291000000405, + "errors": [], + "scope": "Separate-process HTTP probe to a minimal liveness handler in the same Node event loop as the real service; not the full application health route" + }, + { + "kind": "responsiveness", + "round": 1, + "owner": "profiles", + "label": "baseline", + "operationMs": 3155.7248339999987, + "completedHealthRequests": 5, + "maxHealthMs": 3027.892249999999, + "errors": [], + "scope": "Separate-process HTTP probe to a minimal liveness handler in the same Node event loop as the real service; not the full application health route" + }, + { + "kind": "responsiveness", + "round": 1, + "owner": "attention", + "label": "combined", + "operationMs": 183.60120800000004, + "completedHealthRequests": 4, + "maxHealthMs": 71.33616699999766, + "errors": [], + "scope": "Separate-process HTTP probe to a minimal liveness handler in the same Node event loop as the real service; not the full application health route" + }, + { + "kind": "responsiveness", + "round": 1, + "owner": "attention", + "label": "baseline", + "operationMs": 2413.9667499999996, + "completedHealthRequests": 3, + "maxHealthMs": 2292.3195000000014, + "errors": [], + "scope": "Separate-process HTTP probe to a minimal liveness handler in the same Node event loop as the real service; not the full application health route" + }, + { + "kind": "responsiveness", + "round": 2, + "owner": "profiles", + "label": "baseline", + "operationMs": 3146.798458000001, + "completedHealthRequests": 4, + "maxHealthMs": 3008.1585419999974, + "errors": [], + "scope": "Separate-process HTTP probe to a minimal liveness handler in the same Node event loop as the real service; not the full application health route" + }, + { + "kind": "responsiveness", + "round": 2, + "owner": "profiles", + "label": "combined", + "operationMs": 230.38120900000285, + "completedHealthRequests": 4, + "maxHealthMs": 83.7714589999996, + "errors": [], + "scope": "Separate-process HTTP probe to a minimal liveness handler in the same Node event loop as the real service; not the full application health route" + }, + { + "kind": "responsiveness", + "round": 2, + "owner": "attention", + "label": "baseline", + "operationMs": 1886.0740839999999, + "completedHealthRequests": 4, + "maxHealthMs": 1770.5739170000015, + "errors": [], + "scope": "Separate-process HTTP probe to a minimal liveness handler in the same Node event loop as the real service; not the full application health route" + }, + { + "kind": "responsiveness", + "round": 2, + "owner": "attention", + "label": "combined", + "operationMs": 180.26520900000105, + "completedHealthRequests": 3, + "maxHealthMs": 64.22629199999938, + "errors": [], + "scope": "Separate-process HTTP probe to a minimal liveness handler in the same Node event loop as the real service; not the full application health route" + } + ], + "responsiveness_limit": "Minimal local HTTP liveness handler on the same event loop as real database-backed service operations, not the assembled application health route." +} diff --git a/doc/mcp-discovery-performance.md b/doc/mcp-discovery-performance.md index ac696b01187..bed7a912efd 100644 --- a/doc/mcp-discovery-performance.md +++ b/doc/mcp-discovery-performance.md @@ -138,3 +138,86 @@ seven samples of ten resolutions. It reports actual query and returned-row counts, process CPU, and elapsed time. The installation dependency returns an empty list for this fixture, which has no installed connections. It does not measure provider work, whole-run latency, or hosted performance. + + +## Local profile comparison on 2026-11-02 + +The profile and app-attention paths were re-profiled together after +[#18681](https://github.com/paperclipai/paperclip/pull/18681) and +[#18673](https://github.com/paperclipai/paperclip/pull/18673). Both fixes replace +nested selector scans with indexes that live for one computation. + +The comparison used Node 24.22.2 on macOS arm64. Service modules were compiled +as ES modules for Node 24 before measurement. No TypeScript loader ran during +measurement. All variants used the same database, fixture, and database client. +Only the service and projection implementation changed between variants. +The fixture had 200 profiles, 1,900 active tools, 695 removed tools, 76,000 +rules, and 12 agents. Complete ordered responses matched before timing. + +Baseline: `1f8f2e195f4897cd1c237bdc068de95780ab0c3c`. The pending-review fix was +`fd59c0183ce64c44696f79604516f4d51ca5662b`. The summary fix was +`e20165b6eb871c29e504a3837e8fe92a17ef969f`; its later benchmark helper change +does not change production code. Five unprofiled samples alternated order, +with forced GC before each operation. Values below are medians. +[All samples](benchmarks/2026-11-02-profile-projection.json) retain each measurement +and the pending-review-only treatment. + +| Service | Baseline CPU | Combined CPU | Baseline elapsed | Combined elapsed | +| --- | ---: | ---: | ---: | ---: | +| `listProfiles` | 3,164 ms | 385 ms | 3,045 ms | 215 ms | +| `listAppsNeedingAttention` | 2,466 ms | 221 ms | 2,378 ms | 159 ms | + +Each version made seven queries per service. Profile listing returned 78,114 +database rows; app attention returned 78,101. These counts did not change. +The gains came from less JavaScript work after the reads. + +Separate Inspector CPU captures used a 1 ms sampling interval. The baseline +showed `profileCoversCatalogScope` and `summarizeProfile` as major CPU costs. +Their repeated scans were absent from the combined capture. Residual work +included pending projection, row conversion, and garbage collection. +Profile capture time was excluded from the timing samples above. + +The committed harness is in `server/scripts/profile-tool-projection/`. +Use Node 24.22.2, install workspace dependencies, and run these commands from +the repository root in a disposable worktree. The compile step writes local +`dist/` output. It is not the full application build. The Git history must +contain the baseline and pending-fix commits named above. + +```sh +git fetch https://github.com/paperclipai/paperclip.git refs/pull/18681/head +pnpm install --frozen-lockfile +node server/scripts/profile-tool-projection/compile.mjs +node --expose-gc --import ./server/scripts/profile-tool-projection/register.mjs server/scripts/profile-tool-projection/benchmark.mjs --output-dir /tmp/paperclip-tool-projection --repeat 5 +node --expose-gc --import ./server/scripts/profile-tool-projection/register.mjs server/scripts/profile-tool-projection/responsiveness.mjs --output-dir /tmp/paperclip-tool-projection +``` + +The compile step writes the baseline and pending-only owner modules beside +the current compiled modules. A build manifest records the current Git HEAD +and the SHA-256 hashes of the four current owner source files. The benchmark +records this manifest, validates complete output equality, then measures +`profiles` and `attention` separately for `baseline`, `pending`, and `combined`. +It writes `measurements.json` and six separate `.cpuprofile` files to the +output directory. The HTTP probe writes `responsiveness.json` there. Keep +CPU profiles private because they contain local source paths. Rebuild and +rerun if the owner files change. Serialize these commands with other heavy +builds and benchmarks on the same host. + +A separate process also sent simple HTTP requests every 20 ms while each +real service operation ran. The local liveness handler shared the service's +Node event loop. There was at most one outstanding probe request. Across +three alternating trials, the median largest probe delay fell from 3,008 ms +to 78 ms during profile listing, and from 2,103 ms to 64 ms during app +attention. All 48 probe requests completed without error. + +These are synthetic service measurements on a shared developer machine. +They are not a full application build, a browser test, or hosted acceptance. +The liveness handler is a minimal test handler, not the application's health +route. Workload, JIT warmup, and host activity affect absolute timings. The +comparison does not establish a page-load percentage or production capacity. +Use a matched deployment window to confirm the effect on user requests. + +Both benchmark scripts use the test database helper. Set +`PAPERCLIP_TEST_POSTGRES_ADMIN_URL` to use a local PostgreSQL server; otherwise +they create and remove an embedded test database. The helper creates a unique +disposable database and removes it afterward. Benchmark setup and fixture +insertion are outside the timing samples. diff --git a/server/scripts/benchmark-profile-review-projection.ts b/server/scripts/benchmark-profile-review-projection.ts index 694f3a25704..1c1265ec049 100644 --- a/server/scripts/benchmark-profile-review-projection.ts +++ b/server/scripts/benchmark-profile-review-projection.ts @@ -1,7 +1,8 @@ import assert from "node:assert/strict"; import { isDeepStrictEqual } from "node:util"; import { pathToFileURL } from "node:url"; -import { createDb, startEmbeddedPostgresTestDatabase, type Db } from "@paperclipai/db"; +import { createDb, type Db } from "@paperclipai/db"; +import { startEmbeddedPostgresTestDatabase } from "../src/__tests__/helpers/embedded-postgres.js"; import { createToolProfileService } from "../src/services/tool-access/profiles/service.js"; import { createToolAppAttention } from "../src/services/tool-access/catalog/attention.js"; import { createProfileProjectionFixture } from "../src/__tests__/helpers/tool-profile-projection-fixture.js"; diff --git a/server/scripts/profile-tool-projection/benchmark.mjs b/server/scripts/profile-tool-projection/benchmark.mjs new file mode 100644 index 00000000000..f4a0ed3b176 --- /dev/null +++ b/server/scripts/profile-tool-projection/benchmark.mjs @@ -0,0 +1,95 @@ +import assert from 'node:assert/strict'; +import { writeFile, mkdir, readFile } from 'node:fs/promises'; +import path from 'node:path'; +import { Session } from 'node:inspector'; +import { createDb } from '@paperclipai/db'; +import { startEmbeddedPostgresTestDatabase } from '../../dist/__tests__/helpers/embedded-postgres.js'; +import { createProfileProjectionFixture } from '../../dist/__tests__/helpers/tool-profile-projection-fixture.js'; +const option = (name, fallback) => { const index = process.argv.indexOf(name); return index < 0 ? fallback : process.argv[index + 1] ?? fallback; }; +const output = path.resolve(option('--output-dir', '/tmp/paperclip-tool-projection')); +const repeats = Number(option('--repeat', '5')); +assert.ok(Number.isInteger(repeats) && repeats > 0, '--repeat must be a positive integer'); +assert.equal(typeof global.gc, 'function', 'Run Node with --expose-gc'); +const build = JSON.parse(await readFile(new URL('../../dist/.profile-projection-build.json', import.meta.url), 'utf8')); +await mkdir(output, { recursive: true }); +const variants = {}; +for (const label of ['baseline', 'pending', 'combined']) { + const p = label === 'combined' ? './service.js' : `./.${label}-service.js`; + const a = label === 'combined' ? './attention.js' : `./.${label}-attention.js`; + variants[label] = { + profileFactory: (await import(new URL(`../../dist/services/tool-access/profiles/${p}`, import.meta.url))).createToolProfileService, + attentionFactory: (await import(new URL(`../../dist/services/tool-access/catalog/${a}`, import.meta.url))).createToolAppAttention, + }; +} +const database = await startEmbeddedPostgresTestDatabase('paperclip-hotpaths-owned-'); +const db = createDb(database.connectionString, { maxConnections: 1 }); +let queries = 0, rows = 0; +const measuredDb = new Proxy(db, { get(target, key) { + if (key !== 'select') return Reflect.get(target, key); + return (...fields) => { + const builder = target.select(...fields), from = builder.from.bind(builder); + builder.from = (...args) => { + const selection = from(...args), execute = selection.execute.bind(selection); + selection.execute = async () => { const result = await execute(); queries++; rows += result.length; return result; }; + return selection; + }; + return builder; + }; +} }); +const records = []; +const emit = record => { records.push(record); console.log(JSON.stringify(record)); }; +try { + const fixture = await createProfileProjectionFixture(db); + const now = () => new Date('2026-02-01T00:00:00Z'); + for (const variant of Object.values(variants)) { + const profiles = variant.profileFactory(measuredDb, now); + const attention = variant.attentionFactory(measuredDb, now); + variant.profiles = () => profiles.listProfiles(fixture.company.id); + variant.attention = () => attention(fixture.company.id); + } + emit({ kind: 'environment', node: process.version, platform: process.platform, arch: process.arch, + build: 'esbuild transpiled ES modules, node24 target; no runtime TypeScript loader', + ...build, forcedGc: true, + fixture: { profiles: 200, activeTools: 1900, removedTools: 695, rules: 76000, agents: 12 }, repeats, + caveat: 'Local synthetic service workload, shared developer host; not hosted page-load acceptance' }); + for (const owner of ['profiles', 'attention']) { + const expected = await variants.baseline[owner](); + for (const label of ['pending', 'combined']) assert.deepEqual(await variants[label][owner](), expected); + emit({ kind: 'correctness', owner, completeOutputEquality: true }); + } + for (let round = 0; round < repeats; round++) { + for (const owner of ['profiles', 'attention']) { + const order = round % 2 ? ['combined', 'pending', 'baseline'] : ['baseline', 'pending', 'combined']; + for (const label of order) { + global.gc?.(); queries = 0; rows = 0; + const started = performance.now(), cpu = process.cpuUsage(); + const result = await variants[label][owner](); + const used = process.cpuUsage(cpu); + emit({ kind: 'timing', round, owner, label, wallMs: performance.now() - started, cpuMs: (used.user + used.system) / 1000, + queries, rows, resultCount: owner === 'profiles' ? result.length : result.items?.length }); + } + } + } + for (const owner of ['profiles', 'attention']) { + for (const label of ['baseline', 'pending', 'combined']) { + global.gc?.(); + const session = new Session(); session.connect(); + const post = (method, args = {}) => new Promise((resolve, reject) => session.post(method, args, (error, value) => error ? reject(error) : resolve(value))); + try { + await post('Profiler.enable'); await post('Profiler.setSamplingInterval', { interval: 1000 }); + await post('Profiler.start'); + await variants[label][owner](); + const { profile } = await post('Profiler.stop'); + await writeFile(`${output}/${owner}-${label}.cpuprofile`, JSON.stringify(profile)); + emit({ kind: 'profile', owner, label, durationMs: (profile.endTime - profile.startTime) / 1000, samples: profile.samples?.length }); + } finally { session.disconnect(); } + } + } +} finally { + try { + await writeFile(`${output}/measurements.json`, JSON.stringify(records, null, 2)); + } finally { + try { await db.$client.end({ timeout: 1 }); } + finally { await database.cleanup(); } + } +} diff --git a/server/scripts/profile-tool-projection/compile.mjs b/server/scripts/profile-tool-projection/compile.mjs new file mode 100644 index 00000000000..c82b2b8adf1 --- /dev/null +++ b/server/scripts/profile-tool-projection/compile.mjs @@ -0,0 +1,56 @@ +import { build, transform } from 'esbuild'; +import { readdir, mkdir, writeFile, cp, readFile } from 'node:fs/promises'; +import { fileURLToPath } from 'node:url'; +import { createHash } from 'node:crypto'; +import { execFileSync } from 'node:child_process'; +import path from 'node:path'; +const root = fileURLToPath(new URL("../../..", import.meta.url)); +process.chdir(root); +async function sources(dir) { + const out = []; + for (const entry of await readdir(dir, { withFileTypes: true })) { + const file = path.join(dir, entry.name); + if (entry.isDirectory() && entry.name !== '__tests__') out.push(...await sources(file)); + else if (entry.isFile() && /\.tsx?$/.test(entry.name) && !/\.d\.ts$|\.test\.tsx?$/.test(entry.name)) out.push(file); + } + return out; +} +async function assets(dir, out) { + for (const entry of await readdir(dir, { withFileTypes: true })) { + const file = path.join(dir, entry.name), target = path.join(out, entry.name); + if (entry.isDirectory() && entry.name !== '__tests__') await assets(file, target); + else if (entry.isFile() && entry.name.endsWith('.json')) { await mkdir(out, { recursive: true }); await cp(file, target); } + } +} +for (const pkg of ['packages/shared', 'packages/db', 'server']) { + const src = path.join(root, pkg, 'src'); + await build({ entryPoints: await sources(src), outbase: src, outdir: path.join(root, pkg, 'dist'), + format: 'esm', platform: 'node', target: 'node24', sourcemap: true, logLevel: 'warning' }); + await assets(src, path.join(root, pkg, 'dist')); +} +await cp('packages/db/src/migrations', 'packages/db/dist/migrations', { recursive: true }); +await cp('packages/shared/src/app-definitions', 'packages/shared/dist/app-definitions', { recursive: true }); +await mkdir('server/dist/__tests__/helpers', { recursive: true }); +await build({ entryPoints: ['server/src/__tests__/helpers/tool-profile-projection-fixture.ts', 'server/src/__tests__/helpers/embedded-postgres.ts'], + outbase: 'server/src/__tests__/helpers', outdir: 'server/dist/__tests__/helpers', format: 'esm', platform: 'node', target: 'node24', sourcemap: true }); +for (const [label, sha] of [['baseline', '1f8f2e195f4897cd1c237bdc068de95780ab0c3c'], ['pending', 'fd59c0183ce64c44696f79604516f4d51ca5662b']]) { + for (const [name, file] of [['projection', 'profiles/projection'], ['service', 'profiles/service'], ['attention', 'catalog/attention']]) { + const relative = `server/src/services/tool-access/${file}.ts`; + let code = execFileSync('git', ['show', `${sha}:${relative}`], { encoding: 'utf8', maxBuffer: 5e6 }); + if (name === 'service') code = code.replace('from "./projection.js"', `from "./.${label}-projection.js"`); + if (name === 'attention') code = code.replace('from "../profiles/projection.js"', `from "../profiles/.${label}-projection.js"`); + const outfile = `server/dist/services/tool-access/${path.dirname(file)}/.${label}-${name}.js`; + const result = await transform(code, { loader: 'ts', format: 'esm', platform: 'node', target: 'node24', sourcefile: relative, sourcemap: 'inline' }); + await writeFile(outfile, result.code); + } +} +const owners = ['profiles/projection', 'profiles/service', 'profiles/resolution', 'catalog/attention']; +const manifest = { + baseline: '1f8f2e195f4897cd1c237bdc068de95780ab0c3c', + pending: 'fd59c0183ce64c44696f79604516f4d51ca5662b', + treatmentHead: execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim(), + sourceSha256: Object.fromEntries(await Promise.all(owners.map(async file => [file, + createHash('sha256').update(await readFile(`server/src/services/tool-access/${file}.ts`)).digest('hex')]))), +}; +await writeFile('server/dist/.profile-projection-build.json', JSON.stringify(manifest, null, 2)); +console.log('Compiled baseline, pending-only, and current owning service modules; no runtime TypeScript loader.'); diff --git a/server/scripts/profile-tool-projection/probe.mjs b/server/scripts/profile-tool-projection/probe.mjs new file mode 100644 index 00000000000..41e43cbd9dd --- /dev/null +++ b/server/scripts/profile-tool-projection/probe.mjs @@ -0,0 +1,20 @@ +import http from 'node:http'; +const agent = new http.Agent({ keepAlive: true }); +let active = false, pending, times = [], port; +function ping() { + if (!active || pending) return; + pending = new Promise(resolve => { + const start = performance.now(); + const request = http.get({ host: '127.0.0.1', port, path: '/api/health', agent }, response => { + response.resume(); response.on('end', () => { times.push(performance.now() - start); resolve(); }); + }); + request.setTimeout(20000, () => request.destroy(new Error('Probe timeout'))); + request.on('error', error => { times.push({ error: error.message }); resolve(); }); + }).finally(() => { pending = null; }); +} +const interval = setInterval(ping, 20); +process.on('message', async message => { + if (message.kind === 'arm') { port = message.port; times = []; active = true; process.send({ kind: 'armed' }); } + if (message.kind === 'stop') { active = false; await pending; process.send({ kind: 'stopped', times }); } + if (message.kind === 'close') { clearInterval(interval); agent.destroy(); process.disconnect(); } +}); diff --git a/server/scripts/profile-tool-projection/register.mjs b/server/scripts/profile-tool-projection/register.mjs new file mode 100644 index 00000000000..3053c27df24 --- /dev/null +++ b/server/scripts/profile-tool-projection/register.mjs @@ -0,0 +1,13 @@ +import { registerHooks } from 'node:module'; +import { pathToFileURL, fileURLToPath } from 'node:url'; +import path from 'node:path'; +const root = fileURLToPath(new URL("../../..", import.meta.url)); +registerHooks({ resolve(specifier, context, next) { + for (const [name, dir] of [['@paperclipai/db', 'packages/db'], ['@paperclipai/shared', 'packages/shared']]) { + if (specifier === name || specifier.startsWith(name + '/')) { + const file = specifier === name ? 'index' : specifier.slice(name.length + 1); + return next(pathToFileURL(path.join(root, dir, 'dist', `${file}.js`)).href, context); + } + } + return next(specifier, context); +} }); diff --git a/server/scripts/profile-tool-projection/responsiveness.mjs b/server/scripts/profile-tool-projection/responsiveness.mjs new file mode 100644 index 00000000000..d38d4b754e5 --- /dev/null +++ b/server/scripts/profile-tool-projection/responsiveness.mjs @@ -0,0 +1,74 @@ +import assert from 'node:assert/strict'; +import path from 'node:path'; +import { createServer } from 'node:http'; +import { fork } from 'node:child_process'; +import { once } from 'node:events'; +import { writeFile, mkdir } from 'node:fs/promises'; +import { createDb } from '@paperclipai/db'; +import { startEmbeddedPostgresTestDatabase } from '../../dist/__tests__/helpers/embedded-postgres.js'; +import { createProfileProjectionFixture } from '../../dist/__tests__/helpers/tool-profile-projection-fixture.js'; +const index = process.argv.indexOf('--output-dir'); +const output = path.resolve(index < 0 ? '/tmp/paperclip-tool-projection' : process.argv[index + 1]); +assert.equal(typeof global.gc, 'function', 'Run Node with --expose-gc'); +await mkdir(output, { recursive: true }); +const database = await startEmbeddedPostgresTestDatabase('paperclip-hotpaths-http-owned-'); +const db = createDb(database.connectionString, { maxConnections: 1 }); +const server = createServer((_request, response) => { response.writeHead(200, { 'Content-Type': 'application/json' }); response.end('{"status":"ok"}'); }); +let probe; +const control = async message => { + const reply = once(probe, 'message', { signal: AbortSignal.timeout(30000) }); + probe.send(message); + return (await reply)[0]; +}; +const records = []; +try { + server.listen(0, '127.0.0.1'); await once(server, 'listening'); + probe = fork(new URL('./probe.mjs', import.meta.url), { execArgv: [], stdio: ['ignore', 'inherit', 'inherit', 'ipc'] }); + const fixture = await createProfileProjectionFixture(db); + const variants = {}, now = () => new Date('2026-02-01T00:00:00Z'); + for (const label of ['baseline', 'combined']) { + const p = label === 'combined' ? 'service' : '.baseline-service'; + const a = label === 'combined' ? 'attention' : '.baseline-attention'; + const profiles = (await import(new URL(`../../dist/services/tool-access/profiles/${p}.js`, import.meta.url))).createToolProfileService(db, now); + const attention = (await import(new URL(`../../dist/services/tool-access/catalog/${a}.js`, import.meta.url))).createToolAppAttention(db, now); + variants[label] = { profiles: () => profiles.listProfiles(fixture.company.id), attention: () => attention(fixture.company.id) }; + } + for (const variant of Object.values(variants)) for (const owner of ['profiles', 'attention']) await variant[owner](); + for (let round = 0; round < 3; round++) { + for (const owner of ['profiles', 'attention']) { + for (const label of round % 2 ? ['combined', 'baseline'] : ['baseline', 'combined']) { + global.gc?.(); + await control({ kind: 'arm', port: server.address().port }); + const start = performance.now(); + await variants[label][owner](); + const operationMs = performance.now() - start; + const result = await control({ kind: 'stop' }); + const good = result.times.filter(value => typeof value === 'number'); + const record = { kind: 'responsiveness', round, owner, label, operationMs, + completedHealthRequests: good.length, maxHealthMs: good.length ? Math.max(...good) : null, + errors: result.times.filter(value => typeof value !== 'number'), + scope: 'Separate-process HTTP probe to a minimal liveness handler in the same Node event loop as the real service; not the full application health route' }; + records.push(record); console.log(JSON.stringify(record)); + } + } + } +} finally { + try { + await writeFile(path.join(output, 'responsiveness.json'), JSON.stringify(records, null, 2)); + } finally { + try { + if (probe && probe.exitCode === null && probe.signalCode === null) { + const exited = once(probe, 'exit'); + if (probe.connected) probe.send({ kind: 'close' }); + else probe.kill(); + await exited; + } + } finally { + try { await new Promise(resolve => server.close(resolve)); } + finally { + try { await db.$client.end({ timeout: 1 }); } + finally { await database.cleanup(); } + } + } + } +} From 554a770c6f8e48e327e19f5208b575a5ec5bce30 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Mon, 2 Nov 2026 16:55:56 -0600 Subject: [PATCH 08/66] perf(apps): scope connection profile configuration reads (#18687) ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - The Apps Permissions page reads saved access profiles for one connection. > - The page fetched every company profile, rule, and binding. > - It used only the app profile and eligible task-granted profiles for that connection. > - This pull request adds a saved configuration read owned by the profile service. > - The route keeps current connection and per-profile management checks. > - The benefit is fewer database rows, less Node work, and a smaller response for the same displayed configuration. ## Linked Issues or Issue Description **Subsystem affected** Server profile configuration reads and the Apps Permissions and Review data caller. **Problem or motivation** One connection page loaded and serialized unrelated company profiles. In the local fixture, this produced a 32.35 MB response with 76,001 rule rows. **Proposed solution** Select the displayed connection's app profile and exact eligible connection-intent profiles before loading their entries and bindings. Return an explicit configuration DTO with complete selected arrays. Keep summary and pending-review work on the existing full API. **Alternatives considered** Filtering the full response in the browser retains the database and Node cost. Truncating the full API would break other clients. Filtering entries to the connection would lose selectors and bindings needed by saves. **Roadmap alignment** This improves the existing governed Apps capability. It does not add a new access policy. Related merged work: #18673 and #18681. Related harness cleanup: #18683. Public PR searches found no duplicate connection configuration projection. ## What Changed - Add `ToolProfileConfiguration` and `GET /api/tool-connections/{connectionId}/profile-configuration`. - Select `app:` at any status and exact active, correctly bound `connection_intent` profiles. Run the existing per-profile visibility filter before configuration queries. - Preserve complete ordered entries and bindings. Keep all save behavior, empty-binding install fallback, and the existing full profile API. - Switch AppDetail and AgentConnectionAccess to the configuration DTO. Use a distinct cache key below the existing profiles prefix so mutation and reconnect invalidations still refresh it. - Add real PostgreSQL differential and authenticated HTTP tests, UI caller coverage, a reproducible benchmark, and access-map documentation. ## Verification Baseline: `3f8ebfff8aa9febd0733c6f65b07485dcadfd0e2`, after both #18673 and #18681. Final head: `b40c1ed7c0833d403e20ee8a5ff9d09f1bad6d22`. Current implementation and reproduction details are in [the performance note](https://github.com/paperclipai/paperclip/blob/b40c1ed7c0833d403e20ee8a5ff9d09f1bad6d22/doc/performance/2026-11-02-connection-profile-configuration.md). - AppDetail: 79 tests passed. Existing access/action saves, install fallback, grant removal, errors, and refresh cases use the new caller. - Real PostgreSQL differential: passed. Full ordered selected configuration equals the old full-list selection. Cases cover extra bindings, sibling selectors, wrong source/key/status/binding, null and non-string agent metadata, fresh mutations, empty app bindings, visibility filtering, and company scope. - Production HTTP authentication: API key and run JWT cases passed. Hidden personal accounts omit spanning profiles. Private connection reads fail. Restored ownership restores visibility. Revoked management grants deny fresh calls. Final UID canonicalization and foreign-company 404 assertions passed on the current head. - Map structural check, token gate, and `git diff --check`: passed. - Required local `pnpm -r typecheck` and `pnpm build`: passed on the current head. `pnpm test:run` was started once. Its first chat shard passed (337 tests, 151.87 seconds); the second shard was intentionally interrupted after the complete final-head CI suite passed. This partial local attempt is not a full local test pass. The same stable-runner general and serialized partitions and Runner lanes passed in [final-head CI](https://github.com/paperclipai/paperclip/actions/runs/41538805231), including aggregate `ci / verify`. No local assertion failures were observed before interruption. - Final-head focused acceptance: four files, 85 tests passed. Full token gates, maps, and module boundaries passed. - Actual browser acceptance passed on the current checkout with isolated real PostgreSQL and a local trusted board. An action saved as Off persisted after reload. Both app agent bindings stayed intact. Removing an agent-only task grant persisted after reload. Requests used only the narrow profile endpoint. No browser exceptions occurred. No responses were mocked. No provider calls or hosted changes were made. The helper stopped the owned runtime and database after the check. - Final exact-head checks: 54 success, one clean neutral Paperclip Review, three skipped, zero pending or failures. Legacy Snyk status: success. No inline review comments. - Greptile reviewed exact head `b40c1ed7c0833d403e20ee8a5ff9d09f1bad6d22`: 5/5, no findings, completed successful check. Five alternating unprofiled pairs ran with real PostgreSQL on macOS/arm64, Node v24.22.2, and forced GC. The fixture has 210 profiles (200 heavy profiles plus ten eligibility cases), 76,001 rules, 1,900 active tools, 695 removed tools, and 12 agents. The selected configuration has two profiles and 1,391 rules. Each pair asserts complete ordered selected DTO equality. | Measurement | Full list | Configuration read | | --- | --- | --- | | Service wall ms, median [min, max] | 200.927 [193.319, 213.390] | 6.414 [5.508, 6.973] | | Node CPU ms, median [min, max] | 262.751 [246.727, 271.573] | 27.525 [22.576, 29.308] | | Select queries / returned rows | 7 / 78,139 | 3 / 1,397 | | Actual response JSON bytes | 32,353,534 | 592,233 | | Serialization wall ms, median [min, max] | 90.714 [88.715, 99.304] | 1.644 [1.637, 1.745] | | Serialization CPU ms, median [min, max] | 90.644 [88.569, 96.778] | 1.654 [1.637, 1.807] | Reproduce: `node --expose-gc --import ./server/node_modules/tsx/dist/loader.mjs server/scripts/benchmark-connection-profile-configuration.ts 5`. The benchmark measures the service and actual response JSON separately. It excludes HTTP and authorization work through an identity visibility callback. Node CPU does not include PostgreSQL CPU. No hosted or foreground-under-load improvement is claimed. Preserved failures: offline install lacked a locked tarball; frozen online install passed. Sandbox localhost and tsx IPC restrictions blocked initial DB/typecheck attempts; permitted retries use the same checks. The first HTTP suite needed the unbuilt plugin SDK; its owning build helper fixed setup. Shared typecheck caught a missing barrel export, now fixed. A broad typecheck inherited the host's unpinned nested pnpm executable; the owned stalled process was stopped and restarted with pinned pnpm 9. The complete current-head CI suite and aggregate verify passed. The separate Paperclip agent review published 5/5 with no findings at this head and completed with a clean neutral conclusion. An added foreign-company assertion expected 403; the existing guard correctly returned 404. Only the test expectation changed. The first real browser attempt saved and reloaded the action correctly, but grant deletion returned 422: the stress fixture had a company binding, which makes that profile the company default. The UI showed the existing deletion error. The second browser seed used the production agent-only intent shape and passed removal and reload. This does not change the stress fixture or product deletion gates. Initial external browser script attempts also needed an ESM extension and the correct dependency path. Logs, first-failure screenshots, traces, and result files are retained in the local evidence directory. ## Risks - SQL selection must stay aligned with connection-intent eligibility. Differential fixtures cover the exact current predicate. - A selected connection can still have many rules. This change preserves all selected configuration. - Per-profile authorization still reads current selected resources and can add work. Connection visibility does not bypass that check. - No migration, default-allow, effective-access, policy-review, provider, or deployment behavior changes. The old endpoint remains available. Rollback restores the old AppDetail caller and removes the additive endpoint. ## Model Used OpenAI Codex, `gpt-6.1-sol`, with code execution and repository tools. The host does not expose the context-window size. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- ...-11-02-connection-profile-configuration.md | 81 +++++++++++++++++++ maps/agent-access-map/app-permissions.md | 9 +++ maps/feature-map/app-permissions.md | 2 + packages/shared/src/index.ts | 1 + packages/shared/src/types/index.ts | 1 + packages/shared/src/types/tool-access.ts | 6 ++ ...chmark-connection-profile-configuration.ts | 57 +++++++++++++ .../connection-management.integration.test.ts | 49 ++++++++++- .../connection-profile-configuration.test.ts | 44 ++++++++++ ...onnection-profile-configuration-fixture.ts | 51 ++++++++++++ server/src/routes/openapi.ts | 9 +++ server/src/routes/tool-access.ts | 9 +++ .../profiles/connection-configuration.ts | 64 +++++++++++++++ .../services/tool-access/profiles/service.ts | 3 + ui/src/api/tools.ts | 3 + ui/src/lib/queryKeys.ts | 2 + ui/src/pages/apps/AppDetail.test.tsx | 25 ++++-- ui/src/pages/apps/AppDetail.tsx | 12 +-- .../apps/app-detail/AgentConnectionAccess.tsx | 4 +- 19 files changed, 415 insertions(+), 17 deletions(-) create mode 100644 doc/performance/2026-11-02-connection-profile-configuration.md create mode 100644 server/scripts/benchmark-connection-profile-configuration.ts create mode 100644 server/src/__tests__/connection-profile-configuration.test.ts create mode 100644 server/src/__tests__/helpers/connection-profile-configuration-fixture.ts create mode 100644 server/src/services/tool-access/profiles/connection-configuration.ts diff --git a/doc/performance/2026-11-02-connection-profile-configuration.md b/doc/performance/2026-11-02-connection-profile-configuration.md new file mode 100644 index 00000000000..bf2d7c51fab --- /dev/null +++ b/doc/performance/2026-11-02-connection-profile-configuration.md @@ -0,0 +1,81 @@ +# Connection Permissions configuration read + +Baseline: `3f8ebfff8aa9febd0733c6f65b07485dcadfd0e2`, which includes #18673 +and #18681. This change removes unrelated profile reads from AppDetail. The +profile service owns selection and projection. The route retains the connection +management guard and the full-list route's per-profile visibility checks. The +legacy full profile endpoint remains unchanged. + +The new configuration DTO contains saved profile fields and complete ordered +entries and bindings. It has no summary or pending-review counts. Selection +matches the old UI: `app:` at any status, plus active +`connection_intent` profiles with exact metadata/key and a matching agent binding. +This is a presentation view, not effective access. Complete bindings preserve +save behavior and the empty-binding install fallback. + +## Local experiment + +Run with Node 24, pnpm 9 and installed dependencies: + +```sh +node --expose-gc --import ./server/node_modules/tsx/dist/loader.mjs \ + server/scripts/benchmark-connection-profile-configuration.ts 5 +``` + +The benchmark uses the existing real PostgreSQL profile fixture: 200 heavy +profiles, 76,000 rules, 1,900 active tools, 695 removed tools, and 12 agents. It +adds ten eligibility cases and one cross-connection rule. The selected connection +has two profiles and 1,391 rules. Each sample asserts equality of the complete +ordered selected configuration against the old full-list selection. Selected +rules are not truncated, even when they refer to another connection. + +Five alternating unprofiled pairs ran in a coordinated quiet window on +macOS/arm64, Node v24.22.2, with forced GC before samples. Times below are +median [minimum, maximum] milliseconds. CPU measures Node process CPU, not +PostgreSQL CPU. The benchmark uses an identity visibility callback and excludes +HTTP, middleware, and authorization work. It separately serializes the actual +old full response and new configuration response. + +| Measurement | Full list | Connection configuration | +| --- | --- | --- | +| Service wall | 200.927 [193.319, 213.390] | 6.414 [5.508, 6.973] | +| Node CPU | 262.751 [246.727, 271.573] | 27.525 [22.576, 29.308] | +| Select queries | 7 | 3 | +| Returned database rows | 78,139 | 1,397 | +| Actual response JSON bytes | 32,353,534 | 592,233 | +| JSON serialization wall | 90.714 [88.715, 99.304] | 1.644 [1.637, 1.745] | +| JSON serialization CPU | 90.644 [88.569, 96.778] | 1.654 [1.637, 1.807] | + +This establishes local service work and payload reduction. It does not establish +hosted latency, foreground responsiveness under load, or live provider behavior. +Cost still grows with the selected profiles' complete configuration. Current +agent authorization can separately read selected profile entries and bindings; +those reads are excluded from this benchmark and remain in the request path. + +## Correctness and acceptance + +- The PostgreSQL differential test covers exact eligibility, null/non-string + agent metadata, inactive intent profiles, wrong key/source/bindings, sibling + and unmanaged profiles, complete binding ordering, fresh entry changes, empty + app bindings, denied profile selection, and company isolation. +- The assembled HTTP integration fixture exercises standard agent keys and run + JWTs. It checks private account visibility, missing connections, and fresh + permission revocation. A profile spanning a hidden account is omitted while + the displayed connection stays readable. +- AppDetail's component suite covers saved access and action toggles, complete + access restatement, install fallback, removing grants, errors, and refresh. + The configuration key is a distinct child of the existing profiles prefix, + so existing mutation and reconnect invalidations still refresh it. + +Preserved setup failures: the offline install lacked a locked package tarball; +a frozen online install passed. The first database attempt failed on sandbox +localhost binding restrictions; the escalated real PostgreSQL run passed. The +first HTTP suite could not resolve the unbuilt plugin SDK; the owning +`ensure-build-deps` command built it and the retry passed. A missing shared +barrel export failed the first shared typecheck and was fixed. The first broad +typecheck inherited an unpinned nested pnpm executable; its stalled owned +processes were stopped and the command was restarted with a pinned pnpm wrapper. + +The added foreign-company HTTP assertion first expected 403. The existing guard +returned 404 to hide the resource, as designed. The test expectation was corrected; +production authorization did not change. diff --git a/maps/agent-access-map/app-permissions.md b/maps/agent-access-map/app-permissions.md index 78877ca1bf1..8bef4add8b2 100644 --- a/maps/agent-access-map/app-permissions.md +++ b/maps/agent-access-map/app-permissions.md @@ -83,6 +83,15 @@ Evidence: `verified` — [authenticated integration fixtures](../../server/src/_ Verification: `passed` — deterministic local fixture; no live provider qualification. Gaps: Provider fixtures do not qualify live provider setup or availability. +### `get-tool-connections-profile-configuration` + +API: `present` — `GET /api/tool-connections/{connectionId}/profile-configuration`; JSON REST. Returns saved app and eligible connection-intent profiles, including complete ordered entries and bindings. Omits summary and pending-review counts. The company-wide full profile list stays available. +Agent access: `available` — agent REST and default native `search_api` / `call_api`. +Policy: `tools:manage_profiles`; the displayed connection and every selected profile use current management-resource authorization. Profiles that span a hidden personal account are omitted. Connection visibility alone does not grant profile visibility. +Evidence: `verified` — [authenticated integration fixture](../../server/src/__tests__/connection-management.integration.test.ts) covers standard agent keys and run JWTs, private accounts, missing resources, and revoked authority on local PostgreSQL. +Verification: `passed` — deterministic local fixture. [Configuration differential test](../../server/src/__tests__/connection-profile-configuration.test.ts) compares complete selected entries and bindings with the full list, including mutations and empty bindings. +Gaps: These fixtures do not qualify live providers or hosted performance. + ### `get-tool-connections-installs` API: `present` — `GET /api/tool-connections/{connectionId}/installs`; JSON REST. diff --git a/maps/feature-map/app-permissions.md b/maps/feature-map/app-permissions.md index 0fd4b55a1df..c725de0d1f7 100644 --- a/maps/feature-map/app-permissions.md +++ b/maps/feature-map/app-permissions.md @@ -49,6 +49,8 @@ Preconditions: follow the [baseline](./README.md#before-driving-a-journey). Prep ### `permissions` +The page loads saved profiles for the displayed connection through the configuration read. It preserves complete app bindings for saves and the install fallback when bindings are empty. It does not request profile summaries or pending-review counts. + Automated: [app detail](../../ui/src/pages/apps/AppDetail.test.tsx) covers agent/action controls, quarantined actions, and navigation using mocked requests. [Installation permission regressions](../../server/src/__tests__/tool-connection-installs.test.ts) diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index da43d193d30..676ca8ed6b0 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -1526,6 +1526,7 @@ export type { ToolProfileSummary, ToolProfileStatus, ToolProfileWithDetails, + ToolProfileConfiguration, ToolRateLimitCounter, ToolRateLimitRule, ToolRateLimitWindowKind, diff --git a/packages/shared/src/types/index.ts b/packages/shared/src/types/index.ts index 2f4c4e616cc..a00d768be26 100644 --- a/packages/shared/src/types/index.ts +++ b/packages/shared/src/types/index.ts @@ -600,6 +600,7 @@ export type { ToolProfileSummary, ToolProfileStatus, ToolProfileWithDetails, + ToolProfileConfiguration, ToolRateLimitCounter, ToolRateLimitRule, ToolRateLimitWindowKind, diff --git a/packages/shared/src/types/tool-access.ts b/packages/shared/src/types/tool-access.ts index 5627ca6a174..961a97a8f99 100644 --- a/packages/shared/src/types/tool-access.ts +++ b/packages/shared/src/types/tool-access.ts @@ -785,6 +785,12 @@ export interface ToolProfileSummary { isCompanyDefault: boolean; } +/** Saved configuration only; no effective-access summary or pending-review calculation. */ +export interface ToolProfileConfiguration extends Omit { + entries: ToolProfileEntry[]; + bindings: ToolProfileBinding[]; +} + export interface ToolProfileWithDetails extends ToolProfile { entries: ToolProfileEntry[]; bindings: ToolProfileBinding[]; diff --git a/server/scripts/benchmark-connection-profile-configuration.ts b/server/scripts/benchmark-connection-profile-configuration.ts new file mode 100644 index 00000000000..aad1bc74cbf --- /dev/null +++ b/server/scripts/benchmark-connection-profile-configuration.ts @@ -0,0 +1,57 @@ +import assert from "node:assert/strict"; +import { createDb, type Db } from "@paperclipai/db"; +import { startEmbeddedPostgresTestDatabase } from "../src/__tests__/helpers/embedded-postgres.js"; +import { createToolProfileService } from "../src/services/tool-access/profiles/service.js"; +import { createConnectionProfileConfigurationFixture, legacyConnectionConfiguration } from "../src/__tests__/helpers/connection-profile-configuration-fixture.js"; + +const repeat = Number(process.argv[2] ?? 5); +if (!Number.isInteger(repeat) || repeat < 1) throw new Error("Positive repeat count required"); +const database = await startEmbeddedPostgresTestDatabase("connection-profile-config-benchmark-"); +const db = createDb(database.connectionString, { maxConnections: 1 }); +let queries = 0, rows = 0; +const measuredDb = new Proxy(db, { get(target, key) { + if (key !== "select") return Reflect.get(target, key); + return (...fields: Parameters) => { + const builder = target.select(...fields), from = builder.from.bind(builder); + builder.from = ((...args: Parameters) => { + const selection = from(...args), execute = selection.execute.bind(selection); + selection.execute = async () => { const result = await execute(); queries++; rows += result.length; return result; }; + return selection; + }) as typeof builder.from; + return builder; + }; +} }); +try { + const f = await createConnectionProfileConfigurationFixture(db); + const service = createToolProfileService(measuredDb, () => new Date("2026-02-01T00:00:00Z")); + const invoke = async (label: "before" | "after") => { + if (label === "before") { + const profiles = await service.listProfiles(f.company.id); + return { profiles, selected: legacyConnectionConfiguration(profiles, f.connection.id) }; + } + const profiles = await service.listConnectionProfileConfiguration(f.company.id, f.connection.id, async rows => rows); + return { profiles, selected: profiles }; + }; + const baseline = await invoke("before"), treatment = await invoke("after"); + assert.deepStrictEqual(treatment.selected, baseline.selected); + console.log(JSON.stringify({ node: process.version, platform: `${process.platform}/${process.arch}`, fixture: { + profiles: baseline.profiles.length, rules: baseline.profiles.reduce((n, p) => n + p.entries.length, 0), + activeTools: 1900, removedTools: 695, agents: 12, selectedProfiles: treatment.profiles.length, + selectedRules: treatment.profiles.reduce((n, p) => n + p.entries.length, 0), + }, equality: "full ordered selected configuration", scope: "real PostgreSQL service + actual response JSON; identity filter, excludes HTTP/auth" })); + for (let round = 0; round < repeat; round++) { + for (const label of round % 2 ? ["after", "before"] as const : ["before", "after"] as const) { + global.gc?.(); queries = 0; rows = 0; + const start = performance.now(), cpu = process.cpuUsage(); + const result = await invoke(label); + const serviceMs = performance.now() - start, used = process.cpuUsage(cpu); + const serializeStart = performance.now(), serializeCpu = process.cpuUsage(); + const json = JSON.stringify({ profiles: result.profiles }); + const serializationMs = performance.now() - serializeStart, serializeUsed = process.cpuUsage(serializeCpu); + assert.deepStrictEqual(result.selected, baseline.selected); + console.log(JSON.stringify({ round, label, serviceMs, cpuMs: (used.user + used.system) / 1000, queries, rows, + jsonBytes: Buffer.byteLength(json), serializationMs, serializationCpuMs: (serializeUsed.user + serializeUsed.system) / 1000, + forcedGc: Boolean(global.gc) })); + } + } +} finally { await db.$client.end({ timeout: 1 }); await database.cleanup(); } diff --git a/server/src/__tests__/connection-management.integration.test.ts b/server/src/__tests__/connection-management.integration.test.ts index a1f3f3e301d..4ddbe7d4021 100644 --- a/server/src/__tests__/connection-management.integration.test.ts +++ b/server/src/__tests__/connection-management.integration.test.ts @@ -6,7 +6,7 @@ import { join } from "node:path"; import { createServer } from "node:http"; import { and, eq, sql } from "drizzle-orm"; import { beforeAll, afterAll, describe, expect, it, vi } from "vitest"; -import { createDb, authUsers, activityLog, agentApiKeys, companies, companySecrets, companySecretVersions, chatConversations, chatDeliveries, chatEndpoints, chatExternalPrincipals, chatIdentityLinks, toolPolicies, toolRateLimitCounters, connectionGrants, connectionGrantMembers, connectionGrantDelegations, toolAccessAuditEvents, toolOauthStates, issueThreadInteractions, connectionIntentDeliveries, agents, companyMemberships, heartbeatRuns, issues, principalPermissionGrants, projects, toolApplications, toolConnections, toolConnectionInstalls, aiConnectionPools, toolMcpGateways, toolMcpGatewayTokens, toolActionRequests, toolCatalogEntries, toolInvocations, toolCallEvents, toolProfiles, toolProfileBindings, plugins, adapterAuthSessions, environments, routines, chatGitHubReviews } from "@paperclipai/db"; +import { createDb, authUsers, activityLog, agentApiKeys, companies, companySecrets, companySecretVersions, chatConversations, chatDeliveries, chatEndpoints, chatExternalPrincipals, chatIdentityLinks, toolPolicies, toolRateLimitCounters, connectionGrants, connectionGrantMembers, connectionGrantDelegations, toolAccessAuditEvents, toolOauthStates, issueThreadInteractions, connectionIntentDeliveries, agents, companyMemberships, heartbeatRuns, issues, principalPermissionGrants, projects, toolApplications, toolConnections, toolConnectionInstalls, aiConnectionPools, toolMcpGateways, toolMcpGatewayTokens, toolActionRequests, toolCatalogEntries, toolInvocations, toolCallEvents, toolProfileEntries, toolProfiles, toolProfileBindings, plugins, adapterAuthSessions, environments, routines, chatGitHubReviews } from "@paperclipai/db"; import { findSatisfiedToolConnection, satisfiedConnectionIntentService } from "../services/satisfied-connection-intents.js"; import { connectionManagementService, continuingConnectionManagement } from "../services/connection-management.js"; import { aiConnectionService } from "../services/ai-connections.js"; @@ -408,6 +408,53 @@ describe("connection management with production default agents", () => { for (const id of profileIds) expect(await f.request("GET", `/tool-profiles/${id}/new-tools?connectionId=${first.body.id}`)).toMatchObject({ status: 200 }); }); + it("keeps connection profile configuration scoped and rechecks each profile against current personal access", async () => { + const f = await fixture(); + const create = async (name: string) => { + const response = await f.request("POST", `/companies/${f.companyId}/tools/connections`, { + name, transport: "mcp_remote", authKind: "none", config: { url: "https://1.1.1.1/mcp" }, + }); + expect(response).toMatchObject({ status: 201 }); + return response.body; + }; + const owned = await create("Configuration owned"), privateConnection = await create("Configuration private"); + const [app, intent] = await server.db.insert(toolProfiles).values([ + { companyId: f.companyId, profileKey: `app:${owned.id}`, name: "App" }, + { companyId: f.companyId, profileKey: `connection-intent:${owned.id}:${f.agentId}`, name: "Intent", + metadata: { source: "connection_intent", connectionId: owned.id, agentId: f.agentId } }, + ]).returning(); + await server.db.insert(toolProfileBindings).values({ companyId: f.companyId, profileId: intent.id, targetType: "agent", targetId: f.agentId }); + await server.db.insert(toolProfileEntries).values([ + { companyId: f.companyId, profileId: app.id, selectorType: "connection", connectionId: privateConnection.id }, + { companyId: f.companyId, profileId: intent.id, selectorType: "connection", connectionId: owned.id }, + ]); + const path = `/tool-connections/${owned.id}/profile-configuration`; + const uidResult = await f.request("GET", `/tool-connections/${encodeURIComponent(owned.uid)}/profile-configuration`); + expect(uidResult).toMatchObject({ status: 200 }); + expect(uidResult.body.profiles.map((p: { id: string }) => p.id).sort()).toEqual([app.id, intent.id].sort()); + const foreign = await fixture(); + expect(await foreign.request("GET", `/tool-connections/${owned.id}/profile-configuration`)).toMatchObject({ status: 404 }); + for (const token of [f.key.token, f.jwt]) { + const result = await f.request("GET", path, undefined, token); + expect(result).toMatchObject({ status: 200 }); + expect(result.body.profiles.map((p: { id: string }) => p.id).sort()).toEqual([app.id, intent.id].sort()); + expect(result.body.profiles.every((p: object) => !("summary" in p) && !("newToolsPendingCount" in p))).toBe(true); + } + // Seeing the displayed connection must not expose a profile spanning a + // second account after that account becomes private to another person. + await server.db.update(toolConnections).set({ credentialPolicy: "per_user", createdByUserId: randomUUID() }).where(eq(toolConnections.id, privateConnection.id)); + for (const token of [f.key.token, f.jwt]) { + expect(await f.request("GET", path, undefined, token)).toMatchObject({ status: 200, body: { profiles: [expect.objectContaining({ id: intent.id })] } }); + expect(await f.request("GET", `/tool-connections/${privateConnection.id}/profile-configuration`, undefined, token)).toMatchObject({ status: 403 }); + expect(await f.request("GET", `/tool-connections/${randomUUID()}/profile-configuration`, undefined, token)).toMatchObject({ status: 404 }); + } + await server.db.update(toolConnections).set({ createdByUserId: f.userId }).where(eq(toolConnections.id, privateConnection.id)); + expect((await f.request("GET", path)).body.profiles).toHaveLength(2); + await server.db.delete(principalPermissionGrants).where(and(eq(principalPermissionGrants.principalId, f.agentId), eq(principalPermissionGrants.permissionKey, "tools:manage_profiles"))); + expect(await f.request("GET", path)).toMatchObject({ status: 403 }); + expect(await f.request("GET", path, undefined, f.jwt)).toMatchObject({ status: 403 }); + }); + it("rechecks authority after remote endpoint validation before committing a connection update", async () => { const f = await fixture(); const created = await f.request("POST", `/companies/${f.companyId}/tools/connections`, { name: "Before validation", transport: "mcp_remote", authKind: "none", transportConfig: { url: "https://1.1.1.1/mcp" } }); diff --git a/server/src/__tests__/connection-profile-configuration.test.ts b/server/src/__tests__/connection-profile-configuration.test.ts new file mode 100644 index 00000000000..526596d3454 --- /dev/null +++ b/server/src/__tests__/connection-profile-configuration.test.ts @@ -0,0 +1,44 @@ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { eq } from "drizzle-orm"; +import { createDb, toolProfileBindings, toolProfileEntries } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { createConnectionProfileConfigurationFixture, legacyConnectionConfiguration } from "./helpers/connection-profile-configuration-fixture.js"; +import { createToolProfileService } from "../services/tool-access/profiles/service.js"; +const support = await getEmbeddedPostgresTestSupport(); +const describeDatabase = support.supported ? describe : describe.skip; +describeDatabase("connection Permissions profile configuration", () => { + let db: ReturnType, database: Awaited>; + beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("connection-profile-configuration-"); db = createDb(database.connectionString); }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }); + it("matches the old selected view with full ordered entries and bindings, before and after changes", async () => { + const f = await createConnectionProfileConfigurationFixture(db, { profiles: 8, tools: 12, rules: 4 }); + const service = createToolProfileService(db, () => new Date("2026-02-01T00:00:00Z")); + const visible = async (rows: T[]) => rows; + const compare = async () => { + const expected = legacyConnectionConfiguration(await service.listProfiles(f.company.id), f.connection.id); + const actual = await service.listConnectionProfileConfiguration(f.company.id, f.connection.id, visible); + expect(actual).toEqual(expected); + expect(actual.map(p => p.id).sort()).toEqual([...f.selectedIds].sort()); + expect(actual.every(p => !("summary" in p) && !("newToolsPendingCount" in p))).toBe(true); + return actual; + }; + let result = await compare(); + expect(result.find(p => p.id === f.selectedIds[0])!.bindings.map(b => b.priority)).toEqual([1, 2]); + expect(result.find(p => p.id === f.selectedIds[1])!.bindings).toHaveLength(2); + await db.insert(toolProfileEntries).values({ companyId: f.company.id, profileId: f.selectedIds[1], selectorType: "tool_name", toolName: "new selection" }); + await compare(); + // An unbound app profile must remain present for the install fallback. + await db.delete(toolProfileBindings).where(eq(toolProfileBindings.profileId, f.selectedIds[0])); + result = await compare(); + expect(result.find(p => p.id === f.selectedIds[0])!.bindings).toEqual([]); + const seen: string[][] = []; + const filtered = await service.listConnectionProfileConfiguration(f.company.id, f.connection.id, async rows => { + seen.push(rows.map(p => p.id)); return rows.filter(p => p.id !== f.selectedIds[1]); + }); + expect(seen[0].sort()).toEqual([...f.selectedIds].sort()); + expect(filtered.map(p => p.id)).toEqual([f.selectedIds[0]]); + expect(await service.listConnectionProfileConfiguration(f.company.id, f.connection.id, async () => [])).toEqual([]); + const foreign = await createConnectionProfileConfigurationFixture(db, { profiles: 2, tools: 3, rules: 2 }); + expect(await service.listConnectionProfileConfiguration(foreign.company.id, f.connection.id, visible)).toEqual([]); + }); +}); diff --git a/server/src/__tests__/helpers/connection-profile-configuration-fixture.ts b/server/src/__tests__/helpers/connection-profile-configuration-fixture.ts new file mode 100644 index 00000000000..1ea46603bfa --- /dev/null +++ b/server/src/__tests__/helpers/connection-profile-configuration-fixture.ts @@ -0,0 +1,51 @@ +import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { agents, toolConnections, toolProfileBindings, toolProfileEntries, toolProfiles, type Db } from "@paperclipai/db"; +import type { ToolProfileConfiguration, ToolProfileWithDetails } from "@paperclipai/shared"; +import { createProfileProjectionFixture } from "./tool-profile-projection-fixture.js"; + +export async function createConnectionProfileConfigurationFixture(db: Db, options: Parameters[1] = {}) { + const f = await createProfileProjectionFixture(db, options); + const companyAgents = await db.select().from(agents).where(eq(agents.companyId, f.company.id)); + const agentId = companyAgents[0]!.id; + const [sibling] = await db.insert(toolConnections).values({ companyId: f.company.id, applicationId: f.application.id, + name: "Sibling account", uid: randomUUID(), transport: "mcp_remote", status: "active" }).returning(); + await db.update(toolProfiles).set({ profileKey: `app:${f.connection.id}`, status: "disabled" }).where(eq(toolProfiles.id, f.profiles[0]!.id)); + const metadata = { source: "connection_intent", connectionId: f.connection.id, agentId }; + await db.update(toolProfiles).set({ profileKey: `connection-intent:${f.connection.id}:${agentId}`, metadata }).where(eq(toolProfiles.id, f.profiles[1]!.id)); + const rejected = await db.insert(toolProfiles).values([ + { profileKey: `app:${sibling.id}`, name: "Sibling app" }, + { profileKey: `connection-intent:${sibling.id}:${agentId}`, name: "Sibling intent", metadata: { ...metadata, connectionId: sibling.id } }, + { profileKey: `connection-intent:${f.connection.id}:wrong-key`, name: "Wrong key", metadata }, + { profileKey: `connection-intent:${f.connection.id}:${companyAgents[1]!.id}`, name: "Wrong binding", metadata: { ...metadata, agentId: companyAgents[1]!.id } }, + { profileKey: `connection-intent:${f.connection.id}:${companyAgents[2]!.id}`, name: "Inactive intent", status: "disabled" as const, metadata: { ...metadata, agentId: companyAgents[2]!.id } }, + { profileKey: `connection-intent:${f.connection.id}:123`, name: "Non-string agent", metadata: { ...metadata, agentId: 123 } }, + { profileKey: `connection-intent:${f.connection.id}:null`, name: "Null agent", metadata: { ...metadata, agentId: null } }, + { profileKey: "unmanaged", name: "Unmanaged", metadata }, + { profileKey: `connection-intent:${f.connection.id}:${companyAgents[3]!.id}`, name: "Wrong source", metadata: { ...metadata, agentId: companyAgents[3]!.id, source: "other" } }, + { profileKey: `connection-intent:${f.connection.id}:${companyAgents[4]!.id}`, name: "No binding", metadata: { ...metadata, agentId: companyAgents[4]!.id } }, + ].map(profile => ({ companyId: f.company.id, ...profile }))).returning(); + const bindingAt = new Date("2026-01-01T00:00:00Z"); + await db.insert(toolProfileBindings).values([ + { profileId: f.profiles[0]!.id, targetType: "agent" as const, targetId: agentId, priority: 2 }, + { profileId: f.profiles[0]!.id, targetType: "agent" as const, targetId: companyAgents[1]!.id, priority: 1 }, + { profileId: f.profiles[1]!.id, targetType: "agent" as const, targetId: agentId, priority: 0 }, + { profileId: f.profiles[1]!.id, targetType: "company" as const, targetId: f.company.id, priority: 3 }, + ...rejected.filter(p => p.name !== "No binding").map(p => ({ profileId: p.id, targetType: "agent" as const, targetId: agentId, priority: 0 })), + ].map(binding => ({ companyId: f.company.id, ...binding, createdAt: bindingAt }))); + // Selected profiles can carry selectors outside this account. Keep all rows. + await db.insert(toolProfileEntries).values({ companyId: f.company.id, profileId: f.profiles[0]!.id, + selectorType: "connection", connectionId: sibling.id, effect: "exclude" }); + return { ...f, sibling, agentId, selectedIds: [f.profiles[0]!.id, f.profiles[1]!.id], rejected }; +} + +/** The old AppDetail/AgentConnectionAccess selection, with only unused derived fields removed. */ +export function legacyConnectionConfiguration(profiles: ToolProfileWithDetails[], connectionId: string): ToolProfileConfiguration[] { + return profiles.filter(profile => { + const agentId = profile.metadata?.agentId; + return profile.profileKey === `app:${connectionId}` || (profile.status === "active" + && profile.metadata?.source === "connection_intent" && profile.metadata.connectionId === connectionId + && typeof agentId === "string" && profile.profileKey === `connection-intent:${connectionId}:${agentId}` + && profile.bindings.some(binding => binding.targetType === "agent" && binding.targetId === agentId)); + }).map(({ summary: _summary, newToolsPendingCount: _pending, ...configuration }) => configuration); +} diff --git a/server/src/routes/openapi.ts b/server/src/routes/openapi.ts index c766d6c4836..772b4706a19 100644 --- a/server/src/routes/openapi.ts +++ b/server/src/routes/openapi.ts @@ -12068,6 +12068,15 @@ registerCurrentRoute({ summary: "Handle a managed Vercel Connect OAuth callback", }); +registerCurrentRoute({ + "x-paperclip-transport": "rest", + "x-paperclip-connection-management": "connections:profiles", + method: "get", + path: "/api/tool-connections/{connectionId}/profile-configuration", + tags: ["tool-access"], + summary: "Read connection Permissions profile configuration without summaries or review counts", +}); + registerCurrentRoute({ "x-paperclip-transport": "rest", "x-paperclip-connection-management": "connections:profiles", diff --git a/server/src/routes/tool-access.ts b/server/src/routes/tool-access.ts index 3f3bb760e17..72209d84746 100644 --- a/server/src/routes/tool-access.ts +++ b/server/src/routes/tool-access.ts @@ -2327,6 +2327,15 @@ function connectorEnrollmentPrincipal(req: Request): string { res.json({ profiles: await filterConnectionManagementRows(db, req, "connections:profiles", await svc.listProfiles(companyId), row => ({ profileId: row.id })) }); }); + router.get("/tool-connections/:connectionId/profile-configuration", managementGuard("connections:profiles", "GET /tool-connections/:connectionId/profile-configuration", { personal: true }), async (req, res) => { + assertBoardOrAgent(req); + const existing = await getAccessibleResource(req, res, svc.getConnection(req.params.connectionId as string), "Tool connection not found"); + if (!existing) return; + const profiles = await svc.listConnectionProfileConfiguration(existing.companyId, existing.id, + rows => filterConnectionManagementRows(db, req, "connections:profiles", rows, row => ({ profileId: row.id }))); + res.json({ profiles }); + }); + router.get("/tool-profiles/:profileId/new-tools", managementGuard("connections:profiles", "GET /tool-profiles/:profileId/new-tools", { personal: true }), async (req, res) => { assertBoardOrAgent(req); const existing = await getAccessibleResource(req, res, svc.getProfile(req.params.profileId as string), "Tool profile not found"); diff --git a/server/src/services/tool-access/profiles/connection-configuration.ts b/server/src/services/tool-access/profiles/connection-configuration.ts new file mode 100644 index 00000000000..0538929ebaf --- /dev/null +++ b/server/src/services/tool-access/profiles/connection-configuration.ts @@ -0,0 +1,64 @@ +import { and, asc, desc, eq, inArray, or, sql } from "drizzle-orm"; +import { toolProfileBindings, toolProfileEntries, toolProfiles, type Db } from "@paperclipai/db"; +import type { ToolProfileConfiguration } from "@paperclipai/shared"; +import { toProfile, toProfileBinding, toProfileEntry } from "./projection.js"; + +type ProfileRow = typeof toolProfiles.$inferSelect; + +/** Saved profiles shown by connection Permissions; this is not effective access. */ +export function createConnectionProfileConfigurationReader(db: Db) { + return async ( + companyId: string, + connectionId: string, + visibleProfiles: (profiles: ProfileRow[]) => Promise, + ): Promise => { + const candidates = await db.select().from(toolProfiles).where(and( + eq(toolProfiles.companyId, companyId), + or( + eq(toolProfiles.profileKey, `app:${connectionId}`), + and( + eq(toolProfiles.status, "active"), + sql`${toolProfiles.metadata}->>'source' = 'connection_intent'`, + sql`${toolProfiles.metadata}->>'connectionId' = ${connectionId}`, + sql`jsonb_typeof(${toolProfiles.metadata}->'agentId') = 'string'`, + sql`${toolProfiles.profileKey} = 'connection-intent:' || ${connectionId} || ':' || (${toolProfiles.metadata}->>'agentId')`, + sql`exists (select 1 from ${toolProfileBindings} where + ${toolProfileBindings.companyId} = ${toolProfiles.companyId} + and ${toolProfileBindings.profileId} = ${toolProfiles.id} + and ${toolProfileBindings.targetType} = 'agent' + and ${toolProfileBindings.targetId} = ${toolProfiles.metadata}->>'agentId')`, + ), + ), + )).orderBy(desc(toolProfiles.updatedAt)); + // Preserve the full-list route's per-profile current-authority checks before + // loading configuration. Connection access alone does not grant profile access. + const profiles = await visibleProfiles(candidates); + if (profiles.length === 0) return []; + const ids = profiles.map(profile => profile.id); + const [entries, bindings] = await Promise.all([ + db.select().from(toolProfileEntries).where(and( + eq(toolProfileEntries.companyId, companyId), inArray(toolProfileEntries.profileId, ids), + )).orderBy(asc(toolProfileEntries.createdAt), asc(toolProfileEntries.id)), + db.select().from(toolProfileBindings).where(and( + eq(toolProfileBindings.companyId, companyId), inArray(toolProfileBindings.profileId, ids), + )).orderBy(asc(toolProfileBindings.priority), asc(toolProfileBindings.createdAt)), + ]); + const entriesByProfile = new Map(); + const bindingsByProfile = new Map(); + for (const entry of entries) { + const group = entriesByProfile.get(entry.profileId) ?? []; + group.push(toProfileEntry(entry)); + entriesByProfile.set(entry.profileId, group); + } + for (const binding of bindings) { + const group = bindingsByProfile.get(binding.profileId) ?? []; + group.push(toProfileBinding(binding)); + bindingsByProfile.set(binding.profileId, group); + } + return profiles.map(profile => ({ + ...toProfile(profile), + entries: entriesByProfile.get(profile.id) ?? [], + bindings: bindingsByProfile.get(profile.id) ?? [], + })); + }; +} diff --git a/server/src/services/tool-access/profiles/service.ts b/server/src/services/tool-access/profiles/service.ts index b0d9f6e8774..223ca9a9169 100644 --- a/server/src/services/tool-access/profiles/service.ts +++ b/server/src/services/tool-access/profiles/service.ts @@ -1,4 +1,5 @@ import type { ConnectionManagementContext } from "../../connection-management.js"; +import { createConnectionProfileConfigurationReader } from "./connection-configuration.js"; import { connectionActorUserId } from "../actor.js"; import { assertProfileEntryInputs } from "../../tool-profile-entry-validation.js"; import { randomUUID } from "node:crypto"; @@ -360,6 +361,8 @@ export function createToolProfileService(db: Db, now: () => Date) { } } return { + listConnectionProfileConfiguration: createConnectionProfileConfigurationReader(db), + listProfiles: async ( companyId: string, ): Promise => { diff --git a/ui/src/api/tools.ts b/ui/src/api/tools.ts index 4bf3021dfc4..1087548dfff 100644 --- a/ui/src/api/tools.ts +++ b/ui/src/api/tools.ts @@ -34,6 +34,7 @@ import type { ToolProfileEntrySelectorType, ToolProfileStatus, ToolProfileWithDetails, + ToolProfileConfiguration, ToolProfileNewToolReviewDecision, ToolProfileNewToolsReview, ToolProfileNewToolsReviewResult, @@ -480,6 +481,8 @@ export const toolsApi = { ), // --- Profiles --- + listConnectionProfileConfiguration: (connectionId: string) => + api.get<{ profiles: ToolProfileConfiguration[] }>(`/tool-connections/${connectionId}/profile-configuration`), listProfiles: (companyId: string) => api.get(`/companies/${companyId}/tools/profiles`), getProfileNewTools: (profileId: string) => diff --git a/ui/src/lib/queryKeys.ts b/ui/src/lib/queryKeys.ts index 13b265befa2..e11a10395e3 100644 --- a/ui/src/lib/queryKeys.ts +++ b/ui/src/lib/queryKeys.ts @@ -88,6 +88,8 @@ export const queryKeys = { actionRequests: (companyId: string, status: string) => ["tools", companyId, "action-requests", status] as const, gateways: (companyId: string) => ["tools", "gateways", companyId] as const, + connectionProfileConfiguration: (companyId: string, connectionId: string) => + ["tools", companyId, "profiles", "connection-configuration", connectionId] as const, profiles: (companyId: string) => ["tools", companyId, "profiles"] as const, profileNewTools: (profileId: string) => ["tools", "profiles", profileId, "new-tools"] as const, diff --git a/ui/src/pages/apps/AppDetail.test.tsx b/ui/src/pages/apps/AppDetail.test.tsx index 3650f68d88d..c227ff9ccbc 100644 --- a/ui/src/pages/apps/AppDetail.test.tsx +++ b/ui/src/pages/apps/AppDetail.test.tsx @@ -19,7 +19,7 @@ const listApplicationsMock = vi.hoisted(() => vi.fn()); const listGalleryMock = vi.hoisted(() => vi.fn()); const listConnectionsMock = vi.hoisted(() => vi.fn()); const listCatalogMock = vi.hoisted(() => vi.fn()); -const listProfilesMock = vi.hoisted(() => vi.fn()); +const listConnectionProfileConfigurationMock = vi.hoisted(() => vi.fn()); const listPoliciesMock = vi.hoisted(() => vi.fn()); const listConnectionActivityMock = vi.hoisted(() => vi.fn()); const listActionRequestsMock = vi.hoisted(() => vi.fn()); @@ -60,7 +60,7 @@ vi.mock("@/api/tools", () => ({ listGallery: (companyId: string) => listGalleryMock(companyId), listConnections: (companyId: string) => listConnectionsMock(companyId), listCatalog: (connectionId: string) => listCatalogMock(connectionId), - listProfiles: (companyId: string) => listProfilesMock(companyId), + listConnectionProfileConfiguration: (connectionId: string) => listConnectionProfileConfigurationMock(connectionId), deleteProfile: (profileId: string) => deleteProfileMock(profileId), listPolicies: (companyId: string) => listPoliciesMock(companyId), listConnectionActivity: (connectionId: string, limit: number) => @@ -409,7 +409,7 @@ describe("AppDetail", () => { }), ], }); - listProfilesMock.mockResolvedValue({ + listConnectionProfileConfigurationMock.mockResolvedValue({ profiles: [ { profileKey: "app:conn-1", @@ -1034,6 +1034,15 @@ describe("AppDetail", () => { }); }); + it("fetches the connection configuration under a distinct child of the profiles cache", async () => { + await renderAppDetail(); + expect(listConnectionProfileConfigurationMock).toHaveBeenCalledWith("conn-1"); + expect(queryKeys.tools.connectionProfileConfiguration("company-1", "conn-1").slice(0, 3)) + .toEqual(queryKeys.tools.profiles("company-1")); + expect(queryKeys.tools.connectionProfileConfiguration("company-1", "conn-1")) + .not.toEqual(queryKeys.tools.connectionProfileConfiguration("company-1", "conn-2")); + }); + it("shows and removes task-granted access without changing the connection's default permissions", async () => { const grant = { id: "task-grant", status: "active", profileKey: "connection-intent:conn-1:agent-1", @@ -1041,7 +1050,7 @@ describe("AppDetail", () => { bindings: [{ targetType: "agent", targetId: "agent-1" }], entries: [{ effect: "include", catalogEntryId: "catalog-write" }], }; - listProfilesMock.mockResolvedValue({ profiles: [grant] }); + listConnectionProfileConfigurationMock.mockResolvedValue({ profiles: [grant] }); listPoliciesMock.mockResolvedValue({ policies: [{ enabled: true, policyType: "require_approval", selectors: { catalogEntryId: "catalog-write" }, @@ -1051,7 +1060,7 @@ describe("AppDetail", () => { const section = Array.from(container.querySelectorAll("section")).find(section => section.textContent?.includes("Additional agent access"))!; expect(section.textContent).toContain("Write issue"); expect(section.textContent).toContain("Ask first"); - listProfilesMock.mockResolvedValue({ profiles: [] }); + listConnectionProfileConfigurationMock.mockResolvedValue({ profiles: [] }); await act(async () => section.querySelector("button")!.click()); await flushReact(); expect(deleteProfileMock).toHaveBeenCalledWith("task-grant"); @@ -1061,7 +1070,7 @@ describe("AppDetail", () => { }); it("keeps the task grant visible when removal fails", async () => { - listProfilesMock.mockResolvedValue({ profiles: [{ + listConnectionProfileConfigurationMock.mockResolvedValue({ profiles: [{ id: "task-grant", status: "active", profileKey: "connection-intent:conn-1:agent-1", metadata: { source: "connection_intent", connectionId: "conn-1", agentId: "agent-1" }, bindings: [{ targetType: "agent", targetId: "agent-1" }], @@ -1078,7 +1087,7 @@ describe("AppDetail", () => { it("persists agent access independently from always-installed agents", async () => { mockParams.tab = "permissions"; - listProfilesMock.mockResolvedValue({ + listConnectionProfileConfigurationMock.mockResolvedValue({ profiles: [{ profileKey: "app:conn-1", entries: [ @@ -1144,7 +1153,7 @@ describe("AppDetail", () => { it("uses the setup-style agent access question without install terminology", async () => { mockParams.tab = "permissions"; - listProfilesMock.mockResolvedValue({ + listConnectionProfileConfigurationMock.mockResolvedValue({ profiles: [{ profileKey: "app:conn-1", entries: [ diff --git a/ui/src/pages/apps/AppDetail.tsx b/ui/src/pages/apps/AppDetail.tsx index c57fa4806c0..bffed20f0fd 100644 --- a/ui/src/pages/apps/AppDetail.tsx +++ b/ui/src/pages/apps/AppDetail.tsx @@ -15,7 +15,7 @@ import { Check, Loader2, Pencil } from "lucide-react"; import type { ToolConnection, ToolPolicy, - ToolProfileWithDetails, + ToolProfileConfiguration, } from "@paperclipai/shared"; import { connectionDisplaySecondaryHint, @@ -133,9 +133,9 @@ function StandardAppDetail({ renderActions, renderAgentSettings, renderConnectio enabled: !!connectionId && needsCatalog, }); const profilesQuery = useQuery({ - queryKey: queryKeys.tools.profiles(selectedCompanyId ?? "__none__"), - queryFn: () => toolsApi.listProfiles(selectedCompanyId!), - enabled: !!selectedCompanyId && (activeTab === "review" || activeTab === "permissions"), + queryKey: queryKeys.tools.connectionProfileConfiguration(selectedCompanyId ?? "__none__", connectionId), + queryFn: () => toolsApi.listConnectionProfileConfiguration(connectionId), + enabled: !!selectedCompanyId && !!connectionId && (activeTab === "review" || activeTab === "permissions"), }); const policiesQuery = useQuery({ queryKey: queryKeys.tools.policies(selectedCompanyId ?? "__none__"), @@ -923,7 +923,7 @@ function StatusBadge({ status }: { status: StatusInfo }) { ); } -function enabledCatalogIds(profile: ToolProfileWithDetails | undefined): Set { +function enabledCatalogIds(profile: ToolProfileConfiguration | undefined): Set { const ids = new Set(); for (const entry of profile?.entries ?? []) { if (entry.effect === "include" && entry.catalogEntryId) ids.add(entry.catalogEntryId); @@ -959,7 +959,7 @@ function askFirstCatalogIds(policies: ToolPolicy[], connectionId: string): Set Date: Mon, 2 Nov 2026 15:01:52 -0800 Subject: [PATCH 09/66] fix: Compare native finaliser lease expiry at millisecond precision when releasing an expired owner (#17214) > [!NOTE] > Opened by Prospector's automated issue-fix pipeline. A maintainer reviews and decides; nothing here merges on its own. ## Thinking Path - Issue #16715 reports a defect in paperclipai/paperclip. - Root cause, as the fixing agent read it: Lease renewal writes 'lease_expires_at' from the database clock ('now() + interval'), so the stored value has microseconds. 'agentExecutionsHaveStopped' reads that value into a millisecond JS 'Date' and then releases the row with an exact 'eq' on it. The update never matches, so the stop check returns false on every tick and the agent stays in 'pausing'. - This pull request: Compare native finaliser lease expiry at millisecond precision when releasing an expired owner ## What Changed - `server/src/services/agent-execution-stop.ts`: The release update now truncates the stored lease expiry to milliseconds before comparing it with the value it read, using the same 'date_trunc('milliseconds', ...) = ...::timestamptz' pattern as 'chat-channels/activity.ts'. The other guards stay the same: owner, generation, phase and the null result. This was the only exact-match comparison on a lease expiry in 'server/src'. ## Verification - Reproducing test(s), added in this PR: `server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts` - They fail on the base commit and pass with the fix, each twice in a fresh sandbox. - 3 independent attempts wrote 3 reproductions; this fix passes every one of them. - The repository's compile command passes with the fix applied. - The full test suite passes except for the 10 file(s) that already fail on the base commit. ## Risks - Written by an automated pipeline, not a person: review it as you would an outside contribution. - Risk tier of the touched paths: 2. - The scope reviewer found it also changes, beyond the report: agentExecutionsHaveStopped for an expired finaliser row whose lease_expires_at is already millisecond-precise (written from a JS Date) - The scope reviewer found it also changes, beyond the report: Finaliser row whose lease was renewed or taken over between the select and the update - The scope reviewer found it also changes, beyond the report: Rows with null leaseExpiresAt ## Model Used - codex:gpt-6.1-sol, opus, sonnet (Anthropic Claude, via Claude Code; OpenAI, via the Codex CLI), writing and checking the change in an isolated sandbox. - opus (Anthropic Claude, via Claude Code), reviewing it independently. ## Linked Issues Fixes #16715 Other pull requests that name #16715: - #17204 (open): fix(heartbeat): stop loading full run rows and stale pids when pausing an agent - #16745 (open): fix(agent-lifecycle): release an expired native finalizer lease at millisecond precision ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [ ] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with 'Fixes: #' / 'Closes #' / 'Refs #' OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub '#NNN' / 'github.com/paperclipai/paperclip' URLs) - [ ] My branch name describes the change (e.g. 'docs/...', 'fix/...') and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [ ] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [ ] I will address all Greptile and reviewer comments before requesting merge Co-authored-by: commitperclip[bot] <284812584+commitperclip[bot]@users.noreply.github.com> --- .../agent-lifecycle/__tests__/lifecycle.test.ts | 15 +++++++++++++++ server/src/services/agent-execution-stop.ts | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts b/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts index 404e76d6b30..ab4e53d0839 100644 --- a/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts +++ b/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts @@ -414,6 +414,21 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id)))[0]).toMatchObject({ leaseOwner: null, leaseExpiresAt: null }); }); + it("releases an expired native owner whose renewed lease expiry has microsecond precision", async () => { + const agent = await hire(); + const [issue] = await db.insert(issues).values({ companyId, title: "Renewed native run" }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: agent.id, nativeIssueId: issue.id, + status: "cancelled", invocationSource: "on_demand", runtimeMode: "native", + startedAt: new Date(), finishedAt: new Date(), processPid: 2_000_000_000 }).returning(); + await db.insert(nativeRunFinalizations).values({ companyId, issueId: issue.id, runId: run.id, + phase: "executing", leaseOwner: "stopped-server", leaseExpiresAt: new Date(0) }); + await db.update(nativeRunFinalizations) + .set({ leaseExpiresAt: sql`'2020-01-01T00:00:00.123456Z'::timestamptz` }) + .where(eq(nativeRunFinalizations.runId, run.id)); + expect(await agentExecutionsHaveStopped(db, [agent.id])).toBe(true); + expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id)))[0]).toMatchObject({ leaseOwner: null, leaseExpiresAt: null }); + }); + it("does not advance termination while a cancelled process still runs", async () => { const agent = await hire(); const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: agent.id, diff --git a/server/src/services/agent-execution-stop.ts b/server/src/services/agent-execution-stop.ts index 3fad14e89b1..92e9e5964bb 100644 --- a/server/src/services/agent-execution-stop.ts +++ b/server/src/services/agent-execution-stop.ts @@ -64,7 +64,7 @@ export async function agentExecutionsHaveStopped(db: Db, agentIds: string[]) { const [cleared] = await db.update(nativeRunFinalizations).set({ leaseOwner: null, leaseExpiresAt: null, updatedAt: now }) .where(and(eq(nativeRunFinalizations.runId, coordinator.runId), eq(nativeRunFinalizations.leaseOwner, coordinator.leaseOwner!), - eq(nativeRunFinalizations.leaseExpiresAt, coordinator.leaseExpiresAt!), + sql`date_trunc('milliseconds', ${nativeRunFinalizations.leaseExpiresAt}) = ${coordinator.leaseExpiresAt!.toISOString()}::timestamptz`, eq(nativeRunFinalizations.controllerGeneration, coordinator.controllerGeneration), eq(nativeRunFinalizations.phase, coordinator.phase), isNull(nativeRunFinalizations.resultId))) .returning({ runId: nativeRunFinalizations.runId }); From 58308a3e6343bb4ef71fa7f87899d730dab2e312 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Mon, 2 Nov 2026 17:12:22 -0600 Subject: [PATCH 10/66] perf(apps): scope sidebar review counts to one connection (#18688) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip helps people manage AI agents and their governed app access. > - The Apps sidebar displays a Review badge for one connection. > - It polls the full company attention service every 30 seconds to get two counts. > - That service also loads every profile, rule and active catalog entry. > - This pull request adds an authorized connection read for only those counts. > - The same badge now avoids unrelated profile evaluation and database result decoding. ## Linked Issues or Issue Description **Current behavior** The contextual sidebar polls company-wide attention. It needs only one connection's signed pending action count and whether any catalog entries are quarantined. **Proposed behavior** Both sidebar variants poll a connection-scoped count endpoint with existing connection read authority. The full attention API stays unchanged. **Reason and benefit** The retained 200-profile fixture performs seven service queries and returns 80,115 rows for this badge. The new reader performs one query and returns one row. This reduces local service CPU and avoids unnecessary decoded profile/rule data. Related merged evaluator fixes are already in the baseline: Refs #18673, Refs #18681. A search found no existing PR for this scoped count reader. ROADMAP.md contains no duplicate planned work. ## What Changed - Add the narrow shared DTO, typed client, guarded REST/OpenAPI route and catalog-owned reader. - Count independent aggregates with both company boundaries and exact signed-argument semantics. Keep archived counts at zero. - Update both contextual sidebar variants. Keep the exact badge, 30 second interval and existing attention-prefix invalidation. - Add PostgreSQL differential tests, authenticated HTTP tests, sidebar tests, a repeatable benchmark and access documentation. ## Verification - Baseline: `3f8ebfff8aa9febd0733c6f65b07485dcadfd0e2`, including both merged evaluator fixes. Service treatment: `0827f7d0b3`; route/fixture refinement: `230c42ee15`; final local existence gate: `390a218254`. - Passed: focused PostgreSQL differential test; assembled HTTP test with board session and agent JWT; 14 sidebar component tests; token gates; map checks; diff check. - Fixture: 200 profiles, 76,000 rules, 1,900 active tools, 695 removed references, 12 agents, a sibling connection and 2,000 unrelated historical invocations. Complete normalized two-count results are equal on both paths. - Quiet local measurement: seven alternating warm pairs after two warmup pairs, Node 24.22.2, macOS, real disposable PostgreSQL. Legacy/new service CPU median: 213.891/1.680 ms; ranges: 209.329–228.552/0.515–6.137 ms. Wall median: 156.867/1.248 ms; ranges: 151.739–162.289/1.115–1.386 ms. Queries: 7/1. Returned rows: 80,115/1. - Run `cd server && node --expose-gc --import ./node_modules/tsx/dist/loader.mjs scripts/benchmark-connection-review-counts.ts`. See `doc/connection-review-counts.md` for scope and failure history. Byte-volume accounting runs outside timing; no HTTP payload reduction is claimed. - Initial raw scalar SQL failed with ambiguous columns. Typed subqueries fixed it and passed real PostgreSQL equality. Initial fixture agents lacked permissions and correctly received 403. The final fixture grants explicit authority and tests revocation. An initial benchmark included byte serialization in timing; the reported run excludes it. - Passed required local `pnpm -r typecheck` and `pnpm build`. The initial sandbox typecheck could not open the tsx IPC socket; the escalated rerun passed. Full monolithic `pnpm test:run` was not run locally: repeating a 15–25 minute suite serially across both performance PRs adds no distinct evidence. Broad suite acceptance uses final-head CI, which runs the same general/serialized test partitions and the native Runner lane. Final-head CI is in progress. - Passed built-static Chromium journey on an owned local-trusted server and disposable PostgreSQL: Review 3 appears, only scoped counts are requested by this sidebar, resolving requests/clearing quarantines then reloading removes the badge, and archived counts are zero. The first ad hoc launcher omitted required decision-service options and failed at startup; using the normal no-op wake callback fixed that harness. - Greptile identified an invalid resolved-request fixture label. Corrected `denied` to the declared `rejected`, then reran the real PostgreSQL differential test. The count SQL and non-pending exclusion are unchanged. - The first final-head CI failed the static cross-tenant existence-oracle guard because the new explicit resource company assertion had no nearby `hasCompanyAccess` gate. Existing management admission already rejects foreign resources with 404. The route now also follows the standard local two-step gate. The unchanged guard plus authenticated HTTP negative cases passed (3 tests). No allowlist or test was weakened. - These local service measurements do not prove hosted performance. No deployment is included. ## Risks Low risk. New read admission reuses existing current-authority checks. The old attention API is unchanged. Counts still require database work over relevant pending/catalog rows. No cache, schema migration or telemetry is added. Roll back this PR if the badge differs or scoped reads deny previously permitted users. ## Model Used OpenAI `gpt-6.1-sol` (Sol 6.1), with reasoning, tool use and code execution. The host does not expose the configured context-window size. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- doc/connection-review-counts.md | 101 ++++++++++++++++++ maps/agent-access-map/app-permissions.md | 9 ++ maps/feature-map/app-permissions.md | 6 ++ packages/shared/src/index.ts | 1 + packages/shared/src/types/index.ts | 1 + packages/shared/src/types/tool-access.ts | 5 + .../benchmark-connection-review-counts.ts | 48 +++++++++ .../tool-connection-review-counts-fixture.ts | 17 +++ ...tool-connection-review-counts-http.test.ts | 56 ++++++++++ .../tool-connection-review-counts.test.ts | 45 ++++++++ server/src/routes/openapi.ts | 9 ++ server/src/routes/tool-access.ts | 8 ++ server/src/services/tool-access.ts | 2 + .../tool-access/catalog/review-counts.ts | 33 ++++++ ui/src/api/tools.ts | 3 + .../AppConnectionSidebar.production.tsx | 20 ++-- .../components/AppConnectionSidebar.test.tsx | 44 ++++++++ ui/src/components/AppConnectionSidebar.tsx | 20 ++-- ui/src/lib/queryKeys.ts | 2 + 19 files changed, 406 insertions(+), 24 deletions(-) create mode 100644 doc/connection-review-counts.md create mode 100644 server/scripts/benchmark-connection-review-counts.ts create mode 100644 server/src/__tests__/helpers/tool-connection-review-counts-fixture.ts create mode 100644 server/src/__tests__/tool-connection-review-counts-http.test.ts create mode 100644 server/src/__tests__/tool-connection-review-counts.test.ts create mode 100644 server/src/services/tool-access/catalog/review-counts.ts diff --git a/doc/connection-review-counts.md b/doc/connection-review-counts.md new file mode 100644 index 00000000000..603e73d6192 --- /dev/null +++ b/doc/connection-review-counts.md @@ -0,0 +1,101 @@ +# Connection review counts + +The contextual App sidebar uses `GET /api/tool-connections/:connectionId/review-counts` +for its Review badge. The response contains `pendingActionRequestCount` and +`quarantinedCatalogEntryCount`. The badge adds the pending count and one when the +quarantine count is positive. Both sidebar variants keep the 30 second cadence. +The key is a child of the existing company attention prefix so current mutations +and events still invalidate it. + +The route uses existing connection read admission. It resolves UUIDs and UIDs, +checks company and current actor/responsible-user authority, and enforces personal +account restrictions before reading counts. An authorized archived connection +returns zero counts, as its absent legacy attention item did. Missing and hidden +connections retain normal admission errors. This read adds no execution authority. + +The catalog owner reads two independent scalar aggregates in one SQL statement. +Pending counts join requests to invocations by invocation ID and constrain both +company IDs. SQL NULL signed arguments are excluded; empty signed text counts. +Multiple requests for one invocation count separately. No profile, rule or active +catalog projection runs for this read. The company-wide attention API is unchanged. + +## Local performance proof (2026-11-02) + +Baseline: `3f8ebfff8aa9febd0733c6f65b07485dcadfd0e2` on current master, including +both prior evaluator fixes (#18673 and #18681). Treatment is the count reader in +this change. Node v24.22.2, macOS, disposable real embedded PostgreSQL, one database +connection, no cross-request cache. The comparison calls the actual legacy +attention service and selects one connection's two counts, then calls the new +reader. All normalized results equal `{pendingActionRequestCount: 2, +quarantinedCatalogEntryCount: 3}`. + +Fixture: 200 profiles, 76,000 rules, 1,900 active tools, 695 removed references, +12 agents, one sibling connection, six quarantine entries, signed/unsigned/resolved +requests and 2,000 unrelated historical invocations. Reproduction: + +```sh +cd server +node --expose-gc --import ./node_modules/tsx/dist/loader.mjs scripts/benchmark-connection-review-counts.ts +``` + +After two warmup pairs, seven alternating measured pairs in a coordinated quiet +window produced: + +| Service measurement | Legacy attention → selected counts | Scoped counts | +| --- | --- | --- | +| Queries | 7 | 1 | +| Returned application rows | 80,115 | 1 | +| Process CPU median (range), ms | 213.891 (209.329–228.552) | 1.680 (0.515–6.137) | +| Wall median (range), ms | 156.867 (151.739–162.289) | 1.248 (1.115–1.386) | + +Separate work-volume instrumentation serialized database result values to JSON: +32,978,153 versus 66 bytes. This measures decoded application values, not network +bytes or HTTP response size. The legacy HTTP response never contains the rule +arrays. Work-volume serialization is disabled during every timed sample. + +These are local source-service measurements, not deployed latency or capacity +claims. They exclude HTTP admission, network transport and serialization. They do +not measure PostgreSQL CPU. Scoped SQL still scans relevant pending/catalog data +using existing indexes; no migration or new cache was needed. + +## Correctness and preserved failures + +PostgreSQL differential tests cover sibling/company boundaries, multiple requests, +NULL and empty signed text, unsigned/resolved requests, null invocation connections, +cross-company historical references, unhealthy-only connections, archive/restore, +and changed rows between reads. Assembled HTTP tests use actual human sessions +and agent JWT authentication with explicit permissions. They exercise UID lookup, +missing/foreign resources, personal grant revocation and membership revocation. +Both sidebar variants have badge, disabled-before-identity and prefix invalidation +component tests. + +The first SQL attempt embedded column references directly into scalar raw SQL. +Real PostgreSQL rejected the resulting ambiguous unqualified columns. Typed +Drizzle subqueries fixed that failure; the passing differential test exercises the +corrected query. An initial benchmark serialized result values inside its timed +instrumentation and inflated CPU. That run was discarded; the results above come +from a fresh alternating run with byte accounting outside timing. Initial HTTP +fixtures lacked agent membership/management grants and correctly received 403; +the accepted fixture supplies both and retains revocation denial checks. + +The built static UI was also exercised in Chromium against an owned local-trusted +server and disposable PostgreSQL. The connection Permissions page showed Review 3, +issued the scoped count request, and made no company attention request. After +resolving requests and clearing quarantines, reload removed the badge. An archived +connection returned zero counts. This browser path uses board authority; separate +authenticated HTTP fixtures prove delegated agent authority. The first ad hoc +browser launcher omitted required decision-service options and failed at startup; +adding the standard no-op wake callback allowed the same journey to pass. No +product behavior was changed to recover that harness setup failure. + +Required local recursive typecheck and full build passed. The first sandboxed +typecheck could not open the tsx IPC socket; the escalated rerun passed. Greptile +identified an invalid resolved-request fixture label (`denied`); it was corrected +to the declared `rejected` status. This preserves the non-pending exclusion that +the differential test verifies. + +The first final-head CI run flagged the new explicit resource company assertion +because the route lacked a local `hasCompanyAccess` existence gate. Management +admission already rejects foreign resources with 404, but the route now also +follows the repository's two-step gate pattern. The static existence-oracle guard +and actual authenticated HTTP negative cases verify that pattern together. diff --git a/maps/agent-access-map/app-permissions.md b/maps/agent-access-map/app-permissions.md index 8bef4add8b2..22675ec3973 100644 --- a/maps/agent-access-map/app-permissions.md +++ b/maps/agent-access-map/app-permissions.md @@ -404,6 +404,15 @@ Evidence: `source-only` — [route](../../server/src/routes/tool-access.ts); rev Verification: `not-run` — individual operation not executed in this audit; shared authorization and representative family flows are exercised by the authenticated integration suite. Follow the route schema, inspect persistence/audit, and assert owner, scope and revoked-authority denial. Gaps: This individual operation has source review but no recorded HTTP verification. Provider availability and live setup remain unqualified. +### `get-tool-connections-review-counts` + +API: `present` — `GET /api/tool-connections/{connectionId}/review-counts`; JSON REST. +Agent access: `available` — agent REST and default native `search_api` / `call_api`. +Policy: `tools:manage_connections`; existing connection read admission checks current company membership, resolved UUID/UID, responsible-user authority and personal account access. Authorized archived connections return zero counts. +Evidence: `verified` — [PostgreSQL and authenticated HTTP fixtures](../../server/src/__tests__/tool-connection-review-counts-http.test.ts) exercise the assembled server and actual board session and agent JWT middleware. +Verification: `passed` — deterministic local fixture with UID resolution, archive, foreign/missing connection, personal grant revocation and membership revocation. +Gaps: Local fixtures do not prove deployed performance or provider availability. + ### `get-tool-connections-usage` API: `present` — `GET /api/tool-connections/{connectionId}/usage`; JSON REST. diff --git a/maps/feature-map/app-permissions.md b/maps/feature-map/app-permissions.md index c725de0d1f7..921a493d5c9 100644 --- a/maps/feature-map/app-permissions.md +++ b/maps/feature-map/app-permissions.md @@ -162,3 +162,9 @@ Sources: Source: [PermissionsPanel.tsx](../../ui/src/pages/apps/app-detail/Permi ### `delegated-configuration` See the canonical [delegated configuration guidance and acceptance evidence](connection-setup.md#delegated-configuration). + +The contextual connection sidebar reads only the selected connection's pending +signed action request and quarantined action counts. Its Review badge is the +pending request count plus one when any action is quarantined. It refreshes every +30 seconds and after existing attention invalidations. Archived connections show +no badge. The full Apps attention view keeps its existing company-wide read. diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 676ca8ed6b0..81150730214 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -1466,6 +1466,7 @@ export type { ToolAppAttentionItem, ToolAppAttentionReason, ToolAppsAttentionResponse, + ToolConnectionReviewCounts, ToolActionRequestListItem, ToolActionRequestsResponse, ToolConnectionActivityResponse, diff --git a/packages/shared/src/types/index.ts b/packages/shared/src/types/index.ts index a00d768be26..14b31713338 100644 --- a/packages/shared/src/types/index.ts +++ b/packages/shared/src/types/index.ts @@ -507,6 +507,7 @@ export type { ToolAppAttentionItem, ToolAppAttentionReason, ToolAppsAttentionResponse, + ToolConnectionReviewCounts, ToolActionRequestListItem, ToolActionRequestsResponse, ToolConnectionActivityResponse, diff --git a/packages/shared/src/types/tool-access.ts b/packages/shared/src/types/tool-access.ts index 961a97a8f99..39e7e92d3ad 100644 --- a/packages/shared/src/types/tool-access.ts +++ b/packages/shared/src/types/tool-access.ts @@ -1020,6 +1020,11 @@ export interface ToolAppAttentionItem { reasons: ToolAppAttentionReason[]; } +export interface ToolConnectionReviewCounts { + pendingActionRequestCount: number; + quarantinedCatalogEntryCount: number; +} + export interface ToolAppsAttentionResponse { generatedAt: Date | string; apps: ToolAppAttentionItem[]; diff --git a/server/scripts/benchmark-connection-review-counts.ts b/server/scripts/benchmark-connection-review-counts.ts new file mode 100644 index 00000000000..ac425f55aac --- /dev/null +++ b/server/scripts/benchmark-connection-review-counts.ts @@ -0,0 +1,48 @@ +import assert from "node:assert/strict"; +import { createDb, startEmbeddedPostgresTestDatabase, type Db } from "@paperclipai/db"; +import { createConnectionReviewCountsFixture } from "../src/__tests__/helpers/tool-connection-review-counts-fixture.js"; +import { createToolAppAttention } from "../src/services/tool-access/catalog/attention.js"; +import { getToolConnectionReviewCounts } from "../src/services/tool-access/catalog/review-counts.js"; +const database = await startEmbeddedPostgresTestDatabase("connection-review-counts-benchmark-"); +const db = createDb(database.connectionString, { maxConnections: 1 }); +let queries = 0, rows = 0, returnedValueBytes = 0, collectBytes = false; +const measuredDb = new Proxy(db, { get(target, key) { + if (key !== "select") return Reflect.get(target, key); + return (...fields: Parameters) => { + const builder = target.select(...fields), from = builder.from.bind(builder); + builder.from = ((...args: Parameters) => { + const selection = from(...args), execute = selection.execute.bind(selection); + selection.execute = async () => { const result = await execute(); queries++; rows += result.length; if (collectBytes) returnedValueBytes += Buffer.byteLength(JSON.stringify(result)); return result; }; + return selection; + }) as typeof builder.from; + return builder; + }; +} }); +try { + const f = await createConnectionReviewCountsFixture(db, true); + const legacy = createToolAppAttention(measuredDb, () => new Date("2026-02-01T00:00:00Z")); + const invoke = async (label: "before" | "after") => { + if (label === "after") return getToolConnectionReviewCounts(measuredDb, f.company.id, f.connection.id); + const item = (await legacy(f.company.id)).apps.find(app => app.connection.id === f.connection.id); + return { pendingActionRequestCount: item?.pendingActionRequestCount ?? 0, quarantinedCatalogEntryCount: item?.quarantinedCatalogEntryCount ?? 0 }; + }; + const expected = await invoke("before"); + assert.deepEqual(await invoke("after"), expected); + console.log(JSON.stringify({ fixture: { profiles: 200, rules: 76000, activeTools: 1900, removedTools: 695, agents: 12, siblingConnections: 1, extraInvocationHistory: 2000 }, equality: true, expected, node: process.version, platform: process.platform, baseline: "3f8ebfff8a", scope: "service only; excludes HTTP admission and serialization; returnedValueBytes is application JSON value size, not wire bytes" })); + for (const label of ["before", "after"] as const) { + queries = 0; rows = 0; returnedValueBytes = 0; collectBytes = true; + assert.deepEqual(await invoke(label), expected); + console.log(JSON.stringify({ label, kind: "work-counts", queries, rows, returnedValueBytes })); + } + collectBytes = false; + for (let round = -2; round < 7; round++) { + for (const label of round % 2 ? ["after", "before"] as const : ["before", "after"] as const) { + global.gc?.(); queries = 0; rows = 0; returnedValueBytes = 0; + const cpu = process.cpuUsage(), start = performance.now(); + const result = await invoke(label); + const elapsedMs = performance.now() - start, used = process.cpuUsage(cpu); + assert.deepEqual(result, expected); + if (round >= 0) console.log(JSON.stringify({ round, label, elapsedMs, cpuMs: (used.user + used.system) / 1000, queries, rows, forcedGc: Boolean(global.gc) })); + } + } +} finally { await db.$client.end({ timeout: 1 }); await database.cleanup(); } diff --git a/server/src/__tests__/helpers/tool-connection-review-counts-fixture.ts b/server/src/__tests__/helpers/tool-connection-review-counts-fixture.ts new file mode 100644 index 00000000000..ec00fdca602 --- /dev/null +++ b/server/src/__tests__/helpers/tool-connection-review-counts-fixture.ts @@ -0,0 +1,17 @@ +import { randomUUID } from "node:crypto"; +import { toolActionRequests, toolCatalogEntries, toolConnections, toolInvocations, type Db } from "@paperclipai/db"; +import { createProfileProjectionFixture } from "./tool-profile-projection-fixture.js"; + +export async function createConnectionReviewCountsFixture(db: Db, large = false) { + const f = await createProfileProjectionFixture(db, large ? {} : { profiles: 4, tools: 12, rules: 4 }); + const [sibling] = await db.insert(toolConnections).values({ companyId: f.company.id, applicationId: f.application.id, name: "Sibling", uid: randomUUID(), transport: "mcp_remote", status: "active", healthStatus: "error" }).returning(); + await db.insert(toolCatalogEntries).values([f.connection, sibling].flatMap(connection => Array.from({ length: 3 }, (_, i) => ({ companyId: f.company.id, connectionId: connection.id, name: `quarantine_${i}`, toolName: `quarantine_${i}`, versionHash: "v1", status: "quarantined" as const })))); + const [invocation, siblingInvocation, nullInvocation] = await db.insert(toolInvocations).values([f.connection.id, sibling.id, null].map(connectionId => ({ companyId: f.company.id, connectionId, toolName: "review_me" }))).returning(); + const request = (invocationId: string, signedArguments: string | null, status: "pending" | "rejected" = "pending") => ({ companyId: f.company.id, invocationId, signedArguments, status, canonicalArgumentsHash: "hash", canonicalArgumentsSummary: { summary: "Synthetic review request" } }); + const requests = await db.insert(toolActionRequests).values([ + request(invocation.id, "signed"), request(invocation.id, ""), request(invocation.id, null), request(invocation.id, "signed", "rejected"), + request(siblingInvocation.id, "signed"), request(nullInvocation.id, "signed"), + ]).returning(); + if (large) await db.insert(toolInvocations).values(Array.from({ length: 2000 }, () => ({ companyId: f.company.id, connectionId: sibling.id, toolName: "history", status: "succeeded" as const }))); + return { ...f, sibling, invocation, requests }; +} diff --git a/server/src/__tests__/tool-connection-review-counts-http.test.ts b/server/src/__tests__/tool-connection-review-counts-http.test.ts new file mode 100644 index 00000000000..01d7dd223ef --- /dev/null +++ b/server/src/__tests__/tool-connection-review-counts-http.test.ts @@ -0,0 +1,56 @@ +import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { beforeAll, afterAll, describe, expect, it, vi } from "vitest"; +import { companyMemberships, toolApplications, toolConnections, toolCatalogEntries, connectionGrants, principalPermissionGrants } from "@paperclipai/db"; +import { startRunnerApiTestServer } from "./helpers/runner-api-server.js"; +import { accessService } from "../services/access.js"; +import { createLocalAgentJwt } from "../agent-auth-jwt.js"; + +describe("review-counts through assembled authenticated HTTP routes", () => { + let server: Awaited>; + beforeAll(async () => { + vi.stubEnv("PAPERCLIP_AGENT_JWT_SECRET", randomUUID()); + vi.stubEnv("PAPERCLIP_RESPONSIBLE_USER_AUTHZ_MODE", "shadow"); + vi.stubEnv("PAPERCLIP_AUTH_RATE_LIMIT_ENABLED", "false"); + server = await startRunnerApiTestServer({ authPublicBaseUrl: "https://paperclip.fixture.invalid", humanSessions: true }); + }, 90_000); + afterAll(async () => { await server?.close(); vi.unstubAllEnvs(); }); + it("uses current company, personal and private authority for board and actual agent identities", async () => { + const f = await server.fixture({ connectionScenario: "fresh" }); + const access = accessService(server.db); + await access.ensureMembership(f.companyId, "agent", f.agentId, "member", "active"); + await access.ensureMembership(f.companyId, "user", f.responsibleUserId!, "operator", "active"); + await access.ensureRoleDefaultGrants(f.companyId, f.responsibleUserId!, "operator", f.responsibleUserId!); + await server.db.insert(principalPermissionGrants).values({ companyId: f.companyId, principalType: "agent", principalId: f.agentId, permissionKey: "tools:manage_connections", scope: null }); + const cookie = await server.humanSession(f.responsibleUserId!); + const token = createLocalAgentJwt(f.agentId, f.companyId, "paperclip_runner", f.runId, f.responsibleUserId!); + const [application] = await server.db.insert(toolApplications).values({ companyId: f.companyId, name: "Scoped counts", type: "mcp_http" }).returning(); + const [connection] = await server.db.insert(toolConnections).values({ companyId: f.companyId, applicationId: application.id, name: "Scoped counts", uid: `counts-${randomUUID()}`, transport: "mcp_remote", status: "active", createdByUserId: f.responsibleUserId }).returning(); + await server.db.insert(toolCatalogEntries).values({ companyId: f.companyId, connectionId: connection.id, name: "Review", toolName: "review", versionHash: "v1", status: "quarantined" }); + const read = async (id: string, actor: "board" | "agent" = "agent") => { + const response = await fetch(`${server.apiUrl}/api/tool-connections/${id}/review-counts`, { headers: actor === "board" ? { cookie } : { Authorization: `Bearer ${token}` } }); + return { status: response.status, body: await response.json() }; + }; + for (const actor of ["board", "agent"] as const) { + expect(await read(connection.id, actor)).toEqual({ status: 200, body: { pendingActionRequestCount: 0, quarantinedCatalogEntryCount: 1 } }); + expect(await read(connection.uid, actor)).toEqual(await read(connection.id, actor)); + expect((await read(randomUUID(), actor)).status).toBe(404); + } + await server.db.update(toolConnections).set({ status: "archived" }).where(eq(toolConnections.id, connection.id)); + expect(await read(connection.id)).toEqual({ status: 200, body: { pendingActionRequestCount: 0, quarantinedCatalogEntryCount: 0 } }); + await server.db.update(toolConnections).set({ status: "active", credentialPolicy: "per_user", createdByUserId: "another-user" }).where(eq(toolConnections.id, connection.id)); + expect((await read(connection.id)).status).toBe(403); + const [grant] = await server.db.insert(connectionGrants).values({ companyId: f.companyId, connectionId: connection.id, kind: "user", subjectUserId: f.responsibleUserId!, status: "active" }).returning(); + expect((await read(connection.id)).status).toBe(200); + await server.db.update(connectionGrants).set({ status: "revoked" }).where(eq(connectionGrants.id, grant.id)); + expect((await read(connection.id)).status).toBe(403); + await server.db.update(toolConnections).set({ credentialPolicy: "shared", createdByUserId: f.responsibleUserId }).where(eq(toolConnections.id, connection.id)); + const [foreignApplication] = await server.db.insert(toolApplications).values({ companyId: f.foreignCompanyId, name: "Foreign", type: "mcp_http" }).returning(); + const [foreign] = await server.db.insert(toolConnections).values({ applicationId: foreignApplication.id, companyId: f.foreignCompanyId, name: "Foreign", uid: randomUUID(), transport: "mcp_remote" }).returning(); + expect([403, 404]).toContain((await read(foreign.id)).status); + await server.db.delete(principalPermissionGrants).where(eq(principalPermissionGrants.principalId, f.responsibleUserId!)); + await server.db.update(companyMemberships).set({ status: "suspended" }).where(eq(companyMemberships.principalId, f.responsibleUserId!)); + expect([403, 404]).toContain((await read(connection.id)).status); + expect([403, 404]).toContain((await read(connection.id, "board")).status); + }, 30_000); +}); diff --git a/server/src/__tests__/tool-connection-review-counts.test.ts b/server/src/__tests__/tool-connection-review-counts.test.ts new file mode 100644 index 00000000000..4e701cbb0b8 --- /dev/null +++ b/server/src/__tests__/tool-connection-review-counts.test.ts @@ -0,0 +1,45 @@ +import { and, eq } from "drizzle-orm"; +import { beforeAll, afterAll, describe, expect, it } from "vitest"; +import { createDb, companies, toolActionRequests, toolCatalogEntries, toolConnections, toolInvocations } from "@paperclipai/db"; +import { startRecoveryPostgres } from "./helpers/recovery-postgres.js"; +import { createConnectionReviewCountsFixture } from "./helpers/tool-connection-review-counts-fixture.js"; +import { createToolAppAttention } from "../services/tool-access/catalog/attention.js"; +import { getToolConnectionReviewCounts } from "../services/tool-access/catalog/review-counts.js"; + +const zero = { pendingActionRequestCount: 0, quarantinedCatalogEntryCount: 0 }; +describe("connection review counts on PostgreSQL", () => { + let database: Awaited>; + let db: ReturnType; + beforeAll(async () => { database = await startRecoveryPostgres("connection-review-counts-"); db = createDb(database.connectionString); }, 90_000); + afterAll(async () => { await db?.$client.end(); await database?.cleanup(); }); + it("preserves the complete legacy badge projection without fanout and recomputes after changes", async () => { + const f = await createConnectionReviewCountsFixture(db); + const legacy = createToolAppAttention(db, () => new Date()); + const compare = async (id: string) => { + const item = (await legacy(f.company.id)).apps.find(app => app.connection.id === id); + const expected = item ? { pendingActionRequestCount: item.pendingActionRequestCount, quarantinedCatalogEntryCount: item.quarantinedCatalogEntryCount } : zero; + const result = await getToolConnectionReviewCounts(db, f.company.id, id); + expect(result).toEqual(expected); + return result; + }; + expect(await compare(f.connection.id)).toEqual({ pendingActionRequestCount: 2, quarantinedCatalogEntryCount: 3 }); + expect(await compare(f.sibling.id)).toEqual({ pendingActionRequestCount: 1, quarantinedCatalogEntryCount: 3 }); + const [foreign] = await db.insert(companies).values({ name: "Foreign review counts", issuePrefix: "FRC" }).returning(); + // Historical schema permits cross-company references: neither side can contribute. + const [invocation] = await db.insert(toolInvocations).values({ companyId: foreign.id, connectionId: f.connection.id, toolName: "foreign" }).returning(); + await db.insert(toolActionRequests).values([ + { companyId: f.company.id, invocationId: invocation.id, signedArguments: "signed", canonicalArgumentsHash: "hash", canonicalArgumentsSummary: { summary: "Synthetic review request" } }, + { companyId: foreign.id, invocationId: f.invocation.id, signedArguments: "signed", canonicalArgumentsHash: "hash", canonicalArgumentsSummary: { summary: "Synthetic review request" } }, + ]); + await db.insert(toolCatalogEntries).values({ companyId: foreign.id, connectionId: f.connection.id, name: "foreign", toolName: "foreign", versionHash: "v1", status: "quarantined" }); + expect(await compare(f.connection.id)).toEqual({ pendingActionRequestCount: 2, quarantinedCatalogEntryCount: 3 }); + await db.update(toolConnections).set({ status: "archived" }).where(eq(toolConnections.id, f.connection.id)); + expect(await compare(f.connection.id)).toEqual(zero); + await db.update(toolConnections).set({ status: "active" }).where(eq(toolConnections.id, f.connection.id)); + expect(await compare(f.connection.id)).toEqual({ pendingActionRequestCount: 2, quarantinedCatalogEntryCount: 3 }); + await db.update(toolActionRequests).set({ status: "rejected" }).where(eq(toolActionRequests.companyId, f.company.id)); + await db.update(toolCatalogEntries).set({ status: "active" }).where(and(eq(toolCatalogEntries.companyId, f.company.id), eq(toolCatalogEntries.status, "quarantined"))); + expect(await compare(f.connection.id)).toEqual(zero); + expect(await compare(f.sibling.id)).toEqual(zero); // unhealthy-only remains a zero badge + }); +}); diff --git a/server/src/routes/openapi.ts b/server/src/routes/openapi.ts index 772b4706a19..e19d5035a83 100644 --- a/server/src/routes/openapi.ts +++ b/server/src/routes/openapi.ts @@ -11769,6 +11769,15 @@ registerCurrentRoute({ summary: "Revoke a tool connection grant", }); +registerCurrentRoute({ + "x-paperclip-transport": "rest", + "x-paperclip-connection-management": "connections:read", + method: "get", + path: "/api/tool-connections/{connectionId}/review-counts", + tags: ["tool-access"], + summary: "Get pending action request and quarantined catalog entry counts for one connection", +}); + registerCurrentRoute({ "x-paperclip-transport": "rest", "x-paperclip-connection-management": "connections:read", diff --git a/server/src/routes/tool-access.ts b/server/src/routes/tool-access.ts index 72209d84746..889c9f11733 100644 --- a/server/src/routes/tool-access.ts +++ b/server/src/routes/tool-access.ts @@ -1921,6 +1921,14 @@ function connectorEnrollmentPrincipal(req: Request): string { res.json(delegation); }); + router.get("/tool-connections/:connectionId/review-counts", managementGuard("connections:read", "GET /tool-connections/:connectionId/review-counts", { personal: true }), async (req, res) => { + assertBoardOrAgent(req); + const { resource } = connectionRequestContext(req)!; + if (!hasCompanyAccess(req, resource.companyId)) throw notFound("Tool connection not found"); + assertCompanyAccess(req, resource.companyId); + res.json(await svc.getConnectionReviewCounts(resource.companyId, resource.connectionId!)); + }); + router.get("/tool-connections/:connectionId/usage", managementGuard("connections:read", "GET /tool-connections/:connectionId/usage", { personal: true }), async (req, res) => { assertBoardOrAgent(req); const connection = await getAccessibleResource(req, res, svc.getConnection(req.params.connectionId as string), "Tool connection not found"); diff --git a/server/src/services/tool-access.ts b/server/src/services/tool-access.ts index ee3717e231d..3e3f6292565 100644 --- a/server/src/services/tool-access.ts +++ b/server/src/services/tool-access.ts @@ -18,6 +18,7 @@ import { isOAuthEndpointRejection } from "./tool-access/catalog/failures.js"; import { createCatalogDefaultAccess, toPolicy } from "./tool-access/catalog/default-access.js"; import { createToolCatalogDiscovery } from "./tool-access/catalog/discovery.js"; import { createConnectionHealth } from "./tool-access/catalog/health.js"; +import { getToolConnectionReviewCounts } from "./tool-access/catalog/review-counts.js"; import { createToolAppAttention } from "./tool-access/catalog/attention.js"; import { createToolCatalogPublication } from "./tool-access/catalog/publication.js"; import { createToolCatalogReader } from "./tool-access/catalog/reader.js"; @@ -4737,6 +4738,7 @@ export function toolAccessService( }, listAppsNeedingAttention, + getConnectionReviewCounts: (companyId: string, connectionId: string) => getToolConnectionReviewCounts(db, companyId, connectionId), sweepConnectionHealth, scheduledHealth: connectionHealthSchedule(db, checkConnectionHealth), diff --git a/server/src/services/tool-access/catalog/review-counts.ts b/server/src/services/tool-access/catalog/review-counts.ts new file mode 100644 index 00000000000..dc287b7f163 --- /dev/null +++ b/server/src/services/tool-access/catalog/review-counts.ts @@ -0,0 +1,33 @@ +import { and, count, eq, isNotNull, sql } from "drizzle-orm"; +import { toolActionRequests, toolCatalogEntries, toolConnections, toolInvocations, type Db } from "@paperclipai/db"; +import type { ToolConnectionReviewCounts } from "@paperclipai/shared"; + +/** Counts for one already-authorized connection. Independent aggregates avoid catalog/request fanout. */ +export async function getToolConnectionReviewCounts( + db: Db, + companyId: string, + connectionId: string, +): Promise { + const pending = db.select({ count: count() }).from(toolActionRequests) + .innerJoin(toolInvocations, eq(toolInvocations.id, toolActionRequests.invocationId)) + .where(and( + eq(toolActionRequests.companyId, companyId), + eq(toolInvocations.companyId, companyId), + eq(toolInvocations.connectionId, connectionId), + eq(toolActionRequests.status, "pending"), + isNotNull(toolActionRequests.signedArguments), + )); + const quarantined = db.select({ count: count() }).from(toolCatalogEntries).where(and( + eq(toolCatalogEntries.companyId, companyId), + eq(toolCatalogEntries.connectionId, connectionId), + eq(toolCatalogEntries.status, "quarantined"), + )); + const [counts] = await db.select({ + pendingActionRequestCount: sql`case when ${toolConnections.status} = 'archived' then 0 else (${pending}) end`.mapWith(Number), + quarantinedCatalogEntryCount: sql`case when ${toolConnections.status} = 'archived' then 0 else (${quarantined}) end`.mapWith(Number), + }).from(toolConnections).where(and( + eq(toolConnections.companyId, companyId), eq(toolConnections.id, connectionId), + )); + // Missing/hidden resources are rejected by route admission before this reader. + return counts ?? { pendingActionRequestCount: 0, quarantinedCatalogEntryCount: 0 }; +} diff --git a/ui/src/api/tools.ts b/ui/src/api/tools.ts index 1087548dfff..8ab5b4ed9e1 100644 --- a/ui/src/api/tools.ts +++ b/ui/src/api/tools.ts @@ -43,6 +43,7 @@ import type { ReorderToolPolicies, AppDefinition, ToolAppsAttentionResponse, + ToolConnectionReviewCounts, ToolConnectionActivityResponse, ToolConnectionLifecycleEventType, ToolConnectionTestAgentsResponse, @@ -334,6 +335,8 @@ export const toolsApi = { `/companies/${companyId}/tools/apps/${connectionId}/finish`, input, ), + getConnectionReviewCounts: (connectionId: string) => + api.get(`/tool-connections/${connectionId}/review-counts`), listAppsAttention: (companyId: string) => api.get(`/companies/${companyId}/tools/apps/attention`), listApplications: (companyId: string) => diff --git a/ui/src/components/AppConnectionSidebar.production.tsx b/ui/src/components/AppConnectionSidebar.production.tsx index 8eb84c326da..7cec3dab4cc 100644 --- a/ui/src/components/AppConnectionSidebar.production.tsx +++ b/ui/src/components/AppConnectionSidebar.production.tsx @@ -50,13 +50,6 @@ export function AppDetailSidebar(props: AppDetailSidebarProps) { queryFn: () => toolsApi.listGallery(selectedCompanyId!), enabled: !!selectedCompanyId, }); - const attentionQuery = useQuery({ - queryKey: queryKeys.apps.attention(selectedCompanyId ?? "__none__"), - queryFn: () => toolsApi.listAppsAttention(selectedCompanyId!), - enabled: !!selectedCompanyId, - refetchInterval: 30_000, - }); - const connection = connectionQuery.data; const application = props.kind === "application" ? (applicationsQuery.data?.applications ?? []).find((app) => app.id === props.applicationId) @@ -72,12 +65,15 @@ export function AppDetailSidebar(props: AppDetailSidebarProps) { application ?? undefined, ); const reviewConnectionId = connection?.id ?? previousConnection?.id ?? null; - const attentionItem = reviewConnectionId - ? attentionQuery.data?.apps.find((app) => app.connection.id === reviewConnectionId) - : null; + const reviewCountsQuery = useQuery({ + queryKey: queryKeys.apps.connectionReviewCounts(selectedCompanyId ?? "__none__", reviewConnectionId ?? "__none__"), + queryFn: () => toolsApi.getConnectionReviewCounts(reviewConnectionId!), + enabled: !!selectedCompanyId && !!reviewConnectionId, + refetchInterval: 30_000, + }); const reviewCount = - (attentionItem?.pendingActionRequestCount ?? 0) + - ((attentionItem?.quarantinedCatalogEntryCount ?? 0) > 0 ? 1 : 0); + (reviewCountsQuery.data?.pendingActionRequestCount ?? 0) + + ((reviewCountsQuery.data?.quarantinedCatalogEntryCount ?? 0) > 0 ? 1 : 0); return (