diff --git a/doc/idle-sleep-safety.md b/doc/idle-sleep-safety.md index 3f80dd1fbe6..a07fc4d9f6f 100644 --- a/doc/idle-sleep-safety.md +++ b/doc/idle-sleep-safety.md @@ -97,6 +97,90 @@ Independent SQL writers and additional processes need an explicit wake protocol; process-local notifications do not provide cross-replica delivery. Worker quiet alone is not evidence that the host or database actually suspended. +## Retained completion history + +The retained chat, tool-review, browser, decision, transfer, public-MCP and +external-agent families use their owning domain's completion receipts. Inspection +does not delete history, expire credentials, cancel work or shorten retention. +A terminal label alone is insufficient. Missing, contradictory, malformed or +foreign company/task/run/binding references keep the instance awake. + +Chat input can remain work after processing because its exact wake still owns +the conversation head. Publications retain provider ambiguity, endpoint leases +and file-transfer obligations. Tool reviews retain invocation completion, +interaction projection and continuation delivery. Browser runs require drained +events and settled/released accounting; a stopped browser must belong to the +same closed session. Decision success requires the selected option's full effect +manifest, matching ordinal/type/target receipts and audit records. The owning +dismissal, explicit recovery stop and reviewed archive paths have separate proofs. +Their continuation and archive-notification outboxes remain independent barriers. +Failed company transfers remain resumable. Completed transfers require their +recorded manifest, exact part receipts and matching counts. Completed/cancelled +transfers still block while their registered local spool contains cleanup work. + +Chat control tokens can remain issued or become expired after another actor +resolves their canonical interaction. Their exact interaction, publication and +conversation receipts establish completion; a token status alone does not. +Provider submission receipts do not exempt pending source wakes or publications. +Photon attachment provenance remains immutable history after ordinary attachment +removal, but any surviving metadata must agree with its recorded identities and +hashes. This proof establishes completed registration, not storage cleanup. + +Public-MCP upload cleanup requires both the existing cleanup stamp and a new +verified-cleanup stamp. Ordinary attachment and issue removal write this proof +only after successful object deletion and an absence readback. Legacy cleanup +stamps are not backfilled: the existing bounded opportunistic sweep can verify +expired or orphaned uploads. Reissuing an uncommitted upload clears both stamps; +a committed/deleted request replay preserves them. An expired or cleaned upload +cannot accept a still-valid old transport ticket. + +Voice report refresh remains work for the owner's full 24-hour window after a +provider session ends, even when the report is available. Live session recovery, +unconfirmed inbound cleanup and ambiguous reply submission still block. A +completed provider submission does not prove spoken playback. Speko setup also +requires the existing successful whole-configuration receipt after its individual +tool and webhook writes. Discord command registration retains its own future +check; a completed command invocation does not retire that deadline. + +Waiting agent conversations can be passive containers under both ordinary and +scheduled inspection. The same narrow proof requires the exact conversation +agent/company/user, waiting state, no checkout, active run, accounting, monitor +or pending delivery, and a finished wake for every accepted user-comment outbox +entry. An idle presentation state does not exempt an ordinary runnable task. + +Sandbox creation records in `pending` or `needs_reconciliation` remain blockers, +including manual uncertainty after company deletion. The pending-only catalog +omission predates the new startup-cleanup recovery path; no complete-instance +unsafe sleep outcome has been demonstrated. A late identified resource transfers +its obligation to the exact cleanup lease, and only the owning successful cleanup +can resolve that creation. Settled startup cancellations without a committed +native result manifest remain conservatively blocked by native finalization; +their cleanup marker alone is not a history exemption. + +Workspace runtime inspection also reads the owner's process-local resource +state. Startup reconstruction must succeed before that probe becomes quiet. +Adopted/live services, provisioning/control/replacement work, live allocation +claims and quarantined exposure ports remain barriers alongside durable rows. +Reset, failed reconstruction and unresolved desired-state restart cannot reuse an +earlier quiet proof. The probe performs no SQL, network request or cleanup and +adds no timer. A stopped row cannot hide a surviving registered process. + +MCP completion receipts do not exempt their downstream task, subscription or +upload cleanup. Active grants/subscriptions and Dot/Muse bindings/credentials +retain their transport/deadline blockers. Revoked or expired credential history +does not grant authority on the next request. A fenced external assignment is +not worker termination proof, and Muse transport notification is not input +consumption. Released Muse holds require their actual settlement/attestation +path. Dot stop holds remain unsupported because that owner has no completed +worker-stop receipt. Grok Bot shares Dot tables but has independent routine and +dispatch/recovery obligations; even revoked Grok bindings retain their blocker. + +Action/provider variants without an established receipt proof remain blocked. +This includes Photon notice lineage and unqualified legacy action results, +adapter-managed runtime resources, and active persistent transports. These +limits are not evidence of deployed wake support. Existing hosting admission, +generation, accounting, finalization and resource-cleanup checks still apply. + ## Native reply recovery Committed native replies use the shared delivery coordinator and its persisted @@ -518,8 +602,9 @@ operation by itself. Multiple replicas require shared fencing. The protocol does not make existing in-memory-only debt survive an unplanned process crash. Related public PR #13413 adds runtime-service controller requirements and uses -the same `purpose: "idle"` / `ownerId` vocabulary. This report conservatively -blocks all retained runtime-service state instead of approving such services. +the same `purpose: "idle"` / `ownerId` vocabulary. Retained stopped local-process +rows use the domain and process-reconstruction proof described above; live, +uncertain and adapter-managed runtime resources remain blockers. ## Muse maintenance diff --git a/packages/db/src/migrations/0425_awesome_living_lightning.sql b/packages/db/src/migrations/0425_awesome_living_lightning.sql new file mode 100644 index 00000000000..b90f13b2b22 --- /dev/null +++ b/packages/db/src/migrations/0425_awesome_living_lightning.sql @@ -0,0 +1 @@ +ALTER TABLE "mcp_attachment_uploads" ADD COLUMN IF NOT EXISTS "cleanup_verified_at" timestamp with time zone; \ No newline at end of file diff --git a/packages/db/src/migrations/meta/0419_snapshot.json b/packages/db/src/migrations/meta/0425_snapshot.json similarity index 99% rename from packages/db/src/migrations/meta/0419_snapshot.json rename to packages/db/src/migrations/meta/0425_snapshot.json index ec874170b7b..4f8d0f0bc90 100644 --- a/packages/db/src/migrations/meta/0419_snapshot.json +++ b/packages/db/src/migrations/meta/0425_snapshot.json @@ -1,6 +1,6 @@ { - "id": "52cf22f3-3bd9-4b0f-8d23-b422bb0d2540", - "prevId": "c777fa84-85a6-418f-bea1-0d60b56cf071", + "id": "8d9f4abf-57c7-443a-aa8b-3e3fd4f09dae", + "prevId": "cb093c1d-3e54-4ab2-8009-9c243097224c", "version": "7", "dialect": "postgresql", "tables": { @@ -2233,6 +2233,74 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.agent_file_roots": { + "name": "agent_file_roots", + "schema": "", + "columns": { + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "generation": { + "name": "generation", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "entries": { + "name": "entries", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "agent_file_roots_company_id_agent_id_agents_company_id_id_fk": { + "name": "agent_file_roots_company_id_agent_id_agents_company_id_id_fk", + "tableFrom": "agent_file_roots", + "tableTo": "agents", + "columnsFrom": [ + "company_id", + "agent_id" + ], + "columnsTo": [ + "company_id", + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "agent_file_roots_company_id_agent_id_pk": { + "name": "agent_file_roots_company_id_agent_id_pk", + "columns": [ + "company_id", + "agent_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.agent_identity_keys": { "name": "agent_identity_keys", "schema": "", @@ -29705,6 +29773,241 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.execution_workspace_snapshots": { + "name": "execution_workspace_snapshots", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "execution_workspace_id": { + "name": "execution_workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_operation_id": { + "name": "source_operation_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_run_id": { + "name": "source_run_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "parent_snapshot_id": { + "name": "parent_snapshot_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "generation": { + "name": "generation", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "epoch": { + "name": "epoch", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "format": { + "name": "format", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'paperclip.remote-workspace-archive.v1'" + }, + "object_key": { + "name": "object_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sha256": { + "name": "sha256", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "byte_size": { + "name": "byte_size", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "key": { + "name": "key", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "manifest_digest": { + "name": "manifest_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "delete_after": { + "name": "delete_after", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "execution_workspace_snapshots_operation_uniq": { + "name": "execution_workspace_snapshots_operation_uniq", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "execution_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source_operation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "execution_workspace_snapshots_generation_uniq": { + "name": "execution_workspace_snapshots_generation_uniq", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "execution_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "generation", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "execution_workspace_snapshots_workspace_idx": { + "name": "execution_workspace_snapshots_workspace_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "execution_workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "execution_workspace_snapshots_cleanup_idx": { + "name": "execution_workspace_snapshots_cleanup_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "delete_after", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.execution_workspaces": { "name": "execution_workspaces", "schema": "", @@ -50177,6 +50480,12 @@ "primaryKey": false, "notNull": false }, + "created_by_run_id": { + "name": "created_by_run_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, "dispatch_fingerprint": { "name": "dispatch_fingerprint", "type": "text", @@ -52779,6 +53088,12 @@ "primaryKey": false, "notNull": false }, + "next_eval_reason": { + "name": "next_eval_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, "next_eval_at": { "name": "next_eval_at", "type": "timestamp with time zone", @@ -62133,6 +62448,12 @@ "primaryKey": false, "notNull": false }, + "cleanup_verified_at": { + "name": "cleanup_verified_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, "created_at": { "name": "created_at", "type": "timestamp with time zone", diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index f1c5074d6e5..b42ac91d174 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -2962,6 +2962,13 @@ "when": 1793712676822, "tag": "0424_routine_creation_run", "breakpoints": true + }, + { + "idx": 425, + "version": "7", + "when": 1793724587619, + "tag": "0425_awesome_living_lightning", + "breakpoints": true } ] -} +} \ No newline at end of file diff --git a/packages/db/src/schema/public_mcp.ts b/packages/db/src/schema/public_mcp.ts index ec42c436739..7011a37aa0c 100644 --- a/packages/db/src/schema/public_mcp.ts +++ b/packages/db/src/schema/public_mcp.ts @@ -188,6 +188,9 @@ export const mcpAttachmentUploads = pgTable("mcp_attachment_uploads", { storageProvider: text("storage_provider").notNull(), expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(), cleanedAt: timestamp("cleaned_at", { withTimezone: true }), + // Written only after storage deletion and an authoritative absence readback. + // Legacy cleanedAt can predate truthful local filesystem error handling. + cleanupVerifiedAt: timestamp("cleanup_verified_at", { withTimezone: true }), createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), }, (t) => [ uniqueIndex("mcp_attachment_upload_request_uq").on(t.companyId, t.userId, t.requestId), diff --git a/server/src/__tests__/chat-channels.integration.test.ts b/server/src/__tests__/chat-channels.integration.test.ts index a24284fb64f..25d571411f2 100644 --- a/server/src/__tests__/chat-channels.integration.test.ts +++ b/server/src/__tests__/chat-channels.integration.test.ts @@ -1,3 +1,5 @@ +import { chatHistoryActionBlocks } from "./helpers/chat-history-proof.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { resolveCommittedNativeRecoveryNotices, pendingNativeRecoveryNotices } from "../services/native-runtime/native-recovery-notices.js"; import { issueRecoveryActionService } from "../services/issue-recovery-actions.js"; import { projectCommittedRun } from "../services/native-runtime/native-run-projection.js"; @@ -287,7 +289,8 @@ import { startEmbeddedPostgresTestDatabase, } from "./helpers/embedded-postgres.js"; -const externalTestDatabaseUrl = process.env.PAPERCLIP_TEST_DATABASE_URL; +const historyTestDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_HISTORY_TEST_DATABASE_URL"); +const externalTestDatabaseUrl = historyTestDatabaseUrl ?? process.env.PAPERCLIP_TEST_DATABASE_URL; const embeddedPostgresSupport = externalTestDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); @@ -1125,6 +1128,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { // Do not expire or time out its connections on those virtual clock jumps. db = createDb(externalTestDatabaseUrl ?? tempDb!.connectionString, { idleTimeoutSeconds: 0, connectTimeoutSeconds: 0, maxLifetimeSeconds: 0, + ...(historyTestDatabaseUrl ? { maxConnections: 4 } : {}), }); }, 90_000); @@ -3009,6 +3013,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { const [review] = await readChatGitHubReviewsByEndpointId(f.endpoint.id); expect(review.assessment?.findings[0].category).toBe(categories[0]); expect((await f.action()).status).toBe("processed"); + expect(await chatHistoryActionBlocks(db, (await f.action()).id)).toBe(false); expect(f.responseComments.size).toBe(1); }); it("publishes one score heading when a newly configured bot submits a formatted summary", async () => { @@ -3072,6 +3077,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { const [response] = await db.select().from(chatActions).where(and(eq(chatActions.deliveryId, delivery.id), eq(chatActions.kind, "github_response_comment"))); expect(response).toMatchObject({ status: "cancelled", result: { final: true, code: "passive_response_wait" } }); + expect(await chatHistoryActionBlocks(db, response.id)).toBe(false); const writes = f.responseWrites.length; const updatedAt = response.updatedAt; await responses.processPending(); @@ -3317,6 +3323,9 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { const body = [...f.responseComments.values()][0].body; expect(body).not.toMatch(/Review queued|Working on this/); expect(body).toContain("paperclip-response-state:final"); + const responses = await db.select().from(chatActions).where(and(eq(chatActions.endpointId, f.endpoint.id), eq(chatActions.kind, "github_response_comment"))); + expect(responses.length).toBeGreaterThan(0); + for (const receipt of responses) expect(await chatHistoryActionBlocks(db, receipt.id)).toBe(false); }); it("retires a legacy coalesced GitHub placeholder before replacing its receipt", async () => { const f = await publicationRetryFixture(); @@ -3428,6 +3437,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { expect([...f.responseComments.values()].some(comment => comment.body.includes("Newer progress"))).toBe(true); const [settled] = await db.select().from(chatActions).where(eq(chatActions.id, first.id)); expect(settled.result?.final).toBe(false); + expect(await chatHistoryActionBlocks(db, settled.id)).toBe(true); }); it.each([false, true])("recovers an approved queued GitHub review without another approval (revoked: %s)", async revoked => { const f = await publicationRetryFixture(); @@ -3463,6 +3473,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { await db.update(chatActions).set({ result: { ...action.result, retryAt: new Date(0).toISOString() } }).where(eq(chatActions.id, action.id)); await f.service.processPending(); expect((await f.action()).status).toBe(revoked ? "cancelled" : "processed"); + expect(await chatHistoryActionBlocks(db, (await f.action()).id)).toBe(false); if (revoked) await expect(gateway.executeTool({ sessionToken: session.token, tool: tool.name, parameters: f.assessment })).rejects.toThrow(); else expect(await gateway.executeTool({ sessionToken: session.token, tool: tool.name, parameters: f.assessment })) .toMatchObject({ status: "replayed", result: { data: { status: "processed", score: 3 } } }); @@ -4833,6 +4844,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { expect(f.responseWrites.filter(write => write.method === "POST")).toHaveLength(2); const [deleted] = await db.select().from(chatActions).where(and(eq(chatActions.endpointId, f.endpoint.id), eq(chatActions.kind, "github_response_comment"), sql`${chatActions.result}->>'id' = ${String(firstId)}`)); expect(deleted).toMatchObject({ status: "cancelled", result: { final: true, code: "response_deleted" } }); + expect(await chatHistoryActionBlocks(db, deleted.id)).toBe(false); await githubResponseCommentService(db, f.providerFetch).processPending(); expect(f.responseComments.size).toBe(1); await expect(githubRunReplyState(db, { ...f.session, endpointId: f.endpoint.id })).resolves.toBe("confirmed"); @@ -9849,6 +9861,9 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { expect(f.provider.mock.calls.filter(([url]) => String(url).endsWith("apps.manifest.create"))).toHaveLength(0); expect((await f.service.get(f.endpoint.id)).status).not.toBe("revoked"); expect((await f.service.get(f.endpoint.id)).setup.slackRegistration).toMatchObject({ status: "configured", appId: f.appId }); + const [reconnectReceipt] = await db.select().from(chatActions).where(and(eq(chatActions.endpointId, f.endpoint.id), eq(chatActions.kind, "slack_managed_reconnect"))); + expect(reconnectReceipt).toMatchObject({ status: "processed", result: null }); + expect(await chatHistoryActionBlocks(db, reconnectReceipt.id)).toBe(false); await f.service.slackRegistration.managed.provision(f.endpoint.id, actor, reconnectInput); await f.provision(); expect(f.provider.mock.calls.filter(([url]) => String(url).endsWith("apps.managedInstall"))).toHaveLength(1); @@ -17125,12 +17140,13 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { context.endpoint.id, ); expect(context.posts).toEqual([]); - if (change === "edited comment") + if (change === "edited comment") { expect(await context.receipt()).toMatchObject({ status: "cancelled", result: { attempts: 0, code: "source_changed_or_unavailable" }, }); - else expect(await context.receipt()).toBeUndefined(); + expect(await chatHistoryActionBlocks(db, (await context.receipt())!.id)).toBe(false); + } else expect(await context.receipt()).toBeUndefined(); expect(context.wakeup).not.toHaveBeenCalled(); } finally { await retirePublicationFixture(context.service, context.endpoint.id); @@ -17811,6 +17827,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { result: { code: "github_webhook_ingress_runtime_superseded" }, }); expect((await context.ingress())!.payload).not.toHaveProperty("body"); + expect(await chatHistoryActionBlocks(db, (await context.ingress())!.id)).toBe(false); expect(sdkDispatch).not.toHaveBeenCalled(); expect( await db @@ -17862,6 +17879,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { result: { code: "github_webhook_recovery_source_changed" }, }); expect(cancelled!.payload).not.toHaveProperty("body"); + expect(await chatHistoryActionBlocks(db, cancelled!.id)).toBe(false); expect(cancelled!.result?.retryable).not.toBe(true); expect(context.requests).toEqual([]); expect( @@ -43636,6 +43654,11 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { expect(original?.state).toBe("published"); expect(original?.payload.card?.actions).toEqual(expect.arrayContaining([expect.objectContaining({ type: "callback" })])); if (kind === "connection_intent") expect(original?.payload.card?.actions).toContainEqual({ type: "link", label: "Review connection", url: expect.any(String) }); + const controls = (await db.select().from(chatActions).where(and(eq(chatActions.endpointId, endpoint.id), + eq(sql`${chatActions.payload}->>'interactionId'`, interaction.id)))) + .filter(row => ["confirmation_response", "question_form_open", "question_form_submit", "question_answer"].includes(row.kind)); + expect(controls.length).toBeGreaterThan(0); + for (const control of controls) expect(await chatHistoryActionBlocks(db, control.id)).toBe(true); if (kind === "connection_intent") await interactions.resolveConnectionIntent(issue, interaction.id, { version: 1, outcome: "declined" }, { userId: "owner-user" }); else if (kind === "request_item_verdicts") await interactions.submitItemVerdicts(issue, interaction.id, { verdicts: [{ id: "docs", verdict: "approve" }] }, { userId: "owner-user" }); else await interactions.rejectInteraction(issue, interaction.id, {}, { userId: "owner-user" }); @@ -43645,6 +43668,11 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { expect(terminal?.state).toBe("published"); expect(terminal?.providerMessageId).toBe(original.providerMessageId); expect(terminal?.payload.card?.actions).toBeUndefined(); + for (const control of controls) expect(await chatHistoryActionBlocks(db, control.id)).toBe(false); + const control = controls[0]!; + await db.update(chatActions).set({ payload: { ...control.payload, interactionId: randomUUID() } }).where(eq(chatActions.id, control.id)); + expect(await chatHistoryActionBlocks(db, control.id)).toBe(true); + await db.update(chatActions).set({ payload: control.payload }).where(eq(chatActions.id, control.id)); } finally { await service.shutdown(); if (previousPublicUrl === undefined) delete process.env.PAPERCLIP_PUBLIC_URL; @@ -81351,6 +81379,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { expect(context.consentCards).toHaveLength(1); expect(context.fileCards).toEqual([]); expect(context.uploadRequest).not.toHaveBeenCalled(); + expect(await chatHistoryActionBlocks(db, intent!.id)).toBe(true); const before = await context.service.getPublicationBatchStatus( context.endpoint.id, context.conversation.id, @@ -81379,6 +81408,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { body: context.bytes, }); expect(context.fileCards).toHaveLength(1); + expect(await chatHistoryActionBlocks(db, intent!.id)).toBe(false); const status = await context.service.getPublicationBatchStatus( context.endpoint.id, context.conversation.id, diff --git a/server/src/__tests__/chat-run-milestone-recovery-work.test.ts b/server/src/__tests__/chat-run-milestone-recovery-work.test.ts index b1d3cd9bf24..f5fb96b4e1d 100644 --- a/server/src/__tests__/chat-run-milestone-recovery-work.test.ts +++ b/server/src/__tests__/chat-run-milestone-recovery-work.test.ts @@ -22,7 +22,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu const errors: unknown[] = []; beforeAll(async () => { if (!external) temporary = await startEmbeddedPostgresTestDatabase("chat-recovery-milestones-"); - db = createDb(external ?? temporary!.connectionString, { idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + db = createDb(external ?? temporary!.connectionString, { maxConnections: 4, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); fixtures = createChatMilestoneFixtures(db); }, 60_000); beforeEach(async () => { @@ -77,6 +77,9 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu }); it("wakes an older failed run when recovery refreshes its quota classification", async () => { + // Keep both idempotence checks before the provider's midnight reset even + // when this file runs near midnight and quiet() advances the fake clock. + vi.setSystemTime(new Date("2026-11-02T12:00:00.000Z")); const runId = randomUUID(), old = new Date(Date.now() - 48 * 60 * 60_000); await db.insert(heartbeatRuns).values({ id: runId, companyId: f.companyId, agentId: f.agentId, status: "failed", runtimeMode: "legacy", responsibleUserId: f.userId, @@ -96,6 +99,9 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu await flush(); expect(await publications()).toEqual([expect.objectContaining({ idempotencyKey: `run:${runId}:failed:${f.endpointId}` })]); await quiet(); + const [monitored] = await db.select({ nextCheckAt: issues.monitorNextCheckAt }).from(issues).where(eq(issues.id, f.issueId)); + expect(monitored.nextCheckAt).toEqual(new Date("2026-11-03T00:00:00.000Z")); + expect(monitored.nextCheckAt!.getTime()).toBeGreaterThan(Date.now()); expect(await recovery.reconcileStrandedAssignedIssues()).toMatchObject({ providerQuotaMonitored: 0, skipped: 1 }); await flush(); await quiet(); expect(await publications()).toHaveLength(1); }); diff --git a/server/src/__tests__/environment-cleanup-work.test.ts b/server/src/__tests__/environment-cleanup-work.test.ts index bd9419fa970..91fa30faadf 100644 --- a/server/src/__tests__/environment-cleanup-work.test.ts +++ b/server/src/__tests__/environment-cleanup-work.test.ts @@ -28,12 +28,13 @@ describe.skipIf(!support.supported)("environment cleanup scheduling", () => { let companyId: string, environmentId: string, draining: boolean, failing: boolean; let coordinator: ReturnType | undefined; let worker: ReturnType | undefined; - let releases: ReturnType, inspections: ReturnType, reads: ReturnType; + let releases: ReturnType, reads: ReturnType; + let inspections: ReturnType Promise<{ providerLeaseId: string | null; state: string }>>>; const errors: unknown[] = [], disposers: Array<() => void> = []; beforeAll(async () => { if (!external) database = await startEmbeddedPostgresTestDatabase("environment-cleanup-work-"); - db = createDb(external ?? database!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); - await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`))); + db = createDb(external ?? database!.connectionString, { maxConnections: 4, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + await Promise.all(Array.from({ length: 4 }, () => db.execute(sql`select pg_sleep(0.01)`))); }, 90_000); beforeEach(async () => { await db.execute(sql`TRUNCATE sandbox_creation_operations, companies, environments CASCADE`); @@ -48,6 +49,9 @@ describe.skipIf(!support.supported)("environment cleanup scheduling", () => { afterEach(async () => { worker?.stop(); await coordinator?.stop(); coordinator = undefined; worker = undefined; for (const stop of disposers.splice(0)) stop(); + // Creation ownership intentionally survives company deletion; company + // cascades therefore cannot isolate this case from the next test suite. + await db.delete(sandboxCreationOperations).where(eq(sandboxCreationOperations.companyId, companyId)); vi.useRealTimers(); vi.restoreAllMocks(); expect(errors).toEqual([]); }); @@ -61,12 +65,13 @@ describe.skipIf(!support.supported)("environment cleanup scheduling", () => { let historySettled = !history; worker = registerEnvironmentCleanupWork(db, coordinator, { sweepPending: sweepOverride ?? (backoffMs => heartbeat.sweepPendingCleanupLeases({ backoffMs })), - historical: history ? { hasPending: history.hasPending, + historical: history ? { hasPending: () => history.hasPending(), sweep: async () => { const result = await history.sweep(); historySettled = true; return result; } } : undefined, canRun: () => !draining, }); await worker.ready; await vi.waitFor(() => expect(historySettled).toBe(true)); + return history || undefined; } async function settle(ms = 30_000, wait = true) { await vi.advanceTimersByTimeAsync(ms); @@ -116,7 +121,7 @@ describe.skipIf(!support.supported)("environment cleanup scheduling", () => { ])); await quiet(); }); - it.each([false, true])("settles a deleted-company creation as manual history across restart without idle polling (contextual=%s)", async contextual => { + it.each([false, true])("retains deleted-company creation uncertainty across a quiet worker restart (contextual=%s)", async contextual => { await start(); await quiet(); const created = await createOwnedSandbox(db, { companyId, environmentId, provider: "fixture", driverKey: "fixture", config: {}, metadata: contextual ? { runnerTaskLease: true, runnerTaskCreationContextId: randomUUID() } : {} }, @@ -126,8 +131,11 @@ describe.skipIf(!support.supported)("environment cleanup scheduling", () => { await vi.waitFor(async () => expect(await db.select().from(sandboxCreationOperations)).toEqual([ expect.objectContaining({ state: "needs_reconciliation", metadata: expect.objectContaining({ creationReconciliationReason: "company_deleted" }) }), ])); - if (contextual) await recordSandboxCreationResource(db, { id: String(created.metadata!.creationOperationId), companyId, environmentId, - providerLeaseId: created.providerLeaseId }); + if (contextual) { + if (!created.providerLeaseId) throw new Error("Fixture creation returned no provider lease identity"); + await recordSandboxCreationResource(db, { id: String(created.metadata!.creationOperationId), companyId, environmentId, + providerLeaseId: created.providerLeaseId }); + } await quiet(); worker!.stop(); await coordinator!.stop(); await start(); await quiet(); @@ -135,9 +143,84 @@ describe.skipIf(!support.supported)("environment cleanup scheduling", () => { const ownerId = randomUUID(); const hold = { ownerId, draining: true, startedAt: new Date(Date.now() - 1000), expiresAt: new Date(Date.now() + 60_000), activeRuns: 0, pendingWakes: 0 }; expect(await readIdleSleepSafety(db, () => hold, Date.now, ownerId, async () => "none")) - .toEqual({ version: 1, backgroundWork: "none" }); + .toEqual({ version: 1, backgroundWork: "present" }); await quiet(); }); + it("discharges an empty historical metadata notification during provider spacing", async () => { + const lease = await pending(); + await environmentService(db).releaseLease(lease.id, "released", { cleanupStatus: "success" }); + const history = await start(true); + await settle(30_000); + expect(await history!.hasPending()).toBe(false); + const checked = inspections.mock.calls.length; + await environmentService(db).updateLeaseMetadata(lease.id, { driver: "sandbox", presentation: "changed" }); + await vi.waitFor(() => { expect(idleWorkSnapshot().active).toBe(0); expect(coordinator!.nextWakeAt()).toBeNull(); }); + expect(inspections).toHaveBeenCalledTimes(checked); + expect(await history!.hasPending()).toBe(false); + const count = reads.mock.calls.length; + await vi.advanceTimersByTimeAsync(300_000); + expect(reads).toHaveBeenCalledTimes(count); expect(inspections).toHaveBeenCalledTimes(checked); + }); + it.each([false, true])("preserves historical debt arriving during an empty fast-path read (unresolved transaction=%s)", async unresolved => { + const passive = await pending(), debt = await pending(); + await environmentService(db).releaseLease(passive.id, "released", { cleanupStatus: "success" }); + await environmentService(db).releaseLease(debt.id, "failed", { cleanupStatus: "success" }); + const history = (await start(true))!; + await settle(30_000); expect(inspections).toHaveBeenCalledTimes(1); + let readStarted = false, releaseRead!: () => void, writeReady!: () => void, releaseWrite!: () => void; + const readGate = new Promise(resolve => { releaseRead = resolve; }); + const written = new Promise(resolve => { writeReady = resolve; }); + const writeGate = new Promise(resolve => { releaseWrite = resolve; }); + const original = history.hasPending; + const check = vi.spyOn(history, "hasPending").mockImplementationOnce(async () => { + const pending = await original(); expect(pending).toBe(false); readStarted = true; await readGate; return pending; + }); + let writing: Promise | undefined; + try { + await environmentService(db).updateLeaseMetadata(passive.id, { presentation: "changed" }); + await vi.waitFor(() => expect(readStarted).toBe(true)); + writing = db.transaction(async tx => { + await environmentService(tx as unknown as Db).updateLeaseMetadata(debt.id, { driver: "sandbox", changed: true }); + writeReady(); if (unresolved) await writeGate; + }); + await written; + if (!unresolved) await writing; + releaseRead(); + await vi.advanceTimersByTimeAsync(1); + expect(idleWorkSnapshot().active).toBeGreaterThan(0); + expect(inspections).toHaveBeenCalledTimes(1); + releaseWrite(); await writing; + await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1); expect(await original()).toBe(true); }); + expect(idleWorkSnapshot().active).toBeGreaterThan(0); + const ownerId = randomUUID(); + // One drain hold must retain its identity while the safety read runs. + const hold = { ownerId, draining: true, startedAt: new Date(Date.now() - 1000), + expiresAt: new Date(Date.now() + 60_000), activeRuns: 0, pendingWakes: 0 }; + expect((await readIdleSleepSafety(db, () => hold, Date.now, ownerId, async () => "none")).backgroundWork).toBe("present"); + await settle(300_000); + await vi.waitFor(() => { expect(inspections).toHaveBeenCalledTimes(2); expect(idleWorkSnapshot().active).toBe(0); }); + expect(await original()).toBe(false); await quiet(); + } finally { releaseRead(); releaseWrite(); await writing; check.mockRestore(); } + }); + it("retains a failed historical fast-path read through drain and clears it after resume", async () => { + const lease = await pending(); + await environmentService(db).releaseLease(lease.id, "released", { cleanupStatus: "success" }); + const history = (await start(true))!; + await settle(30_000); + vi.spyOn(history, "hasPending").mockRejectedValueOnce(new Error("historical debt read unavailable")); + await environmentService(db).updateLeaseMetadata(lease.id, { presentation: "changed" }); + await vi.waitFor(() => expect(errors).toHaveLength(1)); + expect(idleWorkSnapshot().active).toBeGreaterThan(0); expect(coordinator!.nextWakeAt()).not.toBeNull(); + errors.length = 0; draining = true; + const count = reads.mock.calls.length; + await settle(60_000, false); + expect(reads).toHaveBeenCalledTimes(count); expect(idleWorkSnapshot().active).toBeGreaterThan(0); + draining = false; await settle(); + await vi.waitFor(() => { expect(idleWorkSnapshot().active).toBe(0); expect(coordinator!.nextWakeAt()).toBeNull(); }); + expect(inspections).not.toHaveBeenCalled(); await quiet(); + worker!.stop(); await coordinator!.stop(); + expect(idleWorkSnapshot().active).toBe(0); + }); it("verifies ordinary cleanup carrying creation provenance and stops provider polling", async () => { const lease = await pending(); await environmentService(db).updateLeaseMetadata(lease.id, { driver: "sandbox", creationOperationId: randomUUID() }); diff --git a/server/src/__tests__/helpers/chat-history-proof.ts b/server/src/__tests__/helpers/chat-history-proof.ts new file mode 100644 index 00000000000..60dcef89966 --- /dev/null +++ b/server/src/__tests__/helpers/chat-history-proof.ts @@ -0,0 +1,11 @@ +import { sql } from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; +import { idleChatHistoryChecks } from "../../services/chat-channels/idle-history-proof.js"; + +/** Scope the actual shared predicate to a real producer's receipt. Other pending + * fixture work remains independently observable through its owning gates. */ +export async function chatHistoryActionBlocks(db: Db, actionId: string): Promise { + const check = idleChatHistoryChecks.find(check => check.reason === "chat_actions")!; + const [row] = await db.execute<{ blocked: boolean }>(sql`SELECT EXISTS (${sql.raw(check.query)} AND a.id = ${actionId}::uuid) AS blocked`); + return row.blocked; +} diff --git a/server/src/__tests__/idle-completed-history.test.ts b/server/src/__tests__/idle-completed-history.test.ts new file mode 100644 index 00000000000..389422f543c --- /dev/null +++ b/server/src/__tests__/idle-completed-history.test.ts @@ -0,0 +1,2208 @@ +import { createHash, generateKeyPairSync, randomBytes, randomUUID } from "node:crypto"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import net from "node:net"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { eq, sql } from "drizzle-orm"; +import * as tables from "@paperclipai/db"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { completedHistoryChecks, readIdleSleepSafety } from "../services/idle-sleep-safety.js"; +import { readIdleIssueSchedule } from "../services/idle-issue-schedule.js"; +import { companyTransferRunService } from "../services/company-transfer-runs.js"; +import { inspectIdleSpool } from "../services/idle-local-work.js"; + +const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_HISTORY_TEST_DATABASE_URL"); +const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); + +(support.supported ? describe : describe.skip)("retained completed history", () => { + let database: Awaited> | undefined; + let db: ReturnType; + const now = new Date(); + const ownerId = randomUUID(); + const hold = () => ({ ownerId, draining: true, startedAt: new Date(now.getTime() - 1000), + expiresAt: new Date(Date.now() + 60_000), activeRuns: 0, pendingWakes: 0 }); + const read = () => readIdleSleepSafety(db, hold, Date.now, ownerId, async () => "none"); + const blocked = async (reason: string) => { + const check = completedHistoryChecks.find(check => check.reason === reason); + if (!check) throw new Error(`Missing family ${reason}`); + const started = performance.now(); + const [row] = await db.execute<{ blocked: boolean }>(sql`SELECT EXISTS (${sql.raw(check.query)}) AS blocked`); + const elapsed = performance.now() - started; + if (elapsed > 1000) console.info(`History predicate ${reason}: ${Math.round(elapsed)}ms`); + return row.blocked; + }; + const blockedActionKind = async (kind: string) => { + const query = completedHistoryChecks.find(check => check.reason === "chat_actions")!.query; + const [row] = await db.execute<{ blocked: boolean }>(sql`SELECT EXISTS (${sql.raw(query)} AND a.kind = ${kind}) AS blocked`); + return row.blocked; + }; + beforeAll(async () => { + if (!externalDatabaseUrl) database = await startEmbeddedPostgresTestDatabase("idle-history-"); + db = tables.createDb(externalDatabaseUrl ?? database!.connectionString, { maxConnections: 4 }); + }, 90_000); + afterEach(async () => { await db.transaction(async tx => { + await tx.execute(sql`SET LOCAL client_min_messages = warning`); + await tx.execute(sql`TRUNCATE companies CASCADE`); + }); }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }); + + async function seed() { + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID(); + await db.insert(tables.companies).values({ id: companyId, name: "Retained history", issuePrefix: randomUUID().slice(0, 8) }); + await db.insert(tables.agents).values({ id: agentId, companyId, name: "Idle agent", status: "idle", adapterType: "process" }); + await db.insert(tables.issues).values({ id: issueId, companyId, title: "Finished work", status: "done", assigneeAgentId: agentId }); + await db.insert(tables.heartbeatRuns).values({ id: runId, companyId, agentId, status: "succeeded", + invocationSource: "on_demand", finishedAt: now, contextSnapshot: { issueId } }); + return { companyId, agentId, issueId, runId }; + } + + async function connection(f: Awaited>) { + const [application] = await db.insert(tables.toolApplications).values({ companyId: f.companyId, + applicationKey: randomUUID(), name: "History fixture", type: "rest_api" }).returning(); + const [connection] = await db.insert(tables.toolConnections).values({ companyId: f.companyId, + applicationId: application.id, name: "Disabled draft", uid: randomUUID(), transport: "rest_api", status: "draft", enabled: false }).returning(); + const [grant] = await db.insert(tables.connectionGrants).values({ companyId: f.companyId, connectionId: connection.id, + kind: "organization", status: "revoked", revokedAt: now }).returning(); + return { connectionId: connection.id, grantId: grant.id }; + } + + async function mcp(f: Awaited>, agent = false) { + const userId = randomUUID(), clientId = randomUUID(); + await db.insert(tables.authUsers).values({ id: userId, name: "History user", email: `${userId}@example.test`, createdAt: now, updatedAt: now }); + await db.insert(tables.mcpOauthClients).values({ id: clientId, name: "History client", redirectUris: ["https://example.test/callback"] }); + const [grant] = await db.insert(tables.mcpOauthGrants).values({ companyId: f.companyId, userId, clientId, + resource: "https://example.test/mcp", purpose: agent ? "agent" : "personal", agentId: agent ? f.agentId : null, + scopes: agent ? ["paperclip:agent", "offline_access"] : ["paperclip:read"], revokedAt: now }).returning(); + return { userId, grantId: grant.id }; + } + + async function decisionFixture() { + const { decisionService } = await import("../services/decisions.js"); + const f = await seed(), p = await mcp(f); + await db.insert(tables.companyMemberships).values({ companyId: f.companyId, principalType: "user", principalId: p.userId, + status: "active", membershipRole: "member" }); + await db.update(tables.heartbeatRuns).set({ status: "running", responsibleUserId: p.userId }).where(eq(tables.heartbeatRuns.id, f.runId)); + await db.update(tables.issues).set({ status: "in_progress", responsibleUserId: p.userId }).where(eq(tables.issues.id, f.issueId)); + const actor = { type: "agent" as const, companyId: f.companyId, agentId: f.agentId, runId: f.runId, source: "agent_jwt" as const, + onBehalfOfUserId: p.userId, onBehalfOfMemberships: [{ companyId: f.companyId, membershipRole: "member", status: "active" }] }; + const userActor = { type: "board" as const, userId: p.userId, companyIds: [f.companyId], source: "session" as const, + memberships: [{ companyId: f.companyId, membershipRole: "member", status: "active" }] }; + const service = decisionService(db, { wakeOriginAgent: async () => null }); + const finish = () => db.update(tables.heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(tables.heartbeatRuns.id, f.runId)); + return { ...f, ...p, actor, userActor, service, finish }; + } + + async function connectedMcp(f: Awaited>) { + const { createPublicMcpOAuth, hashMcpSecret } = await import("../services/public-mcp/oauth.js"); + const { createMcpApiDispatch } = await import("../services/public-mcp/capabilities.js"); + const { issueRoutes } = await import("../routes/issues.js"); + const { createStorageService } = await import("../storage/service.js"); + const { createLocalDiskStorageProvider } = await import("../storage/local-disk-provider.js"); + const { instanceSettingsService } = await import("../services/instance-settings.js"); + const { Router } = await import("express"); + await instanceSettingsService(db).updateExperimental({ enablePublicMcp: true }); + const p = await mcp(f), origin = "https://history.example", resource = `${origin}/mcp/paperclip`; + await db.insert(tables.companyMemberships).values({ companyId: f.companyId, principalType: "user", principalId: p.userId, + membershipRole: "owner", status: "active" }); + await db.update(tables.mcpOauthGrants).set({ revokedAt: null, resource, scopes: ["paperclip:read", "paperclip:write"] }) + .where(eq(tables.mcpOauthGrants.id, p.grantId)); + const token = randomBytes(32).toString("base64url"); + await db.insert(tables.mcpOauthTokens).values({ grantId: p.grantId, kind: "access", tokenHash: hashMcpSecret(token), + expiresAt: new Date(Date.now() + 600_000) }); + const directory = await mkdtemp(path.join(tmpdir(), "history-mcp-storage-")); + const router = Router(), storage = createStorageService(createLocalDiskStorageProvider(directory)); + router.use(issueRoutes(db, storage)); + const oauth = createPublicMcpOAuth(db, { origin, resource }), api = createMcpApiDispatch(router); + return { ...p, token, oauth, api, storage, cleanup: () => rm(directory, { recursive: true, force: true }) }; + } + + async function slackFixture() { + const f = await seed(), c = await connection(f), p = await mcp(f); + await db.update(tables.toolConnections).set({ enabled: true, status: "active" }).where(eq(tables.toolConnections.id, c.connectionId)); + await db.insert(tables.companyMemberships).values({ companyId: f.companyId, principalType: "user", principalId: p.userId, + status: "active", membershipRole: "owner" }); + const [endpoint] = await db.insert(tables.chatEndpoints).values({ companyId: f.companyId, connectionId: c.connectionId, + assignedAgentId: f.agentId, publicId: randomUUID(), provider: "slack", status: "active", providerAccountId: "TWORKSPACE", + botExternalId: "UBOT", allowDirectMessages: true, publicationMode: "automatic" }).returning(); + const [conversation] = await db.insert(tables.chatConversations).values({ companyId: f.companyId, endpointId: endpoint.id, + issueId: f.issueId, externalConversationId: "DCHANNEL", externalThreadId: "slack:DCHANNEL:123.456", externalLabel: "Personal", + state: "active", isDirectMessage: true }).returning(); + const [principal] = await db.insert(tables.chatExternalPrincipals).values({ companyId: f.companyId, provider: "slack", + providerAccountId: "TWORKSPACE", externalId: "UREQUESTER", kind: "user", isBot: false }).returning(); + const authority = { endpoint, issueId: f.issueId, conversation, userId: p.userId, slackUserId: "UREQUESTER", principalId: principal.id, + deliveryId: null, identityContextId: null, revision: "fixture-revision", workMode: "standard", botToken: "fixture-bot-token" }; + return { ...f, ...p, ...c, endpoint, conversation, principal, authority }; + } + + it("checks all 32 families on the real migrated schema and keeps an empty instance eligible", async () => { + expect(new Set(completedHistoryChecks.map(c => c.reason)).size).toBe(32); + for (const check of completedHistoryChecks) expect(await blocked(check.reason), check.reason).toBe(false); + expect(await read()).toEqual({ version: 1, backgroundWork: "none" }); + }); + + it.each([false, true])("keeps real sandbox creation uncertainty blocked through owned cleanup, manualRemaining=%s", async manualRemaining => { + const { createOwnedSandbox, recoverSandboxCreations, recordSandboxCreationResource } = await import("../services/sandbox-creation-operations.js"); + const { environmentRuntimeService } = await import("../services/environment-runtime.js"); + const { heartbeatService } = await import("../services/heartbeat.js"); + const { nextEnvironmentCleanupAt } = await import("../services/environment-cleanup-state.js"); + const f = await seed(), environmentId = randomUUID(); + const directory = await mkdtemp(path.join(tmpdir(), "history-sandbox-creation-")); + await db.insert(tables.environments).values({ id: environmentId, name: environmentId, driver: "sandbox" }); + const snapshot = () => readIdleSleepSafety(db, hold, Date.now, ownerId, async () => "none", + undefined, undefined, false, false, { collectDiagnostics: true }); + const reason = async (key: string) => (await snapshot()).diagnostics!.checks.find(check => check.reason === key)!.state; + const uncertain = async (contextual: boolean) => { + const providerLeaseId = randomUUID(); + const result = await createOwnedSandbox(db, { companyId: f.companyId, environmentId, heartbeatRunId: f.runId, + provider: "history-fixture", driverKey: "history-fixture", config: {}, requestKey: providerLeaseId, + metadata: { driver: "sandbox", ...(contextual ? { runnerTaskLease: true, runnerTaskCreationContextId: randomUUID() } : {}) } }, + async () => { + await writeFile(path.join(directory, providerLeaseId), "owned provider resource"); + return { providerLeaseId }; + }); + const id = String(result.metadata!.creationOperationId); + // Advance the actual creation owner's saved deadline, not its status. + await db.update(tables.sandboxCreationOperations).set({ nextAttemptAt: new Date(0) }) + .where(eq(tables.sandboxCreationOperations.id, id)); + await recoverSandboxCreations(db, async () => null); + return { id, providerLeaseId }; + }; + try { + expect(await read()).toEqual({ version: 1, backgroundWork: "none" }); + const contextual = await uncertain(true); + const current = async (id: string) => (await db.select().from(tables.sandboxCreationOperations) + .where(eq(tables.sandboxCreationOperations.id, id)))[0]; + expect(await current(contextual.id)).toMatchObject({ state: "needs_reconciliation", leaseId: null }); + // Isolated RED for the old pending-only catalog: there is no live run, + // finalization, lease, local worker, or other durable blocker to mask it. + expect(await read()).toEqual({ version: 1, backgroundWork: "present" }); + expect(await reason("sandbox_creation_operations")).toBe("present"); + await recordSandboxCreationResource(db, { id: contextual.id, companyId: f.companyId, environmentId, + providerLeaseId: contextual.providerLeaseId }); + expect(await nextEnvironmentCleanupAt(db)).not.toBeNull(); + expect(await reason("sandbox_creation_operations")).toBe("present"); + await recoverSandboxCreations(db, async () => { throw new Error("Known late resource must use its exact owner"); }); + const pending = await current(contextual.id); + expect(pending).toMatchObject({ state: "cleanup_pending", leaseId: expect.any(String) }); + expect(await reason("sandbox_creation_operations")).toBe("none"); + expect(await reason("environment_leases")).toBe("present"); + expect(await read()).toEqual({ version: 1, backgroundWork: "present" }); + const manual = manualRemaining ? await uncertain(false) : undefined; + if (manual) { + expect(await current(manual.id)).toMatchObject({ state: "needs_reconciliation", leaseId: null }); + expect(await reason("sandbox_creation_operations")).toBe("present"); + } + let destroyed = 0; + const runtime = environmentRuntimeService(db, { drivers: [{ driver: "sandbox", + acquireRunLease: async () => { throw new Error("Unexpected provider acquire"); }, + releaseRunLease: async () => { throw new Error("Unexpected provider release"); }, + resolveCapabilities: async () => { throw new Error("Unexpected capability resolution"); }, + retryPendingSandboxTeardown: async ({ lease }: { lease: { providerLeaseId: string | null } }) => { + expect(lease.providerLeaseId).toBe(contextual.providerLeaseId); + const resource = path.join(directory, lease.providerLeaseId!); + expect(await readFile(resource, "utf8")).toBe("owned provider resource"); + await rm(resource); + await expect(readFile(resource)).rejects.toMatchObject({ code: "ENOENT" }); + destroyed++; + return { providerLeaseId: lease.providerLeaseId, state: "destroyed" as const }; + } }] }); + await heartbeatService(db, { environmentRuntime: runtime }).sweepPendingCleanupLeases({ backoffMs: 0 }); + expect(destroyed).toBe(1); + expect(await current(contextual.id)).toMatchObject({ state: "resolved", leaseId: pending.leaseId }); + expect(await reason("environment_leases")).toBe("none"); + expect(await reason("sandbox_creation_operations")).toBe(manual ? "present" : "none"); + expect(await read()).toEqual({ version: 1, backgroundWork: manual ? "present" : "none" }); + if (manual) { + expect(await current(manual.id)).toMatchObject({ state: "needs_reconciliation", leaseId: null }); + expect(await readFile(path.join(directory, manual.providerLeaseId), "utf8")).toBe("owned provider resource"); + } + // A different creation is still real pending work after this one settles. + await createOwnedSandbox(db, { companyId: f.companyId, environmentId, provider: "history-fixture", + driverKey: "history-fixture", config: {}, metadata: {} }, async () => { + const providerLeaseId = randomUUID(); + await writeFile(path.join(directory, providerLeaseId), "another pending provider resource"); + return { providerLeaseId }; + }); + expect(await read()).toEqual({ version: 1, backgroundWork: "present" }); + } finally { + await db.delete(tables.sandboxCreationOperations).where(eq(tables.sandboxCreationOperations.companyId, f.companyId)); + await db.delete(tables.environments).where(eq(tables.environments.id, environmentId)); + await rm(directory, { recursive: true, force: true }); + } + }); + + it.each(["completed", "cancelled"] as const)("retains a %s transfer without hiding its actual spool", async status => { + const f = await seed(); + const result = await companyTransferRunService.resumeOrCreate(db, { companyId: f.companyId, direction: "import", + actorKey: "board:history", idempotencyKey: randomUUID(), containerRef: { kind: "upload" } }); + expect(await blocked("company_transfer_runs")).toBe(true); + if (status === "completed") { + await companyTransferRunService.recordManifest(db, result.run.id, { chunks: [{ name: "tasks.jsonl" }], blobs: [] }, "a".repeat(64)); + await companyTransferRunService.start(db, result.run.id); + expect(await companyTransferRunService.completePart(db, result.run.id, "tasks.jsonl")).toBe(true); + expect(await companyTransferRunService.claimApply(db, result.run.id)).toBe(true); + await companyTransferRunService.complete(db, result.run.id); + } + else await companyTransferRunService.cancel(db, result.run.id); + expect(await blocked("company_transfer_runs")).toBe(false); + const directory = await mkdtemp(path.join(tmpdir(), "history-spool-")); + try { + await writeFile(path.join(directory, result.run.id), "unfinished cleanup"); + const local = () => inspectIdleSpool(directory); + expect(await readIdleSleepSafety(db, hold, Date.now, ownerId, local)).toEqual({ version: 1, backgroundWork: "present" }); + await rm(path.join(directory, result.run.id)); + expect(await readIdleSleepSafety(db, hold, Date.now, ownerId, local)).toEqual({ version: 1, backgroundWork: "none" }); + expect(await companyTransferRunService.getRun(db, result.run.id)).toMatchObject({ status }); + const next = await companyTransferRunService.resumeOrCreate(db, { companyId: f.companyId, direction: "import", + actorKey: "board:history", idempotencyKey: randomUUID(), containerRef: { kind: "upload" } }); + expect(await blocked("company_transfer_runs")).toBe(true); + await companyTransferRunService.fail(db, next.run.id, "retry remains available"); + expect(await blocked("company_transfer_runs")).toBe(true); + } finally { await rm(directory, { recursive: true, force: true }); } + }); + + it("uses the attachment owner to commit bytes and clean an abandoned upload", async () => { + const { createPublicMcpTransfers } = await import("../services/public-mcp/file-transfers.js"); + const { default: express } = await import("express"); + const { default: request } = await import("supertest"); + const f = await seed(), p = await connectedMcp(f); + try { + const transfers = createPublicMcpTransfers(db, p.oauth, p.api, p.storage); + const principal = await p.oauth.authenticate(p.token), bytes = Buffer.from("verified retained attachment"); + const args = { companyId: f.companyId, taskId: f.issueId, requestId: randomUUID(), filename: "result.txt", contentType: "text/plain", + byteSize: bytes.length, sha256: createHash("sha256").update(bytes).digest("hex") }; + const upload = await transfers.getUploadUrl(principal, args); + expect(await blocked("mcp_attachment_uploads")).toBe(true); + const app = express(); app.use(transfers.router); + const url = new URL(String(upload.url)); + const result = await request(app).put(url.pathname + url.search).set("Content-Type", "text/plain").send(bytes).timeout({ response: 5000, deadline: 10_000 }); + expect(result.status).toBe(200); + expect(await blocked("mcp_attachment_uploads")).toBe(false); + await db.update(tables.mcpAttachmentUploads).set({ cleanedAt: now, cleanupVerifiedAt: now }).where(eq(tables.mcpAttachmentUploads.id, String(upload.uploadId))); + expect(await blocked("mcp_attachment_uploads")).toBe(true); + await db.update(tables.mcpAttachmentUploads).set({ cleanedAt: null, cleanupVerifiedAt: null }).where(eq(tables.mcpAttachmentUploads.id, String(upload.uploadId))); + const abandonedArgs = { ...args, requestId: randomUUID() }; + const abandoned = await transfers.getUploadUrl(principal, abandonedArgs); + expect(await blocked("mcp_attachment_uploads")).toBe(true); + const [old] = await db.select().from(tables.mcpAttachmentUploads).where(eq(tables.mcpAttachmentUploads.id, String(abandoned.uploadId))); + await p.storage.putFile({ companyId: f.companyId, namespace: "mcp-transfers", objectKey: old.objectKey, + originalFilename: args.filename, contentType: args.contentType, body: bytes }); + // Model the old owner's optimistic cleanup: its stamp exists while bytes + // survive. It must stay blocked until actual deletion and readback. + await db.update(tables.mcpAttachmentUploads).set({ expiresAt: new Date(0), cleanedAt: new Date() }).where(eq(tables.mcpAttachmentUploads.id, old.id)); + expect(await p.storage.headObject(f.companyId, old.objectKey)).toMatchObject({ exists: true }); + expect(await blocked("mcp_attachment_uploads")).toBe(true); + await transfers.getDownloadUrl(principal, result.body.attachment.id); + expect(await p.storage.headObject(f.companyId, old.objectKey)).toEqual({ exists: false }); + expect(await blocked("mcp_attachment_uploads")).toBe(false); + const verified = (await db.select().from(tables.mcpAttachmentUploads).where(eq(tables.mcpAttachmentUploads.id, old.id)))[0]; + const staleUrl = new URL(String(abandoned.url)); + expect((await request(app).put(staleUrl.pathname + staleUrl.search).set("Content-Type", "text/plain").send(bytes) + .timeout({ response: 5000, deadline: 10_000 })).status).toBe(403); + expect(await p.storage.headObject(f.companyId, old.objectKey)).toEqual({ exists: false }); + expect((await db.select().from(tables.mcpAttachmentUploads).where(eq(tables.mcpAttachmentUploads.id, old.id)))[0]).toEqual(verified); + const reissued = await transfers.getUploadUrl(principal, abandonedArgs); + expect((await db.select().from(tables.mcpAttachmentUploads).where(eq(tables.mcpAttachmentUploads.id, old.id)))[0]) + .toMatchObject({ cleanedAt: null, cleanupVerifiedAt: null }); + expect(await blocked("mcp_attachment_uploads")).toBe(true); + const secondUrl = new URL(String(reissued.url)); + const second = await request(app).put(secondUrl.pathname + secondUrl.search).set("Content-Type", "text/plain").send(bytes) + .timeout({ response: 5000, deadline: 10_000 }); + expect(second.status).toBe(200); + for (const attachment of [result.body.attachment, second.body.attachment]) { + expect(await p.api(principal, "DELETE", `/attachments/${attachment.id}`)).toEqual({ ok: true }); + } + const [removed] = await db.select().from(tables.mcpAttachmentUploads).where(eq(tables.mcpAttachmentUploads.id, String(upload.uploadId))); + expect(removed).toMatchObject({ attachmentId: result.body.attachment.id, cleanedAt: expect.any(Date), cleanupVerifiedAt: expect.any(Date) }); + expect(await p.storage.headObject(f.companyId, removed.objectKey)).toEqual({ exists: false }); + expect(await blocked("mcp_attachment_uploads")).toBe(false); + await expect(transfers.getUploadUrl(principal, args)).rejects.toThrow(); + expect((await db.select().from(tables.mcpAttachmentUploads).where(eq(tables.mcpAttachmentUploads.id, removed.id)))[0]).toEqual(removed); + expect(await blocked("mcp_attachment_uploads")).toBe(false); + await db.update(tables.mcpAttachmentUploads).set({ cleanupVerifiedAt: null }).where(eq(tables.mcpAttachmentUploads.id, removed.id)); + expect(await blocked("mcp_attachment_uploads")).toBe(true); + } finally { await p.cleanup(); } + }, 60_000); + + it("renews only an expired live agent attachment's publication repair window", async () => { + const { createPublicMcpTransfers } = await import("../services/public-mcp/file-transfers.js"); + const { createPublicMcpOAuth } = await import("../services/public-mcp/oauth.js"); + const { instanceSettingsService } = await import("../services/instance-settings.js"); + const { default: express } = await import("express"); + const { default: request } = await import("supertest"); + const f = await seed(), p = await connectedMcp(f); + try { + const board = await p.oauth.authenticate(p.token), sessionId = randomUUID(); + const resource = `${p.oauth.config.origin}/mcp/runner`; + await instanceSettingsService(db).updateExperimental({ enableOpenAiDot: true }); + await db.update(tables.mcpOauthGrants).set({ purpose: "agent", agentId: f.agentId, + resource, scopes: ["paperclip:agent", "offline_access"] }).where(eq(tables.mcpOauthGrants.id, p.grantId)); + await db.update(tables.issues).set({ status: "in_progress" }).where(eq(tables.issues.id, f.issueId)); + await db.update(tables.heartbeatRuns).set({ status: "running", finishedAt: null, runtimeMode: "native", + driverKind: "grok_bot_mcp", nativeIssueId: f.issueId }).where(eq(tables.heartbeatRuns.id, f.runId)); + const oauth = createPublicMcpOAuth(db, { ...p.oauth.config, resource }); + const principal = await oauth.authenticate(p.token); + let ownerAvailable = true; + let ownerRunId = f.runId; + let pauseRepair = false, repairLocked!: () => void, releaseRepair!: () => void; + const locked = new Promise(resolve => { repairLocked = resolve; }); + const released = new Promise(resolve => { releaseRepair = resolve; }); + // Named work-session boundary; OAuth, transfer HTTP, storage, attachment + // registration and native publication receipt all use their real owners. + const transfers = createPublicMcpTransfers(db, oauth, p.api, p.storage, { agentSession: { + authorize: async (actor, session, taskId, _write, queryDb) => { + if (!ownerAvailable || actor.grant.id !== p.grantId || actor.grant.agentId !== f.agentId + || session !== sessionId || taskId !== f.issueId) throw new Error("work session unavailable"); + if (pauseRepair && queryDb) { pauseRepair = false; repairLocked(); await released; } + return { agentId: f.agentId, runId: ownerRunId }; + }, + } }); + const app = express(); app.use(transfers.router); + const bytes = Buffer.from("expired live publication repair"); + const args = { companyId: f.companyId, taskId: f.issueId, requestId: randomUUID(), filename: "repair.txt", + contentType: "text/plain", byteSize: bytes.length, sha256: createHash("sha256").update(bytes).digest("hex") }; + const upload = await transfers.getUploadUrl(principal, args, sessionId); + const put = async (value: unknown) => { + const url = new URL(String(value)); + return request(app).put(url.pathname + url.search).set("Content-Type", "text/plain").send(bytes) + .timeout({ response: 5000, deadline: 10_000 }); + }; + const initial = await put(upload.url); + expect(initial.status).toBe(200); + const uploadId = String(upload.uploadId), attachmentId = initial.body.attachment.id; + const current = async () => (await db.select().from(tables.mcpAttachmentUploads) + .where(eq(tables.mcpAttachmentUploads.id, uploadId)))[0]; + const loseReceipt = async () => { + const [run] = await db.select().from(tables.heartbeatRuns).where(eq(tables.heartbeatRuns.id, f.runId)); + const receipts = { ...run.resultJson?.semanticToolReceipts as Record }; + delete receipts[`external-upload:${uploadId}`]; + await db.update(tables.heartbeatRuns).set({ resultJson: { ...run.resultJson, semanticToolReceipts: receipts } }) + .where(eq(tables.heartbeatRuns.id, f.runId)); + await db.update(tables.mcpAttachmentUploads).set({ expiresAt: new Date(0) }) + .where(eq(tables.mcpAttachmentUploads.id, uploadId)); + }; + await loseReceipt(); + const expired = await current(); + expect((await put(upload.url)).status).toBe(403); + const repair = await transfers.getUploadUrl(principal, args, sessionId); + const renewed = await current(); + expect(renewed).toEqual({ ...expired, expiresAt: expect.any(Date) }); + expect(renewed.expiresAt.getTime()).toBeGreaterThan(Date.now()); + const repaired = await put(repair.url); + expect(repaired.status).toBe(200); + expect(repaired.body).toMatchObject({ attachment: { id: attachmentId }, publication: { + source: "authenticated_upload", evidenceRef: `deliverable:${attachmentId}` } }); + expect(await transfers.getUploadUrl(principal, args, sessionId)).toMatchObject({ status: "completed", + attachment: { id: attachmentId }, publication: repaired.body.publication }); + expect(await current()).toEqual(renewed); + expect(await db.select().from(tables.issueAttachments).where(eq(tables.issueAttachments.issueId, f.issueId))).toHaveLength(1); + await loseReceipt(); + const beforeDenied = await current(); + ownerAvailable = false; + await expect(transfers.getUploadUrl(principal, args, sessionId)).rejects.toThrow("work session unavailable"); + expect(await current()).toEqual(beforeDenied); + ownerAvailable = true; + ownerRunId = randomUUID(); + await expect(transfers.getUploadUrl(principal, args, sessionId)).rejects.toThrow(); + expect(await current()).toEqual(beforeDenied); + ownerRunId = f.runId; + await db.update(tables.mcpAttachmentUploads).set({ cleanedAt: new Date(), cleanupVerifiedAt: new Date() }) + .where(eq(tables.mcpAttachmentUploads.id, uploadId)); + const contradictory = await current(); + await expect(transfers.getUploadUrl(principal, args, sessionId)).rejects.toThrow(); + expect(await current()).toEqual(contradictory); + expect((await put(repair.url)).status).toBe(403); + await db.update(tables.mcpAttachmentUploads).set({ cleanedAt: null, cleanupVerifiedAt: null }) + .where(eq(tables.mcpAttachmentUploads.id, uploadId)); + // Deletion owns metadata/cleanup settlement. Overlap it with the locked + // repair to prove renewal cannot revive a pointer or recreate its bytes. + pauseRepair = true; + const repairing = transfers.getUploadUrl(principal, args, sessionId); + await locked; + const deleting = p.api(board, "DELETE", `/attachments/${attachmentId}`); + let overlappingRepair: Awaited; + try { + await vi.waitFor(async () => expect(await p.storage.headObject(f.companyId, renewed.objectKey)).toEqual({ exists: false }), + { timeout: 5000, interval: 20 }); + } finally { + releaseRepair(); + [overlappingRepair] = await Promise.all([repairing, deleting]); + } + const deleted = await current(); + expect(deleted).toMatchObject({ attachmentId, cleanedAt: expect.any(Date), cleanupVerifiedAt: expect.any(Date) }); + await expect(transfers.getUploadUrl(principal, args, sessionId)).rejects.toThrow(); + expect(await current()).toEqual(deleted); + expect((await put(repair.url)).status).toBe(403); + expect((await put(overlappingRepair.url)).status).toBe(403); + expect(await p.storage.headObject(f.companyId, deleted.objectKey)).toEqual({ exists: false }); + } finally { await p.cleanup(); } + }, 60_000); + + it.each(["delete_failed", "object_survived", "head_failed"] as const)("isolates orphan cleanup failure from another company's actual file transfers: %s", async failure => { + const { createPublicMcpTransfers } = await import("../services/public-mcp/file-transfers.js"); + const { default: express } = await import("express"); + const { default: request } = await import("supertest"); + const f = await seed(), p = await connectedMcp(f), other = await seed(), q = await connectedMcp(other); + const originalDelete = p.storage.deleteObject, originalHead = p.storage.headObject, originalGet = p.storage.getObject; + try { + const principal = await p.oauth.authenticate(p.token), unrelated = await q.oauth.authenticate(q.token); + const transfers = createPublicMcpTransfers(db, p.oauth, p.api, p.storage); + const app = express(); app.use(transfers.router); + const bytes = Buffer.from("authorized bytes survive unrelated cleanup failure"); + const args = (owner: typeof f) => ({ companyId: owner.companyId, taskId: owner.issueId, requestId: randomUUID(), + filename: "result.txt", contentType: "text/plain", byteSize: bytes.length, sha256: createHash("sha256").update(bytes).digest("hex") }); + const put = async (link: { url?: unknown }) => { + const url = new URL(String(link.url)); + return request(app).put(url.pathname + url.search).set("Content-Type", "text/plain").send(bytes) + .timeout({ response: 5000, deadline: 10_000 }); + }; + const badLink = await transfers.getUploadUrl(principal, args(f)); + const badResult = await put(badLink); expect(badResult.status).toBe(200); + const goodLink = await transfers.getUploadUrl(unrelated, args(other)); + const goodResult = await put(goodLink); expect(goodResult.status).toBe(200); + const [bad] = await db.select().from(tables.mcpAttachmentUploads).where(eq(tables.mcpAttachmentUploads.id, String(badLink.uploadId))); + const abandonedLink = await transfers.getUploadUrl(unrelated, args(other)); + const [abandoned] = await db.select().from(tables.mcpAttachmentUploads).where(eq(tables.mcpAttachmentUploads.id, String(abandonedLink.uploadId))); + await p.storage.putFile({ companyId: other.companyId, namespace: "mcp-transfers", objectKey: abandoned.objectKey, + originalFilename: "result.txt", contentType: "text/plain", body: bytes }); + p.storage.deleteObject = async (companyId, objectKey) => { + if (companyId === f.companyId && objectKey === bad.objectKey) { + if (failure === "delete_failed") throw new Error("controlled old object DELETE failure"); + if (failure === "object_survived") return; + } + return originalDelete(companyId, objectKey); + }; + p.storage.headObject = async (companyId, objectKey) => { + if (companyId === f.companyId && objectKey === bad.objectKey && failure === "head_failed") { + throw new Error("controlled old object HEAD failure"); + } + return originalHead(companyId, objectKey); + }; + expect(await p.api(principal, "DELETE", `/attachments/${badResult.body.attachment.id}`)).toEqual({ ok: true }); + await db.update(tables.mcpAttachmentUploads).set({ expiresAt: new Date(0) }) + .where(sql`${tables.mcpAttachmentUploads.id} IN (${bad.id}, ${abandoned.id})`); + const [uncertain] = await db.select().from(tables.mcpAttachmentUploads).where(eq(tables.mcpAttachmentUploads.id, bad.id)); + expect(uncertain).toMatchObject({ attachmentId: badResult.body.attachment.id, cleanedAt: null, cleanupVerifiedAt: null }); + // The old owner fails in this sweep, but the unrelated link and another + // object's verified cleanup must commit independently of that failure. + const download = await transfers.getDownloadUrl(unrelated, goodResult.body.attachment.id); + const downloadUrl = new URL(download.url); + const response = await request(app).get(downloadUrl.pathname + downloadUrl.search).timeout({ response: 5000, deadline: 10_000 }); + expect(response.status).toBe(200); expect(response.body).toEqual(bytes); + expect((await db.select().from(tables.mcpAttachmentUploads).where(eq(tables.mcpAttachmentUploads.id, bad.id)))[0]).toEqual(uncertain); + expect((await db.select().from(tables.mcpAttachmentUploads).where(eq(tables.mcpAttachmentUploads.id, abandoned.id)))[0]) + .toMatchObject({ cleanedAt: expect.any(Date), cleanupVerifiedAt: expect.any(Date) }); + expect(await originalHead(other.companyId, abandoned.objectKey)).toEqual({ exists: false }); + const next = await transfers.getUploadUrl(unrelated, args(other)); + expect((await put(next)).status).toBe(200); + expect(await blocked("mcp_attachment_uploads")).toBe(true); + await expect(transfers.getDownloadUrl(principal, goodResult.body.attachment.id)).rejects.toThrow(); + await expect(transfers.getUploadUrl(principal, args(other))).rejects.toThrow(); + p.storage.deleteObject = originalDelete; p.storage.headObject = originalHead; + await transfers.getDownloadUrl(unrelated, goodResult.body.attachment.id); + expect((await db.select().from(tables.mcpAttachmentUploads).where(eq(tables.mcpAttachmentUploads.id, bad.id)))[0]) + .toMatchObject({ attachmentId: uncertain.attachmentId, cleanedAt: expect.any(Date), cleanupVerifiedAt: expect.any(Date) }); + expect(await originalHead(f.companyId, bad.objectKey)).toEqual({ exists: false }); + expect(await blocked("mcp_attachment_uploads")).toBe(false); + // Only opportunistic cleanup errors are isolated. The caller's own + // transfer storage failure still reaches the existing route error path. + p.storage.getObject = async () => { throw new Error("controlled requested object GET failure"); }; + expect((await request(app).get(downloadUrl.pathname + downloadUrl.search) + .timeout({ response: 5000, deadline: 10_000 })).status).toBe(503); + } finally { + p.storage.deleteObject = originalDelete; p.storage.headObject = originalHead; p.storage.getObject = originalGet; + await p.cleanup(); await q.cleanup(); + } + }, 60_000); + + async function abandonedMcpBacklog() { + const { createPublicMcpTransfers } = await import("../services/public-mcp/file-transfers.js"); + const { default: express } = await import("express"); + const { default: request } = await import("supertest"); + const f = await seed(), p = await connectedMcp(f), other = await seed(), q = await connectedMcp(other); + try { + const principal = await p.oauth.authenticate(p.token), unrelated = await q.oauth.authenticate(q.token); + const hooks: { afterRead?: () => void } = {}; + const api: typeof p.api = async (...args) => { const result = await p.api(...args); hooks.afterRead?.(); return result; }; + const transfers = createPublicMcpTransfers(db, p.oauth, api, p.storage); + const app = express(); app.use(transfers.router); + const bytes = Buffer.from("real abandoned upload bytes"); + const args = (owner: typeof f) => ({ companyId: owner.companyId, taskId: owner.issueId, requestId: randomUUID(), + filename: "retained.txt", contentType: "text/plain", byteSize: bytes.length, + sha256: createHash("sha256").update(bytes).digest("hex") }); + const anchorArgs = args(other), link = await transfers.getUploadUrl(unrelated, anchorArgs); + if (!("url" in link)) throw new Error("Expected a new anchor upload link"); + const url = new URL(String(link.url)); + const uploaded = await request(app).put(url.pathname + url.search).set("Content-Type", "text/plain").send(bytes) + .timeout({ response: 5000, deadline: 10_000 }); + expect(uploaded.status).toBe(200); + for (let index = 0; index < 22; index++) { + const allocated = await transfers.getUploadUrl(principal, args(f)); + if (!("uploadId" in allocated)) throw new Error("Expected a new abandoned upload identity"); + const [upload] = await db.select().from(tables.mcpAttachmentUploads) + .where(eq(tables.mcpAttachmentUploads.id, String(allocated.uploadId))); + await p.storage.putFile({ companyId: f.companyId, namespace: "mcp-transfers", objectKey: upload.objectKey, + originalFilename: upload.originalFilename, contentType: upload.contentType, body: bytes }); + } + await db.update(tables.mcpAttachmentUploads).set({ expiresAt: new Date(0) }) + .where(eq(tables.mcpAttachmentUploads.companyId, f.companyId)); + const read = () => db.select().from(tables.mcpAttachmentUploads) + .where(eq(tables.mcpAttachmentUploads.companyId, f.companyId)).orderBy(tables.mcpAttachmentUploads.id); + const uploads = await read(); expect(uploads).toHaveLength(22); + const trigger = (kind: "upload" | "download", owner = transfers) => kind === "upload" + ? owner.getUploadUrl(unrelated, anchorArgs) + : owner.getDownloadUrl(unrelated, uploaded.body.attachment.id); + return { f, p, q, unrelated, api, hooks, transfers, createPublicMcpTransfers, trigger, uploads, read, + cleanup: async () => { await p.cleanup(); await q.cleanup(); } }; + } catch (error) { await p.cleanup(); await q.cleanup(); throw error; } + } + + it.each(["delete_failed", "object_survived", "head_failed"] as const)("advances bounded MCP cleanup through failed rows and retries after wrap: %s", async failure => { + const b = await abandonedMcpBacklog(); + const originalDelete = b.p.storage.deleteObject, originalHead = b.p.storage.headObject; + const good = b.uploads.at(-1)!, failed = b.uploads.slice(0, -1); + const failedKeys = new Set(failed.map(upload => upload.objectKey)); + const idsByKey = new Map(b.uploads.map(upload => [upload.objectKey, upload.id])); + const attempted: string[] = []; + try { + b.p.storage.deleteObject = async (companyId, objectKey) => { + attempted.push(idsByKey.get(objectKey)!); + if (failedKeys.has(objectKey)) { + if (failure === "delete_failed") throw new Error("controlled backlog DELETE failure"); + if (failure === "object_survived") return; + } + return originalDelete(companyId, objectKey); + }; + b.p.storage.headObject = async (companyId, objectKey) => { + if (failedKeys.has(objectKey) && failure === "head_failed") throw new Error("controlled backlog HEAD failure"); + return originalHead(companyId, objectKey); + }; + for (const kind of ["upload", "download"] as const) { + const before = attempted.length; + await b.trigger(kind); + expect(attempted.length - before).toBeLessThanOrEqual(20); + } + expect(new Set(attempted)).toEqual(new Set(b.uploads.map(upload => upload.id))); + const after = await b.read(); + expect(after.find(upload => upload.id === good.id)).toMatchObject({ cleanedAt: expect.any(Date), cleanupVerifiedAt: expect.any(Date) }); + expect(after.filter(upload => upload.id !== good.id)).toEqual(failed); + expect(await originalHead(b.f.companyId, good.objectKey)).toEqual({ exists: false }); + expect(await blocked("mcp_attachment_uploads")).toBe(true); + // Restore the same storage owner. Future real calls must wrap and retry + // every failed identity; cursor movement alone cannot complete a row. + b.p.storage.deleteObject = async (companyId, objectKey) => { attempted.push(idsByKey.get(objectKey)!); return originalDelete(companyId, objectKey); }; + b.p.storage.headObject = originalHead; + for (const kind of ["download", "upload"] as const) { + const before = attempted.length; + await b.trigger(kind); + expect(attempted.length - before).toBeLessThanOrEqual(20); + } + for (const upload of await b.read()) { + expect(upload).toMatchObject({ cleanedAt: expect.any(Date), cleanupVerifiedAt: expect.any(Date) }); + expect(await originalHead(b.f.companyId, upload.objectKey)).toEqual({ exists: false }); + } + expect(await blocked("mcp_attachment_uploads")).toBe(false); + } finally { b.p.storage.deleteObject = originalDelete; b.p.storage.headObject = originalHead; await b.cleanup(); } + }, 60_000); + + it("coalesces concurrent MCP cleanup and revisits a locked row after wrap", async () => { + const b = await abandonedMcpBacklog(), lockedUpload = b.uploads[0]!; + const originalDelete = b.p.storage.deleteObject, originalHead = b.p.storage.headObject; + let releaseLock!: () => void, enteredLock!: () => void, releaseStorage!: () => void, enteredStorage!: () => void; + const locked = new Promise(resolve => { enteredLock = resolve; }); + const unlock = new Promise(resolve => { releaseLock = resolve; }); + const storageEntered = new Promise(resolve => { enteredStorage = resolve; }); + const continueStorage = new Promise(resolve => { releaseStorage = resolve; }); + let first: Promise | undefined, second: Promise | undefined; + const lock = db.transaction(async tx => { + await tx.select().from(tables.mcpAttachmentUploads).where(eq(tables.mcpAttachmentUploads.id, lockedUpload.id)).for("update"); + enteredLock(); await unlock; + }); + const attempted: string[] = []; + try { + await locked; + b.p.storage.deleteObject = async (companyId, objectKey) => { + attempted.push(objectKey); + if (attempted.length === 1) { enteredStorage(); await continueStorage; } + return originalDelete(companyId, objectKey); + }; + first = b.trigger("download"); await storageEntered; + let secondAuthorized!: () => void; + const authorized = new Promise(resolve => { secondAuthorized = resolve; }); + b.hooks.afterRead = secondAuthorized; + second = b.trigger("upload"); await authorized; + // Authority has returned. Drain its promise continuations before + // releasing the real storage operation; this is not a wall-clock wait. + await new Promise(resolve => setImmediate(resolve)); + releaseStorage(); await Promise.all([first, second]); + expect(attempted).toHaveLength(20); + const pending = (await b.read()).filter(upload => !upload.cleanupVerifiedAt); + expect(pending).toHaveLength(2); + expect(pending).toContainEqual(lockedUpload); + expect(await originalHead(b.f.companyId, lockedUpload.objectKey)).toMatchObject({ exists: true }); + expect(await blocked("mcp_attachment_uploads")).toBe(true); + releaseLock(); await lock; + await b.trigger("download"); await b.trigger("upload"); + expect((await b.read()).every(upload => upload.cleanupVerifiedAt !== null)).toBe(true); + expect(await originalHead(b.f.companyId, lockedUpload.objectKey)).toEqual({ exists: false }); + expect(await blocked("mcp_attachment_uploads")).toBe(false); + } finally { + releaseStorage(); releaseLock(); await Promise.allSettled([lock, first, second]); + b.p.storage.deleteObject = originalDelete; await b.cleanup(); + } + }, 60_000); + + it("retries the same MCP cleanup batch after its real transaction rolls back", async () => { + const b = await abandonedMcpBacklog(); + const originalDelete = b.p.storage.deleteObject; + let rejectCommit = true; + const transaction: typeof db.transaction = (callback, config) => db.transaction(async tx => { + const result = await callback(tx); + if (rejectCommit) { rejectCommit = false; throw new Error("controlled cleanup transaction rollback"); } + return result; + }, config); + const faultDb = new Proxy(db, { get(target, property, receiver) { + if (property === "transaction") return transaction; + const value = Reflect.get(target, property, receiver); + return typeof value === "function" ? value.bind(target) : value; + } }); + const transfers = b.createPublicMcpTransfers(faultDb, b.p.oauth, b.api, b.p.storage); + const attempted: string[] = []; + try { + b.p.storage.deleteObject = async (companyId, objectKey) => { attempted.push(objectKey); return originalDelete(companyId, objectKey); }; + await expect(b.trigger("download", transfers)).rejects.toThrow("controlled cleanup transaction rollback"); + expect(attempted).toHaveLength(20); + expect(await b.read()).toEqual(b.uploads); + expect(await blocked("mcp_attachment_uploads")).toBe(true); + const abortedBatch = attempted.slice(); + await b.trigger("upload", transfers); + expect(attempted.slice(abortedBatch.length)).toEqual(abortedBatch); + expect((await b.read()).filter(upload => upload.cleanupVerifiedAt !== null)).toHaveLength(20); + expect(await blocked("mcp_attachment_uploads")).toBe(true); + await b.trigger("download", transfers); + expect((await b.read()).every(upload => upload.cleanupVerifiedAt !== null)).toBe(true); + expect(await blocked("mcp_attachment_uploads")).toBe(false); + } finally { b.p.storage.deleteObject = originalDelete; await b.cleanup(); } + }, 60_000); + + it.each(["delete_failed", "object_survived", "head_failed"] as const)("retains MCP cleanup uncertainty after actual attachment removal: %s", async failure => { + const { createPublicMcpTransfers } = await import("../services/public-mcp/file-transfers.js"); + const { default: express } = await import("express"); + const { default: request } = await import("supertest"); + const f = await seed(), p = await connectedMcp(f); + try { + const principal = await p.oauth.authenticate(p.token), bytes = Buffer.from("retained uncertain attachment"); + const transfers = createPublicMcpTransfers(db, p.oauth, p.api, p.storage); + const upload = await transfers.getUploadUrl(principal, { companyId: f.companyId, taskId: f.issueId, requestId: randomUUID(), + filename: "uncertain.txt", contentType: "text/plain", byteSize: bytes.length, sha256: createHash("sha256").update(bytes).digest("hex") }); + const app = express(); app.use(transfers.router); + const url = new URL(String(upload.url)); + const result = await request(app).put(url.pathname + url.search).set("Content-Type", "text/plain").send(bytes) + .timeout({ response: 5000, deadline: 10_000 }); + expect(result.status).toBe(200); + const [before] = await db.select().from(tables.mcpAttachmentUploads).where(eq(tables.mcpAttachmentUploads.id, String(upload.uploadId))); + const originalDelete = p.storage.deleteObject, originalHead = p.storage.headObject; + if (failure === "delete_failed") p.storage.deleteObject = async () => { throw new Error("controlled storage removal failure"); }; + if (failure === "object_survived") p.storage.deleteObject = async () => {}; + if (failure === "head_failed") p.storage.headObject = async () => { throw new Error("controlled storage readback failure"); }; + try { expect(await p.api(principal, "DELETE", `/attachments/${result.body.attachment.id}`)).toEqual({ ok: true }); } + finally { p.storage.deleteObject = originalDelete; p.storage.headObject = originalHead; } + expect(await db.select().from(tables.issueAttachments).where(eq(tables.issueAttachments.id, result.body.attachment.id))).toHaveLength(0); + expect((await db.select().from(tables.mcpAttachmentUploads).where(eq(tables.mcpAttachmentUploads.id, before.id)))[0]) + .toMatchObject({ attachmentId: before.attachmentId, cleanedAt: null, cleanupVerifiedAt: null }); + expect((await p.storage.headObject(f.companyId, before.objectKey)).exists).toBe(failure !== "head_failed"); + expect(await blocked("mcp_attachment_uploads")).toBe(true); + // The existing bounded opportunistic owner can later retire this orphan + // only by a real successful delete and readback, preserving its identity. + await db.update(tables.mcpAttachmentUploads).set({ expiresAt: new Date(0) }).where(eq(tables.mcpAttachmentUploads.id, before.id)); + const pending = await transfers.getUploadUrl(principal, { companyId: f.companyId, taskId: f.issueId, requestId: randomUUID(), + filename: "new.txt", contentType: "text/plain", byteSize: 1, sha256: createHash("sha256").update("x").digest("hex") }); + expect((await db.select().from(tables.mcpAttachmentUploads).where(eq(tables.mcpAttachmentUploads.id, before.id)))[0]) + .toMatchObject({ attachmentId: before.attachmentId, cleanupVerifiedAt: expect.any(Date) }); + expect(await p.storage.headObject(f.companyId, before.objectKey)).toEqual({ exists: false }); + expect(await blocked("mcp_attachment_uploads")).toBe(true); // newly allocated upload remains pending + expect(pending.uploadId).toBeTruthy(); + } finally { await p.cleanup(); } + }, 60_000); + + it.each([false, true])("preserves verified MCP cleanup after ordinary issue deletion, storageFailure=%s", async storageFailure => { + const { createPublicMcpTransfers } = await import("../services/public-mcp/file-transfers.js"); + const { default: express } = await import("express"); + const { default: request } = await import("supertest"); + const f = await seed(), p = await connectedMcp(f); + try { + const principal = await p.oauth.authenticate(p.token), bytes = Buffer.from("removed with parent"); + const transfers = createPublicMcpTransfers(db, p.oauth, p.api, p.storage); + const upload = await transfers.getUploadUrl(principal, { companyId: f.companyId, taskId: f.issueId, requestId: randomUUID(), + filename: "parent.txt", contentType: "text/plain", byteSize: bytes.length, sha256: createHash("sha256").update(bytes).digest("hex") }); + const app = express(); app.use(transfers.router); + const url = new URL(String(upload.url)); + const result = await request(app).put(url.pathname + url.search).set("Content-Type", "text/plain").send(bytes) + .timeout({ response: 5000, deadline: 10_000 }); + expect(result.status).toBe(200); + const originalDelete = p.storage.deleteObject; + if (storageFailure) p.storage.deleteObject = async () => { throw new Error("controlled parent storage failure"); }; + try { await p.api(principal, "DELETE", `/issues/${f.issueId}`); } + finally { p.storage.deleteObject = originalDelete; } + expect(await db.select().from(tables.issues).where(eq(tables.issues.id, f.issueId))).toHaveLength(0); + const [removed] = await db.select().from(tables.mcpAttachmentUploads).where(eq(tables.mcpAttachmentUploads.id, String(upload.uploadId))); + expect(removed.attachmentId).toBe(result.body.attachment.id); + expect(removed.cleanupVerifiedAt !== null).toBe(!storageFailure); + expect((await p.storage.headObject(f.companyId, removed.objectKey)).exists).toBe(storageFailure); + expect(await blocked("mcp_attachment_uploads")).toBe(storageFailure); + } finally { await p.cleanup(); } + }, 60_000); + + it("retains the actual rejected tool review and only its acknowledged delivery", async () => { + const { commitToolActionReview } = await import("../services/tool-action-review.js"); + const { toolActionDeliveryService } = await import("../services/tool-action-delivery.js"); + const f = await seed(), requestId = randomUUID(), invocationId = randomUUID(), interactionId = randomUUID(); + await db.update(tables.issues).set({ status: "in_progress" }).where(eq(tables.issues.id, f.issueId)); + await db.insert(tables.toolInvocations).values({ id: invocationId, companyId: f.companyId, issueId: f.issueId, runId: f.runId, + actorType: "agent", actorId: f.agentId, agentId: f.agentId, toolName: "fixture:write", status: "awaiting_approval" }); + await db.insert(tables.issueThreadInteractions).values({ id: interactionId, companyId: f.companyId, issueId: f.issueId, + kind: "request_confirmation", payload: { version: 1, prompt: "Approve fixture action?", toolAction: { actionRequestId: requestId, invocationId } } }); + await db.insert(tables.toolActionRequests).values({ id: requestId, companyId: f.companyId, issueId: f.issueId, + invocationId, interactionId, requestedByAgentId: f.agentId, canonicalArgumentsHash: "a".repeat(64), canonicalArgumentsSummary: { summary: "fixture" } }); + expect(await blocked("tool_action_requests")).toBe(true); + await commitToolActionReview(db, { companyId: f.companyId, actionRequestId: requestId, decision: "rejected", actor: { userId: "local-board" } }); + expect(await blocked("tool_action_requests")).toBe(false); + expect(await blocked("tool_action_deliveries")).toBe(true); + await db.update(tables.issues).set({ status: "done" }).where(eq(tables.issues.id, f.issueId)); + await toolActionDeliveryService(db, { wakeup: async () => { throw new Error("Closed issue must retire its receipt"); } }).deliver(requestId); + expect(await blocked("tool_action_deliveries")).toBe(false); + await db.update(tables.toolInvocations).set({ completedAt: null }).where(eq(tables.toolInvocations.id, invocationId)); + expect(await blocked("tool_action_requests")).toBe(true); + expect(await blocked("tool_action_deliveries")).toBe(true); + }); + + it("accepts MCP completion receipts and cleanup while preserving unknown writes and pending uploads", async () => { + const f = await seed(), p = await mcp(f); + const [receipt] = await db.insert(tables.mcpMutationReceipts).values({ ...p, companyId: f.companyId, + operation: "paperclip_create_task", requestId: randomUUID(), argumentsHash: "a", status: "completed", result: { id: f.issueId } }).returning(); + const [upload] = await db.insert(tables.mcpAttachmentUploads).values({ ...p, companyId: f.companyId, taskId: f.issueId, + requestId: randomUUID(), argumentsHash: "a", originalFilename: "history.txt", contentType: "text/plain", byteSize: 1, + sha256: "a".repeat(64), objectKey: "history/abandoned", storageProvider: "local_disk", expiresAt: now, cleanedAt: now, cleanupVerifiedAt: now }).returning(); + expect(await blocked("mcp_mutation_receipts")).toBe(false); + expect(await blocked("mcp_attachment_uploads")).toBe(false); + expect(await read()).toEqual({ version: 1, backgroundWork: "none" }); + await db.update(tables.mcpMutationReceipts).set({ status: "unknown" }).where(eq(tables.mcpMutationReceipts.id, receipt.id)); + expect(await blocked("mcp_mutation_receipts")).toBe(true); + await db.update(tables.mcpMutationReceipts).set({ status: "completed", result: null }).where(eq(tables.mcpMutationReceipts.id, receipt.id)); + expect(await blocked("mcp_mutation_receipts")).toBe(true); + await db.update(tables.mcpAttachmentUploads).set({ cleanedAt: null }).where(eq(tables.mcpAttachmentUploads.id, upload.id)); + expect(await blocked("mcp_attachment_uploads")).toBe(true); + }); + + it("retains the real MCP mutation receipt and still blocks the actual task it creates", async () => { + const { createPublicMcpExecutor } = await import("../services/public-mcp/capabilities.js"); + const f = await seed(), p = await connectedMcp(f); + try { + // Pausing is the product's ordinary admission control. No provider turn + // is run merely to exercise a mutation receipt or task creation. + await db.update(tables.agents).set({ status: "paused" }).where(eq(tables.agents.id, f.agentId)); + const execute = createPublicMcpExecutor(db, p.oauth, p.api); + const input = { companyId: f.companyId, title: "Actual MCP producer", description: "Retained receipt", + requestId: randomUUID(), assigneeAgentId: f.agentId }; + const result = await execute(p.token, "paperclip_create_task", input); + expect(result).toHaveProperty("task.status", "todo"); + expect(await blocked("mcp_mutation_receipts")).toBe(false); + expect(await execute(p.token, "paperclip_create_task", input)).toEqual(result); + expect(await db.select().from(tables.mcpMutationReceipts)).toHaveLength(1); + expect(await read()).toEqual({ version: 1, backgroundWork: "present" }); + const receipt = (await db.select().from(tables.mcpMutationReceipts))[0]; + await db.update(tables.mcpMutationReceipts).set({ status: "reserved" }).where(eq(tables.mcpMutationReceipts.id, receipt.id)); + expect(await blocked("mcp_mutation_receipts")).toBe(true); + } finally { await p.cleanup(); } + }, 60_000); + + it.each([200, 410])("uses actual MCP subscription and callback delivery receipts for HTTP %s", async status => { + const { createPublicMcpEvents } = await import("../services/public-mcp/events.js"); + const { logActivity } = await import("../services/activity-log.js"); + const f = await seed(), p = await connectedMcp(f); + try { + let received = 0; + const events = createPublicMcpEvents(db, p.oauth, p.api, { fetch: async (_url, init) => { + const body = JSON.parse(String(init.body)); + if (body.type === "verification") return Response.json({ challenge: body.challenge }); + received += 1; + return new Response(null, { status }); + } }); + const principal = await p.oauth.authenticate(p.token); + await events.subscribe(principal, { name: "paperclip.task.status_changed", arguments: { companyId: f.companyId, taskId: f.issueId }, + delivery: { mode: "webhook", url: "https://receiver.example/events", secret: `whsec_${randomBytes(32).toString("base64")}` } }); + await logActivity(db, { companyId: f.companyId, actorType: "system", actorId: "history", + action: "issue.updated", entityType: "issue", entityId: f.issueId, details: { status: "done" } }); + await events.tick(); + expect(received).toBeGreaterThan(0); + expect(await blocked("mcp_event_deliveries")).toBe(false); + expect((await db.select().from(tables.mcpEventDeliveries)).every(row => row.finishedAt !== null)).toBe(true); + expect(await blocked("mcp_event_subscriptions")).toBe(status !== 410); + expect(await blocked("mcp_oauth_grants")).toBe(true); + } finally { await p.cleanup(); } + }, 60_000); + + it.each(["delivered", "filtered", "inactive", "delivery_exhausted", "http_400", "http_503"])("retires finished MCP event outcome %s only with its source", async outcome => { + const f = await seed(), p = await mcp(f); + const subscriptionId = randomUUID(); + await db.insert(tables.mcpEventSubscriptions).values({ id: subscriptionId, companyId: f.companyId, grantId: p.grantId, + name: "paperclip.task.updated", taskId: f.issueId, arguments: { taskId: f.issueId }, deliveryMaterial: {}, verifiedAt: now, + expiresAt: now, stoppedAt: now }); + const [activity] = await db.insert(tables.activityLog).values({ companyId: f.companyId, actorType: "system", actorId: "history", + action: "issue.updated", entityType: "issue", entityId: f.issueId }).returning(); + const [delivery] = await db.insert(tables.mcpEventDeliveries).values({ subscriptionId, activityId: activity.id, + event: {}, finishedAt: now, outcome, attempts: 6 }).returning(); + expect(await blocked("mcp_event_subscriptions")).toBe(false); + expect(await blocked("mcp_event_deliveries")).toBe(false); + await db.update(tables.mcpEventDeliveries).set({ finishedAt: null }).where(eq(tables.mcpEventDeliveries.id, delivery.id)); + expect(await blocked("mcp_event_deliveries")).toBe(true); + }); + + it.each([false, true])("requires actual Speko configure completion after tool receipts, defaultsFailure=%s", async defaultsFailure => { + const { chatChannelService } = await import("../services/chat-channels.js"); + const { instanceSettingsService } = await import("../services/instance-settings.js"); + const f = await seed(), p = await mcp(f), oldKey = process.env.PAPERCLIP_SECRETS_MASTER_KEY; + process.env.PAPERCLIP_SECRETS_MASTER_KEY = "a".repeat(64); + await db.insert(tables.companyMemberships).values({ companyId: f.companyId, principalType: "user", principalId: p.userId, + membershipRole: "owner", status: "active" }); + await instanceSettingsService(db).updateExperimental({ enableChatConnectors: true }); + await db.insert(tables.principalPermissionGrants).values({ companyId: f.companyId, principalType: "user", principalId: p.userId, + permissionKey: "agents:configure" }); + const tools: Record[] = [], hooks: Record[] = []; + let failDefaults = defaultsFailure; + const chat = chatChannelService(db, { heartbeat: { wakeup: async () => { throw new Error("No call expected"); } }, + publicBaseUrl: "https://history.example", webhookPublicBaseUrl: "https://history.example", fetch: async (input, init) => { + const pathname = new URL(String(input)).pathname, method = init?.method ?? "GET"; + if (pathname === "/v1/agents/agent_test") { + if (method === "PATCH" && failDefaults) return new Response("Defaults failed", { status: 503 }); + return Response.json({ id: "agent_test", organizationId: "org_test", name: "History voice" }); + } + if (pathname.startsWith("/v1/agents/agent_test/tools")) { + if (method === "GET") return Response.json(tools); + const tool = { ...JSON.parse(String(init?.body)), id: pathname.split("/").length > 5 ? pathname.split("/").at(-1)! : `tool_${tools.length}` }; + const index = tools.findIndex(old => old.id === tool.id); if (index < 0) tools.push(tool); else tools[index] = tool; + return Response.json({ id: tool.id }); + } + if (pathname.startsWith("/v1/webhooks")) { + if (method === "GET") return Response.json({ data: hooks }); + const hook = { ...JSON.parse(String(init?.body)), id: "hook_test", filterTags: {} }; hooks.splice(0, hooks.length, hook); + return Response.json({ id: hook.id }); + } + throw new Error(`Unexpected Speko provider boundary ${method} ${pathname}`); + } }); + try { + const endpoint = await chat.create(f.companyId, { provider: "speko", assignedAgentId: f.agentId }, p.userId); + let configureAttempts = 0; + const configure = () => chat.configure(endpoint.id, { action: configureAttempts++ === 0 ? "configure" : "reconnect", + credentials: { apiKey: "fixture-api-key", agentId: "agent_test", signingSecret: `whsec_${Buffer.alloc(32, 7).toString("base64")}` } }, p.userId); + if (defaultsFailure) { + await expect(configure()).rejects.toMatchObject({ status: 422 }); + const partial = (await db.select().from(tables.chatActions)).filter(a => a.endpointId === endpoint.id); + expect(partial).toHaveLength(4); + expect(partial.every(a => a.status === "completed")).toBe(true); + for (const receipt of partial) expect(await blockedActionKind(receipt.kind)).toBe(true); + failDefaults = false; + } + await configure(); + const receipts = (await db.select().from(tables.chatActions)).filter(a => a.endpointId === endpoint.id); + expect(receipts).toHaveLength(4); + expect(await blockedActionKind("speko_tool_setup")).toBe(false); + expect(await blockedActionKind("speko_lifecycle_setup")).toBe(false); + const [activity] = await db.select().from(tables.activityLog).where(eq(tables.activityLog.action, + defaultsFailure ? "chat_endpoint.reconnected" : "chat_endpoint.credentials_verified")); + expect(activity.details).toMatchObject({ endpointId: endpoint.id, provider: "speko" }); + } finally { + await chat.shutdown(); + if (oldKey === undefined) delete process.env.PAPERCLIP_SECRETS_MASTER_KEY; else process.env.PAPERCLIP_SECRETS_MASTER_KEY = oldKey; + } + }, 60_000); + + it("retires actual GitHub omission navigation only after its exact native response publication settles", async () => { + const { createGitHubPublicationPolicy } = await import("../services/chat-channels/github/publication-policy.js"); + const { ensureNativeCompletionContract } = await import("../services/native-runtime/completion-contracts.js"); + const { chatChannelService } = await import("../services/chat-channels.js"); + const f = await slackFixture(); + const [endpoint] = await db.update(tables.chatEndpoints).set({ provider: "github" }).where(eq(tables.chatEndpoints.id, f.endpoint.id)).returning(); + const contract = await ensureNativeCompletionContract({ db, companyId: f.companyId, issue: { id: f.issueId, title: "Omitted source", description: null }, actorId: "history" }); + const [comment] = await db.insert(tables.issueComments).values({ companyId: f.companyId, issueId: f.issueId, + authorAgentId: f.agentId, createdByRunId: f.runId, body: "Source unavailable; see the task" }).returning(); + await db.update(tables.heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: f.issueId, completionContractId: contract.row.id, + resultJson: { presentationDecision: { commentId: comment.id } } }).where(eq(tables.heartbeatRuns.id, f.runId)); + const [result] = await db.insert(tables.nativeRunResults).values({ companyId: f.companyId, issueId: f.issueId, runId: f.runId, + completionContractId: contract.row.id, schemaStatus: "accepted", serverFingerprint: randomUUID(), canonicalSha256: "a".repeat(64), resultJson: {} }).returning(); + await db.insert(tables.nativeRunFinalizations).values({ companyId: f.companyId, issueId: f.issueId, runId: f.runId, + phase: "committed", resultId: result.id }); + const [publication] = await db.insert(tables.chatPublications).values({ companyId: f.companyId, endpointId: endpoint.id, + conversationId: f.conversation.id, issueId: f.issueId, commentId: comment.id, idempotencyKey: randomUUID(), payload: { text: comment.body } }).returning(); + const unused = async (): Promise => { throw new Error("Unexpected GitHub publication boundary"); }; + const policy = createGitHubPublicationPolicy(db, { isExplicitOperatorPublication: () => false, getTaskBaseUrl: () => "https://history.example", + // Exact current native-input authority is the named consumer boundary; + // the actual policy validates canonical owner rows and writes the marker. + failedChatRetrySource: async () => ({ endpointId: endpoint.id, conversationId: f.conversation.id, + sources: [{ omissionReasons: { download_unavailable: 1 } }] }) as never, + isExternalActionAuthorizationChange: () => false, FailedChatRetryPublicationReadError: class extends Error {}, endpointRecord: unused, + runIdFromMilestonePublication: () => null, fetchImpl: unused, stageTerminalReceiptReactionRemovals: unused, + runtimeContextForRecord: unused, scheduleMessageProcessing: unused, processReceiptReaction: unused }); + const text = await db.transaction(tx => policy.githubCurrentOmissionPreparedText(tx, publication, endpoint)); + expect(text).toContain("Open this Paperclip task"); + const [receipt] = await db.select().from(tables.chatActions).where(eq(tables.chatActions.kind, "github_omission_navigation")); + expect(receipt).toMatchObject({ status: "processed", result: null }); + expect(await blockedActionKind("github_omission_navigation")).toBe(true); + await db.update(tables.chatPublications).set({ payload: { ...publication.payload, text: String(text) }, state: "delivery_unknown" }) + .where(eq(tables.chatPublications.id, publication.id)); + const chat = chatChannelService(db, { heartbeat: { wakeup: unused }, fetch: unused }); + try { await chat.resolvePublication(endpoint.id, publication.id, "mark_delivered", f.userId); } + finally { await chat.shutdown(); } + expect(await blockedActionKind("github_omission_navigation")).toBe(false); + await db.update(tables.chatActions).set({ payload: { ...receipt.payload, resultId: randomUUID() } }).where(eq(tables.chatActions.id, receipt.id)); + expect(await blockedActionKind("github_omission_navigation")).toBe(true); + }); + + it("retires an actual Discord status-command receipt while preserving registration deadline work", async () => { + const { reconcileStoredDiscordCommandRegistration } = await import("../services/chat-discord-command-registration-store.js"); + const { createDiscordCommands } = await import("../services/chat-channels/discord/commands.js"); + const { chatCredentialMutationLease } = await import("../services/chat-credential-mutation-lease.js"); + const f = await slackFixture(), applicationId = String(1_400_000_000_000_000_000n + BigInt(Date.now()) + BigInt(randomBytes(2).readUInt16BE())), guildId = "1457808928258658549", channelId = "333333333333333333"; + const [endpoint] = await db.update(tables.chatEndpoints).set({ provider: "discord", botExternalId: applicationId, + providerAccountId: guildId, capabilities: { slashCommands: true } }).where(eq(tables.chatEndpoints.id, f.endpoint.id)).returning(); + const [principal] = await db.update(tables.chatExternalPrincipals).set({ provider: "discord", externalId: "444444444444444444", + providerAccountId: guildId }).where(eq(tables.chatExternalPrincipals.id, f.principal.id)).returning(); + const context = { generation: 1, credentialFingerprint: "c".repeat(64), localEpoch: 0, version: "fixture", discordGatewayOwned: true }; + const scope = { companyId: f.companyId, endpointId: endpoint.id, applicationId, guildId }; + const remote: Record[] = []; + const registered = await reconcileStoredDiscordCommandRegistration(db, { scope, runtimeFence: { generation: context.generation, credentialFingerprint: context.credentialFingerprint }, botToken: "fixture-token", + authorize: async tx => { + expect((await tx.select().from(tables.chatEndpoints).where(eq(tables.chatEndpoints.id, endpoint.id)).for("update"))[0].status).toBe("active"); + }, fetch: async (_url, init) => { + if ((init?.method ?? "GET") === "GET") return Response.json(remote); + const command = { ...JSON.parse(String(init?.body)), id: "888888888888888888", application_id: applicationId, version: "999999999999999999" }; + remote.splice(0, remote.length, command); return Response.json(command); + } }); + expect(registered.kind).toBe("registered"); + const leaseKey = "discord_gateway", token = randomUUID(); + await db.insert(tables.chatEndpointLeases).values({ companyId: f.companyId, endpointId: endpoint.id, leaseKey, token, + expiresAt: new Date(Date.now() + 60_000) }); + const record = { endpoint, assignedAgentName: "Idle agent", connectionName: "Discord", applicationId: randomUUID(), credentialSecretRefs: [] }; + const unused = async (): Promise => { throw new Error("No Discord transport/control boundary expected"); }; + const service = createDiscordCommands(db, { endpointRecord: async () => record, runtimeContextForRecord: () => context, + runtimeGeneration: () => 1, credentialFingerprint: () => context.credentialFingerprint, resolveCredentialRefs: unused, + fetchImpl: unused, withCredentialMutationLease: chatCredentialMutationLease(db), runtimeCallbackRecord: async () => record, + ensurePrincipal: async () => ({ principal, userId: f.userId }), runtimeCallbackEndpoint: async () => endpoint, + nonDirectDestinationAllowed: () => true, lockCurrentPrincipalAuthorization: async () => ({ allowed: true, userId: f.userId }), + getDiscordGatewayOwnership: () => ({ leaseKey, token }), discordGatewayRuntimeIsCurrent: () => true, + stageAuthorizedTaskControlPublication: unused, scheduleMessageProcessing: unused, processPendingPublications: unused }); + const thread = `discord:${guildId}:${channelId}`; + const result = await service.handleDiscordNativeCommand({ endpointId: endpoint.id, provider: "discord", transport: "discord_gateway", + event: { command: "/paperclip status", text: "", channelId: thread, channel: { id: thread }, + user: { userId: principal.externalId, isBot: false, isMe: false }, raw: { id: "777777777777777777", application_id: applicationId, + type: 2, version: 1, channel_id: channelId, channel: { id: channelId, type: 0 }, guild_id: guildId, context: 0, + authorizing_integration_owners: { "0": guildId }, data: { id: "888888888888888888", type: 1, name: "paperclip", + options: [{ type: 1, name: "status" }] }, user: { id: principal.externalId, username: "operator", bot: false } } } } as never, context); + expect(result).toMatchObject({ kind: "accepted" }); + expect(await blockedActionKind("discord_native_command")).toBe(false); + expect(await blockedActionKind("discord_command_registration")).toBe(true); + const [receipt] = await db.select().from(tables.chatActions).where(eq(tables.chatActions.kind, "discord_native_command")); + await db.update(tables.chatActions).set({ result: { ...receipt.result, kind: "unknown" } }).where(eq(tables.chatActions.id, receipt.id)); + expect(await blockedActionKind("discord_native_command")).toBe(true); + }); + + it.each(["live_photo_video", "heif_jpeg_preview"] as const)("retains actual Photon %s provenance after source-bound storage registration", async relationKind => { + const { createInboundAttachments } = await import("../services/chat-channels/inbound-attachments.js"); + const { createInboundWakeups } = await import("../services/chat-channels/inbound/wakeups.js"); + const { downloadPhotonAttachment } = await import("../services/photon/attachments.js"); + const { issueService } = await import("../services/issues.js"); + const { createStorageService } = await import("../storage/service.js"); + const { createLocalDiskStorageProvider } = await import("../storage/local-disk-provider.js"); + const f = await slackFixture(), directory = await mkdtemp(path.join(tmpdir(), "history-photon-provenance-")); + const [endpoint] = await db.update(tables.chatEndpoints).set({ provider: "imessage-photon" }).where(eq(tables.chatEndpoints.id, f.endpoint.id)).returning(); + const [comment] = await db.insert(tables.issueComments).values({ companyId: f.companyId, issueId: f.issueId, body: "Source-bound photo" }).returning(); + const providerEventId = randomUUID(); + const [delivery] = await db.insert(tables.chatDeliveries).values({ companyId: f.companyId, endpointId: endpoint.id, + conversationId: f.conversation.id, principalId: f.principal.id, providerEventId, deduplicationKey: randomUUID(), + eventKind: "message", state: "processing", attempts: 1, normalizedEvent: { providerEventId, kind: "message" } }).returning(); + const unused = async (): Promise => { throw new Error("Unexpected Photon admission boundary"); }; + const wakes = createInboundWakeups(db, { authorizeInboundWakeup: unused, authorizeInboundWorkStart: unused, + attachmentOmissionDetail: () => null, isExternalActionAuthorizationChange: () => false, redactError: String, + fetchImpl: unused, heartbeat: { wakeup: unused } }); + try { + await db.transaction(tx => wakes.stageInboundWakeup(tx, { endpoint, deliveryId: delivery.id, conversation: f.conversation, + commentId: comment.id, principalId: f.principal.id, actorUserId: null })); + const storage = createStorageService(createLocalDiskStorageProvider(directory)); + const bytes = relationKind === "heif_jpeg_preview" + ? await readFile(new URL("./photon/fixtures/synthetic.heic", import.meta.url)) + : Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/6S8AAAAASUVORK5CYII=", "base64"); + const type = relationKind === "heif_jpeg_preview" ? "image/heic" : "image/png"; + let body = bytes, closed = false; + if (relationKind === "live_photo_video") { + const companion = Buffer.from("Actual bounded companion bytes"), locator = { kind: "photon_attachment" as const, + lineId: "line", chatGuid: "chat", messageGuid: "message", attachmentGuid: "attachment" }; + const info = { guid: "attachment", totalBytes: bytes.length, mimeType: type, fileName: "photo.png", isSticker: false, isHidden: false }; + const stream = { async *[Symbol.asyncIterator]() { + yield { type: "header", info, companionInfo: { kind: "live-photo-video", mimeType: "video/mp4", totalBytes: companion.length, fileName: "motion.mp4" } }; + yield { type: "primaryChunk", data: bytes }; yield { type: "companionChunk", data: companion }; + }, close: async () => { closed = true; } }; + body = await downloadPhotonAttachment({ messages: { get: async () => ({ guid: "message", chatGuids: ["chat"], content: { attachments: [info] } }) }, + attachments: { downloadStream: () => stream } } as never, "line", locator); + expect(closed).toBe(true); + } + const service = createInboundAttachments(db, { storage, authorizeInboundWakeup: async () => ({ delivery, conversation: f.conversation }), + lifecycleRuntimeFence: () => ({ generation: 1, credentialFingerprint: "c".repeat(64) }), sanitizeFilename: value => value ?? null, + issuesSvc: issueService(db), isExternalActionAuthorizationChange: () => false, redactError: String }); + const result = await service.ingestAttachments({ endpoint, endpointRuntime: {} as never, deliveryId: delivery.id, + issueId: f.issueId, issueCommentId: comment.id, actorUserId: null, + attachments: [{ type: "image", name: relationKind === "heif_jpeg_preview" ? "photo.heic" : "photo.png", mimeType: type, + size: bytes.length, fetchData: async () => body }] }); + expect(result.omissionReasons).toEqual({}); + expect(result.storedIds).toHaveLength(2); + const receipt = (await db.select().from(tables.chatActions)).find(a => a.kind === (relationKind === "heif_jpeg_preview" ? "attachment_derivative" : "attachment_companion"))!; + expect(receipt).toMatchObject({ status: "processed", payload: { kind: relationKind } }); + expect(await blockedActionKind(receipt.kind)).toBe(false); + expect(await blockedActionKind("inbound_wakeup")).toBe(true); + const assets = await db.select().from(tables.assets); + for (const asset of assets) expect((await storage.headObject(f.companyId, asset.objectKey)).exists).toBe(true); + const { chatChannelService } = await import("../services/chat-channels.js"); + const chat = chatChannelService(db, { heartbeat: { wakeup: unused }, fetch: unused }); + try { + await expect(chat.publishBoardMessage(endpoint.id, f.conversation.id, "Reuse an already bound attachment", randomUUID(), f.userId, + [result.storedIds[0]])).rejects.toMatchObject({ status: 409 }); + expect(await blockedActionKind("board_send_rejected")).toBe(false); + } finally { await chat.shutdown(); } + await db.update(tables.chatActions).set({ payload: { ...receipt.payload, derivativeSha256: "0".repeat(64) } }).where(eq(tables.chatActions.id, receipt.id)); + expect(await blockedActionKind(receipt.kind)).toBe(true); + await db.update(tables.chatActions).set({ payload: receipt.payload }).where(eq(tables.chatActions.id, receipt.id)); + const [otherIssue] = await db.insert(tables.issues).values({ companyId: f.companyId, title: "Wrong lineage" }).returning(); + await db.update(tables.issueAttachments).set({ issueId: otherIssue.id }).where(eq(tables.issueAttachments.id, String(receipt.payload.derivativeAttachmentId))); + expect(await blockedActionKind(receipt.kind)).toBe(true); + await db.update(tables.issueAttachments).set({ issueId: f.issueId }).where(eq(tables.issueAttachments.id, String(receipt.payload.derivativeAttachmentId))); + const { issueRoutes } = await import("../routes/issues.js"); + const { createMcpApiDispatch } = await import("../services/public-mcp/capabilities.js"); + const { Router } = await import("express"); + const router = Router(); router.use(issueRoutes(db, storage)); + const api = createMcpApiDispatch(router), actor = { type: "board", userId: f.userId, companyIds: [f.companyId], source: "session", + memberships: [{ companyId: f.companyId, membershipRole: "owner", status: "active" }] }; + for (const id of result.storedIds) expect(await api({ actor } as never, "DELETE", `/attachments/${id}`)).toEqual({ ok: true }); + expect(await db.select().from(tables.issueAttachments)).toHaveLength(0); + for (const asset of assets) expect((await storage.headObject(f.companyId, asset.objectKey)).exists).toBe(false); + expect(await blockedActionKind(receipt.kind)).toBe(false); + expect(await blockedActionKind("inbound_wakeup")).toBe(true); + } finally { wakes.unregisterDeferredWakeOwner(); await rm(directory, { recursive: true, force: true }); } + }, 60_000); + + it("retires actual Photon interaction, retry and response-notice history with their canonical owners", async () => { + const { issueThreadInteractionService } = await import("../services/issue-thread-interactions.js"); + const { recordPhotonPublicationRetry } = await import("../services/chat-channels/imessage-photon/publication-retry.js"); + const { stagePhotonResponseNotice } = await import("../services/chat-channels/imessage-photon/response-notices.js"); + const { chatChannelService } = await import("../services/chat-channels.js"); + const f = await seed(), c = await connection(f); + await db.update(tables.issues).set({ status: "in_progress" }).where(eq(tables.issues.id, f.issueId)); + const [endpoint] = await db.insert(tables.chatEndpoints).values({ companyId: f.companyId, connectionId: c.connectionId, + assignedAgentId: f.agentId, provider: "imessage-photon", publicId: randomUUID(), status: "active", publicationMode: "automatic" }).returning(); + const [conversation] = await db.insert(tables.chatConversations).values({ companyId: f.companyId, endpointId: endpoint.id, + issueId: f.issueId, externalConversationId: "photon-personal", externalThreadId: "photon-personal", externalLabel: "Personal", state: "active" }).returning(); + const interactions = issueThreadInteractionService(db); + const interaction = await interactions.create({ id: f.issueId, companyId: f.companyId }, { kind: "ask_user_questions", + continuationPolicy: "wake_assignee", payload: { version: 1, title: "Choose", submitLabel: "Continue", questions: [{ + id: "choice", prompt: "Choose", selectionMode: "single", required: true, allowOther: false, + options: [{ id: "yes", label: "Yes" }, { id: "no", label: "No" }] }] } }, { agentId: f.agentId }); + const [token] = await db.select().from(tables.chatActions); + expect(token).toMatchObject({ kind: "photon_interaction", status: "issued", conversationId: conversation.id }); + expect(await blockedActionKind("photon_interaction")).toBe(true); + await interactions.answerQuestions({ id: f.issueId, companyId: f.companyId }, interaction.id, + { answers: [{ questionId: "choice", optionIds: ["yes"] }] }, { userId: "local-board" }); + expect(await blockedActionKind("photon_interaction")).toBe(false); + const [previousSource] = await db.select().from(tables.chatPublications); + const [source] = await db.update(tables.chatPublications).set({ state: "delivery_unknown" }) + .where(eq(tables.chatPublications.id, previousSource.id)).returning(); + await db.transaction(async tx => { + await recordPhotonPublicationRetry(tx, endpoint.id, source, "local-board"); + await stagePhotonResponseNotice(tx, { publication: source, reference: String(token.payload.reference), questionIndex: 0, + key: `photon-response:${token.payload.reference}:1`, text: "Answer saved" }); + }); + expect(await blockedActionKind("photon_publication_retry")).toBe(true); + expect(await blockedActionKind("photon_response_notice")).toBe(true); + const unused = async (): Promise => { throw new Error("No Photon network expected"); }; + const service = chatChannelService(db, { heartbeat: { wakeup: unused }, fetch: unused }); + try { + for (const publication of await db.select().from(tables.chatPublications)) { + await db.update(tables.chatPublications).set({ state: "delivery_unknown" }).where(eq(tables.chatPublications.id, publication.id)); + await service.resolvePublication(endpoint.id, publication.id, "mark_delivered", "local-board"); + } + expect(await blocked("chat_actions")).toBe(false); + await db.update(tables.chatActions).set({ payload: { ...token.payload, interactionId: randomUUID() } }).where(eq(tables.chatActions.id, token.id)); + expect(await blockedActionKind("photon_interaction")).toBe(true); + } finally { await service.shutdown(); } + }); + + it("retains actual Slack connect discovery while its separate notice remains pending", async () => { + const { createSlackCommands, slackCommandForAgent } = await import("../services/chat-channels/slack/commands.js"); + const { createProviderEffects } = await import("../services/chat-channels/provider-effects.js"); + const { chatCredentialMutationLease } = await import("../services/chat-credential-mutation-lease.js"); + const f = await slackFixture(), unused = async (): Promise => { throw new Error("Unexpected discovery boundary"); }; + const effects = createProviderEffects(db, { endpointRecord: unused, runtimeCallbackEndpoint: async () => f.endpoint, + withCredentialMutationLease: chatCredentialMutationLease(db), runtimeFor: unused, invalidateRuntime: unused, + lockCurrentPrincipalAuthorization: unused, redactError: String, MAX_ERROR_TEXT: 500, + nonDirectDestinationAllowed: () => false, lockTelegramCallbackNoticeAuthority: unused }); + const command = slackCommandForAgent("Idle agent", f.endpoint.publicId), scheduled: string[] = []; + const service = createSlackCommands(db, { runtime: { get: () => undefined }, runtimeFor: unused, handleSdkMessage: unused, + stageProviderEffect: effects.stageProviderEffect, scheduleProviderEffect: id => { scheduled.push(id); }, + ensurePrincipal: async () => ({ principal: f.principal, userId: null, linkedDenied: false }), + runtimeCallbackEndpoint: async () => f.endpoint, getPublicBaseUrl: () => "https://history.example", + canonicalProviderResourceId: unused, nonDirectDestinationAllowed: () => false, sponsorAllowsGuest: unused, + latestSlackDmControlThreadId: unused, scheduleMessageProcessing: unused, processSlackTaskStart: unused }); + await service.handleSlackSlashCommand({ endpointId: f.endpoint.id, provider: "slack", transport: "slack_socket", + event: { command, text: "connect", triggerId: randomUUID(), channel: { id: "DCHANNEL" }, + user: { userId: "UREQUESTER", isBot: false, isMe: false }, raw: {} } } as never, + { generation: 1, credentialFingerprint: "c".repeat(64), localEpoch: 0, version: "fixture" }, + { endpoint: f.endpoint, assignedAgentName: "Idle agent", connectionName: "Slack", applicationId: randomUUID(), credentialSecretRefs: [] }, command, "connect"); + expect(scheduled).toHaveLength(1); + expect((await db.select().from(tables.chatIdentityLinks))[0]).toMatchObject({ status: "pending", confirmationTokenHash: expect.any(String) }); + expect(await blockedActionKind("slack_connect")).toBe(false); + expect(await blockedActionKind("provider_effect")).toBe(true); + expect(await blocked("chat_actions")).toBe(true); + }); + + it("retains the actual Telegram stop-subscription ownership receipt", async () => { + const { createTelegramMaintenance } = await import("../services/chat-channels/telegram/maintenance.js"); + const { chatCredentialMutationLease } = await import("../services/chat-credential-mutation-lease.js"); + const f = await seed(), c = await connection(f); + await db.update(tables.toolConnections).set({ enabled: true, status: "active" }).where(eq(tables.toolConnections.id, c.connectionId)); + const [endpoint] = await db.insert(tables.chatEndpoints).values({ companyId: f.companyId, connectionId: c.connectionId, + assignedAgentId: f.agentId, provider: "telegram", publicId: randomUUID(), status: "active", botExternalId: "1234567" }).returning(); + const unused = async (): Promise => { throw new Error("Unexpected Telegram provider boundary"); }; + const context = { generation: 3, credentialFingerprint: "c".repeat(64) }, withCredentialMutationLease = chatCredentialMutationLease(db); + const service = createTelegramMaintenance(db, { endpointRecord: unused, runtimeContextForRecord: () => context, + withCredentialMutationLease, resolveCredentials: unused, clearCredentials: unused, + runtimeCallbackEndpoint: async () => endpoint, getWebhookPublicBaseUrl: () => "https://history.example", + fetchImpl: unused, runtimeGeneration: () => 3, credentialFingerprint: () => context.credentialFingerprint, + redactError: String, credentialCheckTimeoutMs: 1000 }); + await withCredentialMutationLease(endpoint, lease => service.recordTelegramStopSubscription(endpoint, context, lease)); + expect(await service.hasTelegramStopSubscription(endpoint, context)).toBe(true); + expect(await blocked("chat_actions")).toBe(false); + const [receipt] = await db.select().from(tables.chatActions); + await db.update(tables.chatActions).set({ payload: { ...receipt.payload, webhookUrlSha256: "unknown" } }).where(eq(tables.chatActions.id, receipt.id)); + expect(await blocked("chat_actions")).toBe(true); + }); + + it("retains the actual Slack private-source boundary and governed completed write", async () => { + const { recordSlackReadBoundary } = await import("../services/connectors/slack-access.js"); + const { registerSlackTaskAuthority } = await import("../services/connectors/slack-authority.js"); + const { executeSlackWrite } = await import("../services/connectors/slack-writes.js"); + const f = await slackFixture(); + await recordSlackReadBoundary(db, f.authority, { id: "CPRIVATE", is_private: true }); + expect(await blocked("chat_actions")).toBe(false); + const invocationId = randomUUID(); + await db.insert(tables.toolInvocations).values({ id: invocationId, companyId: f.companyId, agentId: f.agentId, + issueId: f.issueId, runId: f.runId, connectionId: f.connectionId, actorType: "agent", actorId: f.agentId, + toolName: "slack_post_message", status: "executing" }); + const unregister = registerSlackTaskAuthority(db, async () => f.authority); + const methods: string[] = []; + try { + await executeSlackWrite(db, { companyId: f.companyId, agentId: f.agentId, issueId: f.issueId, runId: f.runId }, + "slack_post_message", { channel: "DCHANNEL", text: "Confirmed message", idempotencyKey: randomUUID() }, invocationId, + async input => { + const method = new URL(String(input)).pathname.split("/").at(-1)!; methods.push(method); + if (method === "conversations.info") return Response.json({ ok: true, channel: { id: "DCHANNEL", is_im: true, user: "UREQUESTER" } }); + if (method === "users.info") return Response.json({ ok: true, user: { id: "UREQUESTER", team_id: "TWORKSPACE", is_bot: false } }); + if (method === "chat.postMessage") return Response.json({ ok: true, channel: "DCHANNEL", ts: "123.456" }); + throw new Error(`Unexpected Slack boundary ${method}`); + }); + expect(methods).toContain("chat.postMessage"); + expect((await db.select().from(tables.chatActions)).find(a => a.kind === "slack_tool_write")) + .toMatchObject({ status: "processed", result: { channel: "DCHANNEL", ts: "123.456" } }); + expect(await blocked("chat_actions")).toBe(false); + await db.update(tables.chatActions).set({ status: "uncertain" }).where(eq(tables.chatActions.kind, "slack_tool_write")); + expect(await blocked("chat_actions")).toBe(true); + } finally { unregister(); } + }); + + it("retires actual Slack invitation, test-message and mirror receipts without hiding their publication", async () => { + const { slackTestMessageService } = await import("../services/connectors/slack/test-message.js"); + const { mirrorSlackBoardComment } = await import("../services/slack-board-messages.js"); + const { chatChannelService } = await import("../services/chat-channels.js"); + const { chatCredentialMutationLease } = await import("../services/chat-credential-mutation-lease.js"); + const { instanceSettingsService } = await import("../services/instance-settings.js"); + const f = await slackFixture(); + await instanceSettingsService(db).updateExperimental({ enableChatConnectors: true }); + const unused = async (): Promise => { throw new Error("Unexpected provider boundary"); }; + const credentialLease = chatCredentialMutationLease(db); + const send = slackTestMessageService(db, { withLock: (id, task) => credentialLease(f.endpoint, task), + runtimeBotToken: async () => "test-token", linkInstaller: async () => {}, ensureAccount: async () => f.principal.id, + createLinkIntent: async () => ({ confirmationUrl: "https://example.test/link", expiresAt: new Date(Date.now() + 600_000).toISOString() }), + fetch: async input => { + const method = new URL(String(input)).pathname.split("/").at(-1); + if (method === "users.info") return Response.json({ ok: true, user: { id: "UREQUESTER", team_id: "TWORKSPACE", is_bot: false } }); + if (method === "chat.postMessage") return Response.json({ ok: true, channel: "DCHANNEL", ts: "123.456" }); + throw new Error(`Unexpected setup boundary ${method}`); + } }); + expect(await send(f.endpoint.id, { userId: f.userId }, { requestId: randomUUID(), slackUserId: "UREQUESTER" })) + .toMatchObject({ status: "connection_pending" }); + expect(await blocked("chat_actions")).toBe(false); + await db.insert(tables.chatIdentityLinks).values({ companyId: f.companyId, endpointId: f.endpoint.id, + principalId: f.principal.id, paperclipUserId: f.userId, status: "linked", confirmedAt: now }); + expect(await send(f.endpoint.id, { userId: f.userId }, { requestId: randomUUID(), slackUserId: "UREQUESTER" })) + .toMatchObject({ status: "sent" }); + expect(await blocked("chat_actions")).toBe(false); + const [comment] = await db.insert(tables.issueComments).values({ companyId: f.companyId, issueId: f.issueId, + authorType: "user", authorUserId: f.userId, body: "Mirror this board comment" }).returning(); + await db.transaction(tx => mirrorSlackBoardComment(tx, comment)); + const [publication] = await db.select().from(tables.chatPublications); + expect(publication).toMatchObject({ state: "pending", commentId: comment.id }); + expect(await blocked("chat_actions")).toBe(true); + const service = chatChannelService(db, { heartbeat: { wakeup: unused }, fetch: unused }); + try { + await db.update(tables.chatPublications).set({ state: "delivery_unknown" }).where(eq(tables.chatPublications.id, publication.id)); + await service.resolvePublication(f.endpoint.id, publication.id, "mark_delivered", f.userId); + expect(await blocked("chat_actions")).toBe(false); + await db.update(tables.chatActions).set({ status: "uncertain" }).where(eq(tables.chatActions.kind, "slack_test_message")); + expect(await blocked("chat_actions")).toBe(true); + } finally { await service.shutdown(); } + }); + + it("retires the actual shared GitHub response target only after its request is final", async () => { + const { githubIngressFixture } = await import("./helpers/github-webhook-ingress.js"); + const { writeGitHubResponseComment } = await import("../services/chat-github-response-comments.js"); + const { stagePullResponse } = await import("../services/chat-channels/github/response-target.js"); + const { withGitHubPublicationLease } = await import("../services/chat-github-publication-lease.js"); + const f = await githubIngressFixture(db), providerEventId = randomUUID(); + const [issue] = await db.insert(tables.issues).values({ companyId: f.companyId, title: "GitHub reply", status: "done" }).returning(); + const [conversation] = await db.insert(tables.chatConversations).values({ companyId: f.companyId, endpointId: f.endpoint.id, + issueId: issue.id, externalConversationId: "repository:418", externalThreadId: "github:org/repo:7", externalLabel: "PR7" }).returning(); + const [delivery] = await db.insert(tables.chatDeliveries).values({ companyId: f.companyId, endpointId: f.endpoint.id, + conversationId: conversation.id, providerEventId, deduplicationKey: randomUUID(), eventKind: "action", normalizedEvent: { providerEventId, kind: "action" } }).returning(); + const source = { endpoint: f.endpoint, conversation, delivery, number: 7, replyId: null, repositoryId: "418" }; + await stagePullResponse(db, source); + expect(await blocked("chat_actions")).toBe(false); // initialized target owns no request yet + let posted: { id: number; html_url: string; body: string; user: { login: string | null } } | undefined; + const calls: string[] = []; + const api = { request: async (pathname: string, options?: { method?: string; body?: unknown }): Promise => { + calls.push(`${options?.method ?? "GET"} ${pathname}`); + if (!options?.method) return (pathname.includes("?per_page=") ? [] : posted) as T; + posted = { id: 123, html_url: "https://github.com/org/repo/pull/7#issuecomment-123", + body: (options.body as { body: string }).body, user: { login: f.endpoint.botUsername } }; + return posted as T; + } }; + const publish = (final: boolean) => withGitHubPublicationLease(db, { companyId: f.companyId, endpointId: f.endpoint.id, + repositoryId: "418", number: 7 }, async () => { throw new Error("No lease network expected"); }, lease => + writeGitHubResponseComment(db, source, api, lease, { body: final ? "Final result" : "Working", final }, async () => {})); + await publish(false); + expect(await blocked("chat_actions")).toBe(true); + await publish(true); + expect(calls).toContain("PATCH /issues/comments/123"); + expect(await blocked("chat_actions")).toBe(false); + const request = (await db.select().from(tables.chatActions)).find(a => a.kind === "github_response_comment")!; + expect(request).toMatchObject({ status: "processed", result: { final: true, id: "123" } }); + await db.update(tables.chatActions).set({ result: { ...request.result, final: false } }).where(eq(tables.chatActions.id, request.id)); + expect(await blocked("chat_actions")).toBe(true); + }); + + it("retires actual GitHub terminal review checks despite their retained retry hints", async () => { + const { githubIngressFixture } = await import("./helpers/github-webhook-ingress.js"); + const { githubReviewCheckService } = await import("../services/chat-github-checks.js"); + const { secretService } = await import("../services/secrets.js"); + const { defaultGitHubReviewPolicy } = await import("@paperclipai/shared"); + const oldKey = process.env.PAPERCLIP_SECRETS_MASTER_KEY; + process.env.PAPERCLIP_SECRETS_MASTER_KEY = "a".repeat(64); + try { + const f = await githubIngressFixture(db); + const privateKey = generateKeyPairSync("rsa", { modulusLength: 2048, + privateKeyEncoding: { type: "pkcs8", format: "pem" }, publicKeyEncoding: { type: "spki", format: "pem" } }).privateKey; + const refs = []; + for (const [key, value] of Object.entries({ appId: "12", installationId: "34", privateKey })) { + const secret = await secretService(db).create(f.companyId, { name: `GitHub ${key}`, key, provider: "local_encrypted", value }); + refs.push({ configPath: `credentials.${key}`, secretId: secret.id, versionSelector: "latest" as const }); + await secretService(db).createBinding({ companyId: f.companyId, secretId: secret.id, targetType: "tool_connection", + targetId: f.endpoint.connectionId, configPath: `credentials.${key}`, versionSelector: "latest" }); + } + await db.update(tables.toolConnections).set({ enabled: true, status: "active", credentialSecretRefs: refs }) + .where(eq(tables.toolConnections.id, f.endpoint.connectionId)); + await db.insert(tables.chatGitHubConfigurations).values({ companyId: f.companyId, endpointId: f.endpoint.id, updatedByUserId: "local-board", + configuration: { version: 1, toolsEnabled: true, responsibleUserId: "local-board", memberAccess: "all_linked", people: [], + repositories: {}, defaults: { ...defaultGitHubReviewPolicy(), ratingThreshold: 3 } } }); + await db.insert(tables.chatEndpointResources).values({ companyId: f.companyId, endpointId: f.endpoint.id, + providerResourceId: "org/repo", type: "repository", label: "org/repo", enabled: true, + availability: "available", metadata: { providerRepositoryId: "418" } }); + const event = { event: "opened" as const, deliveryId: randomUUID(), repositoryId: "418", repository: "org/repo", pullNumber: 7, + title: "Saved review", body: "", baseSha: "a".repeat(40), headSha: "b".repeat(40), baseBranch: "master", + author: { id: "42", login: "octocat", isBot: false }, sender: { id: "42", login: "octocat" }, draft: false, labels: [] }; + const calls: string[] = []; + let stale = false; + const service = githubReviewCheckService(db, async input => { + const pathname = new URL(String(input)).pathname; calls.push(pathname); + if (pathname === "/app/installations/34/access_tokens") return Response.json({ token: "test-token" }); + if (pathname === "/repos/org/repo/pulls/7") return Response.json({ head: { sha: stale ? "c".repeat(40) : event.headSha } }); + if (pathname.endsWith("/check-runs")) return Response.json(pathname.includes("/commits/") + ? { check_runs: [] } : { id: 123, html_url: "https://github.com/org/repo/runs/123" }); + throw new Error(`Unexpected GitHub boundary ${pathname}`); + }); + await service.enqueue(f.endpoint, event, false, "manual review required"); + const [pending] = await db.select().from(tables.chatActions).where(eq(tables.chatActions.kind, "github_review_check")); + expect(await blocked("chat_actions")).toBe(true); + await service.processPending(); + const [completed] = await db.select().from(tables.chatActions).where(eq(tables.chatActions.id, pending.id)); + expect(completed, JSON.stringify({ result: completed.result, calls })).toMatchObject({ status: "processed", result: { state: "manual_required", retryable: true } }); + expect(calls).toContain("/repos/org/repo/check-runs"); + expect(await blocked("chat_actions")).toBe(false); + stale = true; + await service.enqueue(f.endpoint, { ...event, deliveryId: randomUUID() }, false, "stale head"); + const next = (await db.select().from(tables.chatActions)).find(a => a.status === "received")!; + await service.processPending(); + expect((await db.select().from(tables.chatActions).where(eq(tables.chatActions.id, next.id)))[0]) + .toMatchObject({ status: "cancelled", result: { code: "stale_head" } }); + expect(await blocked("chat_actions")).toBe(false); + await service.enqueue(f.endpoint, { ...event, deliveryId: randomUUID() }, false, "connection removed"); + await db.update(tables.chatEndpointResources).set({ enabled: false }).where(eq(tables.chatEndpointResources.endpointId, f.endpoint.id)); + const last = (await db.select().from(tables.chatActions)).find(a => a.status === "received")!; + await service.processPending(); + expect((await db.select().from(tables.chatActions).where(eq(tables.chatActions.id, last.id)))[0]) + .toMatchObject({ status: "cancelled", result: { code: "connection_changed" } }); + expect(await blocked("chat_actions")).toBe(false); + await db.update(tables.chatActions).set({ result: { state: "queued", retryable: true } }).where(eq(tables.chatActions.id, completed.id)); + expect(await blocked("chat_actions")).toBe(true); + } finally { if (oldKey === undefined) delete process.env.PAPERCLIP_SECRETS_MASTER_KEY; else process.env.PAPERCLIP_SECRETS_MASTER_KEY = oldKey; } + }, 60_000); + + it("retires real question-form tokens only after their exact canonical answer commits", async () => { + const { issueThreadInteractionService } = await import("../services/issue-thread-interactions.js"); + const { claimChatQuestionFormSubmission, completeChatQuestionFormSubmission } = await import("../services/chat-question-forms.js"); + const f = await seed(), c = await connection(f); + await db.update(tables.issues).set({ status: "in_progress" }).where(eq(tables.issues.id, f.issueId)); + const [endpoint] = await db.insert(tables.chatEndpoints).values({ companyId: f.companyId, connectionId: c.connectionId, + assignedAgentId: f.agentId, publicId: randomUUID(), provider: "slack", status: "active", publicationMode: "automatic", + capabilities: { actions: true, modals: true } }).returning(); + const [conversation] = await db.insert(tables.chatConversations).values({ companyId: f.companyId, endpointId: endpoint.id, + issueId: f.issueId, externalConversationId: "question-channel", externalThreadId: "question-thread", externalLabel: "Question", state: "active" }).returning(); + const [principal] = await db.insert(tables.chatExternalPrincipals).values({ companyId: f.companyId, provider: "slack", + providerAccountId: "workspace", externalId: "answering-user" }).returning(); + const service = issueThreadInteractionService(db); + const interaction = await service.create({ id: f.issueId, companyId: f.companyId }, { + kind: "ask_user_questions", continuationPolicy: "wake_assignee", title: "Two questions", + payload: { version: 1, title: "Two questions", submitLabel: "Continue", questions: ["first", "second"].map(id => ({ + id, prompt: id, selectionMode: "single" as const, required: true, allowOther: false, + options: [{ id: "yes", label: "Yes" }, { id: "no", label: "No" }] })) }, + }, { agentId: f.agentId }); + const actions = await db.select().from(tables.chatActions); + const open = actions.find(a => a.kind === "question_form_open"), submit = actions.find(a => a.kind === "question_form_submit"); + expect(open).toMatchObject({ conversationId: conversation.id, status: "issued", result: null }); + expect(submit).toMatchObject({ status: "issued", result: null }); + expect(await blocked("chat_actions")).toBe(true); + await service.answerQuestions({ id: f.issueId, companyId: f.companyId }, interaction.id, + { answers: ["first", "second"].map(questionId => ({ questionId, optionIds: ["yes"] })) }, { userId: "local-board" }, { + afterResolveInTransaction: async (tx, resolved) => { + expect(await claimChatQuestionFormSubmission(tx, { actionRowId: submit!.id, principalId: principal.id })).toBe(true); + expect(await completeChatQuestionFormSubmission(tx, { actionRowId: submit!.id, interactionId: resolved.id })).toBe(true); + }, + }); + expect((await db.select().from(tables.issueThreadInteractions).where(eq(tables.issueThreadInteractions.id, interaction.id)))[0]) + .toMatchObject({ status: "answered", resolvedAt: expect.any(Date) }); + expect(await blocked("chat_actions")).toBe(false); + await db.update(tables.chatActions).set({ result: { code: "question_form_answered", interactionId: randomUUID() } }) + .where(eq(tables.chatActions.id, submit!.id)); + expect(await blocked("chat_actions")).toBe(true); + }); + + it("retires a staged inbound wake only after its actual delivery owner rejects it", async () => { + const { githubIngressFixture } = await import("./helpers/github-webhook-ingress.js"); + const { createInboundWakeups } = await import("../services/chat-channels/inbound/wakeups.js"); + const { chatChannelService } = await import("../services/chat-channels.js"); + const f = await githubIngressFixture(db); + const [issue] = await db.insert(tables.issues).values({ companyId: f.companyId, title: "Rejected input" }).returning(); + const [conversation] = await db.insert(tables.chatConversations).values({ companyId: f.companyId, endpointId: f.endpoint.id, + issueId: issue.id, externalConversationId: "rejected", externalThreadId: "rejected", externalLabel: "Rejected" }).returning(); + const [principal] = await db.insert(tables.chatExternalPrincipals).values({ companyId: f.companyId, provider: "github", + providerAccountId: "test", externalId: "42" }).returning(); + const [comment] = await db.insert(tables.issueComments).values({ companyId: f.companyId, issueId: issue.id, body: "Input" }).returning(); + const providerEventId = randomUUID(); + const [delivery] = await db.insert(tables.chatDeliveries).values({ companyId: f.companyId, endpointId: f.endpoint.id, + conversationId: conversation.id, providerEventId, deduplicationKey: randomUUID(), eventKind: "message", state: "received", + normalizedEvent: { providerEventId, kind: "message" } }).returning(); + const unused = async (): Promise => { throw new Error("No provider/admission boundary expected"); }; + const wakes = createInboundWakeups(db, { authorizeInboundWakeup: unused, authorizeInboundWorkStart: unused, + attachmentOmissionDetail: () => null, isExternalActionAuthorizationChange: () => false, redactError: String, + fetchImpl: unused, heartbeat: { wakeup: unused } }); + const service = chatChannelService(db, { heartbeat: { wakeup: unused }, fetch: unused }); + try { + await db.transaction(tx => wakes.stageInboundWakeup(tx, { endpoint: f.endpoint, deliveryId: delivery.id, + conversation, commentId: comment.id, principalId: principal.id, actorUserId: null })); + expect(await blocked("chat_actions")).toBe(true); + await wakes.settleRejectedInboundWakeups(delivery.id); + expect((await db.select().from(tables.chatActions))[0].status).toBe("preparing"); + await service.configure(f.endpoint.id, { action: "pause" }, "local-board"); + await wakes.settleRejectedInboundWakeups(delivery.id); + expect((await db.select().from(tables.chatActions))[0]).toMatchObject({ status: "failed", + result: { code: "inbound_wakeup_delivery_rejected" } }); + expect(await db.select().from(tables.agentWakeupRequests)).toHaveLength(0); + expect(await blocked("chat_actions")).toBe(false); + expect(await blocked("chat_deliveries")).toBe(false); + await db.update(tables.chatDeliveries).set({ nextAttemptAt: new Date(Date.now() + 60_000) }).where(eq(tables.chatDeliveries.id, delivery.id)); + expect(await blocked("chat_actions")).toBe(true); + } finally { wakes.unregisterDeferredWakeOwner(); await service.shutdown(); } + }); + + it("retires actual Teams consent and provider-effect receipts only after the transfer closes", async () => { + const { teamsFileTransferService } = await import("../services/chat-teams-file-transfers.js"); + const { projectTeamsFilePublication } = await import("../services/chat-teams-file-publication.js"); + const { installTeamsFileConsentHook, restoreTeamsFileConsentBinding } = await import("../services/chat-teams-file-consent.js"); + const oldKey = process.env.PAPERCLIP_SECRETS_MASTER_KEY; + process.env.PAPERCLIP_SECRETS_MASTER_KEY = "a".repeat(64); + try { + const f = await seed(), c = await connection(f), bytes = Buffer.from("Actual Teams transfer"); + const authority = { companyId: f.companyId, endpointId: randomUUID(), conversationId: randomUUID(), issueId: f.issueId, + publicationId: randomUUID(), commentId: randomUUID(), attachmentId: randomUUID(), principalId: randomUUID(), + authorizedUserId: "local-board", runtimeGeneration: 3, credentialFingerprint: "c".repeat(64), conversationGeneration: 2, + sourceDigest: "d".repeat(64), tenantId: randomUUID(), botAppId: randomUUID(), aadObjectId: randomUUID(), + providerConversationId: "a:exact-personal", providerUserId: "29:exact-recipient", + sha256: createHash("sha256").update(bytes).digest("hex"), byteSize: bytes.length, filename: "file.txt" }; + await db.insert(tables.chatEndpoints).values({ id: authority.endpointId, companyId: f.companyId, connectionId: c.connectionId, + assignedAgentId: f.agentId, publicId: randomUUID(), provider: "microsoft-teams", status: "active", + providerAccountId: authority.tenantId, botExternalId: authority.botAppId }); + await db.insert(tables.chatConversations).values({ id: authority.conversationId, companyId: f.companyId, + endpointId: authority.endpointId, issueId: f.issueId, externalConversationId: authority.providerConversationId, + externalThreadId: `teams:${Buffer.from(authority.providerConversationId).toString("base64url")}`, externalLabel: "Personal", + isDirectMessage: true, sessionGeneration: 2 }); + await db.insert(tables.chatExternalPrincipals).values({ id: authority.principalId, companyId: f.companyId, provider: "microsoft-teams", + providerAccountId: authority.tenantId, externalId: authority.aadObjectId }); + await db.insert(tables.issueComments).values({ id: authority.commentId, companyId: f.companyId, issueId: f.issueId, + authorUserId: "local-board", body: "Share this file" }); + const [asset] = await db.insert(tables.assets).values({ companyId: f.companyId, provider: "local_disk", objectKey: randomUUID(), + contentType: "text/plain", byteSize: bytes.length, sha256: authority.sha256, originalFilename: authority.filename }).returning(); + await db.insert(tables.issueAttachments).values({ id: authority.attachmentId, companyId: f.companyId, issueId: f.issueId, + assetId: asset.id, issueCommentId: authority.commentId }); + await db.insert(tables.chatPublications).values({ id: authority.publicationId, companyId: f.companyId, + endpointId: authority.endpointId, conversationId: authority.conversationId, issueId: f.issueId, commentId: authority.commentId, + idempotencyKey: randomUUID(), payload: { text: "Share file", attachmentIds: [authority.attachmentId] } }); + const service = teamsFileTransferService(db, { authorize: async () => ({ ...authority }), loadBytes: async () => bytes, + postConsent: async () => ({ id: "card-1" }), postFileInfo: async () => ({ id: "file-info-1" }), + uploadRequest: async () => Response.json({ id: "item-1", name: "file.txt", size: bytes.length }, { status: 201 }), + project: projectTeamsFilePublication }); + const created = await service.issue(authority, new Date(Date.now() + 60_000)); + await service.process(f.companyId, created.id); + const [pending] = await db.select().from(tables.chatTeamsFileTransfers); + expect(pending.phase).toBe("awaiting_consent"); + expect((await db.select().from(tables.chatActions)).map(a => a.kind)).toContain("teams_file_effect_intent"); + expect(await blocked("chat_actions")).toBe(true); + const binding = await restoreTeamsFileConsentBinding({ companyId: f.companyId, endpointId: authority.endpointId, + transferId: created.id, authorityDigest: pending.authorityDigest }, pending.privateState.binding as Record); + const handlers = new Map Promise<{ status: number }>>(); + installTeamsFileConsentHook({ on: (name, callback) => { handlers.set(name, callback); } }, { + companyId: f.companyId, endpointId: authority.endpointId, tenantId: authority.tenantId, botAppId: authority.botAppId, + onConsent: event => service.recordConsent(event) }); + expect((await handlers.get("file.consent.decline")!({ activity: { type: "invoke", name: "fileConsent/invoke", channelId: "msteams", + id: "decline-1", from: { id: authority.providerUserId, aadObjectId: authority.aadObjectId }, + recipient: { id: `28:${authority.botAppId}` }, conversation: { id: authority.providerConversationId, conversationType: "personal", tenantId: authority.tenantId }, + replyToId: "card-1", value: { action: "decline", context: { schema: binding.schema, token: binding.token, action: "decline" } } } })).status).toBe(200); + expect((await db.select().from(tables.chatTeamsFileTransfers))[0]).toMatchObject({ phase: "declined", attemptId: null, attemptExpiresAt: null }); + expect((await db.select().from(tables.chatActions)).map(a => a.kind)).toEqual(expect.arrayContaining(["teams_file_effect_intent", "teams_file_consent"])); + expect(await blocked("chat_actions")).toBe(false); + expect(await blocked("chat_publications")).toBe(false); + const [receipt] = await db.select().from(tables.chatActions).where(eq(tables.chatActions.kind, "teams_file_consent")); + await db.update(tables.chatActions).set({ payload: { ...receipt.payload, receiptDigest: "wrong" } }).where(eq(tables.chatActions.id, receipt.id)); + expect(await blocked("chat_actions")).toBe(true); + } finally { if (oldKey === undefined) delete process.env.PAPERCLIP_SECRETS_MASTER_KEY; else process.env.PAPERCLIP_SECRETS_MASTER_KEY = oldKey; } + }); + + it("measures both chat predicates against 10000 valid retained deliveries and 30000 completed actions", async () => { + const { githubIngressFixture } = await import("./helpers/github-webhook-ingress.js"); + const { chatChannelService } = await import("../services/chat-channels.js"); + const f = await githubIngressFixture(db, undefined, 256), providerEventId = randomUUID(); + const [delivery] = await db.insert(tables.chatDeliveries).values({ companyId: f.companyId, endpointId: f.endpoint.id, + providerEventId, deduplicationKey: randomUUID(), eventKind: "message", state: "received", + normalizedEvent: { providerEventId, kind: "message" } }).returning(); + const action = await f.insertAction({ deliveryId: delivery.id }); + await f.service.processGitHubWebhookIngress(action.id); + const unused = async (): Promise => { throw new Error("No external boundary expected"); }; + const service = chatChannelService(db, { heartbeat: { wakeup: unused }, fetch: unused }); + try { await service.configure(f.endpoint.id, { action: "pause" }, "local-board"); } + finally { await service.shutdown(); } + expect(await blocked("chat_deliveries")).toBe(false); + expect(await blocked("chat_actions")).toBe(false); + // Retention load clones a receipt produced above; it is measurement data, + // not a substitute for the producer regressions in the surrounding cases. + await db.execute(sql`INSERT INTO chat_deliveries (company_id, endpoint_id, provider_event_id, deduplication_key, event_kind, + state, normalized_event, processed_at) + SELECT ${f.companyId}::uuid, ${f.endpoint.id}::uuid, 'retained:' || n, 'retained:' || n, d.event_kind, d.state, + jsonb_set(d.normalized_event, '{providerEventId}', to_jsonb('retained:' || n)), d.processed_at + FROM chat_deliveries d CROSS JOIN generate_series(1, 9999) n WHERE d.id = ${delivery.id}::uuid`); + await db.execute(sql`INSERT INTO chat_actions (company_id, endpoint_id, delivery_id, kind, provider_action_id, payload, status, result) + SELECT a.company_id, a.endpoint_id, d.id, a.kind, 'retained:' || d.id || ':' || n, a.payload, a.status, a.result + FROM chat_actions a JOIN chat_deliveries d ON d.endpoint_id = a.endpoint_id CROSS JOIN generate_series(1, 3) n + WHERE a.id = ${action.id}::uuid AND d.id <> ${delivery.id}::uuid`); + await db.execute(sql`ANALYZE chat_deliveries`); + await db.execute(sql`ANALYZE chat_actions`); + const current = completedHistoryChecks.filter(c => ["chat_deliveries", "chat_actions"].includes(c.reason)); + const baselinePath = process.env.PAPERCLIP_HISTORY_QUERY_BASELINE_PATH; + const baseline: { reason: string; query: string }[] = baselinePath ? JSON.parse(await readFile(baselinePath, "utf8")) : []; + const measurements = []; + for (const [revision, checks] of [["published_aaf55692", baseline], ["repair", current]] as const) { + for (const check of checks) { + const started = performance.now(); + const plan = await db.transaction(async tx => { + await tx.execute(sql`SET LOCAL statement_timeout = '5s'`); + return tx.execute(sql`EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) SELECT EXISTS (${sql.raw(check.query)}) AS blocked`); + }); + measurements.push({ revision, reason: check.reason, wallMs: performance.now() - started, plan }); + } + } + const evidence = { deliveries: 10000, actions: 29998, measurements }; + if (process.env.PAPERCLIP_HISTORY_QUERY_PLANS_PATH) await writeFile(process.env.PAPERCLIP_HISTORY_QUERY_PLANS_PATH, + JSON.stringify(evidence, null, 2)); + console.info(`Retained chat history query plans ${JSON.stringify(evidence)}`); + expect(await blocked("chat_deliveries")).toBe(false); + expect(await blocked("chat_actions")).toBe(false); + }, 60_000); + + it("uses actual chat ingress, lifecycle application and operator publication receipts", async () => { + const { githubIngressFixture } = await import("./helpers/github-webhook-ingress.js"); + const { chatCredentialMutationLease } = await import("../services/chat-credential-mutation-lease.js"); + const { createChatResourceReconciliation } = await import("../services/chat-channels/resource-reconciliation.js"); + const { chatChannelService } = await import("../services/chat-channels.js"); + const { projectSafeChatPublication } = await import("../services/chat-publication-projection.js"); + const f = await githubIngressFixture(db), action = await f.insertAction(); + expect(await blocked("chat_actions")).toBe(true); + await f.service.processGitHubWebhookIngress(action.id); + expect(f.calls).toHaveLength(1); + expect(await blocked("chat_actions")).toBe(false); + const [issue] = await db.insert(tables.issues).values({ companyId: f.companyId, title: "Chat delivery", status: "done" }).returning(); + const [conversation] = await db.insert(tables.chatConversations).values({ companyId: f.companyId, endpointId: f.endpoint.id, + issueId: issue.id, externalConversationId: "fixture", externalThreadId: "fixture", externalLabel: "History", state: "completed" }).returning(); + const effect = { kind: "resource" as const, provider: "github" as const, providerEventId: randomUUID(), providerResourceId: "repository:fixture", + resourceType: "repository", label: "Fixture", availability: "removed" as const }; + const [delivery] = await db.insert(tables.chatDeliveries).values({ companyId: f.companyId, endpointId: f.endpoint.id, + providerEventId: `lifecycle:${effect.providerEventId}`, deduplicationKey: randomUUID(), eventKind: "uninstallation", state: "processing", attempts: 1, + normalizedEvent: { providerEventId: `lifecycle:${effect.providerEventId}`, kind: "uninstallation", lifecycle: effect } }).returning(); + expect(await blocked("chat_deliveries")).toBe(true); + const lease = chatCredentialMutationLease(db); + const resources = createChatResourceReconciliation(db, { endpointRecord: async () => ({ endpoint: f.endpoint }), + withCredentialMutationLease: lease, runtimeGeneration: () => 1, maxErrorText: 500 }); + await lease(f.endpoint, guard => resources.applyProviderAvailability({ currentEndpoint: f.endpoint, effect, + credentialLease: guard, staleResourceEffect: false, claimed: delivery, candidate: { attempts: 0 } })); + expect(await blocked("chat_deliveries")).toBe(false); + const firstAction = await f.insertAction({ deliveryId: delivery.id }); + expect(await blocked("chat_deliveries")).toBe(true); + await f.service.processGitHubWebhookIngress(firstAction.id); + expect(await blocked("chat_deliveries")).toBe(false); + const secondAction = await f.insertAction({ deliveryId: delivery.id }); + expect(await blocked("chat_deliveries")).toBe(true); // one completed sibling cannot hide another pending action + await f.service.processGitHubWebhookIngress(secondAction.id); + expect(await blocked("chat_deliveries")).toBe(false); + await db.update(tables.chatDeliveries).set({ eventKind: "message", + normalizedEvent: { providerEventId: delivery.providerEventId, kind: "message" } }).where(eq(tables.chatDeliveries.id, delivery.id)); + expect(await blocked("chat_deliveries")).toBe(true); // ordinary input requires its own consumed wake, not arbitrary completed actions + await db.update(tables.chatDeliveries).set({ eventKind: "uninstallation", normalizedEvent: {} }).where(eq(tables.chatDeliveries.id, delivery.id)); + expect(await blocked("chat_deliveries")).toBe(true); + const [publication] = await db.insert(tables.chatPublications).values({ companyId: f.companyId, endpointId: f.endpoint.id, conversationId: conversation.id, + issueId: issue.id, idempotencyKey: randomUUID(), state: "delivery_unknown", + payload: projectSafeChatPublication({ classification: "external", source: "explicit_board_send", text: "Retained result" }) }).returning(); + expect(await blocked("chat_publications")).toBe(true); + const service = chatChannelService(db, { heartbeat: { wakeup: async () => { throw new Error("No turn expected"); } }, + fetch: async () => { throw new Error("No external network expected"); } }); + try { + await service.resolvePublication(f.endpoint.id, publication.id, "mark_delivered", "local-board"); + expect(await blocked("chat_publications")).toBe(false); + await db.update(tables.chatPublications).set({ publishedAt: null }).where(eq(tables.chatPublications.id, publication.id)); + expect(await blocked("chat_publications")).toBe(true); + } finally { await service.shutdown(); } + }); + + it("retains a real acknowledged chat completion across later task versions", async () => { + const { issueService } = await import("../services/issues.js"); + const { prepareChatCompletionTurn } = await import("../services/chat-completion-delivery.js"); + const { instanceSettingsService } = await import("../services/instance-settings.js"); + const f = await seed(), service = issueService(db); + await instanceSettingsService(db).updateExperimental({ enableAgentChat: true }); + await db.update(tables.issues).set({ status: "in_progress", conversationAgentId: f.agentId, + conversationUserId: "local-board", conversationState: "waiting" }).where(eq(tables.issues.id, f.issueId)); + await db.update(tables.heartbeatRuns).set({ contextSnapshot: { issueId: f.issueId, conversationSessionGeneration: 0 } }).where(eq(tables.heartbeatRuns.id, f.runId)); + const task = await service.create(f.companyId, { title: "Actual handoff", status: "todo", createdByAgentId: f.agentId, actorRunId: f.runId }); + await service.update(task.id, { status: "done" }); + expect(await blocked("chat_completion_deliveries")).toBe(true); + const [turn] = await db.insert(tables.heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, status: "running", + contextSnapshot: { issueId: f.issueId, wakeReason: "chat_task_completed", conversationSessionGeneration: 0 } }).returning(); + await prepareChatCompletionTurn(db, turn); + const reply = await service.addComment(f.issueId, "The result is ready.", { agentId: f.agentId, runId: turn.id }, { completionReply: true }); + expect(await blocked("chat_completion_deliveries")).toBe(true); // actual reply run still owns work + await db.update(tables.heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(tables.heartbeatRuns.id, turn.id)); + expect(await blocked("chat_completion_deliveries")).toBe(false); + await service.update(task.id, { status: "todo" }); + expect(await blocked("chat_completion_deliveries")).toBe(false); // retained delivered version remains history + await db.update(tables.issueComments).set({ createdByRunId: f.runId }).where(eq(tables.issueComments.id, reply.id)); + expect(await blocked("chat_completion_deliveries")).toBe(true); + }); + + it("uses the real browser accounting and verified provider stop writers", async () => { + const { toolAccessService } = await import("../services/tool-access.js"); + const { browserUseService } = await import("../services/browser-use.js"); + const f = await seed(), userId = "browser-history-reviewer", providerRun = randomUUID(), providerSession = randomUUID(), providerBrowser = randomUUID(); + const oldKey = process.env.PAPERCLIP_SECRETS_MASTER_KEY; + process.env.PAPERCLIP_SECRETS_MASTER_KEY = "a".repeat(64); + let providerStatus = "running", stopped = false, stopCalls = 0; + const request = async (url: string, init: RequestInit = {}) => { + const route = new URL(url).pathname.replace("/api/v4", ""); + if (route === "/profiles") return Response.json({ items: [], totalItems: 0 }); + if (route === "/runs" && init.method === "POST") return Response.json({ id: providerRun, sessionId: providerSession, status: providerStatus, model: "fixture" }); + if (route.endsWith("/status")) return Response.json({ status: providerStatus }); + if (route.endsWith("/events")) return Response.json({ events: [], nextAfter: 0, hasMore: false }); + if (route.endsWith("/cancel")) { providerStatus = "cancelled"; return Response.json({}); } + if (route === `/runs/${providerRun}`) return Response.json({ status: providerStatus, totalCostUsd: "0.15", model: "fixture", result: "Finished" }); + if (route === "/browsers") return Response.json({ items: [{ id: providerBrowser, status: stopped ? "stopped" : "active" }], totalItems: 1 }); + if (route === `/browsers/${providerBrowser}`) { + if (init.method === "PATCH") { stopped = true; stopCalls++; } + return Response.json({ id: providerBrowser, agentSessionId: providerSession, status: stopped ? "stopped" : "active" }); + } + throw new Error(`Unexpected browser provider route ${route}`); + }; + try { + await db.insert(tables.companyMemberships).values({ companyId: f.companyId, principalType: "user", principalId: userId, membershipRole: "admin", status: "active" }); + const access = toolAccessService(db, { remoteHttpRequest: request, remoteHttpEndpointLookup: async () => [{ address: "8.8.8.8", family: 4 }] }); + const actor = { actorType: "user" as const, actorId: userId }; + const connected = await access.connectGalleryApp(f.companyId, { galleryKey: "browser-use-cloud", connectionMethodKey: "cloud-v4", + credentialValues: { "credentials.apiKey": "bu_fixture_secret" } }, actor); + await access.finishGalleryAppConnection(f.companyId, connected.connectionId, { enabledCatalogEntryIds: connected.catalog.map(entry => entry.id), + askFirstCatalogEntryIds: [], access: { agentIds: [f.agentId] } }, actor); + const [grant] = await db.select().from(tables.connectionGrants).where(eq(tables.connectionGrants.connectionId, connected.connectionId)); + await db.update(tables.heartbeatRuns).set({ status: "running", finishedAt: null, responsibleUserId: userId }).where(eq(tables.heartbeatRuns.id, f.runId)); + await db.update(tables.issues).set({ status: "in_progress" }).where(eq(tables.issues.id, f.issueId)); + const service = browserUseService(db, request); + const started = await service.execute({ companyId: f.companyId, agentId: f.agentId, issueId: f.issueId, runId: f.runId }, grant, + randomUUID(), "browser_start", { task: "Read the retained fixture", maxCostUsd: 1 }) as { sessionId: string }; + for (const reason of ["browser_use_sessions", "browser_use_runs"]) expect(await blocked(reason)).toBe(true); + providerStatus = "completed"; + await db.update(tables.browserUseSessions).set({ nextPollAt: new Date(0) }).where(eq(tables.browserUseSessions.id, started.sessionId)); + await service.sweep(connected.connectionId); + expect(await blocked("browser_use_browsers")).toBe(true); + const [session] = await db.select().from(tables.browserUseSessions).where(eq(tables.browserUseSessions.id, started.sessionId)); + await service.control(session, "end"); + await db.update(tables.browserUseSessions).set({ nextPollAt: new Date(0) }).where(eq(tables.browserUseSessions.id, started.sessionId)); + await service.sweep(connected.connectionId); + expect(stopCalls).toBeGreaterThan(0); + await db.update(tables.heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(tables.heartbeatRuns.id, f.runId)); + for (const reason of ["browser_use_sessions", "browser_use_runs", "browser_use_browsers"]) expect(await blocked(reason), reason).toBe(false); + const [run] = await db.select().from(tables.browserUseRuns).where(eq(tables.browserUseRuns.sessionId, started.sessionId)); + expect(run.result?.paperclipBudgetHold).toMatchObject({ state: "settled", accountedCentsExact: "15.0000000" }); + await db.update(tables.browserUseRuns).set({ eventsDrained: 0 }).where(eq(tables.browserUseRuns.id, run.id)); + expect(await blocked("browser_use_runs")).toBe(true); + } finally { + if (oldKey === undefined) delete process.env.PAPERCLIP_SECRETS_MASTER_KEY; else process.env.PAPERCLIP_SECRETS_MASTER_KEY = oldKey; + } + }); + + it("keeps browser accounting, run ownership and provider resource cleanup independent", async () => { + const f = await seed(), c = await connection(f), invocationId = randomUUID(); + const [session] = await db.insert(tables.browserUseSessions).values({ ...f, ...c, status: "closed", providerSessionId: randomUUID() }).returning(); + const [run] = await db.insert(tables.browserUseRuns).values({ companyId: f.companyId, sessionId: session.id, heartbeatRunId: f.runId, + invocationId, providerRunId: randomUUID(), status: "completed", eventsDrained: 1, + result: { paperclipBudgetHold: { invocationId, state: "settled", accountedCentsExact: "0.0000000" } } }).returning(); + const [browser] = await db.insert(tables.browserUseBrowsers).values({ companyId: f.companyId, sessionId: session.id, + providerBrowserId: randomUUID(), status: "stopped" }).returning(); + for (const reason of ["browser_use_sessions", "browser_use_runs", "browser_use_browsers"]) expect(await blocked(reason)).toBe(false); + await db.update(tables.browserUseRuns).set({ eventsDrained: 0 }).where(eq(tables.browserUseRuns.id, run.id)); + expect(await blocked("browser_use_sessions")).toBe(true); + expect(await blocked("browser_use_runs")).toBe(true); + await db.update(tables.browserUseRuns).set({ eventsDrained: 1 }).where(eq(tables.browserUseRuns.id, run.id)); + await db.update(tables.browserUseBrowsers).set({ status: "active" }).where(eq(tables.browserUseBrowsers.id, browser.id)); + expect(await blocked("browser_use_sessions")).toBe(true); + expect(await blocked("browser_use_browsers")).toBe(true); + }); + + it("uses one conversation proof for ordinary and scheduled reports and detects the comment-to-wake crash window", async () => { + const { issueService } = await import("../services/issues.js"); + const { settleConversationTurn } = await import("../services/agent-conversations.js"); + const { instanceSettingsService } = await import("../services/instance-settings.js"); + await instanceSettingsService(db).updateExperimental({ enableAgentChat: true }); + const f = await seed(); + const service = issueService(db); + const conversation = await service.create(f.companyId, { title: "History chat", assigneeAgentId: f.agentId, + conversationAgentId: f.agentId, conversationUserId: "local-board", conversationState: "waiting", status: "in_review" }); + expect(await read()).toEqual({ version: 1, backgroundWork: "none" }); + expect((await readIdleIssueSchedule(db, ownerId, Date.now())).blocked).toBe(false); + const comment = await service.addComment(conversation.id, "A new question", { userId: "local-board" }, + { clientRequestId: randomUUID() }); + expect(await read()).toEqual({ version: 1, backgroundWork: "present" }); + // Preserve the original comment but reproduce the crash before enqueue. + await db.update(tables.issues).set({ conversationState: "waiting" }).where(eq(tables.issues.id, conversation.id)); + expect((await readIdleIssueSchedule(db, ownerId, Date.now())).blocked).toBe(true); + const runId = randomUUID(); + const [run] = await db.insert(tables.heartbeatRuns).values({ id: runId, companyId: f.companyId, agentId: f.agentId, + invocationSource: "on_demand", status: "succeeded", finishedAt: now, + contextSnapshot: { issueId: conversation.id, commentId: comment.id, conversationSessionGeneration: 0 } }).returning(); + await db.insert(tables.agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agentId, + idempotencyKey: `conversation-comment:${comment.id}`, source: "on_demand", triggerDetail: "manual", reason: "issue_commented", + status: "completed", finishedAt: now, runId, payload: { issueId: conversation.id } }); + await db.insert(tables.issueComments).values({ companyId: f.companyId, issueId: conversation.id, + authorAgentId: f.agentId, createdByRunId: runId, body: "The question is answered." }); + await db.update(tables.issues).set({ conversationState: "active", status: "in_progress" }).where(eq(tables.issues.id, conversation.id)); + expect(await settleConversationTurn(db, run)).toBe(true); + expect(await read()).toEqual({ version: 1, backgroundWork: "none" }); + expect((await readIdleIssueSchedule(db, ownerId, Date.now())).blocked).toBe(false); + await db.update(tables.heartbeatRuns).set({ costAccountingPending: true }).where(eq(tables.heartbeatRuns.id, runId)); + expect(await read()).toEqual({ version: 1, backgroundWork: "present" }); + }, 60_000); + + it("uses actual secret and binding approval receipts, including ciphertext scrubbing", async () => { + const { createSecretProposalsService } = await import("../services/secret-proposals.js"); + const f = await seed(), p = await mcp(f), oldKey = process.env.PAPERCLIP_SECRETS_MASTER_KEY; + process.env.PAPERCLIP_SECRETS_MASTER_KEY = "a".repeat(64); + try { + await db.update(tables.heartbeatRuns).set({ status: "running", finishedAt: null }).where(eq(tables.heartbeatRuns.id, f.runId)); + await db.update(tables.issues).set({ status: "in_progress", executionRunId: f.runId }).where(eq(tables.issues.id, f.issueId)); + const service = createSecretProposalsService(db); + const context = { companyId: f.companyId, heartbeatRunId: f.runId, registerForRedaction: () => {} }; + const secret = await service.createSecret(context, { name: "history/token", key: "HISTORY_TOKEN", value: "synthetic-history-secret", justification: "Retained completed approval" }); + expect(await blocked("company_secret_proposals")).toBe(true); + const approved = await service.approve(f.companyId, secret.id, { resolvedByUserId: p.userId }); + const binding = await service.createBinding(context, { secretId: approved.createdSecretId!, configPath: "env.HISTORY_TOKEN", + bindingTargetPolicy: "self_and_reports", justification: "Use the approved credential" }); + await service.approve(f.companyId, binding.id, { resolvedByUserId: p.userId }); + await db.update(tables.heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(tables.heartbeatRuns.id, f.runId)); + expect(await blocked("company_secret_proposals")).toBe(false); + await db.update(tables.companySecretProposals).set({ appliedBindingConfigPath: null }).where(eq(tables.companySecretProposals.id, binding.id)); + expect(await blocked("company_secret_proposals")).toBe(true); + } finally { if (oldKey === undefined) delete process.env.PAPERCLIP_SECRETS_MASTER_KEY; else process.env.PAPERCLIP_SECRETS_MASTER_KEY = oldKey; } + }); + + it("uses actual retention archive notification dispatch and durable acknowledgment", async () => { + const { decisionRetentionService } = await import("../services/decision-retention.js"); + const { decisionArchiveDeliveryService } = await import("../services/decision-archive-delivery.js"); + const f = await seed(), activityAt = new Date("2026-01-01T00:00:00.000Z"); + await db.update(tables.agents).set({ status: "active" }).where(eq(tables.agents.id, f.agentId)); + await db.update(tables.issues).set({ createdByAgentId: f.agentId }).where(eq(tables.issues.id, f.issueId)); + await db.insert(tables.decisionRetention).values({ companyId: f.companyId, sourceKind: "review", sourceId: f.issueId, sourceActivityAt: activityAt, version: 1 }); + expect(await decisionRetentionService(db).autoArchive({ companyId: f.companyId, items: [{ sourceKind: "review", subject: { id: f.issueId }, + activityAt: activityAt.toISOString(), retentionVersion: 1, keep: false, archivedAt: null }], now })).toBe(1); + expect(await blocked("decision_archive_notification_outbox")).toBe(true); + let notifications = 0; + expect(await decisionArchiveDeliveryService(db, { notifyOriginAgent: async batch => { + expect(batch).toMatchObject({ companyId: f.companyId, agentId: f.agentId, items: [{ sourceId: f.issueId }] }); notifications++; + } }).deliver()).toMatchObject({ delivered: 1 }); + expect(notifications).toBe(1); + expect(await blocked("decision_archive_notification_outbox")).toBe(false); + await db.update(tables.decisionArchiveNotificationOutbox).set({ deliveredAt: null }); + expect(await blocked("decision_archive_notification_outbox")).toBe(true); + }); + + it.each(["rejected", "withdrawn", "expired"] as const)("accepts actual %s secret-proposal scrubbing and refuses retained ciphertext", async status => { + const { createSecretProposalsService } = await import("../services/secret-proposals.js"); + const f = await seed(); + const [proposal] = await db.insert(tables.companySecretProposals).values({ companyId: f.companyId, kind: "secret", + proposedName: "history/token", proposedKey: "TOKEN", justification: "History fixture", proposedByAgentId: f.agentId, + originRunId: f.runId, originIssueId: f.issueId, expiresAt: now, valueCiphertext: { pending: true } }).returning(); + expect(await blocked("company_secret_proposals")).toBe(true); + const result = await createSecretProposalsService(db).transition(f.companyId, proposal.id, status, { proposerAgentId: f.agentId }); + expect(result).toMatchObject({ status, valueCiphertext: null }); + expect(await blocked("company_secret_proposals")).toBe(false); + await db.update(tables.companySecretProposals).set({ valueCiphertext: { unfinished: true } }).where(eq(tables.companySecretProposals.id, proposal.id)); + expect(await blocked("company_secret_proposals")).toBe(true); + }, 60_000); + + it("retains actual decided effects, and reblocks when one effect lacks its completion receipt", async () => { + const { decisionService } = await import("../services/decisions.js"); + const previous = process.env.PAPERCLIP_DECISION_SIGNING_SECRET; + process.env.PAPERCLIP_DECISION_SIGNING_SECRET = "0123456789abcdef0123456789abcdef"; + try { + const f = await seed(), p = await mcp(f); + await db.insert(tables.companyMemberships).values({ companyId: f.companyId, principalType: "user", principalId: p.userId, + status: "active", membershipRole: "member" }); + await db.update(tables.heartbeatRuns).set({ status: "running", responsibleUserId: p.userId }).where(eq(tables.heartbeatRuns.id, f.runId)); + await db.update(tables.issues).set({ status: "in_progress", responsibleUserId: p.userId }).where(eq(tables.issues.id, f.issueId)); + const actor = { type: "agent" as const, companyId: f.companyId, agentId: f.agentId, runId: f.runId, source: "agent_jwt" as const, + onBehalfOfUserId: p.userId, onBehalfOfMemberships: [{ companyId: f.companyId, membershipRole: "member", status: "active" }] }; + const userActor = { type: "board" as const, userId: p.userId, companyIds: [f.companyId], source: "session" as const, + memberships: [{ companyId: f.companyId, membershipRole: "member", status: "active" }] }; + const service = decisionService(db, { wakeOriginAgent: async () => null }); + const decision = await service.create({ companyId: f.companyId, actor, agentId: f.agentId, runId: f.runId, + title: "Save this comment?", body: "Completion proof", continuationPolicy: "none", + options: [{ id: "yes", label: "Yes", effects: [{ type: "comment_on_issue", targetIssueId: f.issueId, + staleness: "lenient", bodyMarkdown: "Saved by the decision owner." }] }] }); + expect(await blocked("decisions")).toBe(true); + await service.decide({ id: decision.id, optionId: "yes", decidedByUserId: p.userId, userActor }); + await db.update(tables.heartbeatRuns).set({ status: "succeeded", finishedAt: now }).where(eq(tables.heartbeatRuns.id, f.runId)); + expect(await blocked("decisions")).toBe(false); + expect(await blocked("decision_effect_executions")).toBe(false); + await db.update(tables.decisions).set({ chosenOptionId: "another-option" }).where(eq(tables.decisions.id, decision.id)); + expect(await blocked("decisions")).toBe(true); + await db.update(tables.decisions).set({ chosenOptionId: "yes" }).where(eq(tables.decisions.id, decision.id)); + const [effect] = await db.select().from(tables.decisionEffectExecutions).where(eq(tables.decisionEffectExecutions.decisionId, decision.id)); + await db.delete(tables.decisionEffectExecutions).where(eq(tables.decisionEffectExecutions.id, effect.id)); + expect(await blocked("decisions")).toBe(true); + await db.insert(tables.decisionEffectExecutions).values(effect); + await db.update(tables.decisionEffectExecutions).set({ executedAt: null }).where(eq(tables.decisionEffectExecutions.decisionId, decision.id)); + expect(await blocked("decisions")).toBe(true); + expect(await blocked("decision_effect_executions")).toBe(true); + } finally { + if (previous === undefined) delete process.env.PAPERCLIP_DECISION_SIGNING_SECRET; + else process.env.PAPERCLIP_DECISION_SIGNING_SECRET = previous; + } + }, 60_000); + + it.each(["zero_effect", "dismissed", "cancelled", "expired", "partial", "failed"] as const)("proves the actual decision owner branch %s", async branch => { + const previous = process.env.PAPERCLIP_DECISION_SIGNING_SECRET; + process.env.PAPERCLIP_DECISION_SIGNING_SECRET = "0123456789abcdef0123456789abcdef"; + try { + const f = await decisionFixture(); + const effects = branch === "zero_effect" ? [] : [ + { type: "comment_on_issue" as const, targetIssueId: f.issueId, staleness: "lenient" as const, bodyMarkdown: "First effect" }, + ...(branch === "partial" ? [{ type: "comment_on_issue" as const, targetIssueId: f.issueId, + staleness: "strict" as const, bodyMarkdown: "Snapshot is now stale" }] : []), + ]; + const decision = await f.service.create({ companyId: f.companyId, actor: f.actor, agentId: f.agentId, runId: f.runId, + title: "Decision branch", body: branch, continuationPolicy: "none", options: [{ id: "yes", label: "Yes", effects }] }); + if (branch === "dismissed") await f.service.dismiss(decision.id, f.userId, f.userActor); + else if (branch === "cancelled") await f.service.cancel(decision.id, { actorType: "agent", actorId: f.agentId, runId: f.runId }); + else if (branch === "expired") { + await db.update(tables.decisions).set({ expiresAt: new Date(0) }).where(eq(tables.decisions.id, decision.id)); + await f.service.sweepTtlExpiry(); + } else await f.service.decide({ id: decision.id, optionId: "yes", decidedByUserId: f.userId, + userActor: branch === "failed" ? { type: "none", source: "none" } : f.userActor }); + await f.finish(); + const saved = await f.service.get(decision.id); + expect(saved?.executionStatus).toBe(branch === "partial" ? "partial" : branch === "failed" ? "failed" + : branch === "cancelled" || branch === "expired" ? null : "succeeded"); + expect(await blocked("decisions")).toBe(false); + expect(await blocked("decision_effect_executions")).toBe(false); + } finally { + if (previous === undefined) delete process.env.PAPERCLIP_DECISION_SIGNING_SECRET; else process.env.PAPERCLIP_DECISION_SIGNING_SECRET = previous; + } + }, 60_000); + + it.each(["missing_decider", "missing_option"] as const)("requires the actual %s recovery stop receipt after partial effects", async reason => { + const previous = process.env.PAPERCLIP_DECISION_SIGNING_SECRET; + process.env.PAPERCLIP_DECISION_SIGNING_SECRET = "0123456789abcdef0123456789abcdef"; + try { + const f = await decisionFixture(); + const decision = await f.service.create({ companyId: f.companyId, actor: f.actor, agentId: f.agentId, runId: f.runId, + title: "Recover interrupted effect ownership", body: reason, continuationPolicy: "none", options: [{ id: "yes", label: "Yes", + effects: [{ type: "comment_on_issue", targetIssueId: f.issueId, staleness: "lenient", bodyMarkdown: "Already persisted effect" }] }] }); + await f.service.decide({ id: decision.id, optionId: "yes", decidedByUserId: f.userId, userActor: f.userActor }); + // Model a stranded claim whose durable option/decider was lost after an + // already-written effect. Recovery must record its own final stop. + await db.update(tables.decisions).set({ executionStatus: "running", updatedAt: new Date(Date.now() - 120_000), + ...(reason === "missing_decider" ? { decidedByUserId: null } : { chosenOptionId: "missing" }) }).where(eq(tables.decisions.id, decision.id)); + expect(await blocked("decisions")).toBe(true); + expect(await f.service.recoverEffects()).toEqual({ resumed: 1 }); + await f.finish(); + expect(await f.service.get(decision.id)).toMatchObject({ executionStatus: "partial", metadata: { effectRecoveryError: reason } }); + expect(await blocked("decisions")).toBe(false); + await db.delete(tables.activityLog).where(sql`${tables.activityLog.entityId} = ${decision.id} AND ${tables.activityLog.action} = 'decision.recovery_failed'`); + expect(await blocked("decisions")).toBe(true); + } finally { + if (previous === undefined) delete process.env.PAPERCLIP_DECISION_SIGNING_SECRET; else process.env.PAPERCLIP_DECISION_SIGNING_SECRET = previous; + } + }); + + it.each(["archive", "keep", "manifest_failure"] as const)("checks the separate reviewed retention manifest for %s", async branch => { + const { hashAttentionArchiveManifest } = await import("../services/decision-retention.js"); + const previous = process.env.PAPERCLIP_DECISION_SIGNING_SECRET; + process.env.PAPERCLIP_DECISION_SIGNING_SECRET = "0123456789abcdef0123456789abcdef"; + try { + const f = await decisionFixture(), activityAt = new Date("2026-04-01T00:00:00.000Z"); + await db.insert(tables.decisionRetention).values({ companyId: f.companyId, sourceKind: "review", sourceId: f.issueId, + sourceActivityAt: activityAt, version: 1 }); + const manifest = [{ companyId: f.companyId, sourceKind: "review" as const, sourceId: f.issueId, + expectedVersion: 1, activityAt: activityAt.toISOString(), reason: "Reviewed history" }]; + const decision = await f.service.create({ companyId: f.companyId, actor: f.actor, agentId: f.agentId, runId: f.runId, + title: "Archive reviewed set", body: "Only the selected source", continuationPolicy: "none", + options: [{ id: "archive", label: "Archive", style: "destructive", effects: [] }, { id: "keep", label: "Keep", effects: [] }], + metadata: { kind: "attention_archive_proposal", manifestHash: branch === "manifest_failure" ? "invalid" : hashAttentionArchiveManifest(manifest) }, + additionalTargetSnapshots: { [`attention:review:${f.issueId}`]: { status: "attention", assigneeAgentId: null, + assigneeUserId: null, updatedAt: activityAt.toISOString(), attentionArchive: manifest[0] } } }); + const outcome = await f.service.decide({ id: decision.id, optionId: branch === "keep" ? "keep" : "archive", + decidedByUserId: f.userId, userActor: f.userActor }); + expect(outcome.executionStatus).toBe(branch === "manifest_failure" ? "failed" : "succeeded"); + await f.finish(); + expect(await blocked("decisions")).toBe(false); + if (branch === "archive") { + await db.update(tables.decisions).set({ targetSnapshots: {} }).where(eq(tables.decisions.id, decision.id)); + expect(await blocked("decisions")).toBe(true); + } + } finally { + if (previous === undefined) delete process.env.PAPERCLIP_DECISION_SIGNING_SECRET; else process.env.PAPERCLIP_DECISION_SIGNING_SECRET = previous; + } + }, 60_000); + + it("settles retained Muse assignments through the real pair, runner receipt and terminal reconciliation paths", async () => { + const { instanceSettingsService } = await import("../services/instance-settings.js"); + const { museIdentity, museCredentialHash } = await import("../services/muse-identity.js"); + const { museRunnerBroker } = await import("../services/muse-runner-broker.js"); + const { createHeartbeatLifecycle } = await import("../services/heartbeat/run-lifecycle.js"); + const f = await seed(), p = await mcp(f); + await instanceSettingsService(db).updateExperimental({ enableNativeRunner: true, enableMuse: true }); + await db.insert(tables.companyMemberships).values([ + { companyId: f.companyId, principalType: "user", principalId: p.userId, membershipRole: "owner", status: "active" }, + { companyId: f.companyId, principalType: "agent", principalId: f.agentId, membershipRole: "member", status: "active" }, + ]); + const adapterConfig = { provider: "muse", allowUnmeteredProvider: true, lifecycle: "per_turn" }; + await db.update(tables.agents).set({ status: "active", adapterType: "paperclip_runner", adapterConfig }).where(eq(tables.agents.id, f.agentId)); + const ticket = randomBytes(32).toString("base64url"); + const [binding] = await db.insert(tables.museAgentBindings).values({ companyId: f.companyId, agentId: f.agentId, operatorId: p.userId, + ticketHash: museCredentialHash(ticket), ticketExpiresAt: new Date(Date.now() + 600_000) }).returning(); + await db.update(tables.agents).set({ adapterConfig: { ...adapterConfig, museBindingId: binding.id } }).where(eq(tables.agents.id, f.agentId)); + const identity = museIdentity(db), credentials = await identity.pair(ticket, "2"); + const broker = museRunnerBroker(db, { publicOrigin: "https://paperclip.example", + heartbeat: { wakeup: async () => null, cancelRun: async () => null } }); + await db.update(tables.museAgentBindings).set({ status: "ready", receiverContactAt: new Date(), verifiedReplyAt: new Date() }) + .where(eq(tables.museAgentBindings.id, binding.id)); + const subject = await identity.authenticate(credentials.accessToken), sessionId = randomUUID(), turnId = `turn_${randomUUID()}`; + await db.update(tables.issues).set({ status: "in_progress", responsibleUserId: p.userId, + checkoutRunId: f.runId, executionRunId: f.runId }).where(eq(tables.issues.id, f.issueId)); + await db.update(tables.heartbeatRuns).set({ status: "running", runtimeMode: "native", driverKind: "muse_external", + nativeIssueId: f.issueId, nativeSessionId: sessionId, nativePhase: "turn_running", responsibleUserId: p.userId, finishedAt: null }) + .where(eq(tables.heartbeatRuns.id, f.runId)); + await db.insert(tables.nativeRunFinalizations).values({ runId: f.runId, companyId: f.companyId, issueId: f.issueId, + phase: "running", controllerGeneration: 1, leaseExpiresAt: new Date(Date.now() + 600_000) }); + const execution = { binding: { companyId: f.companyId, agentId: f.agentId, runId: f.runId }, + provider: { binding: await broker.snapshot(f.companyId, f.agentId, binding.id) } }; + const port = broker.port(execution); + const ref = { ...execution.provider.binding, runId: f.runId, normalizedSessionId: sessionId, turnId, assignmentRevision: 1 }; + const detach = await port.attach(async command => { + // Named external runner boundary: persist the real broker's receipt for + // the exact command it emitted, without invoking paid native execution. + await port.settle({ sourceEventId: `result_${command.requestId}`, payload: { binding: ref, requestId: command.requestId, + outcome: command.action === "consume_input" ? { status: "consumed", ...command.input } + : { status: command.action === "accept" ? "accepted" : "completed" } } }); + }); + try { + await port.dispatch({ sourceEventId: `offer_${randomUUID()}`, payload: { kind: "assignment", binding: ref, tools: [], + acceptByUnixMs: Date.now() + 600_000, expiresAtUnixMs: Date.now() + 3_600_000 } }); + const [assignment] = await db.select().from(tables.museRunnerAssignments).where(eq(tables.museRunnerAssignments.runId, f.runId)); + expect(await blocked("muse_runner_assignments")).toBe(true); + await broker.act(subject, { version: 1, command: "accept", assignmentId: assignment.id, requestId: randomUUID() }); + const { digestPaperclipSemanticContent } = await import("../vendor/paperclip-runner/index.js"); + const nativeRequestId = "history_question", response = { schema: "paperclip.question_response.v1", answers: [] }; + const inputDigest = digestPaperclipSemanticContent({ requestId: nativeRequestId, turnId, response }); + await port.inputAvailable({ sourceEventId: randomUUID(), payload: { binding: ref, requestId: nativeRequestId, turnId, inputDigest, response } }); + expect(await blocked("muse_input_deliveries")).toBe(true); + await broker.act(subject, { version: 1, command: "consume_input", assignmentId: assignment.id, requestId: randomUUID(), + nativeRequestId, inputDigest, continuationReceiptId: randomUUID(), continuationPersisted: true }); + await broker.act(subject, { version: 1, command: "finish", assignmentId: assignment.id, requestId: randomUUID(), result: {} }); + expect(await blocked("muse_runner_assignments")).toBe(true); + const lifecycle = createHeartbeatLifecycle(db, { emitTerminalAgentTaskRun: () => {} } as unknown as + import("../services/heartbeat/run-lifecycle.js").HeartbeatLifecycleDependencies); + await lifecycle.setRunStatus(f.runId, "succeeded", { finishedAt: new Date() }); + await broker.reconcileTerminalAssignments(f.companyId, f.agentId, binding.id); + for (const family of ["muse_runner_assignments", "muse_runner_operations", "muse_mailbox_items", "muse_input_deliveries", "external_agent_holds"]) { + expect(await blocked(family), family).toBe(false); + } + await db.update(tables.museRunnerOperations).set({ status: "unknown" }).where(eq(tables.museRunnerOperations.assignmentId, assignment.id)); + expect(await blocked("muse_runner_assignments")).toBe(true); + expect(await blocked("muse_runner_operations")).toBe(true); + await db.update(tables.museRunnerOperations).set({ status: "settled" }).where(eq(tables.museRunnerOperations.assignmentId, assignment.id)); + expect(await blocked("muse_agent_bindings")).toBe(true); + expect(await blocked("muse_credentials")).toBe(true); + await broker.revoke(f.companyId, f.agentId, p.userId); + expect(await blocked("muse_agent_bindings")).toBe(false); + expect(await blocked("muse_credentials")).toBe(true); // cleanup authority still lives for its owner TTL + await db.update(tables.museCredentials).set({ expiresAt: new Date(0) }).where(eq(tables.museCredentials.bindingId, binding.id)); + expect(await blocked("muse_credentials")).toBe(false); + } finally { await detach(); } + }, 60_000); + + it("settles actual Dot runner assignments, operation receipts and mailbox history", async () => { + const { dotRunnerBroker } = await import("../services/dot-runner-broker.js"); + const { instanceSettingsService } = await import("../services/instance-settings.js"); + const { createHeartbeatLifecycle } = await import("../services/heartbeat/run-lifecycle.js"); + const f = await seed(), p = await mcp(f, true), sessionId = randomUUID(), turnId = `turn_${randomUUID()}`; + await instanceSettingsService(db).updateExperimental({ enablePublicMcp: true, enableOpenAiDot: true }); + await db.insert(tables.companyMemberships).values({ companyId: f.companyId, principalType: "user", principalId: p.userId, membershipRole: "owner", status: "active" }); + await db.update(tables.mcpOauthGrants).set({ revokedAt: null }).where(eq(tables.mcpOauthGrants.id, p.grantId)); + const [grant] = await db.select().from(tables.mcpOauthGrants).where(eq(tables.mcpOauthGrants.id, p.grantId)); + const [binding] = await db.insert(tables.dotAgentBindings).values({ companyId: f.companyId, agentId: f.agentId, operatorId: p.userId, + grantId: p.grantId, providerKind: "openai_dot", status: "connected" }).returning(); + await db.update(tables.agents).set({ status: "active", adapterType: "paperclip_runner", + adapterConfig: { provider: "openai_dot", dotBindingId: binding.id, allowUnmeteredProvider: true } }).where(eq(tables.agents.id, f.agentId)); + await db.update(tables.issues).set({ status: "in_progress", responsibleUserId: p.userId, checkoutRunId: f.runId, executionRunId: f.runId }).where(eq(tables.issues.id, f.issueId)); + await db.update(tables.heartbeatRuns).set({ status: "running", runtimeMode: "native", driverKind: "openai_dot_mcp", finishedAt: null, + nativeIssueId: f.issueId, nativeSessionId: sessionId, nativePhase: "turn_running", responsibleUserId: p.userId }).where(eq(tables.heartbeatRuns.id, f.runId)); + await db.insert(tables.nativeRunFinalizations).values({ runId: f.runId, companyId: f.companyId, issueId: f.issueId, + phase: "running", controllerGeneration: 1, leaseExpiresAt: new Date(Date.now() + 600_000) }); + const broker = dotRunnerBroker(db, { heartbeat: { wakeup: async () => null, cancelRun: async () => null } }); + const { createPublicMcpOAuth } = await import("../services/public-mcp/oauth.js"); + const { createPublicMcpEvents } = await import("../services/public-mcp/events.js"); + const principal = { grant, actor: { type: "agent" as const, agentId: f.agentId, companyId: f.companyId }, + company: { id: f.companyId, name: "History", issuePrefix: "H", status: "active" } }; + const events = createPublicMcpEvents(db, createPublicMcpOAuth(db, { origin: "https://example.test", resource: "https://example.test/mcp" }, { mode: "openai_dot_agent" }), + async () => { throw new Error("No personal API expected"); }, { enableDotRunner: true, fetch: async (_url, init) => { + const body = JSON.parse(String(init.body)); + return body.type === "verification" ? Response.json({ challenge: body.challenge }) : new Response(null, { status: 204 }); + } }); + await events.subscribe(principal, { name: "paperclip.dot.mailbox_updated", arguments: { companyId: f.companyId, bindingId: binding.id }, + delivery: { mode: "webhook", url: "https://receiver.example/dot", secret: `whsec_${randomBytes(32).toString("base64")}` } }); + await events.tick(); + const challenge = (await broker.mailbox(principal)).items.find(item => item.kind === "readiness_challenge")!; + await broker.confirmChallenge(principal, String(challenge.references.challenge)); + const execution = { binding: { companyId: f.companyId, agentId: f.agentId, runId: f.runId }, + provider: { kind: "openai_dot" as const, binding: await broker.snapshot(f.companyId, f.agentId, binding.id) } }; + const ref = { ...execution.provider.binding, runId: f.runId, normalizedSessionId: sessionId, turnId, assignmentRevision: 1 }; + const port = broker.port(execution); + const detach = await port.attach(async command => { + await port.settle({ sourceEventId: randomUUID(), payload: { binding: ref, requestId: command.requestId, + outcome: { status: command.action === "accept" ? "accepted" : "completed" } } }); + }); + try { + await port.dispatch({ sourceEventId: randomUUID(), payload: { kind: "assignment", binding: ref, tools: [], + acceptByUnixMs: Date.now() + 600_000, expiresAtUnixMs: Date.now() + 3_600_000 } }); + const [assignment] = await db.select().from(tables.dotRunnerAssignments).where(eq(tables.dotRunnerAssignments.runId, f.runId)); + expect(await blocked("dot_runner_assignments")).toBe(true); + expect(await blocked("dot_mailbox_items")).toBe(true); + await broker.operation(principal, assignment.id, randomUUID(), "accept", {}); + await broker.operation(principal, assignment.id, randomUUID(), "finish", { result: {} }); + const lifecycle = createHeartbeatLifecycle(db, { emitTerminalAgentTaskRun: () => {} } as unknown as + import("../services/heartbeat/run-lifecycle.js").HeartbeatLifecycleDependencies); + await lifecycle.setRunStatus(f.runId, "succeeded", { finishedAt: new Date() }); + for (const reason of ["dot_runner_assignments", "dot_runner_operations"]) expect(await blocked(reason), reason).toBe(false); + // The independent readiness mailbox is retained until its binding is revoked. + await broker.revoke(f.companyId, f.agentId, p.userId); + expect(await blocked("dot_mailbox_items")).toBe(false); + await db.update(tables.dotRunnerOperations).set({ status: "dispatched", outcome: null }).where(eq(tables.dotRunnerOperations.assignmentId, assignment.id)); + expect(await blocked("dot_runner_assignments")).toBe(true); + expect(await blocked("dot_runner_operations")).toBe(true); + } finally { await detach(); } + }, 60_000); + + it("uses the actual Dot revocation writer but preserves Grok routine and recovery blockers", async () => { + const { dotRunnerBroker } = await import("../services/dot-runner-broker.js"); + const f = await seed(), p = await mcp(f, true); + await db.update(tables.mcpOauthGrants).set({ revokedAt: null }).where(eq(tables.mcpOauthGrants.id, p.grantId)); + const [binding] = await db.insert(tables.dotAgentBindings).values({ companyId: f.companyId, agentId: f.agentId, + operatorId: p.userId, grantId: p.grantId, status: "ready", providerKind: "openai_dot" }).returning(); + expect(await blocked("dot_agent_bindings")).toBe(true); + await dotRunnerBroker(db).revoke(f.companyId, f.agentId, p.userId); + expect(await blocked("dot_agent_bindings")).toBe(false); + expect(await blocked("mcp_oauth_grants")).toBe(false); + const [secret] = await db.insert(tables.companySecrets).values({ companyId: f.companyId, key: "GROK_ROUTINE", + name: "Grok routine material", provider: "local_encrypted" }).returning(); + const [grok] = await db.insert(tables.dotAgentBindings).values({ companyId: f.companyId, agentId: f.agentId, + operatorId: p.userId, status: "revoked", providerKind: "grok_bot", revokedAt: now }).returning(); + await db.insert(tables.grokBotRoutines).values({ companyId: f.companyId, agentId: f.agentId, bindingId: grok.id, + bindingGeneration: grok.generation, botName: "History routine", secretId: secret.id, secretVersion: 1, + recoveryRequiredAt: new Date(), recoveryReason: "work_session" }); + expect(await blocked("dot_agent_bindings")).toBe(true); + expect(await read()).toEqual({ version: 1, backgroundWork: "present" }); + expect(await db.select().from(tables.grokBotRoutines)).toHaveLength(1); + expect((await db.select().from(tables.dotAgentBindings).where(eq(tables.dotAgentBindings.id, binding.id)))[0].status).toBe("revoked"); + }, 60_000); + + it("retains a stale runtime exclusivity lease only after its exact owner and resources finish", async () => { + const { workspaceRuntimeLeaseService } = await import("../services/workspace-runtime-leases.js"); + const f = await seed(); + const [workspace] = await db.insert(tables.executionWorkspaces).values({ companyId: f.companyId, name: "History workspace", + mode: "shared_workspace", strategyType: "project_primary", providerType: "local_fs" }).returning(); + const lease = await workspaceRuntimeLeaseService(db).claim({ companyId: f.companyId, executionWorkspaceId: workspace.id, + action: "start", owner: { actorType: "agent", agentId: f.agentId, issueId: f.issueId, runId: f.runId } }); + expect(lease.outcome).toBe("created"); + expect(await blocked("execution_workspace_runtime_leases")).toBe(false); + const [runtime] = await db.insert(tables.workspaceRuntimeServices).values({ id: randomUUID(), companyId: f.companyId, + executionWorkspaceId: workspace.id, issueId: f.issueId, serviceName: "Unstarted history", scopeType: "execution_workspace", + scopeId: workspace.id, status: "stopped", lifecycle: "shared", provider: "local_process", stoppedAt: now }).returning(); + expect(await blocked("workspace_runtime_services")).toBe(false); + await db.update(tables.workspaceRuntimeServices).set({ status: "running", stoppedAt: null }).where(eq(tables.workspaceRuntimeServices.id, runtime.id)); + expect(await blocked("workspace_runtime_services")).toBe(true); + expect(await blocked("execution_workspace_runtime_leases")).toBe(true); + await db.update(tables.workspaceRuntimeServices).set({ status: "stopped", stoppedAt: now, exposureHandle: "uncleaned" }) + .where(eq(tables.workspaceRuntimeServices.id, runtime.id)); + expect(await blocked("workspace_runtime_services")).toBe(true); + await db.update(tables.executionWorkspaceRuntimeLeases).set({ ownerKey: `issue:${randomUUID()}` }) + .where(eq(tables.executionWorkspaceRuntimeLeases.id, lease.lease!.id)); + expect(await blocked("execution_workspace_runtime_leases")).toBe(true); + }); + + it("inspects an actual owned runtime process across stopped-row fallback, adoption, termination and reconstruction failure", async () => { + const runtime = await import("../services/workspace-runtime.js"); + const supervisor = await import("../services/local-service-supervisor.js"); + let ownedProcess: Awaited> = null; + const directory = await mkdtemp(path.join(tmpdir(), "history-owned-runtime-")); + const previousHome = process.env.PAPERCLIP_HOME, previousInstance = process.env.PAPERCLIP_INSTANCE_ID; + process.env.PAPERCLIP_HOME = directory; + process.env.PAPERCLIP_INSTANCE_ID = "history-owned-runtime"; + const f = await seed(); + const [workspace] = await db.insert(tables.executionWorkspaces).values({ companyId: f.companyId, name: "Owned local runtime", + mode: "shared_workspace", strategyType: "project_primary", providerType: "local_fs", cwd: directory }).returning(); + try { + await runtime.resetRuntimeServicesForTests({ terminateProcesses: true }); + expect(runtime.readWorkspaceRuntimeIdleWork()).toBe("unknown"); + await runtime.reconcilePersistedRuntimeServicesOnStartup(db); + expect(runtime.readWorkspaceRuntimeIdleWork()).toBe("none"); + const server = net.createServer(); + await new Promise((resolve, reject) => { server.once("error", reject); server.listen(0, "127.0.0.1", resolve); }); + const port = (server.address() as net.AddressInfo).port; + await new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())); + const command = "node -e \"require('node:http').createServer((req,res)=>res.end('history')).listen(Number(process.env.PORT),'127.0.0.1')\""; + const refs = await runtime.startRuntimeServicesForWorkspaceControl({ db, + actor: { id: null, name: "History operator", companyId: f.companyId }, issue: null, + executionWorkspaceId: workspace.id, adapterEnv: {}, + workspace: { baseCwd: directory, source: "project_primary", projectId: null, workspaceId: null, + repoUrl: null, repoRef: null, strategy: "project_primary", cwd: directory, branchName: null, + worktreePath: null, warnings: [], created: false, branchCreatedByRuntime: false }, + config: { desiredState: "running", workspaceRuntime: { services: [{ name: "history-http", command, port, + expose: { urlTemplate: "http://127.0.0.1:{{port}}" }, + readiness: { type: "http", urlTemplate: "http://127.0.0.1:{{port}}", timeoutSec: 10, intervalMs: 50 }, + lifecycle: "shared", reuseScope: "execution_workspace", stopPolicy: { type: "manual" } }] } } }); + expect(refs).toHaveLength(1); + ownedProcess = await supervisor.findLocalServiceRegistryRecordByRuntimeServiceId({ runtimeServiceId: refs[0].id, profileKind: "workspace-runtime" }); + expect(ownedProcess?.pid).toBeGreaterThan(0); + expect(runtime.readWorkspaceRuntimeIdleWork()).toBe("present"); + await db.update(tables.workspaceRuntimeServices).set({ status: "stopped", stoppedAt: new Date(), url: null }) + .where(eq(tables.workspaceRuntimeServices.id, refs[0].id)); + expect(await blocked("workspace_runtime_services")).toBe(false); + expect(await readIdleSleepSafety(db, hold, Date.now, ownerId, async () => runtime.readWorkspaceRuntimeIdleWork())) + .toEqual({ version: 1, backgroundWork: "present" }); + await runtime.resetRuntimeServicesForTests({ simulateSupervisorExit: true }); + expect(runtime.readWorkspaceRuntimeIdleWork()).toBe("unknown"); + const adopted = await runtime.reconcilePersistedRuntimeServicesOnStartup(db); + expect(adopted.adopted).toBe(1); + expect(runtime.readWorkspaceRuntimeIdleWork()).toBe("present"); + let enterCleanup!: () => void, releaseCleanup!: () => void; + const cleanupEntered = new Promise(resolve => { enterCleanup = resolve; }); + const cleanupGate = new Promise(resolve => { releaseCleanup = resolve; }); + const failedControl = Object.create(db) as typeof db; + failedControl.select = (() => ({ from: () => ({ where: async () => { + enterCleanup(); await cleanupGate; throw new Error("controlled stop cleanup failure"); + } }) })) as typeof db.select; + const stopping = runtime.stopRuntimeServicesForExecutionWorkspace({ db: failedControl, + executionWorkspaceId: workspace.id, workspaceCwd: directory }); + await cleanupEntered; + expect(runtime.readWorkspaceRuntimeIdleWork()).toBe("present"); + let releaseReconstruction!: () => void; + const reconstructionGate = new Promise(resolve => { releaseReconstruction = resolve; }); + const staleReconstruction = Object.create(db) as typeof db; + let firstSelect = true; + staleReconstruction.select = ((...fields: unknown[]) => { + const query = Reflect.apply(db.select, db, fields); + if (!firstSelect) return query; + firstSelect = false; + return { from: (...from: unknown[]) => { + const selected = Reflect.apply(query.from, query, from); + return { where: (...where: unknown[]) => reconstructionGate.then(() => Reflect.apply(selected.where, selected, where)) }; + } }; + }) as typeof db.select; + const earlierReconstruction = runtime.reconcilePersistedRuntimeServicesOnStartup(staleReconstruction); + releaseCleanup(); + await expect(stopping).rejects.toThrow("controlled stop cleanup failure"); + releaseReconstruction(); + await earlierReconstruction; + expect(runtime.readWorkspaceRuntimeIdleWork()).toBe("unknown"); // older successful flight cannot erase the failed control epoch + await expect(fetch(`http://127.0.0.1:${port}`, { signal: AbortSignal.timeout(1000) })).rejects.toThrow(); + ownedProcess = null; + expect(runtime.readWorkspaceRuntimeIdleWork()).toBe("unknown"); + await runtime.reconcilePersistedRuntimeServicesOnStartup(db); + await runtime.stopRuntimeServicesForExecutionWorkspace({ db, executionWorkspaceId: workspace.id, workspaceCwd: directory }); + expect(runtime.readWorkspaceRuntimeIdleWork()).toBe("none"); + expect(await blocked("workspace_runtime_services")).toBe(false); + const unavailable = Object.create(db) as typeof db; + unavailable.select = () => { throw new Error("controlled startup database failure"); }; + await expect(runtime.reconcilePersistedRuntimeServicesOnStartup(unavailable)).rejects.toThrow("controlled startup"); + expect(runtime.readWorkspaceRuntimeIdleWork()).toBe("unknown"); + await runtime.reconcilePersistedRuntimeServicesOnStartup(db); + expect(runtime.readWorkspaceRuntimeIdleWork()).toBe("none"); + await runtime.resetRuntimeServicesForTests(); + expect(runtime.readWorkspaceRuntimeIdleWork()).toBe("unknown"); + } finally { + await runtime.resetRuntimeServicesForTests({ terminateProcesses: true }); + if (ownedProcess) await supervisor.terminateLocalService({ pid: ownedProcess.pid, + processGroupId: ownedProcess.processGroupId ?? ownedProcess.pid, port: ownedProcess.port }); + if (previousHome === undefined) delete process.env.PAPERCLIP_HOME; else process.env.PAPERCLIP_HOME = previousHome; + if (previousInstance === undefined) delete process.env.PAPERCLIP_INSTANCE_ID; else process.env.PAPERCLIP_INSTANCE_ID = previousInstance; + await rm(directory, { recursive: true, force: true }); + } + }, 60_000); +}); diff --git a/server/src/__tests__/sandbox-creation-operations.test.ts b/server/src/__tests__/sandbox-creation-operations.test.ts index 7c5c1924ecb..920b5e65679 100644 --- a/server/src/__tests__/sandbox-creation-operations.test.ts +++ b/server/src/__tests__/sandbox-creation-operations.test.ts @@ -10,6 +10,7 @@ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } import { eq, sql } from "drizzle-orm"; import { agents, builtInManagedResources, companies, companySecretVersions, createDb, environmentLeases, environments, heartbeatRuns, plugins, sandboxCreationOperations } from "@paperclipai/db"; import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { environmentRuntimeService } from "../services/environment-runtime.js"; import { environmentService } from "../services/environments.js"; import { HttpError } from "../errors.js"; @@ -18,8 +19,9 @@ import { readIdleSleepSafety } from "../services/idle-sleep-safety.js"; import { nextEnvironmentCleanupAt } from "../services/environment-cleanup-state.js"; import type { PluginWorkerManager } from "../services/plugin-worker-manager.js"; +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_HISTORY_TEST_DATABASE_URL"); describe("durable sandbox creation through the runtime entry point", () => { - let database: Awaited>; + let database: Awaited> | undefined; let db: ReturnType; let companyId: string, environmentId: string, pluginId: string, runId: string, spool: string; let creates: number, destroys: number, loseReply: boolean, failDestroy: boolean; @@ -27,8 +29,8 @@ describe("durable sandbox creation through the runtime entry point", () => { const resources = new Map(); let workers: PluginWorkerManager; beforeAll(async () => { - database = await startEmbeddedPostgresTestDatabase("sandbox-creation-operations-"); - db = createDb(database.connectionString); + if (!external) database = await startEmbeddedPostgresTestDatabase("sandbox-creation-operations-"); + db = createDb(external ?? database!.connectionString, { maxConnections: 4, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); }, 90_000); beforeEach(async () => { await db.execute(sql`truncate sandbox_creation_operations, companies, environments, plugins cascade`); @@ -267,7 +269,7 @@ describe("durable sandbox creation through the runtime entry point", () => { return secret; } it.each(["deleted", "disabled", "archived", "version-disabled", "vault-disabled", "vault-coming-soon"] as const)( - "retains a %s creation credential as quiet manual reconciliation across restart", async state => { + "retains a %s creation credential with a quiet worker while sleep remains blocked across restart", async state => { const secrets = secretOwner.secretService(db); const vault = state.startsWith("vault-") ? await secrets.createProviderConfig(companyId, { provider: "local_encrypted", displayName: "Creation vault", status: "ready", config: {}, @@ -281,8 +283,10 @@ describe("durable sandbox creation through the runtime entry point", () => { const inspectCalls = () => vi.mocked(workers.call).mock.calls.filter(value => value[1] === "environmentInspectLease").length; await due(); await runtime().flushDeferredOrphanCleanups(); expect(await db.select().from(sandboxCreationOperations)).toEqual([ - expect.objectContaining({ state: "needs_reconciliation" }), + expect.objectContaining({ companyId, state: "needs_reconciliation", + metadata: expect.objectContaining({ creationReconciliationReason: "creation_identity_unavailable" }) }), ]); + expect(resources.size).toBe(1); expect(await db.select().from(environmentLeases)).toEqual([]); expect(inspectCalls()).toBe(0); expect(await nextEnvironmentCleanupAt(db)).toBeNull(); @@ -293,12 +297,12 @@ describe("durable sandbox creation through the runtime entry point", () => { await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, runId)); const ownerId = randomUUID(); const hold = { ownerId, draining: true, startedAt: new Date(Date.now() - 1000), expiresAt: new Date(Date.now() + 60_000), activeRuns: 0, pendingWakes: 0 }; - // An installed plugin is a separate blocker until its worker proves quiet. + // A quiet plugin worker cannot discharge the retained creation uncertainty. expect(await readIdleSleepSafety(db, () => hold, Date.now, ownerId, async () => "none")) .toEqual({ version: 1, backgroundWork: "present" }); expect(await readIdleSleepSafety(db, () => hold, Date.now, ownerId, async () => "none", async () => ({ backgroundWork: "none", pluginIds: [pluginId] }))) - .toEqual({ version: 1, backgroundWork: "none" }); + .toEqual({ version: 1, backgroundWork: "present" }); expect(await readdir(spool)).toEqual([]); }); it("retries a temporary creation credential resolution failure without losing ownership", async () => { diff --git a/server/src/__tests__/storage-local-provider.test.ts b/server/src/__tests__/storage-local-provider.test.ts index e35500896fa..d25963d9fdf 100644 --- a/server/src/__tests__/storage-local-provider.test.ts +++ b/server/src/__tests__/storage-local-provider.test.ts @@ -119,6 +119,32 @@ describe("local disk storage provider", () => { await expect(service.getObject("company-1", stored.objectKey)).rejects.toMatchObject({ status: 404 }); }); + it.each(["EACCES", "EIO"])("retains storage uncertainty for %s instead of reporting absence", async code => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-storage-")); + tempRoots.push(root); + const service = createStorageService(createLocalDiskStorageProvider(root)); + const stored = await service.putFile({ companyId: "company-1", namespace: "issues/issue-1", originalFilename: "retained.txt", + contentType: "text/plain", body: Buffer.from("must not disappear") }); + const failure = Object.assign(new Error("controlled filesystem failure"), { code }); + const unlink = vi.spyOn(fs, "unlink").mockRejectedValueOnce(failure); + try { await expect(service.deleteObject("company-1", stored.objectKey)).rejects.toBe(failure); } + finally { unlink.mockRestore(); } + expect(await service.headObject("company-1", stored.objectKey)).toMatchObject({ exists: true }); + const stat = vi.spyOn(fs, "stat").mockRejectedValueOnce(failure); + try { await expect(service.headObject("company-1", stored.objectKey)).rejects.toBe(failure); } + finally { stat.mockRestore(); } + expect(await readStreamToBuffer((await service.getObject("company-1", stored.objectKey)).stream)).toEqual(Buffer.from("must not disappear")); + }); + + it("treats a non-directory object path as absent", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-storage-")); + tempRoots.push(root); + await fs.writeFile(path.join(root, "company-1"), "not a directory"); + const service = createStorageService(createLocalDiskStorageProvider(root)); + expect(await service.headObject("company-1", "company-1/object")).toEqual({ exists: false }); + await expect(service.deleteObject("company-1", "company-1/object")).resolves.toBeUndefined(); + }); + it.each(["headObject", "deleteObject"] as const)("cancels %s even if a provider stops responding", async method => { const provider = createLocalDiskStorageProvider(os.tmpdir()); const request = vi.spyOn(provider, method).mockImplementation(async () => new Promise(() => {})); diff --git a/server/src/__tests__/voice-session-store.integration.test.ts b/server/src/__tests__/voice-session-store.integration.test.ts index ed0b95194cc..de061793644 100644 --- a/server/src/__tests__/voice-session-store.integration.test.ts +++ b/server/src/__tests__/voice-session-store.integration.test.ts @@ -1,3 +1,5 @@ +import { chatHistoryActionBlocks } from "./helpers/chat-history-proof.js"; +import { voiceCallReports } from "../services/voice/voice-call-reports.js"; import { subscribeDatabaseWork } from "@paperclipai/db"; import * as voiceReplyNotifications from "../services/voice/voice-reply-notifications.js"; import { voiceReplySchedule } from "../services/voice/voice-reply-schedule.js"; @@ -40,15 +42,16 @@ import { buildPaperclipRuntimeMcpServers } from "../services/heartbeat.js"; import { createToolGatewayService } from "../services/tool-gateway.js"; import { initializeRunIdentity } from "../services/run-identity.js"; import { syncSpekoVoiceTools, spekoToolsForSession, executeSpekoVoiceTool } from "../services/voice/speko-agent-tools.js"; -const external = resolveExternalTestDatabaseUrl("PAPERCLIP_VOICE_TEST_DATABASE_URL"); +const historyExternal = resolveExternalTestDatabaseUrl("PAPERCLIP_HISTORY_TEST_DATABASE_URL"); +const external = historyExternal ?? resolveExternalTestDatabaseUrl("PAPERCLIP_VOICE_TEST_DATABASE_URL"); const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); (support.supported ? describe : describe.skip)("durable Speko voice sessions", () => { let db: ReturnType; let database: Awaited> | undefined; beforeAll(async () => { if (!external) database = await startEmbeddedPostgresTestDatabase("paperclip-speko-store-"); - if (external) await applyPendingMigrations(external); - db = createDb(external ?? database!.connectionString, { idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + if (external && !historyExternal) await applyPendingMigrations(external); + db = createDb(external ?? database!.connectionString, { idleTimeoutSeconds: 0, maxLifetimeSeconds: 0, ...(historyExternal ? { maxConnections: 4 } : {}) }); }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }); @@ -75,7 +78,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu return { companyId, endpointId, sessionId, token: reserved.token, envelope, webhookId: `msg_${toolId}`, fingerprint: createHash("sha256").update(JSON.stringify(envelope)).digest("hex"), accept }; }; async function publication(state: "streaming" | "published" = "published") { - const [row] = await db.insert(chatPublications).values({ companyId, endpointId, conversationId, issueId, idempotencyKey: randomUUID(), state, payload: { text: "Approved result" } }).returning(); + const [row] = await db.insert(chatPublications).values({ companyId, endpointId, conversationId, issueId, idempotencyKey: randomUUID(), state, publishedAt: state === "published" ? new Date() : null, payload: { text: "Approved result" } }).returning(); return row; } return { companyId, endpointId, agentId, connectionId, issueId, conversationId, callerId, sessionId, store, request, reserved, accept, toolInput, publication }; @@ -164,6 +167,13 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu const report = f.sign({ type: "call.report", session_id: f.call.providerSessionId, organization_id: f.event.organization_id }); await f.service.inbound.lifecycle(f.endpoint.publicId, report.body, report.headers); await settle(); expect(f.transport.callReport).toHaveBeenCalledTimes(2); expect(onError).not.toHaveBeenCalled(); + const receipts = (await db.select().from(chatActions).where(eq(chatActions.endpointId, f.endpointId))) + .filter(row => row.kind === "speko_lifecycle_receipt"); + expect(receipts.length).toBeGreaterThan(0); + for (const receipt of receipts) expect(await chatHistoryActionBlocks(db, receipt.id)).toBe(true); + await db.update(chatVoiceSessions).set({ endedAt: new Date(Date.now() - 2 * 86_400_000) }).where(eq(chatVoiceSessions.companyId, f.companyId)); + expect(await voiceCallReports(db, f.transport.callReport).load({ kind: "company", id: f.companyId })).toEqual([]); + for (const receipt of receipts) expect(await chatHistoryActionBlocks(db, receipt.id)).toBe(false); } finally { await coordinator.stop(); vi.useRealTimers(); const ended = f.sign({ type: "call.status", call_id: f.call.providerSessionId, status: "ended" }); @@ -1381,7 +1391,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu it("sends long approved phone answers as durable ordered parts and resumes after a rate limit", async () => { const f = await pushFixture(), text = "😀 answer ".repeat(4000) + "FINAL"; await db.update(chatPublications).set({payload: {text}}).where(eq(chatPublications.id, f.publication.id)); - const [later] = await db.insert(chatPublications).values({companyId: f.companyId, endpointId: f.endpointId, conversationId: f.conversationId, issueId: f.issueId, idempotencyKey: randomUUID(), state: "published", payload: {text: "Later answer B"}}).returning(); + const [later] = await db.insert(chatPublications).values({companyId: f.companyId, endpointId: f.endpointId, conversationId: f.conversationId, issueId: f.issueId, idempotencyKey: randomUUID(), state: "published", publishedAt: new Date(), payload: {text: "Later answer B"}}).returning(); await f.store.enqueuePublication(f.companyId, later.id); f.transport.sendCallMessage.mockResolvedValueOnce({messageId: "part-0"}).mockRejectedValueOnce(new SpekoProviderError("provider_unavailable", false, 429)); await f.push(); @@ -1405,6 +1415,13 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu for (const call of calls) {expect(call[1].length).toBeLessThanOrEqual(16_000); expect(/^[\uDC00-\uDFFF]|[\uD800-\uDBFF]$/.test(call[1])).toBe(false);} const [delivered] = await db.select().from(chatVoiceReplies).where(eq(chatVoiceReplies.id, reply.id)); expect(delivered.deliveredAt).toBeInstanceOf(Date); expect(delivered.spokenAt).toBeNull(); + const accepted = (await db.select().from(chatActions).where(eq(chatActions.endpointId, f.endpointId))) + .filter(row => row.kind === "speko_voice_reply_push"); + for (const receipt of accepted) expect(await chatHistoryActionBlocks(db, receipt.id)).toBe(true); + await f.service.end(f.companyId, f.session.id, f.input.caller); + await db.update(chatVoiceSessions).set({ endedAt: new Date(Date.now() - 2 * 86_400_000) }).where(eq(chatVoiceSessions.companyId, f.companyId)); + expect(await voiceCallReports(db, f.transport.callReport).load({ kind: "company", id: f.companyId })).toEqual([]); + for (const receipt of accepted) expect(await chatHistoryActionBlocks(db, receipt.id)).toBe(false); }); it("does not retry an uncertain long-answer part or send its suffix", async () => { const f = await pushFixture(); @@ -1416,6 +1433,13 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu expect(f.transport.sendCallMessage).toHaveBeenCalledTimes(2); const [reply] = await db.select().from(chatVoiceReplies).where(eq(chatVoiceReplies.publicationId, f.publication.id)); expect(reply.deliveredAt).toBeNull(); + await f.service.end(f.companyId, f.session.id, f.input.caller); + await db.update(chatVoiceSessions).set({ endedAt: new Date(Date.now() - 2 * 86_400_000) }).where(eq(chatVoiceSessions.companyId, f.companyId)); + expect(await voiceCallReports(db, f.transport.callReport).load({ kind: "company", id: f.companyId })).toEqual([]); + const uncertain = (await db.select().from(chatActions).where(eq(chatActions.endpointId, f.endpointId))) + .filter(row => row.kind === "speko_voice_reply_push" && row.status === "unknown"); + expect(uncertain).toHaveLength(1); + expect(await chatHistoryActionBlocks(db, uncertain[0]!.id)).toBe(true); }); it("pushes a published phone answer exactly once across concurrent dispatch and restart", async () => { const f = await pushFixture(); @@ -2027,6 +2051,15 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu expect(tools).toHaveLength(3); expect(f.transport.configureTool).toHaveBeenCalledTimes(7); expect(f.transport.configureTool.mock.calls.slice(-3).every((call) => Boolean(call[3]))).toBe(true); + const receipts = (await db.select().from(chatActions).where(eq(chatActions.endpointId, f.endpointId))) + .filter(row => ["speko_tool_setup", "speko_lifecycle_setup"].includes(row.kind)); + expect(receipts).toHaveLength(4); + for (const receipt of receipts) expect(await chatHistoryActionBlocks(db, receipt.id)).toBe(true); // real24h refresh obligation + await db.update(chatVoiceSessions).set({ endedAt: new Date(Date.now() - 2 * 86_400_000) }).where(eq(chatVoiceSessions.companyId, f.companyId)); + expect(await voiceCallReports(db, f.transport.callReport).load({ kind: "company", id: f.companyId })).toEqual([]); + // The resource helper alone has no post-configuration commit receipt; + // defaults/runtime completion belongs to the ordinary configure owner. + for (const receipt of receipts) expect(await chatHistoryActionBlocks(db, receipt.id)).toBe(true); }); it.each(["current", "legacy"])("moves only receipt-owned %s tools and hooks when callback origin and signing secret change", async proofVersion => { diff --git a/server/src/modules/external-agents/idle-history-proof.ts b/server/src/modules/external-agents/idle-history-proof.ts new file mode 100644 index 00000000000..cbed05b1f77 --- /dev/null +++ b/server/src/modules/external-agents/idle-history-proof.ts @@ -0,0 +1,89 @@ +import { finishedIdleRun } from "../../services/idle-work-proof.js"; + +function completedAssignment(alias: string, provider: "dot" | "muse"): string { + const muse = provider === "muse"; + const identity = muse ? 'muse' : 'openai_dot'; + return `EXISTS (SELECT 1 FROM ${provider}_agent_bindings binding JOIN heartbeat_runs run + ON run.id = ${alias}.run_id AND run.company_id = binding.company_id AND run.agent_id = binding.agent_id + WHERE binding.id = ${alias}.binding_id AND binding.company_id = ${alias}.company_id + ${muse ? '' : "AND binding.provider_kind = 'openai_dot'"} + AND binding.agent_id = ${alias}.agent_id AND ${alias}.binding_generation > 0 + AND ${alias}.binding_generation <= binding.generation AND ${finishedIdleRun('run')} + AND ((${alias}.status = 'settled' AND ${muse + ? `run.status = 'succeeded' AND ${alias}.accepted_result_at IS NOT NULL AND ${alias}.finalized_at = run.finished_at` + : `${alias}.settled_at IS NOT NULL`}) + OR (${alias}.status = 'fenced' AND (${alias}.${muse ? 'claimed_at' : 'accepted_at'} IS NULL + OR EXISTS (SELECT 1 FROM external_agent_holds h WHERE h.provider = '${identity}' + AND ${muse ? 'true' : 'false'} + AND h.assignment_id = ${alias}.id AND h.company_id = ${alias}.company_id + AND h.agent_id = ${alias}.agent_id AND h.run_id = ${alias}.run_id + AND h.binding_id = ${alias}.binding_id AND h.binding_generation = ${alias}.binding_generation + AND h.released_at IS NOT NULL AND NOT h.worker_unknown AND NOT h.native_effects_unknown)))) + AND NOT EXISTS (SELECT 1 FROM external_agent_holds h WHERE h.provider = '${identity}' + AND h.assignment_id = ${alias}.id AND (h.company_id <> ${alias}.company_id + OR h.run_id <> ${alias}.run_id OR h.binding_id <> ${alias}.binding_id + OR h.binding_generation <> ${alias}.binding_generation OR h.agent_id <> ${alias}.agent_id + OR h.released_at IS NULL OR h.worker_unknown OR h.native_effects_unknown)))`; +} + +function completedOperation(alias: string, provider: "dot" | "muse"): string { + return `${alias}.status IN (${provider === 'dot' ? "'completed', 'rejected'" : "'settled', 'rejected'"}) + AND jsonb_typeof(${alias}.outcome) = 'object' + AND EXISTS (SELECT 1 FROM ${provider}_runner_assignments a WHERE a.id = ${alias}.assignment_id + AND a.company_id = ${alias}.company_id AND (${completedAssignment('a', provider)}))`; +} + +function assignmentCheck(provider: "dot" | "muse") { + return { reason: `${provider}_runner_assignments`, query: `SELECT 1 FROM ${provider}_runner_assignments a WHERE ( + (${completedAssignment('a', provider)}) AND NOT EXISTS (SELECT 1 FROM ${provider}_runner_operations o + WHERE o.assignment_id = a.id AND (o.company_id <> a.company_id OR (${completedOperation('o', provider)}) IS NOT TRUE))) IS NOT TRUE` }; +} + +function mailboxCheck(provider: "dot" | "muse") { + return { reason: `${provider}_mailbox_items`, query: `SELECT 1 FROM ${provider}_mailbox_items m WHERE ( + EXISTS (SELECT 1 FROM ${provider}_agent_bindings b WHERE b.id = m.binding_id AND b.company_id = m.company_id + AND m.binding_generation > 0 AND m.binding_generation <= b.generation) + AND ((m.assignment_id IS NOT NULL AND m.kind IN ('assignment', 'operation_result', 'follow_up', 'authority_revoked'${provider === 'muse' ? ", 'input_available'" : ''}) + AND EXISTS (SELECT 1 FROM ${provider}_runner_assignments a WHERE a.id = m.assignment_id + AND a.company_id = m.company_id AND a.binding_id = m.binding_id AND a.binding_generation = m.binding_generation + AND (${completedAssignment('a', provider)}))) + OR (m.assignment_id IS NULL AND m.kind = 'readiness_challenge' AND EXISTS ( + SELECT 1 FROM ${provider}_agent_bindings b WHERE b.id = m.binding_id AND b.company_id = m.company_id + AND b.status = 'revoked' AND b.revoked_at IS NOT NULL)))) IS NOT TRUE` }; +} + +export const idleExternalAgentHistoryChecks = [ + { reason: "dot_agent_bindings", query: `SELECT 1 FROM dot_agent_bindings b WHERE ( + b.provider_kind = 'openai_dot' AND b.status = 'revoked' AND b.revoked_at IS NOT NULL AND b.pairing_code_hash IS NULL + AND b.challenge_hash IS NULL AND EXISTS (SELECT 1 FROM agents a WHERE a.id = b.agent_id AND a.company_id = b.company_id) + AND (b.grant_id IS NULL OR EXISTS (SELECT 1 FROM mcp_oauth_grants g + WHERE g.id = b.grant_id AND g.company_id = b.company_id AND g.user_id = b.operator_id + AND g.agent_id = b.agent_id AND g.purpose = 'agent' AND g.revoked_at IS NOT NULL))) IS NOT TRUE` }, + { reason: "muse_agent_bindings", query: `SELECT 1 FROM muse_agent_bindings b WHERE ( + b.status = 'revoked' AND b.revoked_at IS NOT NULL AND b.ticket_hash IS NULL AND b.challenge_hash IS NULL + AND ((b.qualification_id IS NULL AND b.detector_removal_requested_at IS NULL) + OR (b.detector_removed_at IS NOT NULL AND b.detector_removed_at >= b.detector_removal_requested_at)) + AND EXISTS (SELECT 1 FROM agents a WHERE a.id = b.agent_id AND a.company_id = b.company_id) + AND (b.grant_id IS NULL OR EXISTS (SELECT 1 FROM mcp_oauth_grants g + WHERE g.id = b.grant_id AND g.company_id = b.company_id AND g.user_id = b.operator_id + AND g.agent_id = b.agent_id AND g.purpose = 'agent' AND g.revoked_at IS NOT NULL))) IS NOT TRUE` }, + { reason: "muse_credentials", query: `SELECT 1 FROM muse_credentials c WHERE ( + c.kind IN ('access', 'refresh', 'signal', 'cleanup', 'detector_cleanup') + AND (c.revoked_at IS NOT NULL OR c.consumed_at IS NOT NULL OR c.expires_at <= now()) + AND EXISTS (SELECT 1 FROM muse_agent_bindings b WHERE b.id = c.binding_id AND b.company_id = c.company_id + AND c.binding_generation > 0 AND c.binding_generation <= b.generation)) IS NOT TRUE` }, + assignmentCheck('dot'), assignmentCheck('muse'), + { reason: "dot_runner_operations", query: `SELECT 1 FROM dot_runner_operations o WHERE (${completedOperation('o', 'dot')}) IS NOT TRUE` }, + { reason: "muse_runner_operations", query: `SELECT 1 FROM muse_runner_operations o WHERE (${completedOperation('o', 'muse')}) IS NOT TRUE` }, + mailboxCheck('dot'), mailboxCheck('muse'), + { reason: "muse_input_deliveries", query: `SELECT 1 FROM muse_input_deliveries i WHERE ( + EXISTS (SELECT 1 FROM muse_runner_assignments a WHERE a.id = i.assignment_id AND a.company_id = i.company_id + AND a.turn_id = i.turn_id AND (${completedAssignment('a', 'muse')}))) IS NOT TRUE` }, + { reason: "external_agent_holds", query: `SELECT 1 FROM external_agent_holds h WHERE ( + h.provider = 'muse' AND h.released_at IS NOT NULL AND NOT h.worker_unknown AND NOT h.native_effects_unknown + AND EXISTS (SELECT 1 FROM muse_runner_assignments a WHERE a.id = h.assignment_id + AND a.company_id = h.company_id AND a.agent_id = h.agent_id AND a.run_id = h.run_id + AND a.binding_id = h.binding_id AND a.binding_generation = h.binding_generation + AND (h.operator_attested_at IS NOT NULL OR (a.status = 'settled' AND a.accepted_result_at IS NOT NULL)) + AND (${completedAssignment('a', 'muse')}))) IS NOT TRUE` }, +] as const; diff --git a/server/src/modules/external-agents/index.ts b/server/src/modules/external-agents/index.ts index 606d549b843..9d1b6e22c19 100644 --- a/server/src/modules/external-agents/index.ts +++ b/server/src/modules/external-agents/index.ts @@ -4,3 +4,5 @@ export { withExternalAdmissionGuard, withExternalExecutionAdmission, assertNoExt export { dotRunnerBroker, createDotRunnerMcpTools } from "./dot.js"; export { notifyExternalAgentSettlement, pendingMuseTerminalBindings, type ExternalAgentSettlementScope } from "./muse-terminal-settlement.js"; + +export { idleExternalAgentHistoryChecks } from "./idle-history-proof.js"; diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.ts index bce9f2db1d5..e25a5fc647d 100644 --- a/server/src/routes/issues.ts +++ b/server/src/routes/issues.ts @@ -1,3 +1,4 @@ +import { confirmMcpAttachmentCleanup, recordMcpAttachmentCleanup, type McpAttachmentCleanup } from "../services/public-mcp/upload-cleanup.js"; import { notifyDeferredExecutionWork } from "../services/deferred-execution-notifications.js"; import { notifyNativeRecoveryNotices } from "../services/native-recovery-notice-notifications.js"; import { notifyNativeStatusReassessment } from "../services/native-status-reassessment-notifications.js"; @@ -16343,6 +16344,7 @@ export function issueRoutes( for (const attachment of attachments) { try { await storage.deleteObject(attachment.companyId, attachment.objectKey); + await recordMcpAttachmentCleanup(db, await confirmMcpAttachmentCleanup(db, storage, attachment)); } catch (err) { logger.warn( { err, issueId: id, attachmentId: attachment.id }, @@ -20227,8 +20229,10 @@ export function issueRoutes( if (!(await assertDeliverableMutationAllowedByRunContext(req, res, issue))) return; + let mcpCleanup: McpAttachmentCleanup | null = null; try { await storage.deleteObject(attachment.companyId, attachment.objectKey); + mcpCleanup = await confirmMcpAttachmentCleanup(db, storage, attachment); } catch (err) { logger.warn( { err, attachmentId }, @@ -20236,7 +20240,7 @@ export function issueRoutes( ); } - const removed = await svc.removeAttachment(attachmentId); + const removed = await svc.removeAttachment(attachmentId, { mcpCleanup }); if (!removed) { res.status(404).json({ error: "Attachment not found" }); return; diff --git a/server/src/services/browser-use-idle-proof.ts b/server/src/services/browser-use-idle-proof.ts new file mode 100644 index 00000000000..c61baf1914f --- /dev/null +++ b/server/src/services/browser-use-idle-proof.ts @@ -0,0 +1,49 @@ +import { finishedIdleRun } from "./idle-work-proof.js"; + +/** Mirrors sweep/account: terminal provider status cannot settle usage. */ +export function completedBrowserRun(run: string): string { + const hold = `${run}.result->'paperclipBudgetHold'`; + return `${run}.status IN ('completed', 'failed', 'cancelled') + AND ${run}.events_drained = 1 AND ${run}.recovery_cursor IS NULL + AND jsonb_typeof(${hold}) = 'object' AND ${hold}->>'invocationId' = ${run}.invocation_id::text + AND ((${run}.provider_run_id IS NOT NULL AND ${hold}->>'state' = 'settled' + AND ${hold}->>'accountedCentsExact' ~ '^[0-9]+([.][0-9]+)?$') + OR (${run}.provider_run_id IS NULL AND ${run}.status = 'failed' AND ${hold}->>'state' = 'released')) + -- account() replaces held authority with the authoritative terminal + -- summary and ledger charge. It retains prior diagnostic flags in JSON. + AND (${hold}->>'state' = 'settled' OR COALESCE(${hold}->>'billingUnknown', 'false') = 'false') + AND EXISTS (SELECT 1 FROM browser_use_sessions browser_parent_session JOIN heartbeat_runs browser_parent_run + ON browser_parent_run.id = ${run}.heartbeat_run_id AND browser_parent_run.company_id = browser_parent_session.company_id + AND browser_parent_run.agent_id = browser_parent_session.agent_id + WHERE browser_parent_session.id = ${run}.session_id AND browser_parent_session.company_id = ${run}.company_id + AND (browser_parent_run.native_issue_id = browser_parent_session.issue_id + OR browser_parent_run.context_snapshot->>'issueId' = browser_parent_session.issue_id::text + OR browser_parent_run.context_snapshot->>'taskId' = browser_parent_session.issue_id::text) + AND (browser_parent_run.native_issue_id IS NULL OR browser_parent_run.native_issue_id = browser_parent_session.issue_id) + AND (browser_parent_run.context_snapshot->>'issueId' IS NULL + OR browser_parent_run.context_snapshot->>'issueId' = browser_parent_session.issue_id::text) + AND (browser_parent_run.context_snapshot->>'taskId' IS NULL + OR browser_parent_run.context_snapshot->>'taskId' = browser_parent_session.issue_id::text) + AND ${finishedIdleRun('browser_parent_run')})`; +} + +export const idleBrowserHistoryChecks = [ + { reason: "browser_use_sessions", query: `SELECT 1 FROM browser_use_sessions s WHERE ( + (s.status = 'closed' OR (s.status = 'failed' AND s.provider_session_id IS NULL)) + AND s.lease_token IS NULL AND s.lease_until IS NULL AND s.stop_requested IS NULL + AND EXISTS (SELECT 1 FROM issues i JOIN agents a ON a.id = s.agent_id AND a.company_id = i.company_id + JOIN tool_connections c ON c.id = s.connection_id AND c.company_id = i.company_id + JOIN connection_grants g ON g.id = s.grant_id AND g.company_id = i.company_id AND g.connection_id = c.id + WHERE i.id = s.issue_id AND i.company_id = s.company_id) + AND NOT EXISTS (SELECT 1 FROM browser_use_runs r WHERE r.session_id = s.id + AND (r.company_id <> s.company_id OR (${completedBrowserRun('r')}) IS NOT TRUE)) + AND NOT EXISTS (SELECT 1 FROM browser_use_browsers b WHERE b.session_id = s.id + AND (b.company_id <> s.company_id OR b.status <> 'stopped'))) IS NOT TRUE` }, + { reason: "browser_use_runs", query: `SELECT 1 FROM browser_use_runs r WHERE (${completedBrowserRun('r')}) IS NOT TRUE` }, + // stopped is written only after client.stop followed by client.browser + // confirms stopped. Parent/session/run checks remain independent barriers. + { reason: "browser_use_browsers", query: `SELECT 1 FROM browser_use_browsers b WHERE ( + b.status = 'stopped' AND EXISTS (SELECT 1 FROM browser_use_sessions s + WHERE s.id = b.session_id AND s.company_id = b.company_id AND s.status = 'closed' + AND s.stop_requested IS NULL AND s.lease_token IS NULL AND s.lease_until IS NULL)) IS NOT TRUE` }, +] as const; diff --git a/server/src/services/chat-channels/discord/idle-history-proof.ts b/server/src/services/chat-channels/discord/idle-history-proof.ts new file mode 100644 index 00000000000..e90be06c0f6 --- /dev/null +++ b/server/src/services/chat-channels/discord/idle-history-proof.ts @@ -0,0 +1,25 @@ +/** Command acceptance is final history; a staged control publication remains an + * independent obligation and must settle before its command receipt is quiet. */ +export function completedDiscordAction(a: string): string { + return `${a}.kind = 'discord_native_command' AND ${a}.status = 'processed' + AND ${a}.payload->>'version' = '1' AND ${a}.payload->'invocation'->>'schema' = 'paperclip.discord.native-command.v1' + AND ${a}.payload->'invocation'->>'command' IN ('status', 'new', 'close') + AND ${a}.payload->'invocation'->>'digest' ~ '^[a-f0-9]{64}$' + AND ${a}.provider_action_id = 'discord-native-command:' || (${a}.payload->'invocation'->>'interactionId') + AND ${a}.result->>'kind' = 'discord_native_command_recorded' AND length(${a}.result->>'content') > 0 + AND EXISTS (SELECT 1 FROM chat_external_principals principal WHERE principal.id = ${a}.principal_id + AND principal.company_id = ${a}.company_id AND principal.provider = 'discord' + AND principal.external_id = ${a}.payload->'invocation'->>'actorExternalId') + AND ((${a}.conversation_id IS NULL AND ${a}.payload->'target' = 'null'::jsonb) + OR EXISTS (SELECT 1 FROM chat_conversations conversation WHERE conversation.id = ${a}.conversation_id + AND conversation.company_id = ${a}.company_id AND conversation.endpoint_id = ${a}.endpoint_id + AND conversation.id::text = ${a}.payload->'target'->>'conversationId' + AND conversation.issue_id::text = ${a}.payload->'target'->>'issueId' + AND conversation.session_generation::text = ${a}.payload->'target'->>'sessionGeneration')) + AND (${a}.result->'publicationId' = 'null'::jsonb OR EXISTS ( + SELECT 1 FROM chat_publications publication WHERE publication.id::text = ${a}.result->>'publicationId' + AND publication.company_id = ${a}.company_id AND publication.endpoint_id = ${a}.endpoint_id + AND publication.conversation_id = ${a}.conversation_id + AND ((publication.state = 'published' AND publication.published_at IS NOT NULL) + OR (publication.state IN ('failed', 'cancelled') AND publication.next_attempt_at IS NULL))))`; +} diff --git a/server/src/services/chat-channels/github/idle-history-proof.ts b/server/src/services/chat-channels/github/idle-history-proof.ts new file mode 100644 index 00000000000..696460494f3 --- /dev/null +++ b/server/src/services/chat-channels/github/idle-history-proof.ts @@ -0,0 +1,65 @@ +function completedIngress(a: string): string { + return `(${a}.status = 'processed' AND ${a}.result->>'httpStatus' ~ '^2[0-9][0-9]$' AND ${a}.result->>'retryable' = 'false') + OR (${a}.status = 'failed' AND ${a}.result->>'retryable' = 'false' AND ( + ${a}.result->>'code' = 'github_webhook_ingress_payload_invalid' + OR ${a}.result->>'httpStatus' ~ '^4[0-9][0-9]$' + OR (jsonb_typeof(${a}.result->'attempts') = 'number' AND (${a}.result->>'attempts')::numeric >= 5))) + OR (${a}.status = 'cancelled' AND ${a}.result->>'retryable' = 'false' + AND ${a}.result->>'code' IN ('github_webhook_ingress_runtime_superseded', 'github_webhook_recovery_source_changed'))`; +} + +export function completedGithubAction(a: string): string { + return `(${a}.kind = 'github_omission_navigation' AND ${a}.status = 'processed' AND ${a}.result IS NULL + AND ${a}.payload->>'version' = '1' AND ${a}.payload->>'preparedTextSha256' ~ '^[a-f0-9]{64}$' + AND ${a}.provider_action_id = 'github-omission-navigation:' || (${a}.payload->>'publicationId') + AND EXISTS (SELECT 1 FROM chat_publications publication JOIN heartbeat_runs run + ON run.id::text = ${a}.payload->>'runId' AND run.company_id = publication.company_id + AND run.native_issue_id = publication.issue_id AND run.runtime_mode = 'native' + JOIN native_run_finalizations finalization ON finalization.run_id = run.id AND finalization.company_id = run.company_id + AND finalization.issue_id = run.native_issue_id AND finalization.result_id::text = ${a}.payload->>'resultId' AND finalization.phase = 'committed' + WHERE publication.id::text = ${a}.payload->>'publicationId' AND publication.company_id = ${a}.company_id + AND publication.endpoint_id = ${a}.endpoint_id AND publication.conversation_id = ${a}.conversation_id + AND run.status = 'succeeded' AND run.finished_at IS NOT NULL AND run.cost_accounting_pending = false + AND ((publication.state = 'published' AND publication.published_at IS NOT NULL) + OR (publication.state IN ('failed', 'cancelled') AND publication.next_attempt_at IS NULL)))) + OR (${a}.kind = 'github_response_comment' AND ${a}.result->>'final' = 'true' + AND (${a}.status = 'processed' OR (${a}.status = 'cancelled' AND ${a}.result->>'code' + IN ('run_already_answered', 'passive_response_wait', 'response_deleted')))) + OR (${a}.kind = 'github_pr_response_comment' AND ${a}.status = 'processed' + AND (${a}.result IS NULL OR (${a}.result->>'final' = 'true' AND length(${a}.result->>'id') > 0 + AND length(${a}.result->>'url') > 0)) + AND EXISTS (SELECT 1 FROM chat_conversations conversation WHERE conversation.id = ${a}.conversation_id + AND conversation.company_id = ${a}.company_id AND conversation.endpoint_id = ${a}.endpoint_id) + AND NOT EXISTS (SELECT 1 FROM chat_actions request WHERE request.kind = 'github_response_comment' + AND request.company_id = ${a}.company_id AND request.endpoint_id = ${a}.endpoint_id + AND request.conversation_id = ${a}.conversation_id AND (request.result->>'final' = 'true') IS NOT TRUE)) + OR (${a}.kind = 'github_webhook_ingress' AND (${completedIngress(a)})) + OR (${a}.kind = 'github_review_check' AND ( + (${a}.status = 'processed' AND (${a}.result->>'state' IN ('manual_required', 'incomplete', 'error') + OR (${a}.result->>'code' = 'assessment_owns_check' AND EXISTS ( + SELECT 1 FROM chat_github_reviews review WHERE review.id::text = ${a}.result->>'reviewId' + AND review.company_id = ${a}.company_id AND review.endpoint_id = ${a}.endpoint_id AND review.assessment IS NOT NULL)))) + OR (${a}.status = 'cancelled' AND ${a}.result->>'code' IN ('connection_changed', 'stale_head') + AND (${a}.result->>'code' <> 'stale_head' OR NOT EXISTS ( + SELECT 1 FROM chat_github_reviews review JOIN chat_deliveries delivery + ON delivery.id::text = review.delivery_id AND delivery.endpoint_id = review.endpoint_id + WHERE review.company_id = ${a}.company_id AND review.endpoint_id = ${a}.endpoint_id + AND review.state IN ('queued', 'running') AND review.assessment IS NULL + AND delivery.normalized_event->'githubAutomatic'->'context'->>'deliveryId' = ${a}.payload->'event'->>'deliveryId'))))) + OR (${a}.kind = 'github_review_publication' AND ( + (${a}.status = 'processed' AND ( + (${a}.payload->>'operation' IN ('comment', 'update_comment', 'formal_review') + AND length(${a}.result->>'id') > 0 AND length(${a}.result->>'url') > 0) + OR (${a}.payload->>'operation' = 'assessment' AND EXISTS (SELECT 1 FROM chat_github_reviews review + WHERE review.id::text = ${a}.payload->>'reviewId' AND review.id::text = ${a}.result->>'reviewId' + AND review.company_id = ${a}.company_id AND review.endpoint_id = ${a}.endpoint_id + AND review.assessment IS NOT NULL AND review.check_url = ${a}.result->>'checkUrl')))) + OR (${a}.status = 'cancelled' AND ${a}.result->>'retryable' = 'false' + AND ${a}.result->>'code' IN ('publication_binding_unavailable', 'stale_head', 'authorization_changed')))) + OR (${a}.kind = 'github_webhook_recovery' AND ( + (${a}.status = 'cancelled' AND ${a}.result->>'code' IN ('not_eligible_for_recovery', 'source_changed_or_unavailable')) + OR (${a}.status = 'processed' AND EXISTS (SELECT 1 FROM chat_actions ingress + WHERE ingress.kind = 'github_webhook_ingress' AND ingress.company_id = ${a}.company_id + AND ingress.endpoint_id = ${a}.endpoint_id AND ingress.payload->>'deliveryId' = ${a}.payload->'original'->>'guid' + AND (${completedIngress('ingress')})))))`; +} diff --git a/server/src/services/chat-channels/idle-history-proof.ts b/server/src/services/chat-channels/idle-history-proof.ts new file mode 100644 index 00000000000..992c71d8645 --- /dev/null +++ b/server/src/services/chat-channels/idle-history-proof.ts @@ -0,0 +1,129 @@ +import { finishedIdleRun, finishedIdleWake } from "../idle-work-proof.js"; +import { completedGithubAction } from "./github/idle-history-proof.js"; +import { completedSlackAction } from "./slack/idle-history-proof.js"; +import { completedExpiredChatControl, completedQuestionFormAction } from "../chat-question-form-idle-proof.js"; +import { completedPhotonAction } from "./imessage-photon/idle-history-proof.js"; +import { completedDiscordAction } from "./discord/idle-history-proof.js"; +import { completedVoiceAction } from "../voice/idle-history-proof.js"; +import { completedTeamsAction } from "./microsoft-teams/idle-history-proof.js"; +import { completedTelegramAction } from "./telegram/idle-history-proof.js"; + +/** A processed inbound receipt can still own an unconsumed wake. */ +function completedAction(a: string): string { + const wake = `EXISTS (SELECT 1 FROM agent_wakeup_requests w WHERE w.company_id = ${a}.company_id + AND w.id = ${a}.id AND w.agent_id::text = ${a}.payload->>'agentId' + AND w.payload->>'issueId' = ${a}.payload->>'issueId' AND ${finishedIdleWake('w')})`; + return `jsonb_typeof(${a}.payload) = 'object' + AND EXISTS (SELECT 1 FROM chat_endpoints e WHERE e.id = ${a}.endpoint_id AND e.company_id = ${a}.company_id) + AND (${a}.delivery_id IS NULL OR EXISTS (SELECT 1 FROM chat_deliveries d + WHERE d.id = ${a}.delivery_id AND d.company_id = ${a}.company_id AND d.endpoint_id = ${a}.endpoint_id)) + AND CASE + WHEN ${a}.kind = 'inbound_wakeup' THEN (${a}.status IN ('processed', 'failed') AND ${wake}) + OR (${a}.status = 'failed' AND ${a}.result->>'code' = 'inbound_wakeup_delivery_rejected' + AND EXISTS (SELECT 1 FROM chat_deliveries rejected WHERE rejected.id = ${a}.delivery_id + AND rejected.company_id = ${a}.company_id AND rejected.endpoint_id = ${a}.endpoint_id + AND rejected.state IN ('filtered', 'failed') AND rejected.next_attempt_at IS NULL)) + WHEN ${a}.kind IN ('question_form_open', 'question_form_submit', 'question_form_open_attempt', 'question_answer') + THEN (${completedQuestionFormAction(a)}) + WHEN ${a}.kind = 'interaction_wakeup' THEN ${a}.status = 'processed' AND + (${a}.result->>'code' = 'interaction_wakeup_no_longer_applicable' OR EXISTS ( + SELECT 1 FROM agent_wakeup_requests w WHERE w.company_id = ${a}.company_id + AND w.idempotency_key = 'interaction:' || (${a}.payload->>'interactionId') || ':' || (${a}.payload->>'interactionStatus') + AND ${finishedIdleWake('w')})) + WHEN ${a}.kind = 'failed_run_retry' THEN ${a}.status = 'processed' + AND ${a}.result->>'code' = 'failed_chat_retry_durable' AND ${wake} + WHEN ${a}.kind = 'provider_effect' THEN + (${a}.status = 'processed' AND length(${a}.result->>'providerMessageId') > 0 AND length(${a}.result->>'threadId') > 0) + OR (${a}.status = 'cancelled' AND ${a}.result->>'code' = 'provider_effect_no_longer_authorized') + OR (${a}.status = 'failed' AND ${a}.result->>'code' = 'provider_effect_failed' AND ${a}.result->>'retryable' = 'false') + WHEN ${a}.kind = 'receipt_reaction' THEN + (${a}.status = 'processed' AND jsonb_typeof(${a}.result->'attempts') = 'number' AND (${a}.result->>'attempts')::numeric > 0) + OR (${a}.status = 'cancelled' AND ${a}.result->>'code' IN ('receipt_reaction_superseded_by_terminal_publication', 'receipt_reaction_runtime_superseded')) + WHEN ${a}.kind = 'confirmation_response' THEN (${completedExpiredChatControl(a)}) OR (${a}.status = 'processed' + AND ${a}.result->>'code' IN ('confirmation_resolution_committed_by_provider', 'confirmation_resolution_reconciled') + AND EXISTS (SELECT 1 FROM issue_thread_interactions interaction WHERE interaction.id::text = ${a}.result->>'interactionId' + AND interaction.company_id = ${a}.company_id AND interaction.status = ${a}.result->>'interactionStatus' + AND interaction.resolved_at IS NOT NULL)) + WHEN ${a}.kind = 'board_send_rejected' THEN ${a}.status = 'processed' + AND ${a}.result->>'code' = 'chat_board_send_attachments_already_bound' + AND ${a}.payload->>'schemaVersion' = '1' AND length(${a}.payload->>'requestedByUserId') > 0 + AND jsonb_typeof(${a}.payload->'attachmentIds') = 'array' AND jsonb_array_length(CASE WHEN jsonb_typeof(${a}.payload->'attachmentIds') = 'array' THEN ${a}.payload->'attachmentIds' ELSE '[]'::jsonb END) BETWEEN 1 AND 20 + AND NOT EXISTS (SELECT 1 FROM jsonb_array_elements_text(CASE WHEN jsonb_typeof(${a}.payload->'attachmentIds') = 'array' THEN ${a}.payload->'attachmentIds' ELSE '[]'::jsonb END) rejected_id(value) + WHERE rejected_id.value !~ '^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$') + AND EXISTS (SELECT 1 FROM chat_conversations conversation WHERE conversation.id = ${a}.conversation_id + AND conversation.company_id = ${a}.company_id AND conversation.endpoint_id = ${a}.endpoint_id + AND conversation.issue_id::text = ${a}.payload->>'issueId' + AND ${a}.provider_action_id = 'board-send-rejected:' || conversation.id::text || ':' || (${a}.payload->>'idempotencyKey')) + WHEN ${a}.kind = 'task_control_authorization' THEN ${a}.status IN ('processed', 'cancelled') + AND EXISTS (SELECT 1 FROM chat_publications p WHERE p.company_id = ${a}.company_id AND p.endpoint_id = ${a}.endpoint_id + AND p.conversation_id = ${a}.conversation_id AND ${a}.provider_action_id = 'task-control-authorization:' || p.id::text + AND ((p.state = 'published' AND p.published_at IS NOT NULL) OR p.state = 'cancelled')) + ELSE (${[completedGithubAction, completedSlackAction, completedTelegramAction, completedTeamsAction, completedPhotonAction, completedDiscordAction, completedVoiceAction] + .map(proof => `(${proof(a)})`).join(' OR ')}) END`; +} + +export const idleChatHistoryChecks = [ + { reason: "chat_deliveries", query: `WITH candidate_deliveries AS MATERIALIZED ( + SELECT d.id, d.company_id, d.endpoint_id, d.state, d.event_kind FROM chat_deliveries d WHERE ( + EXISTS (SELECT 1 FROM chat_endpoints e WHERE e.id = d.endpoint_id AND e.company_id = d.company_id) + AND jsonb_typeof(d.normalized_event) = 'object' + AND d.normalized_event->>'providerEventId' = d.provider_event_id + AND d.normalized_event->>'kind' = d.event_kind + AND d.event_kind IN ('mention', 'message', 'direct_message', 'message_updated', 'message_deleted', 'message_restored', + 'reaction_added', 'reaction_removed', 'action', 'modal_submitted', 'modal_closed', 'slash_command', 'file_shared', 'installation', 'uninstallation', 'unknown') + AND d.next_attempt_at IS NULL AND (d.state = 'failed' OR (d.state IN ('processed', 'filtered') AND d.processed_at IS NOT NULL)) + AND (d.conversation_id IS NULL OR EXISTS (SELECT 1 FROM chat_conversations c WHERE c.id = d.conversation_id + AND c.company_id = d.company_id AND c.endpoint_id = d.endpoint_id)) +) IS TRUE + ), action_outcomes AS MATERIALIZED ( + SELECT a.delivery_id, a.kind, (${completedAction('a')}) IS TRUE AS completed + FROM chat_actions a JOIN candidate_deliveries candidate ON candidate.id = a.delivery_id + ), action_receipts AS MATERIALIZED ( + SELECT delivery_id, bool_and(completed) AS completed, + bool_or(kind = 'inbound_wakeup' AND completed) AS consumed + FROM action_outcomes GROUP BY delivery_id + ) SELECT 1 FROM chat_deliveries d WHERE ( + EXISTS (SELECT 1 FROM chat_endpoints e WHERE e.id = d.endpoint_id AND e.company_id = d.company_id) + AND jsonb_typeof(d.normalized_event) = 'object' + AND d.normalized_event->>'providerEventId' = d.provider_event_id + AND d.normalized_event->>'kind' = d.event_kind + AND d.event_kind IN ('mention', 'message', 'direct_message', 'message_updated', 'message_deleted', 'message_restored', + 'reaction_added', 'reaction_removed', 'action', 'modal_submitted', 'modal_closed', 'slash_command', 'file_shared', 'installation', 'uninstallation', 'unknown') + AND d.next_attempt_at IS NULL AND (d.state = 'failed' OR (d.state IN ('processed', 'filtered') AND d.processed_at IS NOT NULL)) + AND (d.conversation_id IS NULL OR EXISTS (SELECT 1 FROM chat_conversations c WHERE c.id = d.conversation_id + AND c.company_id = d.company_id AND c.endpoint_id = d.endpoint_id)) +) IS NOT TRUE + UNION ALL SELECT 1 FROM candidate_deliveries d + WHERE d.id IN (SELECT receipt.delivery_id FROM action_receipts receipt WHERE NOT receipt.completed) + OR (d.state = 'processed' AND d.event_kind IN ('mention', 'message', 'direct_message') + AND d.id NOT IN (SELECT receipt.delivery_id FROM action_receipts receipt WHERE receipt.consumed))` }, + { reason: "chat_publications", query: `SELECT 1 FROM chat_publications p WHERE ( + (p.state = 'published' AND p.published_at IS NOT NULL AND p.redacted_error IS NULL + OR (p.state IN ('failed', 'cancelled') AND p.next_attempt_at IS NULL)) + AND EXISTS (SELECT 1 FROM chat_conversations c JOIN chat_endpoints e + ON e.id = c.endpoint_id AND e.company_id = c.company_id + JOIN issues i ON i.id = p.issue_id AND i.company_id = c.company_id + WHERE c.id = p.conversation_id AND c.company_id = p.company_id AND c.endpoint_id = p.endpoint_id) + AND NOT EXISTS (SELECT 1 FROM chat_endpoint_leases l WHERE l.company_id = p.company_id + AND l.endpoint_id = p.endpoint_id AND l.lease_key LIKE 'publication:' || p.id::text || ':%' AND l.expires_at > now()) + AND NOT EXISTS (SELECT 1 FROM chat_teams_file_transfers f WHERE f.publication_id = p.id + AND (f.company_id <> p.company_id OR f.phase NOT IN ('delivered', 'declined', 'expired', 'cancelled') + OR f.attempt_id IS NOT NULL OR f.attempt_expires_at IS NOT NULL))) IS NOT TRUE` }, + { reason: "chat_actions", query: `SELECT 1 FROM chat_actions a WHERE (${completedAction('a')}) IS NOT TRUE` }, + { reason: "chat_completion_deliveries", query: `SELECT 1 FROM chat_completion_deliveries d WHERE ( + d.status IN ('delivered', 'superseded', 'exhausted') + AND EXISTS (SELECT 1 FROM chat_task_handoffs h JOIN issues task ON task.id = h.task_id AND task.company_id = h.company_id + JOIN issues source ON source.id = h.conversation_id AND source.company_id = h.company_id + WHERE h.task_id = d.task_id AND h.company_id = d.company_id + AND source.conversation_agent_id = h.agent_id + AND (d.status <> 'delivered' OR (d.status_version <= task.status_version + AND EXISTS (SELECT 1 FROM issue_comments c WHERE c.id = d.response_comment_id + AND c.company_id = d.company_id AND c.issue_id = h.conversation_id AND c.author_agent_id = h.agent_id + AND c.created_by_run_id = d.target_run_id)))) + AND (d.target_run_id IS NULL AND d.status <> 'delivered' OR EXISTS ( + SELECT 1 FROM heartbeat_runs r JOIN chat_task_handoffs h ON h.task_id = d.task_id + WHERE r.id = d.target_run_id AND r.company_id = d.company_id AND r.agent_id = h.agent_id + AND h.company_id = d.company_id AND r.context_snapshot->>'conversationSessionGeneration' = h.session_generation::text + AND (r.native_issue_id = h.conversation_id OR r.context_snapshot->>'issueId' = h.conversation_id::text) + AND ${finishedIdleRun('r')}))) IS NOT TRUE` }, +] as const; diff --git a/server/src/services/chat-channels/imessage-photon/events.ts b/server/src/services/chat-channels/imessage-photon/events.ts index 75f6d2cc92b..882a57e7a7a 100644 --- a/server/src/services/chat-channels/imessage-photon/events.ts +++ b/server/src/services/chat-channels/imessage-photon/events.ts @@ -1,11 +1,10 @@ +import { stagePhotonResponseNotice } from "./response-notices.js"; import { notifyChatReceiver } from "../runtime/receiver-work-notifications.js"; import type { EndpointRow, DbTransaction, RuntimeContext, LifecycleRuntimeFence } from "../../chat-channels/runtime-types.js"; -import { notifyChatPublicationWork } from "../../chat-work-notifications.js"; import { nativeSha256 } from "../../native-runtime/canonical.js"; -import { projectSafeChatPublicationText } from "../../chat-publication-projection.js"; import { PhotonAnswerValidationError, nativePhotonInteraction, photonResponseCommand, parsePhotonQuestionAnswer, type PhotonPromptReceipt, type PhotonInteractionBinding, type PhotonDraft } from "../../photon/interactions.js"; import { validateNativeQuestionResponseInput } from "../../native-runtime/native-question-bridge.js"; import type { AskUserQuestionsAnswer, AskUserQuestionsInteraction, IssueThreadInteraction } from "@paperclipai/shared"; @@ -368,41 +367,9 @@ export function createPhotonEvents(db: Db, options: PhotonEventsOptions) { } const enqueueResponse = async (key: string, text: string) => { await assertCurrent(); - await db.transaction(async (tx) => { - await tx - .insert(chatActions) - .values({ - companyId: endpoint.companyId, - endpointId: endpoint.id, - conversationId: conversation.id, - kind: "photon_response_notice", - providerActionId: `photon-notice:${key}`, - payload: { - reference, - questionIndex, - publicationId: binding.publicationId, - }, - status: "processed", - }) - .onConflictDoNothing(); - await notifyChatPublicationWork(tx); - await tx - .insert(chatPublications) - .values({ - companyId: endpoint.companyId, - endpointId: endpoint.id, - conversationId: conversation.id, - issueId: conversation.issueId, - idempotencyKey: key, - payload: { - ...publication.payload, - card: undefined, - text: projectSafeChatPublicationText(text), - }, - state: "pending", - }) - .onConflictDoNothing(); - }); + await db.transaction(tx => stagePhotonResponseNotice(tx, { + publication, reference, questionIndex, key, text, + })); scheduleMessageProcessing(() => processPendingPublications().then(() => undefined), ); diff --git a/server/src/services/chat-channels/imessage-photon/idle-history-proof.ts b/server/src/services/chat-channels/imessage-photon/idle-history-proof.ts new file mode 100644 index 00000000000..2e10387d722 --- /dev/null +++ b/server/src/services/chat-channels/imessage-photon/idle-history-proof.ts @@ -0,0 +1,58 @@ +/** Photon tokens and immutable provenance receipts do not own a turn once their + * exact canonical owner has settled. Receiver ownership remains a separate gate. */ +export function completedPhotonAction(a: string): string { + const publication = (id: string) => `EXISTS (SELECT 1 FROM chat_publications publication + WHERE publication.id::text = ${id} AND publication.company_id = ${a}.company_id + AND publication.endpoint_id = ${a}.endpoint_id AND publication.conversation_id = ${a}.conversation_id + AND ((publication.state = 'published' AND publication.published_at IS NOT NULL) + OR (publication.state IN ('failed', 'cancelled') AND publication.next_attempt_at IS NULL)))`; + return `(${a}.kind = 'photon_interaction' AND (${a}.status IN ('issued', 'processed') OR (${a}.status = 'expired' AND ${a}.result->>'code' = 'interaction_resolved_elsewhere')) + AND ${a}.payload->>'version' = '1' AND length(${a}.payload->>'reference') > 0 + AND ${a}.provider_action_id = 'photon:' || (${a}.payload->>'reference') + AND EXISTS (SELECT 1 FROM issue_thread_interactions interaction JOIN chat_publications publication + ON publication.id::text = ${a}.payload->>'publicationId' AND publication.company_id = interaction.company_id + AND publication.issue_id = interaction.issue_id AND publication.payload->>'interactionId' = interaction.id::text + JOIN chat_conversations conversation ON conversation.id = publication.conversation_id + AND conversation.company_id = interaction.company_id AND conversation.issue_id = interaction.issue_id + WHERE interaction.id::text = ${a}.payload->>'interactionId' AND interaction.company_id = ${a}.company_id + AND interaction.kind IN ('ask_user_questions', 'request_confirmation') + AND interaction.status IN ('answered', 'accepted', 'rejected', 'cancelled', 'expired') AND interaction.resolved_at IS NOT NULL + AND publication.endpoint_id = ${a}.endpoint_id AND conversation.id = ${a}.conversation_id + AND conversation.session_generation::text = ${a}.payload->>'sessionGeneration')) + OR (${a}.kind = 'photon_publication_retry' AND ${a}.status = 'processed' + AND length(${a}.result->>'authorizedByUserId') > 0 AND ${a}.payload->>'attempt' ~ '^[1-9][0-9]*$' + AND ${a}.provider_action_id = 'photon-retry:' || (${a}.payload->>'publicationId') || ':' || (${a}.payload->>'attempt') + AND ${publication(`${a}.payload->>'publicationId'`)}) + OR (${a}.kind = 'photon_response_notice' AND ${a}.status = 'processed' AND ${a}.result IS NULL + AND length(${a}.payload->>'reference') > 0 AND ${a}.payload->>'questionIndex' ~ '^[0-9]+$' + AND EXISTS (SELECT 1 FROM chat_publications notice WHERE notice.company_id = ${a}.company_id + AND notice.endpoint_id = ${a}.endpoint_id AND notice.conversation_id = ${a}.conversation_id + AND ${a}.provider_action_id = 'photon-notice:' || notice.idempotency_key + AND ((notice.state = 'published' AND notice.published_at IS NOT NULL) + OR (notice.state IN ('failed', 'cancelled') AND notice.next_attempt_at IS NULL))) + AND EXISTS (SELECT 1 FROM chat_publications source WHERE source.id::text = ${a}.payload->>'publicationId' + AND source.company_id = ${a}.company_id AND source.endpoint_id = ${a}.endpoint_id + AND source.conversation_id = ${a}.conversation_id)) + OR (${a}.kind IN ('attachment_derivative', 'attachment_companion') AND ${a}.status = 'processed' AND ${a}.result IS NULL + AND EXISTS (SELECT 1 FROM chat_endpoints endpoint WHERE endpoint.id = ${a}.endpoint_id + AND endpoint.company_id = ${a}.company_id AND endpoint.provider = 'imessage-photon') + AND ${a}.delivery_id IS NOT NULL + AND ((${a}.kind = 'attachment_derivative' AND ${a}.payload->>'kind' = 'heif_jpeg_preview') + OR (${a}.kind = 'attachment_companion' AND ${a}.payload->>'kind' = 'live_photo_video')) + AND ${a}.payload->>'originalAttachmentId' ~ '^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$' + AND ${a}.payload->>'derivativeAttachmentId' ~ '^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$' + AND ${a}.payload->>'originalAttachmentId' <> ${a}.payload->>'derivativeAttachmentId' + AND ${a}.payload->>'originalSha256' ~ '^[a-f0-9]{64}$' AND ${a}.payload->>'derivativeSha256' ~ '^[a-f0-9]{64}$' + AND ${a}.provider_action_id = 'photon-' || (${a}.payload->>'kind') || ':' || ${a}.delivery_id::text || ':' || (${a}.payload->>'originalAttachmentId') + AND NOT EXISTS (SELECT 1 FROM issue_attachments attachment WHERE attachment.id::text IN + (${a}.payload->>'originalAttachmentId', ${a}.payload->>'derivativeAttachmentId') + AND (attachment.company_id <> ${a}.company_id OR NOT EXISTS (SELECT 1 FROM assets asset + WHERE asset.id = attachment.asset_id AND asset.company_id = ${a}.company_id AND asset.sha256 = CASE + WHEN attachment.id::text = ${a}.payload->>'originalAttachmentId' THEN ${a}.payload->>'originalSha256' + ELSE ${a}.payload->>'derivativeSha256' END))) + AND NOT EXISTS (SELECT 1 FROM issue_attachments original JOIN issue_attachments derivative + ON derivative.id::text = ${a}.payload->>'derivativeAttachmentId' + WHERE original.id::text = ${a}.payload->>'originalAttachmentId' + AND (original.company_id <> derivative.company_id OR original.issue_id <> derivative.issue_id + OR original.issue_comment_id IS DISTINCT FROM derivative.issue_comment_id)))`; +} diff --git a/server/src/services/chat-channels/imessage-photon/response-notices.ts b/server/src/services/chat-channels/imessage-photon/response-notices.ts new file mode 100644 index 00000000000..18a7c65b3aa --- /dev/null +++ b/server/src/services/chat-channels/imessage-photon/response-notices.ts @@ -0,0 +1,27 @@ +import { chatActions, chatPublications, type Db } from "@paperclipai/db"; +import { notifyChatPublicationWork } from "../../chat-work-notifications.js"; +import { projectSafeChatPublicationText } from "../../chat-publication-projection.js"; + +type Tx = Parameters[0]>[0]; + +/** Keep the notice marker and its outbox row in the same owner transaction. */ +export async function stagePhotonResponseNotice(tx: Tx, input: { + publication: typeof chatPublications.$inferSelect; + reference: string; + questionIndex: number; + key: string; + text: string; +}) { + const { publication, reference, questionIndex, key, text } = input; + await tx.insert(chatActions).values({ + companyId: publication.companyId, endpointId: publication.endpointId, conversationId: publication.conversationId, + kind: "photon_response_notice", providerActionId: `photon-notice:${key}`, + payload: { reference, questionIndex, publicationId: publication.id }, status: "processed", + }).onConflictDoNothing(); + await notifyChatPublicationWork(tx); + await tx.insert(chatPublications).values({ + companyId: publication.companyId, endpointId: publication.endpointId, conversationId: publication.conversationId, + issueId: publication.issueId, idempotencyKey: key, + payload: { ...publication.payload, card: undefined, text: projectSafeChatPublicationText(text) }, state: "pending", + }).onConflictDoNothing(); +} diff --git a/server/src/services/chat-channels/microsoft-teams/idle-history-proof.ts b/server/src/services/chat-channels/microsoft-teams/idle-history-proof.ts new file mode 100644 index 00000000000..9d90d550623 --- /dev/null +++ b/server/src/services/chat-channels/microsoft-teams/idle-history-proof.ts @@ -0,0 +1,35 @@ +/** Teams keeps immutable provider-effect and recipient-intent receipts after + * delivery. Only the exact closed transfer/publication retires that authority. */ +export function completedTeamsAction(a: string): string { + return `${a}.status = 'processed' AND ${a}.kind IN ('teams_file_effect_intent', 'teams_file_consent', 'teams_board_file_intent') + AND EXISTS (SELECT 1 FROM chat_teams_file_transfers transfer JOIN chat_publications publication + ON publication.id = transfer.publication_id AND publication.company_id = transfer.company_id + WHERE transfer.company_id = ${a}.company_id AND transfer.endpoint_id = ${a}.endpoint_id + AND transfer.conversation_id = ${a}.conversation_id + AND publication.id::text = ${a}.payload->>'publicationId' + AND publication.endpoint_id = ${a}.endpoint_id AND publication.conversation_id = ${a}.conversation_id + AND transfer.phase IN ('delivered', 'declined', 'expired', 'cancelled') + AND transfer.attempt_id IS NULL AND transfer.attempt_expires_at IS NULL + AND ((publication.state = 'published' AND publication.published_at IS NOT NULL) + OR (publication.state IN ('failed', 'cancelled') AND publication.next_attempt_at IS NULL)) + AND CASE WHEN ${a}.kind = 'teams_file_effect_intent' THEN + ${a}.payload->>'schema' = 'paperclip.teams.file-effect-intent.v1' + AND ${a}.payload->>'transferId' = transfer.id::text + AND ${a}.provider_action_id = 'teams-file-effect:' || (${a}.payload->>'attemptId') + AND ${a}.payload->>'stage' IN ('consent', 'upload', 'file_info') + WHEN ${a}.kind = 'teams_file_consent' THEN + ${a}.payload->>'schema' = 'paperclip.teams.consent-receipt.v1' + AND ${a}.payload->>'transferId' = transfer.id::text + AND ${a}.payload->>'receiptDigest' = transfer.response_digest + AND ${a}.payload->>'action' IN ('accept', 'decline') + AND ${a}.principal_id IS NOT DISTINCT FROM transfer.principal_id + ELSE ${a}.payload->>'schema' = 'paperclip.teams.board-file-intent.v1' + AND ${a}.provider_action_id = 'teams_board_file_intent:' || publication.id::text + AND ${a}.payload->>'issueId' = transfer.issue_id::text + AND ${a}.payload->>'attachmentId' = transfer.attachment_id::text + AND ${a}.payload->>'commentId' = publication.comment_id::text + AND ${a}.payload->>'sourceScopeSha256' ~ '^[a-f0-9]{64}$' + AND ${a}.payload->>'recipientBindingSha256' ~ '^[a-f0-9]{64}$' + AND ${a}.payload->>'byteIdentitySha256' ~ '^[a-f0-9]{64}$' + AND ${a}.payload->>'commentBodySha256' ~ '^[a-f0-9]{64}$' END)`; +} diff --git a/server/src/services/chat-channels/slack/idle-history-proof.ts b/server/src/services/chat-channels/slack/idle-history-proof.ts new file mode 100644 index 00000000000..1db0169861a --- /dev/null +++ b/server/src/services/chat-channels/slack/idle-history-proof.ts @@ -0,0 +1,77 @@ +/** Connector rows below are immutable authority/confirmed-call receipts. Their + * own downstream publication, interaction and run rows keep independent gates. */ +function completedSlackConnectorAction(a: string): string { + const principal = `EXISTS (SELECT 1 FROM chat_external_principals principal WHERE principal.id = ${a}.principal_id + AND principal.company_id = ${a}.company_id AND principal.provider = 'slack')`; + const issue = `EXISTS (SELECT 1 FROM issues issue WHERE issue.id::text = ${a}.payload->>'issueId' + AND issue.company_id = ${a}.company_id AND (${a}.conversation_id IS NULL OR EXISTS ( + SELECT 1 FROM chat_conversations conversation WHERE conversation.id = ${a}.conversation_id + AND conversation.company_id = issue.company_id AND conversation.issue_id = issue.id + AND conversation.endpoint_id = ${a}.endpoint_id)))`; + return `(${a}.kind = 'slack_private_source' AND ${a}.status = 'processed' AND ${a}.result IS NULL + AND ${issue} AND ${principal} AND length(${a}.payload->>'channelId') > 0 + AND length(${a}.payload->>'requesterId') > 0 + AND ${a}.provider_action_id = 'slack-private:' || (${a}.payload->>'issueId') || ':' + || (${a}.payload->>'channelId') || ':' || (${a}.payload->>'requesterId')) + OR (${a}.kind = 'slack_connect' AND ${a}.status = 'processed' + AND ${a}.payload->>'version' = '1' AND ${a}.result->>'code' = 'slack_identity_discovered' + AND length(${a}.payload->>'channelId') > 0 AND ${principal} + AND EXISTS (SELECT 1 FROM chat_external_principals principal WHERE principal.id = ${a}.principal_id + AND principal.external_id = ${a}.payload->>'userId')) + OR (${a}.kind IN ('slack_test_message', 'slack_test_message_invitation') AND ${a}.status = 'processed' + AND length(${a}.payload->>'userId') > 0 AND ${a}.payload->>'slackUserId' ~ '^[UW][A-Z0-9]+$' + AND CASE WHEN ${a}.kind = 'slack_test_message' THEN ${a}.result->>'slackUrl' ~ '^https://app[.]slack[.]com/client/[^/]+/D[A-Z0-9]+$' + ELSE ${principal} AND ${a}.result->>'expiresAt' ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}T' END) + OR (${a}.kind = 'slack_managed_reconnect' AND ${a}.status = 'processed' AND ${a}.result IS NULL + AND ${a}.provider_action_id LIKE 'slack_managed_reconnect:%' + AND length(${a}.payload->>'registrationRequestId') > 0 + AND EXISTS (SELECT 1 FROM chat_slack_manager_grants grant_record WHERE grant_record.id::text = ${a}.payload->>'grantId' + AND grant_record.company_id = ${a}.company_id)) + OR (${a}.kind = 'slack_tool_write' AND ${a}.status = 'processed' AND jsonb_typeof(${a}.result) = 'object' + AND ${a}.payload->>'name' IN ('slack_open_dm', 'slack_post_message', 'slack_update_message', 'slack_delete_message', + 'slack_add_reaction', 'slack_remove_reaction', 'slack_add_pin', 'slack_remove_pin', 'slack_add_bookmark', + 'slack_remove_bookmark', 'slack_create_canvas', 'slack_edit_canvas', 'slack_replace_canvas_section', + 'slack_share_list', 'slack_edit_list', 'slack_create_list', 'slack_create_list_item', 'slack_update_list_item', + 'slack_invite', 'slack_create_channel', 'slack_upload_file') + AND ${a}.payload->>'hash' ~ '^[a-f0-9]{64}$' AND jsonb_typeof(${a}.payload->'args') = 'object' + AND length(${a}.payload->>'userId') > 0 AND length(${a}.payload->>'revision') > 0 + AND ${a}.provider_action_id = 'slack-tool:' || (${a}.payload->'binding'->>'issueId') || ':' || (${a}.payload->'args'->>'idempotencyKey') + AND EXISTS (SELECT 1 FROM tool_invocations invocation JOIN heartbeat_runs run ON run.id = invocation.run_id + AND run.company_id = invocation.company_id AND run.agent_id = invocation.agent_id + WHERE invocation.id::text = ${a}.payload->>'invocationId' AND invocation.company_id = ${a}.company_id + AND invocation.run_id::text = ${a}.payload->'binding'->>'runId' + AND invocation.agent_id::text = ${a}.payload->'binding'->>'agentId' + AND invocation.issue_id::text = ${a}.payload->'binding'->>'issueId' + AND ${a}.payload->'binding'->>'companyId' = ${a}.company_id::text)) + OR (${a}.kind = 'slack_board_message' AND ${issue} AND EXISTS ( + SELECT 1 FROM issue_comments comment JOIN chat_publications publication + ON publication.comment_id = comment.id AND publication.company_id = comment.company_id + WHERE comment.id::text = ${a}.payload->>'commentId' AND comment.company_id = ${a}.company_id + AND comment.issue_id::text = ${a}.payload->>'issueId' AND comment.author_user_id = ${a}.payload->>'userId' + AND publication.endpoint_id = ${a}.endpoint_id AND publication.conversation_id = ${a}.conversation_id + AND ((publication.state = 'published' AND publication.published_at IS NOT NULL) + OR (publication.state IN ('failed', 'cancelled') AND publication.next_attempt_at IS NULL))) + AND ((${a}.status = 'cancelled' AND ${a}.result->>'code' = 'slack_board_work_not_authorized') + OR (${a}.status = 'processed' AND (${a}.result IS NULL OR (${a}.result->>'resumeApplied' = 'true' + AND EXISTS (SELECT 1 FROM agent_wakeup_requests wake WHERE wake.company_id = ${a}.company_id + AND wake.idempotency_key = 'slack-board-comment:' || ${a}.id::text AND wake.finished_at IS NOT NULL + AND wake.status IN ('completed', 'failed', 'cancelled', 'skipped', 'timed_out')))))))`; +} + +export function completedSlackAction(a: string): string { + return `(${a}.kind = 'slack_session_sync' AND ( + (${a}.status = 'processed' AND ${a}.result->>'outcome' IN ('updated', 'unsupported') + AND (${a}.result->>'outcome' <> 'updated' OR ${a}.result->>'sessionStatus' <> 'processing')) + OR (${a}.status = 'cancelled' AND ${a}.result->>'code' IN ('slack_session_no_published_state', 'slack_session_destination_changed')) + OR (${a}.status = 'failed' AND ${a}.result->>'code' = 'slack_session_sync_rejected'))) + OR (${a}.kind = 'slash_task_start' AND ${a}.status = 'processed' + AND length(${a}.result->>'providerMessageId') > 0 AND length(${a}.result->>'threadId') > 0) + OR (${a}.kind = 'slack_file_upload_receipt' AND ${a}.status = 'processed' + AND ${a}.result->>'code' = 'slack_file_upload_identity_confirmed') + OR (${a}.kind = 'slack_session_stop' AND ${a}.status IN ('processed', 'cancelled') + AND ${a}.result->>'code' IN ('slack_session_stop_cancelled', 'slack_session_stop_target_superseded', 'slack_session_stop_no_current_work') + AND NOT EXISTS (SELECT 1 FROM heartbeat_runs r WHERE r.id::text = ${a}.result->>'runId' + AND (r.company_id <> ${a}.company_id OR r.status NOT IN ('succeeded', 'failed', 'cancelled', 'timed_out', 'interrupted') + OR r.finished_at IS NULL OR r.cost_accounting_pending))) + OR (${completedSlackConnectorAction(a)})`; +} diff --git a/server/src/services/chat-channels/telegram/idle-history-proof.ts b/server/src/services/chat-channels/telegram/idle-history-proof.ts new file mode 100644 index 00000000000..87dea9045e1 --- /dev/null +++ b/server/src/services/chat-channels/telegram/idle-history-proof.ts @@ -0,0 +1,20 @@ +export function completedTelegramAction(a: string): string { + return `(${a}.kind = 'telegram_stop_subscription' AND ${a}.status = 'processed' + AND ${a}.result->>'code' = 'telegram_stop_subscription_confirmed' AND ${a}.payload->>'version' = '1' + AND ${a}.payload->>'botUserId' ~ '^[1-9][0-9]{0,15}$' + AND ${a}.payload->>'credentialFingerprint' ~ '^[a-f0-9]{64}$' + AND ${a}.payload->>'webhookUrlSha256' ~ '^[a-f0-9]{64}$' + AND ${a}.payload->>'runtimeGeneration' ~ '^[0-9]+$' + AND ${a}.provider_action_id = 'telegram-stop-subscription:' || (${a}.payload->>'runtimeGeneration') || ':' + || (${a}.payload->>'credentialFingerprint') || ':' || (${a}.payload->>'webhookUrlSha256')) + OR (${a}.kind = 'telegram_maintenance' AND ( + (${a}.status = 'processed' AND (${a}.result->>'providerConfirmed' = 'true' + OR ${a}.result->>'code' = 'telegram_stop_subscription_confirmed')) + OR (${a}.status = 'cancelled' AND ${a}.result->>'code' = 'telegram_maintenance_credentials_superseded') + OR (${a}.status = 'failed' AND ${a}.result->>'retryable' = 'false' + AND ${a}.result->>'code' = 'telegram_maintenance_failed'))) + OR (${a}.kind = 'telegram_draft' AND ( + (${a}.status = 'processed' AND ${a}.result->>'phase' = 'published' AND length(${a}.result->>'providerMessageId') > 0) + OR (${a}.status = 'cancelled' AND (${a}.result->>'phase' = 'stopped' + OR ${a}.result->>'code' = 'telegram_draft_presentation_stopped'))))`; +} diff --git a/server/src/services/chat-question-form-idle-proof.ts b/server/src/services/chat-question-form-idle-proof.ts new file mode 100644 index 00000000000..b82cbb36c82 --- /dev/null +++ b/server/src/services/chat-question-form-idle-proof.ts @@ -0,0 +1,54 @@ +/** Form-open tokens stay issued: resolving the canonical question revokes their + * authority. A processed submission additionally needs its exact answer receipt. */ +export function completedQuestionFormAction(a: string): string { + const resolved = `EXISTS (SELECT 1 FROM issue_thread_interactions interaction + JOIN chat_conversations conversation ON conversation.issue_id = interaction.issue_id AND conversation.company_id = interaction.company_id + JOIN chat_publications publication ON publication.id::text = ${a}.payload->>'publicationId' + AND publication.company_id = interaction.company_id AND publication.issue_id = interaction.issue_id + AND publication.conversation_id = conversation.id AND publication.endpoint_id = conversation.endpoint_id + WHERE interaction.id::text = ${a}.payload->>'interactionId' AND interaction.company_id = ${a}.company_id + AND (interaction.kind = 'ask_user_questions' OR (interaction.kind = 'request_item_verdicts' AND EXISTS ( + SELECT 1 FROM chat_endpoints endpoint WHERE endpoint.id = ${a}.endpoint_id AND endpoint.company_id = ${a}.company_id AND endpoint.provider = 'slack'))) AND interaction.status IN ('answered', 'cancelled', 'expired') + AND interaction.resolved_at IS NOT NULL AND conversation.id = ${a}.conversation_id + AND conversation.endpoint_id = ${a}.endpoint_id AND publication.payload->>'interactionId' = interaction.id::text)`; + return `(${a}.kind IN ('question_form_open', 'question_form_submit') AND ${a}.payload->>'version' = '1' + AND ${resolved} AND ( + (${a}.kind = 'question_form_open' AND (${a}.status = 'issued' OR (${a}.status = 'expired' + AND ${a}.result->>'code' IN ('question_form_submitted', 'interaction_resolved_elsewhere'))) AND EXISTS ( + SELECT 1 FROM chat_actions submit WHERE submit.company_id = ${a}.company_id AND submit.endpoint_id = ${a}.endpoint_id + AND submit.conversation_id = ${a}.conversation_id AND submit.kind = 'question_form_submit' + AND submit.provider_action_id = ${a}.payload->>'formActionId' + AND submit.payload->>'formActionId' = submit.provider_action_id + AND submit.payload->>'interactionId' = ${a}.payload->>'interactionId' + AND submit.payload->>'publicationId' = ${a}.payload->>'publicationId' + AND submit.payload->>'expiresAt' = ${a}.payload->>'expiresAt' + AND (submit.status = 'issued' OR (submit.status = 'expired' AND submit.result->>'code' = 'interaction_resolved_elsewhere') OR (submit.status = 'processed' + AND submit.result->>'code' = 'question_form_answered' + AND submit.result->>'interactionId' = submit.payload->>'interactionId')))) + OR (${a}.kind = 'question_form_submit' AND ${a}.payload->>'formActionId' = ${a}.provider_action_id + AND (${a}.status = 'issued' OR (${a}.status = 'expired' AND ${a}.result->>'code' = 'interaction_resolved_elsewhere') OR (${a}.status = 'processed' AND ${a}.result->>'code' = 'question_form_answered' + AND ${a}.result->>'interactionId' = ${a}.payload->>'interactionId'))))) + OR (${a}.kind = 'question_answer' AND ${resolved} + AND (${a}.status = 'issued' OR (${a}.status = 'processed' + AND ${a}.result->>'interactionId' = ${a}.payload->>'interactionId') + OR (${a}.status = 'expired' AND ${a}.result->>'code' IN ('interaction_resolved_by_sibling_action', 'interaction_resolved_elsewhere')))) + OR (${a}.kind = 'question_form_open_attempt' AND ${a}.status = 'processed' + AND ${a}.result->>'code' = 'question_form_opened')`; +} + +/** The terminal interaction writer expires its unused callback tokens after a + * board/sibling resolution. Expiry alone never proves canonical completion. */ +export function completedExpiredChatControl(a: string): string { + return `${a}.kind = 'confirmation_response' AND ${a}.status = 'expired' + AND ${a}.result->>'code' = 'interaction_resolved_elsewhere' AND ${a}.payload->>'version' = '1' + AND ${a}.payload->>'decision' IN ('accept', 'reject') AND EXISTS ( + SELECT 1 FROM issue_thread_interactions interaction JOIN chat_publications publication + ON publication.id::text = ${a}.payload->>'publicationId' AND publication.company_id = interaction.company_id + AND publication.issue_id = interaction.issue_id AND publication.payload->>'interactionId' = interaction.id::text + JOIN chat_conversations conversation ON conversation.id = publication.conversation_id + AND conversation.company_id = interaction.company_id AND conversation.issue_id = interaction.issue_id + WHERE interaction.id::text = ${a}.payload->>'interactionId' AND interaction.company_id = ${a}.company_id + AND interaction.kind IN ('request_confirmation', 'request_checkbox_confirmation', 'suggest_tasks', 'connection_intent') + AND interaction.status IN ('accepted', 'rejected', 'cancelled', 'expired') AND interaction.resolved_at IS NOT NULL + AND publication.endpoint_id = ${a}.endpoint_id AND conversation.id = ${a}.conversation_id)`; +} diff --git a/server/src/services/conversation-idle-proof.ts b/server/src/services/conversation-idle-proof.ts new file mode 100644 index 00000000000..2ce2cf48d0e --- /dev/null +++ b/server/src/services/conversation-idle-proof.ts @@ -0,0 +1,45 @@ +import { finishedIdleRun, finishedIdleWake } from "./idle-work-proof.js"; + +/** The same positive container proof is used by ordinary and scheduled sleep. + * waiting/in_review is set by settleConversationTurn, but is not sufficient: + * a user comment may commit before the wake is enqueued. */ +export function passiveIdleConversation(i: "i"): string { + return `${i}.conversation_agent_id IS NOT NULL AND length(${i}.conversation_user_id) > 0 + AND ${i}.conversation_state = 'waiting' AND ${i}.status = 'in_review' + AND ${i}.assignee_agent_id = ${i}.conversation_agent_id AND ${i}.assignee_user_id IS NULL + AND (${i}.conversation_user_id = 'local-board' OR EXISTS (SELECT 1 FROM company_memberships membership + WHERE membership.company_id = ${i}.company_id AND membership.principal_type = 'user' + AND membership.principal_id = ${i}.conversation_user_id AND membership.status = 'active')) + AND ${i}.conversation_session_generation >= 0 + AND ${i}.checkout_run_id IS NULL AND ${i}.execution_run_id IS NULL + AND ${i}.monitor_next_check_at IS NULL AND ${i}.monitor_wake_requested_at IS NULL + AND (${i}.execution_policy IS NULL OR (jsonb_typeof(${i}.execution_policy) = 'object' + AND COALESCE(${i}.execution_policy->'monitor', 'null'::jsonb) = 'null'::jsonb)) + AND (${i}.execution_state IS NULL OR (jsonb_typeof(${i}.execution_state) = 'object' + AND ${i}.execution_state->>'status' = 'idle' + AND COALESCE(${i}.execution_state->'monitor', 'null'::jsonb) = 'null'::jsonb + AND COALESCE(${i}.execution_state->'currentParticipant', 'null'::jsonb) = 'null'::jsonb + AND COALESCE(${i}.execution_state->'currentStageId', 'null'::jsonb) = 'null'::jsonb + AND COALESCE(${i}.execution_state->'reviewRequest', 'null'::jsonb) = 'null'::jsonb)) + AND EXISTS (SELECT 1 FROM agents a JOIN companies c ON c.id = a.company_id + WHERE a.id = ${i}.conversation_agent_id AND a.company_id = ${i}.company_id + AND c.status = 'active' AND a.status IN ('idle', 'active', 'paused', 'terminated')) + AND NOT EXISTS (SELECT 1 FROM heartbeat_runs r WHERE + (r.native_issue_id = ${i}.id OR r.context_snapshot->>'issueId' = ${i}.id::text OR r.context_snapshot->>'taskId' = ${i}.id::text) + AND (r.company_id <> ${i}.company_id OR r.agent_id <> ${i}.conversation_agent_id OR (${finishedIdleRun('r')}) IS NOT TRUE)) + AND NOT EXISTS (SELECT 1 FROM agent_wakeup_requests w WHERE + (w.payload->>'issueId' = ${i}.id::text OR w.payload->>'taskId' = ${i}.id::text) + AND (w.company_id <> ${i}.company_id OR w.agent_id <> ${i}.conversation_agent_id + OR (${finishedIdleWake('w')}) IS NOT TRUE)) + AND NOT EXISTS (SELECT 1 FROM issue_comments comment WHERE comment.issue_id = ${i}.id + AND comment.deleted_at IS NULL AND comment.author_user_id IS NOT NULL AND comment.author_agent_id IS NULL + AND comment.origin = 'comment' AND comment.client_request_id IS NOT NULL + AND NOT EXISTS (SELECT 1 FROM agent_wakeup_requests w WHERE w.company_id = ${i}.company_id + AND w.agent_id = ${i}.conversation_agent_id AND w.idempotency_key = 'conversation-comment:' || comment.id::text + AND ${finishedIdleWake('w')})) + AND NOT EXISTS (SELECT 1 FROM issue_thread_interactions interaction WHERE interaction.issue_id = ${i}.id + AND (interaction.company_id <> ${i}.company_id OR interaction.status = 'pending')) + AND NOT EXISTS (SELECT 1 FROM chat_completion_deliveries d JOIN chat_task_handoffs h ON h.task_id = d.task_id + WHERE h.conversation_id = ${i}.id AND (h.company_id <> ${i}.company_id OR d.company_id <> ${i}.company_id + OR d.status NOT IN ('delivered', 'superseded', 'exhausted')))`; +} diff --git a/server/src/services/decision-idle-proof.ts b/server/src/services/decision-idle-proof.ts new file mode 100644 index 00000000000..d95f476bbd1 --- /dev/null +++ b/server/src/services/decision-idle-proof.ts @@ -0,0 +1,134 @@ +import { finishedIdleRun } from "./idle-work-proof.js"; + +/** executeDecisionEffects visits every selected ordinal, including failures. + * Its final aggregate cannot prove a missing row or another option's effect. */ +function completedEffectManifest(d: string): string { + const options = `jsonb_array_elements(CASE WHEN jsonb_typeof(${d}.options) = 'array' THEN ${d}.options ELSE '[]'::jsonb END)`; + return `EXISTS (SELECT 1 FROM ${options} option_row WHERE option_row->>'id' = ${d}.chosen_option_id + AND jsonb_typeof(option_row->'effects') = 'array' + AND EXISTS (SELECT 1 FROM activity_log selection WHERE selection.company_id = ${d}.company_id + AND selection.entity_type = 'decision' AND selection.entity_id = ${d}.id::text + AND selection.action IN ('decision.decided', 'decision.dismissed') + AND selection.details->>'chosenOptionId' = ${d}.chosen_option_id + AND selection.details->>'decidedByUserId' = ${d}.decided_by_user_id) + AND (SELECT count(*) FROM ${options} duplicate_option WHERE duplicate_option->>'id' = ${d}.chosen_option_id) = 1 + AND NOT EXISTS (SELECT 1 FROM jsonb_array_elements(option_row->'effects') WITH ORDINALITY expected(effect, ordinal) + WHERE NOT EXISTS (SELECT 1 FROM decision_effect_executions execution JOIN issues target + ON target.id = execution.target_issue_id AND target.company_id = ${d}.company_id + JOIN activity_log audit ON audit.id = execution.activity_log_id AND audit.company_id = ${d}.company_id + AND audit.entity_type = 'decision' AND audit.entity_id = ${d}.id::text + AND audit.details->>'chosenOptionId' = ${d}.chosen_option_id + AND audit.details->>'executionId' = execution.id::text + WHERE execution.decision_id = ${d}.id AND execution.effect_index = expected.ordinal - 1 + AND execution.effect_type = expected.effect->>'type' AND execution.target_issue_id::text = expected.effect->>'targetIssueId' + AND execution.status IN ('executed', 'failed', 'skipped') AND execution.executed_at IS NOT NULL + AND audit.action = 'decision.effect_' || execution.status)) + AND NOT EXISTS (SELECT 1 FROM decision_effect_executions execution WHERE execution.decision_id = ${d}.id + AND NOT EXISTS (SELECT 1 FROM jsonb_array_elements(option_row->'effects') WITH ORDINALITY expected(effect, ordinal) + WHERE execution.effect_index = expected.ordinal - 1 AND execution.effect_type = expected.effect->>'type' + AND execution.target_issue_id::text = expected.effect->>'targetIssueId')) + AND CASE WHEN ${d}.execution_status = 'succeeded' THEN NOT EXISTS (SELECT 1 FROM decision_effect_executions e + WHERE e.decision_id = ${d}.id AND e.status <> 'executed') + WHEN ${d}.execution_status = 'partial' THEN EXISTS (SELECT 1 FROM decision_effect_executions e + WHERE e.decision_id = ${d}.id AND e.status = 'executed') AND EXISTS (SELECT 1 FROM decision_effect_executions e + WHERE e.decision_id = ${d}.id AND e.status IN ('failed', 'skipped')) + ELSE NOT EXISTS (SELECT 1 FROM decision_effect_executions e WHERE e.decision_id = ${d}.id AND e.status = 'executed') + AND EXISTS (SELECT 1 FROM decision_effect_executions e WHERE e.decision_id = ${d}.id AND e.status IN ('failed', 'skipped')) END)`; +} + +function completedDecisionOutcome(d: string): string { + return `(${d}.status IN ('expired', 'cancelled') AND ${d}.execution_status IS NULL AND ${d}.chosen_option_id IS NULL + AND NOT EXISTS (SELECT 1 FROM decision_effect_executions e WHERE e.decision_id = ${d}.id) + AND EXISTS (SELECT 1 FROM activity_log audit WHERE audit.company_id = ${d}.company_id + AND audit.entity_type = 'decision' AND audit.entity_id = ${d}.id::text AND audit.action = 'decision.' || ${d}.status)) + OR (${d}.status = 'decided' AND ${d}.decided_at IS NOT NULL AND ${d}.execution_status IN ('succeeded', 'partial', 'failed') + AND ((COALESCE(${d}.metadata->>'kind', '') <> 'attention_archive_proposal' AND (${completedEffectManifest(d)})) + OR (${completedArchiveOutcome(d)}) + OR (${d}.chosen_option_id = 'dismissed' AND ${d}.execution_status = 'succeeded' AND ${d}.metadata->>'dismissed' = 'true' + AND NOT EXISTS (SELECT 1 FROM decision_effect_executions e WHERE e.decision_id = ${d}.id) + AND EXISTS (SELECT 1 FROM activity_log audit WHERE audit.company_id = ${d}.company_id + AND audit.entity_type = 'decision' AND audit.entity_id = ${d}.id::text AND audit.action = 'decision.dismissed' + AND audit.details->>'chosenOptionId' = 'dismissed' AND audit.details->>'decidedByUserId' = ${d}.decided_by_user_id)) + OR (${d}.metadata->>'effectRecoveryError' IN ('missing_decider', 'missing_option') + AND ${d}.execution_status IN ('failed', 'partial') + AND EXISTS (SELECT 1 FROM activity_log audit WHERE audit.company_id = ${d}.company_id + AND audit.entity_type = 'decision' AND audit.entity_id = ${d}.id::text AND audit.action = 'decision.recovery_failed' + AND audit.details->>'reason' = ${d}.metadata->>'effectRecoveryError' + AND audit.details->>'executionStatus' = ${d}.execution_status + AND audit.details->>'chosenOptionId' IS NOT DISTINCT FROM ${d}.chosen_option_id))))`; +} + +/** Archive applies the reviewed set in one transaction. Every catch is final + * (execution_status=failed); it neither replays the set nor leaves per-effect + * claims. Continuation and notification queues remain independent checks. */ +function completedArchiveOutcome(d: string): string { + return `${d}.metadata->>'kind' = 'attention_archive_proposal' + AND NOT EXISTS (SELECT 1 FROM decision_effect_executions e WHERE e.decision_id = ${d}.id) + AND EXISTS (SELECT 1 FROM jsonb_array_elements(CASE WHEN jsonb_typeof(${d}.options) = 'array' + THEN ${d}.options ELSE '[]'::jsonb END) selected WHERE selected->>'id' = ${d}.chosen_option_id + AND selected->'effects' = '[]'::jsonb + AND (SELECT count(*) FROM jsonb_array_elements(${d}.options) duplicate WHERE duplicate->>'id' = ${d}.chosen_option_id) = 1) + AND ((${d}.execution_status = 'failed' AND ${d}.chosen_option_id = 'archive' + AND jsonb_typeof(${d}.metadata->'archiveProposalError') = 'string' + AND length(trim(${d}.metadata->>'archiveProposalError')) > 0) + OR (${d}.execution_status = 'succeeded' AND ${d}.metadata->>'archiveProposalError' IS NULL + AND (${d}.chosen_option_id <> 'archive' OR ( + jsonb_typeof(${d}.target_snapshots) = 'object' AND ${d}.target_snapshots <> '{}'::jsonb + AND ${d}.metadata->>'manifestHash' ~ '^[a-f0-9]{64}$' + AND NOT EXISTS (SELECT 1 FROM jsonb_each(${d}.target_snapshots) snapshot WHERE ( + snapshot.key = 'attention:' || (snapshot.value->'attentionArchive'->>'sourceKind') || ':' || (snapshot.value->'attentionArchive'->>'sourceId') + AND snapshot.value->'attentionArchive'->>'companyId' = ${d}.company_id::text + AND jsonb_typeof(snapshot.value->'attentionArchive'->'expectedVersion') = 'number' + AND snapshot.value->'attentionArchive'->>'expectedVersion' ~ '^[1-9][0-9]*$' + AND EXISTS (SELECT 1 FROM decision_retention retention WHERE retention.company_id = ${d}.company_id + AND retention.source_kind = snapshot.value->'attentionArchive'->>'sourceKind' + AND retention.source_id = snapshot.value->'attentionArchive'->>'sourceId' + AND retention.version::numeric > (snapshot.value->'attentionArchive'->>'expectedVersion')::numeric) + AND EXISTS (SELECT 1 FROM activity_log audit WHERE audit.company_id = ${d}.company_id + AND audit.actor_id = 'decision-archive-proposal' AND audit.action = 'decision_retention.archived' + AND audit.entity_type = 'attention_source' AND audit.entity_id = snapshot.value->'attentionArchive'->>'sourceId' + AND audit.details->>'sourceKind' = snapshot.value->'attentionArchive'->>'sourceKind' + AND audit.created_at >= ${d}.decided_at AND audit.responsible_user_id = ${d}.decided_by_user_id)) IS NOT TRUE)))))`; +} + +function completedDecision(alias: string): string { + return `(${completedDecisionOutcome(alias)}) + AND ${alias}.continuation_policy IN ('none', 'wake_origin_agent') + AND jsonb_typeof(${alias}.metadata) = 'object' + AND COALESCE(${alias}.metadata->>'continuationPending', 'false') = 'false' + AND (${alias}.continuation_policy = 'none' OR ${alias}.metadata->>'continuationDeliveredAt' IS NOT NULL) + AND EXISTS (SELECT 1 FROM heartbeat_runs r JOIN issues i ON i.id = ${alias}.origin_issue_id + AND i.company_id = r.company_id WHERE r.id = ${alias}.origin_run_id + AND r.company_id = ${alias}.company_id AND r.agent_id = ${alias}.origin_agent_id AND ${finishedIdleRun('r')}) + AND NOT EXISTS (SELECT 1 FROM decision_effect_executions e WHERE e.decision_id = ${alias}.id + AND (e.status NOT IN ('executed', 'failed', 'skipped') OR e.executed_at IS NULL))`; +} + +export const idleDecisionHistoryChecks = [ + { reason: "decisions", query: `SELECT 1 FROM decisions d WHERE (${completedDecision('d')}) IS NOT TRUE` }, + { reason: "decision_effect_executions", query: `SELECT 1 FROM decision_effect_executions e WHERE ( + e.status IN ('executed', 'failed', 'skipped') AND e.executed_at IS NOT NULL + AND EXISTS (SELECT 1 FROM decisions d JOIN issues i ON i.id = e.target_issue_id AND i.company_id = d.company_id + WHERE d.id = e.decision_id AND (${completedDecision('d')}))) IS NOT TRUE` }, + { reason: "decision_archive_notification_outbox", query: `SELECT 1 FROM decision_archive_notification_outbox o WHERE ( + o.status = 'delivered' AND o.delivered_at IS NOT NULL + AND o.archive_version > 0 AND EXISTS (SELECT 1 FROM decision_retention retention + WHERE retention.company_id = o.company_id AND retention.source_kind = o.source_kind + AND retention.source_id = o.source_id AND retention.archive_version >= o.archive_version) + AND EXISTS (SELECT 1 FROM agents a JOIN issues i ON i.company_id = a.company_id + WHERE a.id = o.origin_agent_id AND i.id = o.origin_issue_id AND i.company_id = o.company_id)) IS NOT TRUE` }, + { reason: "company_secret_proposals", query: `SELECT 1 FROM company_secret_proposals p WHERE ( + p.kind IN ('secret', 'binding') AND p.status IN ('approved', 'rejected', 'withdrawn', 'expired') + AND p.resolved_at IS NOT NULL AND p.value_ciphertext IS NULL AND p.ciphertext_scrubbed_at IS NOT NULL + AND EXISTS (SELECT 1 FROM heartbeat_runs r WHERE r.id = p.origin_run_id + AND r.company_id = p.company_id AND r.agent_id = p.proposed_by_agent_id AND ${finishedIdleRun('r')}) + AND (p.origin_issue_id IS NULL OR EXISTS (SELECT 1 FROM issues i + WHERE i.id = p.origin_issue_id AND i.company_id = p.company_id)) + AND (p.status <> 'approved' OR (p.kind = 'secret' AND EXISTS (SELECT 1 FROM company_secrets s + WHERE s.id = p.created_secret_id AND s.company_id = p.company_id)) + OR (p.kind = 'binding' AND p.applied_binding_config_path = p.config_path AND EXISTS ( + SELECT 1 FROM agents a WHERE a.id = p.target_id AND a.company_id = p.company_id))) + AND (p.interaction_id IS NULL OR EXISTS (SELECT 1 FROM issue_thread_interactions i + WHERE i.id = p.interaction_id AND i.company_id = p.company_id AND i.issue_id = p.origin_issue_id + AND i.status IN ('accepted', 'rejected', 'cancelled', 'expired') AND i.resolved_at IS NOT NULL))) IS NOT TRUE` }, +] as const; diff --git a/server/src/services/environment-cleanup-work.ts b/server/src/services/environment-cleanup-work.ts index 2370762e2ce..a5fecf17adf 100644 --- a/server/src/services/environment-cleanup-work.ts +++ b/server/src/services/environment-cleanup-work.ts @@ -22,7 +22,17 @@ export function registerEnvironmentCleanupWork( onSettled: () => { requested++; }, run: async () => { if (!input.canRun()) { requested++; next = null; return; } - if (Date.now() < earliest) { next = earliest; return; } + if (Date.now() < earliest) { + next = earliest; + if (queue === DELIVERY_QUEUES.historicalCleanup) { + // A settled metadata invalidation is not itself cleanup debt. A + // cheap owner read can discharge it without advancing the provider + // sweep cadence; later notifications keep their own generation. + const generation = requested; + if (await deadline() === null) { next = null; reconciled = generation; } + } + return; + } const generation = ++requested; earliest = Date.now() + minimumSpacing; await run(); diff --git a/server/src/services/idle-issue-schedule.ts b/server/src/services/idle-issue-schedule.ts index 5037c099d97..cb08fff2e81 100644 --- a/server/src/services/idle-issue-schedule.ts +++ b/server/src/services/idle-issue-schedule.ts @@ -5,6 +5,7 @@ import { AGENT_STATUSES, issueExecutionMonitorPolicySchema, PROVIDER_QUOTA_MONIT import type { ScheduledWake } from "./idle-scheduled-work.js"; import { onboardingFirstTaskAwaitingUserSql } from "./recovery/onboarding-first-task-wait.js"; import { completedAgentUnblockRequest, responseDrivenIssueWait } from "./idle-work-proof.js"; +import { passiveIdleConversation } from "./conversation-idle-proof.js"; const processGeneration = randomUUID(); const limit = 10_000; @@ -31,6 +32,7 @@ export async function readIdleIssueSchedule(tx: Pick, ownerId: st untouched_onboarding: boolean; response_driven_wait: boolean; unblock_request_finished: boolean; response_wait_generation: unknown; + passive_conversation: boolean; }>(sql`SELECT i.id, i.company_id, i.status, i.updated_at, i.assignee_agent_id, i.assignee_user_id, i.checkout_run_id, i.execution_run_id, i.execution_policy, i.execution_state, i.unblock_descriptor, i.conversation_agent_id, i.monitor_next_check_at, i.monitor_wake_requested_at, i.monitor_attempt_count, @@ -39,6 +41,7 @@ export async function readIdleIssueSchedule(tx: Pick, ownerId: st (i.assignee_agent_id IS NULL OR a.id IS NOT NULL) AS scoped, (${sql.raw(responseDrivenIssueWait('i'))}) AS response_driven_wait, (${sql.raw(completedAgentUnblockRequest('i'))}) AS unblock_request_finished, + (${sql.raw(passiveIdleConversation('i'))}) AS passive_conversation, (SELECT jsonb_agg(jsonb_build_array(w.id, w.updated_at, w.kind, w.status, w.source_run_id, w.effective_resolver_policy, w.continuation_policy, w.addressee_agent_id, w.addressee_user_id) ORDER BY w.id) FROM issue_thread_interactions w WHERE w.issue_id = i.id AND w.status = 'pending') AS response_wait_generation, @@ -62,7 +65,7 @@ export async function readIdleIssueSchedule(tx: Pick, ownerId: st if (!statuses.has(row.status) || row.company_status !== "active" || row.scoped !== true || (row.assignee_agent_id && !agentStatuses.has(row.agent_status ?? "")) || row.unsafe_dependencies !== false || row.checkout_run_id || row.execution_run_id - || row.conversation_agent_id || row.monitor_wake_requested_at + || (row.conversation_agent_id && row.passive_conversation !== true) || row.monitor_wake_requested_at || (row.execution_state !== null && state?.status !== "idle") || (row.execution_policy !== null && !policy)) return { blocked: true }; if (row.status === "blocked" && row.dependency_ready) return { blocked: true }; @@ -82,7 +85,7 @@ export async function readIdleIssueSchedule(tx: Pick, ownerId: st } if (policy?.monitor != null) return { blocked: true }; const unblock = record(row.unblock_descriptor); - const passive = row.status === "backlog" + const passive = row.passive_conversation === true || row.status === "backlog" || (row.status === "todo" && !row.assignee_agent_id) || (row.status === "todo" && row.assignee_agent_id && row.agent_status === "idle" && row.untouched_onboarding === true) || (row.status === "blocked" && (row.unblock_descriptor === null || unblock?.owner === "board" diff --git a/server/src/services/idle-sleep-safety.ts b/server/src/services/idle-sleep-safety.ts index c773958a546..96d06daae0e 100644 --- a/server/src/services/idle-sleep-safety.ts +++ b/server/src/services/idle-sleep-safety.ts @@ -10,6 +10,23 @@ import { verifiedResourceCleanup } from "./cleanup-verification-proof.js"; import { companyHoldMismatch } from "../modules/agent-lifecycle/index.js"; import { pendingBudgetRunCancellationPredicate } from "./heartbeat/budget-cancellation.js"; import { nativeResponsePresentationPendingQuery } from "./native-runtime/native-response-presentation-query.js"; +import { passiveIdleConversation } from "./conversation-idle-proof.js"; +import { idleChatHistoryChecks } from "./chat-channels/idle-history-proof.js"; +import { idleToolActionHistoryChecks } from "./tool-action-idle-proof.js"; +import { idleBrowserHistoryChecks } from "./browser-use-idle-proof.js"; +import { idleDecisionHistoryChecks } from "./decision-idle-proof.js"; +import { idleWorkspaceHistoryChecks } from "./workspace-history-idle-proof.js"; +import { idleMcpHistoryChecks } from "./public-mcp/idle-history-proof.js"; +import { idleExternalAgentHistoryChecks } from "../modules/external-agents/index.js"; + +export const completedHistoryChecks = [ + ...idleChatHistoryChecks, ...idleToolActionHistoryChecks, ...idleBrowserHistoryChecks, + ...idleDecisionHistoryChecks, ...idleWorkspaceHistoryChecks, ...idleMcpHistoryChecks, + ...idleExternalAgentHistoryChecks, +] as const; + +const ordinaryIssues = `SELECT 1 FROM issues i WHERE i.status NOT IN ('done', 'cancelled') + AND (${passiveIdleConversation('i')}) IS NOT TRUE`; export interface IdleSleepDrainStatus { ownerId?: string; @@ -58,7 +75,7 @@ const WORK_CHECKS = [ state NOT IN ('saved', 'unchanged', 'resolved', 'unavailable', 'conflict') OR process_stopped_at IS NULL OR next_attempt_at IS NOT NULL OR receipt ? 'baseline' OR receipt->>'cleanupPending' = 'true'`), - workCheck("sandbox_creation_operations", `SELECT 1 FROM sandbox_creation_operations WHERE state = 'pending'`), + workCheck("sandbox_creation_operations", `SELECT 1 FROM sandbox_creation_operations WHERE state IN ('pending', 'needs_reconciliation')`), workCheck("budget_reservations", `SELECT 1 FROM budget_reservations WHERE state NOT IN ('settled', 'released')`), workCheck("budget_policies", `SELECT 1 FROM budget_policies WHERE enforcement_version > enforcement_delivered_version`), workCheck("agents.lifecycle", `SELECT 1 FROM agents WHERE lifecycle_state IN @@ -78,7 +95,7 @@ const WORK_CHECKS = [ SELECT 1 FROM heartbeat_runs r WHERE r.id = w.run_id AND r.company_id = w.company_id AND r.agent_id = w.agent_id AND ${finishedRun("r")})))`), - workCheck("issues", `SELECT 1 FROM issues WHERE status NOT IN ('done', 'cancelled')`), + workCheck("issues", ordinaryIssues), // A completed issue can still need its first watchdog review, including a // retry after immediate evaluation failed before creating a review/run. workCheck("issue_watchdogs", `SELECT 1 FROM issue_watchdogs WHERE status <> 'disabled'`), @@ -263,23 +280,7 @@ const WORK_CHECKS = [ OR effect.delivery_state <> 'delivered' OR effect.delivered_at IS NULL OR effect.next_attempt_at IS NOT NULL)) ) IS NOT TRUE`), - // These less common work sources fail closed on any retained state. Their - // terminal-state exceptions can be added with tests for the owning service. - ...[ - "chat_deliveries", "chat_publications", "chat_actions", - "chat_completion_deliveries", "tool_action_requests", "tool_action_deliveries", - "workspace_runtime_services", - "execution_workspace_runtime_leases", - "company_transfer_runs", "decisions", "decision_effect_executions", - "decision_archive_notification_outbox", "browser_use_sessions", "browser_use_runs", - "browser_use_browsers", - "company_secret_proposals", - "mcp_oauth_grants", "mcp_mutation_receipts", "mcp_event_subscriptions", - "mcp_event_deliveries", "mcp_attachment_uploads", "dot_agent_bindings", - "dot_runner_assignments", "dot_runner_operations", "dot_mailbox_items", - "muse_agent_bindings", "muse_credentials", "muse_runner_assignments", - "muse_runner_operations", "muse_mailbox_items", "muse_input_deliveries", "external_agent_holds", - ].map((table) => workCheck(table, `SELECT 1 FROM ${table}`)), + ...completedHistoryChecks.map(check => workCheck(check.reason, check.query)), ] as const; function sameQuietHold(before: IdleSleepDrainStatus, after: IdleSleepDrainStatus, now: number): boolean { diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 2e1bcd372cf..3bd4b50fa24 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -1,3 +1,4 @@ +import { recordMcpAttachmentCleanup, type McpAttachmentCleanup } from "./public-mcp/upload-cleanup.js"; import type { IssueDisplayStatus } from "@paperclipai/shared"; import { issueDisplayStatusSql } from "./issue-display-status.js"; import { notifyStrandedAgentStatus, notifyActiveRunSilence } from "./live-run-recovery-notifications.js"; @@ -13953,7 +13954,7 @@ export function issueService(db: Db) { .where(eq(issueAttachments.id, id)) .then((rows) => rows[0] ?? null), - removeAttachment: async (id: string) => + removeAttachment: async (id: string, options?: { mcpCleanup?: McpAttachmentCleanup | null }) => db.transaction(async (tx) => { const existing = await tx .select({ @@ -13983,6 +13984,11 @@ export function issueService(db: Db) { await notifyIssueRecovery(tx, { companyId: existing.companyId }); await tx.delete(issueAttachments).where(eq(issueAttachments.id, id)); await tx.delete(assets).where(eq(assets.id, existing.assetId)); + if (options?.mcpCleanup?.id === existing.id && options.mcpCleanup.companyId === existing.companyId + && options.mcpCleanup.issueId === existing.issueId && options.mcpCleanup.provider === existing.provider + && options.mcpCleanup.objectKey === existing.objectKey) { + await recordMcpAttachmentCleanup(tx, options.mcpCleanup); + } return existing; }), diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 04e4ce3f093..725f07f2ea0 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -10314,6 +10314,11 @@ describe("native warm session supervision", () => { it("blocks idle sleep for a released local warm session until its close checkpoint finishes", async () => { await closeIdleWarmNativeSessionsForRestart(); + const { reconcilePersistedRuntimeServicesOnStartup, readWorkspaceRuntimeIdleWork } = await import("../workspace-runtime.js"); + // The real server reconstructs the workspace owner before startup can + // become quiet. This fixture has no runtime rows or desired services. + await reconcilePersistedRuntimeServicesOnStartup(leaseDb()); + expect(readWorkspaceRuntimeIdleWork()).toBe("none"); markIdleIngressTracked(); markIdleStartupComplete({ scheduledBackups: false }); let finishClose!: () => void; const checkpoint = new Promise(resolve => { finishClose = resolve; }); diff --git a/server/src/services/public-mcp/file-transfers.ts b/server/src/services/public-mcp/file-transfers.ts index eb268e76bc8..108fd3be52c 100644 --- a/server/src/services/public-mcp/file-transfers.ts +++ b/server/src/services/public-mcp/file-transfers.ts @@ -1,10 +1,11 @@ import { createHash, randomBytes, randomUUID } from "node:crypto"; import { pipeline } from "node:stream/promises"; import express, { Router } from "express"; -import { and, eq, gt, isNull, lt, sql } from "drizzle-orm"; +import { and, asc, eq, gt, isNull, lt, sql } from "drizzle-orm"; import { type Db, mcpAttachmentUploads, mcpFileTickets, mcpOauthGrants, issues } from "@paperclipai/db"; import { z } from "zod"; import { MAX_ATTACHMENT_BYTES, isAllowedContentType } from "../../attachment-types.js"; +import { logger } from "../../middleware/logger.js"; import type { StorageService } from "../../storage/types.js"; import { authorizationService } from "../authorization.js"; import { issueService } from "../issues.js"; @@ -78,16 +79,42 @@ export function createPublicMcpTransfers(db: Db, oauth: PublicMcpOAuth, api: Api await taskAccess(p, attachment.issueId, false, queryDb, sessionId); return attachment; } - async function sweep() { + // Each transfer service lives for the app lifetime. Advance this bounded + // traversal only after commit, and wrap so uncertain rows are retried. + let cleanupAfterId: string | undefined; + let pendingSweep: Promise | undefined; + function sweep() { + if (!pendingSweep) pendingSweep = runSweep().finally(() => { pendingSweep = undefined; }); + return pendingSweep; + } + async function runSweep() { await db.delete(mcpFileTickets).where(lt(mcpFileTickets.expiresAt, new Date())); - // Serialize cleanup against completion/reissue. Never remove a committed file. - await db.transaction(async tx => { - const abandoned = await tx.select().from(mcpAttachmentUploads).where(and(isNull(mcpAttachmentUploads.attachmentId), isNull(mcpAttachmentUploads.cleanedAt), lt(mcpAttachmentUploads.expiresAt, new Date()))).limit(20).for("update", { skipLocked: true }); + // Serialize cleanup against completion/reissue. Requalify old optimistic + // cleanup and orphaned removed attachments, never a still-linked file. + cleanupAfterId = await db.transaction(async tx => { + const abandoned = await tx.select().from(mcpAttachmentUploads).where(and( + eq(mcpAttachmentUploads.storageProvider, storage.provider), + isNull(mcpAttachmentUploads.cleanupVerifiedAt), lt(mcpAttachmentUploads.expiresAt, new Date()), + cleanupAfterId ? gt(mcpAttachmentUploads.id, cleanupAfterId) : undefined, + sql`(${mcpAttachmentUploads.attachmentId} IS NULL OR NOT EXISTS ( + SELECT 1 FROM issue_attachments attachment WHERE attachment.id = ${mcpAttachmentUploads.attachmentId}))`, + )).orderBy(asc(mcpAttachmentUploads.id)).limit(20).for("update", { skipLocked: true }); for (const upload of abandoned) { - if (upload.storageProvider !== storage.provider) continue; - await storage.deleteObject(upload.companyId, upload.objectKey); - await tx.update(mcpAttachmentUploads).set({ cleanedAt: new Date() }).where(eq(mcpAttachmentUploads.id, upload.id)); + try { + await storage.deleteObject(upload.companyId, upload.objectKey); + if ((await storage.headObject(upload.companyId, upload.objectKey)).exists !== false) { + throw new Error("MCP upload storage cleanup could not be confirmed"); + } + } catch { + // Cleanup is opportunistic across companies. Retain this uncertainty + // without denying an unrelated caller's authorized file transfer. + logger.warn({ companyId: upload.companyId, uploadId: upload.id }, "MCP upload cleanup remains unverified"); + continue; + } + const verifiedAt = new Date(); + await tx.update(mcpAttachmentUploads).set({ cleanedAt: verifiedAt, cleanupVerifiedAt: verifiedAt }).where(eq(mcpAttachmentUploads.id, upload.id)); } + return abandoned.length === 20 ? abandoned.at(-1)!.id : undefined; }); } async function ticket(p: McpPrincipal, target: { uploadId?: string; attachmentId?: string }, sessionId?: string) { @@ -133,6 +160,7 @@ export function createPublicMcpTransfers(db: Db, oauth: PublicMcpOAuth, api: Api const published = await db.transaction(async tx => { const [upload] = await tx.select().from(mcpAttachmentUploads).where(eq(mcpAttachmentUploads.id, row.uploadId!)).for("update"); if (!upload || upload.companyId !== principal.grant.companyId || upload.userId !== principal.grant.userId || upload.workSessionId !== row.workSessionId || upload.grantId && upload.grantId !== row.grantId || upload.storageProvider !== storage.provider) throw unavailable(); + if (upload.expiresAt <= new Date() || upload.cleanedAt || upload.cleanupVerifiedAt) throw unavailable(); if (upload.sha256 !== digest || upload.byteSize !== bytes.length || req.get("Content-Type")?.split(";")[0]?.trim().toLowerCase() !== upload.contentType) throw new McpCapabilityError("Upload bytes, size or Content-Type do not match the requested file."); await taskAccess(principal, upload.taskId, principal.grant.purpose !== "agent", tx as unknown as Db, row.workSessionId ?? undefined); const stored = upload.attachmentId ? null : await storage.putFile({ companyId: upload.companyId, namespace: "mcp-transfers", objectKey: upload.objectKey, originalFilename: upload.originalFilename, contentType: upload.contentType, body: bytes }); @@ -200,7 +228,11 @@ export function createPublicMcpTransfers(db: Db, oauth: PublicMcpOAuth, api: Api objectKey: `${p.grant.companyId}/mcp-transfers/${id}`, storageProvider: storage.provider, expiresAt: new Date(Date.now() + lifetime) }).returning(); } if (!found || found.argumentsHash !== hash) throw new McpCapabilityError("requestId was already used for a different upload."); - await tx.update(mcpAttachmentUploads).set({ expiresAt: new Date(Date.now() + lifetime), cleanedAt: null }).where(eq(mcpAttachmentUploads.id, found.id)); + // A committed identity is immutable, including after user deletion. + // A rejected replay must not erase verified cleanup or permit new bytes. + if (!found.attachmentId) await tx.update(mcpAttachmentUploads) + .set({ expiresAt: new Date(Date.now() + lifetime), cleanedAt: null, cleanupVerifiedAt: null }) + .where(eq(mcpAttachmentUploads.id, found.id)); return found; }); if (upload.attachmentId) { @@ -214,6 +246,26 @@ export function createPublicMcpTransfers(db: Db, oauth: PublicMcpOAuth, api: Api ...(publication ? { publication } : {}) }; // An older upload can lack publication proof. Verify the same bytes // through another bounded PUT; never manufacture a receipt from metadata. + await db.transaction(async tx => { + const [current] = await tx.select().from(mcpAttachmentUploads) + .where(eq(mcpAttachmentUploads.id, upload.id)).for("update"); + if (!current || current.companyId !== p.grant.companyId || current.userId !== p.grant.userId + || current.argumentsHash !== hash || current.workSessionId !== sessionId + || current.grantId && current.grantId !== p.grant.id + || current.attachmentId !== attachment.id || current.storageProvider !== storage.provider + || current.cleanedAt || current.cleanupVerifiedAt) throw unavailable(); + const owner = await workSession(p, sessionId, current.taskId, true, tx as unknown as Db); + const live = await attachmentAccess(p, attachment.id, tx as unknown as Db, sessionId); + if (!owner || live.issueId !== current.taskId || live.provider !== current.storageProvider + || live.objectKey !== current.objectKey || live.originalFilename !== current.originalFilename + || live.byteSize !== current.byteSize || live.sha256 !== current.sha256 + || live.originatingRunId !== owner.runId || live.createdByAgentId !== owner.agentId) throw unavailable(); + // Keep the committed attachment and cleanup identity immutable. A + // concurrent deletion still removes that attachment/stamps this row; + // PUT rechecks both and never stores bytes for a committed upload. + await tx.update(mcpAttachmentUploads).set({ expiresAt: new Date(Date.now() + lifetime) }) + .where(eq(mcpAttachmentUploads.id, current.id)); + }); } return { ...await ticket(p, { uploadId: upload.id }, sessionId), method: "PUT", headers: { "Content-Type": a.contentType }, uploadId: upload.id, maxBytes: MAX_ATTACHMENT_BYTES, instructions: "Send the exact file bytes with the given Content-Type. Success saves the attachment and returns its ID. An agent upload also returns a verified publication receipt; cite its evidenceRef in your completion report. There is no separate upload completion tool. Do not share this temporary file credential. Reuse requestId after a lost response." }; diff --git a/server/src/services/public-mcp/idle-history-proof.ts b/server/src/services/public-mcp/idle-history-proof.ts new file mode 100644 index 00000000000..b861eee3775 --- /dev/null +++ b/server/src/services/public-mcp/idle-history-proof.ts @@ -0,0 +1,40 @@ +/** Completion receipts are retained for idempotency. Fixed aliases only. */ +export const idleMcpHistoryChecks = [ + { reason: "mcp_oauth_grants", query: `SELECT 1 FROM mcp_oauth_grants g WHERE + (g.revoked_at IS NOT NULL AND EXISTS (SELECT 1 FROM companies c WHERE c.id = g.company_id)) IS NOT TRUE` }, + { reason: "mcp_mutation_receipts", query: `SELECT 1 FROM mcp_mutation_receipts m WHERE ( + m.status = 'completed' AND jsonb_typeof(m.result) = 'object' + AND EXISTS (SELECT 1 FROM mcp_oauth_grants g WHERE g.id = m.grant_id + AND g.company_id = m.company_id AND g.user_id = m.user_id)) IS NOT TRUE` }, + // Stopped subscriptions no longer scan for events. Expiry alone is not a + // handoff of responsibility for active subscriptions or unfinished delivery. + { reason: "mcp_event_subscriptions", query: `SELECT 1 FROM mcp_event_subscriptions s WHERE ( + s.stopped_at IS NOT NULL AND EXISTS (SELECT 1 FROM mcp_oauth_grants g + WHERE g.id = s.grant_id AND g.company_id = s.company_id)) IS NOT TRUE` }, + { reason: "mcp_event_deliveries", query: `SELECT 1 FROM mcp_event_deliveries d WHERE ( + d.finished_at IS NOT NULL AND (d.outcome IN ('delivered', 'filtered', 'inactive', 'delivery_exhausted') + OR (d.outcome ~ '^http_[1-5][0-9][0-9]$' AND d.outcome !~ '^http_2' + AND (d.attempts >= 6 OR (d.outcome NOT IN ('http_408', 'http_429') AND d.outcome !~ '^http_5')))) + AND EXISTS (SELECT 1 FROM mcp_event_subscriptions s WHERE s.id = d.subscription_id + AND EXISTS (SELECT 1 FROM mcp_oauth_grants g WHERE g.id = s.grant_id AND g.company_id = s.company_id) + AND ((d.activity_id IS NOT NULL AND d.mailbox_item_id IS NULL AND EXISTS ( + SELECT 1 FROM activity_log a WHERE a.id = d.activity_id AND a.company_id = s.company_id)) + OR (d.mailbox_item_id IS NOT NULL AND d.activity_id IS NULL AND EXISTS ( + SELECT 1 FROM dot_mailbox_items m WHERE m.id = d.mailbox_item_id + AND m.company_id = s.company_id AND m.binding_id = s.binding_id))))) IS NOT TRUE` }, + // Upload identity survives user deletion. Only the new owner-verified + // absence stamp can retire cleanup; legacy cleaned_at alone was optimistic. + { reason: "mcp_attachment_uploads", query: `SELECT 1 FROM mcp_attachment_uploads u WHERE ( + EXISTS (SELECT 1 FROM companies c WHERE c.id = u.company_id) + AND NOT EXISTS (SELECT 1 FROM issues contradiction WHERE contradiction.id = u.task_id AND contradiction.company_id <> u.company_id) + AND (u.grant_id IS NULL OR EXISTS (SELECT 1 FROM mcp_oauth_grants g WHERE g.id = u.grant_id + AND g.company_id = u.company_id AND g.user_id = u.user_id)) + AND ((u.cleaned_at IS NOT NULL AND u.cleanup_verified_at IS NOT NULL AND u.cleanup_verified_at >= u.cleaned_at + AND (u.attachment_id IS NULL OR NOT EXISTS (SELECT 1 FROM issue_attachments live WHERE live.id = u.attachment_id))) + OR (u.cleaned_at IS NULL AND u.cleanup_verified_at IS NULL AND u.attachment_id IS NOT NULL + AND EXISTS (SELECT 1 FROM issues i WHERE i.id = u.task_id AND i.company_id = u.company_id) AND EXISTS ( + SELECT 1 FROM issue_attachments a JOIN assets stored ON stored.id = a.asset_id + WHERE a.id = u.attachment_id AND a.company_id = u.company_id AND a.issue_id = u.task_id + AND stored.company_id = u.company_id AND stored.object_key = u.object_key + AND stored.provider = u.storage_provider)))) IS NOT TRUE` }, +] as const; diff --git a/server/src/services/public-mcp/upload-cleanup.ts b/server/src/services/public-mcp/upload-cleanup.ts new file mode 100644 index 00000000000..205b5107c8d --- /dev/null +++ b/server/src/services/public-mcp/upload-cleanup.ts @@ -0,0 +1,38 @@ +import { and, eq } from "drizzle-orm"; +import { mcpAttachmentUploads, type Db } from "@paperclipai/db"; +import type { StorageService } from "../../storage/types.js"; + +type AttachmentStorage = { + id: string; + companyId: string; + issueId: string; + provider: string; + objectKey: string; +}; + +export type McpAttachmentCleanup = AttachmentStorage & { verifiedAt: Date }; + +function uploadIdentity(attachment: AttachmentStorage) { + return and(eq(mcpAttachmentUploads.companyId, attachment.companyId), + eq(mcpAttachmentUploads.taskId, attachment.issueId), eq(mcpAttachmentUploads.attachmentId, attachment.id), + eq(mcpAttachmentUploads.storageProvider, attachment.provider), eq(mcpAttachmentUploads.objectKey, attachment.objectKey)); +} + +/** Called after deletion by the attachment owner. A resolved delete alone can + * be optimistic; only the exact upload's storage absence authorizes cleanup. */ +export async function confirmMcpAttachmentCleanup(db: Db, storage: StorageService, attachment: AttachmentStorage): Promise { + const [upload] = await db.select({ id: mcpAttachmentUploads.id }).from(mcpAttachmentUploads).where(uploadIdentity(attachment)).limit(1); + if (!upload) return null; + if (storage.provider !== attachment.provider || (await storage.headObject(attachment.companyId, attachment.objectKey)).exists !== false) { + throw new Error("MCP attachment storage cleanup could not be confirmed"); + } + return { ...attachment, verifiedAt: new Date() }; +} + +/** Keep attachmentId as immutable replay identity. For attachment removal this + * runs in the metadata deletion transaction; issue removal stamps each object + * after its existing post-transaction storage cleanup completes. */ +export async function recordMcpAttachmentCleanup(db: Pick, cleanup: McpAttachmentCleanup | null | undefined) { + if (!cleanup) return; + await db.update(mcpAttachmentUploads).set({ cleanedAt: cleanup.verifiedAt, cleanupVerifiedAt: cleanup.verifiedAt }).where(uploadIdentity(cleanup)); +} diff --git a/server/src/services/remote-workspace-snapshot-retention.test.ts b/server/src/services/remote-workspace-snapshot-retention.test.ts index 26528e6b024..d008d393d38 100644 --- a/server/src/services/remote-workspace-snapshot-retention.test.ts +++ b/server/src/services/remote-workspace-snapshot-retention.test.ts @@ -13,13 +13,15 @@ import { createStorageService } from "../storage/service.js"; import { createS3StorageProvider } from "../storage/s3-provider.js"; import type { StorageService } from "../storage/types.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "../__tests__/helpers/external-test-database.js"; import { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; import { idleWorkSnapshot } from "./task-admission.js"; import { issueRecoveryActionService } from "./issue-recovery-actions.js"; import { notifyRemoteWorkspaceSnapshots, registerRemoteWorkspaceSnapshotRetentionWork, remoteWorkspaceSnapshotRetentionService } from "./remote-workspace-snapshot-retention.js"; -const url = process.env.PAPERCLIP_TEST_DATABASE_URL; -if (url && !/^remote_workspace_[a-z_]*test$/.test(new URL(url).pathname.slice(1))) throw new Error("Disposable remote_workspace_*test database required"); +const historyUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_HISTORY_TEST_DATABASE_URL"); +const url = historyUrl ?? process.env.PAPERCLIP_TEST_DATABASE_URL; +if (!historyUrl && url && !/^remote_workspace_[a-z_]*test$/.test(new URL(url).pathname.slice(1))) throw new Error("Disposable remote_workspace_*test database required"); const support = url ? { supported: true } : await getEmbeddedPostgresTestSupport(); async function versionedS3Fixture() { @@ -95,7 +97,8 @@ describe.skipIf(!support.supported)("remote workspace snapshot retention", () => let temporary: Awaited> | undefined; const now = new Date("2026-10-30T12:00:00Z"); beforeAll(async () => { - if (url) { await applyPendingMigrations(url); db = createDb(url); } + if (historyUrl) db = createDb(historyUrl, { maxConnections: 4 }); + else if (url) { await applyPendingMigrations(url); db = createDb(url); } else { temporary = await startEmbeddedPostgresTestDatabase("workspace-snapshot-retention-"); db = createDb(temporary.connectionString); } }, 90_000); afterAll(async () => { if (url) await closeRegisteredClients(url, { scope: "database" }); else await temporary?.cleanup(); }); @@ -320,6 +323,13 @@ describe.skipIf(!support.supported)("remote workspace snapshot retention", () => await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: f.issueId, kind: "active_run_watchdog", ownerType: "board", cause: "workspace_recovery", fingerprint: randomUUID(), nextAction: "Recover saved files" }); const owner = service(), coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => { throw error; } }); + const remove = storage.deleteObject.bind(storage); + let secondDeleteHeld = false, releaseSecondDelete!: () => void; + const secondDeleteReleased = new Promise(resolve => { releaseSecondDelete = resolve; }); + storage.deleteObject = async (...args) => { + if (args[1] === f.keys[1]) { secondDeleteHeld = true; await secondDeleteReleased; } + await remove(...args); + }; const worker = registerRemoteWorkspaceSnapshotRetentionWork(coordinator, owner, () => {}); try { await worker.ready; @@ -328,8 +338,23 @@ describe.skipIf(!support.supported)("remote workspace snapshot retention", () => const resolved = await issueRecoveryActionService(db).resolveActiveForIssue({ companyId: f.companyId, sourceIssueId: f.issueId, status: "resolved", outcome: "restored", resolutionNote: "Verified saved files" }); expect(resolved?.status).toBe("resolved"); - await vi.waitFor(async () => expect((await storage.headObject(f.companyId, f.keys[0]!)).exists).toBe(false)); - expect(await owner.nextDeadlineAt()).toBeNull(); - } finally { await worker.stop(); await coordinator.stop(); } + try { + await vi.waitFor(() => expect(secondDeleteHeld).toBe(true)); + expect((await storage.headObject(f.companyId, f.keys[0]!)).exists).toBe(false); + expect((await storage.headObject(f.companyId, f.keys[1]!)).exists).toBe(true); + expect((await db.select().from(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.id, f.row.id)))[0]) + .toMatchObject({ id: f.row.id, status: "delete_requested", deleteAfter: now }); + expect(await owner.nextDeadlineAt()).toBe(now.getTime()); + expect(idleWorkSnapshot().active).toBeGreaterThan(0); + } finally { releaseSecondDelete(); } + await vi.waitFor(async () => { + for (const key of f.keys) expect((await storage.headObject(f.companyId, key)).exists).toBe(false); + expect((await db.select().from(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.id, f.row.id)))[0]) + .toMatchObject({ id: f.row.id, status: "deleted", deleteAfter: null }); + expect(await owner.nextDeadlineAt()).toBeNull(); + expect(idleWorkSnapshot().active).toBe(0); + expect(coordinator.nextWakeAt()).toBeNull(); + }); + } finally { releaseSecondDelete(); await worker.stop(); await coordinator.stop(); } }); }); diff --git a/server/src/services/tool-action-idle-proof.ts b/server/src/services/tool-action-idle-proof.ts new file mode 100644 index 00000000000..a000039d235 --- /dev/null +++ b/server/src/services/tool-action-idle-proof.ts @@ -0,0 +1,28 @@ +import { finishedIdleRun } from "./idle-work-proof.js"; + +function completedRequest(a: string): string { + return `${a}.status IN ('executed', 'failed', 'rejected', 'expired', 'cancelled') AND ${a}.resolved_at IS NOT NULL + AND EXISTS (SELECT 1 FROM tool_invocations v WHERE v.id = ${a}.invocation_id AND v.company_id = ${a}.company_id + AND v.issue_id IS NOT DISTINCT FROM ${a}.issue_id AND v.completed_at IS NOT NULL + AND v.status IN ('succeeded', 'failed', 'denied', 'cancelled', 'timed_out', 'rate_limited') + AND (v.run_id IS NULL OR EXISTS (SELECT 1 FROM heartbeat_runs r WHERE r.id = v.run_id + AND r.company_id = v.company_id AND ${finishedIdleRun('r')}))) + AND (${a}.interaction_id IS NULL OR EXISTS (SELECT 1 FROM issue_thread_interactions i WHERE i.id = ${a}.interaction_id + AND i.company_id = ${a}.company_id AND i.issue_id = ${a}.issue_id AND i.resolved_at IS NOT NULL + AND i.payload->'toolAction'->>'actionRequestId' = ${a}.id::text + AND i.payload->'toolAction'->>'invocationId' = ${a}.invocation_id::text + AND i.status IN ('accepted', 'rejected', 'expired', 'cancelled') + AND (${a}.status = 'rejected' OR i.result->'toolAction'->>'status' = CASE + WHEN ${a}.status = 'cancelled' THEN 'expired' ELSE ${a}.status END)))`; +} + +export const idleToolActionHistoryChecks = [ + { reason: "tool_action_requests", query: `SELECT 1 FROM tool_action_requests a WHERE (${completedRequest('a')}) IS NOT TRUE` }, + // delivered_at is the domain's acknowledgment. It also retires outcomes + // after issue completion/reassignment. Accepted continuations stay covered by + // the global wake/run checks, including coalesced run accounting. + { reason: "tool_action_deliveries", query: `SELECT 1 FROM tool_action_deliveries d WHERE ( + d.delivered_at IS NOT NULL AND EXISTS (SELECT 1 FROM tool_action_requests a + WHERE a.id = d.action_request_id AND a.company_id = d.company_id + AND a.issue_id = d.issue_id AND a.interaction_id = d.interaction_id AND (${completedRequest('a')}))) IS NOT TRUE` }, +] as const; diff --git a/server/src/services/voice/idle-history-proof.ts b/server/src/services/voice/idle-history-proof.ts new file mode 100644 index 00000000000..2c76095f8e2 --- /dev/null +++ b/server/src/services/voice/idle-history-proof.ts @@ -0,0 +1,71 @@ +import { VOICE_CALL_REPORT_WINDOW_MS } from "./voice-call-reports.js"; + +/** Available reports still own refresh work for the owner's whole 24h window. + * The accepted push receipt proves submission, never unobservable playback. */ +export function completedVoiceAction(a: string): string { + const closedSession = (s: string) => `${s}.state IN ('ended', 'failed', 'expired') AND ${s}.ended_at IS NOT NULL + AND COALESCE(${s}.error_code, '') <> 'cleanup_pending' + AND (${s}.provider_session_id IS NULL OR ${s}.ended_at < now() - interval '${VOICE_CALL_REPORT_WINDOW_MS} milliseconds')`; + const submitted = (alias: string) => `(${alias}.status = 'completed' AND length(${alias}.result->>'messageId') > 0 + AND ${alias}.result->>'playback' = 'unknown' AND ${alias}.result->>'acceptedAt' ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}T') + OR (${alias}.status = 'call_ended' AND ${alias}.result->>'errorCode' = 'provider_unavailable') + OR (${alias}.status = 'failed' AND ${alias}.result->>'errorCode' IN + ('credentials_rejected', 'insufficient_credits', 'provider_unavailable', 'voice_authority_unavailable'))`; + // Nonnull company/endpoint keys let PostgreSQL hash each debt set once, + // rather than rescanning every old session for every retained voice marker. + const settled = `(${a}.company_id, ${a}.endpoint_id) NOT IN ( + SELECT session.company_id, session.endpoint_id FROM chat_voice_sessions session + WHERE (${closedSession('session')}) IS NOT TRUE) + AND (${a}.company_id, ${a}.endpoint_id) NOT IN ( + SELECT admission.company_id, admission.endpoint_id FROM chat_voice_inbound_calls admission + WHERE admission.state NOT IN ('denied', 'expired', 'ended') + OR admission.cleanup_confirmed_at IS NULL OR admission.cleanup_next_attempt_at IS NOT NULL) + AND (${a}.company_id, ${a}.endpoint_id) NOT IN ( + SELECT push.company_id, push.endpoint_id FROM chat_actions push + WHERE push.kind = 'speko_voice_reply_push' AND (${submitted('push')}) IS NOT TRUE)`; + // Configuration records this existing audit event only after tools, webhook, + // voice defaults and runtime setup all succeeded. A partial setup cannot use + // individual resource receipts to certify the rest of the owning operation. + const setupCommitted = `EXISTS (SELECT 1 FROM activity_log receipt JOIN chat_endpoints endpoint + ON endpoint.id = ${a}.endpoint_id AND endpoint.company_id = ${a}.company_id + WHERE receipt.company_id = ${a}.company_id AND receipt.entity_type = 'tool_connection' + AND receipt.entity_id = endpoint.connection_id::text AND receipt.details->>'endpointId' = endpoint.id::text + AND receipt.details->>'provider' = 'speko' AND receipt.action IN ('chat_endpoint.credentials_verified', 'chat_endpoint.reconnected') + AND receipt.created_at >= ${a}.updated_at)`; + return `EXISTS (SELECT 1 FROM chat_endpoints endpoint WHERE endpoint.id = ${a}.endpoint_id + AND endpoint.company_id = ${a}.company_id AND endpoint.provider = 'speko') AND ${settled} AND ( + (${a}.kind = 'speko_tool_setup' AND ${a}.status = 'completed' AND ${setupCommitted} + AND ${a}.payload->>'name' IN ('submit_request', 'get_updates', 'answer_question') + AND ${a}.payload->>'fingerprint' ~ '^[a-f0-9]{64}$' AND ${a}.payload->>'ownershipDefinitionHash' ~ '^[a-f0-9]{64}$' + AND length(${a}.payload->>'ownershipAgentId') > 0 AND length(${a}.result->>'toolId') > 0 + AND ${a}.provider_action_id = 'speko_tool:' || (${a}.payload->>'name') || ':' || (${a}.payload->>'fingerprint')) + OR (${a}.kind = 'speko_lifecycle_setup' AND ${a}.status = 'completed' AND ${setupCommitted} + AND ${a}.payload->>'fingerprint' ~ '^[a-f0-9]{64}$' AND length(${a}.payload->>'ownershipAgentId') > 0 + AND ${a}.payload->>'ownershipUrl' LIKE 'https://%/events' AND length(${a}.result->>'webhookId') > 0 + AND ${a}.provider_action_id = 'speko_lifecycle:' || (${a}.payload->>'fingerprint')) + OR (${a}.kind = 'speko_lifecycle_receipt' AND ${a}.status = 'completed' AND ${a}.result->>'accepted' = 'true' + AND ${a}.payload->>'type' IN ('call.status', 'call.report') AND ${a}.payload->>'fingerprint' ~ '^[a-f0-9]{64}$' + AND ${a}.provider_action_id LIKE 'speko_event:%' AND ( + EXISTS (SELECT 1 FROM chat_voice_sessions session WHERE session.id::text = ${a}.payload->>'sessionId' + AND session.company_id = ${a}.company_id AND session.endpoint_id = ${a}.endpoint_id AND ${closedSession('session')}) + OR (${a}.payload->'sessionId' = 'null'::jsonb AND EXISTS ( + SELECT 1 FROM chat_voice_inbound_calls admission WHERE admission.company_id = ${a}.company_id + AND admission.endpoint_id = ${a}.endpoint_id AND admission.state IN ('denied', 'expired', 'ended') + AND admission.cleanup_confirmed_at IS NOT NULL AND admission.cleanup_next_attempt_at IS NULL)))) + OR (${a}.kind = 'speko_voice_reply_push' AND (${submitted(a)}) + AND ${a}.payload->>'textDigest' ~ '^[a-f0-9]{64}$' + AND ${a}.payload->>'partIndex' ~ '^[0-9]+$' AND ${a}.payload->>'partCount' ~ '^[1-9][0-9]*$' + AND CASE WHEN ${a}.payload->>'partIndex' ~ '^[0-9]+$' AND ${a}.payload->>'partCount' ~ '^[1-9][0-9]*$' + THEN (${a}.payload->>'partIndex')::numeric < (${a}.payload->>'partCount')::numeric ELSE false END + AND EXISTS (SELECT 1 FROM chat_voice_sessions session JOIN chat_voice_replies reply + ON reply.session_id = session.id AND reply.company_id = session.company_id + JOIN chat_publications publication ON publication.id = reply.publication_id AND publication.company_id = reply.company_id + WHERE session.id::text = ${a}.payload->>'sessionId' AND session.company_id = ${a}.company_id + AND session.endpoint_id = ${a}.endpoint_id AND session.conversation_id = ${a}.conversation_id + AND session.provider_session_id = ${a}.payload->>'providerSessionId' AND session.generation::text = ${a}.payload->>'generation' + AND reply.id::text = ${a}.payload->>'replyId' AND publication.id::text = ${a}.payload->>'publicationId' + AND ${a}.provider_action_id = 'voice_reply_push:' || reply.id::text || CASE WHEN ${a}.payload->>'partIndex' = '0' + THEN '' ELSE ':part:' || (${a}.payload->>'partIndex') END + AND publication.endpoint_id = session.endpoint_id AND publication.conversation_id = session.conversation_id + AND publication.issue_id = session.issue_id AND publication.state = 'published' AND publication.published_at IS NOT NULL AND ${closedSession('session')})))`; +} diff --git a/server/src/services/voice/voice-call-reports.ts b/server/src/services/voice/voice-call-reports.ts index 8f597f9f16b..0290554133d 100644 --- a/server/src/services/voice/voice-call-reports.ts +++ b/server/src/services/voice/voice-call-reports.ts @@ -13,7 +13,8 @@ import { SpekoProviderError } from "./speko-provider.js"; type Entry = { id: string; companyId: string; endpointId: string; connectionId: string; dueAt: number }; type Report = SpekoCallReport; -const WINDOW_MS = 24 * 60 * 60 * 1000; +export const VOICE_CALL_REPORT_WINDOW_MS = 24 * 60 * 60 * 1000; +const WINDOW_MS = VOICE_CALL_REPORT_WINDOW_MS; const eligible = (now: number) => and(inArray(chatVoiceSessions.state, ["ended", "failed", "expired"]), isNotNull(chatVoiceSessions.providerSessionId), gte(chatVoiceSessions.endedAt, new Date(now - WINDOW_MS))); diff --git a/server/src/services/workspace-history-idle-proof.ts b/server/src/services/workspace-history-idle-proof.ts new file mode 100644 index 00000000000..d6d3f3ce428 --- /dev/null +++ b/server/src/services/workspace-history-idle-proof.ts @@ -0,0 +1,52 @@ +import { finishedIdleRun } from "./idle-work-proof.js"; + +export const idleWorkspaceHistoryChecks = [ + // Failed imports are intentionally resumable. Completed/cancelled rows can + // still owe spool deletion, independently inspected by idle-local-work. + { reason: "company_transfer_runs", query: `SELECT 1 FROM company_transfer_runs t WHERE ( + t.status IN ('completed', 'cancelled') AND t.finished_at IS NOT NULL + AND t.direction IN ('import', 'export') AND jsonb_typeof(t.container_ref) = 'object' + AND jsonb_typeof(t.completed_parts) = 'array' + AND (t.status = 'cancelled' OR (t.error IS NULL AND t.manifest_sha256 ~ '^[a-f0-9]{64}$' + AND CASE WHEN jsonb_typeof(t.manifest->'chunks') = 'array' AND jsonb_typeof(t.manifest->'blobs') = 'array' THEN + t.chunk_count = jsonb_array_length(t.manifest->'chunks') AND t.blob_count = jsonb_array_length(t.manifest->'blobs') + AND NOT EXISTS (SELECT 1 FROM jsonb_array_elements((t.manifest->'chunks') || (t.manifest->'blobs')) part + WHERE jsonb_typeof(part->'name') IS DISTINCT FROM 'string' OR NOT t.completed_parts ? (part->>'name')) + AND NOT EXISTS (SELECT 1 FROM jsonb_array_elements_text(t.completed_parts) done + WHERE NOT EXISTS (SELECT 1 FROM jsonb_array_elements((t.manifest->'chunks') || (t.manifest->'blobs')) part WHERE part->>'name' = done)) + WHEN jsonb_typeof(t.manifest->'parts') = 'array' THEN + t.chunk_count = jsonb_array_length(t.manifest->'parts') AND t.blob_count = 0 + AND NOT EXISTS (SELECT 1 FROM jsonb_array_elements(t.manifest->'parts') WITH ORDINALITY part(value, ordinal) + WHERE NOT t.completed_parts ? ('part-' || (ordinal - 1)::text)) + AND NOT EXISTS (SELECT 1 FROM jsonb_array_elements_text(t.completed_parts) done + WHERE NOT EXISTS (SELECT 1 FROM jsonb_array_elements(t.manifest->'parts') WITH ORDINALITY part(value, ordinal) + WHERE done = 'part-' || (ordinal - 1)::text)) + ELSE false END)) + AND (t.company_id IS NOT NULL OR (t.direction = 'import' AND t.status = 'cancelled')) + AND (t.company_id IS NULL OR EXISTS (SELECT 1 FROM companies c WHERE c.id = t.company_id))) IS NOT TRUE` }, + { reason: "workspace_runtime_services", query: `SELECT 1 FROM workspace_runtime_services s WHERE ( + s.status = 'stopped' AND s.stopped_at IS NOT NULL AND s.provider = 'local_process' + AND s.exposure_handle IS NULL + AND (s.exposure IS NULL OR (jsonb_typeof(s.exposure) = 'object' + AND s.exposure->>'state' = 'removed' AND s.url IS NULL + AND COALESCE(s.exposure->'publicUrl', 'null'::jsonb) = 'null'::jsonb)) + AND (s.execution_workspace_id IS NULL OR EXISTS (SELECT 1 FROM execution_workspaces w + WHERE w.id = s.execution_workspace_id AND w.company_id = s.company_id)) + AND (s.issue_id IS NULL OR EXISTS (SELECT 1 FROM issues i WHERE i.id = s.issue_id AND i.company_id = s.company_id)) + AND (s.started_by_run_id IS NULL OR EXISTS (SELECT 1 FROM heartbeat_runs r + WHERE r.id = s.started_by_run_id AND r.company_id = s.company_id AND ${finishedIdleRun('r')}))) IS NOT TRUE` }, + // This is an exclusivity row, not a provider lease. The domain only + // evaluates staleness on the next claim; no expiry job is owed by history. + { reason: "execution_workspace_runtime_leases", query: `SELECT 1 FROM execution_workspace_runtime_leases l WHERE ( + EXISTS (SELECT 1 FROM execution_workspaces w WHERE w.id = l.execution_workspace_id AND w.company_id = l.company_id) + AND ((l.owner_issue_id IS NOT NULL AND l.owner_key = 'issue:' || l.owner_issue_id::text + AND EXISTS (SELECT 1 FROM issues i WHERE i.id = l.owner_issue_id AND i.company_id = l.company_id + AND i.status IN ('done', 'cancelled'))) + OR (l.owner_issue_id IS NULL AND l.owner_run_id IS NOT NULL AND l.owner_key = 'run:' || l.owner_run_id::text + AND EXISTS (SELECT 1 FROM heartbeat_runs r WHERE r.id = l.owner_run_id AND r.company_id = l.company_id + AND r.agent_id = l.owner_agent_id AND ${finishedIdleRun('r')}))) + AND (l.owner_run_id IS NULL OR EXISTS (SELECT 1 FROM heartbeat_runs r WHERE r.id = l.owner_run_id + AND r.company_id = l.company_id AND r.agent_id = l.owner_agent_id AND ${finishedIdleRun('r')})) + AND NOT EXISTS (SELECT 1 FROM workspace_runtime_services s WHERE s.execution_workspace_id = l.execution_workspace_id + AND (s.company_id <> l.company_id OR s.status <> 'stopped' OR s.stopped_at IS NULL))) IS NOT TRUE` }, +] as const; diff --git a/server/src/services/workspace-runtime.ts b/server/src/services/workspace-runtime.ts index 96f0b39edff..8a6211968a6 100644 --- a/server/src/services/workspace-runtime.ts +++ b/server/src/services/workspace-runtime.ts @@ -46,6 +46,7 @@ import { import { and, desc, eq, gte, inArray, isNull, lte, ne, or, sql } from "drizzle-orm"; import { asNumber, asString, parseObject, renderTemplate } from "../adapters/utils.js"; import { conflict, unprocessable } from "../errors.js"; +import { registerIdleLocalWorkProbe, type IdleLocalWork } from "./idle-local-work.js"; import { withExecutionWorkspaceLifecycleLock } from "./execution-workspace-lifecycle-lock.js"; import { resolveHomeAwarePath } from "../home-paths.js"; import { hasVerifiedWorktreeSeedManifest, isVerifiedWorktreeSeedManifest } from "../worktree-seed-manifest.js"; @@ -283,6 +284,23 @@ const quarantinedRuntimeExposurePorts = new Set(); * companion together or not at all. */ const exposurePortPairClaims = new ExposurePortPairClaims(); +let runtimeReconstructionComplete = false; +let runtimeStopsInFlight = 0; +let runtimeReconstructionEpoch = 0; +let runtimeReconciliation: ReturnType | null = null; + +/** Read owner memory only. Durable stopped rows do not describe a surviving + * registry process, an allocation being published, or unfinished exposure cleanup. */ +export function readWorkspaceRuntimeIdleWork(): IdleLocalWork { + if (!runtimeReconstructionComplete || runtimeReconciliation !== null) return "unknown"; + return runtimeStopsInFlight || runtimeServicesById.size || runtimeProvisionByWorkspace.size || runtimeControlStartByOwner.size + || runtimeReplacementClaimsByReuseKey.size || quarantinedRuntimeExposurePorts.size + || exposurePortPairClaims.activePorts().size + || [...inFlightAllocatedPorts.values()].some(until => until > Date.now()) ? "present" : "none"; +} +// This registry and its probe share the application process lifetime, like the +// native-session owner. Stopping one resource must not unregister its siblings. +registerIdleLocalWorkProbe(readWorkspaceRuntimeIdleWork); /** * Execution-workspace statuses that still hold an exclusive lease. `archived` * is the only terminal state; everything else — including `idle` — is a lane an @@ -525,6 +543,8 @@ type ProcessOutputAccumulator = { export async function resetRuntimeServicesForTests( opts: { terminateProcesses?: boolean; simulateSupervisorExit?: boolean } = {}, ) { + runtimeReconstructionComplete = false; + runtimeReconstructionEpoch += 1; if (opts.terminateProcesses) { for (const serviceId of [...runtimeServicesById.keys()]) { await stopRuntimeService(serviceId).catch(() => undefined); @@ -7155,7 +7175,21 @@ async function cleanupRecordExposure( await persistRuntimeServiceRecord(record.db, record).catch(() => undefined); } +async function withRuntimeStopWork(stop: () => Promise): Promise { + runtimeStopsInFlight += 1; + try { return await stop(); } + catch (error) { + runtimeReconstructionComplete = false; + runtimeReconstructionEpoch += 1; + throw error; + } finally { runtimeStopsInFlight -= 1; } +} + async function stopRuntimeService(serviceId: string) { + return withRuntimeStopWork(() => stopRuntimeServiceRecord(serviceId)); +} + +async function stopRuntimeServiceRecord(serviceId: string) { const record = runtimeServicesById.get(serviceId); if (!record) return; clearIdleTimer(record); @@ -8296,7 +8330,11 @@ export async function releaseRuntimeServicesForRun(runId: string) { } } -export async function stopRuntimeServicesForExecutionWorkspace(input: { +export async function stopRuntimeServicesForExecutionWorkspace(input: Parameters[0]) { + return withRuntimeStopWork(() => stopRuntimeServicesForExecutionWorkspaceOwned(input)); +} + +async function stopRuntimeServicesForExecutionWorkspaceOwned(input: { db?: Db; executionWorkspaceId: string; workspaceCwd?: string | null; @@ -8351,7 +8389,11 @@ export async function stopRuntimeServicesForExecutionWorkspace(input: { } } -export async function stopRuntimeServicesForProjectWorkspace(input: { +export async function stopRuntimeServicesForProjectWorkspace(input: Parameters[0]) { + return withRuntimeStopWork(() => stopRuntimeServicesForProjectWorkspaceOwned(input)); +} + +async function stopRuntimeServicesForProjectWorkspaceOwned(input: { db?: Db; projectWorkspaceId: string; runtimeServiceId?: string | null; @@ -8577,6 +8619,19 @@ export async function refreshPersistedRuntimeServiceHealth(input: { } export async function reconcilePersistedRuntimeServicesOnStartup(db: Db) { + if (runtimeReconciliation) return runtimeReconciliation; + runtimeReconstructionComplete = false; + const epoch = ++runtimeReconstructionEpoch; + runtimeReconciliation = reconstructRuntimeServices(db).then(result => { + if (epoch === runtimeReconstructionEpoch) { + runtimeReconstructionComplete = result.restartFailed === 0 && result.exposureReservationDrift.length === 0; + } + return result; + }).finally(() => { runtimeReconciliation = null; }); + return runtimeReconciliation; +} + +async function reconstructRuntimeServices(db: Db) { const rows = await db .select() .from(workspaceRuntimeServices) diff --git a/server/src/storage/local-disk-provider.ts b/server/src/storage/local-disk-provider.ts index 9ee950117f3..b92f1eea05f 100644 --- a/server/src/storage/local-disk-provider.ts +++ b/server/src/storage/local-disk-provider.ts @@ -30,8 +30,10 @@ function resolveWithin(baseDir: string, objectKey: string): string { async function statOrNull(filePath: string) { try { return await fs.stat(filePath); - } catch { - return null; + } catch (error) { + const code = (error as NodeJS.ErrnoException | null)?.code; + if (code === "ENOENT" || code === "ENOTDIR") return null; + throw error; } } @@ -89,8 +91,9 @@ export function createLocalDiskStorageProvider(baseDir: string): StorageProvider const filePath = resolveWithin(root, input.objectKey); try { await fs.unlink(filePath); - } catch { - // idempotent delete + } catch (error) { + const code = (error as NodeJS.ErrnoException | null)?.code; + if (code !== "ENOENT" && code !== "ENOTDIR") throw error; } }, };