From 5847f047c7329e81296f1345cb54b9850056dfef Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Sun, 11 Oct 2026 09:30:45 -0700 Subject: [PATCH 01/18] fix(connections): retire requests for already available tools (#15856) ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - Connection cards let users provide the access an agent needs. > - Users can complete setup in Apps while an earlier card stays pending. > - A stale card can then cancel recovery despite valid existing access. > - This change verifies existing access and expires the obsolete request. > - It records a system outcome and sends one durable continuation. > - Approval, identity, permission, and workspace recovery gates remain in place. ## Linked Issues or Issue Description **What happened?** An ordinary tool connection request stayed pending after the addressed user connected the service in Apps. A later conversation retry was cancelled with `issue_waiting_for_response`. **Expected behavior** A request expires when the same user and requesting agent already have usable access. The agent gets a durable tool refresh. The system records no new human approval. **Steps to reproduce** 1. Have an agent request a tool connection on an active task. 2. Connect the service through Apps with the addressed user's existing grant, agent install, and allowed tools. 3. Let a conversation run stop and retry. The pending card still blocks the retry on the base branch. Related: #12345 introduced the connection request flow. This fix uses its existing access checks and delivery queue. ## What Changed - Verify company, task ownership, user identity, membership, health, grants, installs, and task tool policy before expiring an ordinary request. - Preserve AI account, channel, additional permission, and upstream provider requests on their existing paths. - Save the system outcome, audit event, and continuation in one transaction. Notify an idle delivery worker when a request commits. - Recheck access and other questions or approvals before delivery, retry, final dispatch, and native projection. Refresh system observations for an authorized replacement account. - Rearm delivery when access changes before provider work starts. Verify the host's admission receipt against execution evidence before allowing retry without a repair incident. - Require the current delivery claim when marking an attempt complete. An old completion cannot overwrite a rearmed continuation. - Add embedded-Postgres regressions for permissions, policy, concurrency, restart, replacement accounts, both dispatch gates, and the delivery race. Keep genuine workspace repair holds blocking. ## Verification - Latest-head connection, wake-context, dispatch, and legacy recovery suites: 124 tests passed. The connection suite includes 44 real-Postgres cases. - Server typecheck passed. Full `pnpm -r typecheck` and `pnpm build` passed on the latest head. - `pnpm test:run` was run locally. Its general-server group passed 18,583 tests and failed 20. That long run started before the review fixes, then loaded new test cases against modules transformed before those fixes. The affected latest-head suites pass in a fresh process. - One full-run failure also reproduces on unchanged base commit de9ab8e7e06: native-session-resume's damaged-prior-epoch replacement case. It is outside this change. Full-run groups after general-server did not run because that command stopped on failure. - Apex reviewed commit 96b96297f41 at 5/5. All review threads are resolved. All latest-head CI gates passed. The branch has no merge conflicts. ## Risks A false readiness result could retire a request too early. The fix reuses the runtime access selection rules, makes no new grants, and rechecks access at delivery. Ordinary expired requests do not authorize continuation. A failed workspace restore still requires repair through the existing gate. No schema migration is needed. ## Model Used OpenAI GPT-6 through Codex, with repository inspection, code editing, shell execution, and test tools. The session identifies the model family as GPT-6. It does not expose a more specific runtime model ID or context window size. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass (focused suites; full local limitations are listed above) - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- doc/execution-semantics.md | 23 ++ .../satisfied-connection-intents.test.ts | 388 ++++++++++++++++++ .../modules/run-dispatch/adapters/postgres.ts | 54 ++- .../src/modules/run-dispatch/domain/policy.ts | 8 + .../run-dispatch/domain/wake-context.test.ts | 7 + .../run-dispatch/domain/wake-context.ts | 5 +- .../services/connection-intent-delivery.ts | 76 +++- .../src/services/issue-thread-interactions.ts | 1 + .../native-interaction-bridge.ts | 16 +- .../src/services/pre-dispatch-review-wait.ts | 20 +- .../services/satisfied-connection-intents.ts | 133 ++++++ server/src/services/tool-gateway.ts | 6 + 12 files changed, 703 insertions(+), 34 deletions(-) create mode 100644 server/src/__tests__/satisfied-connection-intents.test.ts create mode 100644 server/src/services/satisfied-connection-intents.ts diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 0be0461e71e..58e352d9a17 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1602,6 +1602,29 @@ question does not start setup. The real connection card keeps user identity, access grants, the decision, and continuation together. This guidance does not approve a connection or bypass its normal user decision. +An ordinary tool connection request becomes obsolete when an already authorized +connection is usable by its requesting agent with the addressed user's identity. +Startup and periodic connection delivery reconcile that condition, retire the +card as expired, and record a system decision without a fabricated human approval. +The saved continuation is delivered once after rechecking the same access. The +scheduled retry gate also verifies this condition before the next sweep, so the +obsolete card cannot cancel a valid retry. Missing, unhealthy, disabled, ambiguous, +or unauthorized identities remain blocked. Requests for additional tool access, +AI authentication, inbox setup, or an upstream aggregator service keep their normal +approval paths. Task ownership, real questions, approvals, budgets, pauses, and +workspace recovery checks still govern admission. + +New requests notify the connection worker so this reconciliation also starts after an idle startup scan. Readiness uses the gateway's runtime tool +listing policy with issue, project, and run context. System-retired requests are +rechecked against the next run's responsible user and current access, and saved +continuations defer while another question or linked approval remains pending. +Those checks run again during queued and final dispatch admission. An unstarted +system continuation cancelled for a new response wait rearms its delivery in the +same transaction. Verified absence of execution keeps that wait out of workspace +repair incidents. Historical retirements do not bind all future retries to an +old account; an already authorized replacement can refresh the system observation +with its own audit record before delivery. Human decisions remain unchanged. + ## Responses submitted during an active run A confirmation, checkbox confirmation, or question answer is new conversation diff --git a/server/src/__tests__/satisfied-connection-intents.test.ts b/server/src/__tests__/satisfied-connection-intents.test.ts new file mode 100644 index 00000000000..ed37e0b6337 --- /dev/null +++ b/server/src/__tests__/satisfied-connection-intents.test.ts @@ -0,0 +1,388 @@ +import { randomUUID } from "node:crypto"; +import { eq, sql } from "drizzle-orm"; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { activityLog, agentWakeupRequests, agents, approvals, authUsers, companies, companyMemberships, connectionGrants, connectionGrantDelegations, + connectionIntentDeliveries, createDb, environmentLeases, heartbeatRunEvents, heartbeatRuns, issueApprovals, issueRecoveryActions, issueThreadInteractions, issues, projects, toolApplications, + toolCatalogEntries, toolConnectionInstalls, toolConnections, toolProfileBindings, toolProfiles, toolPolicies, +} from "@paperclipai/db"; +import { createPostgresRunDispatchAdapter } from "../modules/run-dispatch/adapters/postgres.js"; +import { connectionIntentDeliveryService } from "../services/connection-intent-delivery.js"; +import { findSatisfiedToolConnection, satisfiedConnectionIntentService } from "../services/satisfied-connection-intents.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { materializeNativeInteractionResponses } from "../services/native-runtime/native-interaction-bridge.js"; +import { DELIVERY_QUEUES, subscribeDeliveryWork } from "../services/delivery-work-notifications.js"; +import { issueThreadInteractionService } from "../services/issue-thread-interactions.js"; +import { legacyExecutionNeedsReconciliationWithEvidence } from "../services/legacy-execution-recovery.js"; +import { remoteTerminationReceipt } from "../services/remote-execution-termination.js"; + +const support = await getEmbeddedPostgresTestSupport(); +const describePostgres = support.supported ? describe : describe.skip; + +describePostgres("already available connection requests", () => { + let db!: ReturnType; + let temp!: Awaited>; + beforeAll(async () => { + temp = await startEmbeddedPostgresTestDatabase("paperclip-satisfied-connections-"); + db = createDb(temp.connectionString); + await db.insert(authUsers).values({ id: "connection-owner", name: "Owner", email: "owner@example.test", createdAt: new Date(), updatedAt: new Date() }); + }, 20_000); + beforeEach(async () => { await db.execute(sql`truncate table companies cascade`); }); + afterAll(async () => { await db.$client.end({ timeout: 1 }); await temp?.cleanup(); }); + + async function seed() { + const companyId = randomUUID(); + const agentId = randomUUID(); + const issueId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Connections", issuePrefix: "CON", defaultResponsibleUserId: "connection-owner" }); + await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: "connection-owner", status: "active", membershipRole: "member" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Assignee", role: "engineer", adapterType: "claude_local", status: "idle", runtimeConfig: { heartbeat: { wakeOnDemand: true } } }); + const [task] = await db.insert(issues).values({ id: issueId, companyId, title: "Use Sentry", status: "in_review", assigneeAgentId: agentId, responsibleUserId: "connection-owner" }).returning(); + const [intent] = await db.insert(issueThreadInteractions).values({ companyId, issueId, kind: "connection_intent", status: "pending", addresseeUserId: "connection-owner", + requestedResolverPolicy: "human_only", effectiveResolverPolicy: "human_only", payload: { version: 1, serviceSlug: "sentry", serviceName: "Sentry", requestingAgentId: agentId, requestingAgentName: "Assignee", phase: "requested" } }).returning(); + const [app] = await db.insert(toolApplications).values({ companyId, applicationKey: "sentry", name: "Sentry", type: "mcp_http", status: "active", metadata: { sourceTemplateKey: "sentry" } }).returning(); + const [connection] = await db.insert(toolConnections).values({ companyId, applicationId: app!.id, name: "Existing Sentry", uid: "sentry", transport: "mcp_remote", authKind: "api_key", credentialPolicy: "per_user", status: "active", enabled: true, healthStatus: "ok", config: { sourceTemplateKey: "sentry" } }).returning(); + const [grant] = await db.insert(connectionGrants).values({ companyId, connectionId: connection!.id, kind: "user", subjectUserId: "connection-owner", status: "active" }).returning(); + await db.insert(toolConnectionInstalls).values({ companyId, connectionId: connection!.id, targetType: "agent", targetId: agentId }); + const [tool] = await db.insert(toolCatalogEntries).values({ companyId, connectionId: connection!.id, toolName: "sentry-read", name: "sentry-read", versionHash: "v1", status: "active", entryKind: "tool" }).returning(); + const [profile] = await db.insert(toolProfiles).values({ companyId, profileKey: "reads", name: "Reads", defaultAction: "allow", status: "active" }).returning(); + await db.insert(toolProfileBindings).values({ companyId, profileId: profile!.id, targetType: "agent", targetId: agentId }); + return { companyId, agentId, issueId, task: task!, intent: intent!, connection: connection!, grant: grant!, tool: tool!, profile: profile! }; + } + async function retry(f: Awaited>) { + const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, issueId: f.issueId, + status: "scheduled_retry", scheduledRetryReason: "transient_failure", scheduledRetryAt: new Date(), runtimeMode: "legacy", + responsibleUserId: "connection-owner", contextSnapshot: { issueId: f.issueId }, resultJson: { conversationContinuation: "continue_conversation_v1" } }).returning(); + return run!; + } + + it("ignores an already available connection before the worker retires its card", async () => { + const f = await seed(); const run = await retry(f); + expect(await createPostgresRunDispatchAdapter(db).evaluateScheduledRetryGate({ companyId: f.companyId, runId: run.id, now: new Date() })).toMatchObject({ allowed: true }); + expect((await db.select().from(issueThreadInteractions))[0].status).toBe("pending"); + }); + + it("keeps a retry under another user's identity waiting for its response", async () => { + const f = await seed(); const run = await retry(f); + await db.update(heartbeatRuns).set({ responsibleUserId: "someone-else" }).where(eq(heartbeatRuns.id, run.id)); + expect(await createPostgresRunDispatchAdapter(db).evaluateScheduledRetryGate({ companyId: f.companyId, runId: run.id, now: new Date() })).toMatchObject({ allowed: false, errorCode: "issue_waiting_for_response" }); + }); + + it("notifies an idle worker when a new connection request commits", async () => { + const f = await seed(); const source = await retry(f); + await db.delete(issueThreadInteractions); + const wakeup = vi.fn().mockResolvedValue(null); + const worker = connectionIntentDeliveryService(db, { wakeup }); + expect(await worker.hasPending()).toBe(false); + const notified = vi.fn(); + const unsubscribe = subscribeDeliveryWork(db, DELIVERY_QUEUES.connection, notified); + try { + await issueThreadInteractionService(db).createConnectionIntent({ id: f.issueId, companyId: f.companyId }, { + payload: f.intent.payload, sourceRunId: source.id, addresseeUserId: "connection-owner", idempotencyKey: "new-card", + }); + expect(notified).toHaveBeenCalledOnce(); + expect(await worker.hasPending()).toBe(true); + await worker.sweepPending(); + expect(wakeup).toHaveBeenCalledOnce(); + } finally { unsubscribe(); } + }); + + it.each(["wrong_identity", "revoked", "disabled", "removed_member", "denied_policy"] as const)("rechecks %s after the card retires", async (reason) => { + const f = await seed(); const run = await retry(f); + await satisfiedConnectionIntentService(db).sweepPending(); + if (reason === "wrong_identity") await db.update(heartbeatRuns).set({ responsibleUserId: "someone-else" }); + if (reason === "revoked") await db.update(connectionGrants).set({ status: "revoked" }); + if (reason === "disabled") await db.update(toolConnections).set({ enabled: false }); + if (reason === "removed_member") await db.delete(companyMemberships); + if (reason === "denied_policy") await db.insert(toolPolicies).values({ companyId: f.companyId, name: "Block this task", policyType: "block", selectors: { issueId: f.issueId } }); + expect(await createPostgresRunDispatchAdapter(db).evaluateScheduledRetryGate({ companyId: f.companyId, runId: run.id, now: new Date() })).toMatchObject({ allowed: false, errorCode: "issue_waiting_for_response" }); + }); + + it.each(["company", "issue", "project"] as const)("uses %s policy when deciding whether tools are ready", async (scope) => { + const f = await seed(); + const selectors = scope === "issue" ? { issueId: f.issueId } : scope === "project" ? { projectId: randomUUID() } : {}; + if (scope === "project") { + const projectId = (selectors as { projectId: string }).projectId; + await db.insert(projects).values({ id: projectId, companyId: f.companyId, name: "Tools" }); + await db.update(issues).set({ projectId }); + } + await db.insert(toolPolicies).values({ companyId: f.companyId, name: "Block tools", policyType: "block", selectors }); + expect(await satisfiedConnectionIntentService(db).sweepPending()).toMatchObject({ satisfied: 0 }); + expect((await db.select().from(issueThreadInteractions))[0].status).toBe("pending"); + }); + + it("retains invocation approval when the authorized connection's tools require it", async () => { + const f = await seed(); + await db.insert(toolPolicies).values({ companyId: f.companyId, name: "Ask before use", policyType: "require_approval", selectors: { issueId: f.issueId } }); + expect(await satisfiedConnectionIntentService(db).sweepPending()).toMatchObject({ satisfied: 1 }); + expect((await db.select().from(toolPolicies))[0].policyType).toBe("require_approval"); + expect(await db.select().from(approvals)).toHaveLength(0); + expect((await db.select().from(issueThreadInteractions))[0].resolvedByUserId).toBeNull(); + }); + + async function addWait(f: Awaited>, kind: "question" | "approval") { + if (kind === "question") { + const [question] = await db.insert(issueThreadInteractions).values({ companyId: f.companyId, issueId: f.issueId, + kind: "ask_user_questions", status: "pending", payload: { version: 1, questions: [] } }).returning(); + return async () => { await db.update(issueThreadInteractions).set({ status: "answered" }).where(eq(issueThreadInteractions.id, question!.id)); }; + } + const [approval] = await db.insert(approvals).values({ companyId: f.companyId, type: "tool_action", status: "pending", payload: {} }).returning(); + await db.insert(issueApprovals).values({ companyId: f.companyId, issueId: f.issueId, approvalId: approval!.id }); + return async () => { await db.update(approvals).set({ status: "approved" }).where(eq(approvals.id, approval!.id)); }; + } + + it.each(["question", "approval"] as const)("defers the saved system continuation for another %s", async (kind) => { + const f = await seed(); const clearWait = await addWait(f, kind); + const wakeup = vi.fn().mockResolvedValue(null); + const worker = connectionIntentDeliveryService(db, { wakeup }); + await worker.sweepPending(); + expect((await db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.id, f.intent.id)))[0].status).toBe("expired"); + expect(wakeup).not.toHaveBeenCalled(); + expect((await db.select().from(connectionIntentDeliveries))[0].deliveredAt).toBeNull(); + await clearWait(); + await db.update(connectionIntentDeliveries).set({ nextAttemptAt: new Date() }); + await worker.sweepPending(); + expect(wakeup).toHaveBeenCalledOnce(); + }); + + it.each(["question", "approval", "revoked"] as const)("rechecks a queued system continuation after %s changes", async (kind) => { + const f = await seed(); const run = await retry(f); + await satisfiedConnectionIntentService(db).sweepPending(); + await db.update(heartbeatRuns).set({ status: "queued", contextSnapshot: { issueId: f.issueId, + connectionIntentResolution: "existing_connection", interactionKind: "connection_intent", interactionStatus: "expired", + mutation: "interaction", wakeReason: "issue_commented", source: "connection_intent.resolved" } }).where(eq(heartbeatRuns.id, run.id)); + if (kind === "revoked") await db.update(connectionGrants).set({ status: "revoked" }); + else await addWait(f, kind); + expect(await createPostgresRunDispatchAdapter(db).cancelStaleQueuedRun({ companyId: f.companyId, runId: run.id, + expectedStatus: "queued", now: new Date() })).toMatchObject({ outcome: "cancelled", errorCode: "issue_waiting_for_response" }); + }); + + async function replaceAccount(f: Awaited>) { + await db.update(toolConnections).set({ enabled: false }).where(eq(toolConnections.id, f.connection.id)); + await db.update(connectionGrants).set({ status: "revoked" }).where(eq(connectionGrants.id, f.grant.id)); + const [connection] = await db.insert(toolConnections).values({ ...f.connection, id: randomUUID(), uid: "sentry-replacement", name: "Replacement Sentry" }).returning(); + await db.insert(connectionGrants).values({ ...f.grant, id: randomUUID(), connectionId: connection!.id, status: "active" }); + await db.insert(toolConnectionInstalls).values({ companyId: f.companyId, connectionId: connection!.id, targetType: "agent", targetId: f.agentId }); + await db.insert(toolCatalogEntries).values({ ...f.tool, id: randomUUID(), connectionId: connection!.id }); + return connection!; + } + + it("allows a later approved replacement instead of permanently binding retries to an old account", async () => { + const f = await seed(); const run = await retry(f); + await satisfiedConnectionIntentService(db).sweepPending(); + const replacement = await replaceAccount(f); + await db.insert(issueThreadInteractions).values({ companyId: f.companyId, issueId: f.issueId, kind: "connection_intent", + status: "accepted", payload: f.intent.payload, addresseeUserId: "connection-owner", resolvedByUserId: "connection-owner", + result: { version: 1, outcome: "accepted", connectionId: replacement.id }, resolvedAt: new Date() }); + expect(await createPostgresRunDispatchAdapter(db).evaluateScheduledRetryGate({ companyId: f.companyId, runId: run.id, now: new Date() })).toMatchObject({ allowed: true }); + }); + + it("refreshes only the system observation for an already authorized replacement", async () => { + const f = await seed(); const run = await retry(f); + await satisfiedConnectionIntentService(db).sweepPending(); + const replacement = await replaceAccount(f); + const input = { db, companyId: f.companyId, issueId: f.issueId, runId: run.id, agentId: f.agentId, interactionIds: [f.intent.id] }; + await expect(materializeNativeInteractionResponses(input)).rejects.toMatchObject({ code: "native_interaction_unresolved" }); + await connectionIntentDeliveryService(db, { wakeup: vi.fn().mockResolvedValue(null) }).sweepPending(); + expect((await db.select().from(issueThreadInteractions))[0]).toMatchObject({ status: "expired", resolvedByUserId: null, + result: { outcome: "expired", connectionId: replacement.id } }); + expect(await materializeNativeInteractionResponses(input)).toEqual([expect.objectContaining({ + response: expect.objectContaining({ result: expect.objectContaining({ connectionId: replacement.id }) }) })]); + expect(await db.select().from(activityLog)).toEqual(expect.arrayContaining([expect.objectContaining({ actorType: "system", + details: expect.objectContaining({ previousConnectionId: f.connection.id, connectionId: replacement.id }) })])); + }); + + it.each(["queued", "running"] as const)("rearms an unstarted %s wait without manufacturing a workspace repair hold", async (status) => { + const f = await seed(); const run = await retry(f); + await satisfiedConnectionIntentService(db).sweepPending(); + const key = `connection-intent:${f.intent.id}:expired`; + const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agentId, + source: "automation", status: "claimed", idempotencyKey: key, runId: run.id }).returning(); + await db.update(connectionIntentDeliveries).set({ deliveredAt: new Date() }); + await db.update(heartbeatRuns).set({ status, startedAt: status === "running" ? new Date() : null, + runtimeModeResolvedAt: status === "running" ? new Date() : null, + controllerBootId: status === "running" ? randomUUID() : null, + executionStage: status === "running" ? "preparing" : null, wakeupRequestId: wake!.id, resultJson: null, + contextSnapshot: { issueId: f.issueId, interactionId: f.intent.id, connectionIntentResolution: "existing_connection", + interactionKind: "connection_intent", interactionStatus: "expired", mutation: "interaction", wakeReason: "issue_commented", source: "connection_intent.resolved" } }).where(eq(heartbeatRuns.id, run.id)); + await db.update(connectionGrants).set({ status: "revoked" }); + const dispatch = createPostgresRunDispatchAdapter(db); + if (status === "queued") { + expect(await dispatch.cancelStaleQueuedRun({ companyId: f.companyId, runId: run.id, + expectedStatus: "queued", now: new Date() })).toMatchObject({ outcome: "cancelled", errorCode: "issue_waiting_for_response" }); + } else { + const invoke = vi.fn(async () => null); + expect(await dispatch.dispatchResolvedInteractionIfCurrent({ companyId: f.companyId, runId: run.id, + expectedStatus: "running", now: new Date(), dispatch: invoke })).toMatchObject({ dispatched: false, + cancellation: { outcome: "cancelled", errorCode: "issue_waiting_for_response" } }); + expect(invoke).not.toHaveBeenCalled(); + } + const cancelled = (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0]; + expect(await legacyExecutionNeedsReconciliationWithEvidence(db, cancelled)).toBe(false); + expect((await db.select().from(connectionIntentDeliveries))[0].deliveredAt).toBeNull(); + expect((await db.select().from(agentWakeupRequests))[0].status).toBe("skipped"); + await db.update(connectionGrants).set({ status: "active" }); + const wakeup = vi.fn(async () => { + await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agentId, source: "automation", status: "queued", idempotencyKey: key }); + return null; + }); + await connectionIntentDeliveryService(db, { wakeup }).sweepPending(); + expect(wakeup).toHaveBeenCalledOnce(); + expect((await db.select().from(connectionIntentDeliveries))[0].deliveredAt).not.toBeNull(); + await db.insert(heartbeatRunEvents).values({ companyId: f.companyId, runId: run.id, agentId: f.agentId, + seq: 2, eventType: "adapter.invoke" }); + expect(await legacyExecutionNeedsReconciliationWithEvidence(db, cancelled)).toBe(true); + }); + + it("does not let an old delivery complete after cancellation rearms its continuation", async () => { + const f = await seed(); const run = await retry(f); + await satisfiedConnectionIntentService(db).sweepPending(); + const key = `connection-intent:${f.intent.id}:expired`; + const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agentId, + source: "automation", status: "claimed", idempotencyKey: key, runId: run.id }).returning(); + await db.update(heartbeatRuns).set({ status: "queued", wakeupRequestId: wake!.id, + contextSnapshot: { issueId: f.issueId, interactionId: f.intent.id, connectionIntentResolution: "existing_connection", + interactionKind: "connection_intent", interactionStatus: "expired", mutation: "interaction", source: "connection_intent.resolved" } }).where(eq(heartbeatRuns.id, run.id)); + let cancelled = false; + // Pause precisely after durableWake() succeeds, before its completion write. + const racingDb = new Proxy(db, { get(target, property) { + if (property !== "update") return Reflect.get(target, property); + return (table: Parameters[0]) => { + const builder = db.update(table); + if (table !== connectionIntentDeliveries) return builder; + return { set(values: Parameters[0]) { + const update = builder.set(values); + if (!values.deliveredAt) return update; + return { async where(condition: Parameters[0]) { + await db.update(connectionGrants).set({ status: "revoked" }); + expect(await createPostgresRunDispatchAdapter(db).cancelStaleQueuedRun({ companyId: f.companyId, + runId: run.id, expectedStatus: "queued", now: new Date() })).toMatchObject({ outcome: "cancelled" }); + cancelled = true; + return update.where(condition); + } }; + } }; + }; + } }); + await connectionIntentDeliveryService(racingDb, { wakeup: vi.fn() }).deliver(f.intent.id); + expect(cancelled).toBe(true); + expect((await db.select().from(connectionIntentDeliveries))[0].deliveredAt).toBeNull(); + await db.update(connectionGrants).set({ status: "active" }); + const wakeup = vi.fn(async () => { await db.insert(agentWakeupRequests).values({ companyId: f.companyId, + agentId: f.agentId, source: "automation", status: "queued", idempotencyKey: key }); return null; }); + await connectionIntentDeliveryService(db, { wakeup }).sweepPending(); + expect(wakeup).toHaveBeenCalledOnce(); + expect((await db.select().from(connectionIntentDeliveries))[0].deliveredAt).not.toBeNull(); + }); + + it("records a system completion and one durable continuation with a one-connection pool", async () => { + const f = await seed(); + const single = createDb(temp.connectionString, { maxConnections: 1 }); + try { + const outcomes = await Promise.all([satisfiedConnectionIntentService(single).sweepPending(), satisfiedConnectionIntentService(db).sweepPending()]); + expect(outcomes.reduce((sum, outcome) => sum + outcome.satisfied, 0)).toBe(1); + expect((await db.select().from(issueThreadInteractions))[0]).toMatchObject({ status: "expired", resolvedByUserId: null, result: { outcome: "expired", connectionId: f.connection.id } }); + expect(await db.select().from(connectionIntentDeliveries)).toHaveLength(1); + expect(await db.select().from(activityLog)).toEqual([expect.objectContaining({ actorType: "system", action: "issue.thread_interaction_resolved" })]); + expect(await db.select().from(connectionGrantDelegations)).toHaveLength(0); + expect(await db.select().from(toolPolicies)).toHaveLength(0); + expect(await db.select().from(connectionGrants)).toHaveLength(1); + expect(await db.select().from(toolConnectionInstalls)).toHaveLength(1); + expect(await satisfiedConnectionIntentService(single).sweepPending()).toMatchObject({ satisfied: 0 }); + } finally { await single.$client.end({ timeout: 1 }); } + }); + + it("discovers pending cards through the existing delivery worker", async () => { + const f = await seed(); const wakeup = vi.fn().mockResolvedValue(null); + const worker = connectionIntentDeliveryService(db, { wakeup }); + expect(await worker.hasPending()).toBe(true); + await worker.sweepPending(); + expect((await db.select().from(issueThreadInteractions))[0].status).toBe("expired"); + expect(wakeup).toHaveBeenCalledOnce(); + expect(wakeup).toHaveBeenCalledWith(f.agentId, expect.objectContaining({ requestedByActorType: "system", idempotencyKey: `connection-intent:${f.intent.id}:expired`, contextSnapshot: expect.objectContaining({ refreshTools: true, responsibleUserId: "connection-owner" }) })); + }); + + it("rechecks access after a restart before it delivers the saved continuation", async () => { + const f = await seed(); + await satisfiedConnectionIntentService(db).sweepPending(); + await db.update(connectionGrants).set({ status: "revoked" }); + const wakeup = vi.fn().mockResolvedValue(null); + const worker = connectionIntentDeliveryService(db, { wakeup }); + await worker.sweepPending(); + expect(wakeup).not.toHaveBeenCalled(); + expect((await db.select().from(connectionIntentDeliveries))[0].deliveredAt).toBeNull(); + await db.update(connectionGrants).set({ status: "active" }); + await db.update(connectionIntentDeliveries).set({ nextAttemptAt: new Date() }); + await worker.sweepPending(); + expect(wakeup).toHaveBeenCalledOnce(); + }); + + it("delivers system retirement to the native runner only while the same connection remains usable", async () => { + const f = await seed(); const run = await retry(f); + await satisfiedConnectionIntentService(db).sweepPending(); + const input = { db, companyId: f.companyId, issueId: f.issueId, runId: run.id, agentId: f.agentId, interactionIds: [f.intent.id] }; + expect(await materializeNativeInteractionResponses(input)).toEqual([expect.objectContaining({ + kind: "connection_intent", response: expect.objectContaining({ status: "expired", result: expect.objectContaining({ connectionId: f.connection.id }) }) })]); + await db.update(connectionGrants).set({ status: "revoked" }); + await expect(materializeNativeInteractionResponses(input)).rejects.toMatchObject({ code: "native_interaction_unresolved" }); + }); + + it.each(["missing_install", "denied_tools", "disabled", "unhealthy", "revoked", "wrong_user", "viewer", "removed_member", "additional_access", "ai", "channel", "upstream", "reassigned", "human_owner", "closed", "other_company"] as const)("preserves a request for %s", async (reason) => { + const f = await seed(); + if (reason === "missing_install") await db.delete(toolConnectionInstalls); + if (reason === "denied_tools") await db.update(toolProfiles).set({ defaultAction: "deny" }); + if (reason === "disabled") await db.update(toolConnections).set({ enabled: false }); + if (reason === "unhealthy") await db.update(toolConnections).set({ healthStatus: "error" }); + if (reason === "revoked") await db.update(connectionGrants).set({ status: "revoked" }); + if (reason === "wrong_user") await db.update(connectionGrants).set({ subjectUserId: "someone-else" }); + if (reason === "viewer") await db.update(companyMemberships).set({ membershipRole: "viewer" }); + if (reason === "removed_member") await db.delete(companyMemberships); + if (["ai", "channel", "upstream", "additional_access"].includes(reason)) { + const extra = reason === "ai" || reason === "channel" ? { purpose: reason } + : reason === "upstream" ? { upstreamService: { slug: "sentry", name: "Sentry" } } + : { accessRequest: { connectionId: f.connection.id, connectionName: "Sentry", tools: [{ catalogEntryId: f.tool.id, toolName: f.tool.toolName, versionHash: "v1", permission: "allowed" }] } }; + await db.update(issueThreadInteractions).set({ payload: { ...f.intent.payload, ...extra } as typeof f.intent.payload }); + } + if (reason === "reassigned") await db.update(issues).set({ assigneeAgentId: null }); + if (reason === "human_owner") await db.update(issues).set({ assigneeUserId: "connection-owner" }); + if (reason === "closed") await db.update(issues).set({ status: "done" }); + if (reason === "other_company") { + const [other] = await db.insert(companies).values({ name: "Other", issuePrefix: "OTH" }).returning(); + await db.update(issueThreadInteractions).set({ companyId: other!.id }); + } + const task = (await db.select().from(issues))[0]; const intent = (await db.select().from(issueThreadInteractions))[0]; + expect(await findSatisfiedToolConnection(db, task, intent)).toBeNull(); + expect(await satisfiedConnectionIntentService(db).sweepPending()).toMatchObject({ satisfied: 0 }); + expect((await db.select().from(issueThreadInteractions))[0].status).toBe("pending"); + expect(await db.select().from(connectionIntentDeliveries)).toHaveLength(0); + }); + + it("keeps a real question blocking even when a sibling connection is available", async () => { + const f = await seed(); const run = await retry(f); + await db.insert(issueThreadInteractions).values({ companyId: f.companyId, issueId: f.issueId, kind: "ask_user_questions", status: "pending", payload: { version: 1, questions: [] } }); + expect(await createPostgresRunDispatchAdapter(db).evaluateScheduledRetryGate({ companyId: f.companyId, runId: run.id, now: new Date() })).toMatchObject({ allowed: false, errorCode: "issue_waiting_for_response" }); + }); + + it("does not bypass a retained workspace hold after connecting", async () => { + const f = await seed(); + const leaseId = randomUUID(); + const workspaceRestoreRecovery = { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [leaseId] }; + const [source] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, issueId: f.issueId, status: "interrupted", finishedAt: new Date(), runtimeMode: "legacy", + contextSnapshot: { issueId: f.issueId }, resultJson: { conversationContinuation: "continue_conversation_v1", workspaceRestoreFailure: "restore_failed", workspaceRestoreRecovery } }).returning(); + const identity = { id: leaseId, companyId: f.companyId, heartbeatRunId: source!.id, provider: "daytona", providerLeaseId: `fixture-${leaseId}` }; + await db.insert(environmentLeases).values({ ...identity, issueId: f.issueId, status: "released", leasePolicy: "retain_on_failure", releasedAt: new Date(), cleanupStatus: "success", + metadata: { remoteExecutionTermination: remoteTerminationReceipt(identity, { providerLeaseId: identity.providerLeaseId, state: "stopped" }) } }); + await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: f.issueId, + kind: "active_run_watchdog", cause: "legacy_execution_requires_reconciliation", ownerType: "board", + fingerprint: `legacy-execution:${source!.id}`, status: "resolved", outcome: "blocked", + nextAction: "Repair the retained workspace before continuing.", evidence: { runId: source!.id, + workspaceRestoreFailure: "restore_failed", automaticRecovery: { replay: "blocked" }, + workspaceRestoreRecovery } }); + const run = await retry(f); + await db.update(heartbeatRuns).set({ retryOfRunId: source!.id, contextSnapshot: { issueId: f.issueId, retryOfRunId: source!.id } }).where(eq(heartbeatRuns.id, run.id)); + await satisfiedConnectionIntentService(db).sweepPending(); + const dispatch = createPostgresRunDispatchAdapter(db); + await dispatch.promoteOrCancelDueRetry({ companyId: f.companyId, runId: run.id, now: new Date() }); + const outcome = await dispatch.cancelStaleQueuedRun({ companyId: f.companyId, runId: run.id, expectedStatus: "queued", now: new Date() }); + expect(outcome).toMatchObject({ outcome: "cancelled", errorCode: "execution_reconciliation_required" }); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0].status).toBe("cancelled"); + }); +}); diff --git a/server/src/modules/run-dispatch/adapters/postgres.ts b/server/src/modules/run-dispatch/adapters/postgres.ts index 693a68a22b1..947e7d33863 100644 --- a/server/src/modules/run-dispatch/adapters/postgres.ts +++ b/server/src/modules/run-dispatch/adapters/postgres.ts @@ -6,9 +6,7 @@ import type { Db } from "@paperclipai/db"; import { agentWakeupRequests, agents, - approvals, - issueApprovals, - issueThreadInteractions, + connectionIntentDeliveries, heartbeatRuns, issueRecoveryActions, issueComments, @@ -22,6 +20,8 @@ import { evaluateAgentInvokabilityFromDb } from "../../../services/agent-invokab import { budgetService, budgetServiceInTransaction } from "../../../services/budgets.js"; import { isCompletedOnboardingHandoffWake } from "../../../services/chat-completion-delivery.js"; import { isHeartbeatWakeOnDemandEnabled } from "../../../services/heartbeat-policy.js"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "../../../services/delivery-work-notifications.js"; +import { connectionContinuationPendingResponse } from "../../../services/satisfied-connection-intents.js"; import { collectDispositionRepairSourceState } from "../../../services/recovery/disposition-repair.js"; import { legacyDispositionEpisode, legacyDispositionFingerprint } from "../../../services/recovery/legacy-continuation.js"; import { appendHeartbeatRunEvent } from "../../../services/heartbeat-run-events.js"; @@ -72,6 +72,7 @@ import { RunDispatchApplicationError } from "../application/types.js"; type HeartbeatRun = typeof heartbeatRuns.$inferSelect; type LoadGateFactsInput = { conversationContinuation: boolean; + responsibleUserId: string | null; runId: string; companyId: string; agentId: string; @@ -84,6 +85,8 @@ type LoadGateFactsResult = | { agentFound: true; facts: ScheduledRetryFacts } | { agentFound: false; issueId: string | null }; type LoadStalenessFactsInput = { + responsibleUserId: string | null; + conversationContinuation: boolean; runId: string; companyId: string; agentId: string; @@ -337,6 +340,7 @@ export function createPostgresRunDispatchAdapter( .select({ id: issues.id, companyId: issues.companyId, + projectId: issues.projectId, status: issues.status, assigneeAgentId: issues.assigneeAgentId, assigneeUserId: issues.assigneeUserId, @@ -364,19 +368,8 @@ export function createPostgresRunDispatchAdapter( facts.issueExecutionRunId = issue.executionRunId; facts.issueCheckoutRunId = issue.checkoutRunId; if (input.conversationContinuation) { - const [interactions, linkedApprovals] = await Promise.all([ - dbOrTx.select({ id: issueThreadInteractions.id }).from(issueThreadInteractions).where(and( - eq(issueThreadInteractions.companyId, input.companyId), - eq(issueThreadInteractions.issueId, issueId), eq(issueThreadInteractions.status, "pending"), - )).limit(1), - dbOrTx.select({ id: approvals.id }).from(issueApprovals).innerJoin(approvals, and( - eq(approvals.id, issueApprovals.approvalId), eq(approvals.companyId, issueApprovals.companyId), - )).where(and( - eq(issueApprovals.companyId, input.companyId), eq(issueApprovals.issueId, issueId), - inArray(approvals.status, ["pending", "revision_requested"]), - )).limit(1), - ]); - facts.pendingResponse = interactions.length > 0 ? "interaction" : linkedApprovals.length > 0 ? "approval" : null; + facts.pendingResponse = await connectionContinuationPendingResponse(dbOrTx as unknown as Db, + issue, input.responsibleUserId, input.runId); } facts.reviewParticipant = await readNativeReviewParticipantFacts(dbOrTx, { companyId: input.companyId, issueId, agentId: input.agentId, @@ -464,6 +457,7 @@ export function createPostgresRunDispatchAdapter( companyId: run.companyId, agentId: run.agentId, conversationContinuation: run.runtimeMode === "legacy" && hasConversationContinuationPolicy(run.resultJson), + responsibleUserId: run.responsibleUserId, contextSnapshot: parseObject(run.contextSnapshot), scheduledRetryReason: run.scheduledRetryReason, retryReasonOverride: input.retryReasonOverride, @@ -519,6 +513,9 @@ export function createPostgresRunDispatchAdapter( const issueQuery = dbOrTx .select({ id: issues.id, + companyId: issues.companyId, + projectId: issues.projectId, + assigneeUserId: issues.assigneeUserId, status: issues.status, assigneeAgentId: issues.assigneeAgentId, executionRunId: issues.executionRunId, @@ -600,6 +597,9 @@ export function createPostgresRunDispatchAdapter( unresolvedBlockerIssueIds: readiness.unresolvedBlockerIssueIds, unresolvedBlockerCount: readiness.unresolvedBlockerCount, } : null, + pendingResponse: issue && (input.conversationContinuation || context.connectionIntentResolution === "existing_connection") + ? await connectionContinuationPendingResponse(dbOrTx as unknown as Db, issue, input.responsibleUserId, input.runId) + : null, issueFound: issue !== null, issueStatus: issue?.status ?? null, issueAssigneeAgentId: issue?.assigneeAgentId ?? null, @@ -770,6 +770,7 @@ export function createPostgresRunDispatchAdapter( companyId: run.companyId, agentId: run.agentId, conversationContinuation: run.runtimeMode === "legacy" && hasConversationContinuationPolicy(run.resultJson), + responsibleUserId: run.responsibleUserId, contextSnapshot: parseObject(run.contextSnapshot), scheduledRetryReason: run.scheduledRetryReason, retryReasonOverride: run.scheduledRetryReason, @@ -881,6 +882,7 @@ export function createPostgresRunDispatchAdapter( decision: Extract, { stale: true }>, expectedStatus: "queued" | "running", now: Date, + providerDispatchNotStarted = false, ): Promise { const [row] = await tx .update(heartbeatRuns) @@ -892,6 +894,9 @@ export function createPostgresRunDispatchAdapter( resultJson: { ...parseObject(run.resultJson), stopReason: decision.errorCode, + ...(providerDispatchNotStarted && decision.errorCode === "issue_waiting_for_response" + ? { preDispatchResponseWait: { version: 1, providerWorkStarted: false } } + : {}), ...(decision.errorCode === "execution_reconciliation_required" ? { executionWait: decision.details } : {}), @@ -927,6 +932,20 @@ export function createPostgresRunDispatchAdapter( ); } + const context = parseObject(row.contextSnapshot); + if (decision.errorCode === "issue_waiting_for_response" && context.connectionIntentResolution === "existing_connection" + && context.source === "connection_intent.resolved" && context.interactionKind === "connection_intent" + && typeof context.interactionId === "string") { + // The partial wake-key uniqueness fence permits another delivery after + // this skipped receipt. Rearm atomically so restored access or a later + // answer can resume the untouched request after a restart. + await tx.update(connectionIntentDeliveries).set({ deliveredAt: null, nextAttemptAt: now }).where(and( + eq(connectionIntentDeliveries.companyId, row.companyId), + eq(connectionIntentDeliveries.interactionId, context.interactionId), + )); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.connection); + } + await tx .update(issues) .set({ @@ -976,6 +995,8 @@ export function createPostgresRunDispatchAdapter( runId: run.id, companyId: run.companyId, agentId: run.agentId, + responsibleUserId: run.responsibleUserId, + conversationContinuation: run.runtimeMode === "legacy" && hasConversationContinuationPolicy(run.resultJson), issueId, contextSnapshot, scheduledRetryReason: run.scheduledRetryReason, @@ -1129,6 +1150,7 @@ export function createPostgresRunDispatchAdapter( decision, input.expectedStatus, input.now, + true, // This locked gate has not handed off to the provider callback. ); return { dispatched: false as const, cancellation }; } diff --git a/server/src/modules/run-dispatch/domain/policy.ts b/server/src/modules/run-dispatch/domain/policy.ts index ec4b729839f..9c26cba3eb1 100644 --- a/server/src/modules/run-dispatch/domain/policy.ts +++ b/server/src/modules/run-dispatch/domain/policy.ts @@ -108,6 +108,7 @@ export type ScheduledRetryFacts = { }; export type QueuedRunStalenessErrorCode = + | "issue_waiting_for_response" | "execution_reconciliation_required" | "issue_dependencies_blocked" | "issue_not_found" @@ -129,6 +130,7 @@ export type StalenessDecision = }; export type QueuedRunFacts = { + pendingResponse?: "interaction" | "approval" | null; /** Rechecked for automatic native replacements immediately before dispatch. */ dependenciesBlocked?: DependencyBlockFacts | null; runId: string; @@ -546,6 +548,12 @@ export function decideQueuedRunStaleness( } } + if (facts.pendingResponse) { + return { stale: true, errorCode: "issue_waiting_for_response", + reason: "Cancelled because the continuation is still waiting for an authorized response", + details: { issueId: facts.issueId, pendingResponse: facts.pendingResponse } }; + } + if (facts.continuationParkApplies && facts.continuationParksExecutor) { return { stale: true, diff --git a/server/src/modules/run-dispatch/domain/wake-context.test.ts b/server/src/modules/run-dispatch/domain/wake-context.test.ts index ac6b84eb68e..f8d196d21ce 100644 --- a/server/src/modules/run-dispatch/domain/wake-context.test.ts +++ b/server/src/modules/run-dispatch/domain/wake-context.test.ts @@ -9,6 +9,13 @@ import { } from "./wake-context.js"; describe("wake context", () => { + it("recognizes a retired available connection without accepting ordinary expiry", () => { + const context = { interactionId: "connection", interactionStatus: "expired", interactionKind: "connection_intent", + connectionIntentResolution: "existing_connection", mutation: "interaction", wakeReason: "issue_commented" }; + expect(isResolvedInteractionContinuationWakeContext(context)).toBe(true); + expect(isResolvedInteractionContinuationWakeContext({ ...context, interactionKind: "request_confirmation" })).toBe(false); + expect(isResolvedInteractionContinuationWakeContext({ ...context, connectionIntentResolution: undefined })).toBe(false); + }); it("requires the subscription-specific non-assignee receipt for a subscription wait", () => { expect(isNonAssigneeWorkspaceBusyRetry("ai_connection_busy", { aiConnectionBusyDeferredWhileAssignee: false, diff --git a/server/src/modules/run-dispatch/domain/wake-context.ts b/server/src/modules/run-dispatch/domain/wake-context.ts index 2100b357d9c..a25833758f0 100644 --- a/server/src/modules/run-dispatch/domain/wake-context.ts +++ b/server/src/modules/run-dispatch/domain/wake-context.ts @@ -95,7 +95,10 @@ export function isResolvedInteractionContinuationWakeContext(contextSnapshot: un const interactionId = readNonEmptyString(context.interactionId); const interactionStatus = readNonEmptyString(context.interactionStatus); if (!interactionId || !interactionStatus) return false; - if (!RESOLVED_INTERACTION_CONTINUATION_STATUSES.has(interactionStatus)) return false; + const retiredConnection = interactionStatus === "expired" + && context.interactionKind === "connection_intent" + && context.connectionIntentResolution === "existing_connection"; + if (!retiredConnection && !RESOLVED_INTERACTION_CONTINUATION_STATUSES.has(interactionStatus)) return false; const mutation = readNonEmptyString(context.mutation); const wakeReason = readNonEmptyString(context.wakeReason); diff --git a/server/src/services/connection-intent-delivery.ts b/server/src/services/connection-intent-delivery.ts index c128d309fcd..3a5c8dfba44 100644 --- a/server/src/services/connection-intent-delivery.ts +++ b/server/src/services/connection-intent-delivery.ts @@ -1,6 +1,7 @@ import { connectionIntentService } from "./connection-intents.js"; +import { connectionContinuationPendingResponse, findSatisfiedToolConnection, satisfiedConnectionIntentService } from "./satisfied-connection-intents.js"; import { isAiConnectionConfigurationFailure } from "./ai-auth-failure.js"; -import { and, eq, isNull, lte, asc, notInArray, desc, sql } from "drizzle-orm"; +import { and, eq, isNull, lte, asc, inArray, notInArray, desc, sql } from "drizzle-orm"; import { connectionIntentDeliveries, issueThreadInteractions, issues, agentWakeupRequests, companyMemberships, heartbeatRuns, chatConversations, chatEndpoints, type Db } from "@paperclipai/db"; import type { heartbeatService } from "./heartbeat.js"; import { issueService } from "./issues.js"; @@ -13,10 +14,11 @@ export async function wakeConnectionIntentAfterResolution( input: { loaded: { issue: { id: string; assigneeAgentId: string | null; status: string }; - interaction: { id: string; resolvedAt?: string | Date | null; payload?: unknown }; + interaction: { id: string; resolvedAt?: string | Date | null; payload?: unknown; addresseeUserId?: string | null }; }; status: string; actorId: string; + actorType?: "user" | "system"; }, ) { const agentId = input.loaded.issue.assigneeAgentId; @@ -37,7 +39,7 @@ export async function wakeConnectionIntentAfterResolution( mutation: "interaction", }, idempotencyKey: `connection-intent:${input.loaded.interaction.id}:${input.status}`, - requestedByActorType: "user", + requestedByActorType: input.actorType ?? "user", requestedByActorId: input.actorId, contextSnapshot: { issueId: input.loaded.issue.id, @@ -48,6 +50,9 @@ export async function wakeConnectionIntentAfterResolution( mutation: "interaction", wakeReason: "issue_commented", source: "connection_intent.resolved", + ...(input.actorType === "system" && input.loaded.interaction.addresseeUserId + ? { responsibleUserId: input.loaded.interaction.addresseeUserId, connectionIntentResolution: "existing_connection" } + : {}), ...(interactionResolvedAt ? { interactionResolvedAt } : {}), @@ -64,6 +69,7 @@ export async function wakeConnectionIntentAfterResolution( export function connectionIntentDeliveryService(db: Db, heartbeat: Pick) { + const satisfiedIntents = satisfiedConnectionIntentService(db); // Only a repaired AI-authentication failure may reopen a blocked task. An old // card must never resume a newer failure, a reassignment, or a manual hold. async function restoreAiBlockedTask(loaded: { @@ -120,11 +126,16 @@ export function connectionIntentDeliveryService(db: Db, heartbeat: Pick { + const [task] = await tx.select().from(issues).where(and(eq(issues.id, loaded.issue.id), eq(issues.companyId, claimed.companyId))).for("update"); + const [card] = await tx.select().from(issueThreadInteractions).where(and( + eq(issueThreadInteractions.id, interactionId), eq(issueThreadInteractions.companyId, claimed.companyId), + eq(issueThreadInteractions.issueId, loaded.issue.id))).for("update"); + if (!task || !card || card.status !== "expired" || card.result?.outcome !== "expired") return null; + const ready = await findSatisfiedToolConnection(tx as unknown as Db, task, { ...card, status: "pending" }); + if (!ready) return null; + if (ready.id !== (card.result as { connectionId: string }).connectionId) { + const previousConnectionId = (card.result as { connectionId: string }).connectionId; + // This is a system observation of existing access. It grants nothing + // and never changes or impersonates a human decision. + const [updated] = await tx.update(issueThreadInteractions).set({ result: { ...card.result, + connectionId: ready.id }, updatedAt: new Date() }).where(eq(issueThreadInteractions.id, card.id)).returning(); + Object.assign(card, updated); + await logActivity(tx as unknown as Db, { companyId: claimed.companyId, actorType: "system", actorId: "connection-reconciliation", + action: "issue.thread_interaction_resolved", entityType: "issue", entityId: task.id, + details: { interactionId, interactionKind: "connection_intent", status: "expired", connectionId: ready.id, + previousConnectionId, resolutionSource: "existing_connection_refreshed" } }, publications); + } + return { task, card, waiting: await connectionContinuationPendingResponse(tx as unknown as Db, task, card.addresseeUserId, card.sourceRunId) }; + }); + for (const publication of publications) publishActivity(publication); + if (!current || current.waiting) return; + loaded.issue = current.task; + loaded.interaction = current.card; + interaction = current.card; + } if (interaction.status === "accepted" && payload?.purpose === "ai" && loaded.issue.status === "blocked") { const restored = await restoreAiBlockedTask(loaded); if (!restored) return; @@ -157,21 +200,32 @@ export function connectionIntentDeliveryService(db: Db, heartbeat: Pick 0; + if ((await db.select({ id: connectionIntentDeliveries.interactionId }).from(connectionIntentDeliveries) + .where(isNull(connectionIntentDeliveries.deliveredAt)).limit(1)).length > 0) return true; + return (await db.select({ id: issueThreadInteractions.id }).from(issueThreadInteractions) + .innerJoin(issues, and(eq(issues.id, issueThreadInteractions.issueId), eq(issues.companyId, issueThreadInteractions.companyId))) + .where(and(eq(issueThreadInteractions.kind, "connection_intent"), eq(issueThreadInteractions.status, "pending"), + inArray(issues.status, ["in_progress", "in_review"]), isNull(issues.assigneeUserId))) + .limit(1)).length > 0; } return { deliver, hasPending, tryDeliver: async (id: string) => { try { await deliver(id); } catch { /* Persisted delivery remains due after its lease. */ } }, sweepPending: async () => { + await satisfiedIntents.sweepPending(); const rows = await db.select().from(connectionIntentDeliveries).where(and(isNull(connectionIntentDeliveries.deliveredAt), lte(connectionIntentDeliveries.nextAttemptAt, new Date()))) .orderBy(asc(connectionIntentDeliveries.nextAttemptAt)).limit(50); let failed = 0; diff --git a/server/src/services/issue-thread-interactions.ts b/server/src/services/issue-thread-interactions.ts index 38287d45535..96049788e73 100644 --- a/server/src/services/issue-thread-interactions.ts +++ b/server/src/services/issue-thread-interactions.ts @@ -2739,6 +2739,7 @@ export function issueThreadInteractionService( } await touchIssue(tx, issue.id); inserted = true; + await notifyDeliveryWork(tx, DELIVERY_QUEUES.connection); return row; }); diff --git a/server/src/services/native-runtime/native-interaction-bridge.ts b/server/src/services/native-runtime/native-interaction-bridge.ts index e12f54c977b..42e4d9b06b9 100644 --- a/server/src/services/native-runtime/native-interaction-bridge.ts +++ b/server/src/services/native-runtime/native-interaction-bridge.ts @@ -19,6 +19,7 @@ import { commitNativeStatusDecision } from "./status-decision-committer.js"; import { nativeSha256 } from "./canonical.js"; import { recordNativeAttentionAssessment } from "./work-assessments.js"; import { formatDurableQuestionResponseSummary } from "../question-response-delivery.js"; +import { findSatisfiedToolConnection } from "../satisfied-connection-intents.js"; import { NATIVE_STATUS_ARBITER_POLICY_VERSION, type NativeAuthoritativeIssueStatus, @@ -163,7 +164,7 @@ export async function materializeNativeInteractionResponses(input: { if (requestedIds.size === 0) return []; const [interactions, issue] = await Promise.all([ issueThreadInteractionService(input.db).listForIssue(input.issueId), - input.db.select({ status: issues.status }).from(issues).where(and( + input.db.select().from(issues).where(and( eq(issues.id, input.issueId), eq(issues.companyId, input.companyId), )).limit(1).then((rows) => rows[0] ?? null), @@ -201,6 +202,19 @@ export async function materializeNativeInteractionResponses(input: { if (invocation.status !== expectedInvocationStatus || (request.status !== "rejected" && interaction.result?.toolAction?.status !== request.status)) throw new NativeInteractionBridgeError("native_interaction_governed_result_mismatch", "Tool review outcome does not match its invocation"); } const interactionResult = record(interaction.result); + if (interaction.kind === "connection_intent" && interaction.status === "expired" + && interactionResult.outcome === "expired" && typeof interactionResult.connectionId === "string") { + const [run] = await input.db.select({ responsibleUserId: heartbeatRuns.responsibleUserId }).from(heartbeatRuns) + .where(and(eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.runId), eq(heartbeatRuns.agentId, input.agentId))); + const ready = issue.assigneeAgentId === input.agentId && run?.responsibleUserId === interaction.addresseeUserId + ? await findSatisfiedToolConnection(input.db, issue, { ...interaction, addresseeUserId: interaction.addresseeUserId ?? null, status: "pending" }, input.runId) : null; + if (!ready || ready.id !== interactionResult.connectionId) { + throw new NativeInteractionBridgeError("native_interaction_unresolved", "The existing connection is no longer available"); + } + responses.push({ interactionId: interaction.id, kind: interaction.kind, + response: { status: "expired", result: structuredClone(interaction.result) as unknown as Record } }); + continue; + } const supersessionOutcome = interaction.status === "expired" && ["superseded_by_newer_request", "superseded_by_comment", "stale_target"].includes(String(interactionResult.outcome)); if (supersessionOutcome) { diff --git a/server/src/services/pre-dispatch-review-wait.ts b/server/src/services/pre-dispatch-review-wait.ts index 08ea49b498e..7448d5f8e55 100644 --- a/server/src/services/pre-dispatch-review-wait.ts +++ b/server/src/services/pre-dispatch-review-wait.ts @@ -6,23 +6,33 @@ import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js"; type Run = typeof heartbeatRuns.$inferSelect; type DispatchFields = "startedAt" | "runtimeModeResolvedAt" | "processPid" | "processGroupId" | "processStartedAt" | "nativeIssueId" | "nativeSessionId" | "sessionIdAfter" | - "controllerBootId" | "controllerLeaseExpiresAt" | "executionStage"; + "controllerBootId" | "controllerLeaseExpiresAt" | "executionStage" | "contextSnapshot"; /** The queued-run gate owns this receipt before execution authority is claimed. - * A deliberate review wait has no provider actions to reconcile. The error code + * A deliberate review or connection response wait has no provider actions to reconcile. The error code * alone is not proof; missing or conflicting dispatch evidence retains the hold. */ export function isPreDispatchReviewWait( run: Pick & Partial>, ): boolean { + const finalResponseWait = run.errorCode === "issue_waiting_for_response" && + (run.resultJson?.preDispatchResponseWait as { version?: number; providerWorkStarted?: boolean } | undefined)?.version === 1 && + (run.resultJson?.preDispatchResponseWait as { providerWorkStarted?: boolean } | undefined)?.providerWorkStarted === false; return run.runtimeMode === "legacy" && run.status === "cancelled" && - run.errorCode === "issue_continuation_waiting_on_review" && + (run.errorCode === "issue_continuation_waiting_on_review" || + (run.errorCode === "issue_waiting_for_response" && + run.contextSnapshot?.connectionIntentResolution === "existing_connection" && + run.contextSnapshot?.source === "connection_intent.resolved" && + run.contextSnapshot?.interactionKind === "connection_intent" && + run.contextSnapshot?.interactionStatus === "expired" && + run.contextSnapshot?.mutation === "interaction" && + typeof run.contextSnapshot?.interactionId === "string")) && run.resultJson?.stopReason === run.errorCode && run.resultJson?.timeoutSource === "stale_queued_run_gate" && run.resultJson?.workspaceRestoreFailure !== "restore_unsafe_archive" && - run.startedAt === null && run.runtimeModeResolvedAt === null && + (finalResponseWait || (run.startedAt === null && run.runtimeModeResolvedAt === null)) && run.processPid === null && run.processGroupId === null && run.processStartedAt === null && run.nativeIssueId === null && run.nativeSessionId === null && run.sessionIdAfter === null && - run.controllerBootId === null && run.controllerLeaseExpiresAt === null && run.executionStage === null; + (finalResponseWait || (run.controllerBootId === null && run.controllerLeaseExpiresAt === null && run.executionStage === null)); } /** The row receipt is a candidate, not permission to discard conflicting diff --git a/server/src/services/satisfied-connection-intents.ts b/server/src/services/satisfied-connection-intents.ts new file mode 100644 index 00000000000..49d295ae97f --- /dev/null +++ b/server/src/services/satisfied-connection-intents.ts @@ -0,0 +1,133 @@ +import { and, asc, eq, gt, inArray, isNull, or } from "drizzle-orm"; +import { approvals, companyMemberships, connectionIntentDeliveries, issueApprovals, issueThreadInteractions, issues, type Db } from "@paperclipai/db"; +import { connectionIntentPayloadSchema } from "@paperclipai/shared"; +import { logActivity, publishActivity, type ActivityPublication } from "./activity-log.js"; +import { connectionIntentService } from "./connection-intents.js"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; +import { logger } from "../middleware/logger.js"; +import { createToolGatewayService } from "./tool-gateway.js"; + +type Task = Pick; +type Intent = Pick & Partial>; + +/** Verify existing access only. This never installs tools, grants access, or adopts an identity. */ +export async function findSatisfiedToolConnection(db: Db, task: Task, intent: Intent, policyRunId?: string | null) { + if (intent.kind !== "connection_intent" || intent.status !== "pending" + || intent.companyId !== task.companyId || intent.issueId !== task.id + || task.assigneeUserId || !["in_progress", "in_review"].includes(task.status)) return null; + const parsed = connectionIntentPayloadSchema.safeParse(intent.payload); + if (!parsed.success) return null; + const payload = parsed.data; + // Additional permissions, provider authentication and inbox setup still need + // their governed completion paths. Aggregator cards also name an upstream app. + if (payload.purpose || payload.accessRequest || payload.upstreamService + || payload.requestingAgentId !== task.assigneeAgentId || !intent.addresseeUserId) return null; + if (intent.addresseeUserId !== "local-board") { + const [membership] = await db.select().from(companyMemberships).where(and( + eq(companyMemberships.companyId, task.companyId), eq(companyMemberships.principalType, "user"), + eq(companyMemberships.principalId, intent.addresseeUserId), + )); + if (membership?.status !== "active" || !membership.membershipRole || membership.membershipRole === "viewer") return null; + } + const connection = await connectionIntentService(db).usableConnectionForAgent({ companyId: task.companyId, + agentId: payload.requestingAgentId, responsibleUserId: intent.addresseeUserId, serviceSlug: payload.serviceSlug }); + if (!connection) return null; + // Use the gateway's exact descriptors and listing policy, including task, + // project and routine context. Per-invocation approvals remain enforced. + const access = await createToolGatewayService(db).summarizeConnectionAccessForAgent({ + companyId: task.companyId, connectionId: connection.id, agentId: payload.requestingAgentId, + heartbeatRunId: policyRunId === undefined ? intent.sourceRunId ?? null : policyRunId, + issueId: task.id, projectId: task.projectId, + }); + return access.allowedCount + access.askFirstCount > 0 ? connection : null; +} + +/** Recheck both live requests and system-retired cards against the next run. */ +export async function connectionContinuationPendingResponse(db: Db, task: Task, responsibleUserId: string | null, + policyRunId?: string | null): Promise<"interaction" | "approval" | null> { + const interactions = await db.select().from(issueThreadInteractions).where(and( + eq(issueThreadInteractions.companyId, task.companyId), eq(issueThreadInteractions.issueId, task.id), + or(eq(issueThreadInteractions.status, "pending"), and(eq(issueThreadInteractions.kind, "connection_intent"), eq(issueThreadInteractions.status, "expired"))), + )); + for (const interaction of interactions) { + const retired = interaction.status === "expired" && interaction.result?.outcome === "expired" + && "connectionId" in interaction.result && typeof interaction.result.connectionId === "string"; + if (interaction.status !== "pending" && !retired) continue; + if (retired && (interaction.payload as { requestingAgentId?: string }).requestingAgentId !== task.assigneeAgentId) continue; + if (interaction.addresseeUserId !== responsibleUserId) return "interaction"; + const connection = await findSatisfiedToolConnection(db, task, { ...interaction, status: "pending" }, policyRunId); + // Historical system retirements describe a service need, not a permanent + // binding to an account the user may later replace. Delivery and native + // response projection still validate their own exact recorded connection. + if (!connection) return "interaction"; + } + const [approval] = await db.select({ id: approvals.id }).from(issueApprovals).innerJoin(approvals, and( + eq(approvals.id, issueApprovals.approvalId), eq(approvals.companyId, issueApprovals.companyId), + )).where(and(eq(issueApprovals.companyId, task.companyId), eq(issueApprovals.issueId, task.id), + inArray(approvals.status, ["pending", "revision_requested"]))).limit(1); + return approval ? "approval" : null; +} + +export function satisfiedConnectionIntentService(db: Db) { + let afterId: string | null = null; + return { + async sweepPending() { + const candidates = await db.select({ id: issueThreadInteractions.id, issueId: issues.id, companyId: issues.companyId }) + .from(issueThreadInteractions).innerJoin(issues, and( + eq(issues.id, issueThreadInteractions.issueId), eq(issues.companyId, issueThreadInteractions.companyId), + )).where(and(eq(issueThreadInteractions.kind, "connection_intent"), eq(issueThreadInteractions.status, "pending"), + inArray(issues.status, ["in_progress", "in_review"]), isNull(issues.assigneeUserId), + ...(afterId ? [gt(issueThreadInteractions.id, afterId)] : []))) + .orderBy(asc(issueThreadInteractions.id)).limit(50); + // Rotate past unavailable cards so they cannot starve a later ready one. + afterId = candidates.length === 50 ? candidates[49]!.id : null; + let satisfied = 0; + let failed = 0; + for (const candidate of candidates) { + const publications: ActivityPublication[] = []; + try { + const updated = await db.transaction(async (tx) => { + const txDb = tx as unknown as Db; + const [task] = await tx.select().from(issues).where(and( + eq(issues.id, candidate.issueId), eq(issues.companyId, candidate.companyId), + )).for("update"); + const [intent] = await tx.select().from(issueThreadInteractions).where(and( + eq(issueThreadInteractions.id, candidate.id), eq(issueThreadInteractions.companyId, candidate.companyId), + eq(issueThreadInteractions.issueId, candidate.issueId), + )).for("update"); + if (!task || !intent) return false; + if (intent.addresseeUserId && intent.addresseeUserId !== "local-board") { + await tx.select({ id: companyMemberships.id }).from(companyMemberships).where(and( + eq(companyMemberships.companyId, candidate.companyId), eq(companyMemberships.principalType, "user"), + eq(companyMemberships.principalId, intent.addresseeUserId), + )).for("share"); + } + const connection = await findSatisfiedToolConnection(txDb, task, intent); + if (!connection) return false; + const now = new Date(); + // Retire the obsolete request without inventing a human approval. + await tx.update(issueThreadInteractions).set({ status: "expired", resolvedAt: now, updatedAt: now, + result: { version: 1, outcome: "expired", connectionId: connection.id, + reason: "An already authorized connection is available to the requesting agent." }, + }).where(and(eq(issueThreadInteractions.id, intent.id), eq(issueThreadInteractions.status, "pending"))); + await tx.update(issues).set({ updatedAt: now }).where(and(eq(issues.id, task.id), eq(issues.companyId, task.companyId))); + await tx.insert(connectionIntentDeliveries).values({ interactionId: intent.id, companyId: task.companyId }).onConflictDoNothing(); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.connection); + await logActivity(txDb, { companyId: task.companyId, actorType: "system", actorId: "connection-reconciliation", + action: "issue.thread_interaction_resolved", entityType: "issue", entityId: task.id, + details: { interactionId: intent.id, interactionKind: "connection_intent", status: "expired", + connectionId: connection.id, resolutionSource: "existing_connection" }, + }, publications); + return true; + }); + for (const publication of publications) publishActivity(publication); + if (updated) satisfied += 1; + } catch (error) { + failed += 1; + logger.warn({ err: error, interactionId: candidate.id }, "Could not reconcile an already available connection"); + } + } + return { scanned: candidates.length, satisfied, failed }; + }, + }; +} diff --git a/server/src/services/tool-gateway.ts b/server/src/services/tool-gateway.ts index 73b025d1e4c..550ec59a300 100644 --- a/server/src/services/tool-gateway.ts +++ b/server/src/services/tool-gateway.ts @@ -9119,6 +9119,9 @@ export function createToolGatewayService( companyId: string; connectionId: string; agentId: string; + heartbeatRunId?: string | null; + issueId?: string | null; + projectId?: string | null; }) { await assertAgentInCompany(input.companyId, input.agentId); const tools = await connectedMcpToolsForConnection( @@ -9130,6 +9133,9 @@ export function createToolGatewayService( policyInputForAgentTool({ companyId: input.companyId, agentId: input.agentId, + heartbeatRunId: input.heartbeatRunId, + issueId: input.issueId, + projectId: input.projectId, tool, }), ) From 0bbd586a2f6b5e4516b3c9ecb65642c650679bc5 Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Sun, 11 Oct 2026 09:30:59 -0700 Subject: [PATCH 02/18] fix(server): wait for workspace restore before shutdown lease release (#15857) ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - Server shutdown must stop agent execution and preserve its files. > - Legacy adapters export workspace and instruction changes during settlement. > - Shutdown released their sandbox leases before this export finished. > - The provider closed command access, so the later Git export failed. > - This change stops an owned adapter through its cancellation control and waits for settlement before release. > - A bounded timeout retains the exact sandbox source behind the existing repair gate. ## Linked Issues or Issue Description **What happened?** A server restart interrupted a remote legacy run. Shutdown began releasing its sandbox lease while the adapter was still stopping its provider and restoring files. The sandbox provider closed command admission. The adapter's later Git export failed with `workspaceRestoreFailure: restore_failed`, leaving the task held for workspace repair. **Expected behavior** Shutdown keeps the exact lease available through provider stop, instruction collection, and workspace restore. Cleanup follows adapter settlement. A restore that exceeds the bounded deadline keeps its exact source for repair before later execution. **Steps to reproduce** 1. Start an owned legacy adapter with a sandbox workspace. 2. Send SIGTERM to the server while the adapter is active. 3. Delay adapter settlement until after the shutdown terminal write. 4. On the base branch, shutdown releases the lease without cancelling or joining the adapter. Its later export can encounter closed provider admission. Related: #14311 defines the provider-stop boundary for file collection. #15042 closes run admission during shutdown. This fix orders sandbox release after the adapter's existing file settlement. ## What Changed - Cancel each owned legacy adapter and join all affected runs within one shared 30-second window. - Commit a durable fence on the exact sandbox before exposing interruption. Other cleanup workers defer while export is pending. - Permit the exact aborted owner's stop-only request so shutdown does not block provider termination. - Reload current execution evidence after settlement. Release leases even when the adapter's ordinary terminal write wins the shutdown race. - Claim the current fence before recording timeout retention. An old snapshot cannot create a repair hold after successful settlement. - Recover fenced sources after controller loss, including released stop receipts. Block reuse or destruction of the physical allocation while file recovery is required, including direct resume and destroy calls with historical lease snapshots. Convert failed ordinary cleanup into stop-only recovery after its existing attempt settles. - Allow 90 seconds for shutdown in shipped Compose services. Preserve native runner restart ownership and unregistered process behavior. - Add database regressions for restore order, timeout, concurrent cancellation, another cleanup worker, controller loss, stale snapshots, stop-only settlement, and completion races. Document the shutdown order. ## Verification - The original ordering regression failed on the base implementation. - Latest recovery and environment runtime suites: 190 tests passed. The other four focused suites passed 136 tests, covering run control, shutdown, file repair, server startup, and environment runtime. - Server typecheck passed. Full `pnpm -r typecheck` and `pnpm build` passed on the latest head. - `pnpm test:run` was run locally. Its general-server group passed 18,559 tests and failed seven, with one unhandled error. The long run started before the review fixes, then loaded newer regression cases against older transformed modules. The current focused suites pass in a fresh process. The damaged-prior-epoch native-session-resume failure also reproduces on unchanged base commit de9ab8e7e06. An isolated rerun of the Google Workspace connection case passed. Later full-run groups did not run because general-server stopped on failure. - The prior CI startup import failure is fixed: expiry loads the terminal recorder only when it needs to pin recovery. Its focused startup suite passes. - Apex reviewed commit c564320b16b at 5/5. All review threads are resolved. All latest-head CI gates passed. The branch has no merge conflicts. ## Risks Owned legacy adapters share a 30-second shutdown window. The wait is bounded. Timeout cleanup uses the existing stop-and-retain and workspace repair paths. A timeout can require manual file verification even if a late restore succeeds. No database migration or API change is needed. ## Model Used OpenAI GPT-6 through Codex, with repository inspection, code editing, shell execution, and test tools. The session identifies the model family as GPT-6. It does not expose a more specific runtime model ID or context window size. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass (focused suites; full local limitations are listed above) - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- doc/execution-semantics.md | 17 ++ docker/docker-compose.quickstart.yml | 2 + docker/docker-compose.yml | 2 + server/src/services/environment-runtime.ts | 35 ++- .../src/services/heartbeat/recovery.test.ts | 289 +++++++++++++++++- server/src/services/heartbeat/recovery.ts | 80 ++++- .../legacy-shutdown-workspace-settlement.ts | 137 +++++++++ 7 files changed, 550 insertions(+), 12 deletions(-) create mode 100644 server/src/services/legacy-shutdown-workspace-settlement.ts diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 58e352d9a17..f5632ecb178 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1136,8 +1136,25 @@ An interrupted conversation does not permanently block its task. For local conve Shutdown, process loss, and provider failure use the existing durable failure retry counter and delay. Ordinary failure recovery permits at most two automatic retries in a failure chain. Accepted-interaction infrastructure recovery retains its existing bounded policy. Repeated scheduler visits reuse the same successor; restarting the server does not reset the counter. After exhaustion, automatic attempts stop. A new explicit user message can start a fresh run and failure budget. Productive max-turn continuation and confirmed workspace waits keep their separate existing semantics. +Graceful shutdown interrupts an owned legacy adapter through its cancellation +control, then waits within one shared 30-second window for provider stop and +adapter settlement across all affected runs. Before interruption becomes visible, +it commits a settlement fence on the exact sandbox lease. Cleanup on this server +or another server defers while that fence is pending. If the controller dies, +cleanup records exact stop-only retention and a repair hold after the deadline, +without claiming a successful restore. The shipped Compose services allow 90 +seconds for settlement and subsequent shutdown cleanup. +The sandbox lease remains available during workspace export and instruction +collection. Lease release and recovery classification follow settlement and use +the current execution evidence. If settlement exceeds the deadline with an +active sandbox source, shutdown records a restore failure and pins the exact +source for stop and retention before cleanup. A later run must pass the normal +workspace repair gate. Native runner ownership keeps its existing restart path. + Real gates still apply: company and task ownership, active provider ownership, budget limits, agent availability, dependencies, pending approval/review paths, and explicit pause holds. Native runner reattachment and finalization retain their existing ownership protocol. Process, HTTP, and gateway adapters retain their recovery rules because invoking those adapters can itself repeat an external action rather than start a conversation turn. +A source that already stopped and was released remains fenced until its export settles; restart reconciliation scans these released sources too. Both provider reuse and destruction check the physical allocation for a shutdown fence or file-repair pin, including historical lease rows. + An operator Stop waits for provider termination. Remote sandbox providers may return a stopped/deleted receipt after their control-plane operation completes. Paperclip binds that receipt to the company, run, and exact lease; successful file cleanup, a terminal run row, or an in-sandbox shutdown event is not sufficient. Legacy conversational runs receive their cancellation acknowledgement after all remote leases have confirmed termination. Stop alone never creates a continuation. A user message queued during remote cleanup is reconsidered when the provider confirms termination; it still passes normal admission and adopts pending comment IDs in order. Once stopped, the next explicit wake uses the same queue. A compatible saved ACP session can resume, and an unavailable or incompatible session can start fresh with the full task context. Run credentials and scratch paths remain scoped to the new run. A subtree pause requires Resume; a message does not bypass it. Cancelled runs and runs with a recorded pending stop lose run-scoped API write authority, while reads remain available for diagnostics. This applies to ordinary tasks as well as conversations. Task and interaction mutations recheck that authority in the write transaction, so a cancelled run cannot overwrite a recovery disposition with a late Done or resolve an interaction after revocation. A task handoff that intentionally stops its own run can commit only with a server receipt tied to that exact request. Direct sandbox CLI adapters such as Grok register cancellation before preparation and keep ownership until host-owned termination is verified, including a sandbox acquired before adapter registration. A failed stop does not acknowledge cancellation or abandon an outstanding remote command. Interrupted workspace restore failures remain recorded for recovery; a stop receipt proves termination, not successful file restoration. diff --git a/docker/docker-compose.quickstart.yml b/docker/docker-compose.quickstart.yml index f750bda19b6..8123f5efb71 100644 --- a/docker/docker-compose.quickstart.yml +++ b/docker/docker-compose.quickstart.yml @@ -16,6 +16,8 @@ services: # backstop if something leaks anyway — the container hits its own limit and # dies visibly rather than starving the host of pids. pids_limit: 2048 + # Includes the shared 30s adapter-settlement window and final cleanup. + stop_grace_period: 90s ports: - "${PAPERCLIP_PORT:-3100}:3100" environment: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index dc6dda42383..03e58dfc52e 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -28,6 +28,8 @@ services: # this container hits its own ceiling and dies visibly rather than starving # the whole host of pids. pids_limit: 2048 + # Includes the shared 30s adapter-settlement window and final cleanup. + stop_grace_period: 90s ports: - "3100:3100" environment: diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index de0a695142d..1d63a948523 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -1,3 +1,4 @@ +import { allowLegacyShutdownWorkspaceCleanup, legacyShutdownWorkspaceResourceProtected } from "./legacy-shutdown-workspace-settlement.js"; import { beginIdleTrackedWork } from "./task-admission.js"; import { hasStopOnlyCleanup, prepareSandboxStopAndRetain, readStopOnlyCleanup, settleStopOnlyCleanup, stopOnlyCleanupKey } from "./sandbox-stop-and-retain.js"; import { JsonRpcCallError, readEnvironmentAcquisitionDiagnostic, readEnvironmentCreationCleanupError } from "@paperclipai/plugin-sdk"; @@ -1943,6 +1944,9 @@ function createSandboxEnvironmentDriver( lease.metadata?.agentId === input.agentId, ) : []; + for (let index = reusableCandidateLeases.length - 1; index >= 0; index--) { + if (await legacyShutdownWorkspaceResourceProtected(db, reusableCandidateLeases[index]!)) reusableCandidateLeases.splice(index, 1); + } const reusableExistingLeases = reusableCandidateLeases.filter((lease) => reusableSandboxLeaseScopeMatches({ lease, @@ -2358,6 +2362,9 @@ function createSandboxEnvironmentDriver( lease.metadata?.agentId === input.agentId, ) : []; + for (let index = reusableCandidateLeases.length - 1; index >= 0; index--) { + if (await legacyShutdownWorkspaceResourceProtected(db, reusableCandidateLeases[index]!)) reusableCandidateLeases.splice(index, 1); + } const reusableExistingLeases = reusableCandidateLeases.filter((lease) => reusableSandboxLeaseScopeMatches({ lease, @@ -3327,6 +3334,7 @@ function createSandboxEnvironmentDriver( lease: EnvironmentLease; failureReason: string; }): Promise { + if (await legacyShutdownWorkspaceResourceProtected(db, input.lease)) return null; let cleanupStatus: "success" | "failed" = "success"; let termination: ReturnType; const metadata = input.lease.metadata ?? {}; @@ -3888,6 +3896,7 @@ export function environmentRuntimeService( scopeCondition?: ReturnType; failureReason: string; }): Promise { + if (!(await allowLegacyShutdownWorkspaceCleanup(db, input.leaseRow))) return null; const now = new Date(); const attemptId = randomUUID(); // Persist recovery ownership before any provider work. Re-check scope and @@ -4065,8 +4074,16 @@ export function environmentRuntimeService( // error through `onLeaseReleaseError` for its log path. Keep the order // serial. const released: EnvironmentRuntimeLeaseRecord[] = []; - for (const leaseRow of leaseRows) { + for (let leaseRow of leaseRows) { try { + if (!(await allowLegacyShutdownWorkspaceCleanup(db, leaseRow, new Date(), { + ownerStopOnly: cancelActiveWork === true && providerResourceDisposition === "stop_and_retain", + }))) continue; + if (leaseRow.metadata?.legacyShutdownWorkspaceSettlement) { + leaseRow = (await db.select().from(environmentLeases).where(and( + eq(environmentLeases.id, leaseRow.id), eq(environmentLeases.companyId, leaseRow.companyId), + )))[0] ?? leaseRow; + } const environment = leaseRow.environmentId ? await environmentsSvc.getById(leaseRow.environmentId) : null; @@ -4211,6 +4228,14 @@ export function environmentRuntimeService( environment: Environment | null; lease: EnvironmentLease; }): Promise { + if (!(await allowLegacyShutdownWorkspaceCleanup(db, input.lease))) { + throw new Error("Shutdown workspace settlement is still pending."); + } + if (input.lease.metadata?.legacyShutdownWorkspaceSettlement) { + const current = await environmentsSvc.getLeaseById(input.lease.id); + if (!current) throw new Error("Shutdown workspace source is unavailable."); + input = { ...input, lease: current }; + } const driver = requireDriverKey(getLeaseDriverKey(input.lease, input.environment)); if (!driver.retryPendingSandboxTeardown) { throw new Error( @@ -4407,6 +4432,7 @@ export function environmentRuntimeService( }, async resumeRunLease(input: EnvironmentDriverLeaseInput): Promise { + if (await legacyShutdownWorkspaceResourceProtected(db, input.lease)) return null; const driver = requireDriverKey(getLeaseDriverKey(input.lease, input.environment)); if (!driver.resumeRunLease) { throw new Error(`Environment driver "${driver.driver}" does not support lease resume.`); @@ -4415,6 +4441,13 @@ export function environmentRuntimeService( }, async destroyRunLease(input: EnvironmentDriverLeaseInput): Promise { + if (await legacyShutdownWorkspaceResourceProtected(db, input.lease)) return null; + if (input.lease.metadata?.legacyShutdownWorkspaceSettlement) { + if (!(await allowLegacyShutdownWorkspaceCleanup(db, input.lease))) return null; + const current = await environmentsSvc.getLeaseById(input.lease.id); + if (!current || hasStopOnlyCleanup(current)) return null; + input = { ...input, lease: current }; + } const driver = requireDriverKey(getLeaseDriverKey(input.lease, input.environment)); if (!driver.destroyRunLease) { throw new Error(`Environment driver "${driver.driver}" does not support lease destroy.`); diff --git a/server/src/services/heartbeat/recovery.test.ts b/server/src/services/heartbeat/recovery.test.ts index a925edfaa79..6d76f2df511 100644 --- a/server/src/services/heartbeat/recovery.test.ts +++ b/server/src/services/heartbeat/recovery.test.ts @@ -1,15 +1,17 @@ import { randomUUID } from "node:crypto"; import type { Db } from "@paperclipai/db"; -import { agents, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, nativeRunFinalizations } from "@paperclipai/db"; +import { agents, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, environments, environmentLeases, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations } from "@paperclipai/db"; import { eq } from "drizzle-orm"; import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; -import { environmentRuntimeService } from "../environment-runtime.js"; +import { classifyEnvironmentCapabilities, environmentRuntimeService } from "../environment-runtime.js"; import { environmentService } from "../environments.js"; import { createHeartbeatRecovery, type HeartbeatRecoveryDependencies } from "./recovery.js"; import type { HotRestartIntent } from "../hot-restart.js"; import type { NativeRestartRecoveryClaim, NativeRestartRecoveryDisposition } from "../native-runtime/index.js"; import { NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE } from "../native-runtime/native-runner-ownership.js"; +import { allowLegacyShutdownWorkspaceCleanup, beginLegacyShutdownWorkspaceSettlement, finishLegacyShutdownWorkspaceSettlement, legacyShutdownWorkspaceResourceProtected, sweepLegacyShutdownWorkspaceSettlements } from "../legacy-shutdown-workspace-settlement.js"; +import { adapterExecutionControls, createAdapterExecutionControl } from "../adapter-execution-control.js"; type Run = typeof heartbeatRuns.$inferSelect; @@ -71,7 +73,7 @@ function callbacks(db: Db) { terminateHeartbeatRunProcess: vi.fn(async () => undefined), mergeRunStopMetadataForAgent: vi.fn((_agent, _outcome, options) => options?.resultJson ?? null), classifyAndPersistRunLiveness: vi.fn(async run => run), - releaseEnvironmentLeasesForRun: vi.fn(async () => undefined), + releaseEnvironmentLeasesForRun: vi.fn(async () => undefined), acknowledgeRemoteStop: vi.fn(async () => undefined), resumeRemoteStopComments: vi.fn(async () => undefined), dispatchPendingNativeStatusWakeups: vi.fn(async () => undefined), @@ -184,6 +186,287 @@ describePostgres("heartbeat recovery module ownership", () => { return (await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, issueId, invocationSource: "automation", status: "running", runtimeMode: "legacy" }).returning())[0]; } + it.each(["restored", "restore_failed"])("keeps the sandbox available until adapter workspace settlement: %s", async outcome => { + const run = await seedRun(); + const deps = callbacks(db); + deps.setRunStatusIfRunning.mockImplementation(async (id, status, patch) => { + const [updated] = await db.update(heartbeatRuns).set({ ...patch, status }).where(eq(heartbeatRuns.id, id)).returning(); + return { updated: true, run: updated }; + }); + deps.getRun.mockImplementation(async id => (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)))[0] ?? null); + const control = createAdapterExecutionControl(); + adapterExecutionControls.set(run.id, control); + const drain = createHeartbeatRecovery(db, deps).drainRunningRunsForShutdown("SIGTERM", new Date(), [run.id]); + try { + await vi.waitFor(() => expect(control.controller.signal.aborted).toBe(true)); + expect(deps.releaseEnvironmentLeasesForRun).not.toHaveBeenCalled(); + expect(deps.scheduleBoundedRetryForRun).not.toHaveBeenCalled(); + expect((await deps.getRun(run.id))?.status).toBe("interrupted"); + const resultJson = outcome === "restore_failed" ? { workspaceRestoreFailure: "restore_failed", + workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [randomUUID()] } } + : { workspaceRestored: true }; + await db.update(heartbeatRuns).set({ resultJson }).where(eq(heartbeatRuns.id, run.id)); + control.finish(); + await drain; + expect(deps.releaseEnvironmentLeasesForRun).toHaveBeenCalledOnce(); + expect(deps.classifyAndPersistRunLiveness).toHaveBeenCalledWith(expect.objectContaining({ + resultJson: expect.objectContaining(resultJson) }), expect.anything()); + } finally { + control.finish(); + adapterExecutionControls.delete(run.id); + await drain; + await db.delete(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)); + } + }); + + it("starts all owned shutdown cancellations within one settlement window", async () => { + const runs = await Promise.all([seedRun(), seedRun()]); + const controls = runs.map(run => { + const control = createAdapterExecutionControl(); adapterExecutionControls.set(run.id, control); return control; + }); + const deps = callbacks(db); + deps.setRunStatusIfRunning.mockImplementation(async (id, status, patch) => { + const [updated] = await db.update(heartbeatRuns).set({ ...patch, status }).where(eq(heartbeatRuns.id, id)).returning(); + return { updated: true, run: updated }; + }); + const drain = createHeartbeatRecovery(db, deps).drainRunningRunsForShutdown("SIGTERM", new Date(), runs.map(run => run.id)); + try { + await vi.waitFor(() => expect(controls.every(control => control.controller.signal.aborted)).toBe(true)); + expect(deps.releaseEnvironmentLeasesForRun).not.toHaveBeenCalled(); + } finally { + controls.forEach(control => control.finish()); + await drain; + for (const run of runs) { adapterExecutionControls.delete(run.id); await db.delete(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)); } + } + }); + + it.each(["ephemeral", "reuse_by_environment"] as const)("protects a %s export from another server and pins it after controller loss", async leasePolicy => { + const run = await seedRun(); + await db.update(heartbeatRuns).set({ contextSnapshot: { issueId: run.issueId } }).where(eq(heartbeatRuns.id, run.id)); + const [lease] = await db.insert(environmentLeases).values({ companyId: run.companyId, heartbeatRunId: run.id, + issueId: run.issueId, status: "active", provider: "daytona", providerLeaseId: randomUUID(), leasePolicy, + metadata: { driver: "sandbox", sandboxProviderPlugin: true, pluginId: "fixture-plugin", remoteCwd: "/home/daytona/workspace" } }).returning(); + try { + const deadline = new Date(Date.now() + 30_000); + await beginLegacyShutdownWorkspaceSettlement(db, run, deadline); + await db.update(heartbeatRuns).set({ status: "interrupted", errorCode: "server_shutdown_interrupted", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run.id)); + const foreignDeps = callbacks(db); + const foreign = createHeartbeatRecovery(db, foreignDeps); + expect(await foreign.sweepOrphanedActiveLeases({ backoffMs: 0 })).toMatchObject({ recovered: 0 }); + expect(await environmentRuntimeService(db).releaseRunLeases(run.id)).toEqual([]); + await db.update(environmentLeases).set({ status: "pending_cleanup" }).where(eq(environmentLeases.id, lease.id)); + expect(await foreign.sweepPendingCleanupLeases({ backoffMs: 0 })).toMatchObject({ destroyed: 0 }); + await db.update(environmentLeases).set({ status: "active" }).where(eq(environmentLeases.id, lease.id)); + let source = (await db.select().from(environmentLeases).where(eq(environmentLeases.id, lease.id)))[0]; + expect(source.status).toBe("active"); + expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, run.issueId!))).toHaveLength(0); + // No local adapter map or timeout handler survives a container SIGKILL. + expect(await allowLegacyShutdownWorkspaceCleanup(db, source, new Date(deadline.getTime() + 1))).toBe(true); + source = (await db.select().from(environmentLeases).where(eq(environmentLeases.id, lease.id)))[0]; + expect(source).toMatchObject({ status: "pending_cleanup", leasePolicy: "retain_on_failure", + metadata: { legacyShutdownWorkspaceSettlement: { state: "expired" }, sandboxStopAndRetain: { leaseId: lease.id } } }); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0].resultJson) + .toMatchObject({ workspaceRestoreFailure: "restore_failed", workspaceRestoreRecovery: { leaseIds: [lease.id] } }); + expect(await foreign.sweepOrphanedActiveLeases({ backoffMs: 0 })).toMatchObject({ recovered: 0 }); + } finally { + await db.delete(environmentLeases).where(eq(environmentLeases.id, lease.id)); + await db.delete(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, run.issueId!)); + await db.delete(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)); + } + }); + + async function seedShutdownSource() { + const run = await seedRun(); + const [lease] = await db.insert(environmentLeases).values({ companyId: run.companyId, heartbeatRunId: run.id, + issueId: run.issueId, status: "active", provider: "daytona", providerLeaseId: randomUUID(), leasePolicy: "ephemeral", + metadata: { driver: "sandbox", sandboxProviderPlugin: true, pluginId: "fixture-plugin", remoteCwd: "/home/daytona/workspace" } }).returning(); + await beginLegacyShutdownWorkspaceSettlement(db, run, new Date(Date.now() + 30_000)); + return { run, lease }; + } + async function deleteShutdownSource(f: Awaited>) { + await db.delete(environmentLeases).where(eq(environmentLeases.id, f.lease.id)); + await db.delete(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.run.issueId!)); + await db.delete(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)); + adapterExecutionControls.delete(f.run.id); + } + + it("does not turn a stale pending snapshot into a repair hold after settlement", async () => { + const f = await seedShutdownSource(); + try { + const old = (await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)))[0]; + await db.update(heartbeatRuns).set({ status: "interrupted", contextSnapshot: { issueId: f.run.issueId }, resultJson: { workspaceRestored: true } }).where(eq(heartbeatRuns.id, f.run.id)); + await finishLegacyShutdownWorkspaceSettlement(db, f.run); + const single = createDb(database.connectionString, { maxConnections: 1 }); + try { expect(await allowLegacyShutdownWorkspaceCleanup(single, old, new Date(Date.now() + 60_000))).toBe(true); } + finally { await single.$client.end({ timeout: 1 }); } + expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)))[0]) + .toMatchObject({ status: "active", leasePolicy: "ephemeral", metadata: { legacyShutdownWorkspaceSettlement: { state: "settled" } } }); + expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.run.issueId!))).toHaveLength(0); + } finally { await deleteShutdownSource(f); } + }); + + it("allows the aborted owner to stop and retain its command while excluding outside cleanup", async () => { + const f = await seedShutdownSource(); + const control = createAdapterExecutionControl(); control.controller.abort(); + adapterExecutionControls.set(f.run.id, control); + const stop = vi.fn(async () => ({ providerLeaseId: f.lease.providerLeaseId, state: "stopped" })); + const release = vi.fn(async () => null); + const runtime = environmentRuntimeService(db, { drivers: [{ driver: "sandbox", resolveCapabilities: async () => classifyEnvironmentCapabilities({}), + acquireRunLease: async () => { throw new Error("Unexpected acquire"); }, releaseRunLease: release, + retryPendingSandboxTeardown: stop }] }); + try { + await runtime.releaseRunLeases(f.run.id, "released", undefined, "stop_and_retain", false); + expect(stop).not.toHaveBeenCalled(); + await runtime.releaseRunLeases(f.run.id, "released", undefined, "stop_and_retain", true); + expect(stop).toHaveBeenCalledOnce(); expect(release).not.toHaveBeenCalled(); + expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)))[0]) + .toMatchObject({ status: "released", cleanupStatus: "success", metadata: { remoteExecutionTermination: { state: "stopped" }, + sandboxStopAndRetainReceipt: { leaseId: f.lease.id } } }); + } finally { control.finish(); await deleteShutdownSource(f); } + }); + + it.each(["ephemeral", "reuse_by_environment"] as const)("recovers a stopped %s source after its shutdown controller disappears", async leasePolicy => { + const f = await seedShutdownSource(); + await db.update(environmentLeases).set({ leasePolicy }).where(eq(environmentLeases.id, f.lease.id)); + const control = createAdapterExecutionControl(); control.controller.abort(); adapterExecutionControls.set(f.run.id, control); + const [environmentRow] = await db.insert(environments).values({ name: "Guarded source", driver: "sandbox" }).returning(); + const environment = (await environmentService(db).getById(environmentRow.id))!; + const leaseSnapshot = (await environmentService(db).getLeaseById(f.lease.id))!; + leaseSnapshot.metadata = f.lease.metadata; + const resume = vi.fn(async () => null); const destroy = vi.fn(async () => null); + const stop = vi.fn(async () => ({ providerLeaseId: f.lease.providerLeaseId, state: "stopped" })); + const runtime = environmentRuntimeService(db, { drivers: [{ driver: "sandbox", resolveCapabilities: async () => classifyEnvironmentCapabilities({}), + acquireRunLease: async () => { throw new Error("Unexpected acquire"); }, releaseRunLease: vi.fn(), retryPendingSandboxTeardown: stop, resumeRunLease: resume, destroyRunLease: destroy }] }); + try { + await runtime.releaseRunLeases(f.run.id, "released", undefined, "stop_and_retain", true); + adapterExecutionControls.delete(f.run.id); // SIGKILL before export/finally settled. + await db.update(heartbeatRuns).set({ status: "interrupted", finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.run.id)); + const [source] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)); + expect(source.status).toBe("released"); + // An old reuse receipt must not bypass the current allocation's fence. + expect(await legacyShutdownWorkspaceResourceProtected(db, { provider: source.provider, providerLeaseId: source.providerLeaseId })).toBe(true); + await sweepLegacyShutdownWorkspaceSettlements(db, new Date(Date.now() + 60_000)); + expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)))[0]) + .toMatchObject({ status: "released", leasePolicy: "retain_on_failure", metadata: { + legacyShutdownWorkspaceSettlement: { state: "expired" }, workspaceRestoreRecovery: { runId: f.run.id } } }); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].resultJson) + .toMatchObject({ workspaceRestoreFailure: "restore_failed", workspaceRestoreRecovery: { leaseIds: [f.lease.id] } }); + expect(await legacyShutdownWorkspaceResourceProtected(db, source)).toBe(true); + // Direct operations and pre-fence snapshots cannot bypass the source pin. + expect(await runtime.resumeRunLease({ environment, lease: leaseSnapshot })).toBeNull(); + expect(await runtime.destroyRunLease({ environment, lease: leaseSnapshot })).toBeNull(); + expect(resume).not.toHaveBeenCalled(); expect(destroy).not.toHaveBeenCalled(); + expect(stop).toHaveBeenCalledOnce(); + } finally { control.finish(); await deleteShutdownSource(f); await db.delete(environments).where(eq(environments.id, environment.id)); } + }); + + it.each(["ephemeral", "reuse_by_environment"] as const)("recovers failed ordinary cleanup of a fenced %s source", async leasePolicy => { + const f = await seedShutdownSource(); + const stop = vi.fn(async () => ({ providerLeaseId: f.lease.providerLeaseId, state: "stopped" })); + const destroy = vi.fn(async () => null); + const runtime = environmentRuntimeService(db, { drivers: [{ driver: "sandbox", resolveCapabilities: async () => classifyEnvironmentCapabilities({}), + acquireRunLease: async () => { throw new Error("Unexpected acquire"); }, releaseRunLease: destroy, destroyRunLease: destroy, + retryPendingSandboxTeardown: stop }] }); + try { + await db.update(heartbeatRuns).set({ status: "interrupted", finishedAt: new Date(), contextSnapshot: { issueId: f.run.issueId } }).where(eq(heartbeatRuns.id, f.run.id)); + await db.update(environmentLeases).set({ status: "pending_cleanup", leasePolicy, cleanupStatus: "failed", + releasedAt: new Date(), failureReason: "release_cleanup_failed" }).where(eq(environmentLeases.id, f.lease.id)); + const [source] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)); + expect(await allowLegacyShutdownWorkspaceCleanup(db, source, new Date(Date.now() + 60_000))).toBe(true); + const lease = (await environmentService(db).getLeaseById(f.lease.id))!; + expect(lease).toMatchObject({ status: "pending_cleanup", leasePolicy: "retain_on_failure", metadata: { + legacyShutdownWorkspaceSettlement: { state: "expired" }, sandboxStopAndRetain: { leaseId: f.lease.id } } }); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].resultJson) + .toMatchObject({ workspaceRestoreRecovery: { leaseIds: [f.lease.id] } }); + const deps = callbacks(db); deps.environmentRuntime = runtime; + expect(await createHeartbeatRecovery(db, deps).sweepPendingCleanupLeases({ backoffMs: 0 })).toMatchObject({ destroyed: 1 }); + expect((await environmentService(db).getLeaseById(f.lease.id))!).toMatchObject({ status: "released", cleanupStatus: "success" }); + expect(stop).toHaveBeenCalledOnce(); expect(destroy).not.toHaveBeenCalled(); + } finally { await deleteShutdownSource(f); } + }); + + it("waits for a claimed ordinary cleanup attempt before converting its shutdown source", async () => { + const f = await seedShutdownSource(); const expiresAt = Date.now() + 120_000; + try { + await db.update(heartbeatRuns).set({ status: "interrupted", finishedAt: new Date(), contextSnapshot: { issueId: f.run.issueId } }).where(eq(heartbeatRuns.id, f.run.id)); + const [marked] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)); + await db.update(environmentLeases).set({ status: "pending_cleanup", cleanupStatus: "failed", releasedAt: new Date(), + metadata: { ...marked.metadata, pendingCleanupInFlight: true, pendingCleanupLeaseExpiresAtMs: expiresAt } }).where(eq(environmentLeases.id, f.lease.id)); + const [source] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)); + expect(await allowLegacyShutdownWorkspaceCleanup(db, source, new Date(expiresAt - 1))).toBe(false); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].resultJson?.workspaceRestoreRecovery).toBeUndefined(); + expect(await allowLegacyShutdownWorkspaceCleanup(db, source, new Date(expiresAt + 1))).toBe(true); + expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)))[0]) + .toMatchObject({ status: "pending_cleanup", metadata: { sandboxStopAndRetain: { leaseId: f.lease.id } } }); + } finally { await deleteShutdownSource(f); } + }); + + it("retries lease release when ordinary completion wins the shutdown status race", async () => { + const f = await seedShutdownSource(); const control = createAdapterExecutionControl(); + adapterExecutionControls.set(f.run.id, control); + const deps = callbacks(db); + deps.setRunStatusIfRunning.mockImplementation(async () => { + await db.update(heartbeatRuns).set({ status: "succeeded", resultJson: { workspaceRestored: true } }).where(eq(heartbeatRuns.id, f.run.id)); + // The adapter's first release was deferred by the fence; its finally ended. + control.finish(); return { updated: false, run: null }; + }); + deps.getRun.mockImplementation(async id => (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)))[0] ?? null); + deps.releaseEnvironmentLeasesForRun.mockImplementation(async input => { + expect(input.status).toBe("succeeded"); + expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)))[0].metadata) + .toMatchObject({ legacyShutdownWorkspaceSettlement: { state: "settled" } }); + await db.update(environmentLeases).set({ status: "released", cleanupStatus: "success", releasedAt: new Date() }).where(eq(environmentLeases.id, f.lease.id)); + }); + try { + await createHeartbeatRecovery(db, deps).drainRunningRunsForShutdown("SIGTERM", new Date(), [f.run.id]); + expect(deps.releaseEnvironmentLeasesForRun).toHaveBeenCalledOnce(); + expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)))[0].status).toBe("released"); + expect(deps.scheduleBoundedRetryForRun).not.toHaveBeenCalled(); + } finally { control.finish(); await deleteShutdownSource(f); } + }); + + it("pins an unjoined sandbox restore for repair before bounded shutdown cleanup", async () => { + const run = await seedRun(); + await db.update(heartbeatRuns).set({ contextSnapshot: { issueId: run.issueId } }).where(eq(heartbeatRuns.id, run.id)); + const [lease] = await db.insert(environmentLeases).values({ companyId: run.companyId, heartbeatRunId: run.id, + issueId: run.issueId, status: "active", provider: "daytona", providerLeaseId: randomUUID(), leasePolicy: "ephemeral", + metadata: { driver: "sandbox", sandboxProviderPlugin: true, pluginId: "fixture-plugin", remoteCwd: "/home/daytona/workspace" } }).returning(); + const deps = callbacks(db); + deps.setRunStatusIfRunning.mockImplementation(async (id, status, patch) => { + expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, lease.id)))[0].metadata) + .toMatchObject({ legacyShutdownWorkspaceSettlement: { state: "pending", runId: run.id } }); + const [updated] = await db.update(heartbeatRuns).set({ ...patch, status }).where(eq(heartbeatRuns.id, id)).returning(); + return { updated: true, run: updated }; + }); + deps.getRun.mockImplementation(async id => (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)))[0] ?? null); + deps.releaseEnvironmentLeasesForRun.mockImplementation(async () => { + expect((await deps.getRun(run.id))?.resultJson).toMatchObject({ workspaceRestoreFailure: "restore_failed", + workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [lease.id] } }); + expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, lease.id)))[0]).toMatchObject({ + status: "pending_cleanup", leasePolicy: "retain_on_failure", metadata: { sandboxStopAndRetain: { leaseId: lease.id } } }); + }); + const control = createAdapterExecutionControl(); + adapterExecutionControls.set(run.id, control); + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + const drain = createHeartbeatRecovery(db, deps).drainRunningRunsForShutdown("SIGTERM", new Date(), [run.id]); + try { + await vi.waitFor(() => expect(control.controller.signal.aborted).toBe(true)); + await vi.advanceTimersByTimeAsync(30_000); + await drain; + expect(deps.releaseEnvironmentLeasesForRun).toHaveBeenCalledOnce(); + expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, run.issueId!))) + .toEqual([expect.objectContaining({ evidence: expect.objectContaining({ workspaceRestoreFailure: "restore_failed" }) })]); + } finally { + vi.useRealTimers(); + control.finish(); + adapterExecutionControls.delete(run.id); + await drain; + await db.delete(environmentLeases).where(eq(environmentLeases.id, lease.id)); + await db.delete(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, run.issueId!)); + await db.delete(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)); + } + }); + it("uses the shared live-execution set after construction across two factories", async () => { const run = await seedRun(); const deps = callbacks(db); diff --git a/server/src/services/heartbeat/recovery.ts b/server/src/services/heartbeat/recovery.ts index a727f4ad5f4..4a6db618b5a 100644 --- a/server/src/services/heartbeat/recovery.ts +++ b/server/src/services/heartbeat/recovery.ts @@ -13,7 +13,7 @@ import { } from "../legacy-controller-lease.js"; import { remoteTerminationReceipt } from "../remote-execution-termination.js"; import { runUsedConversationAdapter } from "../conversation-continuation.js"; -import { legacyExecutionNeedsReconciliationWithEvidence } from "../legacy-execution-recovery.js"; +import { legacyExecutionNeedsReconciliationWithEvidence, terminalizeLegacyExecution } from "../legacy-execution-recovery.js"; import { randomUUID } from "node:crypto"; import { and, @@ -72,6 +72,8 @@ import { type HotRestartReportRun, } from "../hot-restart.js"; import { serverVersion } from "../../version.js"; +import { adapterExecutionControls, waitForAdapterStop } from "../adapter-execution-control.js"; +import { allowLegacyShutdownWorkspaceCleanup, beginLegacyShutdownWorkspaceSettlement, finishLegacyShutdownWorkspaceSettlement, sweepLegacyShutdownWorkspaceSettlements } from "../legacy-shutdown-workspace-settlement.js"; import type { environmentService } from "../environments.js"; import type { environmentRuntimeService, ProviderResourceDisposition } from "../environment-runtime.js"; @@ -1086,12 +1088,13 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD const retryRunIds: string[] = []; const restartSuspendedRunIds: string[] = []; - for (const { run, agent } of activeRuns) { + const settlementDeadline = new Date(Date.now() + 30_000); + const settlements = await Promise.allSettled(activeRuns.map(async ({ run, agent }) => { // Shutdown owns only this boot's legacy executions. Expired foreign // owners belong to the reaper, not another container's drain. if (run.runtimeMode === "legacy" && run.controllerBootId && - run.controllerBootId !== legacyControllerBootId) continue; - if (isNativeRunnerOwnershipHeld(run)) continue; + run.controllerBootId !== legacyControllerBootId) return; + if (isNativeRunnerOwnershipHeld(run)) return; if ( run.runtimeMode === "native" && agent.adapterType === "paperclip_runner" @@ -1152,10 +1155,12 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD }, }); restartSuspendedRunIds.push(run.id); - continue; + return; } const message = `Interrupted by graceful server shutdown (${signal})`; const running = runningProcesses.get(run.id); + const control = run.runtimeMode === "legacy" ? adapterExecutionControls.get(run.id) : undefined; + if (control) await beginLegacyShutdownWorkspaceSettlement(db, run, settlementDeadline); try { if (run.runtimeMode === "native") { await cancelHeartbeatNativeRun({ @@ -1165,7 +1170,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD runtimeMode: run.runtimeMode, }); } - if (running) { + if (running && !control) { await terminateHeartbeatRunProcess({ pid: running.child.pid, processGroupId: running.processGroupId, @@ -1173,7 +1178,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD }); } } finally { - runningProcesses.delete(run.id); + if (!control && runningProcesses.get(run.id) === running) runningProcesses.delete(run.id); } const persistedCancellationResult = @@ -1199,8 +1204,54 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD }), }, ); - if (!interruptedStatus.updated || !interruptedStatus.run) continue; + if (!interruptedStatus.updated || !interruptedStatus.run) { + // A competing terminal write does not prove the owned adapter settled. + if (control) { + control.controller.abort(new Error(message)); + try { + await waitForAdapterStop(control.settled, Math.max(1, settlementDeadline.getTime() - Date.now())); + await finishLegacyShutdownWorkspaceSettlement(db, run); + const current = await getRun(run.id, { includeExecutionEvidence: true }); + if (current && HEARTBEAT_RUN_TERMINAL_STATUSES.some(status => status === current.status)) { + await releaseEnvironmentLeasesForRun({ runId: current.id, companyId: current.companyId, + agentId: current.agentId, status: current.status, failureReason: current.error ?? undefined }); + } + } catch { /* The durable fence retains this source after controller loss. */ } + } + return; + } let interrupted = interruptedStatus.run; + if (control) { + // Keep the exact lease open while its adapter stops the provider and + // copies files back. Provider release closes command admission and + // must not race the adapter's git export or instruction collection. + control.controller.abort(new Error(message)); + try { + await waitForAdapterStop(control.settled, Math.max(1, settlementDeadline.getTime() - Date.now()), { + runId: run.id, adapterType: agent.adapterType, runtimeMode: run.runtimeMode, + abortRequested: true, + }, control); + await finishLegacyShutdownWorkspaceSettlement(db, run); + } catch { + logger.warn({ runId: run.id }, "Shutdown adapter settlement timed out; retaining the sandbox through normal cleanup"); + const sources = await db.select().from(environmentLeases).where(and( + eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id), + sql`${environmentLeases.metadata}->'legacyShutdownWorkspaceSettlement'->>'state' = 'pending'`, + )); + // Stop-only settlement can already have released the source. Claim + // its current fence and pin recovery before classifying or retrying. + for (const source of sources) await allowLegacyShutdownWorkspaceCleanup(db, source, + new Date(Math.max(Date.now(), settlementDeadline.getTime()))); + if (runningProcesses.get(run.id) === running && running) { + await terminateHeartbeatRunProcess({ pid: running.child.pid, processGroupId: running.processGroupId, + graceMs: Math.max(1, running.graceSec) * 1000 }); + if (runningProcesses.get(run.id) === running) runningProcesses.delete(run.id); + } + } + // Settlement can save restore-failure evidence after the shutdown CAS. + // Classification and retry must use the current row, not its snapshot. + interrupted = (await getRun(run.id, { includeExecutionEvidence: true })) ?? interrupted; + } await setWakeupStatus(run.wakeupRequestId, "cancelled", { finishedAt: now, error: null, @@ -1243,6 +1294,10 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run), }); interruptedRunIds.push(interrupted.id); + })); + for (const [index, settlement] of settlements.entries()) { + if (settlement.status === "rejected") logger.warn({ runId: activeRuns[index]!.run.id, + errorKind: "shutdown_settlement_failed" }, "Shutdown settlement failed; durable cleanup protection remains in place"); } if (interruptedRunIds.length > 0) { @@ -1436,6 +1491,10 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD let recovered = 0; for (const { lease } of rows) { + if (!(await allowLegacyShutdownWorkspaceCleanup(db, lease))) { + await deferOrphanedActiveLease(lease.id); + continue; + } // A provider resource id names one physical sandbox. A different lease // row can still hold that same resource in a live status, so this sweep // must not tear down a sandbox that a different lease still owns. @@ -1554,6 +1613,10 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD let capped = 0; for (const row of rows) { if (pendingCleanupAttemptsInFlight.has(row.id)) continue; + if (!(await allowLegacyShutdownWorkspaceCleanup(db, row))) { + await deferPendingCleanupLease(row.id); + continue; + } const metadata = { ...(row.metadata ?? {}) } as Record; const attempts = readPendingCleanupRetryAttempts(metadata); @@ -2370,6 +2433,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD // pending_cleanup sweep, so this tick can stop the recovered sandbox. // Isolate the sweep so its failure never hides the reaper result. try { + await sweepLegacyShutdownWorkspaceSettlements(db); const orphanedActiveLeaseSweep = await sweepOrphanedActiveLeases({ backoffMs: staleThresholdMs, }); diff --git a/server/src/services/legacy-shutdown-workspace-settlement.ts b/server/src/services/legacy-shutdown-workspace-settlement.ts new file mode 100644 index 00000000000..fcf2fbd5448 --- /dev/null +++ b/server/src/services/legacy-shutdown-workspace-settlement.ts @@ -0,0 +1,137 @@ +import { and, eq, gt, inArray, ne, or, sql } from "drizzle-orm"; +import { environmentLeases, heartbeatRuns, issues, type Db } from "@paperclipai/db"; +import { adapterExecutionControls } from "./adapter-execution-control.js"; +import { legacyControllerBootId } from "./legacy-controller-lease.js"; +import { LEGACY_WORKSPACE_RECOVERY_SCHEMA } from "./workspace-restore-recovery-state.js"; + +const SCHEMA = "paperclip.legacy-shutdown-workspace-settlement.v1"; +type Lease = Pick; +type Run = Pick; + +export function pendingLegacyShutdownWorkspaceSettlement(lease: Lease) { + const value = lease.metadata?.legacyShutdownWorkspaceSettlement; + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const marker = value as Record; + return marker.schema === SCHEMA && marker.state === "pending" && marker.runId === lease.heartbeatRunId + && typeof marker.deadlineAt === "string" ? marker : null; +} + +/** Historical reuse rows may name the same allocation as the fenced source. */ +export async function legacyShutdownWorkspaceResourceProtected(db: Db, + lease: Pick) { + if (!lease.provider || !lease.providerLeaseId) return false; + const [protectedLease] = await db.select({ id: environmentLeases.id }).from(environmentLeases).where(and( + eq(environmentLeases.provider, lease.provider), eq(environmentLeases.providerLeaseId, lease.providerLeaseId), + or(sql`${environmentLeases.metadata}->'legacyShutdownWorkspaceSettlement'->>'schema' = ${SCHEMA} + and ${environmentLeases.metadata}->'legacyShutdownWorkspaceSettlement'->>'state' = 'pending'`, + sql`${environmentLeases.metadata}->'workspaceRestoreRecovery'->>'schema' = ${LEGACY_WORKSPACE_RECOVERY_SCHEMA}`), + )).limit(1); + return !!protectedLease; +} + +/** Released stop receipts still need reconciliation if export never settled. */ +export async function sweepLegacyShutdownWorkspaceSettlements(db: Db, now = new Date()) { + const rows = await db.select().from(environmentLeases).where(and( + sql`${environmentLeases.metadata}->'legacyShutdownWorkspaceSettlement'->>'schema' = ${SCHEMA}`, + sql`${environmentLeases.metadata}->'legacyShutdownWorkspaceSettlement'->>'state' = 'pending'`, + sql`${environmentLeases.metadata}->'legacyShutdownWorkspaceSettlement'->>'deadlineAt' <= ${now.toISOString()}`, + )).orderBy(sql`${environmentLeases.metadata}->'legacyShutdownWorkspaceSettlement'->>'deadlineAt'`).limit(50); + for (const lease of rows) await allowLegacyShutdownWorkspaceCleanup(db, lease, now); +} + +/** Commit the fence while the run is still live, before exposing interruption. */ +export async function beginLegacyShutdownWorkspaceSettlement(db: Db, run: Run, deadlineAt: Date) { + await db.update(environmentLeases).set({ metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || + ${JSON.stringify({ legacyShutdownWorkspaceSettlement: { schema: SCHEMA, state: "pending", runId: run.id, + controllerBootId: legacyControllerBootId, deadlineAt: deadlineAt.toISOString() } })}::jsonb`, + }).where(and(eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id), + eq(environmentLeases.status, "active"), ne(environmentLeases.provider, "local"), + or(sql`${environmentLeases.metadata}->>'driver' = 'sandbox'`, sql`${environmentLeases.metadata}->>'sandboxProviderPlugin' = 'true'`))); +} + +/** The owner joins settlement, or records exact retention before timeout cleanup. */ +export async function finishLegacyShutdownWorkspaceSettlement(db: Db, run: Run, state: "settled" | "expired" = "settled") { + await db.update(environmentLeases).set({ metadata: sql`jsonb_set(${environmentLeases.metadata}, + '{legacyShutdownWorkspaceSettlement,state}', ${JSON.stringify(state)}::jsonb)` }).where(and( + eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id), + sql`${environmentLeases.metadata}->'legacyShutdownWorkspaceSettlement'->>'schema' = ${SCHEMA}`, + sql`${environmentLeases.metadata}->'legacyShutdownWorkspaceSettlement'->>'controllerBootId' = ${legacyControllerBootId}`, + sql`${environmentLeases.metadata}->'legacyShutdownWorkspaceSettlement'->>'state' = 'pending'`, + )); +} + +/** A restart or another cleanup worker must never infer saved files from timeout. */ +export async function allowLegacyShutdownWorkspaceCleanup(db: Db, lease: Lease, now = new Date(), + options: { ownerStopOnly?: boolean } = {}) { + if (!pendingLegacyShutdownWorkspaceSettlement(lease)) return true; + return db.transaction(async tx => { + const [hint] = await tx.select().from(environmentLeases).where(and( + eq(environmentLeases.id, lease.id), eq(environmentLeases.companyId, lease.companyId), + eq(environmentLeases.heartbeatRunId, lease.heartbeatRunId!), + )); + if (!hint) return false; + if (!pendingLegacyShutdownWorkspaceSettlement(hint)) return true; + // Match the terminal recorder's issue -> run -> lease lock order. Settlement + // and expiry compete on the current lease row, not the caller's old snapshot. + if (hint.issueId) await tx.select({ id: issues.id }).from(issues).where(and( + eq(issues.id, hint.issueId), eq(issues.companyId, lease.companyId))).for("update"); + const [run] = await tx.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, lease.companyId), eq(heartbeatRuns.id, lease.heartbeatRunId!), + eq(heartbeatRuns.runtimeMode, "legacy"), + )).for("update"); + const [current] = await tx.select().from(environmentLeases).where(and( + eq(environmentLeases.id, lease.id), eq(environmentLeases.companyId, lease.companyId), + eq(environmentLeases.heartbeatRunId, lease.heartbeatRunId!), + )).for("update"); + if (!run || !current) return false; + const marker = pendingLegacyShutdownWorkspaceSettlement(current); + if (!marker) return true; + // Stopping and retaining the exact owner's command is a prerequisite for + // some adapters to settle. This exception never permits provider destruction. + if (options.ownerStopOnly && marker.controllerBootId === legacyControllerBootId + && adapterExecutionControls.get(run.id)?.controller.signal.aborted) return true; + const deadline = Date.parse(marker.deadlineAt as string); + if (!Number.isFinite(deadline) || deadline > now.getTime()) return false; + // A killed container cannot certify export. Preserve source and hold in + // this same transaction before any worker may perform provider cleanup. + if (current.status !== "active") { + const { hasConfirmedSandboxStopAndRetain, readStopOnlyCleanup } = await import("./sandbox-stop-and-retain.js"); + if (!(current.status === "released" && hasConfirmedSandboxStopAndRetain(current)) + && current.status !== "pending_cleanup") return false; + // The shutdown marker was committed on this exact active source before + // cancellation. A stop receipt does not certify the later file export. + const [competing] = await tx.select({ id: environmentLeases.id }).from(environmentLeases).where(and( + ne(environmentLeases.id, current.id), eq(environmentLeases.provider, current.provider!), + eq(environmentLeases.providerLeaseId, current.providerLeaseId!), + or(inArray(environmentLeases.status, ["active", "pending_cleanup"]), + gt(environmentLeases.acquiredAt, current.acquiredAt)), + )).limit(1); + if (competing) return false; + if (current.status === "pending_cleanup" && !readStopOnlyCleanup(current)) { + const expiresAt = current.metadata?.pendingCleanupLeaseExpiresAtMs; + if (current.metadata?.pendingCleanupInFlight === true + && (typeof expiresAt !== "number" || !Number.isFinite(expiresAt) || expiresAt > now.getTime())) return false; + // A failed ordinary release can race the shutdown marker. Re-enter the + // recorder's active-source path within this claim transaction so it + // replaces destructive retry with an exact stop-only recovery intent. + await tx.update(environmentLeases).set({ status: "active" }).where(eq(environmentLeases.id, current.id)); + } else { + await tx.update(environmentLeases).set({ leasePolicy: "retain_on_failure", + metadata: sql`${environmentLeases.metadata} || ${JSON.stringify({ workspaceRestoreRecovery: { + schema: LEGACY_WORKSPACE_RECOVERY_SCHEMA, runId: run.id, + } })}::jsonb`, + }).where(eq(environmentLeases.id, current.id)); + } + } + const { terminalizeLegacyExecution } = await import("./legacy-execution-recovery.js"); + const preserved = await terminalizeLegacyExecution({ db: tx as unknown as Db, run, status: run.status, + recordRestoreFailureOnly: true, patch: { resultJson: { workspaceRestoreFailure: "restore_failed" } } }); + if (!preserved) return false; + await tx.update(environmentLeases).set({ metadata: sql`jsonb_set(${environmentLeases.metadata}, + '{legacyShutdownWorkspaceSettlement,state}', '"expired"'::jsonb)` }).where(and( + eq(environmentLeases.id, lease.id), eq(environmentLeases.companyId, lease.companyId), + eq(environmentLeases.heartbeatRunId, run.id), + )); + return true; + }); +} From 4b59e2a2ecc67e98b336b8eb4c09959681672d54 Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Sun, 11 Oct 2026 10:01:57 -0700 Subject: [PATCH 03/18] fix(server): allow idle sleep with saved AI auth and verified lease history (#15897) ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - Automatic idle sleep must preserve accepted work, cleanup and recovery. > - Saved AI login connections are configuration without independent workers. > - Failed ephemeral leases can remain after their resources have stopped. > - Detached subscription observations and quota refreshes can outlive a response. > - This pull request distinguishes inert configuration and verified history from unfinished work. > - The benefit is safe idle eligibility while keeping credentials and audit history. ## Linked Issues or Issue Description **What happened?** An active saved AI login blocks idle sleep after its login and agent runs finish. A failed ephemeral lease also blocks after successful cleanup and provider termination. The connection guard currently hides detached quota and subscription work that can outlive the request that started it. **Expected behavior** An idle instance can sleep with saved AI credentials and verified failed-lease history. Accepted refresh work, recovery journals, unfinished login sessions, pending cleanup and run accounting must still block. **Steps to reproduce** 1. Save an AI connection with purpose `ai`, transport `runtime_auth` and status `active`. 2. Finish login and all agent work, then acquire an owned idle drain. 3. Read the idle safety report. The saved connection alone reports background work. 4. Repeat with a failed ephemeral lease whose exact provider termination and finished run are recorded. Its history also reports background work. **Paperclip version or commit** Based on `0bbd586a2f6`, after updates from master. **Deployment mode** Single-process managed or self-hosted server using the owned idle-drain protocol. Related work: #15522, #15761 and #15851. Searched open and closed pull requests for related idle-sleep changes. The open accounting recovery work in #15879 is separate; this change does not settle or discard accounting debt. ## What Changed - Exempt active saved AI runtime-auth configuration from the connection sleep check. Other connection types and unknown states keep their existing checks. - Track detached subscription discovery and provider observations until their actual promises settle. - Track underlying quota work through an HTTP timeout, including an OAuth exchange and the save of its replacement credential. - Inspect the credential-recovery directory without reading credential bytes. Journals, probes, temporary files, unexpected entries and read failures keep sleep blocked. Empty company directories left by successful recovery permit sleep. - Exempt a failed ephemeral lease only after successful cleanup, a release timestamp, an exact company/run/lease/resource termination receipt, and a finished run without retry or accounting debt. Export and cleanup intents still block. - Use the same failed-lease proof when checking finished login-session cleanup. - Add PostgreSQL and filesystem regression tests and document the behavior. ## Verification - Four focused suites passed: 186 tests covering durable guard decisions, local recovery, detached subscription work and quota timeouts. - `pnpm build` passed. - `pnpm -r typecheck` passed. - `pnpm --filter @paperclipai/server build` and a final server TypeScript check passed after the directory-scanner cleanup. - The final directory-scan recheck passed all 18 tests. - The original-base `pnpm test:run` attempt found three local failures. It was stopped before completion to update the branch after two master changes. All three also reproduce on unchanged master `e733bddc45e`: two Git default-branch fixtures in `workspace-runtime.test.ts` and one native runner recovery fixture in `native-session-resume.test.ts`. - Both Git fixtures pass with a command-scoped `init.defaultBranch=master` setting. The native fixture fails with the same result on both branches. No global Git setting was changed. - All CI gates passed on the final head `ff63497d9bef40f5f00567bc729fb17536cfe136`: [complete CI run](https://github.com/paperclipai/paperclip/actions/runs/38156312722). The only skipped checks are the two path-gated Storybook jobs. - Greptile Apex scored the final head 5/5 with no actionable findings and no unresolved review threads. The branch has no merge conflicts with master. - After the master update, `pnpm build` and `pnpm -r typecheck` passed again. All four focused suites passed again: 186 tests. - Staging deployment and live sleep/wake verification are pending the Core merge and the verified private Cloud build. No production rollout has been started. ## Risks - The failed-lease exception is deliberately narrow. Older records without exact termination evidence or a complete run receipt continue to block. - Pending OAuth recovery blocks sleep until it is recovered. No credential, history, cleanup obligation or accounting receipt is deleted. - The existing owned drain, ingress fence, local-work generation checks, plugin drain and backup checkpoint remain required. - No schema migration or provider mutation is introduced. Roll back to the previous application release to restore the conservative guard behavior. ## Model Used OpenAI GPT-6 (Codex), with repository inspection, code editing, test execution and read-only provider analysis. No additional model version or context-window size was supplied by the runtime. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either linked existing issues or described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links - [x] My branch name describes the change and contains no internal ticket id or instance-derived details - [x] I have run focused tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented risks - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no unresolved comments - [x] I will address all reviewer comments before requesting merge Co-authored-by: Paperclip --- doc/idle-sleep-safety.md | 27 +++- .../__tests__/company-quota-windows.test.ts | 30 ++++- server/src/__tests__/idle-local-work.test.ts | 53 +++++++- .../src/__tests__/idle-sleep-safety.test.ts | 120 +++++++++++++++++- .../__tests__/subscription-refresh.test.ts | 35 +++++ server/src/services/idle-local-work.ts | 28 +++- server/src/services/idle-sleep-safety.ts | 40 +++++- .../src/services/quota-credential-recovery.ts | 4 +- server/src/services/quota-windows.ts | 7 +- server/src/services/subscription-refresh.ts | 9 +- 10 files changed, 333 insertions(+), 20 deletions(-) diff --git a/doc/idle-sleep-safety.md b/doc/idle-sleep-safety.md index b4bf960ce59..e006680d1a2 100644 --- a/doc/idle-sleep-safety.md +++ b/doc/idle-sleep-safety.md @@ -113,17 +113,36 @@ operator reconciliation; this API does not discard it to save compute. The report checks all companies in one bounded read-only database transaction. Queued and orphaned runs, timers, retries, accounting debt, unfinished issues, -cleanup, active routines, integrations, external API credentials, and plugin +cleanup, active routines, integrations, external ingress credentials, and plugin work block sleep. Completed ordinary run history does not. Less common work sources conservatively block on any retained records. Active issue watchdogs block sleep even when their watched issue is complete and no review has started. Missing migrations, malformed configuration and database errors produce unknown without exposing SQL or tenant data. +Active `runtime_auth` connections with purpose `ai` are saved credentials, not +independent workers. Their saved configuration does not block sleep. Login +sessions and runs still pass the existing durable checks. Detached subscription +observations and quota reads remain counted through their actual completion, +including token refresh after a quota response times out. Credential-recovery +journals, partial writes and probes block sleep across restart. Empty company +directories left after successful recovery do not; invalid paths and read +failures remain unknown. Credentials and connection records are retained. +MCP, chat and other connections still block until they have a wake contract. + +A failed ephemeral lease is history only with successful cleanup, a release +timestamp, and an exact provider termination receipt bound to its company, run, +lease and resource. Its run must be finished without retry or accounting debt. +Pending cleanup claims and export/stop-and-retain intents still block. Failure +labels or cleanup success without that proof do not authorize sleep. Lease +history is retained; this check performs no provider teardown or data updates. + Finished login-session history permits sleep only after its promotion claim is -cleared and any retained provider reference has a matching released or expired -lease with successful cleanup and a release timestamp in the same company and -environment. Missing cleanup evidence, `cleanup_pending`, and live login flows +cleared and any retained provider reference has a matching released, expired, +or verified failed ephemeral lease with successful cleanup and a release +timestamp in the same company and environment. The failed lease must satisfy +the same termination and run checks above. Missing cleanup evidence, +`cleanup_pending`, and live login flows remain blockers. A finished coalesced wake is history only when its exact company/agent's linked run has finished without retry or accounting debt. Image-setup history still blocks: promotion can succeed before teardown is diff --git a/server/src/__tests__/company-quota-windows.test.ts b/server/src/__tests__/company-quota-windows.test.ts index 646f821657d..38d0ef238a1 100644 --- a/server/src/__tests__/company-quota-windows.test.ts +++ b/server/src/__tests__/company-quota-windows.test.ts @@ -25,6 +25,7 @@ vi.mock("@paperclipai/adapter-claude-local/server", () => ({ fetchClaudeQuota: mocks.claude, })); import { fetchCompanyQuotaWindows } from "../services/quota-windows.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; let revision = 1; const db = { @@ -192,7 +193,8 @@ describe("connected account quotas", () => { it("bounds a stalled credential lookup and preserves its account identity", async () => { vi.useFakeTimers(); mocks.accounts.mockResolvedValue([account("stalled")]); - mocks.credential.mockReturnValue(new Promise(() => {})); + let finish!: (value: string) => void; + mocks.credential.mockReturnValue(new Promise(resolve => { finish = resolve; })); const pending = fetchCompanyQuotaWindows(db, "company-8", "user-8"); await vi.advanceTimersByTimeAsync(20_001); const [result] = await pending; @@ -203,6 +205,32 @@ describe("connected account quotas", () => { }); expect(result.accountKey).toBeTruthy(); expect(result.error).not.toContain("timed out"); + finish(JSON.stringify({ accessToken: "fixture" })); + await vi.advanceTimersByTimeAsync(0); + }); + + it("retains the actual credential refresh after its HTTP quota deadline expires", async () => { + vi.useFakeTimers(); + const before = idleWorkSnapshot(); + mocks.accounts.mockResolvedValue([account("tracked-refresh")]); + mocks.codex.mockRejectedValueOnce(new Error("401")); + let finish!: (value: string) => void; + mocks.refresh.mockReturnValue(new Promise(resolve => { finish = resolve; })); + const pending = fetchCompanyQuotaWindows(db, "tracked-company", "tracked-user"); + try { + await vi.advanceTimersByTimeAsync(0); + expect(mocks.refresh).toHaveBeenCalledOnce(); + expect(idleWorkSnapshot().active).toBe(before.active + 1); + await vi.advanceTimersByTimeAsync(20_001); + expect((await pending)[0].ok).toBe(false); + expect(idleWorkSnapshot().active).toBe(before.active + 1); + } finally { + finish(JSON.stringify({ accessToken: "fixture" })); + await vi.advanceTimersByTimeAsync(0); + await pending; + } + expect(idleWorkSnapshot().active).toBe(before.active); + expect(idleWorkSnapshot().generation).toBe(before.generation + 2); }); it("does not start a provider request when a credential resolves after its deadline", async () => { vi.useFakeTimers(); diff --git a/server/src/__tests__/idle-local-work.test.ts b/server/src/__tests__/idle-local-work.test.ts index 666c58d0b20..9728c97562b 100644 --- a/server/src/__tests__/idle-local-work.test.ts +++ b/server/src/__tests__/idle-local-work.test.ts @@ -2,7 +2,7 @@ import { promises as fs } from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { inspectIdleSpool, readIdleLocalWork, markIdleIngressTracked, markIdleStartupComplete, registerIdleSpoolDirectory } from "../services/idle-local-work.js"; +import { inspectIdleSpool, inspectIdleQuotaRecovery, idleQuotaRecoveryPath, readIdleLocalWork, markIdleIngressTracked, markIdleStartupComplete, registerIdleSpoolDirectory } from "../services/idle-local-work.js"; let root: string; beforeEach(async () => { root = await fs.mkdtemp(path.join(os.tmpdir(), "idle-spool-")); }); @@ -51,3 +51,54 @@ describe("idle cleanup inspection", () => { }); }); + +describe("idle credential recovery inspection", () => { + const company = "7238bc31-f3ca-42ba-b487-4122e77b8867"; + it("allows missing roots and empty company directories after successful recovery", async () => { + expect(await inspectIdleQuotaRecovery(path.join(root, "missing"))).toBe("none"); + expect(await inspectIdleQuotaRecovery(root)).toBe("none"); + await fs.mkdir(path.join(root, company)); + expect(await inspectIdleQuotaRecovery(root)).toBe("none"); + }); + it.each(["receipt.json", "malformed.json", "partial.tmp", "write.probe", "unexpected"])( + "retains every pending company entry without reading its contents: %s", async name => { + await fs.mkdir(path.join(root, company)); + const file = path.join(root, company, name); + await fs.writeFile(file, "unreadable recovery fixture"); + const read = vi.spyOn(fs, "readFile"); + expect(await inspectIdleQuotaRecovery(root)).toBe("present"); + expect(read).not.toHaveBeenCalled(); + await fs.rm(file); + expect(await inspectIdleQuotaRecovery(root)).toBe("none"); + }, + ); + it("retains unexpected root entries, nested directories and directory symlinks", async () => { + await fs.mkdir(path.join(root, "unexpected")); + expect(await inspectIdleQuotaRecovery(root)).toBe("present"); + await fs.rm(path.join(root, "unexpected"), { recursive: true }); + await fs.symlink(path.join(root, "missing"), path.join(root, company)); + expect(await inspectIdleQuotaRecovery(root)).toBe("present"); + await fs.rm(path.join(root, company)); + await fs.mkdir(path.join(root, company, "nested"), { recursive: true }); + expect(await inspectIdleQuotaRecovery(root)).toBe("present"); + }); + it("fails closed on invalid roots and directory read errors", async () => { + const file = path.join(root, "file"), link = path.join(root, "link"); + await fs.writeFile(file, ""); await fs.symlink(root, link); + expect(await inspectIdleQuotaRecovery(file)).toBe("unknown"); + expect(await inspectIdleQuotaRecovery(link)).toBe("unknown"); + vi.spyOn(fs, "opendir").mockRejectedValue(Object.assign(new Error("private path"), { code: "EACCES" })); + expect(await inspectIdleQuotaRecovery(root)).toBe("unknown"); + }); + it("keeps persisted credential recovery awake after the request and process work have ended", async () => { + vi.stubEnv("PAPERCLIP_HOME", root); + markIdleIngressTracked(); markIdleStartupComplete({ scheduledBackups: false }); + const directory = path.join(idleQuotaRecoveryPath(), company); + await fs.mkdir(directory, { recursive: true }); + const file = path.join(directory, "pending.json"); + await fs.writeFile(file, "encrypted journal fixture"); + expect(await readIdleLocalWork()).toBe("present"); + await fs.rm(file); + expect(await readIdleLocalWork()).toBe("none"); + }); +}); diff --git a/server/src/__tests__/idle-sleep-safety.test.ts b/server/src/__tests__/idle-sleep-safety.test.ts index f8e15241074..4b7d7319722 100644 --- a/server/src/__tests__/idle-sleep-safety.test.ts +++ b/server/src/__tests__/idle-sleep-safety.test.ts @@ -6,7 +6,7 @@ import { agentInstructionWorkingCopies, agents, agentApiKeys, agentWakeupRequests, companies, companySecretProposals, createDb, adapterAuthSessions, environments, environmentLeases, executionWorkspaces, heartbeatRuns, issues, issueWatchdogs, issueThreadInteractions, issueQuestionResponseDeliveries, - projects, routines, type Db, + projects, routines, toolApplications, toolConnections, type Db, } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { readIdleSleepSafety, type IdleSleepDrainStatus } from "../services/idle-sleep-safety.js"; @@ -443,6 +443,124 @@ if (!support.supported) console.warn(`Skipping idle sleep Postgres tests: ${supp expect(await read()).toEqual(present); }); + async function seedAiConnection() { + const seeded = await seed(); + const [application] = await db.insert(toolApplications).values({ + companyId: seeded.companyId, name: "AI fixture", type: "rest_api", + }).returning(); + const [connection] = await db.insert(toolConnections).values({ + companyId: seeded.companyId, applicationId: application!.id, + name: "Saved AI login", uid: randomUUID(), connectionPurpose: "ai", + transport: "runtime_auth", status: "active", enabled: true, + }).returning(); + return { ...seeded, connection: connection! }; + } + + it("allows saved AI credentials while preserving their connection", async () => { + const { connection } = await seedAiConnection(); + expect(await read()).toEqual(none); + expect(await db.select().from(toolConnections).where(eq(toolConnections.id, connection.id))) + .toMatchObject([{ status: "active", enabled: true }]); + }); + + it.each(["draft", "active_login", "pending_run", "accepted_refresh"])( + "keeps AI connections awake for %s work", async kind => { + const { companyId, runId, connection } = await seedAiConnection(); + if (kind === "draft") await db.update(toolConnections).set({ status: "draft" }).where(eq(toolConnections.id, connection.id)); + if (kind === "pending_run") await db.update(heartbeatRuns).set({ status: "queued" }).where(eq(heartbeatRuns.id, runId)); + if (kind === "active_login") { + const [environment] = await db.insert(environments).values({ name: "Active AI login" }).returning(); + await db.insert(adapterAuthSessions).values({ + companyId, environmentId: environment!.id, connectionId: connection.id, + adapterType: "codex_local", startedByUserId: "fixture-user", + publicSessionId: randomUUID(), status: "waiting_for_user", + }); + } + const finish = kind === "accepted_refresh" ? beginIdleTrackedWork() : undefined; + try { expect(await read()).toEqual(kind === "accepted_refresh" ? unknown : present); } + finally { finish?.(); } + }, + ); + + it("retains remote MCP connections alongside inert AI credentials", async () => { + const { companyId, connection } = await seedAiConnection(); + await db.insert(toolConnections).values({ + companyId, applicationId: connection.applicationId, name: "Remote tools", uid: randomUUID(), + connectionPurpose: "tool", transport: "mcp_remote", status: "active", enabled: true, + }); + expect(await read()).toEqual(present); + }); + + async function seedFinishedFailedLease() { + const seeded = await seed(); + await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date(now) }) + .where(eq(heartbeatRuns.id, seeded.runId)); + const [environment] = await db.insert(environments).values({ name: "Failed turn environment", driver: "sandbox" }).returning(); + const leaseId = randomUUID(); + const receipt = { + schema: "paperclip.remote-termination.v1", companyId: seeded.companyId, + runId: seeded.runId, leaseId, provider: "plugin:demo:sandbox", + providerLeaseId: "fixture-resource", state: "destroyed", confirmedAt: new Date(now).toISOString(), + }; + const [lease] = await db.insert(environmentLeases).values({ + id: leaseId, companyId: seeded.companyId, environmentId: environment!.id, + heartbeatRunId: seeded.runId, status: "failed", leasePolicy: "ephemeral", + provider: receipt.provider, providerLeaseId: receipt.providerLeaseId, + acquiredAt: new Date(now - 1000), releasedAt: new Date(now), cleanupStatus: "success", + metadata: { remoteExecutionTermination: receipt }, + }).returning(); + return { ...seeded, environment: environment!, lease: lease!, receipt }; + } + + it.each(["destroyed", "stopped"])("allows failed lease history after exact provider %s proof", async state => { + const { lease, receipt } = await seedFinishedFailedLease(); + await db.update(environmentLeases).set({ metadata: { remoteExecutionTermination: { ...receipt, state } } }) + .where(eq(environmentLeases.id, lease.id)); + expect(await read()).toEqual(none); + expect(await db.select().from(environmentLeases).where(eq(environmentLeases.id, lease.id))) + .toMatchObject([{ status: "failed", cleanupStatus: "success" }]); + }); + + it.each([ + "cleanup_failed", "cleanup_unknown", "unreleased", "retained_policy", "reusable_policy", + "active", "pending_cleanup", "missing_receipt", "wrong_company", "wrong_run", "wrong_lease", + "wrong_provider", "wrong_resource", "not_stopped", "missing_run", "unfinished_run", + "retry", "accounting", "cleanup_in_flight", "pending_export", "pending_retain", + ])("retains failed leases with %s evidence", async kind => { + const { lease, receipt, runId } = await seedFinishedFailedLease(); + const change: Partial = {}; + if (kind === "cleanup_failed") change.cleanupStatus = "failed"; + if (kind === "cleanup_unknown") change.cleanupStatus = null; + if (kind === "unreleased") change.releasedAt = null; + if (kind === "retained_policy") change.leasePolicy = "retain_on_failure"; + if (kind === "reusable_policy") change.leasePolicy = "reuse_by_environment"; + if (kind === "active" || kind === "pending_cleanup") change.status = kind; + if (kind === "missing_receipt") change.metadata = {}; + const fields: Record = { wrong_company: "companyId", wrong_run: "runId", wrong_lease: "leaseId", wrong_provider: "provider", wrong_resource: "providerLeaseId", not_stopped: "state" }; + if (fields[kind]) change.metadata = { remoteExecutionTermination: { ...receipt, [fields[kind]]: "wrong" } }; + if (kind === "missing_run") change.heartbeatRunId = null; + if (kind === "unfinished_run") await db.update(heartbeatRuns).set({ finishedAt: null }).where(eq(heartbeatRuns.id, runId)); + if (kind === "retry") await db.update(heartbeatRuns).set({ scheduledRetryAt: new Date(now + 1000) }).where(eq(heartbeatRuns.id, runId)); + if (kind === "accounting") await db.update(heartbeatRuns).set({ costAccountingPending: true }).where(eq(heartbeatRuns.id, runId)); + if (kind === "cleanup_in_flight") change.metadata = { remoteExecutionTermination: receipt, pendingCleanupInFlight: true }; + if (kind === "pending_export") change.metadata = { remoteExecutionTermination: receipt, nativeWorkspaceExportResume: {} }; + if (kind === "pending_retain") change.metadata = { remoteExecutionTermination: receipt, sandboxStopAndRetain: {} }; + if (Object.keys(change).length) await db.update(environmentLeases).set(change).where(eq(environmentLeases.id, lease.id)); + expect(await read()).toEqual(present); + }); + + it("uses the same failed-lease cleanup proof for finished login history", async () => { + const { companyId, environment, lease } = await seedFinishedFailedLease(); + await db.insert(adapterAuthSessions).values({ + companyId, environmentId: environment.id, adapterType: "codex_local", startedByUserId: "fixture-user", + publicSessionId: randomUUID(), providerLeaseId: lease.providerLeaseId, + status: "authenticated", finishedAt: new Date(now), + }); + expect(await read()).toEqual(none); + await db.update(environmentLeases).set({ metadata: {} }).where(eq(environmentLeases.id, lease.id)); + expect(await read()).toEqual(present); + }); + it("reports a login sandbox awaiting cleanup", async () => { const { companyId } = await seed(); const [environment] = await db.insert(environments).values({ name: "Login fixture" }).returning(); diff --git a/server/src/__tests__/subscription-refresh.test.ts b/server/src/__tests__/subscription-refresh.test.ts index abe237d87f1..cb2f4bddc49 100644 --- a/server/src/__tests__/subscription-refresh.test.ts +++ b/server/src/__tests__/subscription-refresh.test.ts @@ -2,6 +2,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import type { Db } from "@paperclipai/db"; import { refreshCompanySubscriptions, refreshSubscriptionConnection } from "../services/subscription-refresh.js"; import type { SubscriptionConnection } from "../services/subscriptions.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; const mocks = vi.hoisted(() => ({ accounts: vi.fn(), credential: vi.fn(), register: vi.fn(), probe: vi.fn() })); vi.mock("../services/ai-connections.js", () => ({ aiConnectionService: () => ({ subscriptionAccounts: mocks.accounts, credential: mocks.credential }) })); @@ -28,6 +29,40 @@ beforeEach(() => { }); describe("subscription discovery capacity", () => { + it("tracks detached provider refresh through its final database write", async () => { + const db = database(), probe = deferred<{ credentialKey: string }>(), save = deferred(); + const before = idleWorkSnapshot(); + mocks.probe.mockReturnValue(probe.promise); + mocks.register.mockReturnValue(save.promise); + const task = refreshSubscriptionConnection(db, input("tracked"), "tracked"); + try { + await vi.waitFor(() => expect(mocks.probe).toHaveBeenCalledOnce()); + expect(idleWorkSnapshot().active).toBe(before.active + 1); + expect(refreshSubscriptionConnection(db, input("tracked"), "tracked")).toBe(task); + expect(idleWorkSnapshot().active).toBe(before.active + 1); + probe.resolve({ credentialKey: "fixture-key" }); + await vi.waitFor(() => expect(mocks.register).toHaveBeenCalledOnce()); + expect(idleWorkSnapshot().active).toBe(before.active + 1); + } finally { + probe.resolve({ credentialKey: "fixture-key" }); save.resolve("tracked"); await task; + } + expect(idleWorkSnapshot().active).toBe(before.active); + expect(idleWorkSnapshot().generation).toBe(before.generation + 2); + }); + + it("tracks detached discovery while it waits for account credentials", async () => { + const db = database(), accounts = deferred<[]>(); + const before = idleWorkSnapshot(); + mocks.accounts.mockReturnValue(accounts.promise); + const task = refreshCompanySubscriptions(db, "tracked-company", "alice"); + try { + expect(idleWorkSnapshot().active).toBe(before.active + 1); + expect(refreshCompanySubscriptions(db, "tracked-company", "alice")).toBe(task); + expect(idleWorkSnapshot().active).toBe(before.active + 1); + } finally { accounts.resolve([]); await task; } + expect(idleWorkSnapshot().active).toBe(before.active); + }); + it.each([false, true])("waits for a provider slot without dropping accepted discovery (provider fails: %s)", async (fails) => { const db = database(); const gates = Array.from({ length: 5 }, () => deferred()); diff --git a/server/src/services/idle-local-work.ts b/server/src/services/idle-local-work.ts index 2d54ad90309..0338a66330c 100644 --- a/server/src/services/idle-local-work.ts +++ b/server/src/services/idle-local-work.ts @@ -9,6 +9,7 @@ export function registerIdleSpoolDirectory(directory: string): void { spoolDirec export const idleAccountingSpoolPath = () => path.join(resolvePaperclipInstanceRoot(), "accounting-receipts"); export const idleOrphanSpoolPath = () => process.env.SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR ?? path.join(resolvePaperclipInstanceRoot(), "data", "sandbox-orphan-cleanup"); +export const idleQuotaRecoveryPath = () => path.join(resolvePaperclipInstanceRoot(), "quota-credential-recovery"); let startupComplete = false; let ingressTracked = false; let scheduledBackups = false; @@ -33,13 +34,38 @@ export async function inspectIdleSpool(directory: string): Promise { + try { + const info = await fs.lstat(directory); + if (!info.isDirectory() || info.isSymbolicLink()) return "unknown"; + const dir = await fs.opendir(directory); + try { + for (let entry = await dir.read(); entry !== null; entry = await dir.read()) { + if (!entry.isDirectory() || entry.isSymbolicLink() + || !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(entry.name)) return "present"; + const state = await inspectIdleSpool(path.join(directory, entry.name)); + if (state !== "none") return state; + } + return "none"; + } finally { await dir.close(); } + } catch (error) { + return (error as NodeJS.ErrnoException).code === "ENOENT" ? "none" : "unknown"; + } +} + export async function readIdleLocalWork(options: { backupCheckpoint?: boolean } = {}): Promise { if (!startupComplete || !ingressTracked) return "unknown"; // This exemption belongs only to a scan which will finish a new backup // under the same admission hold before it can return an empty report. if ((scheduledBackups && !options.backupCheckpoint) || idleWorkSnapshot().active !== 0) return "present"; const directories = new Set([...spoolDirectories, idleAccountingSpoolPath(), idleOrphanSpoolPath()]); - const results = await Promise.all([...directories].map(inspectIdleSpool)); + const results = await Promise.all([ + ...[...directories].map(inspectIdleSpool), + inspectIdleQuotaRecovery(idleQuotaRecoveryPath()), + ]); if (results.includes("unknown")) return "unknown"; return results.includes("present") ? "present" : "none"; } diff --git a/server/src/services/idle-sleep-safety.ts b/server/src/services/idle-sleep-safety.ts index 3e637a8c0aa..4d1a5f61968 100644 --- a/server/src/services/idle-sleep-safety.ts +++ b/server/src/services/idle-sleep-safety.ts @@ -22,6 +22,32 @@ export type InspectIdlePlugins = (hold: { ownerId: string; expiresAt: number }) pluginIds: string[]; }>; +// A failed turn may have successfully torn down its ephemeral allocation. A +// cleanup label alone is not provider termination proof. Require the exact +// host-owned receipt and finished run, and retain all cleanup/export intents. +// Aliases are fixed at the call sites, never supplied by a client. +function finishedFailedLease(alias: string): string { + const receipt = `${alias}.metadata->'remoteExecutionTermination'`; + return `${alias}.status = 'failed' AND ${alias}.lease_policy = 'ephemeral' + AND ${alias}.cleanup_status = 'success' AND ${alias}.released_at IS NOT NULL + AND COALESCE(${alias}.metadata->>'pendingCleanupInFlight', 'false') = 'false' + AND NOT (COALESCE(${alias}.metadata, '{}'::jsonb) ?| ARRAY['nativeWorkspaceExportResume', 'sandboxStopAndRetain']) + AND ${alias}.provider IS NOT NULL AND ${alias}.provider <> 'local' + AND ${alias}.provider_lease_id IS NOT NULL AND ${alias}.provider_lease_id <> '' + AND ${receipt}->>'schema' = 'paperclip.remote-termination.v1' + AND ${receipt}->>'companyId' = ${alias}.company_id::text + AND ${receipt}->>'runId' = ${alias}.heartbeat_run_id::text + AND ${receipt}->>'leaseId' = ${alias}.id::text + AND ${receipt}->>'provider' = ${alias}.provider + AND ${receipt}->>'providerLeaseId' = ${alias}.provider_lease_id + AND ${receipt}->>'state' IN ('stopped', 'destroyed') + AND EXISTS (SELECT 1 FROM heartbeat_runs r WHERE r.id = ${alias}.heartbeat_run_id + AND r.company_id = ${alias}.company_id + AND r.status IN ('succeeded', 'failed', 'cancelled', 'timed_out', 'interrupted') + AND r.finished_at IS NOT NULL AND r.scheduled_retry_at IS NULL + AND r.cost_accounting_pending = false)`; +} + // These are deliberately conservative. Completed agent runs and workspace // operations are history, not reasons to keep an otherwise idle instance up. // Other background features remain awake until their work and inbound events @@ -60,13 +86,18 @@ const WORK_CHECKS = [ // retry after immediate evaluation failed before creating a review/run. `SELECT 1 FROM issue_watchdogs WHERE status <> 'disabled'`, `SELECT 1 FROM workspace_operations WHERE status NOT IN ('succeeded', 'failed', 'cancelled')`, - `SELECT 1 FROM environment_leases WHERE status NOT IN ('released', 'expired')`, + `SELECT 1 FROM environment_leases l WHERE status NOT IN ('released', 'expired') + AND (${finishedFailedLease("l")}) IS NOT TRUE`, `SELECT 1 FROM routines WHERE status NOT IN ('paused', 'archived')`, `SELECT 1 FROM routine_runs WHERE status NOT IN ('completed', 'failed', 'skipped', 'cancelled')`, `SELECT 1 FROM plugin_jobs WHERE status <> 'paused'`, `SELECT 1 FROM plugin_job_runs WHERE status NOT IN ('succeeded', 'failed', 'cancelled')`, `SELECT 1 FROM plugin_webhook_deliveries WHERE status NOT IN ('succeeded', 'failed')`, - `SELECT 1 FROM tool_connections WHERE status <> 'archived'`, + // Saved AI credentials have no independent worker. Login sessions, accepted + // quota/refresh work and recovery journals have their own checks. Other + // integrations still need an ingress/scheduler wake contract. + `SELECT 1 FROM tool_connections WHERE status <> 'archived' + AND NOT (status = 'active' AND connection_purpose = 'ai' AND transport = 'runtime_auth')`, `SELECT 1 FROM issue_recovery_actions WHERE status NOT IN ('resolved', 'cancelled')`, `SELECT 1 FROM status_cards WHERE archived_at IS NULL`, `SELECT 1 FROM external_objects WHERE NOT is_terminal OR next_refresh_at IS NOT NULL`, @@ -83,7 +114,8 @@ const WORK_CHECKS = [ AND (l.provider_lease_id = s.provider_lease_id OR l.id = CASE WHEN s.provider_lease_id ~* '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' THEN s.provider_lease_id::uuid END) - AND l.status IN ('released', 'expired') AND l.cleanup_status = 'success' + AND (l.status IN ('released', 'expired') OR (${finishedFailedLease("l")})) + AND l.cleanup_status = 'success' AND l.released_at IS NOT NULL)) OR (s.provider_lease_id IS NOT NULL AND EXISTS ( SELECT 1 FROM environment_leases l WHERE l.company_id = s.company_id @@ -91,7 +123,7 @@ const WORK_CHECKS = [ AND (l.provider_lease_id = s.provider_lease_id OR l.id = CASE WHEN s.provider_lease_id ~* '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' THEN s.provider_lease_id::uuid END) - AND (l.status IN ('released', 'expired') AND l.cleanup_status = 'success' + AND ((l.status IN ('released', 'expired') OR (${finishedFailedLease("l")})) AND l.cleanup_status = 'success' AND l.released_at IS NOT NULL) IS NOT TRUE))`, // Answer delivery history is not work once its acknowledged handoff has // finished. Keep pending deliveries, unfinished target runs, and missing or diff --git a/server/src/services/quota-credential-recovery.ts b/server/src/services/quota-credential-recovery.ts index d666641c059..583ae235247 100644 --- a/server/src/services/quota-credential-recovery.ts +++ b/server/src/services/quota-credential-recovery.ts @@ -2,9 +2,9 @@ import { createHash, randomUUID } from "node:crypto"; import { promises as fs } from "node:fs"; import path from "node:path"; import { z } from "zod"; -import { resolvePaperclipInstanceRoot } from "../home-paths.js"; import { localEncryptedProvider } from "../secrets/local-encrypted-provider.js"; import { ensureDurableDirectory, syncDirectory } from "../lib/durable-directory.js"; +import { idleQuotaRecoveryPath } from "./idle-local-work.js"; const recoverySchema = z.object({ companyId: z.string().uuid(), grantId: z.string().uuid(), connectionId: z.string().uuid(), secretId: z.string().uuid(), @@ -13,7 +13,7 @@ const recoverySchema = z.object({ type Recovery = z.infer; export const quotaCredentialHash = (value: string) => createHash("sha256").update(value).digest("hex"); export const quotaCredentialRecoveryPath = (companyId: string, grantId: string) => - path.join(resolvePaperclipInstanceRoot(), "quota-credential-recovery", companyId, `${grantId}.json`); + path.join(idleQuotaRecoveryPath(), companyId, `${grantId}.json`); /** Called only while holding the company credential mutation lock. The local * encrypted provider uses the instance secrets master key, including when the diff --git a/server/src/services/quota-windows.ts b/server/src/services/quota-windows.ts index 2d363969b26..a31b2153fdb 100644 --- a/server/src/services/quota-windows.ts +++ b/server/src/services/quota-windows.ts @@ -8,6 +8,7 @@ import { companySecrets, companySecretVersions, userSecretDefinitions, type Db } import { fetchCodexQuota } from "@paperclipai/adapter-codex-local/server"; import { fetchClaudeQuota } from "@paperclipai/adapter-claude-local/server"; import { aiConnectionService } from "./ai-connections.js"; +import { trackIdleWork } from "./task-admission.js"; const accountRequests = new Map }>(); const publicError = "Subscription quota is currently unavailable. Check usage with your provider."; @@ -51,7 +52,9 @@ export async function fetchCompanyQuotaWindows(db: Db, companyId: string, userId let pending = accountRequests.get(key); if (!pending || pending.expires <= Date.now()) { const controller = new AbortController(); - const result = (async (): Promise => { + // The HTTP deadline can expire while an OAuth exchange still needs to + // save its consumed token. Track the actual work, not its timeout wrapper. + const result = trackIdleWork((async (): Promise => { let credentialResolved = false; try { const value = await service.credential(row); @@ -104,7 +107,7 @@ export async function fetchCompanyQuotaWindows(db: Db, companyId: string, userId return { ...base, ok: false, windows: [], error: publicError, errorFamily }; } - })(); + })()); const bounded: Promise = withQuotaTimeout(base.provider, result, () => controller.abort()).then(quota => { const observation = { ...base, ...quota, ...(quota.ok ? {} : { error: publicError }) }; if (observation.ok && observation.accountKey !== accountKey) { diff --git a/server/src/services/subscription-refresh.ts b/server/src/services/subscription-refresh.ts index cc46f8333c8..24322d099ee 100644 --- a/server/src/services/subscription-refresh.ts +++ b/server/src/services/subscription-refresh.ts @@ -5,6 +5,7 @@ import { subscriptionService, type SubscriptionConnection } from "./subscription import { probeSubscriptionIdentity } from "./subscription-identity.js"; import { logger } from "../middleware/logger.js"; import { tooManyRequests } from "../errors.js"; +import { trackIdleWork } from "./task-admission.js"; const jobs = new WeakMap>>(); const discovery = new WeakMap>>(); @@ -19,7 +20,7 @@ export function refreshSubscriptionConnection(db: Db, input: SubscriptionConnect // Bound provider concurrency across all companies on this instance. Dashboard // discovery waits below; opportunistic run observations remain due when busy. if (active.size >= 4) return Promise.resolve(); - const task = (async () => { + const task = trackIdleWork((async () => { const [claimed] = await db.update(aiSubscriptions).set({ lastCheckedAt: new Date() }).where(and( eq(aiSubscriptions.companyId, input.companyId), eq(aiSubscriptions.id, id), or(isNull(aiSubscriptions.lastCheckedAt), lt(aiSubscriptions.lastCheckedAt, new Date(Date.now() - REFRESH_MS))), @@ -38,7 +39,7 @@ export function refreshSubscriptionConnection(db: Db, input: SubscriptionConnect })().catch(() => { // No raw provider/credential errors are logged or returned to a user. logger.warn({ companyId: input.companyId, subscriptionId: id }, "Subscription observation could not be saved"); - }).finally(() => active!.delete(id)); + }).finally(() => active!.delete(id))); active.set(id, task); return task; } @@ -50,7 +51,7 @@ export function refreshCompanySubscriptions(db: Db, companyId: string, userId: s const existing = active.get(key); if (existing) return existing; if (active.size >= 20) throw tooManyRequests("Subscription account checks are busy. Try again shortly."); - const task = (async () => { + const task = trackIdleWork((async () => { const service = aiConnectionService(db); // Uses the existing credential audience. Being a company admin does not // authorize probing another member's personal credentials. @@ -75,7 +76,7 @@ export function refreshCompanySubscriptions(db: Db, companyId: string, userId: s } } })().catch(() => logger.warn({ companyId }, "Subscription discovery unavailable")) - .finally(() => active!.delete(key)); + .finally(() => active!.delete(key))); active.set(key, task); return task; } From 85f4205de3914aa8025615622bde5b0dc2f6d107 Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Sun, 11 Oct 2026 11:19:13 -0700 Subject: [PATCH 04/18] perf(test): reuse a migrated embedded Postgres template across suites (#15861) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Pull requests to this repository run a large CI matrix. The slowest check sets the time a contributor waits for a green run. > - After #15853, the twelve `server (n/12)` shards are the slowest check in every fully green PR run created since its merge (518s to 548s; `Build` is 482s to 504s). Each shard is about 60s of setup and 454s to 490s of Vitest, with one worker, so its wall time is the sum of its suites. > - From two shard logs, 22 to 24 of the 71 to 74 suites in a shard start an embedded Postgres cluster. Each one runs `initdb`, starts the cluster, and applies all 331 migrations before its first test. The smallest such suite takes 6s with one or two trivial tests. On top of that, the `getEmbeddedPostgresTestSupport` probe starts and stops a second throwaway cluster at import time in 19 of those suites, which is why their per-file import overhead is 3.5s to 4.1s against 0.9s to 1.0s for suites without Postgres. > - A cluster data directory that was shut down cleanly can be copied and started again as an independent cluster, and the migrated schema is the same for every suite. > - This pull request builds one migrated cluster per host, publishes its data directory as a template under the OS temp directory, and starts every later suite from a copy of it. The probe reuses the template instead of starting a cluster. > - The benefit is about 7s to 8s per Postgres suite on the CI runners (measured 3.2s to 0.35s per start and 0.75s to 0.03s per probe locally), about 120s to 170s per server shard, and a faster local test loop for every suite that uses the fixture. ## Linked Issues or Issue Description **What existing behavior does this improve?** The `server (n/12)` PR checks are the slowest check in every fully green PR run since 2026-10-11 05:27Z. Their Vitest step runs suites one at a time, and about a third of each shard's time is embedded Postgres setup that produces the same migrated cluster every time. **Subsystem affected** Test infrastructure: the embedded Postgres test fixture in `@paperclipai/db` (`packages/db/src/test-embedded-postgres.ts`), used by 349 server suites, the db package suites, and the CLI suites. **Current behavior** `startEmbeddedPostgresTestDatabase` runs `initdb`, starts the cluster, creates the database, and applies all 331 migrations for every suite. `getEmbeddedPostgresTestSupport` starts and stops a second throwaway cluster once per test file, because the module-level cache does not survive Vitest's per-file isolation. Measured locally: 3.2s per suite start plus 0.4s cleanup, and 0.75s per probe. On the CI runners the smallest Postgres suite takes 6s and the probe adds 2.5s to 3s of import time per file. **Proposed behavior** The first caller on a host builds one migrated cluster, stops it, and renames its data directory to `$TMPDIR/paperclip-embedded-postgres-templates-/`. Every later caller copies the template (about 90 MB) and starts Postgres on the copy, then still runs `ensurePostgresDatabase` and `applyPendingMigrations`, which are no-ops on a current template. The probe treats a published template as proof that the host can run the cluster. The key hashes the embedded-postgres version, the initdb flags, the user, and the name and content of every migration file plus the journal, so a changed migration builds a new template. `PAPERCLIP_TEST_POSTGRES_TEMPLATE=0` bypasses the cache. Any template failure falls back to the fresh path. A template is a Postgres data directory, and starting it executes its configuration, so the cache is private to the current OS user: the root name carries the uid, the root is created with mode 0700, and both the root and each template are used only when `lstat` shows a real directory (not a symlink) owned by the current user with no group or world write bit. A root that fails those checks (for example one pre-created by another user in a shared `/tmp`) is left alone and the process uses fresh clusters. **Reason and benefit** The slowest check decides the wall-clock time of every PR run. Measured locally, a template start takes 0.3s to 0.4s instead of 3.2s, cleanup 0.15s instead of 0.43s, and the probe 0.03s instead of 0.75s. Four server suites that passed in both modes dropped by 3.4s to 6.0s each; the db `backup-lib` suite dropped from 42.7s to 20.9s. The CI runners are about twice as slow, so the expected saving per server shard is 120s to 170s. **Breaking changes** None. The returned database is fully migrated in both paths, each suite still gets its own cluster and data directory, and a failure on the template path falls back to the existing fresh path. ## What Changed - `packages/db/src/test-embedded-postgres.ts`: adds the template cache. `computeEmbeddedPostgresTemplateKey` hashes the inputs that determine a migrated cluster. `buildEmbeddedPostgresTemplate` starts a fresh cluster through the existing bounded retry, creates the database, applies migrations, waits for the real `stop()` (30s bound), and publishes the data directory with an atomic rename; a concurrent builder that loses the rename discards its copy. `startEmbeddedPostgresFromTemplateWithRetry` copies the template into a fresh temp directory, sets mode 0700, and starts the cluster with the same per-attempt port and directory rotation as the fresh path. `startEmbeddedPostgresTestDatabase` tries the template first and falls back to the fresh path with one warning per process. `probeEmbeddedPostgresSupport` returns supported when a template exists or can be built, and keeps its throwaway-cluster path for the case where the template could not be published. Templates for other keys are pruned after seven days. The connection strings and initdb flags move into shared constants; no other behavior changes. - `packages/db/src/test-embedded-postgres.test.ts`: two unit tests for the key (stable for identical inputs; changes with migration content, a new file, flags, version, or user; hashes this package's real migrations directory) and two real-cluster tests (the published template is complete and private, a suite starts from it in `template` mode with migrations up to date, a table written to one copy is not visible in the next copy, and `PAPERCLIP_TEST_POSTGRES_TEMPLATE=0` runs the `fresh` path). The real-cluster tests skip on hosts without embedded Postgres, like every other fixture suite. - `doc/DEVELOPING.md`: a short section under Test Commands describing the template, its key, and the bypass variable. **Review fixes (second commit), one per finding:** - Private cache root (Greptile P1, Superagent P2). The root is `$TMPDIR/paperclip-embedded-postgres-templates-` (user name on Windows), created with mode 0700. `prepareTemplateRootDir` and `isUsableTemplate` both reject, via `lstat`, anything that is a symlink, not a directory, owned by another uid, or writable by group or others; a rejected root logs one warning and the process uses fresh clusters. A root created by the first commit with a wider mode is narrowed to 0700 if this user owns it. - Template failures never skip suites (Greptile P1). `getEmbeddedPostgresTemplateDir` no longer rejects. A build or root failure resolves `null` with one warning, and `probeEmbeddedPostgresSupport` then runs the throwaway-cluster probe, so only a cluster that cannot start reports `supported: false`. A migration that cannot apply fails the suite on the fresh path instead of being hidden by `describe.skip`. - Idle-based pruning (Greptile P2). Each reuse refreshes the template's timestamp with `utimes`, so the seven-day prune measures idle time. A template that disappears after a process resolved its path is not retried five times: `startEmbeddedPostgresFromTemplateWithRetry` checks the template before each copy and throws at once, the suite falls back to a fresh cluster, and the fallback forgets the path so the next caller in that process rebuilds the template. - The template-path test enables the cache for itself and restores the original `PAPERCLIP_TEST_POSTGRES_TEMPLATE` value afterwards (Greptile P1), so the suite passes under `PAPERCLIP_TEST_POSTGRES_TEMPLATE=0` as well. - Eight new tests: `isUsableTemplate` accepts a private cluster directory and rejects a missing `PG_VERSION`, a leftover `postmaster.pid`, a symlink, and (POSIX) group- or world-writable modes; a world-writable pre-created root and a symlinked root both fall back to fresh clusters, publish nothing, log one warning, and still report `supported: true` from the probe; a ten-day-old template is refreshed on reuse; a pruned template is rebuilt on the next start; and a template build that fails after the cluster started falls through to the fresh probe. The tests that need an empty cache point `os.tmpdir()` at a scratch directory (`TMPDIR`, `TEMP` and `TMP` together, so the redirect holds on Windows as well) and restore it. **Review fixes (third commit, e8f8e9c8), one per finding:** - Fallback test on Windows (Greptile P1). `useScratchTmpdir` set only `TMPDIR`, which Windows `os.tmpdir()` ignores, so the one scratch-root test that runs on every platform would assert on paths outside the scratch root there. The helper now sets and restores `TMPDIR`, `TEMP` and `TMP` together, and that test asserts `os.tmpdir() === scratch` before it checks any path under it. - Timestamp test runs on its own (Greptile P2). The test assumed an earlier test had published a template; under `PAPERCLIP_TEST_POSTGRES_TEMPLATE=0` the probe does not, so a name-filtered run threw `ENOENT` from `utimes`. It now starts and cleans up one suite first, which builds the template when nothing has, and asserts `isUsableTemplate` before ageing it. ## Verification - `pnpm --filter @paperclipai/db exec tsc --noEmit`: clean (both commits). - `pnpm --filter @paperclipai/db exec vitest run src/test-embedded-postgres.test.ts`: 15 pass (12 new across both commits) with the template root absent (first run builds it), with it present, and with `PAPERCLIP_TEST_POSTGRES_TEMPLATE=0`. - After the review fixes: `client.test.ts` and `backup-lib.test.ts` (29 tests) pass on the template path, and `agent-action-audit-routes.test.ts` and `status-cards.test.ts` (36 tests) pass through the server route harness. The second commit's CI run ([38121021162](https://github.com/paperclipai/paperclip/actions/runs/38121021162)) passed every check: `server (n/12)` jobs 277s to 425s, Vitest `Duration` 219s to 365s, no fallback warning in any shard. - Third commit: reproduced the Greptile P2 on the old file first (`PAPERCLIP_TEST_POSTGRES_TEMPLATE=0 vitest run src/test-embedded-postgres.test.ts -t "refreshes the template"` on an empty cache root fails with `ENOENT … utime`), then with the fix the same command passes, and the full file passes 15/15 on a cold cache, a warm cache, and with the bypass set. `tsc --noEmit` clean. **Measured on this PR's first CI run ([38117892083](https://github.com/paperclipai/paperclip/actions/runs/38117892083)), which ran the branch's fixture in every server shard:** | | Before (runs 38114982393, 38115279131, 38115459196) | This PR's run | | --- | --- | --- | | Vitest `Duration` per `server (n/12)` shard | 454s to 490s | 224s to 337s | | `server (n/12)` job | 518s to 548s | 286s to 401s | | Slowest check in the run | a server shard | `Build`, 453s | No shard logged a fixture fallback warning, so every Postgres suite in the run started from the template. - `pnpm --filter @paperclipai/db exec vitest run src/status-card-migrations.test.ts src/nested-skill-folders-migration.test.ts src/chat-github-migration.test.ts src/question-response-delivery-migration.test.ts src/client.test.ts`: 5 files, 24 tests pass on the template path. These suites drop indexes, re-run migration files, and create extra databases on the cluster, so they cover the semantics the template must preserve. - Four server suites run with `PAPERCLIP_TEST_POSTGRES_TEMPLATE=0` and then with the template, all passing in both modes: | Suite | Fresh path | Template path | | --- | --- | --- | | `server/src/__tests__/agent-action-audit-routes.test.ts` (11 tests) | 17.9s | 14.6s | | `server/src/__tests__/status-cards.test.ts` (25 tests) | 22.2s | 18.6s | | `server/src/__tests__/plugin-managed-routines.test.ts` (3 tests) | 18.2s | 14.3s | | `server/src/__tests__/issue-thread-interactions-service.test.ts` (106 tests) | 19.0s | 12.9s | | `packages/db/src/backup-lib.test.ts` (9 tests) | 42.7s | 20.9s | - Per-operation timings on the same host: fresh `initdb` + start + create database + 331 migrations 3.2s, cleanup 0.43s; template copy 0.20s to 0.23s, start from copy 0.03s to 0.04s, `ensurePostgresDatabase` 0.013s, `applyPendingMigrations` fast path 0.03s to 0.04s, stop 0.03s. Probe with the template on disk 0.03s (was 0.75s). Template size 87 MB, 3215 files. - This PR's own server shards run the branch's fixture, so their Vitest `Duration` lines show the saving directly. Expected on this PR's run: `server (n/12)` about 330s to 430s (was 440s to 548s on runs 38114982393, 38115279131 and 38115459196), and the slowest check moves to `Build` (482s to 504s). **Measured on the three fully green `master` PR runs created after #15853 merged** (slowest server shard logs): | Run | Server shards | Vitest `Duration` (slowest shard) | Postgres suites in it | Their reported duration | Their import overhead | | --- | --- | --- | --- | --- | --- | | 38115459196 | 476s to 548s | 483.8s (tests 72%, import 24%) | 22 of 71 | 197s | 83s | | 38114982393 | 409s to 518s | 452.8s (tests 65%, import 30%) | 24 of 74 | 264s | 98s | ## Risks - Low risk to correctness. The template is a cleanly stopped cluster produced by the same `initdb` flags and the same migration runner as the fresh path, and each suite still runs `ensurePostgresDatabase` and `applyPendingMigrations` on its copy. Each suite gets its own copy, its own port, and its own data directory, so isolation between suites is unchanged. - A stale template is prevented by the key. If a migration file, the journal, the embedded-postgres version, the initdb flags, or the user changes, the key changes and a new template is built. Even if a stale template were used, `applyPendingMigrations` on the copy would bring it up to date. - Concurrent first use (for example the file-parallel db project) can build the same template twice. The rename is atomic, so one copy wins and the other is discarded; a template that exists is always complete. The loser pays one extra fresh start, which is what every suite paid before. - The template root lives under the OS temp directory and holds about 90 MB per key. Keys that have not been used for seven days are pruned when a new template is built; each reuse refreshes the timestamp. - A template is executable configuration, so the cache is only used when the root and the template are private directories of the current user (ownership, no symlink, no group or world write bit). On a host where another user pre-created the path, the process logs one warning and uses fresh clusters. Windows has no uid, so there the check is directory-only; its temp directory is already per user. - Any failure on the template path (root checks, build, copy, start, publish) falls back to the fresh path and logs one warning per process, so a broken cache can make a suite slower but never fail it or skip it. The guard tests assert the mode, so a silent fallback on CI would fail `test-embedded-postgres.test.ts` rather than hide. - Hosts that cannot run embedded Postgres still get `supported: false` with the real Postgres output, now from the throwaway probe: a failed template build is one warning, then the probe decides. On such a host the probe costs one extra failed start attempt cycle per file compared with the first commit; every suite on that host skips anyway. ## Model Used - Claude (Anthropic), model ID `claude-fable-5-1`, running as a Paperclip agent through the Claude local adapter with tool use (shell, file edits, GitHub CLI). The agent collected the CI data, made the change, ran the tests, and wrote this description. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green (53 checks green on both run 38117892083 and run 38121021162; the third commit re-runs them) - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups (1/5 on the first commit, 4/5 on the second; every finding so far is fixed and answered, and Superagent confirmed its fix; re-review of the third commit pending) - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Bender (Fable) Co-authored-by: Claude Fable 5.1 Co-authored-by: devinfoley <139239+devinfoley@users.noreply.github.com> --- doc/DEVELOPING.md | 29 ++ .../db/src/test-embedded-postgres.test.ts | 380 +++++++++++++++ packages/db/src/test-embedded-postgres.ts | 449 +++++++++++++++++- 3 files changed, 849 insertions(+), 9 deletions(-) diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 3425955fed7..6fc8fa4f267 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -531,6 +531,35 @@ verifying Vite/HMR behavior. For normal issue work, start with the smallest targeted check that proves the change. Reserve repo-wide typecheck/build/test runs for PR-ready handoff or changes broad enough that narrow checks do not cover the risk. +#### Embedded Postgres template cache + +Suites that call `startEmbeddedPostgresTestDatabase` (from `@paperclipai/db`) +get their own embedded Postgres cluster. The first such suite on a host builds +one fully migrated cluster and publishes its data directory under +`$TMPDIR/paperclip-embedded-postgres-templates-/`. Every later suite +copies that template and starts Postgres on the copy instead of running +`initdb` and all migrations again, and the `getEmbeddedPostgresTestSupport` +probe reuses it instead of starting a throwaway cluster. The key hashes the +embedded-postgres version, the initdb flags, and every migration file, so a +changed migration builds a new template. Each suite still runs +`applyPendingMigrations` on its copy, which is a no-op when the template is +current. Templates for other keys are removed after seven days without use. + +A template is a Postgres data directory, and starting it executes its +configuration, so the cache is private to the current OS user: the root is +created with mode `0700` and is only used when it is a real directory (not a +symlink) owned by the current user with no group or world write bit, and each +template is checked the same way before it is copied. A root that fails those +checks (for example one pre-created by another user in a shared `/tmp`) is left +alone and every suite in that process uses a fresh cluster. + +Set `PAPERCLIP_TEST_POSTGRES_TEMPLATE=0` to bypass the cache and give every +suite a fresh `initdb`, for example when debugging cluster startup itself. A +failure on the template path (root checks, build, copy, start, or publish) +falls back to the fresh path automatically, so the cache can make a suite +slower but never fail it or skip it: a migration that cannot apply still fails +the suite on the fresh path. + ### Task search evaluation The task search relevance rubric and regression corpus are documented in diff --git a/packages/db/src/test-embedded-postgres.test.ts b/packages/db/src/test-embedded-postgres.test.ts index 1d4f78c797d..aeec953d73d 100644 --- a/packages/db/src/test-embedded-postgres.test.ts +++ b/packages/db/src/test-embedded-postgres.test.ts @@ -1,9 +1,21 @@ import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import postgres from "postgres"; import { afterEach, describe, expect, it } from "vitest"; +import { inspectMigrations } from "./client.js"; import { __embeddedPostgresStartMaxAttemptsForTests as MAX_ATTEMPTS, + __embeddedPostgresTemplateRootDirForTests as templateRootDir, + __isUsableEmbeddedPostgresTemplateForTests as isUsableTemplate, + __lastEmbeddedPostgresStartModeForTests as lastStartMode, + __resetEmbeddedPostgresTemplateStateForTests as resetTemplateState, __setEmbeddedPostgresCtorProviderForTests, __startEmbeddedPostgresWithRetryForTests as startWithRetry, + computeEmbeddedPostgresTemplateKey, + EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, + getEmbeddedPostgresTestSupport, + startEmbeddedPostgresTestDatabase, } from "./test-embedded-postgres.js"; // A fake embedded-postgres constructor. It records every constructed instance so @@ -111,3 +123,371 @@ describe("startEmbeddedPostgresWithRetry", () => { expect((error as Error).message).toContain("Address already in use"); }); }); + +describe("computeEmbeddedPostgresTemplateKey", () => { + function writeMigrations(dir: string, files: Record) { + fs.mkdirSync(path.join(dir, "meta"), { recursive: true }); + fs.writeFileSync(path.join(dir, "meta", "_journal.json"), JSON.stringify({ entries: Object.keys(files) })); + for (const [name, content] of Object.entries(files)) { + fs.writeFileSync(path.join(dir, name), content); + } + } + + it("is stable for identical inputs and changes with any migration, flag, or version", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-template-key-")); + try { + const a = path.join(root, "a"); + const b = path.join(root, "b"); + writeMigrations(a, { "0000_one.sql": "create table one();", "0001_two.sql": "create table two();" }); + writeMigrations(b, { "0000_one.sql": "create table one();", "0001_two.sql": "create table two();" }); + const fixed = { packageVersion: "18.1.0", initdbFlags: ["--locale=C"], user: "paperclip" }; + + const keyA = computeEmbeddedPostgresTemplateKey(a, fixed); + expect(keyA).toMatch(/^[0-9a-f]{24}$/); + expect(computeEmbeddedPostgresTemplateKey(a, fixed)).toBe(keyA); + expect(computeEmbeddedPostgresTemplateKey(b, fixed)).toBe(keyA); + + // Content change, new file, flags, version, and user each change the key. + fs.writeFileSync(path.join(b, "0001_two.sql"), "create table two(id int);"); + const contentChanged = computeEmbeddedPostgresTemplateKey(b, fixed); + expect(contentChanged).not.toBe(keyA); + fs.writeFileSync(path.join(b, "0002_three.sql"), "create table three();"); + expect(computeEmbeddedPostgresTemplateKey(b, fixed)).not.toBe(contentChanged); + expect(computeEmbeddedPostgresTemplateKey(a, { ...fixed, initdbFlags: ["--locale=C", "--encoding=UTF8"] })).not.toBe(keyA); + expect(computeEmbeddedPostgresTemplateKey(a, { ...fixed, packageVersion: "18.2.0" })).not.toBe(keyA); + expect(computeEmbeddedPostgresTemplateKey(a, { ...fixed, user: "other" })).not.toBe(keyA); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + + it("hashes the real migrations directory of this package", () => { + const key = computeEmbeddedPostgresTemplateKey(); + expect(key).toMatch(/^[0-9a-f]{24}$/); + expect(computeEmbeddedPostgresTemplateKey()).toBe(key); + }); +}); + +const describeOnPosix = process.platform === "win32" ? describe.skip : describe; + +describe("isUsableTemplate", () => { + let root: string; + + function writeCluster(dir: string): void { + fs.mkdirSync(dir, { recursive: true, mode: 0o700 }); + fs.writeFileSync(path.join(dir, "PG_VERSION"), "18\n"); + } + + afterEach(() => { + fs.rmSync(root, { recursive: true, force: true }); + }); + + it("accepts a private directory that holds a cleanly stopped cluster", () => { + root = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-template-usable-")); + const template = path.join(root, "template"); + writeCluster(template); + expect(isUsableTemplate(template)).toBe(true); + + // A running or crashed cluster leaves `postmaster.pid` behind. + fs.writeFileSync(path.join(template, "postmaster.pid"), "1\n"); + expect(isUsableTemplate(template)).toBe(false); + fs.rmSync(path.join(template, "postmaster.pid")); + expect(isUsableTemplate(template)).toBe(true); + + // A directory without a cluster, or no directory at all. + fs.rmSync(path.join(template, "PG_VERSION")); + expect(isUsableTemplate(template)).toBe(false); + expect(isUsableTemplate(path.join(root, "missing"))).toBe(false); + }); + + it("rejects a symbolic link even when it points at a usable cluster", () => { + root = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-template-usable-")); + const template = path.join(root, "template"); + writeCluster(template); + const link = path.join(root, "link"); + fs.symlinkSync(template, link, "dir"); + expect(isUsableTemplate(template)).toBe(true); + expect(isUsableTemplate(link)).toBe(false); + }); + + describeOnPosix("on POSIX", () => { + it("rejects a cluster directory that group or others can write", () => { + root = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-template-usable-")); + const template = path.join(root, "template"); + writeCluster(template); + for (const mode of [0o720, 0o702, 0o777]) { + fs.chmodSync(template, mode); + expect(isUsableTemplate(template), `mode ${mode.toString(8)}`).toBe(false); + } + // Read bits for others are not a write path; Postgres narrows the copy + // to 0700 before it starts anyway. + fs.chmodSync(template, 0o755); + expect(isUsableTemplate(template)).toBe(true); + fs.chmodSync(template, 0o700); + expect(isUsableTemplate(template)).toBe(true); + }); + }); +}); + +// Real-cluster coverage of the template path. Skipped on hosts that cannot run +// embedded Postgres, like every other suite that uses the fixture. +const support = await getEmbeddedPostgresTestSupport(); +const describeWithPostgres = support.supported ? describe : describe.skip; + +// Points `os.tmpdir()` at a private scratch directory so a test can drive the +// cache root from an empty state. `os.tmpdir()` reads the environment on every +// call: TMPDIR on POSIX, TEMP and then TMP on Windows, so all three are set. +// Returns the scratch directory; `restore` puts the environment and the +// fixture's cached state back. +const TMPDIR_VARIABLES = ["TMPDIR", "TEMP", "TMP"] as const; + +function useScratchTmpdir(prefix: string): { scratch: string; restore(): void } { + const scratch = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + const previous = new Map(TMPDIR_VARIABLES.map((name) => [name, process.env[name]] as const)); + for (const name of TMPDIR_VARIABLES) process.env[name] = scratch; + return { + scratch, + restore() { + for (const [name, value] of previous) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + resetTemplateState(); + fs.rmSync(scratch, { recursive: true, force: true }); + }, + }; +} + +describeWithPostgres("startEmbeddedPostgresTestDatabase template path", () => { + const originalTemplateSetting = process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE; + + afterEach(() => { + if (originalTemplateSetting === undefined) delete process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE; + else process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE = originalTemplateSetting; + }); + + it("publishes a complete template and starts suites from a copy of it", { timeout: EMBEDDED_POSTGRES_TEST_TIMEOUT_MS }, async () => { + // This test covers the template path, so it enables the cache even when + // the suite runs under PAPERCLIP_TEST_POSTGRES_TEMPLATE=0. The first + // start then builds the template if the support probe did not. + delete process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE; + + const database = await startEmbeddedPostgresTestDatabase("paperclip-template-copy-"); + try { + expect(lastStartMode()).toBe("template"); + // The copy is fully migrated and independent of the template. + const state = await inspectMigrations(database.connectionString); + expect(state.status).toBe("upToDate"); + const sql = postgres(database.connectionString, { max: 1 }); + try { + await sql`create table paperclip_template_copy_probe (id int)`; + } finally { + await sql.end(); + } + } finally { + await database.cleanup(); + } + + const rootDir = templateRootDir(); + const templateDir = path.join(rootDir, computeEmbeddedPostgresTemplateKey()); + expect(fs.existsSync(path.join(templateDir, "PG_VERSION"))).toBe(true); + expect(fs.existsSync(path.join(templateDir, "postmaster.pid"))).toBe(false); + expect(isUsableTemplate(templateDir)).toBe(true); + if (process.platform !== "win32") { + // The root is private to this user, and the template is a data + // directory Postgres accepts: both are 0700. + expect(path.basename(rootDir)).toBe(`paperclip-embedded-postgres-templates-${process.getuid?.()}`); + expect(fs.statSync(rootDir).mode & 0o777).toBe(0o700); + expect(fs.statSync(templateDir).mode & 0o777).toBe(0o700); + } + // Writing to the copy never touches the template, so a second copy does + // not see the table. + const second = await startEmbeddedPostgresTestDatabase("paperclip-template-copy-"); + try { + expect(lastStartMode()).toBe("template"); + const sql = postgres(second.connectionString, { max: 1 }); + try { + const rows = await sql`select to_regclass('paperclip_template_copy_probe') as name`; + expect(rows[0]?.name).toBeNull(); + } finally { + await sql.end(); + } + } finally { + await second.cleanup(); + } + }); + + it("refreshes the template's timestamp on every reuse so idle time drives pruning", { timeout: EMBEDDED_POSTGRES_TEST_TIMEOUT_MS }, async () => { + delete process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE; + // Make sure a template exists before ageing it: the support probe does + // not publish one under PAPERCLIP_TEST_POSTGRES_TEMPLATE=0, and a name + // filter can run this test on its own. + resetTemplateState(); + const first = await startEmbeddedPostgresTestDatabase("paperclip-template-touch-"); + await first.cleanup(); + expect(lastStartMode()).toBe("template"); + const templateDir = path.join(templateRootDir(), computeEmbeddedPostgresTemplateKey()); + expect(isUsableTemplate(templateDir)).toBe(true); + // Age the template as if it had not been used for ten days. + const old = new Date(Date.now() - 10 * 24 * 60 * 60 * 1000); + fs.utimesSync(templateDir, old, old); + expect(fs.statSync(templateDir).mtimeMs).toBeLessThan(Date.now() - 9 * 24 * 60 * 60 * 1000); + + resetTemplateState(); + const database = await startEmbeddedPostgresTestDatabase("paperclip-template-touch-"); + try { + expect(lastStartMode()).toBe("template"); + } finally { + await database.cleanup(); + } + expect(fs.statSync(templateDir).mtimeMs).toBeGreaterThan(Date.now() - 60 * 1000); + }); + + it("rebuilds the template when it was pruned after this process resolved it", { timeout: EMBEDDED_POSTGRES_TEST_TIMEOUT_MS }, async () => { + delete process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE; + const scratch = useScratchTmpdir("paperclip-template-pruned-"); + try { + resetTemplateState(); + const first = await startEmbeddedPostgresTestDatabase("paperclip-template-pruned-copy-"); + await first.cleanup(); + expect(lastStartMode()).toBe("template"); + const templateDir = path.join(templateRootDir(), computeEmbeddedPostgresTemplateKey()); + expect(isUsableTemplate(templateDir)).toBe(true); + + // Another process prunes the template while this one still holds its path. + fs.rmSync(templateDir, { recursive: true, force: true }); + const second = await startEmbeddedPostgresTestDatabase("paperclip-template-pruned-copy-"); + await second.cleanup(); + // This start fell back without retrying the missing path; the fallback + // forgot the stale path, so the next start rebuilt the template. + expect(lastStartMode()).toBe("fresh"); + const third = await startEmbeddedPostgresTestDatabase("paperclip-template-pruned-copy-"); + await third.cleanup(); + expect(lastStartMode()).toBe("template"); + expect(isUsableTemplate(templateDir)).toBe(true); + } finally { + scratch.restore(); + } + }); + + it("runs the fresh initdb path when PAPERCLIP_TEST_POSTGRES_TEMPLATE=0", { timeout: EMBEDDED_POSTGRES_TEST_TIMEOUT_MS }, async () => { + process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE = "0"; + const database = await startEmbeddedPostgresTestDatabase("paperclip-template-bypass-"); + try { + expect(lastStartMode()).toBe("fresh"); + const state = await inspectMigrations(database.connectionString); + expect(state.status).toBe("upToDate"); + } finally { + await database.cleanup(); + } + }); + + describeOnPosix("cache root safety", () => { + it("uses fresh clusters when the cache root is not a private directory of this user", { timeout: EMBEDDED_POSTGRES_TEST_TIMEOUT_MS }, async () => { + delete process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE; + const scratch = useScratchTmpdir("paperclip-template-root-"); + const warnings: string[] = []; + const warn = console.warn; + console.warn = (...args: unknown[]) => { + warnings.push(args.map(String).join(" ")); + }; + try { + // Someone pre-created the predictable root path, world-writable, the + // way a foreign user on a shared host would. + const rootDir = templateRootDir(); + expect(rootDir.startsWith(scratch.scratch)).toBe(true); + fs.mkdirSync(rootDir); + fs.chmodSync(rootDir, 0o777); + + resetTemplateState(); + const database = await startEmbeddedPostgresTestDatabase("paperclip-template-root-copy-"); + try { + expect(lastStartMode()).toBe("fresh"); + } finally { + await database.cleanup(); + } + // Nothing was published into the rejected root, and the reason was logged once. + expect(fs.readdirSync(rootDir)).toEqual([]); + expect(warnings.filter((line) => line.includes("template cache root"))).toHaveLength(1); + expect(warnings[0]).toContain("writable by group or others"); + + // The probe still reports support, from the throwaway cluster. + resetTemplateState(); + expect(await getEmbeddedPostgresTestSupport()).toEqual({ supported: true }); + } finally { + console.warn = warn; + scratch.restore(); + } + }); + + it("uses fresh clusters when the cache root path is a symbolic link", { timeout: EMBEDDED_POSTGRES_TEST_TIMEOUT_MS }, async () => { + delete process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE; + const scratch = useScratchTmpdir("paperclip-template-root-"); + const warn = console.warn; + console.warn = () => {}; + try { + const target = path.join(scratch.scratch, "elsewhere"); + fs.mkdirSync(target, { mode: 0o700 }); + fs.symlinkSync(target, templateRootDir(), "dir"); + + resetTemplateState(); + const database = await startEmbeddedPostgresTestDatabase("paperclip-template-root-copy-"); + try { + expect(lastStartMode()).toBe("fresh"); + } finally { + await database.cleanup(); + } + expect(fs.readdirSync(target)).toEqual([]); + } finally { + console.warn = warn; + scratch.restore(); + } + }); + }); +}); + +describe("support probe when the template cannot be built", () => { + afterEach(() => { + __setEmbeddedPostgresCtorProviderForTests(null); + resetTemplateState(); + }); + + it("falls through to the fresh probe instead of reporting the host as unsupported", { timeout: EMBEDDED_POSTGRES_TEST_TIMEOUT_MS }, async () => { + const originalTemplateSetting = process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE; + delete process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE; + const scratch = useScratchTmpdir("paperclip-template-build-fail-"); + const warnings: string[] = []; + const warn = console.warn; + console.warn = (...args: unknown[]) => { + warnings.push(args.map(String).join(" ")); + }; + try { + // This test runs on every platform, so check the redirect took effect + // before asserting on paths under it. + expect(os.tmpdir()).toBe(scratch.scratch); + // The fake cluster "starts" without listening, so the template build + // fails after the start, at the migration step, the way a broken + // migration would. A real broken migration then fails the suite on + // the fresh path instead of being hidden by `describe.skip`. + const { ctor, constructed } = makeFakeCtor(0); + __setEmbeddedPostgresCtorProviderForTests(async () => ctor); + resetTemplateState(); + + expect(await getEmbeddedPostgresTestSupport()).toEqual({ supported: true }); + // One build attempt, then the throwaway probe, both under the scratch root. + expect(constructed).toHaveLength(2); + for (const options of constructed) expect(options.databaseDir.startsWith(scratch.scratch)).toBe(true); + expect(constructed[0]?.databaseDir).toContain("paperclip-embedded-postgres-template-build-"); + expect(constructed[1]?.databaseDir).toContain("paperclip-embedded-postgres-probe-"); + expect(warnings.filter((line) => line.includes("template build failed"))).toHaveLength(1); + // Nothing was published. + const rootDir = templateRootDir(); + expect(fs.existsSync(rootDir) ? fs.readdirSync(rootDir) : []).toEqual([]); + } finally { + console.warn = warn; + scratch.restore(); + if (originalTemplateSetting === undefined) delete process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE; + else process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE = originalTemplateSetting; + } + }); +}); diff --git a/packages/db/src/test-embedded-postgres.ts b/packages/db/src/test-embedded-postgres.ts index 67a52cd1b00..862829c8234 100644 --- a/packages/db/src/test-embedded-postgres.ts +++ b/packages/db/src/test-embedded-postgres.ts @@ -1,7 +1,10 @@ +import { createHash } from "node:crypto"; import fs from "node:fs"; +import { createRequire } from "node:module"; import net from "node:net"; import os from "node:os"; import path from "node:path"; +import { fileURLToPath } from "node:url"; import { applyPendingMigrations, closeRegisteredClients, ensurePostgresDatabase } from "./client.js"; import { createEmbeddedPostgresLogBuffer, @@ -113,8 +116,19 @@ async function getAvailablePort(): Promise { ); } +const EMBEDDED_POSTGRES_USER = "paperclip"; +const EMBEDDED_POSTGRES_PASSWORD = "paperclip"; +const EMBEDDED_POSTGRES_INITDB_FLAGS = ["--encoding=UTF8", "--locale=C", "--lc-messages=C"]; + async function createEmbeddedPostgresTestInstance(tempDirPrefix: string) { const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), tempDirPrefix)); + return await createEmbeddedPostgresTestInstanceAt(dataDir); +} + +// Wraps an embedded-postgres instance around `dataDir`. The caller decides +// whether the directory still needs `initialise()` (a fresh cluster) or already +// holds a copied cluster that only needs `start()`. +async function createEmbeddedPostgresTestInstanceAt(dataDir: string) { const port = await getAvailablePort(); const EmbeddedPostgres = await getEmbeddedPostgresCtor(); // Postgres writes the true reason for a failed start to its output, for @@ -124,11 +138,11 @@ async function createEmbeddedPostgresTestInstance(tempDirPrefix: string) { const logBuffer = createEmbeddedPostgresLogBuffer(); const instance = new EmbeddedPostgres({ databaseDir: dataDir, - user: "paperclip", - password: "paperclip", + user: EMBEDDED_POSTGRES_USER, + password: EMBEDDED_POSTGRES_PASSWORD, port, persistent: true, - initdbFlags: ["--encoding=UTF8", "--locale=C", "--lc-messages=C"], + initdbFlags: EMBEDDED_POSTGRES_INITDB_FLAGS, onLog: (message) => logBuffer.append(message), onError: (message) => logBuffer.append(message), }); @@ -136,6 +150,14 @@ async function createEmbeddedPostgresTestInstance(tempDirPrefix: string) { return { dataDir, port, instance, getRecentLogs: () => logBuffer.getRecentLogs() }; } +function adminConnectionStringFor(port: number): string { + return `postgres://${EMBEDDED_POSTGRES_USER}:${EMBEDDED_POSTGRES_PASSWORD}@127.0.0.1:${port}/postgres`; +} + +function databaseConnectionStringFor(port: number): string { + return `postgres://${EMBEDDED_POSTGRES_USER}:${EMBEDDED_POSTGRES_PASSWORD}@127.0.0.1:${port}/paperclip`; +} + function cleanupEmbeddedPostgresTestDirs(dataDir: string) { fs.rmSync(dataDir, { recursive: true, force: true }); } @@ -237,13 +259,391 @@ async function startEmbeddedPostgresWithRetry(tempDirPrefix: string): Promise<{ ); } +// --------------------------------------------------------------------------- +// Template cluster cache +// +// Every suite that calls `startEmbeddedPostgresTestDatabase` used to pay for +// `initdb`, a cluster start, and all pending migrations (331 files at the time +// of writing) before its first test ran, and the `getEmbeddedPostgresTestSupport` +// probe started and stopped a second throwaway cluster per test file. On the +// serial CI shards that was 5-6s per suite plus 2.5-3s per probe; measured +// locally it is 3.2s per suite plus 0.75s per probe. +// +// A cluster data directory that was shut down cleanly can be copied and started +// again as an independent cluster. So the first caller on a host builds one +// fully migrated cluster, stops it, and publishes the data directory as a +// template under the OS temp directory. Every later caller copies the template +// (about 90 MB, ~0.2s) and starts Postgres on the copy (~0.05s), then still runs +// `ensurePostgresDatabase` and `applyPendingMigrations` so the returned database +// is exactly what the fresh path produces; both are no-ops on a template copy. +// +// The template key hashes everything that determines the cluster's contents: +// the embedded-postgres package version, the initdb flags, the user, and the +// name and content of every migration file plus the journal. A changed +// migration therefore builds a new template instead of reusing a stale one. +// Publishing is a rename, which is atomic on the same filesystem, so a +// concurrent builder either wins the rename or discards its copy and uses the +// winner's; a template that exists is always complete. +// +// A template is a Postgres data directory, and starting one executes its +// configuration (for example `shared_preload_libraries`). The cache root is +// therefore private to the current OS user: its name carries the user id, it +// is created with mode 0700, and it is only used when it is a real directory +// (not a symlink) owned by this user with no group or world write bit. Each +// template is checked the same way before it is copied. On a host with a +// shared temp directory another user can pre-create the predictable path, and +// those checks reject it. Windows has no uid; its temp directory is already +// per user, so the ownership and mode checks are skipped there. +// +// Set PAPERCLIP_TEST_POSTGRES_TEMPLATE=0 to bypass the cache and run the fresh +// path for every suite. Any failure on the template path (root checks, build, +// copy, start, or publish) falls back to the fresh path, so a broken cache can +// make a suite slower but never fail it or skip it: the fresh path still +// reports a migration that cannot apply or a cluster that cannot start. +// --------------------------------------------------------------------------- + +const TEMPLATE_ROOT_DIR_NAME = "paperclip-embedded-postgres-templates"; +// Templates whose key no longer matches are pruned once they have not been +// used for this long. Every reuse refreshes the template's timestamp. +const TEMPLATE_PRUNE_AGE_MS = 7 * 24 * 60 * 60 * 1000; +// Publishing a template requires the builder cluster to have exited for real, +// so its data files are quiescent before the rename. Give that stop more room +// than the per-suite bounded stop; a template build happens once per host. +const TEMPLATE_BUILD_STOP_TIMEOUT_MS = 30_000; + +export type EmbeddedPostgresStartMode = "template" | "fresh"; + +function isTemplateCacheDisabled(): boolean { + return process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE === "0"; +} + +function migrationsDirectory(): string { + return fileURLToPath(new URL("./migrations/", import.meta.url)); +} + +function embeddedPostgresPackageVersion(): string { + try { + const require = createRequire(import.meta.url); + const manifest = require("embedded-postgres/package.json") as { version?: unknown }; + return typeof manifest.version === "string" ? manifest.version : "unknown"; + } catch { + return "unknown"; + } +} + +// Hash of everything that determines the contents of a migrated cluster. Reads +// every migration file, which costs a few milliseconds per process. +export function computeEmbeddedPostgresTemplateKey( + migrationsDir: string = migrationsDirectory(), + input: { packageVersion?: string; initdbFlags?: string[]; user?: string } = {}, +): string { + const hash = createHash("sha256"); + hash.update(`embedded-postgres=${input.packageVersion ?? embeddedPostgresPackageVersion()}\n`); + hash.update(`initdb=${(input.initdbFlags ?? EMBEDDED_POSTGRES_INITDB_FLAGS).join(" ")}\n`); + hash.update(`user=${input.user ?? EMBEDDED_POSTGRES_USER}\n`); + const journalPath = path.join(migrationsDir, "meta", "_journal.json"); + if (fs.existsSync(journalPath)) { + hash.update(`journal\n`); + hash.update(fs.readFileSync(journalPath)); + hash.update(`\n`); + } + const migrationFiles = fs + .readdirSync(migrationsDir) + .filter((name) => name.endsWith(".sql")) + .sort((a, b) => a.localeCompare(b)); + for (const name of migrationFiles) { + hash.update(`${name}\n`); + hash.update(fs.readFileSync(path.join(migrationsDir, name))); + hash.update(`\n`); + } + return hash.digest("hex").slice(0, 24); +} + +// The uid on POSIX, else the user name. Part of the cache root's name so two +// users on one host never share a root. +function templateOwnerTag(): string { + const uid = process.getuid?.(); + if (uid !== undefined) return String(uid); + let username = "user"; + try { + username = os.userInfo().username; + } catch { + // Fall through to the generic tag. + } + return username.replace(/[^A-Za-z0-9_.-]/g, "_"); +} + +function templateRootDir(): string { + return path.join(os.tmpdir(), `${TEMPLATE_ROOT_DIR_NAME}-${templateOwnerTag()}`); +} + +// Why `dir` cannot be trusted as a private directory of the current user, or +// null when it can. Uses `lstat` so a symlink planted at the path is rejected +// rather than followed. On Windows (no uid) only the directory check applies. +function describePrivateDirectoryProblem(dir: string): string | null { + let stat: fs.Stats; + try { + stat = fs.lstatSync(dir); + } catch (error) { + return `cannot be inspected (${(error as NodeJS.ErrnoException).code ?? "unknown error"})`; + } + if (stat.isSymbolicLink()) return "is a symbolic link"; + if (!stat.isDirectory()) return "is not a directory"; + const uid = process.getuid?.(); + if (uid === undefined) return null; + if (stat.uid !== uid) return `is owned by uid ${stat.uid}, not the current user (uid ${uid})`; + if ((stat.mode & 0o022) !== 0) { + return `is writable by group or others (mode ${(stat.mode & 0o777).toString(8)})`; + } + return null; +} + +// Creates the cache root with mode 0700 and verifies it is private to the +// current user. Returns null, after one warning, when it is not: the cache is +// then unused for this process and every suite takes the fresh path. +function prepareTemplateRootDir(): string | null { + const rootDir = templateRootDir(); + try { + fs.mkdirSync(rootDir, { recursive: true, mode: 0o700 }); + const problem = describePrivateDirectoryProblem(rootDir); + if (problem !== null) { + warnTemplateOnce("root", `template cache root ${rootDir} ${problem}; using fresh clusters`); + return null; + } + // A root created by an earlier version may be wider than 0700. It is + // ours (checked above), so narrow it. + if (process.getuid !== undefined && (fs.lstatSync(rootDir).mode & 0o777) !== 0o700) { + fs.chmodSync(rootDir, 0o700); + } + return rootDir; + } catch (error) { + warnTemplateOnce( + "root", + `template cache root ${rootDir} could not be prepared: ${(error as Error).message}; using fresh clusters`, + ); + return null; + } +} + +// A template is usable when it is a private directory of the current user +// that holds a cleanly stopped cluster: `PG_VERSION` present, no +// `postmaster.pid`. The ownership check repeats the root's check so a +// template is never copied from a directory this user does not own. +function isUsableTemplate(dir: string): boolean { + if (describePrivateDirectoryProblem(dir) !== null) return false; + return fs.existsSync(path.join(dir, "PG_VERSION")) && !fs.existsSync(path.join(dir, "postmaster.pid")); +} + +// Records a use of the template so `pruneStaleTemplates` measures idle time, +// not age. Best effort. +function touchTemplate(dir: string): void { + try { + const now = new Date(); + fs.utimesSync(dir, now, now); + } catch { + // The template still works without the refreshed timestamp. + } +} + +const warnedTemplateTopics = new Set(); + +function warnTemplateOnce(topic: string, message: string): void { + if (warnedTemplateTopics.has(topic)) return; + warnedTemplateTopics.add(topic); + console.warn(`[embedded-postgres] ${message}`); +} + +// Best-effort removal of templates for other keys that have not been used for +// a week, plus abandoned build directories. Every reuse refreshes a template's +// timestamp, so a template that any process resolved in the last week stays. +// A suite never depends on the template after its copy is made, so pruning +// can only affect a process that resolved the path and has not copied yet; +// that process then rebuilds the template instead of failing. Never throws. +function pruneStaleTemplates(rootDir: string, keepDir: string): void { + let entries: fs.Dirent[]; + try { + entries = fs.readdirSync(rootDir, { withFileTypes: true }); + } catch { + return; + } + const now = Date.now(); + for (const entry of entries) { + const entryPath = path.join(rootDir, entry.name); + if (entryPath === keepDir || !entry.isDirectory()) continue; + try { + const age = now - fs.statSync(entryPath).mtimeMs; + if (age > TEMPLATE_PRUNE_AGE_MS) fs.rmSync(entryPath, { recursive: true, force: true }); + } catch { + // Another process may be using or removing it; leave it alone. + } + } +} + +// Waits for the raw `stop()` so the builder cluster has really exited before +// its data directory is published. Returns false if the stop overran. +async function stopForTemplatePublish(instance: EmbeddedPostgresInstance): Promise { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + instance.stop().then(() => true, () => false), + new Promise((resolve) => { + timer = setTimeout(() => resolve(false), TEMPLATE_BUILD_STOP_TIMEOUT_MS); + timer.unref?.(); + }), + ]); + } finally { + if (timer) clearTimeout(timer); + } +} + +// Builds a migrated cluster and publishes its data directory at `templateDir`. +// Throws when the cluster cannot start or a migration cannot apply. Returns +// false when the cluster started but the template could not be published; +// callers then use the fresh path for this process. +async function buildEmbeddedPostgresTemplate(templateDir: string): Promise { + const rootDir = path.dirname(templateDir); + const { port, dataDir, instance } = await startEmbeddedPostgresWithRetry( + "paperclip-embedded-postgres-template-build-", + ); + let published = false; + try { + await ensurePostgresDatabase(adminConnectionStringFor(port), "paperclip"); + await applyPendingMigrations(databaseConnectionStringFor(port)); + const stopped = await stopForTemplatePublish(instance); + if (!stopped || !isUsableTemplate(dataDir)) return false; + try { + fs.renameSync(dataDir, templateDir); + published = true; + } catch (error) { + // Another process published the same key first. Use theirs. + const code = (error as NodeJS.ErrnoException).code; + if (code === "EEXIST" || code === "ENOTEMPTY" || code === "EPERM") { + return isUsableTemplate(templateDir); + } + throw error; + } + pruneStaleTemplates(rootDir, templateDir); + return true; + } finally { + if (!published) { + await stopEmbeddedPostgresBounded(instance, () => cleanupEmbeddedPostgresTestDirs(dataDir)); + } + } +} + +let templateDirPromise: Promise | null = null; + +// Resolves the template data directory for this host, building it on first +// use. Resolves null when the cache is disabled, the cache root is not private +// to this user, or the template could not be built or published; every caller +// then takes the fresh path, which reports the real error itself (a cluster +// that cannot start, a migration that cannot apply). Never rejects, so a +// broken cache can never turn into `supported: false` and skip a suite. +async function getEmbeddedPostgresTemplateDir(): Promise { + if (isTemplateCacheDisabled()) return null; + if (!templateDirPromise) { + templateDirPromise = (async () => { + const rootDir = prepareTemplateRootDir(); + if (rootDir === null) return null; + const templateDir = path.join(rootDir, computeEmbeddedPostgresTemplateKey()); + if (isUsableTemplate(templateDir)) { + touchTemplate(templateDir); + return templateDir; + } + try { + const built = await buildEmbeddedPostgresTemplate(templateDir); + return built && isUsableTemplate(templateDir) ? templateDir : null; + } catch (error) { + warnTemplateOnce( + "build", + `template build failed, using fresh clusters: ${(error as Error).message}`, + ); + return null; + } + })(); + } + return await templateDirPromise; +} + +// Forgets the resolved template so the next caller in this process checks the +// cache again. Used after a template start failed, for example because the +// template was pruned by another process after this one resolved its path. +function resetTemplateDirCache(): void { + templateDirPromise = null; +} + +// Copies the template and starts a cluster on the copy. Each attempt uses a +// fresh copy and a fresh port, mirroring `startEmbeddedPostgresWithRetry`. A +// template that is no longer usable (pruned, or no longer private) is not +// retried: the caller falls back and the next caller rebuilds it. +async function startEmbeddedPostgresFromTemplateWithRetry( + templateDir: string, + tempDirPrefix: string, +): Promise<{ port: number; dataDir: string; instance: EmbeddedPostgresInstance }> { + let lastError = new Error("embedded Postgres startup from template failed"); + + for (let attempt = 1; attempt <= EMBEDDED_POSTGRES_START_MAX_ATTEMPTS; attempt += 1) { + if (!isUsableTemplate(templateDir)) { + throw new Error(`embedded Postgres template ${templateDir} is no longer usable`); + } + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), tempDirPrefix)); + const dataDir = path.join(rootDir, "data"); + let created: Awaited> | null = null; + try { + fs.cpSync(templateDir, dataDir, { recursive: true }); + // Postgres refuses a data directory that is group or world accessible. + fs.chmodSync(dataDir, 0o700); + created = await createEmbeddedPostgresTestInstanceAt(dataDir); + await created.instance.start(); + return { port: created.port, dataDir: rootDir, instance: created.instance }; + } catch (error) { + lastError = formatEmbeddedPostgresError(error, { + fallbackMessage: "embedded Postgres startup from template failed", + recentLogs: created?.getRecentLogs(), + }); + await stopEmbeddedPostgresBounded(created?.instance ?? null, () => + cleanupEmbeddedPostgresTestDirs(rootDir), + ); + } + } + + throw new Error( + `Failed to start embedded PostgreSQL from template after ${EMBEDDED_POSTGRES_START_MAX_ATTEMPTS} attempts: ${lastError.message}`, + ); +} + +let templateFallbackWarned = false; +let lastStartMode: EmbeddedPostgresStartMode | null = null; + // Test-only accessors. Production callers use `startEmbeddedPostgresTestDatabase` // or `getEmbeddedPostgresTestSupport`. A test drives the bounded retry directly // so it does not need a real Postgres connection. export const __startEmbeddedPostgresWithRetryForTests = startEmbeddedPostgresWithRetry; export const __embeddedPostgresStartMaxAttemptsForTests = EMBEDDED_POSTGRES_START_MAX_ATTEMPTS; +export const __embeddedPostgresTemplateRootDirForTests = templateRootDir; +export const __isUsableEmbeddedPostgresTemplateForTests = isUsableTemplate; +export function __lastEmbeddedPostgresStartModeForTests(): EmbeddedPostgresStartMode | null { + return lastStartMode; +} +// Forgets the support probe result, the resolved template, and the one-time +// warnings so a test can drive the probe and the cache again in one process. +export function __resetEmbeddedPostgresTemplateStateForTests(): void { + embeddedPostgresSupportPromise = null; + templateDirPromise = null; + templateFallbackWarned = false; + warnedTemplateTopics.clear(); + lastStartMode = null; +} async function probeEmbeddedPostgresSupport(): Promise { + // A published template proves this host can run the cluster: it was built + // here. Reusing it makes the probe free for every file after the first. + // Building it costs one cluster start, the same as the throwaway probe did. + // When the template cannot be built or used, the throwaway probe below + // decides, so only a cluster that cannot start reports `supported: false`; + // a migration that cannot apply fails the suite loudly on the fresh path. + if ((await getEmbeddedPostgresTemplateDir()) !== null) return { supported: true }; + let started: { dataDir: string; instance: EmbeddedPostgresInstance } | null = null; try { @@ -271,17 +671,48 @@ export async function getEmbeddedPostgresTestSupport(): Promise { + const templateDir = await getEmbeddedPostgresTemplateDir(); + + if (templateDir !== null) { + try { + const started = await startEmbeddedPostgresFromTemplateWithRetry(templateDir, tempDirPrefix); + return { ...started, mode: "template" }; + } catch (error) { + // The next caller in this process re-resolves the template: if it was + // pruned it is rebuilt, instead of retrying a path that is gone. + resetTemplateDirCache(); + if (!templateFallbackWarned) { + templateFallbackWarned = true; + console.warn( + `[embedded-postgres] template start failed, using a fresh cluster: ${(error as Error).message}`, + ); + } + } + } + + // The bounded retry hardens the cluster start against the port race. It throws + // with the real Postgres output if every attempt fails. + const started = await startEmbeddedPostgresWithRetry(tempDirPrefix); + return { ...started, mode: "fresh" }; +} + export async function startEmbeddedPostgresTestDatabase( tempDirPrefix: string, ): Promise { - // The bounded retry hardens the cluster start against the port race. It throws - // with the real Postgres output if every attempt fails. - const { port, dataDir, instance } = await startEmbeddedPostgresWithRetry(tempDirPrefix); + const { port, dataDir, instance, mode } = await startEmbeddedPostgresClusterForSuite(tempDirPrefix); + lastStartMode = mode; try { - const adminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${port}/postgres`; - await ensurePostgresDatabase(adminConnectionString, "paperclip"); - const connectionString = `postgres://paperclip:paperclip@127.0.0.1:${port}/paperclip`; + // Both calls are no-ops on a template copy and keep the returned database + // identical to the fresh path even if the template were ever stale. + await ensurePostgresDatabase(adminConnectionStringFor(port), "paperclip"); + const connectionString = databaseConnectionStringFor(port); await applyPendingMigrations(connectionString); return { From 026bdcd046425f0650c6cc8ef1355b0bce750be0 Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Sun, 11 Oct 2026 11:19:44 -0700 Subject: [PATCH 05/18] perf(ci): run the verifier's bundled-dependency installs concurrently (#15868) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Every pull request runs the `Build` check. Its last step is the public npm install verifier, `scripts/verify-grok-npm-install.mjs`, which stages the eighteen public packages and installs them in a clean consumer. > - The verifier stages the packages in one serial loop. Three packages bundle dependencies (`@paperclipai/server`, `@paperclipai/adapter-utils`, `@paperclipai/db`). Each one runs a host-side `npm install` of its bundled closure in-process, one after the other, and nothing in the loop overlaps anything else. The staging phase took 102-116s on recent `Build` jobs. > - `Build` at 433-530s is the slowest check in most PR runs once the server shards drop, so its staging phase sets the wall-clock time of the run. > - The three installs write only their own directories and read nothing from each other. There is no reason to run them one after the other. > - This pull request starts the three installs together as child processes and copies the other fifteen packages while they run. Packing stays serial and in order. > - The benefit is a shorter `Build` job, and a timing line that reports each install, so the saving is measured on this PR's own run instead of estimated. ## Linked Issues or Issue Description Refs #15861. This branch is stacked on the branch of #15861 because the same execution branch holds that open PR. The diff here is this PR's three commits. After #15861 merges, this PR is retargeted to `master` and rebased. **What existing behavior does this improve?** The `Build` PR check. Its public npm install verifier stages the eighteen public packages in one serial loop. **Subsystem affected** CI and release scripts (`scripts/verify-grok-npm-install.mjs`, `scripts/prepare-bundled-package.mjs`, `scripts/prefetch-child.mjs`, `scripts/prefetch-supervisor.mjs`). **Current behavior** The staging loop calls `prepareBundledPackage` in-process for the three packages that bundle dependencies. Each call runs a host-side `npm install` of that package's bundled closure, applies the repo patches, and rewrites the manifest, before the loop moves to the next package. The timing line reports the whole staging phase as one number, 102-116s on recent runs. The three installs are the only network work in that phase. **Proposed behavior** Stage the three bundled packages' sources first, start their installs at the same time as children of the script's own CLI entry (the same path `release.sh` uses), copy the other fifteen packages while the installs run, join the installs, and then pack every package in dependency order as before. Report `stage_sources_ms`, `stage_bundled_ms`, `stage_pack_ms`, `bundled_wait_ms` and one `stage_bundled__ms` per install on the timing line. **Reason and benefit** The staging phase shortens by the two shorter installs. Measured on this PR's first run (below): the three installs sum to 25.9s and take 16.4s of wall time together, so the overlap removes about 9.5s from the phase, and the timing line now shows that packing (35.8s) and the standalone-package build before the loop (about 27s) are the larger remaining costs. The original estimate of 30-50s was too high because it assumed the installs were about half of the phase; they are a quarter. **Breaking changes** None. The packed tarballs are produced by the same script, in the same order, and are checked the same way. ## What Changed - `scripts/stage-bundled-packages.mjs` (new): `startBundledInstalls` starts one child per bundled package through the prefetch helper from #15849, with the child's output in a log file. `bundledInstallTimings` reads each child's wall time for the timing line. `bundledPackageSlug` maps a package name to its log and timing key. - `scripts/verify-grok-npm-install.mjs`: the staging loop is split. The bundled packages' sources are staged first and their installs start together. The other packages are copied while the installs run. The install handles are pushed into the `prefetches` list at once, so an earlier failure stops any install still running before the scratch root is removed. The join comes before the first `npm pack`. The pack loop walks every package in `needed` order and checks each tarball as before. `prepareBundledPackage` is no longer imported. - Timing line: `stage_sources_ms`, `stage_bundled_ms`, `stage_pack_ms`, `bundled_wait_ms` and `stage_bundled_server_ms`, `stage_bundled_adapter_utils_ms`, `stage_bundled_db_ms` are added. `stage_packages_ms` and the download keys are unchanged, so the before and after numbers compare directly. - `scripts/__tests__/stage-bundled-packages.test.mjs` (new): five tests with local Node children and no network. The slug mapping; the children run the CLI entry `release.sh` runs and never an in-process call; two installs run at the same time, each with its own source and target, and their logs and timing keys are correct (each child exits only after the other has started, so serial children would fail); a failed install rejects with its own log tail while the other is still running and is then stopped; and stopping the installs after one fails also stops the stand-in npm process the other wrapper is blocked on. - `scripts/__tests__/grok-public-install-sandbox.test.mjs`: one guard test on the verifier source. The bundled sources are staged before the installs start, the installs are registered for cleanup before the other packages are copied, the join comes before the pack loop, the pack loop is in order, there is still exactly one package pack loop, and the timing keys are reported. - `package.json`: the new test file is in `test:release-registry`. ### Review fix (second commit): stop a prefetch's whole process tree Greptile pointed out that `stopPrefetches` signalled only the Node wrapper of each install, while the wrapper is blocked in `execFileSync('npm', ['install', ...])`. After one install failed, the other wrapper died and its npm process kept writing into the scratch root the verifier was about to remove. - `scripts/prefetch-child.mjs`: on POSIX every prefetch child is started `detached`, so it leads its own process group. `stopPrefetches` signals the group (`process.kill(-pid, ...)`) and waits until no member of the group exists, with `SIGKILL` after the grace period and a second grace period before it gives up so cleanup cannot hang. On Windows the child is signalled directly as before. - `scripts/__tests__/prefetch-child.test.mjs`: two tests. A stopped prefetch's grandchild is gone after `stopPrefetches` returns, and a `SIGINT` to a separate parent process reaches the grandchild while the parent still exits with `SIGINT`. Both are skipped on Windows. - `scripts/__tests__/stage-bundled-packages.test.mjs`: one test, the cancellation case Greptile asked for. The stand-in `prepare-bundled-package` blocks in `execFileSync` on a stand-in npm that announces its pid; after the other install fails, `stopPrefetches` must leave that pid gone. All three new tests fail against the previous helper (`grandchild outlived stopPrefetches`, `outlived the parent's SIGINT`, `stand-in npm outlived stopPrefetches`) and pass with the fix. ### Review fix (third commit): the parent runs no signal handler Greptile pointed out that the second commit forwarded the terminal's signals to the detached groups through a JavaScript handler in the parent. The verifier spends most of its time blocked in `execFileSync` (the standalone builds, every `npm pack`), where no handler can run, and installing the handler also replaced the parent's immediate default termination. So a Ctrl-C during a synchronous phase did nothing until that command returned, and the detached installs and downloads kept running meanwhile. Confirmed with the new test against the second commit's helper: the parent survived the `SIGINT`, exited with code 1 once the synchronous command had died, and its handler never ran. - `scripts/prefetch-supervisor.mjs` (new): every prefetch command now runs under this small Node script as the leader of its own process group. The supervisor holds a pipe from the parent on stdin (which the parent writes to only to request a stop where it cannot signal the group; see the fourth commit). The kernel closes it when the parent exits for any reason, including a signal delivered while the parent is blocked, or `SIGKILL`, and the supervisor then stops its group (`SIGTERM`, `SIGKILL` after the grace period) and says so in the log. It passes group-wide signals on to the command and stays alive until the command exits, so its own exit reports the command's exit code or signal. Run by hand with a terminal or `/dev/null` on stdin it watches nothing. - `scripts/prefetch-child.mjs`: `startPrefetch` spawns the supervisor with the command, with stdin as the pipe. The forwarding handlers are gone; the parent installs no signal handler, so its default termination stays immediate. `stopPrefetches` is unchanged. - `scripts/__tests__/prefetch-child.test.mjs`: the first test asserts the parent has no `SIGINT`/`SIGTERM`/`SIGHUP` listener after a prefetch starts. The "signal to the parent" test now runs for `SIGINT` and `SIGKILL` and checks the supervisor's log line. One new test gives a separate parent its own process group, blocks it in `execFileSync` on a 20s command, and sends `SIGINT` to that group the way a terminal sends Ctrl-C: the parent must die of `SIGINT` within two seconds and the prefetch's grandchild must be gone. Against the previous helper the parent exits with code 1 instead, the grandchild outlives `SIGKILL`, and the handler-count assertion fails. ### Review fix (fourth commit): a deliberate stop on Windows goes through the pipe Greptile pointed out that on Windows, where `startPrefetch` now returns the supervisor, `stopPrefetches` called `child.kill('SIGTERM')` on it. Windows has no process groups and `kill` there ends a process outright, so the supervisor died without running its handler and its command kept running after cleanup returned. - `scripts/prefetch-child.mjs`: without process groups, the graceful stop no longer signals the supervisor. `stopPrefetches` writes a stop request to the pipe the supervisor already watches for the parent's exit (`stdin.end('stop\n')`). Only the `SIGKILL` escalation targets the supervisor itself, after it has had its grace period. POSIX keeps the group signal, so `stopPrefetches` and the existing stop and failure tests are unchanged there. - `scripts/prefetch-supervisor.mjs`: data on stdin is a stop request and EOF is the parent's exit; both end the group the same way (`SIGTERM`, `SIGKILL` after the grace period) and the log line names which one it was. - `scripts/__tests__/prefetch-child.test.mjs`: one new test writes the stop request to a running prefetch's pipe instead of calling `stopPrefetches`. The supervisor must exit with `SIGTERM`, the command's grandchild must be gone, and the log must say `stop requested` and not `parent exited`. The previous supervisor fails it on the log line, since it treated the request as the parent's exit. The Windows kill itself cannot be reproduced on the Linux runners, so the Windows branch of `signalTree` is verified by that supervisor test plus reading its two lines. The stand-in that ignores `SIGTERM` is now skipped on Windows, where `kill` cannot be ignored. ## Verification - `node --test scripts/__tests__/stage-bundled-packages.test.mjs scripts/__tests__/grok-public-install-sandbox.test.mjs scripts/__tests__/prefetch-child.test.mjs`: 30 tests pass (was 19 before this PR, 24 after the first commit, 27 after the second, 29 after the third). The three files pass three consecutive runs. - `node --check` passes on all four scripts. - The verifier itself needs Docker and a network, which this workspace does not have. This PR's own `Build` job runs the branch's verifier. Measured on all four commits' runs ([38132868803](https://github.com/paperclipai/paperclip/actions/runs/38132868803): `Build` 428s, `pnpm build` 112s, verifier step 214s; [38136912842](https://github.com/paperclipai/paperclip/actions/runs/38136912842): `Build` 447s; [38139871529](https://github.com/paperclipai/paperclip/actions/runs/38139871529): `Build` 432s; [38142762538](https://github.com/paperclipai/paperclip/actions/runs/38142762538): `Build` 457s, `pnpm build` 123s, verifier step 231s): | Key | Run 1 | Run 2 | Run 3 | Run 4 | Before | | --- | --- | --- | --- | --- | --- | | `stage_packages_ms` | 80.4s | 84.9s | 81.5s | 89.6s | 102-116s on master runs 38112764439, 38111191543, 38110932348, 38110914319, 38110897745, 38127611057; 84.5s on #15849's run on a runner as fast as these | | `stage_sources_ms` | 1.4s | 0.7s | 0.9s | 0.9s | (inside the number above) | | `stage_bundled_server_ms` / `adapter_utils` / `db` | 16.3s / 5.0s / 4.6s | 14.9s / 3.6s / 3.3s | 16.4s / 5.2s / 4.7s | 17.1s / 4.4s / 3.8s | ran one after another, 21.8-25.9s together | | `stage_bundled_ms` (wall, all three) | 16.4s | 15.0s | 16.5s | 17.2s | 21.8-25.9s | | `bundled_wait_ms` | 15.3s | 14.7s | 16.1s | 16.7s | — | | `stage_pack_ms` | 35.8s | 37.1s | 35.6s | 37.9s | (inside the number above) | | `consumer_install_ms` | 94.4s | 97.8s | 95.7s | 99.8s | 97-134s | | `image_prefetch_ms` / `image_wait_ms` | 33.7s / 0ms | 31.8s / 0ms | 34.7s / 0ms | 33.7s / 0ms | unchanged | | `grok_prefetch_ms` / `grok_wait_ms` | 2.1s / 0ms | 2.1s / 0ms | 2.0s / 0ms | 2.1s / 0ms | unchanged | The overlap removed about 7-10s from the staging phase on each of the four runs (21.8-25.9s of installs in 15.0-17.2s of wall time). The third and fourth commits changed no timing: both waits stay at 0ms and the staging phase sits inside the band of the first two runs. The rest of the phase is the standalone-package build and the server UI dist prep before the loop (about 27-32s) and the eighteen serial `npm pack` plus `tar -tzf` checks (36-37s). - `docker-runner-check.yml` runs the same script after `pnpm build` and needs no change. ## Risks - Three `npm install` processes share the host npm cache at the same time. npm's cache (`cacache`) is designed for concurrent writers, and the three installs have disjoint target directories. If a shared-cache race appears, the failing install's log tail is in the error and the fix is to serialize again. The first CI run showed none. - The install output moves from the CI log into per-install log files in the scratch root. A failure reports the last 2000 characters of the failing install's log; a successful install's output is discarded with the scratch root. - Memory: three concurrent installs on the `Build` runner instead of one. The closures are small (the server's eight bundled packages are the largest), so this stays well under the runner's memory. - Process groups: a detached child no longer receives the terminal's signals on its own. The supervisor stops its group when the parent is gone, which it learns from the kernel closing the stdin pipe, so this holds for a Ctrl-C while the parent is blocked in synchronous work and for `SIGKILL` alike. The parent keeps Node's default signal handling. If the supervisor itself is killed with `SIGKILL` before its command, the command is orphaned and finishes on its own (each install ends within seconds; the image pull within a minute), the same as any orphaned grandchild before this PR. - One extra Node process per prefetch (five per verifier run). Each costs about 40ms to start, inside the time the parent spends staging anyway. - Low risk otherwise. Packing, the tarball checks, the consumer install, and every probe are unchanged. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - Claude Fable 5.1 (Anthropic), model ID `claude-fable-5-1`, run as a Paperclip agent through the Claude Code local adapter with tool use (shell, file edits, GitHub CLI). Reasoning mode: extended thinking. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green on all four commits - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups (on the fourth commit Greptile reviewed 8 files and added no comments; all three earlier threads are resolved) - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Bender (Fable) Co-authored-by: Claude Fable 5.1 Co-authored-by: devinfoley <139239+devinfoley@users.noreply.github.com> --- package.json | 2 +- .../grok-public-install-sandbox.test.mjs | 25 +++++ scripts/__tests__/prefetch-child.test.mjs | 97 +++++++++++++++- .../__tests__/stage-bundled-packages.test.mjs | 104 ++++++++++++++++++ scripts/prefetch-child.mjs | 78 +++++++++++-- scripts/prefetch-supervisor.mjs | 78 +++++++++++++ scripts/stage-bundled-packages.mjs | 32 ++++++ scripts/verify-grok-npm-install.mjs | 58 +++++++--- 8 files changed, 448 insertions(+), 26 deletions(-) create mode 100644 scripts/__tests__/stage-bundled-packages.test.mjs create mode 100644 scripts/prefetch-supervisor.mjs create mode 100644 scripts/stage-bundled-packages.mjs diff --git a/package.json b/package.json index 5a5000234b9..29b999b28e2 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,7 @@ "smoke:posthog-live": "node scripts/smoke/posthog-live.mjs", "smoke:pipelines-tutorial": "./scripts/smoke/pipelines-tutorial-smoke.sh", "smoke:terminal-bench-loop-skill": "node scripts/smoke/terminal-bench-loop-skill-smoke.mjs", - "test:release-registry": "node --test scripts/__tests__/canary-startup-playwright.test.mjs scripts/__tests__/cursor-public-install-sandbox.test.mjs scripts/__tests__/grok-public-install-sandbox.test.mjs scripts/__tests__/prefetch-child.test.mjs scripts/__tests__/retain-runner-qualification-packages.test.mjs scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs scripts/cloud-migrator-artifacts.test.mjs", + "test:release-registry": "node --test scripts/__tests__/canary-startup-playwright.test.mjs scripts/__tests__/cursor-public-install-sandbox.test.mjs scripts/__tests__/grok-public-install-sandbox.test.mjs scripts/__tests__/prefetch-child.test.mjs scripts/__tests__/stage-bundled-packages.test.mjs scripts/__tests__/retain-runner-qualification-packages.test.mjs scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs scripts/cloud-migrator-artifacts.test.mjs", "storybook-visual:baseline": "node scripts/storybook-visual-baseline.mjs", "test:storybook-visual": "node scripts/storybook-visual-baseline.mjs download && node scripts/storybook-visual-baseline.mjs verify && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts", "test:storybook-visual:update": "node scripts/storybook-visual-baseline.mjs download && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts --update-snapshots && node scripts/storybook-visual-baseline.mjs pack", diff --git a/scripts/__tests__/grok-public-install-sandbox.test.mjs b/scripts/__tests__/grok-public-install-sandbox.test.mjs index 5a3dc487e38..11fde52433e 100644 --- a/scripts/__tests__/grok-public-install-sandbox.test.mjs +++ b/scripts/__tests__/grok-public-install-sandbox.test.mjs @@ -110,6 +110,31 @@ test('the image and Grok downloads start before staging and join without weakeni assert.equal(source.match(/rmSync\(root/g).length, 1); }); +test('the bundled-dependency installs run together while the other packages are copied, and pack in order', () => { + const source = readFileSync(new URL('../verify-grok-npm-install.mjs', import.meta.url), 'utf8'); + const position = fragment => { const index = source.indexOf(fragment); assert.notEqual(index, -1, `verifier must contain ${fragment}`); return index; }; + // The installs are children of the script's own CLI entry through the helper + // covered by stage-bundled-packages.test.mjs, never in-process calls. + assert.doesNotMatch(source, /prepareBundledPackage/); + assert.ok(source.includes("import { bundledInstallTimings, startBundledInstalls } from './stage-bundled-packages.mjs'")); + // Only the bundled packages' sources are staged before the installs start; + // the installs are registered for cleanup at once, before anything else can + // fail; the other packages are copied while the installs run; the join comes + // before the first pack; and the pack loop walks every package in order. + const bundledSources = position('for (const entry of entries.filter(entry => entry.bundled)) stageSource(entry)'); + const start = position("startBundledInstalls(entries.filter(entry => entry.bundled), { logDir: prefetched, cwd: repo })"); + const registered = position('prefetches.push(...installs.map(({ handle }) => handle))'); + const otherSources = position('for (const entry of entries.filter(entry => !entry.bundled))'); + const join = position("await awaited('bundled', Promise.all(installs.map(({ handle }) => handle.done)))"); + const packLoop = position('for (const { target } of entries)'); + const firstPack = position("run('npm', ['pack', '--ignore-scripts', '--pack-destination', root], target)"); + assert.ok(bundledSources < start && start < registered && registered < otherSources && otherSources < join && join < packLoop && packLoop < firstPack); + assert.equal(source.match(/run\('npm', \['pack'/g).length, 2, 'one package pack loop plus the lifecycle sentinel'); + // The timing line reports the split staging phase and each install. + for (const key of ['stage_sources_ms', 'stage_bundled_ms', 'stage_pack_ms']) assert.ok(source.includes(`timing.${key} =`)); + assert.ok(source.includes('...bundledInstallTimings(installs)')); +}); + test('the installed Codex probe exercises the public export and rejects incomplete or mismatched packages', () => { const root = realpathSync(mkdtempSync(join(tmpdir(), 'installed-codex-probe-'))); try { diff --git a/scripts/__tests__/prefetch-child.test.mjs b/scripts/__tests__/prefetch-child.test.mjs index bd3d9864bef..c508ef0290c 100644 --- a/scripts/__tests__/prefetch-child.test.mjs +++ b/scripts/__tests__/prefetch-child.test.mjs @@ -1,5 +1,5 @@ import assert from 'node:assert/strict'; -import { spawnSync } from 'node:child_process'; +import { spawn, spawnSync } from 'node:child_process'; import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -25,6 +25,8 @@ const untilLogHas = async (path, text) => { test('a successful prefetch resolves, keeps its output, and is left alone by stop', async () => withScratch(async root => { const handle = start(root, 'ok', "setTimeout(() => process.stdout.write('fetched\\n'), 150)"); + // The parent keeps its default, immediate termination: no handler of ours. + for (const signal of ['SIGINT', 'SIGTERM', 'SIGHUP']) assert.equal(process.listenerCount(signal), 0, `${signal} handler installed in the parent`); await handle.done; assert.equal(handle.child.exitCode, 0); assert.equal(readFileSync(join(root, 'ok.log'), 'utf8'), 'fetched\n'); @@ -66,7 +68,7 @@ test('stopping a running prefetch waits for it to exit so its directory can be r await assert.rejects(handle.done, /slow prefetch failed \(SIGTERM\): started/); })); -test('a prefetch that ignores SIGTERM is killed after the grace period', async () => withScratch(async root => { +test('a prefetch that ignores SIGTERM is killed after the grace period', { skip: process.platform === 'win32' && 'SIGTERM cannot be ignored on Windows' }, async () => withScratch(async root => { const handle = start(root, 'stubborn', "process.on('SIGTERM', () => {}); process.stdout.write('ignoring\\n'); setInterval(() => {}, 1000)"); await untilLogHas(join(root, 'stubborn.log'), 'ignoring'); const stopping = Date.now(); @@ -75,6 +77,97 @@ test('a prefetch that ignores SIGTERM is killed after the grace period', async ( assert.equal(handle.child.signalCode, 'SIGKILL'); })); +// A process is gone once the kernel no longer knows it, or once it is a zombie +// that only waits to be reaped: either way it can no longer write anywhere. +const gone = pid => { + try { process.kill(pid, 0); } catch (error) { return error.code === 'ESRCH'; } + try { return /^\d+ \(.*\) Z /.test(readFileSync(`/proc/${pid}/stat`, 'utf8')); } catch { return false; } +}; +const untilGone = async pid => { + for (let attempt = 0; attempt < 200; attempt++) { + if (gone(pid)) return true; + await new Promise(resolve => setTimeout(resolve, 25)); + } + return false; +}; +// The child starts a process of its own that announces its pid and then runs +// on, the way the bundled-dependency installs block in `npm install`. +const GRANDCHILD_SCRIPT = ` + const { spawn } = require('node:child_process'); + const grandchild = spawn(process.execPath, ['-e', "process.stdout.write('grandchild ' + process.pid + '\\\\n'); setInterval(() => {}, 1000)"], { stdio: 'inherit' }); + grandchild.on('exit', () => process.exit(0)); +`; +const grandchildPid = async log => { await untilLogHas(log, 'grandchild '); return Number(readFileSync(log, 'utf8').match(/grandchild (\d+)/)[1]); }; + +test('stopping a prefetch also stops the processes it started', { skip: process.platform === 'win32' && 'process groups are POSIX only' }, async () => withScratch(async root => { + const handle = start(root, 'tree', GRANDCHILD_SCRIPT); + const pid = await grandchildPid(join(root, 'tree.log')); + assert.ok(!gone(pid), 'the grandchild runs before the stop'); + await stopPrefetches([handle]); + assert.ok(await untilGone(pid), `grandchild ${pid} outlived stopPrefetches`); + assert.notEqual(handle.child.signalCode ?? handle.child.exitCode, null); +})); + +test('a stop request written to the pipe ends the command and the processes it started, with no signal to the supervisor', { skip: process.platform === 'win32' && 'the grandchild check reads /proc' }, async () => withScratch(async root => { + // This is the stop path on a platform without process groups: there `kill` + // would end the supervisor outright and leave its command running, so + // `stopPrefetches` writes a request instead. The supervisor answers it + // the way it answers the parent's exit, and its own exit still reports the + // command's fate. + const handle = start(root, 'tree', GRANDCHILD_SCRIPT); + const pid = await grandchildPid(join(root, 'tree.log')); + handle.child.stdin.end('stop\n'); + await assert.rejects(handle.done, /tree prefetch failed \(SIGTERM\)/); + assert.ok(await untilGone(pid), `grandchild ${pid} outlived the stop request`); + assert.match(readFileSync(join(root, 'tree.log'), 'utf8'), /stop requested; stopping/); + assert.doesNotMatch(readFileSync(join(root, 'tree.log'), 'utf8'), /parent exited/); + await stopPrefetches([handle]); +})); + +// A separate parent process stands in for the verifier: it starts a prefetch +// whose child starts a grandchild, then does `work` (a module-scope statement). +const parentScript = (root, work) => ` + import { execFileSync } from 'node:child_process'; + import { startPrefetch } from ${JSON.stringify(new URL('../prefetch-child.mjs', import.meta.url).href)}; + startPrefetch({ name: 'tree', cmd: process.execPath, args: ['-e', ${JSON.stringify(GRANDCHILD_SCRIPT)}], cwd: ${JSON.stringify(root)}, env: process.env, logPath: ${JSON.stringify(join(root, 'tree.log'))} }); + ${work} +`; + +for (const signal of ['SIGINT', 'SIGKILL']) { + test(`${signal} to the parent ends the processes a prefetch started, and the parent dies of it at once`, { skip: process.platform === 'win32' && 'process groups are POSIX only' }, async () => withScratch(async root => { + // The parent runs no handler, so it dies of the signal immediately, and the + // prefetch tree stops because the parent is gone, not because it was told: + // SIGKILL leaves the parent no chance to tell anyone anything. + const parent = spawn(process.execPath, ['--input-type=module', '-e', parentScript(root, 'setInterval(() => {}, 1000);')], { cwd: root, stdio: 'ignore' }); + const exited = new Promise(resolve => parent.once('exit', (code, signal) => resolve({ code, signal }))); + const pid = await grandchildPid(join(root, 'tree.log')); + parent.kill(signal); + assert.deepEqual(await exited, { code: null, signal }); + assert.ok(await untilGone(pid), `grandchild ${pid} outlived the parent's ${signal}`); + assert.match(readFileSync(join(root, 'tree.log'), 'utf8'), /parent exited; stopping/); + })); +} + +test('Ctrl-C while the parent is blocked in synchronous work stops it at once, and the prefetch tree with it', { skip: process.platform === 'win32' && 'process groups are POSIX only' }, async () => withScratch(async root => { + // The verifier runs `execFileSync` for the standalone builds while prefetches + // run, and a JavaScript signal handler cannot run until that returns. The + // parent gets its own group here so the signal can be sent the way a + // terminal sends Ctrl-C: to the foreground group, which is the parent and + // its synchronous command, and not the detached prefetch tree. + const parent = spawn(process.execPath, ['--input-type=module', '-e', parentScript(root, "execFileSync(process.execPath, ['-e', 'setTimeout(() => {}, 20000)'], { stdio: 'ignore' });")], { cwd: root, stdio: 'ignore', detached: true }); + try { + const exited = new Promise(resolve => parent.once('exit', (code, signal) => resolve({ code, signal }))); + const pid = await grandchildPid(join(root, 'tree.log')); + const signalled = Date.now(); + process.kill(-parent.pid, 'SIGINT'); + assert.deepEqual(await exited, { code: null, signal: 'SIGINT' }); + assert.ok(Date.now() - signalled < 2000, `the parent took ${Date.now() - signalled}ms to die: it waited for the synchronous command`); + assert.ok(await untilGone(pid), `grandchild ${pid} outlived the parent's Ctrl-C`); + } finally { + try { process.kill(-parent.pid, 'SIGKILL'); } catch {} + } +})); + test('an unawaited failure never becomes an unhandled rejection', () => { const script = ` import { startPrefetch } from ${JSON.stringify(new URL('../prefetch-child.mjs', import.meta.url).href)}; diff --git a/scripts/__tests__/stage-bundled-packages.test.mjs b/scripts/__tests__/stage-bundled-packages.test.mjs new file mode 100644 index 00000000000..d42ddf14cf7 --- /dev/null +++ b/scripts/__tests__/stage-bundled-packages.test.mjs @@ -0,0 +1,104 @@ +import assert from 'node:assert/strict'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { stopPrefetches } from '../prefetch-child.mjs'; +import { PREPARE_BUNDLED_PACKAGE_SCRIPT, bundledInstallTimings, bundledPackageSlug, startBundledInstalls } from '../stage-bundled-packages.mjs'; + +// A small local script stands in for `prepare-bundled-package.mjs`; no test +// here needs npm or a network. +const withScratch = async body => { + const root = mkdtempSync(join(tmpdir(), 'stage-bundled-packages-')); + try { await body(root); } finally { rmSync(root, { recursive: true, force: true }); } +}; +const entriesFor = (root, names) => names.map((name, index) => ({ name, stagedSource: join(root, `source-${index}`), target: join(root, `package-${index}`) })); +const fakeScript = (root, body) => { const script = join(root, 'fake-prepare.mjs'); writeFileSync(script, body); return script; }; + +test('package names map to stable log and timing slugs', () => { + assert.equal(bundledPackageSlug('@paperclipai/server'), 'server'); + assert.equal(bundledPackageSlug('@paperclipai/adapter-utils'), 'adapter_utils'); + assert.equal(bundledPackageSlug('@paperclipai/db'), 'db'); +}); + +test('the children run the CLI entry release.sh uses, never an in-process call', () => { + assert.ok(existsSync(PREPARE_BUNDLED_PACKAGE_SCRIPT)); + const source = readFileSync(PREPARE_BUNDLED_PACKAGE_SCRIPT, 'utf8'); + assert.ok(source.includes('if (process.argv[1] === fileURLToPath(import.meta.url))')); + assert.ok(source.includes('prepareBundledPackage(resolve(sourceDir), resolve(destinationDir))')); + const release = readFileSync(new URL('../release.sh', import.meta.url), 'utf8'); + assert.match(release, /node "\$REPO_ROOT\/scripts\/prepare-bundled-package\.mjs" "\$REPO_ROOT\/\$pkg_dir" "\$publish_dir"/); +}); + +test('the installs run at the same time, each with its own source and target', async () => withScratch(async root => { + // Each child records its arguments, announces itself, and exits only once + // the other child has announced itself too. Serial children would deadlock + // here and the five-second guard would turn that into a failure. + const script = fakeScript(root, ` + import { existsSync, writeFileSync } from 'node:fs'; + import { basename, join } from 'node:path'; + const [source, target] = process.argv.slice(2); + const root = join(target, '..'); + writeFileSync(join(root, 'started-' + basename(target)), ''); + console.log('install ' + source + ' -> ' + target); + const other = basename(target) === 'package-0' ? 'package-1' : 'package-0'; + const deadline = Date.now() + 5000; + while (!existsSync(join(root, 'started-' + other))) { + if (Date.now() > deadline) { console.error('the other install never started'); process.exit(3); } + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 20); + } + writeFileSync(join(target, 'package.json'), '{}'); + `); + const entries = entriesFor(root, ['@paperclipai/server', '@paperclipai/db']); + for (const { target } of entries) mkdirSync(target); + const installs = startBundledInstalls(entries, { logDir: root, cwd: root, script }); + assert.deepEqual(installs.map(({ slug }) => slug), ['server', 'db']); + await Promise.all(installs.map(({ handle }) => handle.done)); + for (const [index, { stagedSource, target, slug, handle }] of installs.entries()) { + assert.equal(handle.child.exitCode, 0); + assert.equal(readFileSync(join(root, `bundled-${slug}.log`), 'utf8'), `install ${stagedSource} -> ${target}\n`); + assert.equal(target, join(root, `package-${index}`)); + assert.ok(existsSync(join(target, 'package.json'))); + } + const timings = bundledInstallTimings(installs); + assert.deepEqual(Object.keys(timings), ['stage_bundled_server_ms', 'stage_bundled_db_ms']); + for (const value of Object.values(timings)) assert.ok(Number.isInteger(value) && value >= 0); +})); + +test('stopping the installs after one fails also stops the npm process the other started', { skip: process.platform === 'win32' && 'process groups are POSIX only' }, async () => withScratch(async root => { + // The real wrapper blocks in `execFileSync('npm', ['install', ...])`; here + // the stand-in npm announces its pid and runs on, and the wrapper blocks on + // it the same way. Stopping only the wrapper would leave it writing. + const script = fakeScript(root, ` + import { execFileSync } from 'node:child_process'; + const [, target] = process.argv.slice(2); + if (target.endsWith('package-1')) { console.error('bundled install exploded'); process.exit(2); } + execFileSync(process.execPath, ['-e', "process.stdout.write('npm ' + process.pid + '\\\\n'); setInterval(() => {}, 1000)"], { stdio: 'inherit' }); + `); + const installs = startBundledInstalls(entriesFor(root, ['@paperclipai/server', '@paperclipai/db']), { logDir: root, cwd: root, script }); + await assert.rejects(Promise.all(installs.map(({ handle }) => handle.done)), /bundled-db prefetch failed \(exit 2\)/); + const log = join(root, 'bundled-server.log'); + const logHasPid = () => existsSync(log) && /npm \d+/.test(readFileSync(log, 'utf8')); + for (let attempt = 0; attempt < 200 && !logHasPid(); attempt++) await new Promise(resolve => setTimeout(resolve, 25)); + const npmPid = Number(readFileSync(log, 'utf8').match(/npm (\d+)/)[1]); + const alive = pid => { try { process.kill(pid, 0); return !/^\d+ \(.*\) Z /.test(readFileSync(`/proc/${pid}/stat`, 'utf8')); } catch (error) { return error.code !== 'ESRCH'; } }; + assert.ok(alive(npmPid), 'the stand-in npm runs before the stop'); + await stopPrefetches(installs.map(({ handle }) => handle)); + let stopped = false; + for (let attempt = 0; attempt < 200 && !(stopped = !alive(npmPid)); attempt++) await new Promise(resolve => setTimeout(resolve, 25)); + assert.ok(stopped, `stand-in npm ${npmPid} outlived stopPrefetches`); + assert.notEqual(installs[0].handle.child.signalCode ?? installs[0].handle.child.exitCode, null); +})); + +test('a failed install rejects with the tail of its own log and leaves the others to be stopped', async () => withScratch(async root => { + const script = fakeScript(root, ` + const [, target] = process.argv.slice(2); + if (target.endsWith('package-1')) { console.error('bundled install exploded'); process.exit(2); } + setTimeout(() => {}, 30_000); + `); + const installs = startBundledInstalls(entriesFor(root, ['@paperclipai/server', '@paperclipai/adapter-utils']), { logDir: root, cwd: root, script }); + await assert.rejects(Promise.all(installs.map(({ handle }) => handle.done)), /bundled-adapter_utils prefetch failed \(exit 2\): bundled install exploded/); + assert.equal(installs[0].handle.child.exitCode, null, 'the other install is still running when the failure surfaces'); + await stopPrefetches(installs.map(({ handle }) => handle)); + assert.notEqual(installs[0].handle.child.signalCode ?? installs[0].handle.child.exitCode, null); +})); diff --git a/scripts/prefetch-child.mjs b/scripts/prefetch-child.mjs index bdaf3d25a84..926bd3c2571 100644 --- a/scripts/prefetch-child.mjs +++ b/scripts/prefetch-child.mjs @@ -2,23 +2,72 @@ // output in a log file, join it where the result is needed, and stop it before // the scratch directory it writes into is removed. No shell, no inherited // terminal, and a failure is reported where the caller awaits it. +// +// A child may start processes of its own: the bundled-dependency installs are +// a Node wrapper blocked in `npm install`. Signalling only the wrapper would +// leave npm writing into the scratch root while it is removed, so on POSIX +// each command runs under `prefetch-supervisor.mjs` as the leader of its own +// process group, and stopping signals and waits for the whole group. A group +// of its own no longer receives the terminal's Ctrl-C, and the parent cannot +// forward it: the verifier spends most of its time blocked in `execFileSync`, +// where no JavaScript signal handler can run, and installing one would also +// replace the parent's immediate default termination. Instead the supervisor +// holds a pipe from the parent and stops its group when the pipe closes, which +// the kernel does when the parent exits for any reason. The parent installs +// no signal handlers. +// +// Windows has no process groups and `kill` there ends a process outright, so a +// signalled supervisor could pass nothing on and its command would run on +// after cleanup returned. There a deliberate stop goes through the same pipe +// as a written request, which the supervisor answers by ending its command, +// and only the SIGKILL escalation targets the supervisor itself. import { spawn } from 'node:child_process'; import { closeSync, openSync, readFileSync, statSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +export const PREFETCH_SUPERVISOR_SCRIPT = fileURLToPath(new URL('./prefetch-supervisor.mjs', import.meta.url)); + +const GROUPS = process.platform !== 'win32'; const running = child => child.pid !== undefined && child.exitCode === null && child.signalCode === null; +const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); -// Returns { name, child, done, durationMs }. `done` resolves on exit code 0 -// and otherwise rejects with the tail of the log; the rejection is observed -// here so an unawaited failure never surfaces as an unhandled rejection. +// Signals the child's whole process group (POSIX). Without groups the +// supervisor is asked to stop its command through the pipe it watches, since +// `kill` would end the supervisor at once and leave the command running; only +// SIGKILL is sent to the supervisor itself, after it has had its chance. +const signalTree = (child, signal) => { + if (child.pid === undefined) return; + if (!GROUPS) { + if (!running(child)) return; + if (signal === 'SIGKILL') child.kill(signal); + else if (child.stdin?.writable) child.stdin.end('stop\n'); + return; + } + try { process.kill(-child.pid, signal); } catch (error) { if (error.code !== 'ESRCH') throw error; } +}; +// True while any member of the child's group exists. On a platform without +// groups only the child itself is known. +const treeAlive = child => { + if (child.pid === undefined) return false; + if (!GROUPS) return running(child); + try { process.kill(-child.pid, 0); return true; } catch (error) { return error.code !== 'ESRCH'; } +}; + +// Returns { name, child, done, durationMs }. `child` is the supervisor, whose +// exit code or signal is the command's. `done` resolves on exit code 0 and +// otherwise rejects with the tail of the log; the rejection is observed here +// so an unawaited failure never surfaces as an unhandled rejection. export function startPrefetch({ name, cmd, args, cwd, env, logPath }) { const startedAt = Date.now(); const log = openSync(logPath, 'w'); let child; try { - child = spawn(cmd, args, { cwd, env, stdio: ['ignore', log, log] }); + // stdin is the pipe the supervisor watches; nothing is ever written to it. + child = spawn(process.execPath, [PREFETCH_SUPERVISOR_SCRIPT, cmd, ...args], { cwd, env, stdio: ['pipe', log, log], detached: GROUPS }); } finally { closeSync(log); // the child holds its own descriptor } + child.stdin?.on('error', () => {}); const handle = { name, child, done: null, durationMs: null }; let settled = false; handle.done = new Promise((settle, reject) => { @@ -48,13 +97,22 @@ export function startPrefetch({ name, cmd, args, cwd, env, logPath }) { return handle; } -// Stops every prefetch that is still running and resolves only once all of -// them have exited, so the caller can remove their working directory without -// racing a write. A child that ignores SIGTERM is killed after `graceMs`. +// Stops every prefetch that is still running, together with any process it +// started, and resolves only once all of them have exited, so the caller can +// remove their working directory without racing a write. A tree that ignores +// SIGTERM is killed after `graceMs`; one that still exists `graceMs` after +// that is given up on so cleanup cannot hang. export async function stopPrefetches(prefetches, { graceMs = 5000 } = {}) { await Promise.all(prefetches.map(async ({ child, done }) => { - if (running(child)) child.kill('SIGTERM'); - const escalate = setTimeout(() => { if (running(child)) child.kill('SIGKILL'); }, graceMs); - try { await done; } catch {} finally { clearTimeout(escalate); } + const started = Date.now(); + let termed = false, killed = false; + while (treeAlive(child)) { + const elapsed = Date.now() - started; + if (!termed) { signalTree(child, 'SIGTERM'); termed = true; } + else if (!killed && elapsed >= graceMs) { signalTree(child, 'SIGKILL'); killed = true; } + else if (killed && elapsed >= 2 * graceMs) break; + await sleep(25); + } + try { await done; } catch {} })); } diff --git a/scripts/prefetch-supervisor.mjs b/scripts/prefetch-supervisor.mjs new file mode 100644 index 00000000000..0cd29765c68 --- /dev/null +++ b/scripts/prefetch-supervisor.mjs @@ -0,0 +1,78 @@ +// Child side of prefetch-child.mjs: runs one prefetch command as the leader of +// its own process group and stops the whole group when the parent that started +// it is gone. +// +// The parent holds the write end of this process's stdin. The kernel closes +// that end when the parent exits for any reason: a Ctrl-C delivered while the +// parent is blocked in synchronous work, a plain `kill`, or SIGKILL. Reading +// EOF here is therefore the only parent-death signal needed, and the parent +// runs no signal handler of its own, so its default termination stays +// immediate. The parent writes to the pipe for one reason only: to ask for a +// deliberate stop where it cannot signal the group, which is Windows, where +// `kill` ends a process outright and a signalled supervisor could not pass +// anything on. Group-wide signals the parent sends through `stopPrefetches` +// on POSIX reach this process too; it passes them on to the command and stays +// alive until the command exits, so its own exit reports the command's +// outcome (exit code, or the same signal). +import { spawn } from 'node:child_process'; +import { fstatSync } from 'node:fs'; +import { constants } from 'node:os'; + +const GROUPS = process.platform !== 'win32'; +const GRACE_MS = Number(process.env.PAPERCLIP_PREFETCH_GRACE_MS) || 5000; +const [cmd, ...args] = process.argv.slice(2); +if (!cmd) { + process.stderr.write('prefetch supervisor: usage: prefetch-supervisor.mjs [args...]\n'); + process.exit(64); +} + +const child = spawn(cmd, args, { stdio: ['ignore', 'inherit', 'inherit'] }); +const signalChild = signal => { try { child.kill(signal); } catch {} }; +// On POSIX this process leads the group (the parent starts it detached), so +// the group id is its own pid. Without groups only the command is known. +const signalGroup = signal => { + if (!GROUPS) return signalChild(signal); + try { process.kill(-process.pid, signal); } catch (error) { if (error.code === 'ESRCH') signalChild(signal); } +}; + +// A signal to the group arrives here as well as at the command; passing it on +// is harmless when the command already has it, and covers a signal sent to +// this process alone. The parent owns the SIGKILL escalation in that case. +for (const signal of ['SIGINT', 'SIGTERM', 'SIGHUP']) process.on(signal, () => signalChild(signal)); + +let stopping = false; +const stopGroup = reason => { + if (stopping) return; + stopping = true; + process.stderr.write(`prefetch supervisor: ${reason}; stopping ${cmd}\n`); + signalGroup('SIGTERM'); + setTimeout(() => signalGroup('SIGKILL'), GRACE_MS).unref(); +}; +// Only a pipe can carry a stop request or report the parent's exit (Node +// creates its stdio pipes as socket pairs). Run by hand with a terminal or +// /dev/null on stdin, there is no parent to watch and nothing is read. +let stdinIsPipe = false; +try { const stdin = fstatSync(0); stdinIsPipe = stdin.isFIFO() || stdin.isSocket(); } catch {} +if (stdinIsPipe) { + process.stdin.on('data', () => stopGroup('stop requested')); + const parentGone = () => stopGroup('parent exited'); + process.stdin.on('end', parentGone); + process.stdin.on('close', parentGone); + process.stdin.on('error', parentGone); + process.stdin.resume(); +} + +child.on('error', error => { + process.stderr.write(`prefetch supervisor: ${error.message}\n`); + process.exit(127); +}); +child.on('exit', (code, signal) => { + if (signal !== null) { + // Die of the same signal, so the parent sees the command's fate on this + // process. Dropping the listener restores the default disposition first. + process.removeAllListeners(signal); + try { process.kill(process.pid, signal); } catch {} + process.exit(128 + (constants.signals[signal] ?? 1)); + } + process.exit(code ?? 1); +}); diff --git a/scripts/stage-bundled-packages.mjs b/scripts/stage-bundled-packages.mjs new file mode 100644 index 00000000000..9e3abe6acbb --- /dev/null +++ b/scripts/stage-bundled-packages.mjs @@ -0,0 +1,32 @@ +// The public-install verifier stages three packages that bundle dependencies +// (`@paperclipai/server`, `@paperclipai/adapter-utils`, `@paperclipai/db`). +// Each one is a host-side `npm install` of that package's bundled closure into +// its own pack directory, and none reads what another produces, so they run as +// concurrent children instead of one after another inside the staging loop. +// Each child is the CLI entry of `prepare-bundled-package.mjs`, the same path +// release.sh runs, with its output in a log file that a failure reports. +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { startPrefetch } from './prefetch-child.mjs'; + +export const PREPARE_BUNDLED_PACKAGE_SCRIPT = fileURLToPath(new URL('./prepare-bundled-package.mjs', import.meta.url)); + +// `@paperclipai/adapter-utils` -> `adapter_utils`, for log names and timing keys. +export const bundledPackageSlug = name => name.replace(/^@paperclipai\//, '').replace(/[^a-z0-9]+/gi, '_'); + +// Starts one child per entry (`{ name, stagedSource, target }`) at once and +// returns the entries with their `slug` and prefetch `handle`. The caller joins +// `handle.done` where the packed output is needed and registers the handles +// for cleanup, so an earlier failure stops every install still running. +export function startBundledInstalls(entries, { logDir, cwd, env = process.env, script = PREPARE_BUNDLED_PACKAGE_SCRIPT }) { + return entries.map(entry => { + const slug = bundledPackageSlug(entry.name); + const handle = startPrefetch({ name: `bundled-${slug}`, cmd: process.execPath, args: [script, entry.stagedSource, entry.target], cwd, env, logPath: join(logDir, `bundled-${slug}.log`) }); + return { ...entry, slug, handle }; + }); +} + +// Per-package wall time for the timing line. The helper measures from the +// start to the child's last output; the parent is idle while it waits for the +// installs, so the exit is observed as it happens. +export const bundledInstallTimings = installs => Object.fromEntries(installs.map(({ slug, handle }) => [`stage_bundled_${slug}_ms`, handle.durationMs])); diff --git a/scripts/verify-grok-npm-install.mjs b/scripts/verify-grok-npm-install.mjs index d35c40e8183..a5844b220b4 100644 --- a/scripts/verify-grok-npm-install.mjs +++ b/scripts/verify-grok-npm-install.mjs @@ -8,10 +8,11 @@ import { chmodSync, cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, re import { tmpdir } from 'node:os'; import { basename, dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { materializePublishManifest, prepareBundledPackage } from './prepare-bundled-package.mjs'; +import { materializePublishManifest } from './prepare-bundled-package.mjs'; import { GROK_PUBLIC_INSTALL_IMAGE, GROK_PUBLIC_INSTALL_LIFECYCLE, assertNoBundledCodexPayloads, grokConsumerDockerArgs, installedCodexProbeSource } from './grok-public-install-sandbox.mjs'; import { startPrefetch, stopPrefetches } from './prefetch-child.mjs'; import { retainRunnerQualificationPackages } from './retain-runner-qualification-packages.mjs'; +import { bundledInstallTimings, startBundledInstalls } from './stage-bundled-packages.mjs'; const repo = resolve(dirname(fileURLToPath(import.meta.url)), '..'); assert.equal(process.platform, 'linux', 'Run this verification on disposable EC2 Linux, not a developer host'); const root = mkdtempSync(join(tmpdir(), 'paperclip-grok-public-install-')); @@ -28,13 +29,18 @@ const releaseVersion = `0.0.0-grok-verify.${sourceRevision.slice(0, 12)}`; // `native/grok` path that the post-install assertion inspects. const prefetched = join(root, 'prefetch'); mkdirSync(prefetched); const stagedGrok = join(prefetched, 'grok'); +// Every background child, downloads and bundled-dependency installs alike, is +// stopped in `finally`; `downloads` is the subset the timing line reports as +// prefetches. const prefetches = []; +const downloads = []; const timing = {}; // A prefetch failure surfaces where its result is awaited, never as an // unhandled rejection that would mask an earlier staging error. const prefetch = (name, cmd, args) => { const handle = startPrefetch({ name, cmd, args, cwd: root, env, logPath: join(prefetched, `${name}.log`) }); prefetches.push(handle); + downloads.push(handle); return handle.done; }; let phaseStarted = performance.now(); @@ -61,11 +67,21 @@ try { // Source manifests remain untouched, including independently versioned SDKs. run(process.execPath, [join(repo, 'scripts/build-standalone-public-packages.mjs')], repo); run('bash', [join(repo, 'scripts/prepare-server-ui-dist.sh')], repo); - const tarballs = []; - for (const [index, name] of [...needed].entries()) { + // Each package's published files are staged into `source-` and its + // pack input into `package-`, with in dependency order so the + // tarball list keeps matching `needed`. The packages that bundle dependencies + // each need a host-side `npm install` of their bundled closure; those installs + // write only their own `package-` and read nothing another package + // produces, so they start first and run together while the other packages + // are copied, instead of one after another inside one loop. Packing stays + // serial and in order, because each tarball is checked as it is produced. + const entries = [...needed].map((name, index) => { const { dir, manifest } = packages.get(name); - const target = join(root, `package-${index}`); mkdirSync(target); - const stagedSource = join(root, `source-${index}`); mkdirSync(stagedSource); + const bundled = (manifest.bundleDependencies ?? manifest.bundledDependencies ?? []).length > 0; + return { name, dir, manifest, bundled, target: join(root, `package-${index}`), stagedSource: join(root, `source-${index}`) }; + }); + const stageSource = ({ dir, manifest, stagedSource }) => { + mkdirSync(stagedSource); for (const file of manifest.files ?? ['dist']) { // release.sh stages runtime skills into these public packages before pack. const releaseSkills = file === 'skills' && ['server', 'packages/adapters/claude-local', 'packages/adapters/codex-local'].includes(dir); @@ -76,12 +92,25 @@ try { } const releaseManifest = { ...manifest, version: releaseVersion }; writeFileSync(join(stagedSource, 'package.json'), JSON.stringify(releaseManifest)); - if ((manifest.bundleDependencies ?? manifest.bundledDependencies ?? []).length) { - prepareBundledPackage(stagedSource, target); - } else { - cpSync(stagedSource, target, { recursive: true }); - writeFileSync(join(target, 'package.json'), JSON.stringify(materializePublishManifest(releaseManifest))); - } + return releaseManifest; + }; + const sourcesStarted = performance.now(); + for (const entry of entries.filter(entry => entry.bundled)) stageSource(entry); + const installsStarted = performance.now(); + const installs = startBundledInstalls(entries.filter(entry => entry.bundled), { logDir: prefetched, cwd: repo }); + prefetches.push(...installs.map(({ handle }) => handle)); + for (const entry of entries.filter(entry => !entry.bundled)) { + const releaseManifest = stageSource(entry); + mkdirSync(entry.target); + cpSync(entry.stagedSource, entry.target, { recursive: true }); + writeFileSync(join(entry.target, 'package.json'), JSON.stringify(materializePublishManifest(releaseManifest))); + } + timing.stage_sources_ms = Math.round(performance.now() - sourcesStarted); + await awaited('bundled', Promise.all(installs.map(({ handle }) => handle.done))); + timing.stage_bundled_ms = Math.round(performance.now() - installsStarted); + const packStarted = performance.now(); + const tarballs = []; + for (const { target } of entries) { run('npm', ['pack', '--ignore-scripts', '--pack-destination', root], target); const packed = readdirSync(root).filter(f => f.endsWith('.tgz') && !tarballs.includes(join(root, f))); assert.equal(packed.length, 1); @@ -89,6 +118,7 @@ try { assertNoBundledCodexPayloads(run('tar', ['-tzf', tarball]).toString().trim().split('\n')); tarballs.push(tarball); } + timing.stage_pack_ms = Math.round(performance.now() - packStarted); const assets = join(root, 'assets'); mkdirSync(assets, { mode: 0o755 }); const consumer = join(root, 'consumer'); mkdirSync(consumer); const cache = join(root, 'cache'); mkdirSync(cache); @@ -184,8 +214,10 @@ try { // without re-running it. `*_prefetch_ms` runs from a download's start to its // last output, and `*_wait_ms` is how long the main flow waited for a // prefetch that had not finished yet; zero means the overlap hid it entirely. - const downloads = Object.fromEntries(prefetches.map(({ name, durationMs }) => [`${name}_prefetch_ms`, durationMs])); - console.log(JSON.stringify({ schema: 'paperclip.public-npm-install.timing.v1', ...timing, ...downloads })); + // `stage_sources_ms`, `stage_bundled_ms` and `stage_pack_ms` split the + // staging phase, and `stage_bundled__ms` is each concurrent install. + const downloaded = Object.fromEntries(downloads.map(({ name, durationMs }) => [`${name}_prefetch_ms`, durationMs])); + console.log(JSON.stringify({ schema: 'paperclip.public-npm-install.timing.v1', ...timing, ...downloaded, ...bundledInstallTimings(installs) })); if (process.env.PAPERCLIP_RUNNER_QUALIFICATION_PACKAGES_DIR) { const retained = retainRunnerQualificationPackages({ repo, output: process.env.PAPERCLIP_RUNNER_QUALIFICATION_PACKAGES_DIR, sourceRevision, releaseVersion, env, publicArchives: [...needed].map((name, index) => ({ name, file: tarballs[index] })), From 3d3b8d4371445f5a7a1d8cc8555c0a9f449c651d Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Sun, 11 Oct 2026 11:53:21 -0700 Subject: [PATCH 06/18] fix(external-objects): distinguish hidden GitHub repos from genuinely-missing objects (#15906) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - Issues reference external objects such as GitHub pull requests. The instance resolves their status and shows it in the UI. > - GitHub returns 404 both when a pull request or issue does not exist and when the repository exists but the configured credentials cannot see it. GitHub hides private repos behind a 404 on purpose. > - The resolver treated every 404 as a terminal not-found verdict. It froze references to inaccessible repos forever, and marked them "Not found (Stale)". > - This pull request probes the repository metadata to tell the two cases apart and keep frozen objects re-checkable. > - The benefit is accurate statuses instead of permanent "Not found" labels for repos the instance cannot see. ## Linked Issues or Issue Description No public GitHub issue exists for this report. **What happened?** - The issue properties panel and markdown decoration showed many GitHub pull request and issue references as "Not found (Stale)" even though the objects exist. - GitHub returns 404 for objects the configured credentials cannot access, not only for objects that are truly gone. - The resolver treated every 404 as a terminal not-found verdict, so those references were frozen and showed a misleading "stale" liveness. - Detected-but-unresolved GitHub objects were labeled "Not yet resolved" in inline markdown, which did not match the pill label "Not yet refreshed". **Expected behavior** - A reference to a repo the credentials cannot see should surface as authorization-required, not as a permanent "Not found". - A truly missing object should still be "Not found". - A terminal object (merged, closed, or confirmed not-found) should not be labeled "Stale", because it is never refreshed again. - Inline markdown statuses should match the pill statuses. **Steps to reproduce** 1. Configure GitHub credentials with no access to a private repo. 2. Add a comment or issue description that links to a pull request in that repo. 3. Wait for the external object resolver to run. 4. Observe the reference labeled "Not found (Stale)". **Paperclip version or commit** - `master` at `e733bdd` (before this change). **Deployment mode** - Self-hosted server. ## What Changed - Added a repository visibility probe to the GitHub external object provider. - A 404 for an object now probes the repository metadata with the same credentials: - Repository visible: the object genuinely is gone. Keep the not-found snapshot. - Repository hidden: return an authorization-required result instead of a terminal not-found verdict. - Probe failed (network, rate limit, 5xx): return an unreachable result and retry later. - A rate-limited repository probe (403 with `x-ratelimit-remaining: 0`) is treated as uncertain, not as missing access, so it retries instead of showing a misleading "Requires auth". - Re-check previously frozen GitHub not-found objects on their normal schedule so they self-heal once credentials or repo visibility improve. A frozen object is only re-checked until a probe confirms the repo is visible; once confirmed deleted, it is terminal and no longer polled. - Cleared any terminal verdict when a refresh fails, so a failed refresh does not freeze an object. - Kept terminal objects from displaying the "stale" liveness label. - Matched the inline markdown status label to the pill status label, so detected-but-unresolved GitHub objects say "Not yet refreshed" instead of "Not yet resolved". - Skipped the liveness suffix in markdown links and status icons when it duplicates the status label (e.g. no more "Requires auth (Requires auth)"). - Cached repository visibility probes per company+repo + credential fingerprint, so many objects in the same repo share one probe per refresh cycle instead of each issuing its own — and a verdict obtained under one token is never reused by a later refresh running under a different credential. ## Verification - Added provider tests for visible, hidden, failed, and rate-limited repository probes, per-company/per-repo probe caching, and probe-verified not-found snapshots. - Added service tests for terminal-object liveness, for re-checking frozen GitHub not-found objects, and for ensuring probe-verified not-found objects are never re-queued. - Added UI tests asserting unresolved GitHub objects render as "Not yet refreshed" (not "Not yet resolved") and that failed-first-refresh labels do not repeat the liveness text. - Ran `npx vitest run src/__tests__/external-objects-service.test.ts` in `server/`: 34 passed (incl. provider + service suites). - Ran the external-object markdown/pill/icon suites in `ui/`: 97 passed. - `ci / Typecheck + Release Registry` and all other CI gates pass on this head. Local `tsc` could not run to completion in the execution sandbox (the sandbox memory cap killed it), so CI typecheck is the authoritative gate. ## Risks - Low risk. The change is confined to the GitHub external object provider and the external object service. - A repo that genuinely does not exist and a private repo both return 404 for the probe. Both now surface authorization-required or a retry instead of an immediate not-found freeze, which stays re-checkable and self-heals. - Re-checking frozen not-found objects adds at most one repository probe per affected object until the probe produces a definitive verdict, after which the object is terminal and no longer polled. Probe results are cached per company+repo+credential, so a credential rotation always re-probes instead of inheriting a stale verdict. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - Provider: DeepSeek via OpenRouter, model ID `openrouter/~deepseek/deepseek-v4-flash-latest`, running inside the Paperclip opencode adapter (`opencode_local`). - The model produced and assisted with the reasoning and code changes; the execution workspace, test suite, and git/PR steps above were run by the same agent run. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [ ] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details — the branch name is assigned by the managed execution workspace and the harness requires keeping it - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green on the current head (all `ci / *`, Greptile, Socket, and review checks pass) - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: devinfoley <139239+devinfoley@users.noreply.github.com> Co-authored-by: Paperclip --- .../external-objects-service.test.ts | 367 +++++++++++++++++- server/src/services/external-objects.ts | 34 +- .../github-external-object-provider.ts | 96 ++++- .../components/ExternalObjectStatusIcon.tsx | 3 +- ui/src/components/MarkdownBody.tsx | 5 +- .../components/MarkdownExternalLink.test.tsx | 53 +++ 6 files changed, 546 insertions(+), 12 deletions(-) diff --git a/server/src/__tests__/external-objects-service.test.ts b/server/src/__tests__/external-objects-service.test.ts index ba8b1184541..76555d88dd5 100644 --- a/server/src/__tests__/external-objects-service.test.ts +++ b/server/src/__tests__/external-objects-service.test.ts @@ -328,11 +328,6 @@ describe("GitHub external object provider", () => { }), { ok: false, liveness: "unreachable", errorCode: "github_rate_limited" }, ], - [ - "not-found", - new Response("", { status: 404, headers: { etag: '"missing"' } }), - { ok: true, snapshot: expect.objectContaining({ displayKey: "GitHub Pull Request", iconKey: "github", statusKey: "not_found", statusIconKey: "archive", statusCategory: "archived", statusTone: "muted" }) }, - ], ])("maps %s responses to provider-safe results", async (_name, githubResponse, expected) => { const provider = createGitHubExternalObjectProvider({} as any, { fetch: async () => githubResponse, @@ -348,6 +343,191 @@ describe("GitHub external object provider", () => { expect(result).toEqual(expect.objectContaining(expected)); expect(JSON.stringify(result)).not.toContain("http"); }); + + it("classifies a 404 as not-found when the repository is visible to the credentials", async () => { + const fetch = vi.fn(async (url: string) => { + if (url.endsWith("/repos/acme/app")) { + return new Response(JSON.stringify({ full_name: "acme/app" }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + } + return new Response("", { status: 404, headers: { etag: '"missing"' } }); + }); + const provider = createGitHubExternalObjectProvider({} as any, { fetch, tokenProvider: null }); + const resolver = provider.resolvers.find((entry) => entry.objectType === "pull_request")!; + + const result = await resolver.resolve({ + companyId: "company-1", + object: githubObject("pull/42", "pull_request"), + }); + + expect(fetch).toHaveBeenCalledWith( + "https://api.github.com/repos/acme/app", + expect.objectContaining({ headers: expect.not.objectContaining({ authorization: expect.any(String) }) }), + ); + expect(result).toEqual({ + ok: true, + snapshot: expect.objectContaining({ + displayKey: "GitHub Pull Request", + iconKey: "github", + statusKey: "not_found", + statusIconKey: "archive", + statusCategory: "archived", + statusTone: "muted", + isTerminal: true, + data: expect.objectContaining({ provider: "github", notFound: true, probeVerified: true }), + }), + }); + expect(JSON.stringify(result)).not.toContain("http"); + }); + + it("caches repository-visibility probes per company and repo", async () => { + let probeCalls = 0; + const fetch = vi.fn(async (url: string) => { + if (url.endsWith("/repos/acme/app")) { + probeCalls += 1; + return new Response(JSON.stringify({ full_name: "acme/app" }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + } + return new Response("", { status: 404, headers: { etag: '"missing"' } }); + }); + const provider = createGitHubExternalObjectProvider({} as any, { fetch, tokenProvider: null }); + const resolver = provider.resolvers.find((entry) => entry.objectType === "pull_request")!; + + await resolver.resolve({ companyId: "company-1", object: githubObject("pull/42", "pull_request") }); + await resolver.resolve({ companyId: "company-1", object: githubObject("pull/99", "pull_request") }); + + expect(probeCalls).toBe(1); + expect(fetch.mock.calls.filter(([url]) => String(url).endsWith("/repos/acme/app"))).toHaveLength(1); + }); + + it("does not reuse a cached probe verdict across credential changes", async () => { + const fetch = vi.fn(async (url: string, init?: RequestInit) => { + if (url.endsWith("/repos/acme/app")) { + const auth = (init?.headers as Record | undefined)?.authorization; + if (auth === "Bearer token-A" || auth === "Bearer token-B") { + return new Response(JSON.stringify({ full_name: "acme/app" }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + } + return new Response("", { status: 404 }); + } + return new Response("", { status: 404, headers: { etag: '"missing"' } }); + }); + let token: string | null = "token-A"; + const provider = createGitHubExternalObjectProvider({} as any, { + fetch, + tokenProvider: async () => token, + }); + const resolver = provider.resolvers.find((entry) => entry.objectType === "pull_request")!; + + const visibleVerdict = await resolver.resolve({ companyId: "company-1", object: githubObject("pull/42", "pull_request") }); + + // Same repo, same credentials: probe result is reused. + await resolver.resolve({ companyId: "company-1", object: githubObject("pull/99", "pull_request") }); + const probesUnderTokenA = fetch.mock.calls.filter(([url]) => String(url).endsWith("/repos/acme/app")); + expect(probesUnderTokenA).toHaveLength(1); + + // Credential now only has access to a different repo — cache must not + // serve the stale "visible" verdict, or this object would be frozen as a + // probe-verified not-found forever. + token = "other-token"; + const hiddenVerdict = await resolver.resolve({ companyId: "company-1", object: githubObject("pull/10", "pull_request") }); + + expect(visibleVerdict).toEqual( + expect.objectContaining({ ok: true, snapshot: expect.objectContaining({ isTerminal: true }) }), + ); + expect(hiddenVerdict).toEqual( + expect.objectContaining({ ok: false, liveness: "auth_required", errorCode: "github_repo_not_accessible" }), + ); + expect(fetch.mock.calls.filter(([url]) => String(url).endsWith("/repos/acme/app"))).toHaveLength(2); + }); + + it("classifies a 404 as auth-required when the repository itself is hidden", async () => { + const fetch = vi.fn(async (url: string) => { + if (url.endsWith("/repos/acme/app")) { + return new Response("", { status: 404 }); + } + return new Response("", { status: 404, headers: { etag: '"missing"' } }); + }); + const provider = createGitHubExternalObjectProvider({} as any, { fetch, tokenProvider: null }); + const resolver = provider.resolvers.find((entry) => entry.objectType === "pull_request")!; + + const result = await resolver.resolve({ + companyId: "company-1", + object: githubObject("pull/42", "pull_request"), + }); + + expect(result).toEqual( + expect.objectContaining({ + ok: false, + liveness: "auth_required", + errorCode: "github_repo_not_accessible", + errorMessage: expect.stringContaining("acme/app"), + }), + ); + expect(JSON.stringify(result)).not.toContain("http"); + }); + + it("classifies a 404 as unreachable when the repository probe itself fails", async () => { + const fetch = vi.fn(async (url: string) => { + if (url.endsWith("/repos/acme/app")) { + throw new Error("network down"); + } + return new Response("", { status: 404, headers: { etag: '"missing"' } }); + }); + const provider = createGitHubExternalObjectProvider({} as any, { fetch, tokenProvider: null }); + const resolver = provider.resolvers.find((entry) => entry.objectType === "pull_request")!; + + const result = await resolver.resolve({ + companyId: "company-1", + object: githubObject("pull/42", "pull_request"), + }); + + expect(result).toEqual( + expect.objectContaining({ + ok: false, + liveness: "unreachable", + errorCode: "github_repo_probe_failed", + }), + ); + expect(JSON.stringify(result)).not.toContain("http"); + }); + + it("classifies a 404 as unreachable when the repository probe is rate-limited", async () => { + const fetch = vi.fn(async (url: string) => { + if (url.endsWith("/repos/acme/app")) { + return new Response("", { + status: 403, + headers: { "x-ratelimit-remaining": "0", "x-ratelimit-reset": String(Math.floor(Date.now() / 1000) + 120) }, + }); + } + return new Response("", { status: 404, headers: { etag: '"missing"' } }); + }); + const provider = createGitHubExternalObjectProvider({} as any, { fetch, tokenProvider: null }); + const resolver = provider.resolvers.find((entry) => entry.objectType === "pull_request")!; + + const result = await resolver.resolve({ + companyId: "company-1", + object: githubObject("pull/42", "pull_request"), + }); + + expect(result).toEqual( + expect.objectContaining({ + ok: false, + liveness: "unreachable", + errorCode: "github_repo_probe_failed", + }), + ); + expect(result).not.toEqual( + expect.objectContaining({ ok: false, liveness: "auth_required", errorCode: "github_repo_not_accessible" }), + ); + expect(JSON.stringify(result)).not.toContain("http"); + }); }); describeEmbeddedPostgres("externalObjectService", () => { @@ -786,6 +966,183 @@ describeEmbeddedPostgres("externalObjectService", () => { expect(resolve).toHaveBeenCalledTimes(1); }); + it("keeps terminal objects fresh-for-display after their refresh window passes", async () => { + const { companyId, issueId } = await createIssue(); + const resolve = vi.fn(async () => ({ + ok: true as const, + snapshot: { + statusCategory: "closed" as const, + statusTone: "muted" as const, + statusKey: "closed", + statusLabel: "Closed", + isTerminal: true, + ttlSeconds: 1, + }, + })); + const svc = externalObjectService(db, { + resolvers: [{ providerKey: "url", objectType: "link", resolve }], + github: false, + }); + await svc.syncIssue(issueId); + const object = await db.select().from(externalObjects).then((rows) => rows[0]!); + await svc.refreshObject(object.id, { companyId, force: true }); + // Expire the refresh window so the display path computes liveness from + // scratch. Terminal objects must still show as fresh, never as stale. + await db + .update(externalObjects) + .set({ nextRefreshAt: new Date(0) }) + .where(eq(externalObjects.id, object.id)); + + const summary = await svc.getIssueSummary(issueId); + const displayed = summary.objects[0]; + expect(displayed?.liveness).toBe("fresh"); + expect(displayed?.isTerminal).toBe(true); + + const summaryLater = await svc.getIssueSummaries(companyId, [issueId]).then((map) => map.get(issueId)); + expect(summaryLater?.objects[0]?.liveness).toBe("fresh"); + expect(summaryLater?.staleCount).toBe(0); + expect(summaryLater?.byLiveness?.stale ?? 0).toBe(0); + }); + + it("re-checks previously frozen GitHub not-found objects so they can self-heal", async () => { + const { companyId } = await createIssue(); + const canonical = canonicalizeExternalObjectUrl("https://github.com/acme/app/pull/42"); + if (!canonical) throw new Error("expected canonical url"); + const inserted = await db.insert(externalObjects).values({ + companyId, + providerKey: "github", + objectType: "pull_request", + externalId: "acme/app#pull/42", + sanitizedCanonicalUrl: canonical.sanitizedCanonicalUrl, + canonicalIdentityHash: canonical.canonicalIdentityHash, + displayTitle: "acme/app#42", + statusKey: "not_found", + statusLabel: "Not found", + statusCategory: "archived", + statusTone: "muted", + liveness: "fresh", + isTerminal: true, + nextRefreshAt: new Date(0), + data: { provider: "github", notFound: true }, + }).returning(); + const object = inserted[0]!; + + const snapshot = { + statusKey: "open" as const, + statusLabel: "Open" as const, + statusCategory: "open" as const, + statusTone: "info" as const, + statusIconKey: "git-pull-request" as const, + displayTitle: "acme/app#42", + isTerminal: false, + ttlSeconds: 300, + data: { provider: "github", owner: "acme", repo: "app", number: 42 }, + }; + const resolver: ExternalObjectResolver = { + providerKey: "github", + objectType: "pull_request", + resolve: vi.fn(async () => ({ ok: true as const, snapshot })), + }; + const svc = externalObjectService(db, { resolvers: [resolver], github: false }); + expect(object.isTerminal).toBe(true); + + const refreshed = await svc.refreshDueObjects(companyId); + expect(refreshed).toHaveLength(1); + expect(refreshed[0]?.refreshed).toBe(true); + + const updated = await db.select().from(externalObjects).then((rows) => rows[0]!); + expect(updated.statusLabel).toBe("Open"); + expect(updated.isTerminal).toBe(false); + expect(updated.liveness).toBe("fresh"); + }); + + it("does not re-check probe-verified GitHub not-found objects (deleted references are terminal)", async () => { + const { companyId } = await createIssue(); + const canonical = canonicalizeExternalObjectUrl("https://github.com/acme/app/pull/42"); + if (!canonical) throw new Error("expected canonical url"); + await db.insert(externalObjects).values({ + companyId, + providerKey: "github", + objectType: "pull_request", + externalId: "acme/app#pull/42", + sanitizedCanonicalUrl: canonical.sanitizedCanonicalUrl, + canonicalIdentityHash: canonical.canonicalIdentityHash, + displayTitle: "acme/app#42", + statusKey: "not_found", + statusLabel: "Not found", + statusCategory: "archived", + statusTone: "muted", + liveness: "fresh", + isTerminal: true, + nextRefreshAt: new Date(0), + data: { provider: "github", notFound: true, probeVerified: true }, + }); + const resolver: ExternalObjectResolver = { + providerKey: "github", + objectType: "pull_request", + resolve: vi.fn(async () => ({ + ok: true as const, + snapshot: { + statusKey: "open", + statusLabel: "Open", + statusCategory: "open" as const, + statusTone: "info" as const, + isTerminal: false, + ttlSeconds: 300, + }, + })), + }; + const svc = externalObjectService(db, { resolvers: [resolver], github: false }); + + const refreshed = await svc.refreshDueObjects(companyId); + + expect(refreshed).toEqual([]); + expect(resolver.resolve).not.toHaveBeenCalled(); + }); + + it("does not re-check terminal objects that are not GitHub not-found verdicts", async () => { + const { companyId } = await createIssue(); + const canonical = canonicalizeExternalObjectUrl("https://github.com/acme/app/pull/42"); + if (!canonical) throw new Error("expected canonical url"); + await db.insert(externalObjects).values({ + companyId, + providerKey: "github", + objectType: "pull_request", + externalId: "acme/app#pull/42", + sanitizedCanonicalUrl: canonical.sanitizedCanonicalUrl, + canonicalIdentityHash: canonical.canonicalIdentityHash, + displayTitle: "acme/app#42", + statusKey: "merged", + statusLabel: "Merged", + statusCategory: "succeeded", + statusTone: "success", + liveness: "fresh", + isTerminal: true, + nextRefreshAt: new Date(0), + }); + const resolver: ExternalObjectResolver = { + providerKey: "github", + objectType: "pull_request", + resolve: vi.fn(async () => ({ + ok: true as const, + snapshot: { + statusKey: "open", + statusLabel: "Open", + statusCategory: "open" as const, + statusTone: "info" as const, + isTerminal: false, + ttlSeconds: 300, + }, + })), + }; + const svc = externalObjectService(db, { resolvers: [resolver], github: false }); + + const refreshed = await svc.refreshDueObjects(companyId); + + expect(refreshed).toEqual([]); + expect(resolver.resolve).not.toHaveBeenCalled(); + }); + it("keeps external object identities company-scoped for duplicate urls", async () => { const first = await createIssue(); const second = await createIssue(); diff --git a/server/src/services/external-objects.ts b/server/src/services/external-objects.ts index 1312e62d3f8..6776cb25305 100644 --- a/server/src/services/external-objects.ts +++ b/server/src/services/external-objects.ts @@ -1,5 +1,5 @@ import { randomUUID } from "node:crypto"; -import { and, asc, eq, inArray, isNull, lte, or, sql } from "drizzle-orm"; +import { and, asc, eq, inArray, isNull, lte, not, or, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { companies, documents, externalObjectMentions, externalObjects, issueComments, issueDocuments, issues, plugins } from "@paperclipai/db"; import { @@ -124,6 +124,9 @@ function addSeconds(date: Date, seconds: number) { } function visibleLiveness(object: ExternalObjectRecord, now = new Date()): ExternalObjectLivenessState { + // Terminal objects are never refreshed again, so "stale" (refresh overdue) is + // misleading — their resolved status is final. Keep the last liveness as-is. + if (object.isTerminal) return object.liveness; if (object.liveness === "fresh" && object.nextRefreshAt && object.nextRefreshAt <= now) { return "stale"; } @@ -810,7 +813,7 @@ export function externalObjectService( const [updated] = await db .update(externalObjects) .set({ - liveness: visibleLiveness(object, now) === "fresh" ? "stale" : object.liveness, + liveness: object.isTerminal ? object.liveness : visibleLiveness(object, now) === "fresh" ? "stale" : object.liveness, nextRefreshAt: addSeconds(now, DEFAULT_RETRY_AFTER_SECONDS), refreshStartedAt: null, refreshToken: null, @@ -824,10 +827,15 @@ export function externalObjectService( const result = pluginResult ?? await resolver!.resolve({ companyId: object.companyId, object }); if (!result.ok) { + // A failed refresh means the object's status is currently unknown, not + // terminal. Clear a previous terminal verdict so the object keeps being + // scheduled for automatic refresh and self-heals (e.g. a GitHub "not + // found" that later resolves once repo access improves). const [updated] = await db .update(externalObjects) .set({ liveness: result.liveness, + isTerminal: false, lastErrorAt: now, lastErrorCode: result.errorCode, lastErrorMessage: sanitizeErrorMessage(result.errorMessage), @@ -1024,13 +1032,33 @@ export function externalObjectService( async function refreshDueObjectsUnchecked(companyId: string, limit = 50, now = new Date()) { const staleRefreshStartedBefore = new Date(now.getTime() - DEFAULT_REFRESH_LEASE_SECONDS * 1000); + // Terminal objects are usually skipped forever, but a GitHub "not found" + // verdict is only trustworthy while the repository is actually visible to + // the configured credentials. GitHub hides inaccessible (often private) + // repos behind a 404, so a not-found object can become resolvable later + // when credentials or repo visibility improve. Re-check objects that were + // frozen as "not found" before the repository probe existed so they + // self-heal instead of freezing. Once the provider has probed the repo and + // confirmed it is visible (data.probeVerified = "true"), the not-found + // verdict is definitive, so stop re-polling that object to avoid burning + // the company's GitHub API budget on deleted references forever. + const probeVerifiedNotFound = sql`coalesce(${externalObjects.data}->>'probeVerified', 'false') = 'true'`; + const refreshable = or( + eq(externalObjects.isTerminal, false), + and( + eq(externalObjects.isTerminal, true), + eq(externalObjects.providerKey, "github"), + eq(externalObjects.statusKey, "not_found"), + not(probeVerifiedNotFound), + ), + )!; const due = await db .select({ id: externalObjects.id }) .from(externalObjects) .where( and( eq(externalObjects.companyId, companyId), - eq(externalObjects.isTerminal, false), + refreshable, lte(externalObjects.nextRefreshAt, now), or( isNull(externalObjects.refreshStartedAt), diff --git a/server/src/services/github-external-object-provider.ts b/server/src/services/github-external-object-provider.ts index 928e8501f50..eeb65ca648d 100644 --- a/server/src/services/github-external-object-provider.ts +++ b/server/src/services/github-external-object-provider.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import type { Db } from "@paperclipai/db"; import type { ExternalObjectCanonicalUrl } from "@paperclipai/shared"; import { DEFAULT_GITHUB_TOKEN_SECRET_NAMES } from "./git-credentials.js"; @@ -188,10 +189,54 @@ function notFoundSnapshot(identity: GitHubObjectIdentity, etag: string | null): repo: identity.repo, number: identity.number, notFound: true, + // The repository was probed with the configured credentials and found + // to be visible, so this not-found verdict is definitive, not a stale + // guess from a time when credentials could not see the repo. The due + // sweep uses this flag to avoid re-polling a confirmed-deleted object + // forever and exhausting the company's GitHub API budget. + probeVerified: true, }, }; } +/** + * Probe `/repos/{owner}/{repo}` to classify a 404 on a pull request or issue. + * + * GitHub returns 404 both when the object does not exist and when the repository + * exists but the credentials cannot see it (private repos are deliberately hidden + * behind 404). This probes the repo metadata with the same credentials to tell the + * two apart: + * +* - "visible": the repo returns 2xx — the referenced object really is gone. + * - "hidden": the repo returns 404/403 — the credentials lack access to the repo, + * so the object status cannot be determined and should not be frozen as "Not found". + * - "uncertain": the probe itself failed (network/rate limit) — retry later. + */ +async function probeRepoVisibility( + identity: GitHubObjectIdentity, + opts: { fetch: FetchLike; headers?: Record | undefined }, +): Promise<"visible" | "hidden" | "uncertain"> { + try { + const probe = await opts.fetch( + `${gitHubApiBase(identity.host)}/repos/${encodeURIComponent(identity.owner)}/${encodeURIComponent(identity.repo)}`, + { headers: opts.headers }, + ); + if (probe.ok) return "visible"; + if (probe.status === 404) return "hidden"; + if (probe.status === 403) { + // GitHub can return 403 for both "credentials lack access" and "rate limit + // reached". A rate-limited probe cannot tell the two apart, so treat it as + // uncertain (retry later) instead of implying missing access. + const rateLimitRemaining = probe.headers.get("x-ratelimit-remaining"); + if (rateLimitRemaining === "0") return "uncertain"; + return "hidden"; + } + return "uncertain"; + } catch { + return "uncertain"; + } +} + function pullRequestSnapshot(identity: GitHubObjectIdentity, body: Record, etag: string | null): ExternalObjectResolverSnapshot { const title = asString(body.title); const state = asString(body.state) ?? "unknown"; @@ -338,6 +383,30 @@ export function createGitHubExternalObjectProvider( const tokenProvider = Object.prototype.hasOwnProperty.call(opts, "tokenProvider") && opts.tokenProvider !== undefined ? opts.tokenProvider : ((companyId: string) => defaultTokenProvider(db, companyId, secretNames)); +// Cache repository-visibility probe results so many objects in the same repo + // share a single probe instead of each issuing its own request every refresh + // cycle. Only determined outcomes ("visible"/"hidden") are cached; uncertain + // probes are retried on the next cycle. The cache is keyed per company+repo+ + // credential fingerprint so a verdict obtained under one token is never + // reused after the company rotates its credentials — that would freeze a + // reference as "probe-verified not-found" using an out-of-date result. + + const probeCache = new Map(); + async function cachedProbeRepoVisibility( + companyId: string, + identity: GitHubObjectIdentity, + headers: Record, + ): Promise<"visible" | "hidden" | "uncertain"> { + const credentialFingerprint = createHash("sha256").update(headers.authorization ?? "anonymous").digest("hex"); + const key = `${companyId}\u0000${credentialFingerprint}\u0000${identity.owner.toLowerCase()}\u0000${identity.repo.toLowerCase()}`; + const cached = probeCache.get(key); + if (cached && cached.expiresAt > Date.now()) return cached.result; + const result = await probeRepoVisibility(identity, { fetch: fetchImpl, headers }); + if (result !== "uncertain") { + probeCache.set(key, { result, expiresAt: Date.now() + GITHUB_OBJECT_TTL_SECONDS * 1000 }); + } + return result; + } const detector: ExternalObjectDetector = { key: "github", @@ -414,7 +483,32 @@ export function createGitHubExternalObjectProvider( const etag = response.headers.get("etag"); if (response.status === 404) { - return { ok: true, snapshot: notFoundSnapshot(identity, etag) }; + // GitHub deliberately returns 404 for repositories the credentials cannot + // see, to avoid leaking that a private repo exists. A 404 therefore means + // either "object deleted/never existed" or "repo private / no access". + // Probe the repository metadata to classify: if the repo itself is visible + // the object truly is gone; if it is not, surface credentials-needed + // instead of permanently freezing a reference as "Not found". + const repoVisibility = await cachedProbeRepoVisibility(companyId, identity, headers); + if (repoVisibility === "visible") { + return { ok: true, snapshot: notFoundSnapshot(identity, etag) }; + } + if (repoVisibility === "hidden") { + return { + ok: false, + liveness: "auth_required", + errorCode: "github_repo_not_accessible", + errorMessage: `GitHub returned not-found for ${identity.owner}/${identity.repo}. The repository may be private, deleted, or the configured GitHub credentials may lack access to it.`, + retryAfterSeconds: GITHUB_OBJECT_TTL_SECONDS, + }; + } + return { + ok: false, + liveness: "unreachable", + errorCode: "github_repo_probe_failed", + errorMessage: "GitHub could not verify this repository while checking this object.", + retryAfterSeconds: GITHUB_OBJECT_TTL_SECONDS, + }; } const failure = failureFromGitHubResponse(response); diff --git a/ui/src/components/ExternalObjectStatusIcon.tsx b/ui/src/components/ExternalObjectStatusIcon.tsx index 3c17a941cd7..f529ecc8f6a 100644 --- a/ui/src/components/ExternalObjectStatusIcon.tsx +++ b/ui/src/components/ExternalObjectStatusIcon.tsx @@ -59,7 +59,8 @@ export function ExternalObjectStatusIcon({ const livenessSuffix = liveness === "fresh" || liveness === "unknown" ? "" : ` (${externalObjectLivenessLabel(liveness)})`; - const ariaLabel = `${label ?? externalObjectCategoryLabel(category)}${livenessSuffix}`; + const baseLabel = label ?? externalObjectCategoryLabel(category); + const ariaLabel = `${baseLabel}${baseLabel.includes(externalObjectLivenessLabel(liveness)) ? "" : livenessSuffix}`; // The clock overlay needs a positioned wrapper. Inline mode keeps the icon // tight to the surrounding text; pill mode expects to size by sizeClassName. diff --git a/ui/src/components/MarkdownBody.tsx b/ui/src/components/MarkdownBody.tsx index e45e93fd9b0..9a7b5b34ac1 100644 --- a/ui/src/components/MarkdownBody.tsx +++ b/ui/src/components/MarkdownBody.tsx @@ -37,7 +37,7 @@ import { StatusIcon } from "./StatusIcon"; import { WorkspaceFileLink } from "./WorkspaceFileLink"; import { ExternalObjectStatusIcon } from "./ExternalObjectStatusIcon"; import { - externalObjectCategoryLabel, + externalObjectDisplayStatusLabel, externalObjectLivenessLabel, externalObjectProviderLabel, } from "../lib/external-objects"; @@ -240,9 +240,10 @@ function MarkdownExternalLink({ }) { const provider = externalObjectProviderLabel(reference.providerKey); const displayKey = reference.displayKey?.trim() || provider; - const statusLabel = reference.statusLabel ?? externalObjectCategoryLabel(reference.statusCategory); + const statusLabel = externalObjectDisplayStatusLabel(reference); const livenessLabel = externalObjectLivenessLabel(reference.liveness); const livenessSuffix = reference.liveness === "fresh" || reference.liveness === "unknown" + || livenessLabel.toLowerCase() === statusLabel.toLowerCase() ? "" : ` (${livenessLabel})`; const titleParts = [ diff --git a/ui/src/components/MarkdownExternalLink.test.tsx b/ui/src/components/MarkdownExternalLink.test.tsx index 173b1c46c3e..0f4f9340845 100644 --- a/ui/src/components/MarkdownExternalLink.test.tsx +++ b/ui/src/components/MarkdownExternalLink.test.tsx @@ -91,4 +91,57 @@ describe("MarkdownBody external object decoration", () => { }); expect(html).toContain('aria-label="HubSpot Reconnect (Requires auth): Acme deal"'); }); + + it("labels detected-but-unresolved GitHub objects as not yet refreshed, not not yet resolved", () => { + const html = render("See https://github.com/acme/web/pull/241", { + "https://github.com/acme/web/pull/241": { + providerKey: "github", + objectType: "pull_request", + displayKey: "Github Pull Request", + iconKey: "github", + statusCategory: "unknown", + liveness: "unknown", + statusLabel: null, + displayTitle: "acme/web#241", + }, + }); + expect(html).toContain("data-external-link=\"resolved\""); + expect(html).toContain('aria-label="Github Pull Request Not yet refreshed: acme/web#241"'); + expect(html).not.toContain("Not yet resolved"); + }); + + it("does not repeat the liveness label when the status label equals it", () => { + const html = render("Auth-blocked: https://github.com/acme/web/pull/241", { + "https://github.com/acme/web/pull/241": { + providerKey: "github", + objectType: "pull_request", + displayKey: "Github Pull Request", + iconKey: "github", + statusCategory: "unknown", + liveness: "auth_required", + statusLabel: null, + displayTitle: "acme/web#241", + }, + }); + expect(html).toContain('aria-label="Github Pull Request Requires auth: acme/web#241"'); + expect(html).not.toContain("Requires auth (Requires auth)"); + expect(html).not.toContain("(Requires auth)"); + }); + + it("does not repeat the liveness label when the status label equals it for unreachable", () => { + const html = render("Down: https://github.com/acme/web/pull/241", { + "https://github.com/acme/web/pull/241": { + providerKey: "github", + objectType: "pull_request", + displayKey: "Github Pull Request", + iconKey: "github", + statusCategory: "unknown", + liveness: "unreachable", + statusLabel: null, + displayTitle: "acme/web#241", + }, + }); + expect(html).toContain('aria-label="Github Pull Request Unreachable: acme/web#241"'); + expect(html).not.toContain("(Unreachable)"); + }); }); From 1ac48aff8dc13926539b085ca2e1d67913bcf7dc Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Sun, 11 Oct 2026 12:02:19 -0700 Subject: [PATCH 07/18] perf(ci): pack the verifier's staged packages concurrently (#15888) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Every pull request runs the `Build` check. Its last step is the public npm install verifier, `scripts/verify-grok-npm-install.mjs`, which stages the eighteen public packages, packs them, and installs them in a clean consumer. > - The verifier packs the eighteen packages one after another with `npm pack`. Each pack is a Node start-up plus gzip of that package's files. The timing line added in #15868 shows this loop at 35.6-37.9s on four `Build` runs, the largest part of the staging phase now that the bundled installs overlap. > - `Build` at 433-530s is the slowest check in most PR runs once the server shards drop, so its staging phase sets the wall-clock time of the run. > - Each pack reads only its own `package-` directory and writes one tarball. The runner has four cores and the packs are CPU-bound. There is no reason to run them one after the other. > - This pull request runs the packs as concurrent children through the prefetch helper, one per core, bundled packages first, each into its own directory, and puts the results back into dependency order before every tarball is checked. > - The benefit is a shorter `Build` job, and a timing line that reports the packs' serial cost next to their wall time, so the saving is measured on this PR's own run instead of estimated. ## Linked Issues or Issue Description Refs #15868, #15861. This branch is stacked on the branch of #15868, which is stacked on #15861, because the same execution branch holds both open PRs. The diff here is this PR's one commit. After #15868 merges, this PR is retargeted to `master` and rebased. **What existing behavior does this improve?** The `Build` PR check. Its public npm install verifier packs the eighteen public packages in one serial loop. **Subsystem affected** CI and release scripts (`scripts/verify-grok-npm-install.mjs`, `scripts/prefetch-child.mjs`). **Current behavior** After the bundled installs are joined, the verifier runs `npm pack --pack-destination ` for each of the eighteen packages in turn, finds the new tarball by listing the shared root, and checks it with `tar -tzf`. `stage_pack_ms` was 35.8s, 37.1s, 35.6s and 37.9s on the four runs of #15868. The parent is idle apart from the one pack it waits on, and three of the four cores are idle. **Proposed behavior** Start the packs as children through `startPrefetch`, at most one per core (`os.availableParallelism()`), bundled packages first because they make the largest tarballs. Each child packs into its own `pack-` directory, so the tarball a package produced is read from there instead of guessed from a shared directory. Register each child for cleanup as it starts. Put the results back into dependency order, then check every tarball with `tar -tzf` as before, then copy them into the assets directory as before. Report `stage_pack_check_ms`, `pack_concurrency`, `pack_serial_ms` and `pack_longest_ms` on the timing line, and keep `stage_pack_ms` as the whole step. **Reason and benefit** With four slots the step takes about the longest single pack or a quarter of the serial sum, whichever is larger. The expected saving is 20-25s of the 36-38s, measured on this PR's own `Build` job by `pack_serial_ms` against `stage_pack_ms`. **Measured on this PR's first run ([38146108750](https://github.com/paperclipai/paperclip/actions/runs/38146108750), `Build` 433s, verifier 214s):** `stage_pack_ms` 29.9s against 35.6-37.9s before, so the step shed 6-8s, not the 20-25s estimated. `pack_serial_ms` is 44.4s with `pack_concurrency` 4, and `pack_longest_ms` is 28.3s: one pack (the server with its 513 MB bundled closure, gzip level 9 inside npm) runs for almost the whole step, so the step is bounded by that pack and not by the slot count. The seventeen other packs fit inside it. The serial sum is 7-9s higher than the old serial step, which is the four processes contending for the runner's cores. `stage_packages_ms` 75.8s against 80.4-89.6s on #15868's runs; `image_wait_ms` and `grok_wait_ms` stay 0. **Breaking changes** None. The same packages are packed with the same `npm pack` arguments, the tarballs are checked the same way, and the consumer receives them in the same order. ## What Changed - `scripts/pack-staged-packages.mjs` (new): `packStagedPackages` launches the packs in the order given with at most `concurrency` children at once, through the prefetch helper, and resolves with each entry's tarball path and wall time in the same order. Every child is passed to `register` as it starts. The first failure rejects with the tail of that child's log and nothing further is launched. A child that leaves anything other than one tarball in its directory is a failure. `packConcurrency` is one per core. `packTimings` gives the timing-line keys. - `scripts/verify-grok-npm-install.mjs`: each entry gets its `index` and its own `pack-` destination. The pack loop is replaced by one `packStagedPackages` call, launched bundled-first and registered into the `prefetches` list as each child starts, so an earlier failure stops any pack still running before the scratch root is removed. The results are sorted back into dependency order, every tarball is checked in that order, and the copy into the assets directory is unchanged. Only the lifecycle sentinel is still packed in-process. - Timing line: `stage_pack_check_ms`, `pack_concurrency`, `pack_serial_ms` and `pack_longest_ms` are added. `stage_pack_ms` still covers the whole step, so the before and after numbers compare directly. - `scripts/__tests__/pack-staged-packages.test.mjs` (new): five tests with a local stand-in for npm and no network. The default concurrency and the pack arguments; four packs with two slots run the two slow packs together, start a third only once a slot is free, receive the exact `npm pack` argument list, register every child, and come back in entry order with their tarballs and timing keys although the fast packs finish first; a failed pack rejects with its log tail, launches nothing more, and leaves the running pack to be stopped; a pack that produces two tarballs or none is a failure; and no entries resolve at once. - `scripts/__tests__/grok-public-install-sandbox.test.mjs`: the guard test on the verifier source now checks the import, the bundled-first launch with cleanup registration, the reorder before the checks, the checks before the copy, that only the sentinel is packed in-process, that each pack has its own directory, and the new timing keys. - `package.json`: the new test file is in `test:release-registry`. ## Verification - `node --test scripts/__tests__/pack-staged-packages.test.mjs scripts/__tests__/grok-public-install-sandbox.test.mjs scripts/__tests__/stage-bundled-packages.test.mjs scripts/__tests__/prefetch-child.test.mjs`: 35 tests pass (30 on #15868's head plus the five new ones). The new file passes three consecutive runs. - `node --check` passes on both scripts. - With the real npm 11.12.1 on Node 25.9: two small packages packed through `packStagedPackages` with two slots produce one tarball each in their own `pack-` directory, with the expected `package/` listing from `tar -tzf`. - The verifier itself needs Docker and a network, which this workspace does not have. This PR's own `Build` job runs the branch's verifier; its timing line carries `stage_pack_ms`, `pack_serial_ms`, `pack_longest_ms` and `pack_concurrency`, which is the measurement. Before, on #15868's four runs: `stage_pack_ms` 35.8s / 37.1s / 35.6s / 37.9s. - `docker-runner-check.yml` runs the same script after `pnpm build` and needs no change. ## Risks - Four `npm pack` processes run at once and share the host npm configuration and cache. `npm pack --ignore-scripts` reads the package directory and writes one tarball; it does not write to the cache beyond its per-process log file. If a shared-state race appears, the failing pack's log tail is in the error and the fix is to lower the concurrency. - Memory and CPU: four Node processes instead of one on the four-core `Build` runner, for about thirty seconds. The largest pack (the server with its bundled dependencies) is the same process as before. - The pack output moves from the CI log into per-pack log files in the scratch root. A failure reports the last 2000 characters of the failing pack's log; a successful pack's output is discarded with the scratch root. - The tarball checks (`tar -tzf`) now run after all packs instead of after each pack. A bundled Codex payload is still rejected before any tarball reaches the consumer; the only change is that the error arrives a few seconds later. - Low risk otherwise. The bundled installs, the consumer install, and every probe are unchanged. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - Claude Fable 5.1 (Anthropic), model ID `claude-fable-5-1`, run as a Paperclip agent through the Claude Code local adapter with tool use (shell, file edits, GitHub CLI). Reasoning mode: extended thinking. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green (run 38146108750: `server (11/12)` failed on its first attempt on a teardown `EACCES` while removing the per-worker `PAPERCLIP_HOME` after all 74 files and 1257 tests had passed, in a file this PR does not touch; the re-run of that job passed and every check is green) - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: devinfoley <139239+devinfoley@users.noreply.github.com> Co-authored-by: Claude Fable 5.1 --- package.json | 2 +- .../grok-public-install-sandbox.test.mjs | 23 ++-- .../__tests__/pack-staged-packages.test.mjs | 107 ++++++++++++++++++ scripts/pack-staged-packages.mjs | 64 +++++++++++ scripts/verify-grok-npm-install.mjs | 31 +++-- 5 files changed, 207 insertions(+), 20 deletions(-) create mode 100644 scripts/__tests__/pack-staged-packages.test.mjs create mode 100644 scripts/pack-staged-packages.mjs diff --git a/package.json b/package.json index 29b999b28e2..1d8f7fb078a 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,7 @@ "smoke:posthog-live": "node scripts/smoke/posthog-live.mjs", "smoke:pipelines-tutorial": "./scripts/smoke/pipelines-tutorial-smoke.sh", "smoke:terminal-bench-loop-skill": "node scripts/smoke/terminal-bench-loop-skill-smoke.mjs", - "test:release-registry": "node --test scripts/__tests__/canary-startup-playwright.test.mjs scripts/__tests__/cursor-public-install-sandbox.test.mjs scripts/__tests__/grok-public-install-sandbox.test.mjs scripts/__tests__/prefetch-child.test.mjs scripts/__tests__/stage-bundled-packages.test.mjs scripts/__tests__/retain-runner-qualification-packages.test.mjs scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs scripts/cloud-migrator-artifacts.test.mjs", + "test:release-registry": "node --test scripts/__tests__/canary-startup-playwright.test.mjs scripts/__tests__/cursor-public-install-sandbox.test.mjs scripts/__tests__/grok-public-install-sandbox.test.mjs scripts/__tests__/prefetch-child.test.mjs scripts/__tests__/stage-bundled-packages.test.mjs scripts/__tests__/pack-staged-packages.test.mjs scripts/__tests__/retain-runner-qualification-packages.test.mjs scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs scripts/cloud-migrator-artifacts.test.mjs", "storybook-visual:baseline": "node scripts/storybook-visual-baseline.mjs", "test:storybook-visual": "node scripts/storybook-visual-baseline.mjs download && node scripts/storybook-visual-baseline.mjs verify && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts", "test:storybook-visual:update": "node scripts/storybook-visual-baseline.mjs download && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts --update-snapshots && node scripts/storybook-visual-baseline.mjs pack", diff --git a/scripts/__tests__/grok-public-install-sandbox.test.mjs b/scripts/__tests__/grok-public-install-sandbox.test.mjs index 11fde52433e..3cf43712a1a 100644 --- a/scripts/__tests__/grok-public-install-sandbox.test.mjs +++ b/scripts/__tests__/grok-public-install-sandbox.test.mjs @@ -120,19 +120,28 @@ test('the bundled-dependency installs run together while the other packages are // Only the bundled packages' sources are staged before the installs start; // the installs are registered for cleanup at once, before anything else can // fail; the other packages are copied while the installs run; the join comes - // before the first pack; and the pack loop walks every package in order. + // before the packs start; and the packs run through the helper covered by + // pack-staged-packages.test.mjs, bundled packages first, each registered for + // cleanup as it starts, with the results put back into dependency order + // before every tarball is checked. const bundledSources = position('for (const entry of entries.filter(entry => entry.bundled)) stageSource(entry)'); const start = position("startBundledInstalls(entries.filter(entry => entry.bundled), { logDir: prefetched, cwd: repo })"); const registered = position('prefetches.push(...installs.map(({ handle }) => handle))'); const otherSources = position('for (const entry of entries.filter(entry => !entry.bundled))'); const join = position("await awaited('bundled', Promise.all(installs.map(({ handle }) => handle.done)))"); - const packLoop = position('for (const { target } of entries)'); - const firstPack = position("run('npm', ['pack', '--ignore-scripts', '--pack-destination', root], target)"); - assert.ok(bundledSources < start && start < registered && registered < otherSources && otherSources < join && join < packLoop && packLoop < firstPack); - assert.equal(source.match(/run\('npm', \['pack'/g).length, 2, 'one package pack loop plus the lifecycle sentinel'); - // The timing line reports the split staging phase and each install. - for (const key of ['stage_sources_ms', 'stage_bundled_ms', 'stage_pack_ms']) assert.ok(source.includes(`timing.${key} =`)); + assert.ok(source.includes("import { packConcurrency, packStagedPackages, packTimings } from './pack-staged-packages.mjs'")); + const packs = position("await packStagedPackages([...entries].sort((a, b) => Number(b.bundled) - Number(a.bundled)), { logDir: prefetched, env, concurrency, register: handle => prefetches.push(handle) })"); + const reordered = position('packs.sort((a, b) => a.index - b.index)'); + const checks = position("for (const tarball of tarballs) assertNoBundledCodexPayloads(run('tar', ['-tzf', tarball])"); + const copied = position('for (const tarball of tarballs) {'); + assert.ok(bundledSources < start && start < registered && registered < otherSources && otherSources < join && join < packs && packs < reordered && reordered < checks && checks < copied); + assert.equal(source.match(/run\('npm', \['pack'/g).length, 1, 'only the lifecycle sentinel is packed in-process'); + assert.ok(source.includes("destination: join(root, `pack-${index}`)"), 'each pack writes into its own directory'); + // The timing line reports the split staging phase, each install, and the + // pack step with what the packs would have cost one after another. + for (const key of ['stage_sources_ms', 'stage_bundled_ms', 'stage_pack_ms', 'stage_pack_check_ms']) assert.ok(source.includes(`timing.${key} =`)); assert.ok(source.includes('...bundledInstallTimings(installs)')); + assert.ok(source.includes('Object.assign(timing, packTimings(packs, concurrency))')); }); test('the installed Codex probe exercises the public export and rejects incomplete or mismatched packages', () => { diff --git a/scripts/__tests__/pack-staged-packages.test.mjs b/scripts/__tests__/pack-staged-packages.test.mjs new file mode 100644 index 00000000000..09c19cf64aa --- /dev/null +++ b/scripts/__tests__/pack-staged-packages.test.mjs @@ -0,0 +1,107 @@ +import assert from 'node:assert/strict'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { stopPrefetches } from '../prefetch-child.mjs'; +import { PACK_ARGS, packConcurrency, packStagedPackages, packTimings } from '../pack-staged-packages.mjs'; + +// A small local script stands in for npm; no test here needs npm or a network. +// It is run as `node