Merge master into sleep blocker inventory

Preserve the explicit external database fixture and owned-client cleanup while adopting the upstream PostgreSQL shutdown timeout in both sleep suites.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-11-02 16:49:21 -06:00
commit 4a9974ea03
314 files changed
+10887 -2969

No files matched your search

@@ -4,7 +4,7 @@ import request from "supertest";
import { eq } from "drizzle-orm";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { agents, companies, createDb, heartbeatRunEvents, heartbeatRuns, issueComments, issues } from "@paperclipai/db";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { createLocalAgentJwt } from "../agent-auth-jwt.js";
import { actorMiddleware } from "../middleware/auth.js";
@@ -53,7 +53,7 @@ const support = await getEmbeddedPostgresTestSupport();
it("bounds actual middleware responses and preserves the complete stored run", async () => {
const fixture = await seed();
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
try {
const response = await request(fixture.app).get("/actor").set("Authorization", `Bearer ${fixture.token}`);
@@ -61,7 +61,7 @@ const support = await getEmbeddedPostgresTestSupport();
expect(response.body).toMatchObject({ type: "agent", runId: fixture.runId,
identityContextId: fixture.identity.id, onBehalfOfUserId: "owner" });
} finally { capture.flush(); }
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
&& event.table === "heartbeat_runs" && (event.operation === "select" || event.operation === "execute"));
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThanOrEqual(3);
expect(reads.reduce((sum, event) => sum + Number(event.responseValueBytes), 0)).toBeLessThan(16_384);
@@ -16,7 +16,7 @@ import { createHash, randomUUID } from "node:crypto";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { eq, sql } from "drizzle-orm";
import { agentFileStore, fileHash, inspectAgentFile, inspectAgentDirectory, snapshotAgentFiles, MAX_AGENT_FILE_BYTES, MAX_AGENT_DIRECTORY_BYTES, MAX_AGENT_DIRECTORY_ENTRIES } from "../services/agent-file-store.js";
import { agents, companies, authUsers, companyMemberships, principalPermissionGrants, heartbeatRuns, environmentLeases, environments, agentInstructionWorkingCopies, agentInstructionRevisions, agentInstructionHeads, createDb, fileStorageDb } from "@paperclipai/db";
import { agents, companies, issues, authUsers, companyMemberships, principalPermissionGrants, heartbeatRuns, environmentLeases, environments, agentInstructionWorkingCopies, agentInstructionRevisions, agentInstructionHeads, createDb, fileStorageDb } from "@paperclipai/db";
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { agentInstructionRevisionService } from "../services/agent-instruction-revisions.js";
import { agentInstructionWorkingCopyService, instructionWorkingCopyGuidance } from "../services/agent-instruction-working-copies.js";
@@ -24,6 +24,7 @@ import { resolveManagedInstructionsRoot } from "../services/agent-instructions.j
import { buildNativeRuntimeContext } from "../services/native-runtime/runtime-context.js";
import type { EnvironmentRuntimeService } from "../services/environment-runtime.js";
import { remoteTerminationReceipt } from "../services/remote-execution-termination.js";
import { createReusableLeaseHandoff } from "../services/environment-lease-handoff.js";
import { AgentDirectoryReuseInvalidatedError, agentDirectoryWorkingCopyService } from "../services/agent-directory-working-copies.js";
import { withDirectoryMergeLock } from "@paperclipai/adapter-utils/workspace-restore-merge";
import { heartbeatRunEvents } from "@paperclipai/db";
@@ -123,6 +124,44 @@ describe("persistent agent directories", () => {
for (const copy of [active, completed, transferred]) expect((await fs.stat(copy.localRoot)).isDirectory()).toBe(true);
} finally { await worker.stop(); }
});
it("retires a stale retry on an already cleaned receipt after restart without filesystem work", async () => {
const copy = await run();
await db.update(agentInstructionWorkingCopies).set({ state: "unavailable", processStoppedAt: new Date(),
nextAttemptAt: new Date(), receipt: { schema: "paperclip.agent-files.v1", cleanupPending: false } })
.where(eq(agentInstructionWorkingCopies.runId, copy.runId));
clock(); const worker = coordinator(), removes = vi.spyOn(fs, "rm");
try {
await agentInstructionWorkingCopyService(db).registerDirectoryCleanupWork(worker, () => true, async () => true).ready;
await vi.waitFor(async () => {
await vi.advanceTimersByTimeAsync(100);
expect((await copies.get(companyId, copy.runId))?.nextAttemptAt).toBeNull();
});
await expect.poll(() => idleWorkSnapshot().active).toBe(0);
const reads = vi.spyOn(db, "select");
await vi.advanceTimersByTimeAsync(120_000);
expect(reads).not.toHaveBeenCalled(); expect(removes).not.toHaveBeenCalled();
expect(worker.nextWakeAt()).toBeNull();
expect(await fs.readFile(path.join(root, entryFile), "utf8")).toBe(initial);
} finally { await worker.stop(); }
});
it("honors a cleanup obligation published before stale-deadline settlement", async () => {
const copy = await run(), dueAt = new Date(Date.now() + 30_000);
await db.update(agentInstructionWorkingCopies).set({ state: "unavailable", processStoppedAt: new Date(),
nextAttemptAt: dueAt, receipt: { schema: "paperclip.agent-files.v1", cleanupPending: false } })
.where(eq(agentInstructionWorkingCopies.runId, copy.runId));
const transact = db.transaction.bind(db);
const race = vi.spyOn(db, "transaction").mockImplementationOnce(async (callback, config) => {
await db.update(agentInstructionWorkingCopies).set({ receipt: { ...copy.receipt, cleanupPending: true } })
.where(eq(agentInstructionWorkingCopies.runId, copy.runId));
return transact(callback, config);
});
try {
await copies.release(companyId, copy.runId);
expect(await copies.get(companyId, copy.runId)).toMatchObject({ receipt: { cleanupPending: false }, nextAttemptAt: null });
await expect(fs.stat(copy.localRoot)).rejects.toMatchObject({ code: "ENOENT" });
expect(await fs.readFile(path.join(root, entryFile), "utf8")).toBe(initial);
} finally { race.mockRestore(); }
});
it("reactivates from real collection, preserves saved files, retries failed cleanup, and becomes quiet", async () => {
const copy = await run(); await fs.writeFile(path.join(copy.localRoot, "memory.txt"), "saved memory");
clock(); const worker = coordinator();
@@ -1461,6 +1500,53 @@ describe("persistent agent directories", () => {
return { copy: (await copies.get(companyId, copy.runId))!, lease, environmentId };
}
it.each(["destroyed", "stopped", "broken_handoff", "extra_owner", "foreign_company"] as const)(
"recovers an earlier remote copy only from the exact destroyed successor chain (%s)", async scenario => {
const { copy, lease } = await unavailableRemote();
const issueId = randomUUID(), successorRunId = randomUUID(), successorId = randomUUID();
const acquiredAt = new Date(Date.now() - 2000), handoffAt = new Date(Date.now() - 1000);
await db.insert(issues).values({ id: issueId, companyId, title: "Completed retained workspace", status: "done" });
await db.insert(heartbeatRuns).values({ id: successorRunId, companyId, agentId, invocationSource: "on_demand", status: "succeeded" });
const [previous] = await db.update(environmentLeases).set({ status: "expired", leasePolicy: "reuse_by_environment",
issueId, acquiredAt, releasedAt: handoffAt, cleanupStatus: "success", metadata: { agentId } })
.where(eq(environmentLeases.id, lease.id)).returning();
const successor = { ...previous!, id: successorId, heartbeatRunId: successorRunId, acquiredAt: handoffAt,
releasedAt: new Date(), metadata: { agentId } };
if (scenario === "foreign_company") {
const foreignCompanyId = randomUUID();
await db.insert(companies).values({ id: foreignCompanyId, name: "Unrelated owner", issuePrefix: randomUUID().slice(0, 8) });
successor.companyId = foreignCompanyId;
}
const handoff = createReusableLeaseHandoff(previous!, successor);
const termination = remoteTerminationReceipt(successor, { providerLeaseId: lease.providerLeaseId,
state: scenario === "stopped" ? "stopped" : "destroyed" });
await db.update(environmentLeases).set({ metadata: { agentId, reusableLeaseSuccessor: handoff } })
.where(eq(environmentLeases.id, lease.id));
await db.insert(environmentLeases).values({ ...successor, metadata: { agentId, remoteExecutionTermination: termination,
reusableLeasePredecessor: scenario === "broken_handoff" ? { ...handoff, handoffId: randomUUID() } : handoff } });
if (scenario === "extra_owner") await db.insert(environmentLeases).values({ ...lease, id: randomUUID(), status: "active" });
await db.update(agentInstructionWorkingCopies).set({ state: "pending_collection", attempts: 0, candidateHash: null, candidateBase64: null })
.where(eq(agentInstructionWorkingCopies.runId, copy.runId));
const shell = vi.spyOn(executionTargetTools, "runAdapterExecutionTargetShellCommand").mockRejectedValue(new Error("must not execute"));
try {
await copies.recoverStopped();
const recovered = (await copies.get(companyId, copy.runId))!;
if (scenario === "destroyed") {
expect(recovered).toMatchObject({ state: "unavailable", errorCode: "INSTRUCTION_COLLECTION_UNAVAILABLE",
receipt: { cleanupPending: false }, nextAttemptAt: null });
expect(recovered.processStoppedAt).toBeInstanceOf(Date);
await expect(fs.stat(copy.localRoot)).rejects.toMatchObject({ code: "ENOENT" });
} else {
expect(recovered).toMatchObject({ state: "pending_collection", processStoppedAt: null });
expect(await fs.readFile(path.join(copy.localRoot, entryFile), "utf8")).toBe(initial);
}
expect(shell).not.toHaveBeenCalled();
expect(await fs.readFile(path.join(root, entryFile), "utf8")).toBe(initial);
const [historical] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, lease.id));
expect(historical!.metadata).not.toHaveProperty("remoteExecutionTermination");
} finally { shell.mockRestore(); }
});
it.each([undefined, "stopped", "companyId", "runId", "leaseId", "provider", "providerLeaseId"])(
"defers unavailable cleanup without exact destruction proof (%s)", async proof => {
const { copy } = await unavailableRemote({ state: proof === undefined ? undefined : proof === "stopped" ? "stopped" : "destroyed",
@@ -1,6 +1,6 @@
import { randomUUID } from "node:crypto";
import { agents, companies, costEvents, createDb } from "@paperclipai/db";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import { eq } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
@@ -61,7 +61,7 @@ describe.skipIf(!support.supported)("agent roster read projections", () => {
async function observe<T>(run: (observed: ReturnType<typeof createDb>) => Promise<T>) {
const observed = createDb(database.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(observed, { agents, costEvents }, capture);
try {
const result = await run(observed);
@@ -1496,6 +1496,46 @@ console.log(JSON.stringify({ type: "result", subtype: "success", result: "hello"
} finally { fetchSpy.mockRestore(); }
});
it("reads provider identity only for an authorized credential and preserves connected state when profile access is denied", async () => {
const owner = "identity-owner";
await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" });
const account = await service.save(companyId, owner, { provider: "anthropic", method: "subscription", ownership: "personal", name: "Identity account", loginSessionId: "fixture", agentIds: [], allAgents: true }, "identity-selected-secret");
const fetchSpy = vi.spyOn(globalThis, "fetch").mockResolvedValue(Response.json({ account: { email: "avery@example.test", uuid: "private-account-id" }, organization: { name: "Example team", uuid: "private-org-id" } }));
const app = express();
app.use((req, _res, next) => {
req.actor = req.header("x-test-anonymous") ? { type: "none" }
: req.header("x-agent") ? { type: "agent", agentId, companyId, companyIds: [companyId], onBehalfOfUserId: owner }
: { type: "board", source: "session", userId: req.header("x-test-user") ?? owner, companyIds: [companyId] };
next();
});
app.use("/api", aiConnectionRoutes(db));
app.use((error: { status?: number; message: string }, _req: express.Request, res: express.Response, _next: express.NextFunction) => { res.status(error.status ?? 500).json({ error: error.message }); });
const url = `/api/companies/${companyId}/ai-connections/${account.connectionId}/identity?grantId=${account.grantId}`;
try {
const result = await request(app).get(url);
expect(result.status).toBe(200);
expect(result.headers["cache-control"]).toBe("no-store");
expect(result.body).toMatchObject({ ...account, status: "available", accountLabel: "avery@example.test", organizationLabel: "Example team" });
expect(fetchSpy).toHaveBeenCalledWith("https://api.anthropic.com/api/oauth/profile", expect.objectContaining({ headers: expect.objectContaining({ Authorization: "Bearer identity-selected-secret" }) }));
for (const value of ["identity-selected-secret", "private-account-id", "private-org-id"]) expect(JSON.stringify(result.body)).not.toContain(value);
fetchSpy.mockReset();
expect((await request(app).get(url).set("x-test-user", "bob")).status).toBe(404);
expect((await request(app).get(url.replace(companyId, otherCompanyId))).status).toBe(403);
expect((await request(app).get(url.replace(account.grantId, randomUUID()))).status).toBe(404);
expect((await request(app).get(url).set("x-test-anonymous", "yes")).status).toBe(401);
expect((await request(app).get(url).set("x-agent", "yes")).status).toBe(403);
expect((await request(app).get(url.replace(account.connectionId, "invalid"))).status).toBe(404);
expect(fetchSpy).not.toHaveBeenCalled();
fetchSpy.mockResolvedValue(Response.json({ error: "private-provider-error" }, { status: 403 }));
expect((await request(app).get(url)).body).toMatchObject({ status: "unavailable", errorCode: "permission_denied", accountLabel: null });
expect((await service.list(companyId, owner)).find(item => item.id === account.connectionId)?.status).toBe("connected");
fetchSpy.mockClear();
await db.update(connectionGrants).set({ status: "revoked" }).where(eq(connectionGrants.id, account.grantId));
expect((await request(app).get(url)).body).toMatchObject({ status: "unavailable", errorCode: "connection_unavailable" });
expect(fetchSpy).not.toHaveBeenCalled();
} finally { fetchSpy.mockRestore(); }
});
it("uses the same usage probe for shared accounts across legacy and native runner selections", async () => {
const account = await service.save(companyId, "alice", { provider: "anthropic", method: "subscription", ownership: "shared", name: "Shared usage account", loginSessionId: "fixture", agentIds: [], allAgents: true }, "shared-usage-secret");
const selected = { provider: "anthropic", method: "subscription", mode: "shared", ...account } as const;
@@ -4,7 +4,7 @@ import { randomUUID } from "node:crypto";
import express from "express";
import request from "supertest";
import { eq, sql } from "drizzle-orm";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
import {
agents,
@@ -1164,13 +1164,13 @@ describeEmbeddedPostgres("attention service", () => {
})));
}
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
try {
const feed = await attentionService(db).list(companyId, { userId: "board-user" });
expect(feed.items.filter(item => item.sourceKind === "failed_run").map(item => item.subject.id)).toEqual([failedId]);
} finally { capture.flush(); }
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
&& event.table === "heartbeat_runs" && (event.operation === "select" || event.operation === "execute"));
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThan(0);
expect(reads.reduce((sum, event) => sum + Number(event.rows), 0)).toBeLessThan(64);
@@ -1209,7 +1209,7 @@ describeEmbeddedPostgres("attention service", () => {
});
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
try {
const feed = await attentionService(db).list(companyId, {
@@ -1218,7 +1218,7 @@ describeEmbeddedPostgres("attention service", () => {
expect(feed.items.filter((item) => item.sourceKind === "failed_run").map((item) => item.subject.id).sort())
.toEqual([...keptIds].sort());
} finally { capture.flush(); }
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
&& event.table === "heartbeat_runs" && (event.operation === "select" || event.operation === "execute"));
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThan(0);
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeLessThan(4);
@@ -1407,7 +1407,7 @@ describeEmbeddedPostgres("attention service", () => {
const interactionItems = feed.items.filter((item) => item.sourceKind === "issue_thread_interaction");
const detailsByKind = new Map(interactionItems.map((item) => [item.detail?.kind, item]));
const summaryEvents: Array<Record<string, unknown>> = [], captureNow = Date.now();
const summaryCapture = createDatabaseResponseCapture({ start: captureNow - 100, end: captureNow + 59_000 }, event => summaryEvents.push(event));
const summaryCapture = createFullDatabaseResponseCapture({ start: captureNow - 100, end: captureNow + 59_000 }, event => summaryEvents.push(event));
installDatabaseResponseCapture(db, { documents, assets, issueAttachments }, summaryCapture);
try {
const summary = await attentionService(db).summary(companyId, { userId: "board-user" });
@@ -1418,7 +1418,7 @@ describeEmbeddedPostgres("attention service", () => {
expect(index.totalCount).toBe(feed.totalCount);
} finally { summaryCapture.flush(); }
const cardReads = summaryEvents.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
const cardReads = summaryEvents.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
&& ["documents", "assets", "issue_attachments"].includes(String(event.table)));
expect(cardReads).toEqual([]);
@@ -33,11 +33,13 @@ import { budgetService } from "../services/budgets.js";
import { idleWorkSnapshot } from "../services/task-admission.js";
import { readFreshWorkspaceReadiness } from "../services/fresh-workspace-continuation.js";
import { requestTaskRetry, reconcileTaskChatRetryReceipts } from "../services/task-retry.js";
import { taskInterruptionPresentation } from "../services/task-interruption-presentation.js";
import { withQueuedCommentIdsInWakePayload } from "../services/issue-queued-comment-queue.js";
import { createSystemRecovery } from "../services/system-recovery-state.js";
import { remoteTerminationReceipt } from "../services/remote-execution-termination.js";
import { stageReceiptReactionRemovals } from "../services/chat-receipt-reactions.js";
import type { Db } from "@paperclipai/db";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import { githubResponseAuthorityFields } from "../services/chat-channels/github/response-authority-read.js";
import * as activityLogModule from "../services/activity-log.js";
import * as connectionCredentialBindings from "../services/connection-credential-bindings.js";
@@ -112,6 +114,7 @@ import {
isNotNull,
isNull,
like,
ne,
notInArray,
or,
sql,
@@ -2887,7 +2890,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
const normalizedEvent = { ...delivery.normalizedEvent, unusedContent: bulk };
await db.update(chatDeliveries).set({ normalizedEvent }).where(eq(chatDeliveries.id, delivery.id));
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { chatMessageLinks, chatDeliveries, chatEndpoints, chatConversations,
chatEndpointResources, chatGitHubConfigurations, chatExternalPrincipals, chatIdentityLinks,
companyMemberships, chatActions, heartbeatRuns }, capture);
@@ -2916,7 +2919,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
} };
await db.update(chatDeliveries).set({ normalizedEvent }).where(eq(chatDeliveries.id, delivery.id));
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { chatDeliveries }, capture);
let projected: { event: Record<string, unknown> };
try {
@@ -7152,6 +7155,68 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
expect(await readChatConversationsByEndpointId(f.endpoint.id)).toHaveLength(0);
expect(await readChatGitHubReviewsByEndpointId(f.endpoint.id)).toHaveLength(0);
});
it("offers Retry for a saved automatic PR event after a successful older review", async () => {
const f = await reviewBotFixture();
const payload = {
action: "opened", installation: { id: 2468 },
repository: { id: 97531, full_name: "paperclipai/paperclip", name: "paperclip", owner: { id: 1357, login: "paperclipai" } },
sender: { id: 77, login: "maintainer" },
pull_request: { number: 81, title: "Review successive heads", body: "", draft: false,
base: { sha: "a".repeat(40), ref: "master" }, head: { sha: "b".repeat(40) },
user: { id: 42, login: "octocat", type: "User" }, labels: [] },
};
const send = async (event: typeof payload) => {
await f.service.handleWebhook(f.endpoint.publicId, "github", signedGitHubWebhookRequest({
event: "pull_request", delivery: randomUUID(), payload: event, webhookSecret: f.webhookSecret,
}));
await f.service.processPendingDeliveries();
};
await send(payload);
await expect.poll(async () => {
await f.service.processPendingDeliveries();
return (await readChatConversationsByEndpointId(f.endpoint.id)).length;
}, { timeout: 10_000 }).toBe(1);
const [conversation] = await readChatConversationsByEndpointId(f.endpoint.id);
const [task] = await readIssuesById(conversation.issueId!);
const [original] = await db.select().from(chatActions).where(and(
eq(chatActions.conversationId, conversation.id), eq(chatActions.kind, "inbound_wakeup")));
await expect.poll(async () => (await readAgentWakeupRequestsById(original.id)).length).toBe(1);
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.assignedAgentId,
issueId: task.id, status: "succeeded", finishedAt: new Date(), wakeupRequestId: original.id,
contextSnapshot: { issueId: task.id, taskKey: task.identifier, source: "chat:github", wakeCommentIds: [original.payload.commentId] },
}).returning();
await db.update(agentWakeupRequests).set({ status: "completed", runId: run.id }).where(eq(agentWakeupRequests.id, original.id));
await send({ ...payload, action: "synchronize", pull_request: { ...payload.pull_request, head: { sha: "c".repeat(40) } } });
await expect.poll(async () => {
await f.service.processPendingDeliveries();
return (await db.select().from(chatActions).where(and(
eq(chatActions.conversationId, conversation.id), eq(chatActions.kind, "inbound_wakeup")))).length;
}).toBe(2);
const [pending] = await db.select().from(chatActions).where(and(
eq(chatActions.conversationId, conversation.id), eq(chatActions.kind, "inbound_wakeup"), ne(chatActions.id, original.id)));
await expect.poll(async () => (await readAgentWakeupRequestsById(pending.id)).length).toBe(1);
const [receipt] = await readAgentWakeupRequestsById(pending.id);
await db.update(agentWakeupRequests).set({ status: "deferred_issue_execution", runId: null,
payload: withQueuedCommentIdsInWakePayload(receipt.payload, [String(pending.payload.commentId)]),
requestedAt: new Date(Date.now() - 60_000) }).where(eq(agentWakeupRequests.id, pending.id));
expect(receipt.requestedByActorType).toBe("user");
await db.update(issues).set({ status: "blocked", executionRunId: null }).where(eq(issues.id, task.id));
expect(await taskInterruptionPresentation(db, { ...task, status: "blocked" }))
.toMatchObject({ state: "failed", canRetry: true, intentId: pending.id, observedRunId: run.id });
await db.update(agents).set({ status: "running", runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 } } })
.where(eq(agents.id, f.assignedAgentId));
await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.assignedAgentId, status: "running", startedAt: new Date() });
f.wakeup.mockImplementation(heartbeatService(db).wakeup);
const retry = await requestTaskRetry({ db, companyId: f.companyId, issueId: task.id, actorId: "owner-user",
actorSource: "cloud_tenant", observedRunId: run.id, idempotencyKey: randomUUID(), prepareChatRetry: f.service.prepareFailedChatRunRetry });
await reconcileTaskChatRetryReceipts(db, { companyId: f.companyId });
const [resumed] = await readAgentWakeupRequestsById(pending.id);
expect(resumed.runId).toBeTruthy();
expect((await readHeartbeatRunsById(resumed.runId!))[0]).toMatchObject({ status: "queued", retryOfRunId: null,
contextSnapshot: { wakeCommentIds: [pending.payload.commentId] } });
expect((await readAgentWakeupRequestsById(retry.requestId!))[0]).toMatchObject({ status: "coalesced", runId: resumed.runId });
});
it("admits signed PR events into an ordinary assigned task and deduplicates delivery", async () => {
const f = await reviewBotFixture();
const { buildSkillMentionHref } = await import("@paperclipai/shared");
@@ -62866,7 +62931,8 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
const commentsBeforeDisabledLifecycle = await db
.select({ id: issueComments.id })
.from(issueComments)
.where(eq(issueComments.issueId, conversation.issueId));
.where(eq(issueComments.issueId, conversation.issueId))
.orderBy(issueComments.id);
const priorLifecycleIds = new Set(
(
await db
@@ -62946,7 +63012,8 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
db
.select({ id: issueComments.id })
.from(issueComments)
.where(eq(issueComments.issueId, conversation.issueId)),
.where(eq(issueComments.issueId, conversation.issueId))
.orderBy(issueComments.id),
).resolves.toEqual(commentsBeforeDisabledLifecycle);
await service.shutdown();
});
@@ -64140,6 +64207,163 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
return { ...f, pending, originalDelivery };
}
async function successfulGitHubQueuedInputFixture() {
const f = await currentGitHubTaskRetryFixture(true);
const companyId = f.fixture.companyId, issueId = f.issue.id;
const [saved] = await readAgentWakeupRequestsById(f.pending!.id);
await db.update(agentWakeupRequests).set({ payload: withQueuedCommentIdsInWakePayload(saved.payload,
[String(f.pending!.payload.commentId)]) }).where(eq(agentWakeupRequests.id, saved.id));
const contractId = randomUUID(), resultId = randomUUID();
await db.insert(completionContracts).values({ id: contractId, companyId, issueId, revision: 1,
schemaVersion: "paperclip.completion-contract.v1", policyVersion: "test", risk: "standard",
completionAuthority: "server_arbiter", incompleteCriteriaPolicy: "preserve_non_terminal", contractJson: {},
canonicalSha256: contractId, createdByActorType: "system", createdByActorId: "test" });
await db.update(heartbeatRuns).set({ status: "succeeded", errorCode: null, nativePhase: "committed",
completionContractId: contractId }).where(eq(heartbeatRuns.id, f.runId));
const [result] = await db.insert(nativeRunResults).values({ id: resultId, companyId, issueId, runId: f.runId,
completionContractId: contractId, serverFingerprint: resultId, schemaStatus: "accepted", canonicalSha256: resultId,
resultJson: { terminal: { runTerminalState: "succeeded" }, summary: "Earlier review remains saved" } }).returning();
await db.update(nativeRunFinalizations).set({ phase: "committed", resultId }).where(eq(nativeRunFinalizations.runId, f.runId));
await db.update(agentWakeupRequests).set({ status: "completed" }).where(eq(agentWakeupRequests.id, f.receipt.id));
const input = { db, companyId, issueId, actorId: "owner-user", actorSource: "cloud_tenant",
observedRunId: f.runId, idempotencyKey: randomUUID(), prepareChatRetry: f.service.prepareFailedChatRunRetry };
return { ...f, input, result };
}
it("retries saved GitHub input after a successful run without replaying that run", async () => {
const f = await successfulGitHubQueuedInputFixture();
try {
const { input } = f;
const companyId = f.fixture.companyId;
const before = (await readHeartbeatRunsById(f.runId))[0];
const [request, duplicate] = await Promise.all([requestTaskRetry(input),
requestTaskRetry({ ...input, idempotencyKey: randomUUID() })]);
expect(request.accepted).toBe(true);
expect(duplicate.requestId).toBe(request.requestId);
await reconcileTaskChatRetryReceipts(db, { companyId });
const [source] = await readAgentWakeupRequestsById(f.pending!.id);
expect(source.runId).toBeTruthy();
const [successor] = await readHeartbeatRunsById(source.runId!);
expect(successor).toMatchObject({ status: "queued", retryOfRunId: null,
contextSnapshot: { wakeCommentIds: [f.pending!.payload.commentId] } });
await reconcileTaskChatRetryReceipts(db, { companyId });
expect((await readAgentWakeupRequestsById(request.requestId!))[0])
.toMatchObject({ status: "coalesced", runId: successor.id });
expect((await readHeartbeatRunsByCompanyId(companyId)).filter(run => run.wakeupRequestId === f.pending!.id)).toHaveLength(1);
expect((await readHeartbeatRunsById(f.runId))[0]).toEqual(before);
expect((await db.select().from(nativeRunResults).where(eq(nativeRunResults.id, f.result.id)))[0]).toEqual(f.result);
} finally { await f.service.shutdown(); }
});
it.each(["connection_revoked", "source_rebound", "task_completed", "agent_paused", "cleanup_pending"] as const)(
"keeps saved-input Retry behind current admission gates: %s", async gate => {
const f = await successfulGitHubQueuedInputFixture();
try {
const companyId = f.fixture.companyId;
const request = await requestTaskRetry(f.input);
const before = (await readHeartbeatRunsById(f.runId))[0];
if (gate === "connection_revoked") await db.update(chatEndpoints).set({ status: "paused" }).where(eq(chatEndpoints.id, f.endpoint.id));
if (gate === "source_rebound") await db.update(chatActions).set({ payload: { ...f.pending!.payload, commentId: randomUUID() } })
.where(eq(chatActions.id, f.pending!.id));
if (gate === "task_completed") await db.update(issues).set({ status: "done" }).where(eq(issues.id, f.issue.id));
if (gate === "agent_paused") await db.update(agents).set({ status: "paused" }).where(eq(agents.id, f.fixture.assignedAgentId));
const lease = { id: randomUUID(), companyId, issueId: f.issue.id, heartbeatRunId: f.runId,
provider: "daytona", providerLeaseId: randomUUID(), leasePolicy: "ephemeral" };
if (gate === "cleanup_pending") {
await db.insert(environmentLeases).values({ ...lease, status: "pending_cleanup" });
await db.update(issues).set({ executionRunId: f.runId }).where(eq(issues.id, f.issue.id));
}
await reconcileTaskChatRetryReceipts(db, { companyId });
expect((await readAgentWakeupRequestsById(f.pending!.id))[0].runId).toBeNull();
expect((await readHeartbeatRunsByCompanyId(companyId)).filter(run => run.wakeupRequestId === f.pending!.id)).toHaveLength(0);
expect((await readHeartbeatRunsById(f.runId))[0]).toEqual(before);
expect((await db.select().from(nativeRunResults).where(eq(nativeRunResults.id, f.result.id)))[0]).toEqual(f.result);
if (gate === "cleanup_pending") {
expect((await readAgentWakeupRequestsById(request.requestId!))[0].status).toBe("deferred_issue_execution");
await db.update(environmentLeases).set({ status: "released", cleanupStatus: "success", releasedAt: new Date(),
metadata: { remoteExecutionTermination: remoteTerminationReceipt(lease, { providerLeaseId: lease.providerLeaseId, state: "destroyed" }) },
}).where(eq(environmentLeases.id, lease.id));
await reconcileTaskChatRetryReceipts(db, { companyId });
expect((await readAgentWakeupRequestsById(f.pending!.id))[0].runId).toBeTruthy();
}
} finally { await f.service.shutdown(); }
});
it.each(["issue.comment.reopen", "boardMessageRetry"] as const)(
"resumes saved GitHub input after a successful board turn: %s", async origin => {
const f = await successfulGitHubQueuedInputFixture();
try {
const [previous] = await readHeartbeatRunsById(f.runId);
await db.update(heartbeatRuns).set({ responsibleUserId: f.input.actorId, contextSnapshot: {
...previous.contextSnapshot, ...(origin === "boardMessageRetry" ? { boardMessageRetry: true } : { source: origin }),
} }).where(eq(heartbeatRuns.id, f.runId));
const before = (await readHeartbeatRunsById(f.runId))[0];
const retry = await requestTaskRetry(f.input);
await reconcileTaskChatRetryReceipts(db, { companyId: f.fixture.companyId, requestId: retry.requestId! });
const [saved] = await readAgentWakeupRequestsById(f.pending!.id);
expect(saved.runId).toBeTruthy();
expect((await readHeartbeatRunsById(saved.runId!))[0]).toMatchObject({ retryOfRunId: null,
contextSnapshot: { source: "chat:github", wakeCommentIds: [f.pending!.payload.commentId] } });
expect((await readHeartbeatRunsById(f.runId))[0]).toEqual(before);
} finally { await f.service.shutdown(); }
});
it("keeps a board message queued separately when saved GitHub Retry resumes after cleanup", async () => {
const f = await successfulGitHubQueuedInputFixture();
try {
const companyId = f.fixture.companyId;
const lease = { id: randomUUID(), companyId, issueId: f.issue.id, heartbeatRunId: f.runId,
provider: "daytona", providerLeaseId: randomUUID(), leasePolicy: "ephemeral" };
await db.insert(environmentLeases).values({ ...lease, status: "pending_cleanup" });
await db.update(issues).set({ status: "in_review", executionRunId: f.runId }).where(eq(issues.id, f.issue.id));
const retry = await requestTaskRetry(f.input);
await reconcileTaskChatRetryReceipts(db, { companyId, requestId: retry.requestId! });
expect((await readAgentWakeupRequestsById(f.pending!.id))[0].runId).toBeNull();
const [comment] = await db.insert(issueComments).values({ companyId, issueId: f.issue.id,
authorType: "user", authorUserId: f.input.actorId, body: "A separate board follow-up" }).returning();
const [board] = await db.insert(agentWakeupRequests).values({ companyId, agentId: f.fixture.assignedAgentId,
source: "automation", triggerDetail: "system", reason: "issue_commented", status: "deferred_issue_execution",
requestedByActorType: "user", requestedByActorId: f.input.actorId,
payload: withQueuedCommentIdsInWakePayload({ issueId: f.issue.id, commentId: comment.id,
_paperclipWakeContext: { issueId: f.issue.id, source: "issue.comment.reopen", wakeReason: "issue_commented" },
}, [comment.id]),
}).returning();
await db.update(environmentLeases).set({ status: "released", cleanupStatus: "success", releasedAt: new Date(),
metadata: { remoteExecutionTermination: remoteTerminationReceipt(lease, { providerLeaseId: lease.providerLeaseId, state: "destroyed" }) },
}).where(eq(environmentLeases.id, lease.id));
await reconcileTaskChatRetryReceipts(db, { companyId, requestId: retry.requestId! });
const [saved] = await readAgentWakeupRequestsById(f.pending!.id);
expect(saved.runId).toBeTruthy();
expect((await readHeartbeatRunsById(saved.runId!))[0]).toMatchObject({ status: "queued",
contextSnapshot: { wakeCommentIds: [f.pending!.payload.commentId] } });
expect((await readAgentWakeupRequestsById(board.id))[0]).toEqual(board);
} finally { await f.service.shutdown(); }
});
it("dispatches saved input through the task Retry HTTP endpoint and survives a repeated click", async () => {
const f = await successfulGitHubQueuedInputFixture();
const { issueRoutes } = await import("../routes/issues.js");
const app = express();
app.use(express.json());
app.use((req, _res, next) => { req.actor = boardActor(f.fixture.companyId); next(); });
app.use("/api", issueRoutes(db, createStorageService().storage, { chatRunRetries: f.service }));
app.use(errorHandler);
try {
const body = { observedRunId: f.runId, idempotencyKey: randomUUID() };
const first = await request(app).post(`/api/issues/${f.issue.id}/retry`).send(body);
expect(first.status, JSON.stringify(first.body)).toBe(202);
await expect.poll(async () => (await readAgentWakeupRequestsById(f.pending!.id))[0].runId).toBeTruthy();
const second = await request(app).post(`/api/issues/${f.issue.id}/retry`).send(body);
expect(second.status).toBe(202);
expect(second.body.requestId).toBe(first.body.requestId);
expect((await readHeartbeatRunsByCompanyId(f.fixture.companyId)).filter(run => run.wakeupRequestId === f.pending!.id)).toHaveLength(1);
const detail = await request(app).get(`/api/issues/${f.issue.id}`);
expect(detail.status).toBe(200);
expect(detail.body.interruption).toMatchObject({ state: "working", canRetry: false });
expect((await db.select().from(nativeRunResults).where(eq(nativeRunResults.id, f.result.id)))[0]).toEqual(f.result);
} finally { await f.service.shutdown(); }
});
it.each(["reconnected", "newer_input", "reconnected_newer_input", "snapshot_newer_input"] as const)(
"admits current GitHub task Retry without replaying obsolete input: %s", async kind => {
const f = await currentGitHubTaskRetryFixture(kind.includes("newer_input"));
@@ -64269,7 +64493,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
status: "active", membershipRole: "operator" });
}
const queryEvents: Record<string, unknown>[] = [];
const capture = mode === "backlog_51" ? createDatabaseResponseCapture({ start: Date.now() - 1,
const capture = mode === "backlog_51" ? createFullDatabaseResponseCapture({ start: Date.now() - 1,
end: Date.now() + 59_000 }, event => queryEvents.push(event)) : undefined;
if (capture) installDatabaseResponseCapture(db, { chatActions, chatDeliveries, chatMessageLinks, issueComments,
agentWakeupRequests, chatEndpoints, chatConversations }, capture);
@@ -64289,7 +64513,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
idempotencyKey: mode === "concurrent_clicks" ? randomUUID() : input.idempotencyKey })]);
if (capture) {
capture.flush();
expect(Number(queryEvents.find(event => event.event === "paperclip.db_response_capture.summary.v1")?.startedQueries))
expect(Number(queryEvents.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")?.startedQueries))
.toBeLessThan(180);
}
expect(two.requestId).toBe(one.requestId);
@@ -67489,7 +67713,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
})));
const scope = { companyId: context.fixture.companyId, issueId: context.issue.id };
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { chatActions, agentWakeupRequests }, capture);
let scopes: Array<{ companyId: string; issueId: string }>;
try {
@@ -67506,7 +67730,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
}
expect(events.at(-1)).toMatchObject({ startedQueries: mode === "task" ? 0 : 1,
completedQueries: mode === "task" ? 0 : 1, inFlightAtEnd: 0 });
expect(events.filter(event => event.event === "paperclip.db_response_capture.entry.v1")
expect(events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1")
.reduce((count, event) => count + Number(event.rows), 0)).toBe(mode === "task" ? 0 : scopes.length);
} finally {
await context.service.shutdown(); await other.service.shutdown();
@@ -77214,7 +77438,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
})));
const scope = { companyId: context.fixture.companyId, issueId: context.conversation.issueId };
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { chatActions, agentWakeupRequests }, capture);
let scopes: Array<{ companyId: string; issueId: string }>;
try {
@@ -77231,7 +77455,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
}
expect(events.at(-1)).toMatchObject({ startedQueries: mode === "task" ? 0 : 1,
completedQueries: mode === "task" ? 0 : 1, inFlightAtEnd: 0 });
expect(events.filter(event => event.event === "paperclip.db_response_capture.entry.v1")
expect(events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1")
.reduce((count, event) => count + Number(event.rows), 0)).toBe(mode === "task" ? 0 : scopes.length);
} finally {
await context.service.shutdown(); await other.service.shutdown();
@@ -4,7 +4,7 @@ import { randomUUID } from "node:crypto";
import { and, asc, eq } from "drizzle-orm";
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { agents, agentWakeupRequests, chatCompletionDeliveries as deliveries, chatTaskHandoffs as handoffs,
companies, createDb, documents, environmentLeases, heartbeatRuns, issueComments, issueRecoveryActions, issueThreadInteractions, issues } from "@paperclipai/db";
companies, createDb, documents, environmentLeases, executionWorkspaces, heartbeatRuns, issueComments, issueRecoveryActions, issueThreadInteractions, issues, workspaceOperations } from "@paperclipai/db";
import { recoveryPostgresSupport as support, startRecoveryPostgres } from "./helpers/recovery-postgres.js";
import { settleInterruptedNativeBootstrap, terminalizeLegacyExecution } from "../services/legacy-execution-recovery.js";
import { getExecutionBlocker } from "../services/execution-blocker.js";
@@ -15,6 +15,8 @@ import { documentService } from "../services/documents.js";
import { instanceSettingsService } from "../services/instance-settings.js";
import { chatCompletionDeliveryService, isCompletedOnboardingHandoffWake, prepareChatCompletionTurn, recordChatCompletion, recordChatHandoff } from "../services/chat-completion-delivery.js";
import { heartbeatService, shouldQueueFollowupForRunningIssueWake } from "../services/heartbeat.js";
import { createHeartbeatQueue } from "../services/heartbeat/queue.js";
import { createHeartbeatQueueTestDependencies } from "./helpers/heartbeat-queue.js";
(support.supported ? describe : describe.skip)("chat completion delivery", () => {
let db: ReturnType<typeof createDb>;
@@ -68,6 +70,75 @@ import { heartbeatService, shouldQueueFollowupForRunningIssueWake } from "../ser
})]);
expect(JSON.stringify(run.contextSnapshot?.chatCompletionUpdates)).not.toMatch(/Injected|credentials|Plan accepted|Agent approved/);
});
it("retains a distinct completion receipt behind a real dependency gate", async () => {
const f = await seed();
const blocker = await f.create();
await issueService(db).update(f.sourceId, { status: "blocked", blockedByIssueIds: [blocker.id] });
await f.finish();
const heartbeat = heartbeatService(db);
// The generic dependency signal is replaceable. The outbox's exact wake
// receipt is not: it needs that identity to reconcile a refused admission.
await heartbeat.wakeup(f.agentId, {
source: "automation", triggerDetail: "system", reason: "issue_continuation_needed",
requestedByActorType: "system", requestedByActorId: "dependency-test",
payload: { issueId: f.sourceId }, contextSnapshot: { issueId: f.sourceId },
});
const [delivery] = await f.rows();
const service = chatCompletionDeliveryService(db, heartbeat);
await service.deliver(delivery.id);
const receipts = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId));
expect(receipts).toEqual(expect.arrayContaining([
expect.objectContaining({ idempotencyKey: `chat-completion:${delivery.id}:0`,
reason: "issue_dependencies_blocked", status: "skipped", runId: null,
payload: expect.objectContaining({ chatCompletionDeliveryIds: [delivery.id] }) }),
]));
expect(receipts).toHaveLength(2);
expect(receipts.find(receipt => receipt.idempotencyKey?.startsWith("execution-wait:")))
.toMatchObject({ coalescedCount: 0 });
expect((await f.rows())[0]).toMatchObject({ attempts: 0, status: "pending", targetRunId: null, error: null });
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(1);
expect(await issueService(db).getDependencyReadiness(f.sourceId)).toMatchObject({
isDependencyReady: false, unresolvedBlockerIssueIds: [blocker.id],
});
});
it("admits a saved worker completion after its owning finalize encloses the snapshot", async () => {
const f = await seed(), workspaceId = randomUUID(), workerRunId = randomUUID();
await db.update(issues).set({ responsibleUserId: "operator" }).where(eq(issues.id, f.sourceId));
await db.insert(executionWorkspaces).values({ id: workspaceId, companyId: f.companyId,
mode: "isolated_workspace", strategyType: "managed_remote", name: "Saved worker", providerType: "daytona" });
await db.insert(heartbeatRuns).values({ id: workerRunId, companyId: f.companyId, agentId: f.agentId,
status: "succeeded", contextSnapshot: { issueId: f.task.id } });
await db.update(issues).set({ executionWorkspaceId: workspaceId }).where(eq(issues.id, f.task.id));
await issueService(db).update(f.sourceId, { status: "blocked", blockedByIssueIds: [f.task.id] });
await db.insert(workspaceOperations).values([
{ companyId: f.companyId, issueId: f.task.id, executionWorkspaceId: workspaceId, heartbeatRunId: workerRunId,
phase: "workspace_finalize", status: "succeeded", startedAt: new Date(10_000), finishedAt: new Date(12_000) },
{ companyId: f.companyId, issueId: f.task.id, executionWorkspaceId: workspaceId, heartbeatRunId: workerRunId,
phase: "remote_workspace_snapshot", status: "succeeded", startedAt: new Date(10_100), finishedAt: new Date(11_900) },
]);
await f.finish();
const deps = createHeartbeatQueueTestDependencies(db), issueSvc = issueService(db);
deps.getAgent.mockImplementation(id => db.select().from(agents).where(eq(agents.id, id)).then(rows => rows[0] ?? null));
deps.instanceSettings.getExperimental.mockImplementation(() => instanceSettingsService(db).getExperimental());
deps.issuesSvc.getDependencyReadiness.mockImplementation(issueSvc.getDependencyReadiness);
deps.issuesSvc.listDependencyReadiness.mockImplementation(issueSvc.listDependencyReadiness);
// Admit the real durable run, then hold provider dispatch at the normal drain boundary.
deps.getSchedulingSuppression.mockResolvedValue({ suppressed: true, reason: "task_drain" });
const queue = createHeartbeatQueue(db, deps);
const service = chatCompletionDeliveryService(db, { wakeup: queue.trackWakeup });
const [delivery] = await f.rows();
await service.deliver(delivery.id);
const [queued] = await f.rows();
expect(queued).toMatchObject({ status: "queued", attempts: 0, error: null, targetRunId: expect.any(String) });
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, queued.targetRunId!));
expect(run).toMatchObject({ status: "queued", contextSnapshot: { issueId: f.sourceId,
wakeReason: "chat_task_completed", chatCompletionDeliveryIds: [delivery.id] } });
const prepared = await prepareChatCompletionTurn(db, run);
expect(prepared.contextSnapshot?.chatCompletionUpdates).toEqual([
expect.objectContaining({ id: f.task.id, status: "done", completedAt: expect.any(Date), url: `/issues/${f.task.identifier ?? f.task.id}` }),
]);
expect(deps.executeRun).not.toHaveBeenCalled();
});
it("dispatches a committed completion without activity publication and recovers after restart", async () => {
const f = await seed();
const run = vi.fn(() => f.service.sweepPending());
@@ -17,7 +17,7 @@ import {
projectWorkspaces,
} from "@paperclipai/db";
import { parseFrontmatterMarkdown } from "@paperclipai/shared";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import {
getEmbeddedPostgresTestSupport,
startEmbeddedPostgresTestDatabase,
@@ -651,17 +651,17 @@ describeEmbeddedPostgres("companySkillService.list", () => {
]);
await svc.list(companyId);
const events: Record<string, unknown>[] = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { agents, costEvents }, capture);
let listed: Awaited<ReturnType<typeof svc.list>>;
try { listed = await svc.list(companyId); } finally { capture.flush(); }
expect(listed.find(item => item.id === first.id)?.attachedAgentCount).toBe(2);
expect(listed.find(item => item.id === second.id)?.attachedAgentCount).toBe(1);
expect(listed.find(item => item.id === ambiguous.id)?.attachedAgentCount).toBe(0);
const agentReads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" && event.table === "agents");
const agentReads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "agents");
expect(agentReads).toHaveLength(1);
expect(Number(agentReads[0]?.responseValueBytes)).toBeLessThan(2_000);
expect(events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" && event.table === "cost_events")).toHaveLength(0);
expect(events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "cost_events")).toHaveLength(0);
await db.update(agents).set({ adapterConfig: { paperclipSkillSync: { desiredSkills: [second.key] } } }).where(eq(agents.id, readerId));
const refreshed = await svc.list(companyId, { q: "First" });
expect(refreshed.find(item => item.id === first.id)?.attachedAgentCount).toBe(1);
@@ -680,12 +680,12 @@ describeEmbeddedPostgres("companySkillService.list", () => {
const before = await svc.list(companyId);
expect(before.find(item => item.id === inserted[0]!.id)?.folderPath).toBe("bundled/paperclip-core");
const events: Record<string, unknown>[] = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { companySkills }, capture);
let after: Awaited<ReturnType<typeof svc.list>>;
try { after = await svc.list(companyId); } finally { capture.flush(); }
expect(after).toEqual(before);
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" && event.table === "company_skills");
const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "company_skills");
expect(reads.reduce((sum, event) => sum + Number(event.responseValueBytes), 0)).toBeLessThan(950_000);
const stored = await svc.getById(companyId, inserted[0]!.id);
expect(stored?.markdown).toBe(savedMarkdown);
@@ -762,11 +762,11 @@ describeEmbeddedPostgres("companySkillService.list", () => {
.orderBy(companySkillVersions.id);
expect(before.length).toBeGreaterThan(0);
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { companySkillVersions }, capture);
try { expect(await companySkillService(db).list(companyId)).toEqual(expected); }
finally { capture.flush(); }
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
&& event.table === "company_skill_versions" && event.operation === "select");
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThan(0);
expect(reads.reduce((sum, event) => sum + Number(event.responseValueBytes), 0)).toBeLessThan(4096);
@@ -783,11 +783,11 @@ describeEmbeddedPostgres("companySkillService.list", () => {
? { type, companyId: otherCompanyId, agentId: randomUUID() }
: { type, companyIds: [otherCompanyId] };
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { companySkillVersions }, capture);
try { expect(await companySkillService(db, { actor }).list(companyId)).toEqual([]); }
finally { capture.flush(); }
expect(events.find(event => event.event === "paperclip.db_response_capture.summary.v1")?.completedQueries).toBe(0);
expect(events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")?.completedQueries).toBe(0);
});
it("restores a deleted release after warm validation", async () => {
@@ -6,9 +6,10 @@ import { join } from "node:path";
import { createServer } from "node:http";
import { and, eq, sql } from "drizzle-orm";
import { beforeAll, afterAll, describe, expect, it, vi } from "vitest";
import { createDb, authUsers, activityLog, agentApiKeys, companies, companySecrets, companySecretVersions, chatConversations, chatDeliveries, chatEndpoints, chatExternalPrincipals, chatIdentityLinks, toolPolicies, toolRateLimitCounters, connectionGrants, connectionGrantMembers, connectionGrantDelegations, toolAccessAuditEvents, toolOauthStates, issueThreadInteractions, connectionIntentDeliveries, agents, companyMemberships, heartbeatRuns, issues, principalPermissionGrants, projects, toolApplications, toolConnections, toolConnectionInstalls, toolMcpGateways, toolMcpGatewayTokens, toolActionRequests, toolCatalogEntries, toolInvocations, toolCallEvents, toolProfiles, toolProfileBindings, plugins, adapterAuthSessions, environments, routines, chatGitHubReviews } from "@paperclipai/db";
import { createDb, authUsers, activityLog, agentApiKeys, companies, companySecrets, companySecretVersions, chatConversations, chatDeliveries, chatEndpoints, chatExternalPrincipals, chatIdentityLinks, toolPolicies, toolRateLimitCounters, connectionGrants, connectionGrantMembers, connectionGrantDelegations, toolAccessAuditEvents, toolOauthStates, issueThreadInteractions, connectionIntentDeliveries, agents, companyMemberships, heartbeatRuns, issues, principalPermissionGrants, projects, toolApplications, toolConnections, toolConnectionInstalls, aiConnectionPools, toolMcpGateways, toolMcpGatewayTokens, toolActionRequests, toolCatalogEntries, toolInvocations, toolCallEvents, toolProfiles, toolProfileBindings, plugins, adapterAuthSessions, environments, routines, chatGitHubReviews } from "@paperclipai/db";
import { findSatisfiedToolConnection, satisfiedConnectionIntentService } from "../services/satisfied-connection-intents.js";
import { connectionManagementService, continuingConnectionManagement } from "../services/connection-management.js";
import { aiConnectionService } from "../services/ai-connections.js";
import { redactRunnerApiPayloads } from "../services/native-runtime/runner-api-log-redaction.js";
import * as remoteEndpointGuard from "../services/remote-http-endpoint-guard.js";
import * as remoteHttpFetch from "../services/remote-http-fetch.js";
@@ -87,13 +88,14 @@ describe("connection management with production default agents", () => {
authority: new PaperclipRunnerToolAuthority(server.db, { companyId: base.companyId, agentId: agent.id, issueId: base.issueId, runId: base.runId, apiUrl: server.apiUrl, storage: server.storage }) };
}
it("matches ordinary install capabilities on real PostgreSQL for scoped, retained and fallback resources", async () => {
it.each(["tool", "ai"] as const)("matches ordinary %s install capabilities on real PostgreSQL for scoped, retained and fallback resources", async (connectionPurpose) => {
const f = await server.fixture({ connectionScenario: "fresh" });
const candidateIds = [f.agentId, randomUUID(), randomUUID()];
await server.db.insert(agents).values(candidateIds.slice(1).map((id, index) => ({ id, companyId: f.companyId, name: `Capability target ${index}`, adapterType: "process" })));
const [application] = await server.db.insert(toolApplications).values({ companyId: f.companyId, name: "Capability differential", type: "mcp_http" }).returning();
const [connection] = await server.db.insert(toolConnections).values({ companyId: f.companyId, applicationId: application.id,
uid: randomUUID(), name: "Capability differential", transport: "mcp_remote", authKind: "none", createdByUserId: "different-creator" }).returning();
uid: randomUUID(), name: "Capability differential", connectionPurpose,
transport: connectionPurpose === "ai" ? "runtime_auth" : "mcp_remote", authKind: "none", createdByUserId: "different-creator" }).returning();
await server.db.insert(principalPermissionGrants).values({ companyId: f.companyId, principalType: "user", principalId: f.responsibleUserId!,
permissionKey: "tools:manage_profiles", scope: { targetAgentIds: candidateIds.slice(0, 2) } });
const service = connectionManagementService(server.db);
@@ -177,7 +179,7 @@ describe("connection management with production default agents", () => {
expect(await faulting.describeAgentInstallCapabilities(localActor, resource, [candidateIds[1]])).toEqual([denied]);
});
it("preserves the complete grants GET across human roles and denies a real install mutation after revocation", async () => {
it.each(["tool", "ai"] as const)("preserves the complete %s grants GET across human roles and denies a real install mutation after revocation", async (connectionPurpose) => {
const f = await server.fixture({ connectionScenario: "fresh" });
const candidateIds = [f.agentId, randomUUID(), randomUUID()];
await server.db.insert(agents).values(candidateIds.slice(1).map((id, index) => ({ id, companyId: f.companyId, name: `Grant target ${index}`, adapterType: "process" })));
@@ -198,7 +200,8 @@ describe("connection management with production default agents", () => {
scope: permissionKey === "tools:manage_profiles" ? { targetAgentIds: candidateIds.slice(0, 2) } : null })));
const [application] = await server.db.insert(toolApplications).values({ companyId: f.companyId, name: "Grants response parity", type: "mcp_http" }).returning();
const [connection] = await server.db.insert(toolConnections).values({ companyId: f.companyId, applicationId: application.id,
uid: randomUUID(), name: "Grants response parity", transport: "mcp_remote", authKind: "none", createdByUserId: users.creator }).returning();
uid: randomUUID(), name: "Grants response parity", connectionPurpose,
transport: connectionPurpose === "ai" ? "runtime_auth" : "mcp_remote", authKind: "none", createdByUserId: users.creator }).returning();
const grants = await server.db.insert(connectionGrants).values([
{ companyId: f.companyId, connectionId: connection.id, kind: "organization", createdByUserId: users.creator, isDefault: true },
...[users.creator, users.manager, users.member].map((subjectUserId, index) => ({ companyId: f.companyId, connectionId: connection.id, kind: "user" as const,
@@ -253,6 +256,65 @@ describe("connection management with production default agents", () => {
expect(savedInstalls).toHaveLength(2);
});
it("checks company-wide personal AI installs once per target and preserves pool member authorization", async () => {
const f = await server.fixture({ connectionScenario: "fresh" });
const candidateIds = [f.agentId, ...Array.from({ length: 19 }, () => randomUUID())];
await server.db.insert(agents).values(candidateIds.slice(1).map((id, index) => ({ id, companyId: f.companyId, name: `AI target ${index}`, adapterType: "process" })));
const [application] = await server.db.insert(toolApplications).values({ companyId: f.companyId, name: "AI capability checks", type: "mcp_http" }).returning();
const [connection] = await server.db.insert(toolConnections).values({ companyId: f.companyId, applicationId: application.id,
uid: randomUUID(), name: "Personal AI account", transport: "runtime_auth", connectionPurpose: "ai", authKind: "api_key",
credentialPolicy: "per_user", createdByUserId: f.responsibleUserId }).returning();
const [grant] = await server.db.insert(connectionGrants).values({ companyId: f.companyId, connectionId: connection.id,
kind: "user", subjectUserId: f.responsibleUserId, createdByUserId: f.responsibleUserId }).returning();
await server.db.insert(toolConnectionInstalls).values({ companyId: f.companyId, connectionId: connection.id, targetType: "company", targetId: f.companyId });
await server.db.insert(principalPermissionGrants).values({ companyId: f.companyId, principalType: "user", principalId: f.responsibleUserId!,
permissionKey: "tools:manage_profiles", scope: { targetAgentIds: candidateIds } });
let targetReads = 0;
const observedDb = new Proxy(server.db, { get(target, key, receiver) {
if (key !== "select") return Reflect.get(target, key, receiver);
return (...fields: Parameters<typeof target.select>) => {
const builder = target.select(...fields), from = builder.from.bind(builder);
builder.from = ((...args: Parameters<typeof from>) => {
const query = from(...args), execute = query.execute.bind(query);
query.execute = (async (...executeArgs: Parameters<typeof execute>) => {
if (query.toSQL().sql.includes('"agents"."id" =')) targetReads++;
return execute(...executeArgs);
}) as typeof execute;
return query;
}) as typeof builder.from;
return builder;
};
} });
const service = connectionManagementService(observedDb);
const actor: AuthorizationActor = { type: "board", userId: f.responsibleUserId, source: "cloud_tenant" };
const resource = { type: "connection_management" as const, companyId: f.companyId, connectionId: connection.id, operation: "configure", readOnly: false };
const baseline = [];
for (const targetAgentId of candidateIds) baseline.push(await service.capability(actor, "connections:profiles", { ...resource, targetAgentId }));
expect(baseline.every(result => result.allowed)).toBe(true);
expect(targetReads).toBe(candidateIds.length ** 2);
targetReads = 0;
expect(await service.describeAgentInstallCapabilities(actor, resource, candidateIds)).toEqual(baseline);
expect(targetReads).toBe(candidateIds.length);
const [pool] = await server.db.insert(toolConnections).values({ companyId: f.companyId, applicationId: application.id,
uid: randomUUID(), name: "Pool with private member", transport: "runtime_auth", connectionPurpose: "ai", authKind: "none" }).returning();
await server.db.insert(aiConnectionPools).values({ id: pool.id, companyId: f.companyId, pluginKey: "fixture-pool", config: {
name: "Pool", enabled: true, mode: "round_robin", thresholdPercent: 90,
members: [{ id: randomUUID(), binding: { mode: "delegated", provider: "openai", method: "api_key", connectionId: connection.id, grantId: grant.id }, profile: { provider: "codex", model: "fixture-model" } }],
} });
await server.db.delete(principalPermissionGrants).where(and(eq(principalPermissionGrants.companyId, f.companyId), eq(principalPermissionGrants.principalId, f.responsibleUserId!)));
await server.db.insert(principalPermissionGrants).values({ companyId: f.companyId, principalType: "user", principalId: f.responsibleUserId!,
permissionKey: "tools:manage_profiles", scope: { connectionIds: [pool.id] } });
const poolResource = { ...resource, connectionId: pool.id };
expect((await service.describeAgentInstallCapabilities(actor, poolResource, candidateIds.slice(0, 2))).every(result => result.allowed)).toBe(true);
// The pool grant cannot replace the member account's own authority.
await server.db.update(connectionGrants).set({ status: "revoked" }).where(eq(connectionGrants.id, grant.id));
await server.db.update(toolConnections).set({ createdByUserId: "different-owner" }).where(eq(toolConnections.id, connection.id));
const ordinary = await service.capability(actor, "connections:profiles", { ...poolResource, targetAgentId: candidateIds[0] });
expect(ordinary.allowed).toBe(false);
expect(await service.describeAgentInstallCapabilities(actor, poolResource, candidateIds.slice(0, 2))).toEqual([ordinary, ordinary]);
});
it("creates and configures tool connections and profiles with fresh grants and actual key/JWT authentication", async () => {
const f = await fixture();
const grants = await server.db.select().from(principalPermissionGrants).where(eq(principalPermissionGrants.principalId, f.agentId));
@@ -1642,6 +1704,36 @@ describe("connection management with production default agents", () => {
} finally { provider = null; unregisterServerAdapter("paperclip_runner"); }
});
it("reads only the authorized Anthropic account identity through actual agent authentication", async () => {
const f = await fixture();
const account = await aiConnectionService(server.db).save(f.companyId, f.userId, {
provider: "anthropic", method: "subscription", ownership: "personal", name: "Account identity fixture",
loginSessionId: "fixture", agentIds: [f.agentId], allAgents: false,
}, "selected-identity-credential");
let calls = 0;
provider = async (url, init) => {
calls++;
expect(url).toBe("https://api.anthropic.com/api/oauth/profile");
expect(new Headers(init?.headers).get("authorization")).toBe("Bearer selected-identity-credential");
return Response.json({ account: { uuid: "private-seat", email: "avery@example.test" }, organization: { uuid: "private-org", name: "Example team" } });
};
const path = `/companies/${f.companyId}/ai-connections/${account.connectionId}/identity?grantId=${account.grantId}`;
try {
for (const token of [f.key.token, f.jwt]) {
const response = await f.request("GET", path, undefined, token);
expect(response).toMatchObject({ status: 200, body: { ...account, status: "available", accountLabel: "avery@example.test", organizationLabel: "Example team" } });
for (const secret of ["selected-identity-credential", "private-seat", "private-org"]) expect(JSON.stringify(response)).not.toContain(secret);
}
expect(calls).toBe(2);
expect((await f.request("GET", path.replace(f.companyId, f.foreignCompanyId))).status).toBe(403);
expect((await f.request("GET", path.replace(account.grantId, randomUUID()))).status).toBe(404);
expect(calls).toBe(2);
await server.db.update(connectionGrants).set({ status: "revoked" }).where(eq(connectionGrants.id, account.grantId));
expect(await f.request("GET", path)).toMatchObject({ status: 200, body: { status: "unavailable", errorCode: "connection_unavailable", accountLabel: null } });
expect(calls).toBe(2);
} finally { provider = null; }
});
it("filters AI account and pool listings by the agent's resolved connection scope", async () => {
const f = await fixture();
provider = async url => {
@@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto";
import { and, eq, sql } from "drizzle-orm";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { agents, companies, createDb, heartbeatRuns, issueComments, issueThreadInteractions, issues } from "@paperclipai/db";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { settleConversationTurnForRun } from "../services/agent-conversations.js";
@@ -52,11 +52,11 @@ const support = await getEmbeddedPostgresTestSupport();
it.each(["response", "interaction", "reset"] as const)("bounds %s settlement reads and retains all saved text and run payloads", async reply => {
const input = await seed(reply), before = await sourceDigests(input);
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { heartbeatRuns, issues, issueComments }, capture);
try { expect(await settleConversationTurnForRun(db, input.companyId, input.runId)).toBe(true); }
finally { capture.flush(); }
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
&& ["heartbeat_runs", "issues", "issue_comments"].includes(String(event.table))
&& (event.operation === "select" || event.operation === "execute"));
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThanOrEqual(4);
@@ -0,0 +1,137 @@
import express from "express";
import request from "supertest";
import { QueryPromise } from "drizzle-orm/query-promise";
import { afterEach, expect, it, vi } from "vitest";
import { withDatabaseOperation } from "@paperclipai/db";
import { installDatabaseWorkSignals } from "@paperclipai/db/work-signals";
import { databaseOperationMiddleware } from "../middleware/database-operation.js";
import { withBudgetSpan } from "../services/budget-tracing.js";
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "../services/delivery-work-notifications.js";
import {
createRunPreparationTiming,
measureRunPhase,
withRunExecutionTiming,
withRunResponsePresentationTiming,
withRunWorkspaceFinalizationTiming,
} from "../services/execution-preparation-timing.js";
const { owners } = await vi.hoisted(async () => {
const { AsyncLocalStorage } = await import("node:async_hooks");
return { owners: new AsyncLocalStorage<string>() };
});
vi.mock("@paperclipai/db", async importOriginal => ({
...await importOriginal<typeof import("@paperclipai/db")>(),
withDatabaseOperation: <T>(owner: string, action: () => T): T => owners.run(owner, action),
}));
class OwnerQuery extends QueryPromise<string | undefined> {
async execute() { return owners.getStore(); }
}
afterEach(() => {
vi.unstubAllEnvs();
vi.useRealTimers();
});
it.each(["", "http://collector.test"])("attributes lazy queries to execution, preparation, and budget owners with endpoint %s", async endpoint => {
vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", endpoint);
await withDatabaseOperation("http.request", async () => {
const query = new OwnerQuery();
await expect(withRunExecutionTiming("private-run", () => query)).resolves.toBe("task.execution_attempt");
await expect(withRunWorkspaceFinalizationTiming("private-run", () => query)).resolves.toBe("task.workspace_finalization");
await expect(withRunResponsePresentationTiming("private-run", () => query)).resolves.toBe("task.response_presentation");
await expect(createRunPreparationTiming("private-run")("agent_read", () => query)).resolves.toBe("heartbeat.prepare.agent_read");
await expect(createRunPreparationTiming("private-run", "native.controller")("backend_setup", () => query)).resolves.toBe("native.controller.backend_setup");
await expect(measureRunPhase("skills.inventory", () => query)).resolves.toBe("skills.inventory");
await expect(withBudgetSpan("budget.accounting.transaction", () => query)).resolves.toBe("budget.accounting.transaction");
await expect(query).resolves.toBe("http.request");
});
expect(owners.getStore()).toBeUndefined();
});
it("keeps owner attribution after the span detail limit and restores nested scopes after failure", async () => {
vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.test");
const failure = new Error("private accounting failure");
await withRunExecutionTiming("private-run", async () => {
for (let i = 0; i < 18; i++) {
await expect(measureRunPhase("skills.inventory", () => new OwnerQuery())).resolves.toBe("skills.inventory");
}
await expect(measureRunPhase("execution.budget_reservation", async () => {
await expect(withBudgetSpan("budget.admission.reserve", async () => {
await Promise.resolve();
expect(owners.getStore()).toBe("budget.admission.reserve");
throw failure;
})).rejects.toBe(failure);
return new OwnerQuery();
})).resolves.toBe("execution.budget_reservation");
await expect(new OwnerQuery()).resolves.toBe("task.execution_attempt");
});
expect(owners.getStore()).toBeUndefined();
});
it("keeps concurrent HTTP requests and nested work in fixed scopes without using private URLs", async () => {
vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "");
const app = express();
app.use(databaseOperationMiddleware);
app.use(express.json());
let arrived = 0;
let release!: () => void;
const both = new Promise<void>(resolve => { release = resolve; });
app.all("/:privateId", async (req, res) => {
const before = await new OwnerQuery();
const work = async () => {
if (++arrived === 2) release();
await both;
return new OwnerQuery();
};
const during = req.params.privateId === "first"
? await withBudgetSpan("budget.admission.reserve", work)
: await measureRunPhase("skills.inventory", work);
res.json({ before, during, after: await new OwnerQuery() });
});
const [first, second] = await Promise.all([
request(app).get("/first?secret=private-first").expect(200),
request(app).post("/second?secret=private-second").send({ private: "request-body" }).expect(200),
]);
expect(first.body).toEqual({ before: "http.request", during: "budget.admission.reserve", after: "http.request" });
expect(second.body).toEqual({ before: "http.request", during: "skills.inventory", after: "http.request" });
expect(owners.getStore()).toBeUndefined();
});
it("replaces producer ownership on each delivery dispatch, including retry and pending checks", async () => {
vi.useFakeTimers();
const execute = vi.fn(async () => [{ xid: "42" }]);
const database = installDatabaseWorkSignals({ execute, async transaction<T>(action: (tx: object) => Promise<T>): Promise<T> {
return action({ execute });
} });
const seen: Array<{ queue: string; phase: string; owner: string | undefined }> = [];
const coordinator = createDeliveryWorkCoordinator({ owner: database, canRun: () => true,
onError: (_error, queue) => { seen.push({ queue, phase: "failure", owner: owners.getStore() }); },
});
let fail = true;
try {
const workers = withDatabaseOperation("http.request", () => [DELIVERY_QUEUES.feedback, DELIVERY_QUEUES.email].map(queue =>
coordinator.register(queue, { retryMs: 5000,
run: async () => {
await Promise.resolve();
seen.push({ queue, phase: "run", owner: await new OwnerQuery() });
if (queue === DELIVERY_QUEUES.feedback && fail) { fail = false; throw new Error("retry"); }
},
hasPending: async () => {
seen.push({ queue, phase: "pending", owner: await new OwnerQuery() });
return false;
},
})));
await Promise.all(workers.map(worker => worker.ready));
await withDatabaseOperation("budget.accounting.transaction", () => database.transaction(tx =>
notifyDeliveryWork(tx, DELIVERY_QUEUES.feedback)));
await vi.advanceTimersByTimeAsync(1);
expect(seen.map(({ phase }) => phase).sort()).toEqual(["failure", "pending", "pending", "run", "run", "run"]);
for (const entry of seen) expect(entry.owner).toBe(`delivery.${entry.queue}`);
expect(owners.getStore()).toBeUndefined();
expect(coordinator.nextWakeAt()).toBeNull();
} finally {
await coordinator.stop();
}
});
@@ -1,5 +1,5 @@
import { readIdleSleepSafety } from "../services/idle-sleep-safety.js";
import { createOwnedSandbox } from "../services/sandbox-creation-operations.js";
import { createOwnedSandbox, recordSandboxCreationResource } from "../services/sandbox-creation-operations.js";
import { mkdtemp, writeFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
@@ -116,15 +116,18 @@ describe.skipIf(!support.supported)("environment cleanup scheduling", () => {
]));
await quiet();
});
it("settles a deleted-company creation as manual history across restart without idle polling", async () => {
it.each([false, true])("settles a deleted-company creation as manual history across restart without idle polling (contextual=%s)", async contextual => {
await start(); await quiet();
await createOwnedSandbox(db, { companyId, environmentId, provider: "fixture", driverKey: "fixture", config: {}, metadata: {} },
const created = await createOwnedSandbox(db, { companyId, environmentId, provider: "fixture", driverKey: "fixture", config: {},
metadata: contextual ? { runnerTaskLease: true, runnerTaskCreationContextId: randomUUID() } : {} },
async () => ({ providerLeaseId: "fixture-deleted-company" }));
await db.delete(companies).where(eq(companies.id, companyId));
await settle(31 * 60_000);
await vi.waitFor(async () => expect(await db.select().from(sandboxCreationOperations)).toEqual([
expect.objectContaining({ state: "needs_reconciliation", metadata: expect.objectContaining({ creationReconciliationReason: "company_deleted" }) }),
]));
if (contextual) await recordSandboxCreationResource(db, { id: String(created.metadata!.creationOperationId), companyId, environmentId,
providerLeaseId: created.providerLeaseId });
await quiet();
worker!.stop(); await coordinator!.stop();
await start(); await quiet();
@@ -1,11 +1,12 @@
import { randomUUID } from "node:crypto";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { eq } from "drizzle-orm";
import { agents, companies, createDb, environmentLeases, environments, heartbeatRuns, plugins } from "@paperclipai/db";
import { agents, companies, createDb, environmentLeases, environments, heartbeatRuns, plugins, sandboxCreationOperations } from "@paperclipai/db";
import { resolvePluginTaskResume } from "../services/environment-task-runtime.js";
import { remoteTerminationReceipt } from "../services/remote-execution-termination.js";
import { environmentRuntimeService } from "../services/environment-runtime.js";
import type { PluginWorkerManager } from "../services/plugin-worker-manager.js";
import { NativeExecutionContext } from "../services/native-runtime/native-execution-context.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
const support = await getEmbeddedPostgresTestSupport();
@@ -20,6 +21,108 @@ databaseSuite("environment task lease authority", () => {
});
afterAll(async () => { await stopDb?.(); });
it("replaces a stopped warm candidate within its tracked startup context", async () => {
const companyId = randomUUID(), agentId = randomUUID(), environmentId = randomUUID(), pluginId = randomUUID(), runId = randomUUID(), leaseId = randomUUID();
await db.insert(companies).values({ id: companyId, name: "Warm candidate", issuePrefix: `T${companyId.slice(0, 7).replace(/-/g, "").toUpperCase()}` });
await db.insert(agents).values({ id: agentId, companyId, name: "Fixture", adapterType: "paperclip_runner" });
await db.insert(plugins).values({ id: pluginId, pluginKey: pluginId, packageName: "fixture-warm-replacement", version: "1", status: "ready",
manifestJson: { capabilities: ["environment.drivers.register"], environmentDrivers: [{ driverKey: "tasks", supportsTasks: true }] } as never });
await db.insert(environments).values({ id: environmentId, companyId, name: `Fixture ${environmentId}`, driver: "plugin",
config: { pluginKey: pluginId, driverKey: "tasks", driverConfig: {} } });
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "manual", status: "running" });
await db.insert(environmentLeases).values({ id: leaseId, companyId, environmentId, heartbeatRunId: runId,
provider: "plugin:fixture:tasks", providerLeaseId: "stopped-task", status: "active", leasePolicy: "reuse_by_environment",
metadata: { driver: "plugin", pluginId, pluginKey: pluginId, driverKey: "tasks", runnerTaskLease: true, runnerTaskReused: true } });
const calls: string[] = [];
const workers = { getWorker: () => ({ supportedMethods: ["environmentTask", "environmentAcquireLease"] }),
call: async (_plugin: string, method: string, params: any, _timeout?: number, _sink?: unknown, settled?: () => void) => {
calls.push(method === "environmentTask" ? params.operation.kind : method);
try {
if (method === "environmentAcquireLease") return { providerLeaseId: "replacement-task" };
if (params.operation.kind === "status") return { kind: "status", taskId: params.taskId, phase: "completed", executionStopped: true };
if (params.operation.kind === "describe") return { kind: "description", taskId: params.taskId,
runner: { version: "fixture", digest: `sha256:${"a".repeat(64)}` }, executionContextVersion: 1 };
return { kind: "accepted", taskId: params.taskId };
} finally { settled?.(); }
},
} as unknown as PluginWorkerManager;
const context = new NativeExecutionContext({ binding: { companyId, agentId, runId } } as never, 1_000);
try {
await expect(environmentRuntimeService(db, { pluginWorkerManager: workers }).task({ companyId, leaseId,
operation: { kind: "describe" }, executionContext: context })).resolves.toMatchObject({ kind: "description", taskId: "replacement-task" });
expect(calls).toEqual(["status", "status", "complete", "environmentAcquireLease", "describe"]);
expect(context.pending).toBe(false);
const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, leaseId));
expect(lease.providerLeaseId).toBe("replacement-task");
expect(lease.metadata?.runnerTaskReused).toBe(false);
expect(JSON.stringify(lease.metadata)).not.toContain(context.id);
} finally { context.finish(); }
});
it.each(["late", "post-result", "retiring"])("queues a replacement instead of attaching after %s startup cancellation", async mode => {
const companyId = randomUUID(), agentId = randomUUID(), environmentId = randomUUID(), pluginId = randomUUID(), runId = randomUUID(), leaseId = randomUUID();
await db.insert(companies).values({ id: companyId, name: "Warm candidate", issuePrefix: `T${companyId.slice(0, 7).replace(/-/g, "").toUpperCase()}` });
await db.insert(agents).values({ id: agentId, companyId, name: "Fixture", adapterType: "paperclip_runner" });
await db.insert(plugins).values({ id: pluginId, pluginKey: pluginId, packageName: "fixture-warm-replacement", version: "1", status: "ready",
manifestJson: { capabilities: ["environment.drivers.register"], environmentDrivers: [{ driverKey: "tasks", supportsTasks: true }] } as never });
await db.insert(environments).values({ id: environmentId, companyId, name: `Fixture ${environmentId}`, driver: "plugin",
config: { pluginKey: pluginId, driverKey: "tasks", driverConfig: {} } });
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "manual", status: "running" });
await db.insert(environmentLeases).values({ id: leaseId, companyId, environmentId, heartbeatRunId: runId,
provider: "plugin:fixture:tasks", providerLeaseId: "stopped-task", status: "active", leasePolicy: "reuse_by_environment",
metadata: { driver: "plugin", pluginId, pluginKey: pluginId, driverKey: "tasks", runnerTaskLease: true, runnerTaskReused: true } });
const calls: string[] = [];
let reply!: () => Promise<void>, acquiring!: () => void;
const entered = new Promise<void>(resolve => { acquiring = resolve; });
const workers = { getWorker: () => ({ supportedMethods: ["environmentTask", "environmentAcquireLease"] }),
call: async (_plugin: string, method: string, params: any, _timeout?: number, _sink?: unknown, settled?: () => void, _contextId?: string, observe?: (result: any) => Promise<void>) => {
calls.push(method === "environmentTask" ? params.operation.kind : method);
try {
if (method === "environmentAcquireLease") return await new Promise(resolve => {
reply = async () => { const result = { providerLeaseId: "replacement-task", metadata: { binding: "fresh" } };
await observe?.(result);
if (mode === "retiring") await db.update(environmentLeases).set({ metadata: { driver: "plugin", pluginId, driverKey: "tasks", runnerTaskRetiring: true } }).where(eq(environmentLeases.id, leaseId));
resolve(result); };
acquiring();
});
if (params.operation.kind === "status") return { kind: "status", taskId: params.taskId, phase: "completed", executionStopped: true };
if (params.operation.kind === "describe") return { kind: "description", taskId: params.taskId,
runner: { version: "fixture", digest: `sha256:${"a".repeat(64)}` }, executionContextVersion: 1 };
return { kind: "accepted", taskId: params.taskId };
} finally { settled?.(); }
},
} as unknown as PluginWorkerManager;
const context = new NativeExecutionContext({ binding: { companyId, agentId, runId } } as never, 1_000);
if (mode === "post-result") {
const wait = context.wait.bind(context);
vi.spyOn(context, "wait").mockImplementation(operation => wait(async () => {
const value = await operation(); context.abort(new Error("Cancelled after recording")); return value;
}));
}
try {
const request = environmentRuntimeService(db, { pluginWorkerManager: workers }).task({ companyId, leaseId,
operation: { kind: "describe" }, executionContext: context });
const rejected = expect(request).rejects.toThrow("operation unavailable");
await entered;
if (mode === "late") {
context.abort(new Error("Startup cancelled"));
await rejected;
expect(context.pending).toBe(true);
}
await reply(); await rejected;
expect(context.pending).toBe(false);
const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, leaseId));
expect(lease.providerLeaseId).toBe("stopped-task");
const [creation] = await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.runId, runId));
expect(creation.state).toBe("cleanup_pending");
expect(creation.metadata.runnerTaskCreationContextId).toBe(context.id);
const [cleanup] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, creation.leaseId!));
expect(cleanup).toMatchObject({ heartbeatRunId: runId, providerLeaseId: "replacement-task", status: "pending_cleanup" });
expect(cleanup.metadata).toMatchObject({ runnerTaskLease: true, providerMetadata: { binding: "fresh" } });
expect(calls).not.toContain("describe");
} finally { context.finish(); }
});
it("authorizes a new task lease only after scoped prior execution has stopped", async () => {
const companyId = randomUUID(), agentId = randomUUID(), environmentId = randomUUID(), pluginId = randomUUID();
const oldRunId = randomUUID(), nextRunId = randomUUID(), oldLeaseId = randomUUID(), nextLeaseId = randomUUID();
@@ -2,14 +2,16 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
import { PgDialect } from "drizzle-orm/pg-core";
import { executionWorkspaces, environmentLeases, type Db } from "@paperclipai/db";
import { environmentTaskOperationSchema, parseEnvironmentTaskResult } from "@paperclipai/plugin-sdk";
import { executeEnvironmentTask } from "../services/environment-task-runtime.js";
import { executeEnvironmentTask, settleDispatchedEnvironmentTask } from "../services/environment-task-runtime.js";
import { NativeExecutionContext } from "../services/native-runtime/native-execution-context.js";
const state = vi.hoisted(() => ({ plugin: {} as any, release: vi.fn() }));
const state = vi.hoisted(() => ({ plugin: {} as any, release: vi.fn(), identity: vi.fn(), config: vi.fn() }));
vi.mock("../services/provider-fresh-workspace.js", () => ({
stageProviderFreshWorkspace: async (input: { existing: unknown }) => input.existing,
releaseProviderRecoveryTransfers: state.release,
}));
vi.mock("../services/plugin-registry.js", () => ({ pluginRegistryService: () => ({ getById: vi.fn(async () => state.plugin) }) }));
vi.mock("../services/plugin-registry.js", () => ({ pluginRegistryService: () => ({ getById: vi.fn(async () => state.plugin), getConfig: state.config }) }));
vi.mock("../services/plugin-agent-runtime-state.js", () => ({ pluginAgentRuntimeIdentity: state.identity }));
const leaseId = "10000000-0000-4000-8000-000000000001";
const projectId = "10000000-0000-4000-8000-000000000002";
const secondProjectId = "10000000-0000-4000-8000-000000000003";
@@ -25,7 +27,7 @@ function database(value: ReturnType<typeof row> | null = row(), projectRows = [{
const query: any = { from: (table: unknown) => { selectedRows = table === executionWorkspaces ? workspaceRows : table === environmentLeases ? ownerRows : projectRows; return query; }, leftJoin: () => query, where: vi.fn(() => query), limit: () => Promise.resolve(results.shift()), then: (resolve: (rows: typeof projectRows) => unknown) => Promise.resolve(selectedRows).then(resolve as (rows: unknown[]) => unknown) };
const sets: Record<string, unknown>[] = [];
const update: any = { set: (value: Record<string, unknown>) => { sets.push(value); return update; }, where: () => update,
returning: async () => [{ id: leaseId }], then: (resolve: (value: unknown) => unknown) => Promise.resolve([]).then(resolve) };
returning: async () => [{ id: leaseId, metadata: value?.lease.metadata, heartbeatRunId: value?.lease.heartbeatRunId }], then: (resolve: (value: unknown) => unknown) => Promise.resolve([]).then(resolve) };
const db = { select: () => query, update: () => update,
transaction: vi.fn(async (write: (tx: Db) => Promise<unknown>) => write(db as unknown as Db)) };
return { db: db as unknown as Db, query, sets, transaction: db.transaction };
@@ -37,9 +39,65 @@ const submit = { kind: "submit" as const, preparedLaunchId: "configuration", pro
beforeEach(() => {
state.release.mockReset();
state.identity.mockReset().mockResolvedValue({ instanceSecrets: { launcher: "synthetic-transient-binding" } });
state.config.mockReset().mockResolvedValue({ configJson: { endpoint: "synthetic" } });
state.plugin = { id: "original", pluginKey: "test.provider", status: "ready", manifestJson: { capabilities: ["environment.drivers.register"], environmentDrivers: [{ driverKey: "tasks", supportsTasks: true }] } };
});
describe("plugin-provided Runner execution", () => {
it.each([false, true])("retires an undispatched warm task only with required preparation proof (%s)", async preparationStopped => {
const value = row();
Object.assign(value.lease.metadata, { runnerTaskDispatched: false,
runnerTaskStartupCleanup: { contextId: "startup", requiresPreparationStop: true } });
const fixture = database(value), workers = worker();
fixture.db.select = ((...args: unknown[]) => (database(value).db.select as (...values: unknown[]) => unknown)(...args)) as Db["select"];
let release!: () => void;
const drain = new Promise<void>(resolve => { release = resolve; });
const drainStartup = vi.fn(() => drain);
Object.assign(workers, { drainStartup });
const operations: string[] = [];
workers.call.mockImplementation(async (...args: any[]) => {
const operation = args[2].operation.kind; operations.push(operation);
return operation === "status" ? { kind: "status", taskId: "attempt-1", phase: "cancelled", executionStopped: true, preparationStopped }
: { kind: "accepted", taskId: "attempt-1" };
});
const retiring = settleDispatchedEnvironmentTask(fixture.db, workers as never, { companyId: "company", id: leaseId });
const checked = retiring.then(value => ({ value, error: null }), error => ({ value: false, error }));
await vi.waitFor(() => expect(operations).toContain("stop"));
await vi.waitFor(() => expect(operations).toContain("status"));
expect(drainStartup).toHaveBeenCalledWith("startup");
expect(operations).not.toContain("complete");
release();
const result = await checked;
if (preparationStopped) { expect(result).toEqual({ value: true, error: null }); expect(operations).toEqual(["stop", "status", "complete"]); }
else { expect(result.error?.message).toContain("termination is unconfirmed"); expect(fixture.sets).toContainEqual(expect.objectContaining({ status: "pending_cleanup", cleanupStatus: "failed" })); }
});
it("keeps legacy payloads passive without capability or transport activation", async () => {
const context = new NativeExecutionContext({ binding: { companyId: "company", agentId: "agent", runId: "run" } } as never, 1_000);
const workers = worker();
workers.call.mockImplementation(async (...args: any[]) => { args[5]?.(); return { kind: "description", taskId: "attempt-1", runner: { version: "fixture", digest: `sha256:${"a".repeat(64)}` } }; });
try {
for (const executionContext of [undefined, context]) {
await executeEnvironmentTask(database().db, workers as never, { companyId: "company", leaseId,
operation: { kind: "describe" }, ...(executionContext ? { executionContext } : {}) });
const sent = workers.call.mock.calls.at(-1) as unknown as any[];
expect(sent[2]).not.toHaveProperty("executionContextSupported");
expect(sent[2]).not.toHaveProperty("executionContext");
if (!executionContext) expect(sent).toHaveLength(4);
}
expect(state.identity).not.toHaveBeenCalled();
expect(state.config).not.toHaveBeenCalled();
expect(context.pending).toBe(false);
} finally { context.finish(); }
});
it.each(["json", "wrong-run"])("rejects a %s context before the worker", async mode => {
const context = new NativeExecutionContext({ binding: { companyId: "company", agentId: "agent", runId: mode === "wrong-run" ? "other" : "run" } } as never, 1_000);
const workers = worker();
try {
await expect(executeEnvironmentTask(database().db, workers as never, { companyId: "company", leaseId, operation: { kind: "describe" },
executionContext: mode === "json" ? JSON.parse(JSON.stringify(context)) : context })).rejects.toThrow("native_execution_context");
expect(workers.call).not.toHaveBeenCalled();
} finally { context.finish(); }
});
it("forwards a retained fresh directory intent on later launch calls and checks provider support", async () => {
const value = row();
value.lease.metadata.runnerTaskProjectAuthValidation.projectIds = [];
@@ -8,7 +8,7 @@ import { createRuntimeToolsToken } from "../runtime-tools-token.js";
import { errorHandler } from "../middleware/index.js";
import { randomUUID } from "node:crypto";
import { eq } from "drizzle-orm";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import {
agents,
@@ -87,7 +87,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
});
await db.insert(toolConnections).values(irrelevant);
const observed = createDb(external ?? database!.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(observed, { toolConnections }, capture);
try {
expect(await resolveManagedGitHubIdentitySelection(observed, input.companyId, { agentId: input.agentId, responsibleUserId: "A" }))
@@ -119,7 +119,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
}).where(eq(heartbeatRuns.id, input.runId));
const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId));
const observed = createDb(external ?? database!.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(observed, { heartbeatRuns }, capture);
try {
expect(await resolveGitHubOperationCredentials(observed, input)).toMatchObject({
@@ -689,14 +689,9 @@ describeEmbeddedPostgres("heartbeat dependency-aware queued run selection", () =
});
expect(interactionWake).not.toBeNull();
await waitForCondition(async () => {
const run = await db
.select({ status: heartbeatRuns.status })
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, interactionWake!.id))
.then((rows) => rows[0] ?? null);
return run?.status === "succeeded";
});
// This checks dependency interaction semantics, not provider startup speed.
// Join the admitted execution before inspecting its persisted outcome.
await heartbeat.drainActiveRunExecutions();
const interactionRun = await db
.select({
@@ -1,10 +1,15 @@
import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db";
import { randomUUID } from "node:crypto";
import { eq } from "drizzle-orm";
import { eq, sql } from "drizzle-orm";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { agents, authUsers, companies, companyMemberships, createDb, heartbeatRuns, issues } from "@paperclipai/db";
import { agents, authUsers, companies, companyMemberships, createDb, environmentLeases, environments, heartbeatRuns, issues, nativeRunFinalizations } from "@paperclipai/db";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { heartbeatService } from "../services/heartbeat.js";
import { NativeStartupCleanupPendingError, readNativeStartupCleanup } from "../services/native-runtime/native-startup-cleanup.js";
import { prepareNativeHeartbeatRun } from "../services/native-runtime/prepare-native-run.js";
import { environmentService } from "../services/environments.js";
import { remoteTerminationReceipt } from "../services/remote-execution-termination.js";
import { createIsolatedTestHome } from "./helpers/isolated-test-home.js";
const execute = vi.hoisted(() => vi.fn());
vi.mock("../adapters/index.js", async () => {
@@ -70,4 +75,108 @@ suite("native terminal-run cleanup admission", () => {
expect((await heartbeat.getRun(nextId))?.status).toBe("succeeded");
expect(execute).toHaveBeenCalledTimes(3);
}, 30_000);
it("retries a failed startup intent through timer failure and drain before verified cancellation", async () => {
const home = createIsolatedTestHome(), originalHome = process.env.PAPERCLIP_HOME;
process.env.PAPERCLIP_HOME = home.home;
const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), leaseId = randomUUID();
const contextId = randomUUID(), leaseOwner = randomUUID(), taskId = randomUUID(), deadlineAtMs = Date.now() - 1;
const originalSetTimeout = globalThis.setTimeout;
const retries: Array<{ timer: ReturnType<typeof setTimeout>; fire(): void }> = [];
const timeout = vi.spyOn(globalThis, "setTimeout").mockImplementation(((callback: (...args: unknown[]) => void, delay?: number, ...args: unknown[]) => {
if (typeof delay === "number" && delay <= 30_000 && new Error().stack?.includes("scheduleNativeSessionResumeDispatch")) {
const timer = originalSetTimeout(() => {}, 3_600_000);
retries.push({ timer, fire() { clearTimeout(timer); callback(...args); } });
return timer;
}
return originalSetTimeout(callback, delay, ...args);
}) as typeof setTimeout);
let failSuppression = false, runId = "";
const runtimeEnv = new Proxy<Record<string, string | undefined>>({}, { get(_target, key) {
if (key === "PAPERCLIP_DATABASE_RESTORE_IN_PROGRESS" && failSuppression) {
failSuppression = false;
throw new Error("Scheduling dependency unavailable");
}
return undefined;
} });
const heartbeat = heartbeatService(db, { runtimeEnv });
try {
await db.insert(companies).values({ id: companyId, name: "Intent retry", issuePrefix: "CIR", defaultResponsibleUserId: "owner" });
await db.insert(authUsers).values({ id: "owner", name: "Fixture user", email: "owner@example.test", createdAt: new Date(), updatedAt: new Date() }).onConflictDoNothing();
await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: "owner", status: "active", membershipRole: "member" });
await db.insert(agents).values({ id: agentId, companyId, name: "Worker", role: "engineer", status: "idle", adapterType: "process",
adapterConfig: {}, permissions: {}, runtimeConfig: { heartbeat: { enabled: false, wakeOnDemand: true } } });
await db.insert(issues).values({ id: issueId, companyId, title: "Recover startup", status: "todo", assigneeAgentId: agentId, responsibleUserId: "owner" });
const [environment] = await db.insert(environments).values({ companyId, name: "Controlled provider", driver: "plugin", config: {} }).returning();
await db.execute(sql`create table test_cleanup_intent_fault (enabled boolean not null)`);
await db.execute(sql`insert into test_cleanup_intent_fault values (true)`);
await db.execute(sql`create sequence test_cleanup_intent_attempts`);
await db.execute(sql.raw(`create function test_reject_cleanup_intent() returns trigger language plpgsql as $$ begin
if new.company_id = '${companyId}'::uuid and new.error_code = 'native_startup_cleanup_pending'
and exists (select 1 from test_cleanup_intent_fault where enabled) then
perform nextval('test_cleanup_intent_attempts'); raise exception 'Cleanup intent write unavailable';
end if; return new; end $$`));
await db.execute(sql`create trigger test_cleanup_intent_fault before update on heartbeat_runs for each row execute function test_reject_cleanup_intent()`);
execute.mockReset();
execute.mockImplementation(async (context: { runId: string }) => {
runId = context.runId;
const [run] = await db.update(heartbeatRuns).set({ runtimeMode: "native", runtimeModeResolvedAt: null }).where(eq(heartbeatRuns.id, runId)).returning();
await prepareNativeHeartbeatRun({ db, run, issue: { id: issueId, title: "Recover startup", description: null }, environmentLeaseId: leaseId });
await db.update(agents).set({ adapterType: "paperclip_runner", adapterConfig: { provider: "codex" } }).where(eq(agents.id, agentId));
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId));
await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId, phase: "observed", attempt: 1,
controllerGeneration: 1, leaseOwner, leaseExpiresAt: new Date(Date.now() - 1_000) });
await db.insert(environmentLeases).values({ id: leaseId, companyId, environmentId: environment.id, heartbeatRunId: runId,
issueId, status: "active", provider: "plugin", providerLeaseId: taskId, leasePolicy: "reuse_by_environment", metadata: { runnerTaskLease: true } });
await db.update(heartbeatRuns).set({ runnerProfileJson: sql`coalesce(${heartbeatRuns.runnerProfileJson}, '{}'::jsonb) || ${JSON.stringify({
nativeStartupContext: { schema: "paperclip.native-startup-context.v1", companyId, agentId, runId,
contextId, deadlineAtMs, leaseId, leaseOwner, controllerGeneration: 1, attempt: 1 },
})}::jsonb` }).where(eq(heartbeatRuns.id, runId));
throw new NativeStartupCleanupPendingError(contextId, deadlineAtMs, leaseOwner, false, leaseId);
});
await heartbeat.invoke(agentId, "on_demand", { issueId }, "manual");
await heartbeat.drainActiveRunExecutions();
const owner = () => db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)).then(rows => rows[0]!);
const lease = () => db.select().from(environmentLeases).where(eq(environmentLeases.id, leaseId)).then(rows => rows[0]!);
expect((await db.execute<{ last_value: string }>(sql`select last_value from test_cleanup_intent_attempts`))[0].last_value).toBe("2");
expect(await heartbeat.getRun(runId)).toMatchObject({ status: "running", errorCode: null });
expect((await owner()).leaseOwner).toBe(leaseOwner);
expect(await lease()).toMatchObject({ status: "active", cleanupStatus: null });
expect(readNativeStartupCleanup((await heartbeat.getRun(runId))!)).toBeNull();
const drainedRetries = retries.length;
expect(drainedRetries).toBeGreaterThan(0);
await heartbeat.resumeQueuedRunsAfterDrain();
expect(retries.length).toBeGreaterThan(drainedRetries);
const resumedRetries = retries.length;
failSuppression = true;
retries.at(-1)!.fire();
await vi.waitFor(() => expect(retries.length).toBeGreaterThan(resumedRetries));
expect((await owner()).leaseOwner).toBe(leaseOwner);
expect((await heartbeat.getRun(runId))?.status).toBe("running");
await db.execute(sql`update test_cleanup_intent_fault set enabled = false`);
retries.at(-1)!.fire();
await vi.waitFor(async () => expect(readNativeStartupCleanup((await heartbeat.getRun(runId))!)?.state).toBe("pending"));
await vi.waitFor(async () => expect((await heartbeat.listEvents(runId)).some(event => event.eventType === "native.recovery.transition")).toBe(true));
const current = await lease();
await environmentService(db).releaseLease(leaseId, "released", { cleanupStatus: "success",
remoteExecutionTermination: remoteTerminationReceipt(current, { providerLeaseId: taskId, state: "stopped" }) });
await heartbeat.cancelRun(runId, "Startup cleanup retry completed");
expect(await heartbeat.getRun(runId)).toMatchObject({ status: "cancelled" });
expect(readNativeStartupCleanup((await heartbeat.getRun(runId))!)?.state).toBe("complete");
expect(await owner()).toMatchObject({ leaseOwner: null, controlDeadlineAt: null });
expect(execute).toHaveBeenCalledOnce();
} finally {
await heartbeat.drainActiveRunExecutions();
timeout.mockRestore();
for (const retry of retries) clearTimeout(retry.timer);
await db.execute(sql`drop trigger if exists test_cleanup_intent_fault on heartbeat_runs`);
await db.execute(sql`drop function if exists test_reject_cleanup_intent()`);
await db.execute(sql`drop table if exists test_cleanup_intent_fault`);
await db.execute(sql`drop sequence if exists test_cleanup_intent_attempts`);
if (originalHome === undefined) delete process.env.PAPERCLIP_HOME;
else process.env.PAPERCLIP_HOME = originalHome;
home.cleanup();
}
}, 30_000);
});
@@ -1,7 +1,7 @@
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
import { describe, expect, it } from "vitest";
import { eq, sql } from "drizzle-orm";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import {
agents,
companies,
@@ -63,11 +63,11 @@ describe("P6-11..13 / P6-17 canonical event allocator", () => {
protocolSchemaVersion: 1, canonicalPayload: { sourceSeq } },
});
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { heartbeatRunEvents }, capture);
try { expect((await append(5002)).highestContiguousSourceSeq).toBe(5000); }
finally { capture.flush(); }
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
&& event.table === "heartbeat_run_events" && (event.operation === "select" || event.operation === "execute"));
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThan(0);
expect(reads.reduce((sum, event) => sum + Number(event.rows), 0)).toBeLessThan(16);
@@ -41,3 +41,21 @@ describe("compactRunLogChunk", () => {
expect(compacted).not.toContain("paperclip-flag-secret");
});
});
describe("compactRunLogChunk with long base64 commands", () => {
it("redacts a tool call that writes base64 of source text without stalling", () => {
const typescriptSource = "export const limit = 65536;\nexport function keep(value: string) {\n return value.slice(0, limit);\n}\n";
const blob = Buffer.from(typescriptSource.repeat(1_000)).toString("base64").slice(0, 48 * 1024);
expect(blob).toMatch(/^[A-Za-z0-9]+$/);
const command = `printf '%s' '${blob}' | base64 -d > src/keep.ts && echo '{"GITHUB_TOKEN":"tool-call-secret"}'`;
const chunk = `${JSON.stringify({ type: "assistant", message: { content: [{ type: "tool_use", input: { command } }] } })}\n`;
const startedAt = performance.now();
const compacted = compactRunLogChunk(chunk);
expect(performance.now() - startedAt).toBeLessThan(1_000);
expect(compacted).toContain(blob);
expect(compacted).not.toContain("tool-call-secret");
expect(compacted).toContain("***REDACTED***");
});
});
@@ -1,7 +1,7 @@
import { randomUUID } from "node:crypto";
import { eq } from "drizzle-orm";
import { drizzle } from "drizzle-orm/postgres-js";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import express from "express";
import request from "supertest";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
@@ -268,7 +268,7 @@ describeEmbeddedPostgres("heartbeat run privacy routes", { concurrent: false },
}).where(eq(heartbeatRuns.id, f.privateRunId));
const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.privateRunId));
const observed = createDb(externalDatabaseUrl ?? tempDb!.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(observed, { heartbeatRuns }, capture);
try {
const app = createApp(f.companyId, f.grantedAgentId, observed);
@@ -791,19 +791,7 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
`));
await heartbeat.resumeQueuedRuns();
await waitForCondition(async () => {
const [run, wakeup] = await Promise.all([
db.select({ status: heartbeatRuns.status })
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, runId))
.then((rows) => rows[0] ?? null),
db.select({ status: agentWakeupRequests.status })
.from(agentWakeupRequests)
.where(eq(agentWakeupRequests.id, wakeupRequestId))
.then((rows) => rows[0] ?? null),
]);
return run?.status === "cancelled" && wakeup?.status === "skipped";
});
await heartbeat.drainActiveRunExecutions();
const [run, wakeup, issue] = await Promise.all([
db.select({ status: heartbeatRuns.status, errorCode: heartbeatRuns.errorCode })
@@ -970,16 +958,10 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
expect(ordering).toEqual(["validated", "parked"]);
};
await heartbeat.resumeQueuedRuns();
await waitForCondition(async () => {
const run = await db
.select({ status: heartbeatRuns.status })
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, runId))
.then((rows) => rows[0] ?? null);
return run?.status === "cancelled";
});
await heartbeat.drainActiveRunExecutions();
await parkPromise;
expect((await heartbeat.getRun(runId))?.status).toBe("cancelled");
const issue = await db
.select({ status: issues.status })
.from(issues)
@@ -1108,7 +1090,7 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
});
expect(run).not.toBeNull();
await waitForCondition(async () => countExecuteCallsForRun(run!.id) > 0);
await heartbeat.drainActiveRunExecutions();
expect(countExecuteCallsForRun(run!.id)).toBe(1);
});
@@ -1126,7 +1108,7 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
});
expect(run).not.toBeNull();
await waitForCondition(async () => countExecuteCallsForRun(run!.id) > 0);
await heartbeat.drainActiveRunExecutions();
expect(countExecuteCallsForRun(run!.id)).toBe(1);
});
@@ -1718,13 +1700,7 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
.where(eq(agentWakeupRequests.id, wakeupRequestId));
await heartbeat.resumeQueuedRuns();
await waitForCondition(async () => {
const [deferred] = await db
.select({ status: agentWakeupRequests.status, runId: agentWakeupRequests.runId })
.from(agentWakeupRequests)
.where(eq(agentWakeupRequests.id, deferredWakeupId));
return Boolean(deferred?.runId) && deferred?.status !== "deferred_issue_execution";
});
await heartbeat.drainActiveRunExecutions();
const [deferred] = await db
.select({ status: agentWakeupRequests.status, runId: agentWakeupRequests.runId })
@@ -1788,15 +1764,7 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
});
await heartbeat.resumeQueuedRuns();
await waitForCondition(async () => {
const run = await db
.select({ status: heartbeatRuns.status })
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, runId))
.then((rows) => rows[0] ?? null);
return run?.status === "cancelled";
});
await heartbeat.drainActiveRunExecutions();
const [run, wakeup, issue] = await Promise.all([
db
@@ -1,35 +1,7 @@
import { randomUUID } from "node:crypto";
import {
applyPendingMigrations,
createDb,
getEmbeddedPostgresTestSupport as embeddedSupport,
startEmbeddedPostgresTestDatabase as startEmbedded,
export {
getEmbeddedPostgresTestSupport,
startEmbeddedPostgresTestDatabase,
EMBEDDED_POSTGRES_TEST_TIMEOUT_MS,
type EmbeddedPostgresTestDatabase,
type EmbeddedPostgresTestSupport,
} from "@paperclipai/db";
export type { EmbeddedPostgresTestDatabase, EmbeddedPostgresTestSupport };
export async function getEmbeddedPostgresTestSupport(): Promise<EmbeddedPostgresTestSupport> {
return process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL ? { supported: true } : embeddedSupport();
}
export async function startEmbeddedPostgresTestDatabase(prefix: string): Promise<EmbeddedPostgresTestDatabase> {
const adminUrl = process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL;
if (!adminUrl) return startEmbedded(prefix);
const name = `paperclip_test_${randomUUID().replaceAll("-", "")}`;
const template = process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE;
if (template && !/^paperclip_test_template_[a-f0-9]+$/.test(template)) throw new Error("Invalid disposable test template name");
const admin = createDb(adminUrl, { maxConnections: 1 });
const url = new URL(adminUrl);
url.pathname = `/${name}`;
const cleanup = async () => {
try { await admin.$client.unsafe(`DROP DATABASE IF EXISTS "${name}" WITH (FORCE)`); }
finally { await admin.$client.end({ timeout: 1 }); }
};
try {
await admin.$client.unsafe(`CREATE DATABASE "${name}"${template ? ` TEMPLATE "${template}"` : ""}`);
if (!template) await applyPendingMigrations(url.toString());
return { connectionString: url.toString(), cleanup };
} catch (error) { await cleanup(); throw error; }
}
@@ -0,0 +1,62 @@
import { expect, it } from "vitest";
import { createFullDatabaseResponseCapture } from "./full-database-response-capture.js";
const metadata = { operation: "select", table: "heartbeat_runs", callers: [] };
it("measures every oversized string and complex value without sampling or truncation", () => {
const events: Array<Record<string, unknown>> = [];
const capture = createFullDatabaseResponseCapture({ start: 0, end: 1000 }, event => events.push(event), () => 0);
const text = "private-payload🙂".repeat(30_000);
const nested = { instructions: text, additional: [{ prompt: text }] };
const rows = Object.assign([[text, nested]], { columns: [{ name: "stdout_excerpt" }, { name: "context_snapshot" }] });
const bytes = Buffer.byteLength(text) + Buffer.byteLength(JSON.stringify(nested));
try {
for (let index = 0; index < 17; index++) expect(capture.execute(() => rows, metadata)).toBe(rows);
capture.flush();
expect(events[0]).toMatchObject({ event: "paperclip.test_db_response_capture.entry.v1",
calls: 17, rows: 17, responseValueBytes: 17 * bytes, maxResponseValueBytes: bytes,
estimatedValues: 17, serializationErrors: 0,
});
expect(events[1]).toMatchObject({ event: "paperclip.test_db_response_capture.summary.v1",
startedQueries: 17, completedQueries: 17, inFlightAtEnd: 0, overflowCalls: 0, responseValueBytes: 17 * bytes,
});
expect(JSON.stringify(events)).not.toContain("private-payload");
expect(nested.instructions).toBe(text);
} finally { capture.flush(); }
});
it("preserves asynchronous results and original query errors", async () => {
const events: Array<Record<string, unknown>> = [];
const capture = createFullDatabaseResponseCapture({ start: 0, end: 1000 }, event => events.push(event), () => 0);
const rows = Object.assign([{ value: { nested: "result" } }], { columns: [{ name: "value" }] });
const failure = new Error("private query failure");
try {
await expect(capture.execute(async () => rows, metadata)).resolves.toBe(rows);
await expect(capture.execute(async () => { throw failure; }, metadata)).rejects.toBe(failure);
expect(() => capture.execute(() => { throw failure; }, metadata)).toThrow(failure);
capture.flush();
expect(events[0]).toMatchObject({ calls: 3, rows: 1, queryErrors: 2,
responseValueBytes: Buffer.byteLength(JSON.stringify(rows[0].value)), serializationErrors: 0,
});
expect(events[1]).toMatchObject({ startedQueries: 3, completedQueries: 3, inFlightAtEnd: 0 });
expect(JSON.stringify(events)).not.toContain(failure.message);
} finally { capture.flush(); }
});
it("reports unfinished work at the deadline without cancelling it or counting a late result", async () => {
const events: Array<Record<string, unknown>> = [];
let now = 0;
const capture = createFullDatabaseResponseCapture({ start: 0, end: 1000 }, event => events.push(event), () => now);
let release!: (value: string[]) => void;
const original = new Promise<string[]>(resolve => { release = resolve; });
const pending = capture.execute(() => original, metadata);
now = 1000;
capture.flush();
release(["result"]);
await expect(pending).resolves.toEqual(["result"]);
capture.flush();
expect(events).toHaveLength(1);
expect(events[0]).toMatchObject({ event: "paperclip.test_db_response_capture.summary.v1",
startedQueries: 1, completedQueries: 0, inFlightAtEnd: 1, responseValueBytes: 0,
});
});
@@ -0,0 +1,115 @@
import { randomUUID } from "node:crypto";
import { createDatabaseResponseCapture, type ResponseCaptureWindow } from "@paperclipai/db/response-capture";
export { installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
type Capture = ReturnType<typeof createDatabaseResponseCapture>;
type Metadata = Parameters<Capture["execute"]>[1];
type Entry = Metadata & { calls: number; rows: number; bytes: number; maxBytes: number;
queryErrors: number; serializationErrors: number; estimatedValues: number };
function fullResponseValueSize(value: unknown): { bytes: number; estimatedValues: number } {
const columns = value && typeof value === "object" ? (value as { columns?: unknown }).columns : undefined;
if (!Array.isArray(value) || !Array.isArray(columns)) {
const serialized = JSON.stringify(value, (_, item: unknown) => typeof item === "bigint" ? item.toString() : item);
return { bytes: serialized === undefined ? 0 : Buffer.byteLength(serialized), estimatedValues: 1 };
}
let bytes = 0, estimatedValues = 0;
for (const row of value) {
const cells = Array.isArray(row) ? row : columns.map(column => row[column.name]);
for (const cell of cells) {
if (cell === null || cell === undefined) continue;
if (typeof cell === "string") bytes += Buffer.byteLength(cell);
else if (typeof cell === "boolean") bytes += 1;
else if (typeof cell === "number" || typeof cell === "bigint") bytes += Buffer.byteLength(String(cell));
else {
estimatedValues += 1;
const serialized = JSON.stringify(cell, (_, item: unknown) => typeof item === "bigint" ? item.toString() : item);
if (serialized !== undefined) bytes += Buffer.byteLength(serialized);
}
}
}
return { bytes, estimatedValues };
}
export function createFullDatabaseResponseCapture(
window: ResponseCaptureWindow,
emit: (event: Record<string, unknown>) => void,
now: () => number = Date.now,
): Capture {
const capture = createDatabaseResponseCapture(window, () => {}, now);
const id = randomUUID(), entries = new Map<string, Entry>();
let closed = false, inFlight = 0, started = 0, completed = 0, overflowCalls = 0;
const send = (event: Record<string, unknown>) => { try { emit(event); } catch {} };
function flush() {
if (closed) return;
closed = true;
clearTimeout(timer);
capture.flush();
const ordered = [...entries.values()].sort((a, b) => b.bytes - a.bytes);
for (const [groupIndex, entry] of ordered.entries()) send({
event: "paperclip.test_db_response_capture.entry.v1", captureId: id,
groupIndex, groupPage: Math.floor(groupIndex / 128),
operation: entry.operation, table: entry.table, callers: entry.callers,
calls: entry.calls, rows: entry.rows, responseValueBytes: entry.bytes,
maxResponseValueBytes: entry.maxBytes, estimatedValues: entry.estimatedValues,
queryErrors: entry.queryErrors, serializationErrors: entry.serializationErrors,
});
send({ event: "paperclip.test_db_response_capture.summary.v1", captureId: id,
windowStart: new Date(window.start).toISOString(), windowEnd: new Date(window.end).toISOString(),
closedAt: new Date(now()).toISOString(), startedQueries: started, completedQueries: completed,
inFlightAtEnd: inFlight, groups: entries.size, groupPageSize: 128, groupPages: Math.ceil(ordered.length / 128),
overflowCalls, responseValueBytes: ordered.reduce((sum, entry) => sum + entry.bytes, 0),
rowValuesLogged: false, sqlOrParametersLogged: false, wireBytesMeasured: false,
measurementBoundary: "postgres-js results before Drizzle column decoding",
relationalQueriesMeasured: true, unsafeDriverQueriesMeasured: true,
taggedDriverQueriesMeasured: false, streamingQueriesMeasured: false,
});
entries.clear();
}
const timer = setTimeout(flush, Math.max(0, window.end - now()));
timer.unref();
function finish(metadata: Metadata, value: unknown, failed: boolean) {
if (!capture.active()) return;
inFlight -= 1;
completed += 1;
let key = JSON.stringify(metadata);
if (!entries.has(key) && entries.size >= 2047) {
overflowCalls += 1;
metadata = { operation: "other", table: "other", callers: [] };
key = "overflow";
}
let entry = entries.get(key);
if (!entry) {
entry = { ...metadata, calls: 0, rows: 0, bytes: 0, maxBytes: 0,
queryErrors: 0, serializationErrors: 0, estimatedValues: 0 };
entries.set(key, entry);
}
entry.calls += 1;
if (failed) { entry.queryErrors += 1; return; }
entry.rows += Array.isArray(value) ? value.length : 0;
try {
const size = fullResponseValueSize(value);
entry.bytes += size.bytes;
entry.maxBytes = Math.max(entry.maxBytes, size.bytes);
entry.estimatedValues += size.estimatedValues;
} catch { entry.serializationErrors += 1; }
}
function execute(action: () => unknown, metadata: Metadata) {
if (!capture.active()) return action();
inFlight += 1;
started += 1;
let result: unknown;
try { result = action(); }
catch (error) { finish(metadata, undefined, true); throw error; }
let asynchronous = false;
try { asynchronous = Boolean(result && typeof (result as { then?: unknown }).then === "function"); } catch {}
if (asynchronous) return Promise.resolve(result).then(
value => { finish(metadata, value, false); return value; },
error => { finish(metadata, undefined, true); throw error; },
);
finish(metadata, result, false);
return result;
}
return { ...capture, execute, flush };
}
@@ -0,0 +1,15 @@
import { randomUUID } from "node:crypto";
import { agents, companies, toolApplications, toolCatalogEntries, toolConnections, toolProfileBindings, toolProfileEntries, toolProfiles, type Db } from "@paperclipai/db";
export async function createToolProfileMatchingFixture(db: Db) {
const [company] = await db.insert(companies).values({ name: "Synthetic profile matching", issuePrefix: `BM${randomUUID().slice(0, 6)}` }).returning();
const companyAgents = await db.insert(agents).values(Array.from({ length: 12 }, (_, i) => ({ companyId: company.id, name: `Agent ${i}` }))).returning();
const agent = companyAgents[0]!;
const [app] = await db.insert(toolApplications).values({ companyId: company.id, name: "Synthetic app", type: "mcp_http" }).returning();
const [connection] = await db.insert(toolConnections).values({ companyId: company.id, applicationId: app.id, name: "Synthetic connection", uid: randomUUID(), transport: "mcp_remote" }).returning();
const catalog = await db.insert(toolCatalogEntries).values(Array.from({ length: 1890 }, (_, i) => ({ companyId: company.id, applicationId: app.id, connectionId: connection.id, name: `tool_${i}`, toolName: `tool_${i}`, versionHash: `v${i}`, riskLevel: "read" as const }))).returning();
const [profile] = await db.insert(toolProfiles).values({ companyId: company.id, profileKey: "selected", name: "Selected tools", defaultAction: "deny" }).returning();
await db.insert(toolProfileBindings).values({ companyId: company.id, profileId: profile.id, targetType: "agent", targetId: agent.id });
await db.insert(toolProfileEntries).values(catalog.slice(0, 1575).map((entry, i) => ({ companyId: company.id, profileId: profile.id, selectorType: "catalog_entry" as const, catalogEntryId: entry.id, effect: i % 5 === 0 ? "exclude" as const : "include" as const })));
return { company, agent };
}
@@ -0,0 +1,29 @@
import { randomUUID } from "node:crypto";
import { agents, companies, toolApplications, toolCatalogEntries, toolConnections, toolProfileEntries, toolProfiles, type Db } from "@paperclipai/db";
export async function createProfileProjectionFixture(db: Db, options: { profiles?: number; tools?: number; rules?: number } = {}) {
const profileCount = options.profiles ?? 200, toolCount = options.tools ?? 1900, ruleCount = options.rules ?? 380, maxRuleCount = options.rules ?? 695;
const [company] = await db.insert(companies).values({ name: "Profile projection fixture", issuePrefix: `PP${randomUUID().slice(0, 6)}` }).returning();
await db.insert(agents).values(Array.from({ length: 12 }, (_, i) => ({ companyId: company.id, name: `Agent ${i}` })));
const [application] = await db.insert(toolApplications).values({ companyId: company.id, name: "Projection application", type: "mcp_http" }).returning();
const [connection] = await db.insert(toolConnections).values({ companyId: company.id, applicationId: application.id, name: "Projection connection", uid: randomUUID(), transport: "mcp_remote", status: "active", enabled: true, healthStatus: "ok" }).returning();
const catalog = await db.insert(toolCatalogEntries).values(Array.from({ length: toolCount + maxRuleCount }, (_, i) => ({
companyId: company.id, applicationId: application.id, connectionId: connection.id,
name: `tool_${i}`, toolName: `tool_${i}`, versionHash: `v${i}`, riskLevel: "read" as const,
status: i < toolCount ? "active" as const : "removed" as const,
firstSeenAt: new Date(i < toolCount ? "2026-01-03T00:00:00Z" : "2026-01-01T00:00:00Z"),
}))).returning();
const profiles = await db.insert(toolProfiles).values(Array.from({ length: profileCount }, (_, i) => ({
companyId: company.id, profileKey: `selected-${i}`, name: `Selected profile ${i}`, defaultAction: "deny" as const,
createdAt: new Date("2026-01-02T00:00:00Z"),
}))).returning();
for (let i = 0; i < profiles.length; i++) {
const count = i % 4 < 2 ? maxRuleCount : 2 * ruleCount - maxRuleCount;
const rules = (i % 2 ? catalog.slice(toolCount, toolCount + count) : catalog.slice(0, count)).map(entry => ({
companyId: company.id, profileId: profiles[i]!.id, selectorType: "catalog_entry" as const,
effect: "include" as const, catalogEntryId: entry.id,
}));
for (let offset = 0; offset < rules.length; offset += 500) await db.insert(toolProfileEntries).values(rules.slice(offset, offset + 500));
}
return { company, application, connection, catalog: catalog.slice(0, toolCount), profiles };
}
@@ -5,7 +5,7 @@ import { agentApiKeys, agents, budgetPolicies, chatDeliveries, chatEndpoints, co
companyTransferRuns, connectionEventDeliveries, createDb, heartbeatRuns, issues,
routineTriggers, routines, toolApplications, toolConnections, type Db } from "@paperclipai/db";
import { readIdleSleepSafety } from "../services/idle-sleep-safety.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
const now = Date.UTC(2026, 10, 2);
@@ -25,7 +25,7 @@ if (!support.supported) console.warn(`Skipping idle sleep diagnostics Postgres t
db = createDb(externalDatabaseUrl ?? temporary!.connectionString, { maxConnections: 4 });
}, 60_000);
afterEach(async () => { await db.execute(sql`TRUNCATE companies, plugins, environments CASCADE`); });
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); });
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
const read = (scheduled = false, monitors = false) => readIdleSleepSafety(db, hold, () => now, ownerId,
localNone, undefined, undefined, scheduled, monitors, { collectDiagnostics: true });
+15 -2
View File
@@ -9,7 +9,7 @@ import {
heartbeatRuns, issues, issueWatchdogs, issueThreadInteractions, issueQuestionResponseDeliveries,
projects, projectWorkspaces, routines, routineTriggers, toolApplications, toolConnections, workspaceOperations, executionWorkspaceRuntimeLeases, type Db,
} from "@paperclipai/db";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
import { readIdleSleepSafety, type IdleSleepDrainStatus } from "../services/idle-sleep-safety.js";
@@ -175,7 +175,7 @@ if (!support.supported) console.warn(`Skipping idle sleep Postgres tests: ${supp
db = createDb(externalDatabaseUrl ?? database.connectionString, { maxConnections: 4 });
}, 60_000);
afterEach(async () => { await db.execute(sql`TRUNCATE companies, plugins, environments CASCADE`); });
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); });
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
const read = () => readIdleSleepSafety(db, owned, () => now, ownerId, emptyLocal);
const readScheduled = () => readIdleSleepSafety(db, owned, () => now, ownerId, emptyLocal, undefined, undefined, true);
@@ -206,6 +206,19 @@ if (!support.supported) console.warn(`Skipping idle sleep Postgres tests: ${supp
}
});
it("keeps a failed startup intent awake after its local worker counter settles", async () => {
const f = await seed();
expect(await read()).toEqual(none);
const finish = beginIdleTrackedWork();
await db.update(heartbeatRuns).set({ status: "running", runtimeMode: "native", nativePhase: "observed",
runnerProfileJson: { nativeStartupContext: { contextId: randomUUID(), deadlineAtMs: now - 1 } } }).where(eq(heartbeatRuns.id, f.runId));
finish();
expect(idleWorkSnapshot().active).toBe(0);
expect(await read()).toEqual(present);
await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, f.runId));
expect(await read()).toEqual(none);
});
it("retains committed budget stop intent until its exact generation is delivered", async () => {
const f = await seed();
const [policy] = await db.insert(budgetPolicies).values({ companyId: f.companyId, scopeType: "company", scopeId: f.companyId,
@@ -1,6 +1,6 @@
import { randomUUID } from "node:crypto";
import { activityLog, agents, companies, createDb, heartbeatRuns, issues } from "@paperclipai/db";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import { eq } from "drizzle-orm";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
@@ -55,7 +55,7 @@ describe.skipIf(!support.supported)("creation source run projections", () => {
expect(run.issueId).toBe(binding === "scoped" ? sourceId : null);
expect(run.nativeIssueId).toBeNull();
const observed = createDb(database.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(observed, { heartbeatRuns, issues }, capture);
try {
const result = await resolveIssueCreationSource(observed, child);
@@ -0,0 +1,127 @@
import { randomUUID } from "node:crypto";
import { eq, sql } from "drizzle-orm";
import { expect, it } from "vitest";
import request from "supertest";
import { agents, heartbeatRuns, issueRecoveryActions, issueRelations, issues, statusCards } from "@paperclipai/db";
import { companySearchQuerySchema, createIssueSchema, PROVIDER_QUOTA_MONITOR_SERVICE_NAME } from "@paperclipai/shared";
import { issueDisplayStatusSql } from "../services/issue-display-status.js";
import { issueService } from "../services/issues.js";
import { companySearchService } from "../services/company-search.js";
import { statusCardService } from "../services/status-cards.js";
import { issueRoutes } from "../routes/issues.js";
import {
describeEmbeddedPostgres, routeApp,
seedCompanyWithBoardAccess, useEmbeddedPostgres,
} from "./helpers/route-test-harness.js";
describeEmbeddedPostgres("Monitoring task presentation", () => {
const ctx = useEmbeddedPostgres("paperclip-monitoring-status-", {
resetEach: async (db) => { await db.execute(sql`SET client_min_messages = warning`); await db.execute(sql`TRUNCATE TABLE companies CASCADE`); },
});
async function fixture() {
const company = await seedCompanyWithBoardAccess(ctx.db, "Monitoring");
const agentId = randomUUID();
await ctx.db.insert(agents).values({ id: agentId, companyId: company.companyId, name: "Monitor owner", role: "engineer", status: "paused", adapterType: "process", adapterConfig: {} });
const nextCheckAt = new Date(Date.now() + 60 * 60_000);
const issueId = randomUUID();
await ctx.db.insert(issues).values({
id: issueId, companyId: company.companyId, title: "Check deployment", status: "in_progress",
assigneeAgentId: agentId, createdByUserId: company.actor.userId,
monitorNextCheckAt: nextCheckAt,
executionPolicy: { mode: "normal", stages: [], commentRequired: false, monitor: { nextCheckAt: nextCheckAt.toISOString(), notes: "Check deployment" } },
});
return { ...company, agentId, issueId };
}
async function displayStatus(id: string) {
const [row] = await ctx.db.select({ status: issues.status, displayStatus: issueDisplayStatusSql() }).from(issues).where(eq(issues.id, id));
return row;
}
it("projects a scheduled wait consistently across detail, compact rows, summaries, search and counts", async () => {
const f = await fixture();
const svc = issueService(ctx.db);
expect(await svc.getById(f.issueId)).toMatchObject({ status: "in_progress", displayStatus: "monitoring" });
expect(await svc.list(f.companyId, { displayStatus: "monitoring", listProjection: "index" })).toEqual([expect.objectContaining({ id: f.issueId, status: "in_progress", displayStatus: "monitoring" })]);
expect(await svc.count(f.companyId, { displayStatus: "monitoring" })).toBe(1);
const app = routeApp(ctx.db, f.actor, issueRoutes);
const summary = await request(app).get(`/api/companies/${f.companyId}/issues/summaries?ids=${f.issueId}`).expect(200);
expect(summary.body[0]).toMatchObject({ status: "in_progress", displayStatus: "monitoring" });
const result = await companySearchService(ctx.db).search(f.companyId, companySearchQuerySchema.parse({ q: "deployment", displayStatus: "monitoring" }));
expect(result.results[0]?.issue).toMatchObject({ id: f.issueId, status: "in_progress", displayStatus: "monitoring" });
expect(result.filterOptionCounts.displayStatus?.monitoring).toBe(1);
expect(result.filterOptionCounts.status.in_progress).toBe(1);
const [card] = await ctx.db.insert(statusCards).values({ companyId: f.companyId, interestPrompt: "Watch deployment", mentionedIssueIds: [f.issueId], refreshPolicy: { mode: "manual" } }).returning();
expect(await statusCardService(ctx.db).listMentionedIssues(card!)).toEqual([expect.objectContaining({ id: f.issueId, status: "in_progress", displayStatus: "monitoring" })]);
// Existing agent/API lifecycle filters keep their original meaning.
expect(await svc.count(f.companyId, { status: "in_progress" })).toBe(1);
expect(createIssueSchema.safeParse({ title: "Manual monitoring", status: "monitoring" }).success).toBe(false);
});
it("filters before pagination and keeps In Progress separate from Monitoring", async () => {
const f = await fixture();
await ctx.db.insert(issues).values({ companyId: f.companyId, title: "Ordinary work", status: "in_progress", assigneeAgentId: f.agentId });
const svc = issueService(ctx.db);
expect(await svc.list(f.companyId, { displayStatus: "monitoring", limit: 1, offset: 0 })).toEqual([expect.objectContaining({ id: f.issueId })]);
expect(await svc.list(f.companyId, { displayStatus: "monitoring", limit: 1, offset: 1 })).toEqual([]);
expect(await svc.count(f.companyId, { displayStatus: "in_progress" })).toBe(1);
expect(await svc.count(f.companyId, { status: "in_progress" })).toBe(2);
});
it("shows the same wait indicator in list blocker summaries and task detail relations", async () => {
const f = await fixture();
const [dependent] = await ctx.db.insert(issues).values({ companyId: f.companyId, title: "Ship after deployment", status: "blocked" }).returning();
await ctx.db.insert(issueRelations).values({ companyId: f.companyId, issueId: f.issueId, relatedIssueId: dependent!.id, type: "blocks" });
const svc = issueService(ctx.db);
const list = await svc.list(f.companyId, { includeBlockedBy: true });
expect(list.find(row => row.id === dependent!.id)?.blockedBy).toEqual([expect.objectContaining({ id: f.issueId, displayStatus: "monitoring" })]);
const detail = await request(routeApp(ctx.db, f.actor, issueRoutes)).get(`/api/issues/${dependent!.id}`).expect(200);
expect(detail.body.blockedBy).toEqual([expect.objectContaining({ id: f.issueId, displayStatus: "monitoring" })]);
});
it("returns to In Progress during the check and Monitoring after another yield, then Done", async () => {
const f = await fixture();
const runId = randomUUID();
await ctx.db.insert(heartbeatRuns).values({ id: runId, companyId: f.companyId, agentId: f.agentId, issueId: f.issueId, status: "queued" });
expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress");
await ctx.db.update(heartbeatRuns).set({ status: "running" }).where(eq(heartbeatRuns.id, runId));
await ctx.db.update(issues).set({ executionRunId: runId, checkoutRunId: runId }).where(eq(issues.id, f.issueId));
expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress");
await ctx.db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, runId));
await ctx.db.update(issues).set({ executionRunId: null, checkoutRunId: null }).where(eq(issues.id, f.issueId));
expect((await displayStatus(f.issueId)).displayStatus).toBe("monitoring");
await ctx.db.update(issues).set({ status: "done" }).where(eq(issues.id, f.issueId));
expect(await displayStatus(f.issueId)).toEqual({ status: "done", displayStatus: "done" });
});
it.each(["blocked", "in_review", "todo", "cancelled"])("preserves %s even with a retained monitor", async (status) => {
const f = await fixture();
await ctx.db.update(issues).set({ status }).where(eq(issues.id, f.issueId));
expect((await displayStatus(f.issueId)).displayStatus).toBe(status);
});
it("does not label cleared, consumed, orphaned or quota-recovery schedules Monitoring", async () => {
const f = await fixture();
await ctx.db.update(issues).set({ executionState: { monitor: { status: "cleared" } } }).where(eq(issues.id, f.issueId));
expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress");
await ctx.db.update(issues).set({ executionState: null, monitorNextCheckAt: null }).where(eq(issues.id, f.issueId));
expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress");
await ctx.db.update(issues).set({ monitorNextCheckAt: new Date(), executionPolicy: { monitor: { serviceName: PROVIDER_QUOTA_MONITOR_SERVICE_NAME } } }).where(eq(issues.id, f.issueId));
expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress");
await ctx.db.update(issues).set({ executionPolicy: null }).where(eq(issues.id, f.issueId));
expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress");
});
it("keeps recovery and review holds visible and never writes a new lifecycle state", async () => {
const f = await fixture();
await ctx.db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: f.issueId, kind: "source", ownerType: "board", cause: "process_lost", fingerprint: randomUUID(), nextAction: "Inspect stopped run" });
expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress");
await ctx.db.execute(sql`UPDATE issue_recovery_actions SET status = 'escalated' WHERE source_issue_id = ${f.issueId}`);
expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress");
await ctx.db.execute(sql`UPDATE issue_recovery_actions SET status = 'resolved' WHERE source_issue_id = ${f.issueId}`);
expect((await displayStatus(f.issueId)).displayStatus).toBe("monitoring");
await ctx.db.update(issues).set({ executionState: { currentParticipant: { type: "user", userId: "reviewer" } } }).where(eq(issues.id, f.issueId));
expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress");
});
});
@@ -4,7 +4,7 @@ import { randomUUID } from "node:crypto";
import express from "express";
import request from "supertest";
import { eq, sql } from "drizzle-orm";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import {
agentRuntimeState,
@@ -202,7 +202,7 @@ describeEmbeddedPostgres("issue queued-comment routes", () => {
if (fallback) await db.update(issues).set({ executionRunId: null }).where(eq(issues.id, f.issueId));
const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId));
const observed = createDb(tempDb!.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(observed, { heartbeatRuns }, capture);
try {
const response = await request(app(f.companyId, "queue-owner", undefined, observed))
@@ -238,7 +238,7 @@ describeEmbeddedPostgres("issue queued-comment routes", () => {
}).where(eq(heartbeatRuns.companyId, f.companyId));
const [before] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, f.wakeId));
const observed = createDb(tempDb!.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(observed, { agentWakeupRequests, heartbeatRuns }, capture);
try {
const response = await request(app(f.companyId, "queue-owner", undefined, observed)).get(path).expect(200);
@@ -275,7 +275,7 @@ describeEmbeddedPostgres("issue queued-comment routes", () => {
})),
]);
const observed = createDb(tempDb!.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(observed, { agentWakeupRequests }, capture);
try {
const response = await request(app(f.companyId, "queue-owner", undefined, observed)).get(path).expect(200);
@@ -50,7 +50,7 @@ describeEmbeddedPostgres("bounded issue summaries", () => {
expect(res.headers["cache-control"]).toBe("private, no-store");
expect(res.body.map((row: { id: string }) => row.id)).toEqual([f.rows[1]!.id, f.rows[0]!.id]);
expect(Object.keys(res.body[0]).sort()).toEqual([
"companyId", "externalConversationState", "id", "identifier", "status", "title", "updatedAt",
"companyId", "displayStatus", "externalConversationState", "id", "identifier", "status", "title", "updatedAt",
]);
expect(res.body[0].externalConversationState).toBeNull();
expect(forbiddenRead).not.toHaveBeenCalled();
@@ -0,0 +1,110 @@
import { randomUUID } from "node:crypto";
import { eq, sql } from "drizzle-orm";
import request from "supertest";
import { expect, it } from "vitest";
import { activityLog, agents, authUsers, issues } from "@paperclipai/db";
import { issueRoutes } from "../routes/issues.js";
import { deriveIssueUserContext, issueService } from "../services/issues.js";
import { describeEmbeddedPostgres, resetCompanyIssueFixtures, routeApp, seedCompanyWithBoardAccess, useEmbeddedPostgres } from "./helpers/route-test-harness.js";
describeEmbeddedPostgres("human assignee unread comments", () => {
const ctx = useEmbeddedPostgres("paperclip-assignee-unread-", {
resetEach: async db => {
await db.delete(activityLog);
await db.delete(issues);
await db.delete(agents);
await resetCompanyIssueFixtures(db);
await db.delete(authUsers);
},
});
it.each([false, true])("keeps another human's real comment unread through recency updates (prior read: %s)", async priorRead => {
const { companyId, userId, actor } = await seedCompanyWithBoardAccess(ctx.db, "Assigned human task");
const otherCompany = await seedCompanyWithBoardAccess(ctx.db, "Unrelated company");
const svc = issueService(ctx.db), app = routeApp(ctx.db, actor, issueRoutes);
const issue = await svc.create(companyId, {
title: "Human inbox comment", status: "in_progress", createdByUserId: userId, assigneeUserId: userId,
});
if (priorRead) await request(app).post(`/api/issues/${issue.id}/read`).send({}).expect(200);
const comment = await svc.addComment(issue.id, "Another human supplied an update", { userId: `author-${randomUUID()}` });
const [afterComment] = await ctx.db.select().from(issues).where(eq(issues.id, issue.id));
// Exercise the real insert and recency update. No historical timestamps or
// mocked comment state hide the ordering that used to clear unread status.
expect(afterComment!.updatedAt.getTime()).toBeGreaterThanOrEqual(new Date(comment.createdAt).getTime());
const path = `/api/companies/${companyId}/issues`;
const expectUnread = async (unread: boolean) => {
for (const projection of ["", "&view=rows", "&view=index"]) {
const result = await request(app).get(`${path}?q=${encodeURIComponent(issue.title)}&touchedByUserId=me&limit=100${projection}`).expect(200);
expect(result.body).toHaveLength(1);
expect(result.body[0].isUnreadForMe).toBe(unread);
const filtered = await request(app).get(`${path}?q=${encodeURIComponent(issue.title)}&unreadForUserId=me&limit=100${projection}`).expect(200);
expect(filtered.body.map((row: { id: string }) => row.id)).toEqual(unread ? [issue.id] : []);
}
expect(await svc.countUnreadTouchedByUser(companyId, userId)).toBe(unread ? 1 : 0);
expect(await svc.unreadInboxIssueIds(companyId, userId, sql<boolean>`true`)).toEqual(unread ? [issue.id] : []);
};
await expectUnread(true);
// Legacy compact responses have a short TTL. Check their SQL projection
// once; immediate mutation readback belongs to ordinary/rows/index.
const compact = await request(app).get(`${path}?q=${encodeURIComponent(issue.title)}&touchedByUserId=me&view=compact`).expect(200);
expect(compact.body[0].isUnreadForMe).toBe(true);
const derived = deriveIssueUserContext(afterComment!, userId, {
myLastCommentAt: null, myLastReadAt: null, lastExternalCommentAt: comment.createdAt,
});
expect(derived.isUnreadForMe).toBe(true);
await request(app).post(`/api/issues/${issue.id}/read`).send({}).expect(200);
await expectUnread(false);
await request(app).delete(`/api/issues/${issue.id}/read`).expect(200);
await expectUnread(true);
await svc.addComment(issue.id, "I have read this update", { userId });
await expectUnread(false);
await request(app).get(`/api/companies/${otherCompany.companyId}/issues?unreadForUserId=me`).expect(403);
expect(await svc.countUnreadTouchedByUser(otherCompany.companyId, userId)).toBe(0);
});
it("shows existing comments as unread when a task is subsequently assigned to a human", async () => {
const { companyId, userId, actor } = await seedCompanyWithBoardAccess(ctx.db, "New assignment");
const svc = issueService(ctx.db), app = routeApp(ctx.db, actor, issueRoutes);
const issue = await svc.create(companyId, { title: "Assigned after discussion", status: "backlog" });
await svc.addComment(issue.id, "Context written before assignment", { userId: `author-${randomUUID()}` });
await request(app).patch(`/api/issues/${issue.id}`).send({ assigneeUserId: userId, status: "in_progress" }).expect(200);
const result = await request(app).get(`/api/companies/${companyId}/issues?q=${encodeURIComponent(issue.title)}&unreadForUserId=me`).expect(200);
expect(result.body.map((row: { id: string }) => row.id)).toEqual([issue.id]);
expect(result.body[0].isUnreadForMe).toBe(true);
await request(app).post(`/api/issues/${issue.id}/read`).send({}).expect(200);
expect(await svc.countUnreadTouchedByUser(companyId, userId)).toBe(0);
});
it("excludes deleted external comments but retains the historical touch from a deleted own reply", async () => {
const { companyId, userId } = await seedCompanyWithBoardAccess(ctx.db, "Deleted comment controls");
const svc = issueService(ctx.db);
const issue = await svc.create(companyId, {
title: "Deleted inbox activity", status: "in_progress", assigneeUserId: userId,
});
const authorId = `author-${randomUUID()}`;
const first = await svc.addComment(issue.id, "Keep this live update", { userId: authorId });
const second = await svc.addComment(issue.id, "Remove this later update", { userId: authorId });
const observation = async () => {
const [row] = await svc.list(companyId, { touchedByUserId: userId });
return { row, count: await svc.countUnreadTouchedByUser(companyId, userId) };
};
await svc.tombstoneComment(second.id, { actorType: "user", userId });
expect((await observation()).count).toBe(1);
expect((await observation()).row!.lastExternalCommentAt).toEqual(new Date(first.createdAt));
await svc.tombstoneComment(first.id, { actorType: "user", userId });
const removed = await observation();
expect(removed.count).toBe(0);
expect(removed.row!.isUnreadForMe).toBe(false);
expect(removed.row!.lastExternalCommentAt).toBeNull();
await svc.addComment(issue.id, "New live activity", { userId: authorId });
expect((await observation()).count).toBe(1);
const ownReply = await svc.addComment(issue.id, "I saw the live update", { userId });
expect((await observation()).count).toBe(0);
await svc.tombstoneComment(ownReply.id, { actorType: "user", userId });
const seen = await observation();
expect(seen.count).toBe(0);
expect(seen.row!.isUnreadForMe).toBe(false);
expect(seen.row!.myLastTouchAt).toEqual(new Date(ownReply.createdAt));
});
});
@@ -56,4 +56,30 @@ describeEmbeddedPostgres("issue list status query parsing", () => {
it("returns every status when ?status is absent", async () => {
expect(await listStatuses("")).toEqual(["done", "in_progress", "todo"]);
});
it.each(["", "&view=compact", "&view=compact&projection=index"])("rejects unsupported creation sorting for blocked attention (%s)", async projection => {
const { companyId, actor } = await seedCompanyWithBoardAccess(ctx.db, "Blocked sorting");
await request(routeApp(ctx.db, actor, issueRoutes))
.get(`/api/companies/${companyId}/issues?attention=blocked&sortField=created&sortDir=asc${projection}`)
.expect(422);
});
it.each(["", "&view=compact", "&view=compact&projection=index"])("sorts routine tasks by creation time before pagination (%s)", async (projection) => {
const { companyId, actor } = await seedCompanyWithBoardAccess(ctx.db, "Creation sorting");
const ids = Array.from({ length: 4 }, (_, index) => `00000000-0000-4000-8000-00000000000${index}`);
await ctx.db.insert(issues).values([
...ids.map((id, index) => ({ id, companyId, title: "Routine sorting", originKind: "routine_execution", status: "todo",
createdAt: new Date(index < 2 ? "2026-01-01" : "2026-02-01"),
updatedAt: new Date(index < 2 ? "2026-03-01" : "2026-02-01"), priority: index < 2 ? "critical" : "low" })),
{ companyId, title: "Routine sorting manual", originKind: "manual", createdAt: new Date("2026-04-01") },
]);
const app = routeApp(ctx.db, actor, issueRoutes);
const path = `/api/companies/${companyId}/issues?originKind=routine_execution&includeRoutineExecutions=true&sortField=created&limit=2${projection}`;
for (const sortDir of ["asc", "desc"]) {
const first = await request(app).get(`${path}&sortDir=${sortDir}`).expect(200);
const second = await request(app).get(`${path}&sortDir=${sortDir}&offset=2`).expect(200);
expect([...first.body, ...second.body].map((issue: { id: string }) => issue.id)).toEqual(sortDir === "asc" ? ids : [...ids].reverse());
}
await request(app).get(`${path}&sortDir=invalid`).expect(400);
});
});
@@ -4166,6 +4166,7 @@ const dependencyTestDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_ISSU
await db.delete(executionWorkspaces);
await db.delete(projectWorkspaces);
await db.delete(projects);
await db.delete(heartbeatRuns);
await db.delete(agents);
await db.delete(instanceSettings);
await db.delete(companies);
@@ -4720,6 +4721,42 @@ const dependencyTestDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_ISSU
});
});
it.each(["covered", "foreign-run", "unfinished", "failed-finalize", "late-finish", "later-operation"])(
"requires a successful owning finalize to cover a nested snapshot: %s", async kind => {
const f = await seedSharedWorkspaceDependency();
const runId = randomUUID(), foreignRunId = randomUUID();
await db.insert(heartbeatRuns).values([runId, foreignRunId].map(id => ({
id, companyId: f.companyId, agentId: f.assigneeAgentId, status: "succeeded",
})));
const started = new Date("2026-05-23T22:00:00.000Z");
const finished = new Date("2026-05-23T22:00:02.000Z");
await db.insert(workspaceOperations).values([
{ companyId: f.companyId, executionWorkspaceId: f.executionWorkspaceId, issueId: f.blockerId,
heartbeatRunId: runId, phase: "workspace_finalize",
status: kind === "failed-finalize" ? "failed" : "succeeded", startedAt: started, finishedAt: finished },
{ companyId: f.companyId, executionWorkspaceId: f.executionWorkspaceId, issueId: f.blockerId,
heartbeatRunId: kind === "foreign-run" ? foreignRunId : runId, phase: "remote_workspace_snapshot",
status: kind === "unfinished" ? "running" : "succeeded",
startedAt: new Date(started.getTime() + 100),
finishedAt: kind === "unfinished" ? null : new Date(finished.getTime() + (kind === "late-finish" ? 100 : -100)) },
...(kind === "later-operation" ? [{
companyId: f.companyId, executionWorkspaceId: f.executionWorkspaceId, issueId: f.blockerId,
heartbeatRunId: runId, phase: "worktree_prepare", status: "succeeded",
startedAt: new Date(finished.getTime() + 100), finishedAt: new Date(finished.getTime() + 200),
}] : []),
]);
const ready = kind === "covered";
expect((await svc.listWakeableBlockedDependents(f.blockerId)).map(issue => issue.id))
.toEqual(ready ? [f.dependentId] : []);
await expect(svc.getDependencyReadiness(f.dependentId)).resolves.toMatchObject({
isDependencyReady: ready,
pendingFinalizeBlockerIssueIds: ready ? [] : [f.blockerId],
});
const listed = (await svc.list(f.companyId, { status: "blocked" })).find(issue => issue.id === f.dependentId);
expect(listed?.blockerAttention?.pendingFinalizeBlockerIssueIds ?? []).toEqual(ready ? [] : [f.blockerId]);
},
);
it("reports dependency readiness for blocked issue chains", async () => {
const companyId = randomUUID();
await seedIssueCompany(db, companyId);
@@ -76,7 +76,7 @@ describe("deriveIssueUserContext", () => {
}
});
it("uses issue updated time as fallback touch point for assignee", () => {
it("uses stable creation time for assignees instead of treating task recency as a read", () => {
const context = deriveIssueUserContext(
makeIssue({ assigneeUserId: "user-1", updatedAt: new Date("2026-03-06T15:00:00.000Z") }),
"user-1",
@@ -87,8 +87,8 @@ describe("deriveIssueUserContext", () => {
},
);
expect(context.myLastTouchAt?.toISOString()).toBe("2026-03-06T15:00:00.000Z");
expect(context.isUnreadForMe).toBe(false);
expect(context.myLastTouchAt?.toISOString()).toBe("2026-03-06T10:00:00.000Z");
expect(context.isUnreadForMe).toBe(true);
});
it("uses latest read timestamp to clear unread without requiring a comment", () => {
@@ -1,5 +1,5 @@
import { randomBytes, randomUUID } from "node:crypto";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import { and, eq, sql } from "drizzle-orm";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { agents, companies, createDb, environmentLeases, environments, executionWorkspaces, heartbeatRuns, issueRecoveryActions, issues, projects } from "@paperclipai/db";
@@ -54,12 +54,12 @@ const support = externalTestDatabaseUrl ? { supported: true } : await getEmbedde
runnerProfileJson: { savedProfile: bulk }, stdoutExcerpt: bulk, stderrExcerpt: bulk,
}).where(eq(heartbeatRuns.id, f.runId)).returning();
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
let returned: Awaited<ReturnType<typeof terminalizeLegacyExecution>>;
try { returned = await terminalizeLegacyExecution({ db, run: saved!, status: "failed", patch }); }
finally { capture.flush(); }
const groups = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1");
const groups = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1");
const reads = groups.filter(group => group.table === "heartbeat_runs" && group.operation === "select");
expect(reads.reduce((sum, group) => sum + Number(group.calls), 0)).toBe(1);
expect(reads.reduce((sum, group) => sum + Number(group.responseValueBytes), 0)).toBeLessThan(4096);
@@ -86,7 +86,7 @@ const support = externalTestDatabaseUrl ? { supported: true } : await getEmbedde
patch: { resultJson: { workspaceRestoreFailure: "restore_failed", futureResult: bulk } },
}))!;
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
let returned: Awaited<ReturnType<typeof terminalizeLegacyExecution>>;
try { returned = await terminalizeLegacyExecution({ db, run: failed, status: "failed", recordRestoreFailureOnly,
@@ -94,7 +94,7 @@ const support = externalTestDatabaseUrl ? { supported: true } : await getEmbedde
...(recordRestoreFailureOnly ? { patch: { resultJson: failed.resultJson } } : {}),
}); }
finally { capture.flush(); }
const writes = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
const writes = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
&& event.table === "heartbeat_runs" && event.operation === "update");
expect(writes.reduce((sum, group) => sum + Number(group.rows), 0)).toBe(1);
expect(returned).toMatchObject({ id: f.runId, status: "failed", finishedAt: failed.finishedAt,
@@ -11,7 +11,7 @@ import { setTimeout as delay } from "node:timers/promises";
import { eq } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { agents, companies, completionContracts, createDb, environmentLeases, environments, heartbeatRuns, issues, nativeRunFinalizations, type Db } from "@paperclipai/db";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
import { heartbeatService } from "../services/heartbeat.js";
import { environmentRuntimeService, type EnvironmentDriver } from "../services/environment-runtime.js";
@@ -180,7 +180,7 @@ describe("native environment release scheduling", () => {
}
const events: Record<string, unknown>[] = [];
const now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { nativeRunFinalizations, heartbeatRuns, environmentLeases }, capture);
const settle = vi.fn(async () => {});
const work = nativeEnvironmentSettlementWork(db, settle);
@@ -1,7 +1,7 @@
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db";
import { randomBytes, randomUUID } from "node:crypto";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import { spawn } from "node:child_process";
import os from "node:os";
import { and, eq, sql } from "drizzle-orm";
@@ -222,11 +222,11 @@ suite("settled native run process identity", () => {
const saved = await readRun(runId);
const events: Array<Record<string, unknown>> = [];
const now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
try { expect(await releaseSettledNativeRunProcess(db, runId)).toBe("released"); }
finally { capture.flush(); }
const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!;
const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!;
expect(summary.responseValueBytes).toBeLessThan(8192);
const after = await readRun(runId);
expect(after).toEqual({ ...saved, processPid: null, processGroupId: null, processStartedAt: null,
@@ -0,0 +1,729 @@
import { randomUUID } from "node:crypto";
import { databaseWorkPending } from "@paperclipai/db/work-signals";
import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js";
import { writeFile } from "node:fs/promises";
import path from "node:path";
import type { PluginEnvironmentTaskOperation } from "@paperclipai/plugin-sdk";
import type { PaperclipPluginManifestV1 } from "@paperclipai/shared";
import { and, eq, sql } from "drizzle-orm";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import {
activityLog, agents, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, environmentLeases,
environments, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults, plugins, sandboxCreationOperations,
} from "@paperclipai/db";
import { recoveryPostgresSupport, startRecoveryPostgres } from "./helpers/recovery-postgres.js";
import { createIsolatedTestHome } from "./helpers/isolated-test-home.js";
import { prepareNativeHeartbeatRun } from "../services/native-runtime/prepare-native-run.js";
import {
acknowledgeNativeStartupCleanup, NATIVE_STARTUP_CLEANUP_PENDING, readNativeStartupCleanup,
requestNativeStartupCleanup, verifyNativeStartupCleanup, clearNativeStartupContext,
} from "../services/native-runtime/native-startup-cleanup.js";
import { isNativeRunnerOwnershipHeld } from "../services/native-runtime/native-runner-ownership.js";
import { cancelNativeSession } from "../services/native-runtime/native-session-executor.js";
import * as nativeSessionExecutor from "../services/native-runtime/native-session-executor.js";
import { NativeExecutionContext } from "../services/native-runtime/native-execution-context.js";
import type { NativeExecutionInput } from "../vendor/paperclip-runner/index.js";
import { callbacks } from "../services/heartbeat/__tests__/run-control-fixtures.js";
import { createHeartbeatRunControl } from "../services/heartbeat/run-control.js";
import { nextEnvironmentCleanupAt } from "../services/environment-cleanup-state.js";
import { executionControlSchedule } from "../services/execution-control-work.js";
import { claimNativeRestartRecoveries } from "../services/native-runtime/native-restart-recovery.js";
import { reconcileAbandonedExecutionControl } from "../services/execution-control-reconciliation.js";
import { createPluginWorkerManager, type PluginWorkerManager } from "../services/plugin-worker-manager.js";
import { hasRemoteTerminationReceipt, remoteTerminationReceipt } from "../services/remote-execution-termination.js";
import { createOwnedSandbox, queueSandboxCreationCleanup, recordSandboxCreationResource, recoverSandboxCreations } from "../services/sandbox-creation-operations.js";
import { environmentService } from "../services/environments.js";
import { environmentRuntimeService } from "../services/environment-runtime.js";
import { createHeartbeatRecovery, type HeartbeatRecoveryDependencies } from "../services/heartbeat/recovery.js";
const describePostgres = recoveryPostgresSupport.supported ? describe : describe.skip;
describePostgres("durable cleanup after a native startup timeout", () => {
let temporary: Awaited<ReturnType<typeof startRecoveryPostgres>>;
let db: ReturnType<typeof createDb>;
let home: ReturnType<typeof createIsolatedTestHome>;
const originalHome = process.env.PAPERCLIP_HOME;
const companyId = randomUUID(), agentId = randomUUID();
beforeAll(async () => {
home = createIsolatedTestHome();
process.env.PAPERCLIP_HOME = home.home;
temporary = await startRecoveryPostgres("native-startup-cleanup-");
db = createDb(temporary.connectionString);
await db.insert(companies).values({ id: companyId, name: "Startup cleanup", issuePrefix: "NSC" });
await db.insert(agents).values({ id: agentId, companyId, name: "Runner", role: "engineer", status: "running",
adapterType: "paperclip_runner", adapterConfig: { provider: "codex" }, runtimeConfig: {}, permissions: {} });
}, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
afterAll(async () => {
await temporary?.cleanup();
home?.cleanup();
if (originalHome === undefined) delete process.env.PAPERCLIP_HOME;
else process.env.PAPERCLIP_HOME = originalHome;
});
async function fixture(requiresPreparationStop = false) {
const runId = randomUUID(), issueId = randomUUID(), leaseId = randomUUID(), taskId = randomUUID();
const contextId = randomUUID(), leaseOwner = randomUUID();
await db.insert(issues).values({ id: issueId, companyId, identifier: `NSC-${issueId}`, title: "Timed out startup",
status: "in_progress", priority: "medium", assigneeAgentId: agentId });
const [run] = await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running",
runtimeMode: "native", nativeIssueId: issueId, invocationSource: "manual", contextSnapshot: { issueId } }).returning();
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId));
await prepareNativeHeartbeatRun({ db, run, issue: { id: issueId, title: "Timed out startup", description: null }, environmentLeaseId: leaseId });
await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId, phase: "observed", attempt: 1,
controllerGeneration: 1, controllerBootId: randomUUID(), controllerPid: process.pid,
controllerProcessStartedAt: new Date(), leaseOwner, leaseExpiresAt: new Date(Date.now() + 60_000) });
const [environment] = await db.insert(environments).values({ name: `Cleanup ${runId}`, driver: "plugin", status: "active", config: {} }).returning();
await db.insert(environmentLeases).values({ id: leaseId, companyId, environmentId: environment.id, heartbeatRunId: runId,
issueId, status: "active", provider: "plugin", providerLeaseId: taskId, leasePolicy: "reuse_by_environment",
metadata: { runnerTaskLease: true, runnerTaskReused: true } });
const request = { companyId, agentId, runId, contextId, leaseOwner, deadlineAtMs: Date.now() - 1, requiresPreparationStop };
const reload = () => db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)).then(rows => rows[0]!);
const owner = () => db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)).then(rows => rows[0]!);
const lease = () => db.select().from(environmentLeases).where(eq(environmentLeases.id, leaseId)).then(rows => rows[0]!);
const stopReceipt = async (override: Record<string, unknown> = {}, preparationStopped = false) => {
const current = await lease();
const receipt = remoteTerminationReceipt(current, { providerLeaseId: taskId, state: "stopped" });
await db.update(environmentLeases).set({ status: "released", releasedAt: new Date(), cleanupStatus: "success",
metadata: { ...current.metadata, ...(preparationStopped ? { runnerTaskPreparationStopped: true } : {}), remoteExecutionTermination: { ...receipt, ...override } } }).where(eq(environmentLeases.id, leaseId));
};
const acceptedResult = async () => {
const current = await reload(), resultId = randomUUID();
await db.insert(nativeRunResults).values({ id: resultId, companyId, issueId, runId,
completionContractId: current.completionContractId!, serverFingerprint: resultId, schemaStatus: "accepted",
resultJson: { schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", summary: "Completed work" }, canonicalSha256: "a".repeat(64) });
await db.update(nativeRunFinalizations).set({ resultId, phase: "workspace_finalizing" }).where(eq(nativeRunFinalizations.runId, runId));
return resultId;
};
return { request, reload, owner, lease, stopReceipt, acceptedResult };
}
async function saveEarlyContext(f: Awaited<ReturnType<typeof fixture>>) {
const run = await f.reload(), owner = await f.owner(), lease = await f.lease();
const context = { schema: "paperclip.native-startup-context.v1", companyId, agentId, runId: run.id,
contextId: f.request.contextId, deadlineAtMs: f.request.deadlineAtMs, leaseId: lease.id,
leaseOwner: owner.leaseOwner, controllerGeneration: owner.controllerGeneration, attempt: owner.attempt };
await db.update(heartbeatRuns).set({ runnerProfileJson: { ...run.runnerProfileJson, nativeStartupContext: context } }).where(eq(heartbeatRuns.id, run.id));
return context;
}
it.each(["pending", "needs_reconciliation"])("keeps ownership until a %s replacement queued after the first teardown snapshot is verified stopped", async state => {
const f = await fixture(true), original = await f.lease();
let reply!: (result: { providerLeaseId: string }) => void, creating!: () => void;
const entered = new Promise<void>(resolve => { creating = resolve; });
const acquiring = createOwnedSandbox(db, { companyId, environmentId: original.environmentId!, heartbeatRunId: f.request.runId,
issueId: original.issueId, provider: original.provider!, driverKey: "tasks", config: {}, requestKey: original.providerLeaseId!,
metadata: { runnerTaskLease: true, runnerTaskCreationContextId: f.request.contextId } }, async () => new Promise(resolve => { reply = resolve; creating(); }));
await entered;
await requestNativeStartupCleanup(db, f.request);
await f.stopReceipt({}, true);
await expect(verifyNativeStartupCleanup(db, await f.reload(), async () => {})).rejects.toThrow("cleanup is still pending");
const [pending] = await db.select().from(sandboxCreationOperations).where(and(
eq(sandboxCreationOperations.runId, f.request.runId), eq(sandboxCreationOperations.requestKey, original.providerLeaseId!)));
if (state === "needs_reconciliation") {
await db.update(sandboxCreationOperations).set({ nextAttemptAt: new Date(0) }).where(eq(sandboxCreationOperations.id, pending.id));
await recoverSandboxCreations(db, async () => null);
expect((await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.id, pending.id)))[0].state).toBe(state);
const inspect = vi.fn(async () => null);
await recoverSandboxCreations(db, inspect);
expect(inspect).not.toHaveBeenCalled();
await expect(queueSandboxCreationCleanup(db, { id: pending.id, companyId, environmentId: original.environmentId!,
providerLeaseId: "unproven-task" })).rejects.toThrow("ownership changed");
}
reply({ providerLeaseId: "late-replacement" });
const fresh = await acquiring;
await recordSandboxCreationResource(db, { id: String(fresh.metadata!.creationOperationId), companyId,
environmentId: original.environmentId!, providerLeaseId: fresh.providerLeaseId, providerMetadata: { binding: "actual" } });
if (state === "pending") await db.update(sandboxCreationOperations).set({ nextAttemptAt: new Date(0) })
.where(eq(sandboxCreationOperations.id, String(fresh.metadata!.creationOperationId)));
expect(await nextEnvironmentCleanupAt(db)).not.toBeNull();
const restarted = createDb(temporary.connectionString);
try { await recoverSandboxCreations(restarted, async () => { throw new Error("A recorded task identity needs no new provider inspection"); }); }
finally { await restarted.$client.end(); }
const [creation] = await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.id, String(fresh.metadata!.creationOperationId)));
await expect(verifyNativeStartupCleanup(db, await f.reload(), async () => {})).rejects.toThrow("cleanup is still pending");
const replacement = (await environmentService(db).getLeaseById(creation.leaseId!))!;
await queueSandboxCreationCleanup(db, { id: creation.id, companyId, environmentId: original.environmentId!, providerLeaseId: fresh.providerLeaseId });
expect((await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.id, creation.id)))[0].leaseId).toBe(replacement.id);
await expect(queueSandboxCreationCleanup(db, { id: creation.id, companyId, environmentId: original.environmentId!, providerLeaseId: "foreign-task" })).rejects.toThrow("ownership changed");
await environmentService(db).releaseLease(replacement.id, "released", { cleanupStatus: "success",
remoteExecutionTermination: remoteTerminationReceipt(replacement, { providerLeaseId: replacement.providerLeaseId, state: "stopped" }) });
expect((await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.id, creation.id)))[0].state).toBe("resolved");
await queueSandboxCreationCleanup(db, { id: creation.id, companyId, environmentId: original.environmentId!, providerLeaseId: fresh.providerLeaseId });
expect((await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.id, creation.id)))[0])
.toMatchObject({ state: "resolved", leaseId: replacement.id });
await verifyNativeStartupCleanup(db, await f.reload(), async () => {});
await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, f.request.runId));
await acknowledgeNativeStartupCleanup(db, f.request);
expect((await f.owner()).leaseOwner).toBeNull();
});
it.each(["before_inventory", "before_write", "complete_before_write", "ordinary"])("preserves startup cleanup through orphan recovery: %s", async phase => {
const f = await fixture(), deps = callbacks(db);
await db.update(nativeRunFinalizations).set({ phase: "terminal_failure", leaseExpiresAt: new Date(0) })
.where(eq(nativeRunFinalizations.runId, f.request.runId));
if (phase === "before_inventory") await requestNativeStartupCleanup(db, f.request);
const write = deps.setRunStatusFromLive.getMockImplementation()!;
deps.setRunStatusFromLive.mockImplementation(async (...args) => {
if (phase === "before_write" || phase === "complete_before_write") {
await requestNativeStartupCleanup(db, f.request);
if (phase === "complete_before_write") await verifyNativeStartupCleanup(db, await f.reload(), () => f.stopReceipt());
}
return write(...args);
});
const recovery = createHeartbeatRecovery(db, { ...deps, environmentRuntime: environmentRuntimeService(db),
environmentsSvc: environmentService(db), runtimeEnv: {}, activeRunExecutionPromises: new Set<Promise<void>>(),
mergeRunStopMetadataForAgent: (_agent: unknown, _status: unknown, options: { resultJson?: object }) => options?.resultJson,
classifyAndPersistRunLiveness: async (run: unknown) => run, releaseEnvironmentLeasesForRun: vi.fn(),
releaseIssueExecutionAndPromote: vi.fn(), scheduleInteractionContinuationInfrastructureRetryIfEligible: async () => null,
} as unknown as HeartbeatRecoveryDependencies);
await recovery.recoverActiveExecutions({ runId: f.request.runId }, { staleThresholdMs: 0 });
expect((await f.reload()).status).toBe(phase === "ordinary" ? "failed" : "running");
if (phase === "ordinary") return;
expect((await f.owner()).leaseOwner).toBe(f.request.leaseOwner);
deps.verifyStartupCleanup = run => verifyNativeStartupCleanup(db, run, () => f.stopReceipt());
expect(await createHeartbeatRunControl(db, deps).cancelRunInternal(f.request.runId, "Startup cleanup retry"))
.toMatchObject({ status: "cancelled" });
expect((await f.owner()).leaseOwner).toBeNull();
});
it.each(["request", "acknowledge"])("registers queue intent before cleanup %s writes", async phase => {
const f = await fixture(true);
const writes: boolean[][] = [];
const observe = async (queues: string[], work: () => Promise<unknown>) => {
const transaction = db.transaction.bind(db);
const spy = vi.spyOn(db, "transaction").mockImplementation(((callback: (tx: typeof db) => Promise<unknown>) => transaction(async tx => {
const update = tx.update.bind(tx);
tx.update = ((table: unknown) => {
if ([heartbeatRuns, nativeRunFinalizations, environmentLeases].includes(table as never)) {
writes.push(queues.map(queue => databaseWorkPending(db, queue)));
}
return update(table as never);
}) as typeof tx.update;
return callback(tx as unknown as typeof db);
})) as typeof db.transaction);
try { await work(); } finally { spy.mockRestore(); }
};
if (phase === "request") {
await observe([DELIVERY_QUEUES.executionFinalization, DELIVERY_QUEUES.environmentCleanup], () => requestNativeStartupCleanup(db, f.request));
expect(writes).toEqual([[true, true], [true, true], [true, true]]);
await verifyNativeStartupCleanup(db, await f.reload(), () => f.stopReceipt({}, true));
await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, f.request.runId));
await acknowledgeNativeStartupCleanup(db, f.request);
} else {
await requestNativeStartupCleanup(db, f.request);
await verifyNativeStartupCleanup(db, await f.reload(), () => f.stopReceipt({}, true));
await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, f.request.runId));
await observe([DELIVERY_QUEUES.agentLifecycle, DELIVERY_QUEUES.environmentCleanup], () => acknowledgeNativeStartupCleanup(db, f.request));
expect(writes).toEqual([[true, true]]);
}
});
it("returns the deadline queue to quiet and wakes again for the next startup failure", async () => {
const schedule = executionControlSchedule(db);
expect(await schedule.finalization()).toBeNull();
for (let failure = 0; failure < 2; failure += 1) {
const f = await fixture(true);
await requestNativeStartupCleanup(db, f.request);
expect(await schedule.finalization()).not.toBeNull();
const deps = callbacks(db);
deps.verifyStartupCleanup = run => verifyNativeStartupCleanup(db, run, () => f.stopReceipt({}, true));
const control = createHeartbeatRunControl(db, deps);
await reconcileAbandonedExecutionControl(db, new Date(Date.now() + 1_000), { startupCleanup: async scope => {
if (scope.runId !== f.request.runId) return;
await control.cancelRunInternal(scope.runId, "Startup cleanup retry");
await acknowledgeNativeStartupCleanup(db, scope);
} });
expect((await f.reload()).status).toBe("cancelled");
expect(await f.owner()).toMatchObject({ leaseOwner: null, controlDeadlineAt: null });
expect(await schedule.finalization()).toBeNull();
}
});
it("discovers a crash after warm prepare dispatch but before a cleanup intent was recorded", async () => {
const f = await fixture(true), lease = await f.lease();
await db.update(environmentLeases).set({ metadata: { ...lease.metadata, runnerTaskStartup: {
contextId: f.request.contextId, deadlineAtMs: f.request.deadlineAtMs, runId: f.request.runId,
} } }).where(eq(environmentLeases.id, lease.id));
await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 1_000) })
.where(eq(nativeRunFinalizations.runId, f.request.runId));
expect(readNativeStartupCleanup(await f.reload())).toBeNull();
const onClaim = vi.fn();
const result = await claimNativeRestartRecoveries({ db, restartKind: "hard", runIds: [f.request.runId],
controller: { bootId: randomUUID(), pid: process.pid, processStartedAt: new Date() }, onClaim });
expect(result).toEqual([{ kind: "blocked", runId: f.request.runId, reason: NATIVE_STARTUP_CLEANUP_PENDING }]);
expect(onClaim).not.toHaveBeenCalled();
expect(readNativeStartupCleanup(await f.reload())).toMatchObject({ contextId: f.request.contextId,
state: "pending", requiresPreparationStop: true });
expect(isNativeRunnerOwnershipHeld(await f.reload())).toBe(false);
});
it.each(["wrong_run", "future_deadline", "live_owner", "checkpointed", "missing_marker"])(
"does not infer a timed-out startup from %s", async variant => {
const f = await fixture(true), lease = await f.lease();
if (variant !== "missing_marker") await db.update(environmentLeases).set({ metadata: { ...lease.metadata,
runnerTaskStartup: { contextId: f.request.contextId, runId: variant === "wrong_run" ? randomUUID() : f.request.runId,
deadlineAtMs: variant === "future_deadline" ? Date.now() + 60_000 : f.request.deadlineAtMs },
} }).where(eq(environmentLeases.id, lease.id));
await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() + (variant === "live_owner" ? 60_000 : -1_000)) })
.where(eq(nativeRunFinalizations.runId, f.request.runId));
if (variant === "checkpointed") await db.update(heartbeatRuns).set({ sessionIdAfter: "current-run-checkpoint" })
.where(eq(heartbeatRuns.id, f.request.runId));
await claimNativeRestartRecoveries({ db, restartKind: "hard", runIds: [f.request.runId],
controller: { bootId: randomUUID(), pid: process.pid, processStartedAt: new Date() } });
expect(readNativeStartupCleanup(await f.reload())).toBeNull();
});
it("persists an audited cancellation intent that a new database handle can recover without a generic hold", async () => {
const f = await fixture();
expect(await requestNativeStartupCleanup(db, f.request)).toBe(true);
const run = await f.reload(), marker = readNativeStartupCleanup(run)!;
expect(marker).toMatchObject({ state: "pending", contextId: f.request.contextId, leaseOwner: f.request.leaseOwner,
controllerGeneration: 1, attempt: 1, taskId: (await f.lease()).providerLeaseId });
expect(run).toMatchObject({ status: "running", errorCode: NATIVE_STARTUP_CLEANUP_PENDING });
expect(run.resultJson?.startupCancellation).toMatchObject({ requestedAt: expect.any(String) });
expect(isNativeRunnerOwnershipHeld(run)).toBe(false);
expect(await db.select({ id: activityLog.id }).from(activityLog).where(and(eq(activityLog.id, marker.auditId),
eq(activityLog.companyId, companyId), eq(activityLog.action, "native.startup_cleanup_requested")))).toHaveLength(1);
const restarted = createDb(temporary.connectionString), release = vi.fn(f.stopReceipt);
await verifyNativeStartupCleanup(restarted, await f.reload(), release);
await verifyNativeStartupCleanup(restarted, await f.reload(), release);
expect(release).toHaveBeenCalledOnce();
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("complete");
await acknowledgeNativeStartupCleanup(restarted, { companyId, runId: run.id });
expect((await f.owner()).controlDeadlineAt).not.toBeNull();
await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, run.id));
await acknowledgeNativeStartupCleanup(restarted, { companyId, runId: run.id });
expect(await f.owner()).toMatchObject({ leaseOwner: null, leaseExpiresAt: null, controlDeadlineAt: null, recoveryState: null });
});
it("keeps cleanup bound to the explicit current lease when a released alias belongs to the same run", async () => {
const f = await fixture(true), current = await f.lease();
const aliasId = randomUUID();
await db.insert(environmentLeases).values({ ...current, id: aliasId, providerLeaseId: randomUUID(),
status: "released", releasedAt: new Date(), cleanupStatus: "success" });
expect(await requestNativeStartupCleanup(db, { ...f.request, leaseId: current.id })).toBe(true);
expect(readNativeStartupCleanup(await f.reload())).toMatchObject({ leaseId: current.id, taskId: current.providerLeaseId });
expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, aliasId)))[0]?.metadata?.runnerTaskStartupCleanup).toBeUndefined();
});
it("recovers an exact Stop receipt committed before Core records completion", async () => {
const f = await fixture();
await requestNativeStartupCleanup(db, f.request);
await f.stopReceipt();
const release = vi.fn(async () => { throw new Error("Already retired task must not be released twice"); });
await verifyNativeStartupCleanup(createDb(temporary.connectionString), await f.reload(), release);
expect(release).not.toHaveBeenCalled();
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("complete");
});
it("keeps accelerated preparation pending when only the provider has stopped", async () => {
const f = await fixture(true);
await requestNativeStartupCleanup(db, f.request);
await expect(verifyNativeStartupCleanup(db, await f.reload(), () => f.stopReceipt())).rejects.toThrow(/pending|preparation/i);
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("pending");
expect((await f.owner()).leaseOwner).toBe(f.request.leaseOwner);
});
it("automatically finishes pending cleanup after a late real worker reply and exact preparation Stop proof", async () => {
const f = await fixture(true), lease = await f.lease();
const entrypointPath = path.join(home.home, `cleanup-worker-${f.request.runId}.cjs`);
await writeFile(entrypointPath, `
const rl=require("node:readline").createInterface({input:process.stdin,crlfDelay:Infinity});
const held=[];let stopped=false;
const send=(id,result)=>process.stdout.write(JSON.stringify({jsonrpc:"2.0",id,result})+"\\n");
rl.on("line",line=>{const m=JSON.parse(line);
if(m.method==="initialize")send(m.id,{ok:true,supportedMethods:["environmentTask","configChanged"]});
else if(m.method==="environmentTask"){
const kind=m.params.operation.kind;
if(kind==="prepare")held.push(m.id);
else if(kind==="status")send(m.id,{kind:"status",taskId:m.params.taskId,phase:stopped?"cancelled":"running",executionStopped:stopped,preparationStopped:stopped&&held.length===0});
else {if(kind==="stop")stopped=true;send(m.id,{kind:"accepted",taskId:m.params.taskId});}
}else if(m.method==="configChanged"){for(const id of held.splice(0))send(id,{});send(m.id,{});}
else if(m.method==="shutdown"){send(m.id,{});setImmediate(()=>process.exit(0));}
});
`, { mode: 0o600 });
const manifest: PaperclipPluginManifestV1 = { id: "test.startup-cleanup", apiVersion: 1, version: "1.0.0",
displayName: "Cleanup fixture", description: "Cleanup fixture", author: "Paperclip", categories: ["automation"],
capabilities: [], entrypoints: { worker: "worker.cjs" } };
const manager = createPluginWorkerManager();
await manager.startWorker(manifest.id, { entrypointPath, manifest, config: {},
instanceInfo: { instanceId: "cleanup", hostVersion: "1.0.0" }, apiVersion: 1, hostHandlers: {} });
const context = new NativeExecutionContext({ binding: { companyId, agentId, runId: f.request.runId } } as NativeExecutionInput, 100);
const params = { companyId, agentId, runId: f.request.runId, environmentId: lease.environmentId, issueId: lease.issueId,
driverKey: "tasks", taskId: lease.providerLeaseId!, config: {}, projectIds: [], lease: { providerLeaseId: lease.providerLeaseId! } };
const operation = { kind: "prepare", projectIds: [], requirements: { schema: "paperclip.runner.launch-requirements.v1",
identity: { runnerInstanceId: "fixture", environmentLeaseId: lease.id, runId: f.request.runId,
normalizedSessionId: "session", turnId: "turn", itemId: "item" }, provider: "codex", model: null,
instructions: "Synthetic", runtimeContext: null, settings: {}, approvalPolicy: "never", readOnly: false,
preparedConversation: false, includeCollaborationModeInstructions: false } } as PluginEnvironmentTaskOperation;
const release = async () => {
await manager.call(manifest.id, "environmentTask", { ...params, operation: { kind: "stop" } });
const status = await manager.call(manifest.id, "environmentTask", { ...params, operation: { kind: "status" } });
if (status.kind === "status" && status.executionStopped && status.preparationStopped) {
await manager.call(manifest.id, "environmentTask", { ...params, operation: { kind: "complete" } });
await f.stopReceipt({}, true);
}
};
try {
await expect(manager.call(manifest.id, "environmentTask", { ...params, operation }, 10, undefined, context.trackWorker())).rejects.toThrow("timed out");
if (!context.signal.aborted) await new Promise<void>(resolve => context.signal.addEventListener("abort", () => resolve(), { once: true }));
expect(context.pending).toBe(true);
expect(await requestNativeStartupCleanup(db, { ...f.request, contextId: context.id, deadlineAtMs: context.deadlineAtMs })).toBe(true);
expect(isNativeRunnerOwnershipHeld(await f.reload())).toBe(false);
await expect(cancelNativeSession(f.request.runId, "Startup timeout", { db, scope: "run" })).resolves.toMatchObject({ auditId: expect.any(String) });
await expect(verifyNativeStartupCleanup(db, await f.reload(), release)).rejects.toThrow("pending");
expect((await f.reload()).status).toBe("running");
await manager.call(manifest.id, "configChanged", { companyId, config: { release: true } });
expect(context.pending).toBe(false);
await verifyNativeStartupCleanup(createDb(temporary.connectionString), await f.reload(), release);
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("complete");
expect((await f.lease()).metadata?.runnerTaskPreparationStopped).toBe(true);
} finally { context.finish(); await manager.stopAll(); }
});
it.each([false, true])("verifies an abandoned ephemeral startup without an inactive-owner acknowledgement (receipt=%s)", async stopped => {
const prior = process.env.PAPERCLIP_EPHEMERAL_SERVER;
process.env.PAPERCLIP_EPHEMERAL_SERVER = "true";
try {
const f = await fixture(true);
await requestNativeStartupCleanup(db, f.request);
const deps = callbacks(db);
deps.verifyStartupCleanup = vi.fn(run => verifyNativeStartupCleanup(db, run, async () => { if (stopped) await f.stopReceipt({}, true); }));
const control = createHeartbeatRunControl(db, deps);
if (stopped) {
expect(await control.cancelRunInternal(f.request.runId, "Startup timeout")).toMatchObject({ status: "cancelled" });
expect((await f.owner()).leaseOwner).toBeNull();
} else {
await expect(control.cancelRunInternal(f.request.runId, "Startup timeout")).rejects.toThrow("Task cleanup is still pending");
expect((await f.reload()).status).toBe("running");
expect((await f.owner()).leaseOwner).toBe(f.request.leaseOwner);
}
expect(deps.verifyStartupCleanup).toHaveBeenCalled();
} finally {
if (prior === undefined) delete process.env.PAPERCLIP_EPHEMERAL_SERVER;
else process.env.PAPERCLIP_EPHEMERAL_SERVER = prior;
}
}, 30_000);
it("terminalizes through assembled run control only after verified cleanup succeeds", async () => {
const f = await fixture(true);
await requestNativeStartupCleanup(db, f.request);
let drained = false;
const deps = callbacks(db);
deps.verifyStartupCleanup = run => verifyNativeStartupCleanup(db, run, () => f.stopReceipt({}, drained));
const control = createHeartbeatRunControl(db, deps);
await expect(control.cancelRunInternal(f.request.runId, "Startup timeout")).rejects.toThrow("pending");
expect(await f.reload()).toMatchObject({ status: "running", errorCode: NATIVE_STARTUP_CLEANUP_PENDING });
expect((await f.owner()).leaseOwner).toBe(f.request.leaseOwner);
drained = true;
const result = await control.cancelRunInternal(f.request.runId, "Startup timeout");
expect(result).toMatchObject({ status: "cancelled" });
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("complete");
expect(await f.owner()).toMatchObject({ leaseOwner: null, leaseExpiresAt: null, controlDeadlineAt: null });
});
it("joins cleanup recorded while an operator Stop waits for native startup", async () => {
const f = await fixture(true);
let entered!: () => void, resume!: () => void;
const waiting = new Promise<void>(resolve => { entered = resolve; });
const settled = new Promise<void>(resolve => { resume = resolve; });
const cancel = nativeSessionExecutor.cancelNativeSession;
const spy = vi.spyOn(nativeSessionExecutor, "cancelNativeSession").mockImplementation(async (...args) => {
const result = await cancel(...args);
if (args[0] === f.request.runId) { entered(); await settled; }
return result;
});
let drained = false;
const deps = callbacks(db);
deps.verifyStartupCleanup = vi.fn(run => verifyNativeStartupCleanup(db, run, () => f.stopReceipt({}, drained)));
const control = createHeartbeatRunControl(db, deps);
try {
const stopping = control.cancelRunInternal(f.request.runId, "Operator Stop");
const outcome = stopping.then(value => ({ value }), error => ({ error }));
await waiting;
expect(await requestNativeStartupCleanup(db, f.request)).toBe(true);
resume();
expect(await outcome).toMatchObject({ error: expect.objectContaining({ message: "Task cleanup is still pending" }) });
expect(deps.verifyStartupCleanup).toHaveBeenCalledOnce();
expect((await f.reload()).status).toBe("running");
expect((await f.owner()).leaseOwner).toBe(f.request.leaseOwner);
drained = true;
expect(await control.cancelRunInternal(f.request.runId, "Operator Stop")).toMatchObject({ status: "cancelled" });
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("complete");
expect((await f.owner()).leaseOwner).toBeNull();
} finally { resume(); spy.mockRestore(); }
});
it("rejects a cancellation commit when cleanup arrives after its final read", async () => {
const f = await fixture(true);
const deps = callbacks(db), commit = deps.setRunStatusFromLive.getMockImplementation()!;
deps.setRunStatusFromLive.mockImplementationOnce(async (...args) => {
expect(await requestNativeStartupCleanup(db, f.request)).toBe(true);
return commit(...args);
});
deps.verifyStartupCleanup = vi.fn(run => verifyNativeStartupCleanup(db, run, () => f.stopReceipt({}, true)));
const control = createHeartbeatRunControl(db, deps);
expect(await control.cancelRunInternal(f.request.runId, "Operator Stop")).toMatchObject({ status: "running" });
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("pending");
expect((await f.owner()).leaseOwner).toBe(f.request.leaseOwner);
expect(deps.startNextQueuedRunForAgent).not.toHaveBeenCalled();
expect(await control.cancelRunInternal(f.request.runId, "Operator Stop")).toMatchObject({ status: "cancelled" });
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("complete");
expect((await f.owner()).leaseOwner).toBeNull();
});
it("restarts the real cancellation flow after remote Stop committed before Core acknowledgement", async () => {
const f = await fixture(true);
await requestNativeStartupCleanup(db, f.request);
await f.stopReceipt({}, true);
const restarted = createDb(temporary.connectionString), release = vi.fn(async () => { throw new Error("Stop receipt already persisted"); });
const deps = callbacks(restarted);
deps.verifyStartupCleanup = run => verifyNativeStartupCleanup(restarted, run, release);
const control = createHeartbeatRunControl(restarted, deps);
const reconcile = vi.fn(async (scope: { companyId: string; runId: string }) => {
if (scope.runId !== f.request.runId) return;
expect(scope.companyId).toBe(companyId);
await control.cancelRunInternal(scope.runId, "Startup timeout recovered after restart");
await acknowledgeNativeStartupCleanup(restarted, scope);
});
await reconcileAbandonedExecutionControl(restarted, new Date(Date.now() + 1_000), { startupCleanup: reconcile });
expect(reconcile.mock.calls.some(([scope]) => scope.runId === f.request.runId)).toBe(true);
expect(release).not.toHaveBeenCalled();
expect((await f.reload()).status).toBe("cancelled");
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("complete");
expect(await f.owner()).toMatchObject({ leaseOwner: null, leaseExpiresAt: null, controlDeadlineAt: null });
});
it.each(["providerLeaseId", "runId", "leaseId"])("rejects a Stop receipt for another %s", async field => {
const f = await fixture();
await requestNativeStartupCleanup(db, f.request);
await expect(verifyNativeStartupCleanup(db, await f.reload(), () => f.stopReceipt({ [field]: randomUUID() }))).rejects.toThrow("pending");
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("pending");
});
it("does not issue a Stop against a newer controller generation", async () => {
const f = await fixture();
await requestNativeStartupCleanup(db, f.request);
await db.update(nativeRunFinalizations).set({ controllerGeneration: 2 }).where(eq(nativeRunFinalizations.runId, f.request.runId));
const release = vi.fn(f.stopReceipt);
await expect(verifyNativeStartupCleanup(db, await f.reload(), release)).rejects.toThrow("ownership changed");
expect(release).not.toHaveBeenCalled();
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("pending");
});
it("rejects a stale generation that changes while Stop is in flight", async () => {
const f = await fixture();
await requestNativeStartupCleanup(db, f.request);
await expect(verifyNativeStartupCleanup(db, await f.reload(), async () => {
await f.stopReceipt();
await db.update(nativeRunFinalizations).set({ controllerGeneration: 2 }).where(eq(nativeRunFinalizations.runId, f.request.runId));
})).rejects.toThrow("ownership changed");
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("pending");
});
it("preserves a final result that wins while verified cleanup is in flight", async () => {
const f = await fixture();
await requestNativeStartupCleanup(db, f.request);
let resultId: string;
await expect(verifyNativeStartupCleanup(db, await f.reload(), async () => {
await f.stopReceipt();
resultId = await f.acceptedResult();
})).rejects.toThrow("ownership changed");
expect(await f.owner()).toMatchObject({ resultId: resultId!, phase: "workspace_finalizing" });
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("pending");
});
it.each(["native_adopted_runner_authentication_timeout", "native_execution_ownership_unverified"])("preserves the existing %s hold", async errorCode => {
const f = await fixture();
await db.update(heartbeatRuns).set({ errorCode, nativePhase: "terminal_failure" }).where(eq(heartbeatRuns.id, f.request.runId));
expect(await requestNativeStartupCleanup(db, f.request)).toBe(false);
const current = await f.reload();
expect(current).toMatchObject({ status: "running", errorCode, nativePhase: "terminal_failure" });
expect(readNativeStartupCleanup(current)).toBeNull();
});
it("does not replace a final result with a timeout intent", async () => {
const f = await fixture();
const resultId = await f.acceptedResult();
expect(await requestNativeStartupCleanup(db, f.request)).toBe(false);
expect((await f.owner()).resultId).toBe(resultId);
expect(readNativeStartupCleanup(await f.reload())).toBeNull();
});
it("promotes a pre-capability context before an ordinary deadline after the audited intent write fails", async () => {
const f = await fixture();
await saveEarlyContext(f);
await db.execute(sql`create function reject_startup_intent() returns trigger language plpgsql as $$ begin
if new.action = 'native.startup_cleanup_requested' then raise exception 'Startup intent write unavailable'; end if; return new; end $$`);
await db.execute(sql`create trigger reject_startup_intent before insert on activity_log for each row execute function reject_startup_intent()`);
try { await expect(requestNativeStartupCleanup(db, f.request)).rejects.toThrow(); }
finally { await db.execute(sql`drop trigger reject_startup_intent on activity_log`); await db.execute(sql`drop function reject_startup_intent()`); }
expect(readNativeStartupCleanup(await f.reload())).toBeNull();
await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 1_000) }).where(eq(nativeRunFinalizations.runId, f.request.runId));
await db.update(heartbeatRuns).set({ executionControlDeadlineAt: new Date(Date.now() - 1_000) }).where(eq(heartbeatRuns.id, f.request.runId));
await reconcileAbandonedExecutionControl(db);
expect(readNativeStartupCleanup(await f.reload())).toMatchObject({ contextId: f.request.contextId, requiresPreparationStop: false, state: "pending" });
expect((await f.reload()).status).toBe("running");
expect((await f.owner()).leaseOwner).toBe(f.request.leaseOwner);
await verifyNativeStartupCleanup(db, await f.reload(), () => f.stopReceipt());
await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, f.request.runId));
await acknowledgeNativeStartupCleanup(db, f.request);
});
it.each([false, true])("recovers an early context without replacing stronger preparation proof (prepared=%s)", async prepared => {
const f = await fixture();
await saveEarlyContext(f);
if (prepared) { const lease = await f.lease(); await db.update(environmentLeases).set({ metadata: { ...lease.metadata,
runnerTaskStartup: { contextId: f.request.contextId, deadlineAtMs: f.request.deadlineAtMs, runId: f.request.runId } } }).where(eq(environmentLeases.id, lease.id)); }
await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 1_000) }).where(eq(nativeRunFinalizations.runId, f.request.runId));
const onClaim = vi.fn();
const result = await claimNativeRestartRecoveries({ db, restartKind: "hard", runIds: [f.request.runId],
controller: { bootId: randomUUID(), pid: process.pid, processStartedAt: new Date() }, onClaim });
expect(result).toEqual([{ kind: "blocked", runId: f.request.runId, reason: NATIVE_STARTUP_CLEANUP_PENDING }]);
expect(onClaim).not.toHaveBeenCalled();
expect(readNativeStartupCleanup(await f.reload())).toMatchObject({ requiresPreparationStop: prepared, state: "pending" });
await verifyNativeStartupCleanup(db, await f.reload(), () => f.stopReceipt({}, prepared));
await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, f.request.runId));
await acknowledgeNativeStartupCleanup(db, f.request);
});
it.each([false, true])("defers an early context to a concrete owner deadline (locally active=%s)", async active => {
const f = await fixture();
await saveEarlyContext(f);
const owner = await f.owner();
const result = await claimNativeRestartRecoveries({ db, restartKind: "hard", runIds: [f.request.runId],
liveRunExecutions: active ? new Set([f.request.runId]) : undefined,
controller: { bootId: randomUUID(), pid: process.pid, processStartedAt: new Date() } });
expect(result).toMatchObject([{ kind: "awaiting_evidence", runId: f.request.runId, retryAt: owner.leaseExpiresAt }]);
expect(readNativeStartupCleanup(await f.reload())).toBeNull();
});
it("does not turn an early context into cleanup after the current-run checkpoint", async () => {
const f = await fixture();
await saveEarlyContext(f);
await db.update(heartbeatRuns).set({ sessionIdAfter: "current-checkpoint" }).where(eq(heartbeatRuns.id, f.request.runId));
await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 1_000) }).where(eq(nativeRunFinalizations.runId, f.request.runId));
await claimNativeRestartRecoveries({ db, restartKind: "hard", runIds: [f.request.runId],
controller: { bootId: randomUUID(), pid: process.pid, processStartedAt: new Date() } });
expect(readNativeStartupCleanup(await f.reload())).toBeNull();
});
it("locks the selected lease before choosing the stronger preparation proof", async () => {
const f = await fixture(), lease = await f.lease();
let changed!: () => void, commit!: () => void;
const written = new Promise<void>(resolve => { changed = resolve; });
const release = new Promise<void>(resolve => { commit = resolve; });
const preparing = db.transaction(async tx => {
await tx.update(environmentLeases).set({ metadata: { ...lease.metadata, runnerTaskStartup: {
contextId: f.request.contextId, deadlineAtMs: f.request.deadlineAtMs, runId: f.request.runId,
} } }).where(eq(environmentLeases.id, lease.id));
changed();
await release;
});
await written;
const requesting = requestNativeStartupCleanup(db, f.request);
try {
await vi.waitFor(async () => {
const rows = await db.execute(sql`select query from pg_stat_activity where wait_event_type = 'Lock'
and query like 'select%from "environment_leases"%'`);
expect(rows.length).toBeGreaterThan(0);
});
} finally { commit(); }
await preparing;
await requesting;
expect(readNativeStartupCleanup(await f.reload())).toMatchObject({ requiresPreparationStop: true });
await expect(verifyNativeStartupCleanup(db, await f.reload(), () => f.stopReceipt())).rejects.toThrow("pending");
await verifyNativeStartupCleanup(db, await f.reload(), () => f.stopReceipt({}, true));
});
it("clears only its owned startup context before fallback and leaves no expired cleanup authority", async () => {
const f = await fixture();
await saveEarlyContext(f);
const scope = { companyId, runId: f.request.runId, contextId: f.request.contextId,
leaseOwner: f.request.leaseOwner, controllerGeneration: 1 };
await expect(clearNativeStartupContext(db, { ...scope, controllerGeneration: 2 })).rejects.toThrow("lost authority");
expect((await f.reload()).runnerProfileJson?.nativeStartupContext).toBeDefined();
await clearNativeStartupContext(db, scope);
expect((await f.reload()).runnerProfileJson?.nativeStartupContext).toBeUndefined();
await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 1_000) }).where(eq(nativeRunFinalizations.runId, f.request.runId));
await claimNativeRestartRecoveries({ db, restartKind: "hard", runIds: [f.request.runId],
controller: { bootId: randomUUID(), pid: process.pid, processStartedAt: new Date() } });
expect(readNativeStartupCleanup(await f.reload())).toBeNull();
});
it.each(["cleanup", "terminal"])("does not clear the startup breadcrumb after %s wins", async winner => {
const f = await fixture();
await saveEarlyContext(f);
if (winner === "cleanup") await requestNativeStartupCleanup(db, f.request);
else await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, f.request.runId));
await expect(clearNativeStartupContext(db, { companyId, runId: f.request.runId, contextId: f.request.contextId,
leaseOwner: f.request.leaseOwner, controllerGeneration: 1 })).rejects.toThrow("lost authority");
expect((await f.reload()).runnerProfileJson?.nativeStartupContext).toBeDefined();
if (winner === "cleanup") expect(readNativeStartupCleanup(await f.reload())).toMatchObject({ state: "pending" });
});
it("requires exact task Stop proof when retrying cleanup of a never-dispatched replacement", async () => {
const f = await fixture(true), original = await f.lease(), pluginId = randomUUID();
await db.insert(plugins).values({ id: pluginId, pluginKey: pluginId, packageName: "fixture-abandoned-task", version: "1", status: "ready",
manifestJson: { capabilities: ["environment.drivers.register"], environmentDrivers: [{ driverKey: "tasks", supportsTasks: true }] } as never });
await db.update(environments).set({ companyId, config: { pluginKey: pluginId, driverKey: "tasks", driverConfig: {} } })
.where(eq(environments.id, original.environmentId!));
const fresh = await createOwnedSandbox(db, { companyId, environmentId: original.environmentId!, heartbeatRunId: f.request.runId,
issueId: original.issueId, provider: original.provider!, pluginId, driverKey: "tasks", config: {}, requestKey: original.providerLeaseId!,
metadata: { driver: "plugin", pluginId, pluginKey: pluginId, driverKey: "tasks", runnerTaskLease: true, agentId,
runnerTaskCreationContextId: f.request.contextId } }, async () => ({ providerLeaseId: "late-never-dispatched" }));
await queueSandboxCreationCleanup(db, { id: String(fresh.metadata!.creationOperationId), companyId,
environmentId: original.environmentId!, providerLeaseId: fresh.providerLeaseId });
const creation = async () => (await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.id, String(fresh.metadata!.creationOperationId))))[0]!;
const replacement = async () => (await environmentService(db).getLeaseById((await creation()).leaseId!))!;
expect((await replacement()).metadata?.runnerTaskDispatched).not.toBe(true);
await requestNativeStartupCleanup(db, { ...f.request, leaseId: original.id });
await f.stopReceipt({}, true);
let proof: "void" | "running" | "foreign" | "exact" = "void";
const calls: string[] = [];
const workers = { isRunning: () => true, getWorker: () => ({ supportedMethods: ["environmentTask", "environmentReleaseLease"] }),
drainStartup: vi.fn(async () => undefined),
call: async (_plugin: string, method: string, params: any) => {
calls.push(method === "environmentTask" ? params.operation.kind : method);
if (method === "environmentReleaseLease") return undefined;
if (params.operation.kind === "status") return proof === "void" ? undefined : {
kind: "status", taskId: proof === "foreign" ? "another-task" : params.taskId,
phase: proof === "running" ? "running" : "cancelled", executionStopped: proof !== "running",
};
return { kind: "accepted", taskId: params.taskId };
},
} as unknown as PluginWorkerManager;
const runtime = environmentRuntimeService(db, { pluginWorkerManager: workers });
const recovery = createHeartbeatRecovery(db, { ...callbacks(db), environmentRuntime: runtime,
environmentsSvc: environmentService(db), runtimeEnv: {}, activeRunExecutionPromises: new Set<Promise<void>>(),
acknowledgeRemoteStop: async () => undefined, resumeRemoteStopComments: async () => undefined,
} as unknown as HeartbeatRecoveryDependencies);
const stopDeps = callbacks(db);
stopDeps.verifyStartupCleanup = run => verifyNativeStartupCleanup(db, run, async () => {});
const control = createHeartbeatRunControl(db, stopDeps);
for (const missing of ["void", "running", "foreign"] as const) {
proof = missing;
const swept = await recovery.sweepPendingCleanupLeases({ explicitRetry: { companyId, runId: f.request.runId, actorId: "operator" } });
const cancelled = await control.cancelRunInternal(f.request.runId, "Startup cleanup retry").then(() => true, () => false);
const lease = await replacement();
expect({ destroyed: swept.destroyed, status: lease.status, verifiedReceipt: hasRemoteTerminationReceipt(lease),
creationState: (await creation()).state, cancelled, leaseOwner: (await f.owner()).leaseOwner }).toEqual({
destroyed: 0, status: "pending_cleanup", verifiedReceipt: false, creationState: "cleanup_pending", cancelled: false, leaseOwner: f.request.leaseOwner,
});
}
proof = "exact";
expect((await recovery.sweepPendingCleanupLeases({ explicitRetry: { companyId, runId: f.request.runId, actorId: "operator" } })).destroyed).toBe(1);
expect(calls).toContain("stop");
expect(calls).toContain("status");
expect(calls).toContain("complete");
expect(workers!.drainStartup).toHaveBeenCalledWith(f.request.contextId);
expect(hasRemoteTerminationReceipt(await replacement())).toBe(true);
expect((await creation()).state).toBe("resolved");
expect(await control.cancelRunInternal(f.request.runId, "Startup cleanup retry")).toMatchObject({ status: "cancelled" });
expect((await f.owner()).leaseOwner).toBeNull();
});
});
@@ -332,6 +332,36 @@ describe("clampProviderSpanAttributes", () => {
});
describe("nested worker timing receipts",()=>{
it("keeps lease verification beneath its worker and clamps provider state", () => {
mockRecordSpan.mockReset();
const rootParent = { traceId: "0af7651916cd43dd8448eb211c80319c", spanId: "1111111111111111", traceFlags: 1 };
const verifyParent = { ...rootParent, spanId: "2222222222222222" };
mockRecordSpan.mockReturnValueOnce(rootParent).mockReturnValueOnce(verifyParent);
const now = Date.now();
recordWorkerProviderSpan({ name: "plugin.worker", startTimeMs: now - 100, endTimeMs: now, children: [
{ name: "lease.verify", startTimeMs: now - 90, endTimeMs: now - 10, children: [
{ name: "lease.sentinel", startTimeMs: now - 80, endTimeMs: now - 20,
attributes: { "paperclip.sandbox.startup.state": "archived", token: "private-token" } },
] },
] }, { traceparent: VALID_TRACEPARENT });
expect(mockRecordSpan.mock.calls[1][0]).toMatchObject({ name: "sandbox.daytona.lease.verify", parent: rootParent });
expect(mockRecordSpan.mock.calls[2][0]).toMatchObject({ name: "sandbox.daytona.lease.sentinel", parent: verifyParent,
startTimeMs: now - 80, endTimeMs: now - 20, attributes: { "paperclip.sandbox.startup.state": "archived" } });
expect(JSON.stringify(mockRecordSpan.mock.calls)).not.toContain("private-token");
expect(clampProviderSpanAttributes({ "paperclip.sandbox.startup.state": "private-state" })).toEqual({});
});
it("admits every closed lease phase and rejects dynamic phase names", () => {
mockRecordSpan.mockReset();
const names = ["lease.wait_for_teardown", "lease.lookup", "lease.refresh", "lease.channel_cleanup",
"lease.start", "lease.recover", "lease.verify", "lease.workspace", "lease.sentinel", "lease.shell",
"lease.create", "lease.expiry", "lease.cleanup", "lease.private-resource-id"];
for (const name of names) recordWorkerProviderSpan({ name }, { traceparent: VALID_TRACEPARENT });
expect(mockRecordSpan.mock.calls.map(call => call[0].name)).toEqual([
...names.slice(0, -1).map(name => `sandbox.daytona.${name}`), "sandbox.daytona.other",
]);
});
it("uses host-generated parents, clamps content and bounds child clocks",()=>{
mockRecordSpan.mockReset();
const childParent={traceId:"0af7651916cd43dd8448eb211c80319c",spanId:"1111111111111111",traceFlags:1};
+70
View File
@@ -899,3 +899,73 @@ second-line\" status=401`,
});
});
});
describe("redaction of long letter/digit runs", () => {
const typescriptSource = [
"export function compactRunLogChunk(chunk: string, maxChars = 65536) {",
" const normalized = redactSensitiveText(chunk);",
" if (normalized.length <= maxChars) return normalized;",
" return normalized.slice(0, maxChars);",
"}",
"",
].join("\n");
function base64OfTypeScriptSource(length: number) {
const repeatedSource = typescriptSource.repeat(Math.ceil(length / typescriptSource.length));
return Buffer.from(repeatedSource).toString("base64").slice(0, length);
}
function redactWithinTimeBudget(input: string) {
const startedAt = performance.now();
const output = redactSensitiveText(input);
expect(performance.now() - startedAt).toBeLessThan(1_000);
return output;
}
it("redacts secret field names that carry a prefix in plain and escaped JSON", () => {
const cases = [
[`{"myApiKey": "plain-prefixed-secret"}`, `{"myApiKey": "${REDACTED_EVENT_VALUE}"}`],
[`{"x_auth_token":"snake-prefixed-secret","status":"ok"}`, `{"x_auth_token":"${REDACTED_EVENT_VALUE}","status":"ok"}`],
[`GITHUB_TOKEN: "upper-prefixed-secret"`, `GITHUB_TOKEN: "${REDACTED_EVENT_VALUE}"`],
[`{'db_password': 'single-quoted-secret'}`, `{'db_password': '${REDACTED_EVENT_VALUE}'}`],
[`text"apiKey":"glued-quote-secret"`, `text"apiKey":"${REDACTED_EVENT_VALUE}"`],
[`aGVsbG8apiKey:"glued-run-secret"`, `aGVsbG8apiKey:"${REDACTED_EVENT_VALUE}"`],
[String.raw`{\"myApiKey\":\"escaped-prefixed-secret\"}`, String.raw`{\"myApiKey\":\"${REDACTED_EVENT_VALUE}\"}`],
[
String.raw`{\"x_auth_token\":\"escaped-snake-secret\",\"status\":\"ok\"}`,
String.raw`{\"x_auth_token\":\"${REDACTED_EVENT_VALUE}\",\"status\":\"ok\"}`,
],
[String.raw`aGVsbG8\"GITHUB_TOKEN\":\"escaped-glued-secret\"`, String.raw`aGVsbG8\"GITHUB_TOKEN\":\"${REDACTED_EVENT_VALUE}\"`],
];
for (const [input, expected] of cases) {
expect(redactSensitiveText(input)).toBe(expected);
expect(redactSensitiveText(expected)).toBe(expected);
}
});
it("keeps base64 of source text intact and fast while redacting a secret glued to its end", () => {
const blob = base64OfTypeScriptSource(48 * 1024);
expect(blob).toMatch(/^[A-Za-z0-9]+$/);
const shellCommand = `printf '%s' '${blob}' | base64 -d > src/compact.ts`;
expect(redactWithinTimeBudget(shellCommand)).toBe(shellCommand);
const toolCallLine = JSON.stringify({ type: "tool_use", input: { command: shellCommand } });
expect(redactWithinTimeBudget(toolCallLine)).toBe(toolCallLine);
const serializedToolCallLine = JSON.stringify(toolCallLine);
expect(redactWithinTimeBudget(serializedToolCallLine)).toBe(serializedToolCallLine);
const gluedSecret = redactWithinTimeBudget(`${blob}apiKey:"run-glued-secret"`);
expect(gluedSecret).toBe(`${blob}apiKey:"${REDACTED_EVENT_VALUE}"`);
});
it("scans long dash-separated runs in linear time", () => {
for (const run of ["a-".repeat(60_000), "a--".repeat(40_000)]) {
expect(redactWithinTimeBudget(`${run} Authorization: Bearer dash-run-secret`))
.toBe(`${run} Authorization: ${REDACTED_EVENT_VALUE}`);
expect(redactWithinTimeBudget(`${run}Authorization: Bearer glued-run-secret`))
.toBe(`${run}Authorization: ${REDACTED_EVENT_VALUE}`);
}
});
});
@@ -152,15 +152,20 @@ describeDatabase("remote workspace ordinary file routes", () => {
});
const runtime = { execute, syncOut } as unknown as Pick<EnvironmentRuntimeService, "execute" | "syncOut">;
const remoteFiles = createRemoteWorkspaceFiles(db, { environmentRuntime: runtime });
const app = filesApp(ids.company, remoteFiles);
return { root, ids, authority, execute, syncOut, remoteFiles, app, beforeCopy: (callback: () => Promise<void>) => { beforeCopy = callback; } };
}
function filesApp(companyId: string, remoteFiles: ReturnType<typeof createRemoteWorkspaceFiles>) {
const app = express();
app.use(express.json());
app.use((req, _res, next) => {
req.actor = { type: "board", userId: "local-board", companyIds: [ids.company], source: "local_implicit", isInstanceAdmin: true } as Request["actor"];
req.actor = { type: "board", userId: "local-board", companyIds: [companyId], source: "local_implicit", isInstanceAdmin: true } as Request["actor"];
next();
});
app.use("/api", fileResourceRoutes(db, { remoteFiles }));
app.use(errorHandler);
return { root, ids, authority, execute, syncOut, remoteFiles, app, beforeCopy: (callback: () => Promise<void>) => { beforeCopy = callback; } };
return app;
}
it("lists, resolves, previews, and downloads live bytes without a controller checkout", async () => {
@@ -323,7 +328,7 @@ describeDatabase("remote workspace ordinary file routes", () => {
{ workspace: "execution", path: "docs/result.md" })).rejects.toMatchObject({ status: 422, details: { code: "file_changed" } });
});
async function savedFileFixture() {
async function savedFileFixture(keepLocation = false) {
const f = await fixture(), storageRoot = await fs.mkdtemp(path.join(os.tmpdir(), "saved-file-budget-"));
directories.push(storageRoot);
const storage = createStorageService(createLocalDiskStorageProvider(path.join(storageRoot, "objects")));
@@ -335,11 +340,113 @@ describeDatabase("remote workspace ordinary file routes", () => {
const saved = await uploadWorkspaceSnapshot({ db, storage, companyId: f.ids.company, workspaceId: f.ids.workspace,
snapshotId: snapshot.id, archivePath, manifestDigest: archive.manifestDigest });
await db.update(executionWorkspaceSnapshots).set({ status: "committed" }).where(eq(executionWorkspaceSnapshots.id, snapshot.id));
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { ...f.authority, location: null, snapshotId: snapshot.id } } })
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { ...f.authority, location: keepLocation ? f.authority.location : null, snapshotId: snapshot.id } } })
.where(eq(executionWorkspaces.id, f.ids.workspace));
return { ...f, storage, saved, input: { companyId: f.ids.company, workspaceId: f.ids.workspace, snapshotId: snapshot.id } };
}
it("serves committed saved bytes through ordinary routes while the live allocation is unavailable", async () => {
const f = await savedFileFixture(true);
f.execute.mockRejectedValue(new Error("synthetic live allocation unavailable"));
const load = createSavedWorkspaceLoader(db, f.storage), closed: Promise<void>[] = [];
const loadSavedWorkspace = vi.fn(async (input: Parameters<typeof load>[0]) => {
const saved = await load(input);
let finish!: () => void, fail!: (error: unknown) => void;
closed.push(new Promise<void>((resolve, reject) => { finish = resolve; fail = reject; }));
return { ...saved, cleanup: async () => {
try { await saved.cleanup(); finish(); } catch (error) { fail(error); throw error; }
} };
});
const remoteFiles = createRemoteWorkspaceFiles(db, { loadSavedWorkspace, environmentRuntime: { execute: f.execute, syncOut: f.syncOut } });
const app = filesApp(f.ids.company, remoteFiles), base = `/api/issues/${f.ids.issue}/file-resources`;
const preview = await request(app).get(`${base}/content`).query({ workspace: "execution", path: "docs/result.md" }).expect(200);
expect(preview.body).toMatchObject({ content: { data: "Remote useful result\n" }, resource: { source: { kind: "saved", snapshotId: f.saved.id } } });
const listed = await request(app).get(`${base}/list`).query({ workspace: "execution", path: "docs" }).expect(200);
expect(listed.body).toMatchObject({ workspace: { source: { kind: "saved", snapshotId: f.saved.id } } });
const resolved = await request(app).get(`${base}/resolve`).query({ workspace: "execution", path: "docs/result.md" }).expect(200);
expect(resolved.body).toMatchObject({ source: { kind: "saved", snapshotId: f.saved.id }, capabilities: { preview: true, download: true } });
const download = await request(app).get(`${base}/content`).query({ workspace: "execution", path: "docs/result.md", download: true }).expect(200);
expect(download.text).toBe("Remote useful result\n");
expect(f.execute).not.toHaveBeenCalled(); expect(f.syncOut).not.toHaveBeenCalled();
expect(loadSavedWorkspace).toHaveBeenCalledTimes(4);
// A streamed response can finish at the client before its server finally block.
await Promise.all(closed);
for (const loaded of loadSavedWorkspace.mock.results) await expect(fs.stat((await loaded.value).root)).rejects.toMatchObject({ code: "ENOENT" });
const [workspace] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, f.ids.workspace));
expect(workspace!.metadata?.remoteWorkspace).toMatchObject({ location: f.authority.location, dirty: false, writerRunId: null });
});
it.each(["dirty", "writer"] as const)("keeps %s work on the live path and never substitutes its older save", async state => {
const f = await savedFileFixture(true);
let writerRunId: string | null = null;
if (state === "writer") {
const [agent] = await db.insert(agents).values({ companyId: f.ids.company, name: "Current workspace writer" }).returning();
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.ids.company, agentId: agent!.id, status: "running" }).returning();
writerRunId = run!.id;
}
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { ...f.authority, snapshotId: f.saved.id, dirty: state === "dirty", writerRunId } } })
.where(eq(executionWorkspaces.id, f.ids.workspace));
await fs.writeFile(path.join(f.root, "docs", "result.md"), "Unsaved live result\n");
const loadSavedWorkspace = vi.fn(createSavedWorkspaceLoader(db, f.storage));
const remoteFiles = createRemoteWorkspaceFiles(db, { loadSavedWorkspace, environmentRuntime: { execute: f.execute, syncOut: f.syncOut } });
const app = filesApp(f.ids.company, remoteFiles), route = `/api/issues/${f.ids.issue}/file-resources/content`;
const current = await request(app).get(route).query({ workspace: "execution", path: "docs/result.md" }).expect(200);
expect(current.body).toMatchObject({ resource: { source: { kind: "live" } }, content: { data: "Unsaved live result\n" } });
f.execute.mockRejectedValueOnce(new Error("synthetic live allocation unavailable"));
const unavailable = await request(app).get(route).query({ workspace: "execution", path: "docs/result.md" }).expect(422);
expect(unavailable.body).not.toHaveProperty("content"); expect(loadSavedWorkspace).not.toHaveBeenCalled();
});
it.each([true, false])("preserves the optional saved-loader boundary with a recorded location=%s", async keepLocation => {
const f = await savedFileFixture(keepLocation);
const remoteFiles = createRemoteWorkspaceFiles(db, { environmentRuntime: { execute: f.execute, syncOut: f.syncOut } });
const response = await request(filesApp(f.ids.company, remoteFiles)).get(`/api/issues/${f.ids.issue}/file-resources/content`)
.query({ workspace: "execution", path: "docs/result.md" }).expect(keepLocation ? 200 : 422);
if (keepLocation) {
expect(response.body).toMatchObject({ resource: { source: { kind: "live" } }, content: { data: "Remote useful result\n" } });
expect(f.execute).toHaveBeenCalled();
} else {
expect(response.body).not.toHaveProperty("content"); expect(f.execute).not.toHaveBeenCalled();
}
});
it.each(["company", "workspace", "uncommitted"] as const)("rejects a saved snapshot with an invalid %s binding before object retrieval", async invalid => {
const f = await savedFileFixture(true);
if (invalid === "company") {
const [other] = await db.insert(companies).values({ name: "Other saved-file company" }).returning();
await db.update(executionWorkspaceSnapshots).set({ companyId: other!.id }).where(eq(executionWorkspaceSnapshots.id, f.saved.id));
} else if (invalid === "workspace") {
const [other] = await db.insert(executionWorkspaces).values({ companyId: f.ids.company, mode: "isolated_workspace",
strategyType: "project_primary", providerType: "cloud_sandbox", name: "Other saved-file workspace" }).returning();
await db.update(executionWorkspaceSnapshots).set({ executionWorkspaceId: other!.id }).where(eq(executionWorkspaceSnapshots.id, f.saved.id));
} else await db.update(executionWorkspaceSnapshots).set({ status: "uploaded" }).where(eq(executionWorkspaceSnapshots.id, f.saved.id));
const getObject = vi.spyOn(f.storage, "getObject");
const remoteFiles = createRemoteWorkspaceFiles(db, { loadSavedWorkspace: createSavedWorkspaceLoader(db, f.storage),
environmentRuntime: { execute: f.execute, syncOut: f.syncOut } });
const response = await request(filesApp(f.ids.company, remoteFiles)).get(`/api/issues/${f.ids.issue}/file-resources/content`)
.query({ workspace: "execution", path: "docs/result.md" });
expect(response.status).toBeGreaterThanOrEqual(400); expect(response.body).not.toHaveProperty("content");
expect(getObject).not.toHaveBeenCalled(); expect(f.execute).not.toHaveBeenCalled(); expect(f.syncOut).not.toHaveBeenCalled();
getObject.mockRestore();
});
it("fences a saved read if a new writer claims the workspace during extraction and removes its scratch files", async () => {
const f = await savedFileFixture(true), load = createSavedWorkspaceLoader(db, f.storage);
let savedRoot: string | undefined;
const loadSavedWorkspace = async (input: Parameters<typeof load>[0]) => {
const saved = await load(input); savedRoot = saved.root;
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { ...f.authority, snapshotId: f.saved.id, dirty: true, epoch: 2 } } })
.where(eq(executionWorkspaces.id, f.ids.workspace));
return saved;
};
const remoteFiles = createRemoteWorkspaceFiles(db, { loadSavedWorkspace, environmentRuntime: { execute: f.execute, syncOut: f.syncOut } });
const response = await request(filesApp(f.ids.company, remoteFiles)).get(`/api/issues/${f.ids.issue}/file-resources/content`)
.query({ workspace: "execution", path: "docs/result.md" }).expect(422);
expect(response.body.details.code).toBe("workspace_changed"); expect(response.body).not.toHaveProperty("content");
expect(savedRoot).toBeDefined(); await expect(fs.stat(savedRoot!)).rejects.toMatchObject({ code: "ENOENT" });
expect(f.execute).not.toHaveBeenCalled(); expect(f.syncOut).not.toHaveBeenCalled();
});
it("shares fail-fast saved Files admission across loader instances until owned cleanup completes", async () => {
const f = await savedFileFixture(), availableBytes = async () => 8n * 1024n ** 3n;
const first = createSavedWorkspaceLoader(db, f.storage, { availableBytes });
@@ -625,7 +732,7 @@ describeDatabase("remote workspace ordinary file routes", () => {
});
it.each(["confirmed_stop", "unknown_stop", "dirty", "foreign_receipt"])(
"uses saved bytes only for a clean workspace with an exact stopped receipt (%s)", async state => {
"uses clean saved authority independently of an allocation's stop receipt (%s)", async state => {
const f = await fixture();
const snapshotId = randomUUID();
const agentId = randomUUID();
@@ -650,7 +757,7 @@ describeDatabase("remote workspace ordinary file routes", () => {
const remoteFiles = createRemoteWorkspaceFiles(db, { loadSavedWorkspace });
const content = workspaceFileResourceService(db, undefined, { remoteFiles }).readContent(f.ids.issue,
{ workspace: "execution", path: "docs/result.md" });
if (state === "confirmed_stop") {
if (state !== "dirty") {
await expect(content).resolves.toMatchObject({ content: { data: "Saved stopped result\n" },
resource: { source: { kind: "saved", snapshotId, savedAt: "2026-10-30T12:00:00.000Z" } } });
expect(loadSavedWorkspace).toHaveBeenCalledWith({ companyId: f.ids.company, workspaceId: f.ids.workspace, snapshotId });
@@ -2,7 +2,7 @@ import { randomBytes, randomUUID } from "node:crypto";
import { eq, sql, type SQL } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest";
import { createDb, plugins, type Db } from "@paperclipai/db";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import type { PaperclipPluginManifestV1, PluginEnvironmentDriverDeclaration } from "@paperclipai/shared";
import { resolvePluginSandboxProviderConfigSchema, resolvePluginSandboxProviderDriverByKey } from "../services/plugin-environment-driver.js";
import { resolveEnvironmentDriverConfigForRuntime, resolveSandboxProviderSecretRefPaths } from "../services/environment-config.js";
@@ -39,12 +39,12 @@ describe.skipIf(!support.supported)("sandbox provider schema reads", () => {
const lookup = (client: Db, driverKey = key) => resolvePluginSandboxProviderConfigSchema({ db: client, driverKey });
async function measure<T>(client: Db, operation: () => Promise<T>) {
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event), () => now);
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event), () => now);
installDatabaseResponseCapture(client, { plugins }, capture);
try {
const value = await operation();
capture.flush();
const entries = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1");
const entries = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1");
return { value, bytes: entries.reduce((sum, entry) => sum + Number(entry.responseValueBytes), 0),
calls: entries.reduce((sum, entry) => sum + Number(entry.calls), 0) };
} finally { capture.flush(); }
@@ -1,5 +1,8 @@
import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, UploadPartCommand, DeleteObjectCommand, PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
import { Readable } from "node:stream";
import { createHash } from "node:crypto";
import { createServer, type ServerResponse } from "node:http";
import type { AddressInfo } from "node:net";
import { describe, expect, it, vi } from "vitest";
import { putS3Multipart } from "../storage/s3-multipart.js";
import { createS3StorageProvider } from "../storage/s3-provider.js";
@@ -7,6 +10,94 @@ import { createStorageService } from "../storage/service.js";
import { STORAGE_FAILURE_CLEANUP_MS } from "../storage/failure-cleanup.js";
describe("streamed S3 multipart uploads", () => {
it("sends concurrent exact part bodies through the real S3 HTTP client", async () => {
const received = new Map<number, string>();
const responses = new Map<number, ServerResponse>();
let completedParts = "";
const server = createServer((request, response) => {
const url = new URL(request.url!, "http://localhost");
const digest = createHash("sha256");
let text = "";
request.on("data", chunk => { digest.update(chunk); if (request.method === "POST") text += chunk; });
request.on("end", () => {
if (request.method === "PUT") {
const number = Number(url.searchParams.get("partNumber"));
received.set(number, digest.digest("hex"));
responses.set(number, response);
} else if (url.searchParams.has("uploads")) {
response.end('<InitiateMultipartUploadResult><UploadId>upload</UploadId></InitiateMultipartUploadResult>');
} else {
completedParts = text;
response.end('<CompleteMultipartUploadResult><ETag>complete</ETag></CompleteMultipartUploadResult>');
}
});
});
await new Promise<void>((resolve, reject) => { server.once("error", reject); server.listen(0, "127.0.0.1", resolve); });
const client = new S3Client({ region: "us-east-1", endpoint: `http://127.0.0.1:${(server.address() as AddressInfo).port}`,
forcePathStyle: true, requestChecksumCalculation: "WHEN_REQUIRED", credentials: { accessKeyId: "fixture", secretAccessKey: "fixture" } });
const cancellation = new AbortController();
const body = Readable.from((async function* () {
for (let index = 0; index < 512; index++) yield Buffer.alloc(64 * 1024, Math.floor(index / 128) + 1);
})());
const saving = putS3Multipart(client, "bucket", "key", { objectKey: "key", contentType: "application/octet-stream", contentLength: 32 * 1024 * 1024, body, signal: cancellation.signal }).catch(error => error);
try {
await expect.poll(() => received.size, { timeout: 3000 }).toBe(4);
for (const number of [4, 2, 3, 1]) {
expect(received.get(number)).toBe(createHash("sha256").update(Buffer.alloc(8 * 1024 * 1024, number)).digest("hex"));
responses.get(number)!.writeHead(200, { ETag: `part-${number}` }).end();
}
expect(await saving).toBeUndefined();
expect([...completedParts.matchAll(/<PartNumber>(\d+)<\/PartNumber>/g)].map(match => Number(match[1]))).toEqual([1, 2, 3, 4]);
} finally {
cancellation.abort();
await saving;
client.destroy();
server.closeAllConnections();
await new Promise<void>(resolve => server.close(() => resolve()));
}
});
it("overlaps slow part acknowledgements and completes exact bytes in source order", async () => {
const client = new S3Client({ region: "us-east-1" });
const received = new Map<number, Buffer>();
const acknowledgements = new Map<number, () => void>();
let completed = false;
const send = vi.spyOn(client, "send").mockImplementation(async (command: any) => {
if (command instanceof CreateMultipartUploadCommand) return { UploadId: "upload" };
if (command instanceof UploadPartCommand) {
const chunks: Buffer[] = [];
for await (const chunk of command.input.Body as Readable) chunks.push(chunk);
received.set(command.input.PartNumber, Buffer.concat(chunks));
await new Promise<void>(resolve => acknowledgements.set(command.input.PartNumber, resolve));
return { ETag: `part-${command.input.PartNumber}` };
}
expect(command).toBeInstanceOf(CompleteMultipartUploadCommand);
expect(command.input.MultipartUpload.Parts).toEqual([1, 2, 3, 4].map(PartNumber => ({ PartNumber, ETag: `part-${PartNumber}` })));
completed = true;
return {};
});
const length = 25 * 1024 * 1024;
const body = Readable.from((async function* () {
for (let offset = 0; offset < length; offset += 64 * 1024) yield Buffer.alloc(Math.min(64 * 1024, length - offset), offset / (64 * 1024) % 251);
})());
const saving = putS3Multipart(client, "bucket", "key", { objectKey: "key", contentType: "application/octet-stream", contentLength: length, body });
try {
await expect.poll(() => received.size, { timeout: 1000 }).toBe(4);
expect(completed).toBe(false);
for (const number of [4, 2, 3, 1]) acknowledgements.get(number)!();
await saving;
const actual = Buffer.concat([1, 2, 3, 4].map(number => received.get(number)!));
expect(actual.length).toBe(length);
for (let offset = 0; offset < length; offset += 64 * 1024) {
expect(actual.subarray(offset, Math.min(length, offset + 64 * 1024)).every(byte => byte === offset / (64 * 1024) % 251)).toBe(true);
}
expect(completed).toBe(true);
} finally {
const release = setInterval(() => { for (const acknowledge of acknowledgements.values()) acknowledge(); }, 1);
try { await saving; } finally { clearInterval(release); send.mockRestore(); client.destroy(); }
}
});
it("streams exact ordered parts without buffering the full response", async () => {
const client = new S3Client({ region: "us-east-1" });
const lengths: number[] = [];
@@ -33,6 +124,109 @@ describe("streamed S3 multipart uploads", () => {
expect(lengths).toEqual([8, 8, 1].map(mb => mb * 1024 * 1024));
} finally { send.mockRestore(); client.destroy(); }
});
it("keeps source backpressure and at most four part requests while acknowledgements stall", async () => {
const client = new S3Client({ region: "us-east-1" });
let releaseReading!: () => void, releaseAcknowledgements!: () => void;
const reading = new Promise<void>(resolve => { releaseReading = resolve; });
const acknowledgements = new Promise<void>(resolve => { releaseAcknowledgements = resolve; });
let yielded = 0, active = 0, peak = 0, received = 0;
const send = vi.spyOn(client, "send").mockImplementation(async (command: any) => {
if (command instanceof CreateMultipartUploadCommand) return { UploadId: "upload" };
if (command instanceof UploadPartCommand) {
active++; peak = Math.max(peak, active);
await reading;
for await (const chunk of command.input.Body as Readable) expect(chunk.length).toBeLessThanOrEqual(64 * 1024);
received++;
await acknowledgements;
active--;
return { ETag: `part-${command.input.PartNumber}` };
}
expect(command).toBeInstanceOf(CompleteMultipartUploadCommand);
return {};
});
const body = Readable.from((async function* () {
for (let index = 0; index < 1600; index++) { yielded++; yield Buffer.alloc(64 * 1024); }
})());
const saving = putS3Multipart(client, "bucket", "key", { objectKey: "key", contentType: "application/octet-stream", contentLength: 100 * 1024 * 1024, body });
try {
await expect.poll(() => active).toBe(1);
expect(yielded).toBeLessThanOrEqual(3);
releaseReading();
await expect.poll(() => received).toBe(4);
expect(yielded).toBeLessThanOrEqual(4 * 128 + 3);
expect(peak).toBe(4);
releaseAcknowledgements();
await saving;
expect(received).toBe(13);
expect(yielded).toBe(1600);
expect(peak).toBe(4);
} finally { releaseReading(); releaseAcknowledgements(); await saving; send.mockRestore(); client.destroy(); }
});
it("settles cancelled siblings before aborting a failed multipart upload", async () => {
const client = new S3Client({ region: "us-east-1" });
const cancellations: AbortSignal[] = [];
let failPart!: () => void, releaseSiblings!: () => void;
const settlement = new Promise<void>(resolve => { releaseSiblings = resolve; });
let received = 0, settled = 0, aborted = false, completed = false;
const send = vi.spyOn(client, "send").mockImplementation(async (command: any, options: any) => {
if (command instanceof CreateMultipartUploadCommand) return { UploadId: "upload" };
if (command instanceof UploadPartCommand) {
for await (const _chunk of command.input.Body as Readable) {}
received++;
if (command.input.PartNumber === 2) return new Promise((_resolve, reject) => { failPart = () => reject(new Error("second part failed")); });
cancellations.push(options.abortSignal);
await new Promise<void>(resolve => options.abortSignal.addEventListener("abort", () => resolve(), { once: true }));
await settlement;
settled++;
throw options.abortSignal.reason;
}
if (command instanceof CompleteMultipartUploadCommand) { completed = true; return {}; }
expect(command).toBeInstanceOf(AbortMultipartUploadCommand);
expect(settled).toBe(3);
aborted = true;
return {};
});
const body = Readable.from((async function* () { for (let index = 0; index < 512; index++) yield Buffer.alloc(64 * 1024); })());
const saving = putS3Multipart(client, "bucket", "key", { objectKey: "key", contentType: "application/octet-stream", contentLength: 32 * 1024 * 1024, body }).catch(error => error);
try {
await expect.poll(() => received).toBe(4);
failPart();
await expect.poll(() => cancellations.every(signal => signal.aborted)).toBe(true);
expect(aborted).toBe(false);
releaseSiblings();
expect((await saving).message).toBe("second part failed");
expect(aborted).toBe(true);
expect(completed).toBe(false);
expect(body.destroyed).toBe(true);
} finally { failPart?.(); releaseSiblings(); await saving; send.mockRestore(); client.destroy(); }
});
it.each(["truncated", "excess", "source error"])("rejects %s input without publishing an object", async kind => {
const client = new S3Client({ region: "us-east-1" });
let aborted = false;
const send = vi.spyOn(client, "send").mockImplementation(async (command: any) => {
if (command instanceof CreateMultipartUploadCommand) return { UploadId: "upload" };
if (command instanceof UploadPartCommand) {
for await (const _chunk of command.input.Body as Readable) {}
return { ETag: `part-${command.input.PartNumber}` };
}
expect(command).toBeInstanceOf(AbortMultipartUploadCommand);
aborted = true;
return {};
});
const expected = { truncated: "S3 upload source was truncated", excess: "S3 upload source exceeds its declared size", "source error": "source unavailable" };
const body = Readable.from((async function* () {
for (let index = 0; index < (kind === "excess" ? 274 : 270); index++) yield Buffer.alloc(64 * 1024);
if (kind === "source error") throw new Error("source unavailable");
})());
try {
await expect(putS3Multipart(client, "bucket", "key", { objectKey: "key", contentType: "application/octet-stream", contentLength: 17 * 1024 * 1024, body })).rejects.toThrow(expected[kind]);
expect(aborted).toBe(true);
expect(body.destroyed).toBe(true);
} finally { send.mockRestore(); client.destroy(); }
});
it("aborts an incomplete upload and closes its source on failure", async () => {
const client = new S3Client({ region: "us-east-1" });
const send = vi.spyOn(client, "send").mockImplementation(async (command: any) => {
@@ -79,4 +273,43 @@ describe("streamed S3 multipart uploads", () => {
await expect(storage.putFile({ companyId: "company", namespace: "skill-packages", originalFilename: "next", contentType: "text/plain", body: Buffer.from("next") })).resolves.toMatchObject({ byteSize: 4 });
} finally { send.mockRestore(); vi.useRealTimers(); }
});
it("bounds cancellation while an asynchronous source read is stalled", async () => {
vi.useFakeTimers();
const client = new S3Client({ region: "us-east-1" });
const cancellation = new AbortController();
let release!: () => void, waiting = false, settled = false;
const sourceGate = new Promise<void>(resolve => { release = resolve; });
const body = Readable.from((async function* () {
yield Buffer.alloc(64 * 1024);
waiting = true;
await sourceGate;
yield Buffer.alloc(64 * 1024);
})());
const send = vi.spyOn(client, "send").mockImplementation(async (command: any, options: any) => {
if (command instanceof CreateMultipartUploadCommand) return { UploadId: "upload" };
if (command instanceof UploadPartCommand) {
(command.input.Body as Readable).resume();
return new Promise((_resolve, reject) => options.abortSignal.addEventListener("abort", () => reject(options.abortSignal.reason), { once: true }));
}
expect(command).toBeInstanceOf(AbortMultipartUploadCommand);
return {};
});
const saving = putS3Multipart(client, "bucket", "key", { objectKey: "key", contentType: "text/plain", contentLength: 128 * 1024, body, signal: cancellation.signal })
.catch(error => error).finally(() => { settled = true; });
try {
await vi.advanceTimersByTimeAsync(0);
expect(waiting).toBe(true);
cancellation.abort(new Error("cancelled during source read"));
await vi.advanceTimersByTimeAsync(STORAGE_FAILURE_CLEANUP_MS + 1);
expect(settled).toBe(true);
expect((await saving).message).toBe("cancelled during source read");
expect(body.destroyed).toBe(true);
} finally {
release();
await vi.advanceTimersByTimeAsync(STORAGE_FAILURE_CLEANUP_MS + 1);
await saving;
send.mockRestore(); client.destroy(); vi.useRealTimers();
}
});
});
@@ -5,7 +5,7 @@ import {
agents, companies, createDb, toolApplications, toolCatalogEntries,
toolConnections, toolProfileBindings, toolProfileEntries, toolProfiles, type Db,
} from "@paperclipai/db";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import { toolAccessService } from "../services/tool-access.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
@@ -76,6 +76,20 @@ describeDatabase("effective tool assignments", () => {
expect(full.allowedTools[0].annotations).toEqual(f.read.annotations);
});
it("keeps exclusions local to each profile when effective profiles grant the same tool", async () => {
const f = await fixture();
await db.update(toolProfiles).set({ defaultAction: "allow" }).where(eq(toolProfiles.id, f.profile.id));
await db.insert(toolProfileEntries).values({ companyId: f.company.id, profileId: f.profile.id,
selectorType: "catalog_entry", effect: "exclude", catalogEntryId: f.read.id });
const [second] = await db.insert(toolProfiles).values({ companyId: f.company.id, profileKey: "second", name: "Second", defaultAction: "deny" }).returning();
await db.insert(toolProfileBindings).values({ companyId: f.company.id, profileId: second.id, targetType: "agent", targetId: f.agent.id });
const [include] = await db.insert(toolProfileEntries).values({ companyId: f.company.id, profileId: second.id,
selectorType: "catalog_entry", effect: "include", catalogEntryId: f.read.id }).returning();
expect((await compare(f)).assignments.allowedTools.map(tool => tool.id)).toEqual([f.read.id, f.write.id]);
await db.delete(toolProfileEntries).where(eq(toolProfileEntries.id, include.id));
expect((await compare(f)).assignments.allowedTools.map(tool => tool.id)).toEqual([f.write.id]);
});
it("keeps unresolved exact names and observes changed catalog, profile and binding state", async () => {
const f = await fixture();
await db.insert(toolProfileEntries).values([
@@ -149,14 +163,14 @@ describeDatabase("effective tool assignments", () => {
const before = await service.getEffectiveToolAssignmentsForAgent(f.company.id, f.agent.id);
expect(before.allowedTools).toHaveLength(102);
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { toolCatalogEntries, toolProfileEntries }, capture);
let scoped: Awaited<ReturnType<typeof service.getConnectionAssignmentsForAgent>>;
try { scoped = await service.getConnectionAssignmentsForAgent(f.company.id, f.agent.id, [f.connection.id]); }
finally { capture.flush(); }
expect(Number(events.at(-1)?.completedQueries)).toBeGreaterThan(0);
expect(Number(events.at(-1)?.responseValueBytes)).toBeLessThan(16 * 1024);
const catalogQueries = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
const catalogQueries = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
&& event.table === "tool_catalog_entries" && event.operation === "select");
expect(catalogQueries).toHaveLength(1);
expect(catalogQueries[0]).toMatchObject({ calls: 1, rows: 2 });
@@ -192,13 +206,13 @@ describeDatabase("effective tool assignments", () => {
const service = toolAccessService(db), before = await compare(f);
const savedRules = await db.select().from(toolProfileEntries);
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { toolProfileEntries, toolProfileBindings }, capture);
try {
const scoped = await service.getConnectionAssignmentsForAgent(f.company.id, f.agent.id, [f.connection.id]);
capture.flush();
const rules = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" && event.table === "tool_profile_entries");
const bindings = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" && event.table === "tool_profile_bindings");
const rules = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "tool_profile_entries");
const bindings = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "tool_profile_bindings");
expect(rules).toHaveLength(1);
expect(Number(rules[0].responseValueBytes)).toBeLessThan(4096);
expect(rules[0]).toMatchObject({ calls: 1, rows: 2 });
@@ -0,0 +1,21 @@
import { describe, expect, it } from "vitest";
import { profileCatalogMatcher, profileEntryMatchesCatalog, type ProfileCatalogMatch, type ProfileEntryMatch } from "../services/tool-access/profiles/projection.js";
describe("indexed profile catalog matching", () => {
it("preserves all selector kinds, null matches and exact strings in mixed rule sets", () => {
const catalog: ProfileCatalogMatch[] = Array.from({ length: 75 }, (_, index) => ({
id: `tool-${index}`, applicationId: index % 3 ? `app-${index % 5}` : null,
connectionId: `connection-${index % 7}`, toolName: index % 4 ? `tool-${index}` : "quoted' path\\tool_🙂",
riskLevel: index % 2 ? "read" : "write",
}));
const selectors = ["application", "connection", "catalog_entry", "tool_name", "risk_level", "unknown"];
const rules: ProfileEntryMatch[] = catalog.flatMap(entry => selectors.map(selectorType => ({
selectorType, applicationId: entry.applicationId, connectionId: entry.connectionId,
catalogEntryId: entry.id, toolName: entry.toolName, riskLevel: entry.riskLevel,
})));
for (const selected of [[], ...rules.map(rule => [rule]), rules.slice(0, 25), rules]) {
const matches = profileCatalogMatcher(selected);
expect(catalog.filter(matches)).toEqual(catalog.filter(entry => selected.some(rule => profileEntryMatchesCatalog(rule, entry))));
}
});
});
@@ -0,0 +1,69 @@
import { describe, expect, it } from "vitest";
import { pendingNewToolsForProfile, profileCoversCatalogScope, profileEntryMatchesCatalog, type ProfileCatalogEntry } from "../services/tool-access/profiles/projection.js";
import type { toolProfiles, toolProfileEntries } from "@paperclipai/db";
type Input = Parameters<typeof pendingNewToolsForProfile>[0];
const date = new Date("2026-01-01T00:00:00Z");
const later = new Date("2026-01-02T00:00:00Z");
function profile(overrides: Partial<typeof toolProfiles.$inferSelect> = {}): typeof toolProfiles.$inferSelect {
return { id: "profile", companyId: "company", profileKey: "selected", name: "Selected", description: null, status: "active", defaultAction: "deny", newToolsReviewedAt: null, metadata: {}, createdAt: date, updatedAt: date, ...overrides };
}
function rule(overrides: Partial<typeof toolProfileEntries.$inferSelect>): typeof toolProfileEntries.$inferSelect {
return { id: "rule", companyId: "company", profileId: "profile", selectorType: "catalog_entry", effect: "include", applicationId: null, connectionId: null, catalogEntryId: null, toolName: null, riskLevel: null, conditions: null, createdAt: date, updatedAt: date, ...overrides };
}
function legacy(input: Input) {
if (input.profile.status !== "active" || input.profile.defaultAction !== "deny") return [];
const catalogById = new Map(input.catalog.map(entry => [entry.id, entry]));
return input.catalog.filter(entry => (entry.status === "active" || entry.status === "quarantined")
&& entry.firstSeenAt > (input.profile.newToolsReviewedAt ?? input.profile.createdAt)
&& input.entries.some(rule => profileCoversCatalogScope({ entry: rule, catalogEntry: entry, catalogById }))
&& !input.entries.some(rule => profileEntryMatchesCatalog(rule, entry))).map(entry => ({
catalogEntryId: entry.id, applicationId: entry.applicationId,
applicationName: entry.applicationId ? input.applicationsById?.get(entry.applicationId)?.name ?? null : null,
connectionId: entry.connectionId, connectionName: input.connectionsById?.get(entry.connectionId)?.name ?? null,
toolName: entry.toolName, title: entry.title ?? null, description: entry.description ?? null,
capability: entry.riskLevel, riskLevel: entry.riskLevel, addedAt: entry.firstSeenAt, firstSeenAt: entry.firstSeenAt,
}));
}
const catalog: ProfileCatalogEntry[] = Array.from({ length: 48 }, (_, i): ProfileCatalogEntry => ({
id: `tool-${i}`, applicationId: i % 3 ? `app-${i % 4}` : null,
connectionId: `connection-${i % 6}`, toolName: i % 7 ? `name-${i}` : "quoted' path\\tool_🙂",
riskLevel: i % 2 ? "read" : "write", status: i % 5 ? "active" : "quarantined",
firstSeenAt: i % 11 ? later : date,
}));
describe("pending profile projection", () => {
it("matches the previous ordered projection for every selector, effect, watermark and nullable scope", () => {
const rules = catalog.flatMap(entry => (["include", "exclude"] as const).flatMap(effect => [
rule({ effect, selectorType: "application", applicationId: entry.applicationId }),
rule({ effect, selectorType: "connection", connectionId: entry.connectionId }),
rule({ effect, selectorType: "catalog_entry", catalogEntryId: entry.id }),
rule({ effect, selectorType: "tool_name", toolName: entry.toolName }),
rule({ effect, selectorType: "risk_level", riskLevel: entry.riskLevel }),
]));
const scopes = [rule({ selectorType: "application", applicationId: null }), rule({ selectorType: "connection", connectionId: "connection-1" }), rule({ catalogEntryId: "tool-2" }), rule({ catalogEntryId: "missing" })];
for (const scope of scopes) for (const selected of [[], ...rules.map(entry => [entry]), rules.slice(0, 50), rules]) {
const input = { profile: profile(), catalog, entries: [scope, ...selected], applicationsById: new Map([["app-1", { id: "app-1", name: "Application" }]]), connectionsById: new Map([["connection-1", { id: "connection-1", name: "Connection" }]]) };
expect(pendingNewToolsForProfile(input)).toEqual(legacy(input));
}
for (const changes of [{ status: "archived" as const }, { defaultAction: "allow" as const }, { newToolsReviewedAt: later }]) {
const input = { profile: profile(changes), catalog, entries: scopes };
expect(pendingNewToolsForProfile(input)).toEqual(legacy(input));
}
});
it("does not broaden an individual tool's null application scope to unrelated null applications", () => {
const input = { profile: profile(), entries: [rule({ catalogEntryId: "tool-0" })], catalog };
const ids = pendingNewToolsForProfile(input).map(entry => entry.catalogEntryId);
expect(ids).toContain("tool-6");
expect(ids).not.toContain("tool-3");
expect(pendingNewToolsForProfile(input)).toEqual(legacy(input));
});
it("recomputes catalog references, exclusions and scope after the input changes", () => {
const input = { profile: profile(), entries: [rule({ catalogEntryId: "tool-2" })], catalog };
expect(pendingNewToolsForProfile(input).length).toBeGreaterThan(0);
expect(pendingNewToolsForProfile({ ...input, catalog: catalog.filter(entry => entry.id !== "tool-2") })).toEqual([]);
expect(pendingNewToolsForProfile({ ...input, entries: [rule({ catalogEntryId: "tool-2", effect: "exclude" })] })).toEqual([]);
});
});
@@ -0,0 +1,36 @@
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { eq } from "drizzle-orm";
import { createDb, toolCatalogEntries, toolProfileEntries, toolProfiles } from "@paperclipai/db";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { createProfileProjectionFixture } from "./helpers/tool-profile-projection-fixture.js";
import { createToolProfileService } from "../services/tool-access/profiles/service.js";
import { createToolAppAttention } from "../services/tool-access/catalog/attention.js";
const support = await getEmbeddedPostgresTestSupport();
const describeDatabase = support.supported ? describe : describe.skip;
describeDatabase("pending profile tools through profile and attention services", () => {
let db: ReturnType<typeof createDb>, database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("profile-review-service-"); db = createDb(database.connectionString); });
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); });
it("preserves fresh pending counts in both owning services after review and catalog changes", async () => {
const f = await createProfileProjectionFixture(db, { profiles: 2, tools: 8, rules: 4 });
const now = () => new Date("2026-02-01T00:00:00Z"), service = createToolProfileService(db, now), attention = createToolAppAttention(db, now);
const assertCounts = async (expected: number) => {
const profiles = await service.listProfiles(f.company.id);
expect(profiles.find(p => p.id === f.profiles[0].id)?.newToolsPendingCount).toBe(expected);
expect(profiles.find(p => p.id === f.profiles[1].id)?.newToolsPendingCount).toBe(0);
expect((await service.listProfileNewTools(f.profiles[0].id, f.company.id)).pendingCount).toBe(expected);
expect((await attention(f.company.id)).totals.newToolsPendingReview).toBe(expected);
};
await assertCounts(4);
const [exclude] = await db.insert(toolProfileEntries).values({ companyId: f.company.id, profileId: f.profiles[0].id,
selectorType: "tool_name", effect: "exclude", toolName: f.catalog[4].toolName }).returning();
await assertCounts(3);
await db.update(toolCatalogEntries).set({ status: "removed" }).where(eq(toolCatalogEntries.id, f.catalog[5].id));
await assertCounts(2);
await db.update(toolProfiles).set({ newToolsReviewedAt: now() }).where(eq(toolProfiles.id, f.profiles[0].id));
await assertCounts(0);
const foreign = await createProfileProjectionFixture(db, { profiles: 2, tools: 5, rules: 2 });
expect((await service.listProfiles(foreign.company.id)).map(p => p.companyId)).toEqual([foreign.company.id, foreign.company.id]);
expect((await db.select().from(toolProfileEntries).where(eq(toolProfileEntries.id, exclude.id)))[0].effect).toBe("exclude");
});
});
@@ -122,16 +122,21 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
await quiet(); expect(fetchReport).not.toHaveBeenCalled();
await vi.advanceTimersByTimeAsync(deadline - Date.now());
await vi.waitFor(async () => {
await vi.advanceTimersByTimeAsync(1000);
// Keep time fixed during database I/O. Advancing it on every poll can
// make the first reports due again before the backlog has settled.
const next = coordinator!.nextWakeAt();
if (next !== null && next < deadline + 60_000) {
await vi.advanceTimersByTimeAsync(Math.max(0, next - Date.now()));
}
expect(idleWorkSnapshot().active).toBe(0);
expect(fetchReport).toHaveBeenCalledTimes(105);
}, { timeout: 5000 });
}, { interval: 0, timeout: 15_000 });
expect(fetchReport.mock.calls.map(([call]) => call.id).sort()).toEqual(ids.sort());
expect((await db.select().from(chatVoiceReports).where(eq(chatVoiceReports.companyId, companyId)))
.every(row => row.attempts === 1)).toBe(true);
await quiet();
} finally { reads.mockRestore(); }
});
}, 30_000);
it("leaves configured idle, live calls, old history and unidentified calls quiet", async () => {
await session({ state: "active", endedAt: null });
await session({ endedAt: new Date(Date.now() - day - 1000) });
@@ -145,6 +145,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
expect(await row(call.id)).toBeUndefined(); expect(coordinator!.nextWakeAt()).toBeNull(); await quiet();
});
it("rounds PostgreSQL submillisecond deadlines up and never deletes early", async () => {
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: false });
const call = await admission(); const dueAt = Math.trunc(Date.now()) + 60_000;
await db.execute(sql`UPDATE chat_voice_inbound_calls SET updated_at = to_timestamp(${dueAt - week}::double precision / 1000) + interval '0.0005 seconds' WHERE id = ${call.id}`);
const service = voiceInboundRetention(db), [entry] = await service.load({ kind: "admission", id: call.id });