mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-16 03:31:44 +01:00
Merge master into sleep blocker inventory
Preserve the explicit external database fixture and owned-client cleanup while adopting the upstream PostgreSQL shutdown timeout in both sleep suites. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
commit
4a9974ea03
314 files changed
+10887
-2969
No files matched your search
@@ -4,7 +4,7 @@ import request from "supertest";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { agents, companies, createDb, heartbeatRunEvents, heartbeatRuns, issueComments, issues } from "@paperclipai/db";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { createLocalAgentJwt } from "../agent-auth-jwt.js";
|
||||
import { actorMiddleware } from "../middleware/auth.js";
|
||||
@@ -53,7 +53,7 @@ const support = await getEmbeddedPostgresTestSupport();
|
||||
it("bounds actual middleware responses and preserves the complete stored run", async () => {
|
||||
const fixture = await seed();
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
|
||||
try {
|
||||
const response = await request(fixture.app).get("/actor").set("Authorization", `Bearer ${fixture.token}`);
|
||||
@@ -61,7 +61,7 @@ const support = await getEmbeddedPostgresTestSupport();
|
||||
expect(response.body).toMatchObject({ type: "agent", runId: fixture.runId,
|
||||
identityContextId: fixture.identity.id, onBehalfOfUserId: "owner" });
|
||||
} finally { capture.flush(); }
|
||||
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
|
||||
const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
|
||||
&& event.table === "heartbeat_runs" && (event.operation === "select" || event.operation === "execute"));
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThanOrEqual(3);
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.responseValueBytes), 0)).toBeLessThan(16_384);
|
||||
|
||||
@@ -16,7 +16,7 @@ import { createHash, randomUUID } from "node:crypto";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { agentFileStore, fileHash, inspectAgentFile, inspectAgentDirectory, snapshotAgentFiles, MAX_AGENT_FILE_BYTES, MAX_AGENT_DIRECTORY_BYTES, MAX_AGENT_DIRECTORY_ENTRIES } from "../services/agent-file-store.js";
|
||||
import { agents, companies, authUsers, companyMemberships, principalPermissionGrants, heartbeatRuns, environmentLeases, environments, agentInstructionWorkingCopies, agentInstructionRevisions, agentInstructionHeads, createDb, fileStorageDb } from "@paperclipai/db";
|
||||
import { agents, companies, issues, authUsers, companyMemberships, principalPermissionGrants, heartbeatRuns, environmentLeases, environments, agentInstructionWorkingCopies, agentInstructionRevisions, agentInstructionHeads, createDb, fileStorageDb } from "@paperclipai/db";
|
||||
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { agentInstructionRevisionService } from "../services/agent-instruction-revisions.js";
|
||||
import { agentInstructionWorkingCopyService, instructionWorkingCopyGuidance } from "../services/agent-instruction-working-copies.js";
|
||||
@@ -24,6 +24,7 @@ import { resolveManagedInstructionsRoot } from "../services/agent-instructions.j
|
||||
import { buildNativeRuntimeContext } from "../services/native-runtime/runtime-context.js";
|
||||
import type { EnvironmentRuntimeService } from "../services/environment-runtime.js";
|
||||
import { remoteTerminationReceipt } from "../services/remote-execution-termination.js";
|
||||
import { createReusableLeaseHandoff } from "../services/environment-lease-handoff.js";
|
||||
import { AgentDirectoryReuseInvalidatedError, agentDirectoryWorkingCopyService } from "../services/agent-directory-working-copies.js";
|
||||
import { withDirectoryMergeLock } from "@paperclipai/adapter-utils/workspace-restore-merge";
|
||||
import { heartbeatRunEvents } from "@paperclipai/db";
|
||||
@@ -123,6 +124,44 @@ describe("persistent agent directories", () => {
|
||||
for (const copy of [active, completed, transferred]) expect((await fs.stat(copy.localRoot)).isDirectory()).toBe(true);
|
||||
} finally { await worker.stop(); }
|
||||
});
|
||||
it("retires a stale retry on an already cleaned receipt after restart without filesystem work", async () => {
|
||||
const copy = await run();
|
||||
await db.update(agentInstructionWorkingCopies).set({ state: "unavailable", processStoppedAt: new Date(),
|
||||
nextAttemptAt: new Date(), receipt: { schema: "paperclip.agent-files.v1", cleanupPending: false } })
|
||||
.where(eq(agentInstructionWorkingCopies.runId, copy.runId));
|
||||
clock(); const worker = coordinator(), removes = vi.spyOn(fs, "rm");
|
||||
try {
|
||||
await agentInstructionWorkingCopyService(db).registerDirectoryCleanupWork(worker, () => true, async () => true).ready;
|
||||
await vi.waitFor(async () => {
|
||||
await vi.advanceTimersByTimeAsync(100);
|
||||
expect((await copies.get(companyId, copy.runId))?.nextAttemptAt).toBeNull();
|
||||
});
|
||||
await expect.poll(() => idleWorkSnapshot().active).toBe(0);
|
||||
const reads = vi.spyOn(db, "select");
|
||||
await vi.advanceTimersByTimeAsync(120_000);
|
||||
expect(reads).not.toHaveBeenCalled(); expect(removes).not.toHaveBeenCalled();
|
||||
expect(worker.nextWakeAt()).toBeNull();
|
||||
expect(await fs.readFile(path.join(root, entryFile), "utf8")).toBe(initial);
|
||||
} finally { await worker.stop(); }
|
||||
});
|
||||
it("honors a cleanup obligation published before stale-deadline settlement", async () => {
|
||||
const copy = await run(), dueAt = new Date(Date.now() + 30_000);
|
||||
await db.update(agentInstructionWorkingCopies).set({ state: "unavailable", processStoppedAt: new Date(),
|
||||
nextAttemptAt: dueAt, receipt: { schema: "paperclip.agent-files.v1", cleanupPending: false } })
|
||||
.where(eq(agentInstructionWorkingCopies.runId, copy.runId));
|
||||
const transact = db.transaction.bind(db);
|
||||
const race = vi.spyOn(db, "transaction").mockImplementationOnce(async (callback, config) => {
|
||||
await db.update(agentInstructionWorkingCopies).set({ receipt: { ...copy.receipt, cleanupPending: true } })
|
||||
.where(eq(agentInstructionWorkingCopies.runId, copy.runId));
|
||||
return transact(callback, config);
|
||||
});
|
||||
try {
|
||||
await copies.release(companyId, copy.runId);
|
||||
expect(await copies.get(companyId, copy.runId)).toMatchObject({ receipt: { cleanupPending: false }, nextAttemptAt: null });
|
||||
await expect(fs.stat(copy.localRoot)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
expect(await fs.readFile(path.join(root, entryFile), "utf8")).toBe(initial);
|
||||
} finally { race.mockRestore(); }
|
||||
});
|
||||
it("reactivates from real collection, preserves saved files, retries failed cleanup, and becomes quiet", async () => {
|
||||
const copy = await run(); await fs.writeFile(path.join(copy.localRoot, "memory.txt"), "saved memory");
|
||||
clock(); const worker = coordinator();
|
||||
@@ -1461,6 +1500,53 @@ describe("persistent agent directories", () => {
|
||||
return { copy: (await copies.get(companyId, copy.runId))!, lease, environmentId };
|
||||
}
|
||||
|
||||
it.each(["destroyed", "stopped", "broken_handoff", "extra_owner", "foreign_company"] as const)(
|
||||
"recovers an earlier remote copy only from the exact destroyed successor chain (%s)", async scenario => {
|
||||
const { copy, lease } = await unavailableRemote();
|
||||
const issueId = randomUUID(), successorRunId = randomUUID(), successorId = randomUUID();
|
||||
const acquiredAt = new Date(Date.now() - 2000), handoffAt = new Date(Date.now() - 1000);
|
||||
await db.insert(issues).values({ id: issueId, companyId, title: "Completed retained workspace", status: "done" });
|
||||
await db.insert(heartbeatRuns).values({ id: successorRunId, companyId, agentId, invocationSource: "on_demand", status: "succeeded" });
|
||||
const [previous] = await db.update(environmentLeases).set({ status: "expired", leasePolicy: "reuse_by_environment",
|
||||
issueId, acquiredAt, releasedAt: handoffAt, cleanupStatus: "success", metadata: { agentId } })
|
||||
.where(eq(environmentLeases.id, lease.id)).returning();
|
||||
const successor = { ...previous!, id: successorId, heartbeatRunId: successorRunId, acquiredAt: handoffAt,
|
||||
releasedAt: new Date(), metadata: { agentId } };
|
||||
if (scenario === "foreign_company") {
|
||||
const foreignCompanyId = randomUUID();
|
||||
await db.insert(companies).values({ id: foreignCompanyId, name: "Unrelated owner", issuePrefix: randomUUID().slice(0, 8) });
|
||||
successor.companyId = foreignCompanyId;
|
||||
}
|
||||
const handoff = createReusableLeaseHandoff(previous!, successor);
|
||||
const termination = remoteTerminationReceipt(successor, { providerLeaseId: lease.providerLeaseId,
|
||||
state: scenario === "stopped" ? "stopped" : "destroyed" });
|
||||
await db.update(environmentLeases).set({ metadata: { agentId, reusableLeaseSuccessor: handoff } })
|
||||
.where(eq(environmentLeases.id, lease.id));
|
||||
await db.insert(environmentLeases).values({ ...successor, metadata: { agentId, remoteExecutionTermination: termination,
|
||||
reusableLeasePredecessor: scenario === "broken_handoff" ? { ...handoff, handoffId: randomUUID() } : handoff } });
|
||||
if (scenario === "extra_owner") await db.insert(environmentLeases).values({ ...lease, id: randomUUID(), status: "active" });
|
||||
await db.update(agentInstructionWorkingCopies).set({ state: "pending_collection", attempts: 0, candidateHash: null, candidateBase64: null })
|
||||
.where(eq(agentInstructionWorkingCopies.runId, copy.runId));
|
||||
const shell = vi.spyOn(executionTargetTools, "runAdapterExecutionTargetShellCommand").mockRejectedValue(new Error("must not execute"));
|
||||
try {
|
||||
await copies.recoverStopped();
|
||||
const recovered = (await copies.get(companyId, copy.runId))!;
|
||||
if (scenario === "destroyed") {
|
||||
expect(recovered).toMatchObject({ state: "unavailable", errorCode: "INSTRUCTION_COLLECTION_UNAVAILABLE",
|
||||
receipt: { cleanupPending: false }, nextAttemptAt: null });
|
||||
expect(recovered.processStoppedAt).toBeInstanceOf(Date);
|
||||
await expect(fs.stat(copy.localRoot)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
} else {
|
||||
expect(recovered).toMatchObject({ state: "pending_collection", processStoppedAt: null });
|
||||
expect(await fs.readFile(path.join(copy.localRoot, entryFile), "utf8")).toBe(initial);
|
||||
}
|
||||
expect(shell).not.toHaveBeenCalled();
|
||||
expect(await fs.readFile(path.join(root, entryFile), "utf8")).toBe(initial);
|
||||
const [historical] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, lease.id));
|
||||
expect(historical!.metadata).not.toHaveProperty("remoteExecutionTermination");
|
||||
} finally { shell.mockRestore(); }
|
||||
});
|
||||
|
||||
it.each([undefined, "stopped", "companyId", "runId", "leaseId", "provider", "providerLeaseId"])(
|
||||
"defers unavailable cleanup without exact destruction proof (%s)", async proof => {
|
||||
const { copy } = await unavailableRemote({ state: proof === undefined ? undefined : proof === "stopped" ? "stopped" : "destroyed",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { agents, companies, costEvents, createDb } from "@paperclipai/db";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
@@ -61,7 +61,7 @@ describe.skipIf(!support.supported)("agent roster read projections", () => {
|
||||
|
||||
async function observe<T>(run: (observed: ReturnType<typeof createDb>) => Promise<T>) {
|
||||
const observed = createDb(database.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(observed, { agents, costEvents }, capture);
|
||||
try {
|
||||
const result = await run(observed);
|
||||
|
||||
@@ -1496,6 +1496,46 @@ console.log(JSON.stringify({ type: "result", subtype: "success", result: "hello"
|
||||
} finally { fetchSpy.mockRestore(); }
|
||||
});
|
||||
|
||||
it("reads provider identity only for an authorized credential and preserves connected state when profile access is denied", async () => {
|
||||
const owner = "identity-owner";
|
||||
await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" });
|
||||
const account = await service.save(companyId, owner, { provider: "anthropic", method: "subscription", ownership: "personal", name: "Identity account", loginSessionId: "fixture", agentIds: [], allAgents: true }, "identity-selected-secret");
|
||||
const fetchSpy = vi.spyOn(globalThis, "fetch").mockResolvedValue(Response.json({ account: { email: "avery@example.test", uuid: "private-account-id" }, organization: { name: "Example team", uuid: "private-org-id" } }));
|
||||
const app = express();
|
||||
app.use((req, _res, next) => {
|
||||
req.actor = req.header("x-test-anonymous") ? { type: "none" }
|
||||
: req.header("x-agent") ? { type: "agent", agentId, companyId, companyIds: [companyId], onBehalfOfUserId: owner }
|
||||
: { type: "board", source: "session", userId: req.header("x-test-user") ?? owner, companyIds: [companyId] };
|
||||
next();
|
||||
});
|
||||
app.use("/api", aiConnectionRoutes(db));
|
||||
app.use((error: { status?: number; message: string }, _req: express.Request, res: express.Response, _next: express.NextFunction) => { res.status(error.status ?? 500).json({ error: error.message }); });
|
||||
const url = `/api/companies/${companyId}/ai-connections/${account.connectionId}/identity?grantId=${account.grantId}`;
|
||||
try {
|
||||
const result = await request(app).get(url);
|
||||
expect(result.status).toBe(200);
|
||||
expect(result.headers["cache-control"]).toBe("no-store");
|
||||
expect(result.body).toMatchObject({ ...account, status: "available", accountLabel: "avery@example.test", organizationLabel: "Example team" });
|
||||
expect(fetchSpy).toHaveBeenCalledWith("https://api.anthropic.com/api/oauth/profile", expect.objectContaining({ headers: expect.objectContaining({ Authorization: "Bearer identity-selected-secret" }) }));
|
||||
for (const value of ["identity-selected-secret", "private-account-id", "private-org-id"]) expect(JSON.stringify(result.body)).not.toContain(value);
|
||||
fetchSpy.mockReset();
|
||||
expect((await request(app).get(url).set("x-test-user", "bob")).status).toBe(404);
|
||||
expect((await request(app).get(url.replace(companyId, otherCompanyId))).status).toBe(403);
|
||||
expect((await request(app).get(url.replace(account.grantId, randomUUID()))).status).toBe(404);
|
||||
expect((await request(app).get(url).set("x-test-anonymous", "yes")).status).toBe(401);
|
||||
expect((await request(app).get(url).set("x-agent", "yes")).status).toBe(403);
|
||||
expect((await request(app).get(url.replace(account.connectionId, "invalid"))).status).toBe(404);
|
||||
expect(fetchSpy).not.toHaveBeenCalled();
|
||||
fetchSpy.mockResolvedValue(Response.json({ error: "private-provider-error" }, { status: 403 }));
|
||||
expect((await request(app).get(url)).body).toMatchObject({ status: "unavailable", errorCode: "permission_denied", accountLabel: null });
|
||||
expect((await service.list(companyId, owner)).find(item => item.id === account.connectionId)?.status).toBe("connected");
|
||||
fetchSpy.mockClear();
|
||||
await db.update(connectionGrants).set({ status: "revoked" }).where(eq(connectionGrants.id, account.grantId));
|
||||
expect((await request(app).get(url)).body).toMatchObject({ status: "unavailable", errorCode: "connection_unavailable" });
|
||||
expect(fetchSpy).not.toHaveBeenCalled();
|
||||
} finally { fetchSpy.mockRestore(); }
|
||||
});
|
||||
|
||||
it("uses the same usage probe for shared accounts across legacy and native runner selections", async () => {
|
||||
const account = await service.save(companyId, "alice", { provider: "anthropic", method: "subscription", ownership: "shared", name: "Shared usage account", loginSessionId: "fixture", agentIds: [], allAgents: true }, "shared-usage-secret");
|
||||
const selected = { provider: "anthropic", method: "subscription", mode: "shared", ...account } as const;
|
||||
|
||||
@@ -4,7 +4,7 @@ import { randomUUID } from "node:crypto";
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
|
||||
import {
|
||||
agents,
|
||||
@@ -1164,13 +1164,13 @@ describeEmbeddedPostgres("attention service", () => {
|
||||
})));
|
||||
}
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
|
||||
try {
|
||||
const feed = await attentionService(db).list(companyId, { userId: "board-user" });
|
||||
expect(feed.items.filter(item => item.sourceKind === "failed_run").map(item => item.subject.id)).toEqual([failedId]);
|
||||
} finally { capture.flush(); }
|
||||
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
|
||||
const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
|
||||
&& event.table === "heartbeat_runs" && (event.operation === "select" || event.operation === "execute"));
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThan(0);
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.rows), 0)).toBeLessThan(64);
|
||||
@@ -1209,7 +1209,7 @@ describeEmbeddedPostgres("attention service", () => {
|
||||
});
|
||||
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
|
||||
try {
|
||||
const feed = await attentionService(db).list(companyId, {
|
||||
@@ -1218,7 +1218,7 @@ describeEmbeddedPostgres("attention service", () => {
|
||||
expect(feed.items.filter((item) => item.sourceKind === "failed_run").map((item) => item.subject.id).sort())
|
||||
.toEqual([...keptIds].sort());
|
||||
} finally { capture.flush(); }
|
||||
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
|
||||
const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
|
||||
&& event.table === "heartbeat_runs" && (event.operation === "select" || event.operation === "execute"));
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThan(0);
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeLessThan(4);
|
||||
@@ -1407,7 +1407,7 @@ describeEmbeddedPostgres("attention service", () => {
|
||||
const interactionItems = feed.items.filter((item) => item.sourceKind === "issue_thread_interaction");
|
||||
const detailsByKind = new Map(interactionItems.map((item) => [item.detail?.kind, item]));
|
||||
const summaryEvents: Array<Record<string, unknown>> = [], captureNow = Date.now();
|
||||
const summaryCapture = createDatabaseResponseCapture({ start: captureNow - 100, end: captureNow + 59_000 }, event => summaryEvents.push(event));
|
||||
const summaryCapture = createFullDatabaseResponseCapture({ start: captureNow - 100, end: captureNow + 59_000 }, event => summaryEvents.push(event));
|
||||
installDatabaseResponseCapture(db, { documents, assets, issueAttachments }, summaryCapture);
|
||||
try {
|
||||
const summary = await attentionService(db).summary(companyId, { userId: "board-user" });
|
||||
@@ -1418,7 +1418,7 @@ describeEmbeddedPostgres("attention service", () => {
|
||||
expect(index.totalCount).toBe(feed.totalCount);
|
||||
|
||||
} finally { summaryCapture.flush(); }
|
||||
const cardReads = summaryEvents.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
|
||||
const cardReads = summaryEvents.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
|
||||
&& ["documents", "assets", "issue_attachments"].includes(String(event.table)));
|
||||
expect(cardReads).toEqual([]);
|
||||
|
||||
|
||||
@@ -33,11 +33,13 @@ import { budgetService } from "../services/budgets.js";
|
||||
import { idleWorkSnapshot } from "../services/task-admission.js";
|
||||
import { readFreshWorkspaceReadiness } from "../services/fresh-workspace-continuation.js";
|
||||
import { requestTaskRetry, reconcileTaskChatRetryReceipts } from "../services/task-retry.js";
|
||||
import { taskInterruptionPresentation } from "../services/task-interruption-presentation.js";
|
||||
import { withQueuedCommentIdsInWakePayload } from "../services/issue-queued-comment-queue.js";
|
||||
import { createSystemRecovery } from "../services/system-recovery-state.js";
|
||||
import { remoteTerminationReceipt } from "../services/remote-execution-termination.js";
|
||||
import { stageReceiptReactionRemovals } from "../services/chat-receipt-reactions.js";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
|
||||
import { githubResponseAuthorityFields } from "../services/chat-channels/github/response-authority-read.js";
|
||||
import * as activityLogModule from "../services/activity-log.js";
|
||||
import * as connectionCredentialBindings from "../services/connection-credential-bindings.js";
|
||||
@@ -112,6 +114,7 @@ import {
|
||||
isNotNull,
|
||||
isNull,
|
||||
like,
|
||||
ne,
|
||||
notInArray,
|
||||
or,
|
||||
sql,
|
||||
@@ -2887,7 +2890,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
const normalizedEvent = { ...delivery.normalizedEvent, unusedContent: bulk };
|
||||
await db.update(chatDeliveries).set({ normalizedEvent }).where(eq(chatDeliveries.id, delivery.id));
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { chatMessageLinks, chatDeliveries, chatEndpoints, chatConversations,
|
||||
chatEndpointResources, chatGitHubConfigurations, chatExternalPrincipals, chatIdentityLinks,
|
||||
companyMemberships, chatActions, heartbeatRuns }, capture);
|
||||
@@ -2916,7 +2919,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
} };
|
||||
await db.update(chatDeliveries).set({ normalizedEvent }).where(eq(chatDeliveries.id, delivery.id));
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { chatDeliveries }, capture);
|
||||
let projected: { event: Record<string, unknown> };
|
||||
try {
|
||||
@@ -7152,6 +7155,68 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
expect(await readChatConversationsByEndpointId(f.endpoint.id)).toHaveLength(0);
|
||||
expect(await readChatGitHubReviewsByEndpointId(f.endpoint.id)).toHaveLength(0);
|
||||
});
|
||||
it("offers Retry for a saved automatic PR event after a successful older review", async () => {
|
||||
const f = await reviewBotFixture();
|
||||
const payload = {
|
||||
action: "opened", installation: { id: 2468 },
|
||||
repository: { id: 97531, full_name: "paperclipai/paperclip", name: "paperclip", owner: { id: 1357, login: "paperclipai" } },
|
||||
sender: { id: 77, login: "maintainer" },
|
||||
pull_request: { number: 81, title: "Review successive heads", body: "", draft: false,
|
||||
base: { sha: "a".repeat(40), ref: "master" }, head: { sha: "b".repeat(40) },
|
||||
user: { id: 42, login: "octocat", type: "User" }, labels: [] },
|
||||
};
|
||||
const send = async (event: typeof payload) => {
|
||||
await f.service.handleWebhook(f.endpoint.publicId, "github", signedGitHubWebhookRequest({
|
||||
event: "pull_request", delivery: randomUUID(), payload: event, webhookSecret: f.webhookSecret,
|
||||
}));
|
||||
await f.service.processPendingDeliveries();
|
||||
};
|
||||
await send(payload);
|
||||
await expect.poll(async () => {
|
||||
await f.service.processPendingDeliveries();
|
||||
return (await readChatConversationsByEndpointId(f.endpoint.id)).length;
|
||||
}, { timeout: 10_000 }).toBe(1);
|
||||
const [conversation] = await readChatConversationsByEndpointId(f.endpoint.id);
|
||||
const [task] = await readIssuesById(conversation.issueId!);
|
||||
const [original] = await db.select().from(chatActions).where(and(
|
||||
eq(chatActions.conversationId, conversation.id), eq(chatActions.kind, "inbound_wakeup")));
|
||||
await expect.poll(async () => (await readAgentWakeupRequestsById(original.id)).length).toBe(1);
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.assignedAgentId,
|
||||
issueId: task.id, status: "succeeded", finishedAt: new Date(), wakeupRequestId: original.id,
|
||||
contextSnapshot: { issueId: task.id, taskKey: task.identifier, source: "chat:github", wakeCommentIds: [original.payload.commentId] },
|
||||
}).returning();
|
||||
await db.update(agentWakeupRequests).set({ status: "completed", runId: run.id }).where(eq(agentWakeupRequests.id, original.id));
|
||||
await send({ ...payload, action: "synchronize", pull_request: { ...payload.pull_request, head: { sha: "c".repeat(40) } } });
|
||||
await expect.poll(async () => {
|
||||
await f.service.processPendingDeliveries();
|
||||
return (await db.select().from(chatActions).where(and(
|
||||
eq(chatActions.conversationId, conversation.id), eq(chatActions.kind, "inbound_wakeup")))).length;
|
||||
}).toBe(2);
|
||||
const [pending] = await db.select().from(chatActions).where(and(
|
||||
eq(chatActions.conversationId, conversation.id), eq(chatActions.kind, "inbound_wakeup"), ne(chatActions.id, original.id)));
|
||||
await expect.poll(async () => (await readAgentWakeupRequestsById(pending.id)).length).toBe(1);
|
||||
const [receipt] = await readAgentWakeupRequestsById(pending.id);
|
||||
await db.update(agentWakeupRequests).set({ status: "deferred_issue_execution", runId: null,
|
||||
payload: withQueuedCommentIdsInWakePayload(receipt.payload, [String(pending.payload.commentId)]),
|
||||
requestedAt: new Date(Date.now() - 60_000) }).where(eq(agentWakeupRequests.id, pending.id));
|
||||
expect(receipt.requestedByActorType).toBe("user");
|
||||
await db.update(issues).set({ status: "blocked", executionRunId: null }).where(eq(issues.id, task.id));
|
||||
expect(await taskInterruptionPresentation(db, { ...task, status: "blocked" }))
|
||||
.toMatchObject({ state: "failed", canRetry: true, intentId: pending.id, observedRunId: run.id });
|
||||
await db.update(agents).set({ status: "running", runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 } } })
|
||||
.where(eq(agents.id, f.assignedAgentId));
|
||||
await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.assignedAgentId, status: "running", startedAt: new Date() });
|
||||
f.wakeup.mockImplementation(heartbeatService(db).wakeup);
|
||||
const retry = await requestTaskRetry({ db, companyId: f.companyId, issueId: task.id, actorId: "owner-user",
|
||||
actorSource: "cloud_tenant", observedRunId: run.id, idempotencyKey: randomUUID(), prepareChatRetry: f.service.prepareFailedChatRunRetry });
|
||||
await reconcileTaskChatRetryReceipts(db, { companyId: f.companyId });
|
||||
const [resumed] = await readAgentWakeupRequestsById(pending.id);
|
||||
expect(resumed.runId).toBeTruthy();
|
||||
expect((await readHeartbeatRunsById(resumed.runId!))[0]).toMatchObject({ status: "queued", retryOfRunId: null,
|
||||
contextSnapshot: { wakeCommentIds: [pending.payload.commentId] } });
|
||||
expect((await readAgentWakeupRequestsById(retry.requestId!))[0]).toMatchObject({ status: "coalesced", runId: resumed.runId });
|
||||
});
|
||||
|
||||
it("admits signed PR events into an ordinary assigned task and deduplicates delivery", async () => {
|
||||
const f = await reviewBotFixture();
|
||||
const { buildSkillMentionHref } = await import("@paperclipai/shared");
|
||||
@@ -62866,7 +62931,8 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
const commentsBeforeDisabledLifecycle = await db
|
||||
.select({ id: issueComments.id })
|
||||
.from(issueComments)
|
||||
.where(eq(issueComments.issueId, conversation.issueId));
|
||||
.where(eq(issueComments.issueId, conversation.issueId))
|
||||
.orderBy(issueComments.id);
|
||||
const priorLifecycleIds = new Set(
|
||||
(
|
||||
await db
|
||||
@@ -62946,7 +63012,8 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
db
|
||||
.select({ id: issueComments.id })
|
||||
.from(issueComments)
|
||||
.where(eq(issueComments.issueId, conversation.issueId)),
|
||||
.where(eq(issueComments.issueId, conversation.issueId))
|
||||
.orderBy(issueComments.id),
|
||||
).resolves.toEqual(commentsBeforeDisabledLifecycle);
|
||||
await service.shutdown();
|
||||
});
|
||||
@@ -64140,6 +64207,163 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
return { ...f, pending, originalDelivery };
|
||||
}
|
||||
|
||||
async function successfulGitHubQueuedInputFixture() {
|
||||
const f = await currentGitHubTaskRetryFixture(true);
|
||||
const companyId = f.fixture.companyId, issueId = f.issue.id;
|
||||
const [saved] = await readAgentWakeupRequestsById(f.pending!.id);
|
||||
await db.update(agentWakeupRequests).set({ payload: withQueuedCommentIdsInWakePayload(saved.payload,
|
||||
[String(f.pending!.payload.commentId)]) }).where(eq(agentWakeupRequests.id, saved.id));
|
||||
const contractId = randomUUID(), resultId = randomUUID();
|
||||
await db.insert(completionContracts).values({ id: contractId, companyId, issueId, revision: 1,
|
||||
schemaVersion: "paperclip.completion-contract.v1", policyVersion: "test", risk: "standard",
|
||||
completionAuthority: "server_arbiter", incompleteCriteriaPolicy: "preserve_non_terminal", contractJson: {},
|
||||
canonicalSha256: contractId, createdByActorType: "system", createdByActorId: "test" });
|
||||
await db.update(heartbeatRuns).set({ status: "succeeded", errorCode: null, nativePhase: "committed",
|
||||
completionContractId: contractId }).where(eq(heartbeatRuns.id, f.runId));
|
||||
const [result] = await db.insert(nativeRunResults).values({ id: resultId, companyId, issueId, runId: f.runId,
|
||||
completionContractId: contractId, serverFingerprint: resultId, schemaStatus: "accepted", canonicalSha256: resultId,
|
||||
resultJson: { terminal: { runTerminalState: "succeeded" }, summary: "Earlier review remains saved" } }).returning();
|
||||
await db.update(nativeRunFinalizations).set({ phase: "committed", resultId }).where(eq(nativeRunFinalizations.runId, f.runId));
|
||||
await db.update(agentWakeupRequests).set({ status: "completed" }).where(eq(agentWakeupRequests.id, f.receipt.id));
|
||||
const input = { db, companyId, issueId, actorId: "owner-user", actorSource: "cloud_tenant",
|
||||
observedRunId: f.runId, idempotencyKey: randomUUID(), prepareChatRetry: f.service.prepareFailedChatRunRetry };
|
||||
return { ...f, input, result };
|
||||
}
|
||||
|
||||
it("retries saved GitHub input after a successful run without replaying that run", async () => {
|
||||
const f = await successfulGitHubQueuedInputFixture();
|
||||
try {
|
||||
const { input } = f;
|
||||
const companyId = f.fixture.companyId;
|
||||
const before = (await readHeartbeatRunsById(f.runId))[0];
|
||||
const [request, duplicate] = await Promise.all([requestTaskRetry(input),
|
||||
requestTaskRetry({ ...input, idempotencyKey: randomUUID() })]);
|
||||
expect(request.accepted).toBe(true);
|
||||
expect(duplicate.requestId).toBe(request.requestId);
|
||||
await reconcileTaskChatRetryReceipts(db, { companyId });
|
||||
const [source] = await readAgentWakeupRequestsById(f.pending!.id);
|
||||
expect(source.runId).toBeTruthy();
|
||||
const [successor] = await readHeartbeatRunsById(source.runId!);
|
||||
expect(successor).toMatchObject({ status: "queued", retryOfRunId: null,
|
||||
contextSnapshot: { wakeCommentIds: [f.pending!.payload.commentId] } });
|
||||
await reconcileTaskChatRetryReceipts(db, { companyId });
|
||||
expect((await readAgentWakeupRequestsById(request.requestId!))[0])
|
||||
.toMatchObject({ status: "coalesced", runId: successor.id });
|
||||
expect((await readHeartbeatRunsByCompanyId(companyId)).filter(run => run.wakeupRequestId === f.pending!.id)).toHaveLength(1);
|
||||
expect((await readHeartbeatRunsById(f.runId))[0]).toEqual(before);
|
||||
expect((await db.select().from(nativeRunResults).where(eq(nativeRunResults.id, f.result.id)))[0]).toEqual(f.result);
|
||||
} finally { await f.service.shutdown(); }
|
||||
});
|
||||
|
||||
it.each(["connection_revoked", "source_rebound", "task_completed", "agent_paused", "cleanup_pending"] as const)(
|
||||
"keeps saved-input Retry behind current admission gates: %s", async gate => {
|
||||
const f = await successfulGitHubQueuedInputFixture();
|
||||
try {
|
||||
const companyId = f.fixture.companyId;
|
||||
const request = await requestTaskRetry(f.input);
|
||||
const before = (await readHeartbeatRunsById(f.runId))[0];
|
||||
if (gate === "connection_revoked") await db.update(chatEndpoints).set({ status: "paused" }).where(eq(chatEndpoints.id, f.endpoint.id));
|
||||
if (gate === "source_rebound") await db.update(chatActions).set({ payload: { ...f.pending!.payload, commentId: randomUUID() } })
|
||||
.where(eq(chatActions.id, f.pending!.id));
|
||||
if (gate === "task_completed") await db.update(issues).set({ status: "done" }).where(eq(issues.id, f.issue.id));
|
||||
if (gate === "agent_paused") await db.update(agents).set({ status: "paused" }).where(eq(agents.id, f.fixture.assignedAgentId));
|
||||
const lease = { id: randomUUID(), companyId, issueId: f.issue.id, heartbeatRunId: f.runId,
|
||||
provider: "daytona", providerLeaseId: randomUUID(), leasePolicy: "ephemeral" };
|
||||
if (gate === "cleanup_pending") {
|
||||
await db.insert(environmentLeases).values({ ...lease, status: "pending_cleanup" });
|
||||
await db.update(issues).set({ executionRunId: f.runId }).where(eq(issues.id, f.issue.id));
|
||||
}
|
||||
await reconcileTaskChatRetryReceipts(db, { companyId });
|
||||
expect((await readAgentWakeupRequestsById(f.pending!.id))[0].runId).toBeNull();
|
||||
expect((await readHeartbeatRunsByCompanyId(companyId)).filter(run => run.wakeupRequestId === f.pending!.id)).toHaveLength(0);
|
||||
expect((await readHeartbeatRunsById(f.runId))[0]).toEqual(before);
|
||||
expect((await db.select().from(nativeRunResults).where(eq(nativeRunResults.id, f.result.id)))[0]).toEqual(f.result);
|
||||
if (gate === "cleanup_pending") {
|
||||
expect((await readAgentWakeupRequestsById(request.requestId!))[0].status).toBe("deferred_issue_execution");
|
||||
await db.update(environmentLeases).set({ status: "released", cleanupStatus: "success", releasedAt: new Date(),
|
||||
metadata: { remoteExecutionTermination: remoteTerminationReceipt(lease, { providerLeaseId: lease.providerLeaseId, state: "destroyed" }) },
|
||||
}).where(eq(environmentLeases.id, lease.id));
|
||||
await reconcileTaskChatRetryReceipts(db, { companyId });
|
||||
expect((await readAgentWakeupRequestsById(f.pending!.id))[0].runId).toBeTruthy();
|
||||
}
|
||||
} finally { await f.service.shutdown(); }
|
||||
});
|
||||
|
||||
it.each(["issue.comment.reopen", "boardMessageRetry"] as const)(
|
||||
"resumes saved GitHub input after a successful board turn: %s", async origin => {
|
||||
const f = await successfulGitHubQueuedInputFixture();
|
||||
try {
|
||||
const [previous] = await readHeartbeatRunsById(f.runId);
|
||||
await db.update(heartbeatRuns).set({ responsibleUserId: f.input.actorId, contextSnapshot: {
|
||||
...previous.contextSnapshot, ...(origin === "boardMessageRetry" ? { boardMessageRetry: true } : { source: origin }),
|
||||
} }).where(eq(heartbeatRuns.id, f.runId));
|
||||
const before = (await readHeartbeatRunsById(f.runId))[0];
|
||||
const retry = await requestTaskRetry(f.input);
|
||||
await reconcileTaskChatRetryReceipts(db, { companyId: f.fixture.companyId, requestId: retry.requestId! });
|
||||
const [saved] = await readAgentWakeupRequestsById(f.pending!.id);
|
||||
expect(saved.runId).toBeTruthy();
|
||||
expect((await readHeartbeatRunsById(saved.runId!))[0]).toMatchObject({ retryOfRunId: null,
|
||||
contextSnapshot: { source: "chat:github", wakeCommentIds: [f.pending!.payload.commentId] } });
|
||||
expect((await readHeartbeatRunsById(f.runId))[0]).toEqual(before);
|
||||
} finally { await f.service.shutdown(); }
|
||||
});
|
||||
|
||||
it("keeps a board message queued separately when saved GitHub Retry resumes after cleanup", async () => {
|
||||
const f = await successfulGitHubQueuedInputFixture();
|
||||
try {
|
||||
const companyId = f.fixture.companyId;
|
||||
const lease = { id: randomUUID(), companyId, issueId: f.issue.id, heartbeatRunId: f.runId,
|
||||
provider: "daytona", providerLeaseId: randomUUID(), leasePolicy: "ephemeral" };
|
||||
await db.insert(environmentLeases).values({ ...lease, status: "pending_cleanup" });
|
||||
await db.update(issues).set({ status: "in_review", executionRunId: f.runId }).where(eq(issues.id, f.issue.id));
|
||||
const retry = await requestTaskRetry(f.input);
|
||||
await reconcileTaskChatRetryReceipts(db, { companyId, requestId: retry.requestId! });
|
||||
expect((await readAgentWakeupRequestsById(f.pending!.id))[0].runId).toBeNull();
|
||||
const [comment] = await db.insert(issueComments).values({ companyId, issueId: f.issue.id,
|
||||
authorType: "user", authorUserId: f.input.actorId, body: "A separate board follow-up" }).returning();
|
||||
const [board] = await db.insert(agentWakeupRequests).values({ companyId, agentId: f.fixture.assignedAgentId,
|
||||
source: "automation", triggerDetail: "system", reason: "issue_commented", status: "deferred_issue_execution",
|
||||
requestedByActorType: "user", requestedByActorId: f.input.actorId,
|
||||
payload: withQueuedCommentIdsInWakePayload({ issueId: f.issue.id, commentId: comment.id,
|
||||
_paperclipWakeContext: { issueId: f.issue.id, source: "issue.comment.reopen", wakeReason: "issue_commented" },
|
||||
}, [comment.id]),
|
||||
}).returning();
|
||||
await db.update(environmentLeases).set({ status: "released", cleanupStatus: "success", releasedAt: new Date(),
|
||||
metadata: { remoteExecutionTermination: remoteTerminationReceipt(lease, { providerLeaseId: lease.providerLeaseId, state: "destroyed" }) },
|
||||
}).where(eq(environmentLeases.id, lease.id));
|
||||
await reconcileTaskChatRetryReceipts(db, { companyId, requestId: retry.requestId! });
|
||||
const [saved] = await readAgentWakeupRequestsById(f.pending!.id);
|
||||
expect(saved.runId).toBeTruthy();
|
||||
expect((await readHeartbeatRunsById(saved.runId!))[0]).toMatchObject({ status: "queued",
|
||||
contextSnapshot: { wakeCommentIds: [f.pending!.payload.commentId] } });
|
||||
expect((await readAgentWakeupRequestsById(board.id))[0]).toEqual(board);
|
||||
} finally { await f.service.shutdown(); }
|
||||
});
|
||||
|
||||
it("dispatches saved input through the task Retry HTTP endpoint and survives a repeated click", async () => {
|
||||
const f = await successfulGitHubQueuedInputFixture();
|
||||
const { issueRoutes } = await import("../routes/issues.js");
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
app.use((req, _res, next) => { req.actor = boardActor(f.fixture.companyId); next(); });
|
||||
app.use("/api", issueRoutes(db, createStorageService().storage, { chatRunRetries: f.service }));
|
||||
app.use(errorHandler);
|
||||
try {
|
||||
const body = { observedRunId: f.runId, idempotencyKey: randomUUID() };
|
||||
const first = await request(app).post(`/api/issues/${f.issue.id}/retry`).send(body);
|
||||
expect(first.status, JSON.stringify(first.body)).toBe(202);
|
||||
await expect.poll(async () => (await readAgentWakeupRequestsById(f.pending!.id))[0].runId).toBeTruthy();
|
||||
const second = await request(app).post(`/api/issues/${f.issue.id}/retry`).send(body);
|
||||
expect(second.status).toBe(202);
|
||||
expect(second.body.requestId).toBe(first.body.requestId);
|
||||
expect((await readHeartbeatRunsByCompanyId(f.fixture.companyId)).filter(run => run.wakeupRequestId === f.pending!.id)).toHaveLength(1);
|
||||
const detail = await request(app).get(`/api/issues/${f.issue.id}`);
|
||||
expect(detail.status).toBe(200);
|
||||
expect(detail.body.interruption).toMatchObject({ state: "working", canRetry: false });
|
||||
expect((await db.select().from(nativeRunResults).where(eq(nativeRunResults.id, f.result.id)))[0]).toEqual(f.result);
|
||||
} finally { await f.service.shutdown(); }
|
||||
});
|
||||
|
||||
it.each(["reconnected", "newer_input", "reconnected_newer_input", "snapshot_newer_input"] as const)(
|
||||
"admits current GitHub task Retry without replaying obsolete input: %s", async kind => {
|
||||
const f = await currentGitHubTaskRetryFixture(kind.includes("newer_input"));
|
||||
@@ -64269,7 +64493,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
status: "active", membershipRole: "operator" });
|
||||
}
|
||||
const queryEvents: Record<string, unknown>[] = [];
|
||||
const capture = mode === "backlog_51" ? createDatabaseResponseCapture({ start: Date.now() - 1,
|
||||
const capture = mode === "backlog_51" ? createFullDatabaseResponseCapture({ start: Date.now() - 1,
|
||||
end: Date.now() + 59_000 }, event => queryEvents.push(event)) : undefined;
|
||||
if (capture) installDatabaseResponseCapture(db, { chatActions, chatDeliveries, chatMessageLinks, issueComments,
|
||||
agentWakeupRequests, chatEndpoints, chatConversations }, capture);
|
||||
@@ -64289,7 +64513,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
idempotencyKey: mode === "concurrent_clicks" ? randomUUID() : input.idempotencyKey })]);
|
||||
if (capture) {
|
||||
capture.flush();
|
||||
expect(Number(queryEvents.find(event => event.event === "paperclip.db_response_capture.summary.v1")?.startedQueries))
|
||||
expect(Number(queryEvents.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")?.startedQueries))
|
||||
.toBeLessThan(180);
|
||||
}
|
||||
expect(two.requestId).toBe(one.requestId);
|
||||
@@ -67489,7 +67713,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
})));
|
||||
const scope = { companyId: context.fixture.companyId, issueId: context.issue.id };
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { chatActions, agentWakeupRequests }, capture);
|
||||
let scopes: Array<{ companyId: string; issueId: string }>;
|
||||
try {
|
||||
@@ -67506,7 +67730,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
}
|
||||
expect(events.at(-1)).toMatchObject({ startedQueries: mode === "task" ? 0 : 1,
|
||||
completedQueries: mode === "task" ? 0 : 1, inFlightAtEnd: 0 });
|
||||
expect(events.filter(event => event.event === "paperclip.db_response_capture.entry.v1")
|
||||
expect(events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1")
|
||||
.reduce((count, event) => count + Number(event.rows), 0)).toBe(mode === "task" ? 0 : scopes.length);
|
||||
} finally {
|
||||
await context.service.shutdown(); await other.service.shutdown();
|
||||
@@ -77214,7 +77438,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
})));
|
||||
const scope = { companyId: context.fixture.companyId, issueId: context.conversation.issueId };
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { chatActions, agentWakeupRequests }, capture);
|
||||
let scopes: Array<{ companyId: string; issueId: string }>;
|
||||
try {
|
||||
@@ -77231,7 +77455,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
}
|
||||
expect(events.at(-1)).toMatchObject({ startedQueries: mode === "task" ? 0 : 1,
|
||||
completedQueries: mode === "task" ? 0 : 1, inFlightAtEnd: 0 });
|
||||
expect(events.filter(event => event.event === "paperclip.db_response_capture.entry.v1")
|
||||
expect(events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1")
|
||||
.reduce((count, event) => count + Number(event.rows), 0)).toBe(mode === "task" ? 0 : scopes.length);
|
||||
} finally {
|
||||
await context.service.shutdown(); await other.service.shutdown();
|
||||
|
||||
@@ -4,7 +4,7 @@ import { randomUUID } from "node:crypto";
|
||||
import { and, asc, eq } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { agents, agentWakeupRequests, chatCompletionDeliveries as deliveries, chatTaskHandoffs as handoffs,
|
||||
companies, createDb, documents, environmentLeases, heartbeatRuns, issueComments, issueRecoveryActions, issueThreadInteractions, issues } from "@paperclipai/db";
|
||||
companies, createDb, documents, environmentLeases, executionWorkspaces, heartbeatRuns, issueComments, issueRecoveryActions, issueThreadInteractions, issues, workspaceOperations } from "@paperclipai/db";
|
||||
import { recoveryPostgresSupport as support, startRecoveryPostgres } from "./helpers/recovery-postgres.js";
|
||||
import { settleInterruptedNativeBootstrap, terminalizeLegacyExecution } from "../services/legacy-execution-recovery.js";
|
||||
import { getExecutionBlocker } from "../services/execution-blocker.js";
|
||||
@@ -15,6 +15,8 @@ import { documentService } from "../services/documents.js";
|
||||
import { instanceSettingsService } from "../services/instance-settings.js";
|
||||
import { chatCompletionDeliveryService, isCompletedOnboardingHandoffWake, prepareChatCompletionTurn, recordChatCompletion, recordChatHandoff } from "../services/chat-completion-delivery.js";
|
||||
import { heartbeatService, shouldQueueFollowupForRunningIssueWake } from "../services/heartbeat.js";
|
||||
import { createHeartbeatQueue } from "../services/heartbeat/queue.js";
|
||||
import { createHeartbeatQueueTestDependencies } from "./helpers/heartbeat-queue.js";
|
||||
|
||||
(support.supported ? describe : describe.skip)("chat completion delivery", () => {
|
||||
let db: ReturnType<typeof createDb>;
|
||||
@@ -68,6 +70,75 @@ import { heartbeatService, shouldQueueFollowupForRunningIssueWake } from "../ser
|
||||
})]);
|
||||
expect(JSON.stringify(run.contextSnapshot?.chatCompletionUpdates)).not.toMatch(/Injected|credentials|Plan accepted|Agent approved/);
|
||||
});
|
||||
it("retains a distinct completion receipt behind a real dependency gate", async () => {
|
||||
const f = await seed();
|
||||
const blocker = await f.create();
|
||||
await issueService(db).update(f.sourceId, { status: "blocked", blockedByIssueIds: [blocker.id] });
|
||||
await f.finish();
|
||||
const heartbeat = heartbeatService(db);
|
||||
// The generic dependency signal is replaceable. The outbox's exact wake
|
||||
// receipt is not: it needs that identity to reconcile a refused admission.
|
||||
await heartbeat.wakeup(f.agentId, {
|
||||
source: "automation", triggerDetail: "system", reason: "issue_continuation_needed",
|
||||
requestedByActorType: "system", requestedByActorId: "dependency-test",
|
||||
payload: { issueId: f.sourceId }, contextSnapshot: { issueId: f.sourceId },
|
||||
});
|
||||
const [delivery] = await f.rows();
|
||||
const service = chatCompletionDeliveryService(db, heartbeat);
|
||||
await service.deliver(delivery.id);
|
||||
const receipts = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId));
|
||||
expect(receipts).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ idempotencyKey: `chat-completion:${delivery.id}:0`,
|
||||
reason: "issue_dependencies_blocked", status: "skipped", runId: null,
|
||||
payload: expect.objectContaining({ chatCompletionDeliveryIds: [delivery.id] }) }),
|
||||
]));
|
||||
expect(receipts).toHaveLength(2);
|
||||
expect(receipts.find(receipt => receipt.idempotencyKey?.startsWith("execution-wait:")))
|
||||
.toMatchObject({ coalescedCount: 0 });
|
||||
expect((await f.rows())[0]).toMatchObject({ attempts: 0, status: "pending", targetRunId: null, error: null });
|
||||
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(1);
|
||||
expect(await issueService(db).getDependencyReadiness(f.sourceId)).toMatchObject({
|
||||
isDependencyReady: false, unresolvedBlockerIssueIds: [blocker.id],
|
||||
});
|
||||
});
|
||||
it("admits a saved worker completion after its owning finalize encloses the snapshot", async () => {
|
||||
const f = await seed(), workspaceId = randomUUID(), workerRunId = randomUUID();
|
||||
await db.update(issues).set({ responsibleUserId: "operator" }).where(eq(issues.id, f.sourceId));
|
||||
await db.insert(executionWorkspaces).values({ id: workspaceId, companyId: f.companyId,
|
||||
mode: "isolated_workspace", strategyType: "managed_remote", name: "Saved worker", providerType: "daytona" });
|
||||
await db.insert(heartbeatRuns).values({ id: workerRunId, companyId: f.companyId, agentId: f.agentId,
|
||||
status: "succeeded", contextSnapshot: { issueId: f.task.id } });
|
||||
await db.update(issues).set({ executionWorkspaceId: workspaceId }).where(eq(issues.id, f.task.id));
|
||||
await issueService(db).update(f.sourceId, { status: "blocked", blockedByIssueIds: [f.task.id] });
|
||||
await db.insert(workspaceOperations).values([
|
||||
{ companyId: f.companyId, issueId: f.task.id, executionWorkspaceId: workspaceId, heartbeatRunId: workerRunId,
|
||||
phase: "workspace_finalize", status: "succeeded", startedAt: new Date(10_000), finishedAt: new Date(12_000) },
|
||||
{ companyId: f.companyId, issueId: f.task.id, executionWorkspaceId: workspaceId, heartbeatRunId: workerRunId,
|
||||
phase: "remote_workspace_snapshot", status: "succeeded", startedAt: new Date(10_100), finishedAt: new Date(11_900) },
|
||||
]);
|
||||
await f.finish();
|
||||
const deps = createHeartbeatQueueTestDependencies(db), issueSvc = issueService(db);
|
||||
deps.getAgent.mockImplementation(id => db.select().from(agents).where(eq(agents.id, id)).then(rows => rows[0] ?? null));
|
||||
deps.instanceSettings.getExperimental.mockImplementation(() => instanceSettingsService(db).getExperimental());
|
||||
deps.issuesSvc.getDependencyReadiness.mockImplementation(issueSvc.getDependencyReadiness);
|
||||
deps.issuesSvc.listDependencyReadiness.mockImplementation(issueSvc.listDependencyReadiness);
|
||||
// Admit the real durable run, then hold provider dispatch at the normal drain boundary.
|
||||
deps.getSchedulingSuppression.mockResolvedValue({ suppressed: true, reason: "task_drain" });
|
||||
const queue = createHeartbeatQueue(db, deps);
|
||||
const service = chatCompletionDeliveryService(db, { wakeup: queue.trackWakeup });
|
||||
const [delivery] = await f.rows();
|
||||
await service.deliver(delivery.id);
|
||||
const [queued] = await f.rows();
|
||||
expect(queued).toMatchObject({ status: "queued", attempts: 0, error: null, targetRunId: expect.any(String) });
|
||||
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, queued.targetRunId!));
|
||||
expect(run).toMatchObject({ status: "queued", contextSnapshot: { issueId: f.sourceId,
|
||||
wakeReason: "chat_task_completed", chatCompletionDeliveryIds: [delivery.id] } });
|
||||
const prepared = await prepareChatCompletionTurn(db, run);
|
||||
expect(prepared.contextSnapshot?.chatCompletionUpdates).toEqual([
|
||||
expect.objectContaining({ id: f.task.id, status: "done", completedAt: expect.any(Date), url: `/issues/${f.task.identifier ?? f.task.id}` }),
|
||||
]);
|
||||
expect(deps.executeRun).not.toHaveBeenCalled();
|
||||
});
|
||||
it("dispatches a committed completion without activity publication and recovers after restart", async () => {
|
||||
const f = await seed();
|
||||
const run = vi.fn(() => f.service.sweepPending());
|
||||
|
||||
@@ -17,7 +17,7 @@ import {
|
||||
projectWorkspaces,
|
||||
} from "@paperclipai/db";
|
||||
import { parseFrontmatterMarkdown } from "@paperclipai/shared";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
|
||||
import {
|
||||
getEmbeddedPostgresTestSupport,
|
||||
startEmbeddedPostgresTestDatabase,
|
||||
@@ -651,17 +651,17 @@ describeEmbeddedPostgres("companySkillService.list", () => {
|
||||
]);
|
||||
await svc.list(companyId);
|
||||
const events: Record<string, unknown>[] = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { agents, costEvents }, capture);
|
||||
let listed: Awaited<ReturnType<typeof svc.list>>;
|
||||
try { listed = await svc.list(companyId); } finally { capture.flush(); }
|
||||
expect(listed.find(item => item.id === first.id)?.attachedAgentCount).toBe(2);
|
||||
expect(listed.find(item => item.id === second.id)?.attachedAgentCount).toBe(1);
|
||||
expect(listed.find(item => item.id === ambiguous.id)?.attachedAgentCount).toBe(0);
|
||||
const agentReads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" && event.table === "agents");
|
||||
const agentReads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "agents");
|
||||
expect(agentReads).toHaveLength(1);
|
||||
expect(Number(agentReads[0]?.responseValueBytes)).toBeLessThan(2_000);
|
||||
expect(events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" && event.table === "cost_events")).toHaveLength(0);
|
||||
expect(events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "cost_events")).toHaveLength(0);
|
||||
await db.update(agents).set({ adapterConfig: { paperclipSkillSync: { desiredSkills: [second.key] } } }).where(eq(agents.id, readerId));
|
||||
const refreshed = await svc.list(companyId, { q: "First" });
|
||||
expect(refreshed.find(item => item.id === first.id)?.attachedAgentCount).toBe(1);
|
||||
@@ -680,12 +680,12 @@ describeEmbeddedPostgres("companySkillService.list", () => {
|
||||
const before = await svc.list(companyId);
|
||||
expect(before.find(item => item.id === inserted[0]!.id)?.folderPath).toBe("bundled/paperclip-core");
|
||||
const events: Record<string, unknown>[] = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { companySkills }, capture);
|
||||
let after: Awaited<ReturnType<typeof svc.list>>;
|
||||
try { after = await svc.list(companyId); } finally { capture.flush(); }
|
||||
expect(after).toEqual(before);
|
||||
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" && event.table === "company_skills");
|
||||
const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "company_skills");
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.responseValueBytes), 0)).toBeLessThan(950_000);
|
||||
const stored = await svc.getById(companyId, inserted[0]!.id);
|
||||
expect(stored?.markdown).toBe(savedMarkdown);
|
||||
@@ -762,11 +762,11 @@ describeEmbeddedPostgres("companySkillService.list", () => {
|
||||
.orderBy(companySkillVersions.id);
|
||||
expect(before.length).toBeGreaterThan(0);
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { companySkillVersions }, capture);
|
||||
try { expect(await companySkillService(db).list(companyId)).toEqual(expected); }
|
||||
finally { capture.flush(); }
|
||||
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
|
||||
const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
|
||||
&& event.table === "company_skill_versions" && event.operation === "select");
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThan(0);
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.responseValueBytes), 0)).toBeLessThan(4096);
|
||||
@@ -783,11 +783,11 @@ describeEmbeddedPostgres("companySkillService.list", () => {
|
||||
? { type, companyId: otherCompanyId, agentId: randomUUID() }
|
||||
: { type, companyIds: [otherCompanyId] };
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { companySkillVersions }, capture);
|
||||
try { expect(await companySkillService(db, { actor }).list(companyId)).toEqual([]); }
|
||||
finally { capture.flush(); }
|
||||
expect(events.find(event => event.event === "paperclip.db_response_capture.summary.v1")?.completedQueries).toBe(0);
|
||||
expect(events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")?.completedQueries).toBe(0);
|
||||
});
|
||||
|
||||
it("restores a deleted release after warm validation", async () => {
|
||||
|
||||
@@ -6,9 +6,10 @@ import { join } from "node:path";
|
||||
import { createServer } from "node:http";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { beforeAll, afterAll, describe, expect, it, vi } from "vitest";
|
||||
import { createDb, authUsers, activityLog, agentApiKeys, companies, companySecrets, companySecretVersions, chatConversations, chatDeliveries, chatEndpoints, chatExternalPrincipals, chatIdentityLinks, toolPolicies, toolRateLimitCounters, connectionGrants, connectionGrantMembers, connectionGrantDelegations, toolAccessAuditEvents, toolOauthStates, issueThreadInteractions, connectionIntentDeliveries, agents, companyMemberships, heartbeatRuns, issues, principalPermissionGrants, projects, toolApplications, toolConnections, toolConnectionInstalls, toolMcpGateways, toolMcpGatewayTokens, toolActionRequests, toolCatalogEntries, toolInvocations, toolCallEvents, toolProfiles, toolProfileBindings, plugins, adapterAuthSessions, environments, routines, chatGitHubReviews } from "@paperclipai/db";
|
||||
import { createDb, authUsers, activityLog, agentApiKeys, companies, companySecrets, companySecretVersions, chatConversations, chatDeliveries, chatEndpoints, chatExternalPrincipals, chatIdentityLinks, toolPolicies, toolRateLimitCounters, connectionGrants, connectionGrantMembers, connectionGrantDelegations, toolAccessAuditEvents, toolOauthStates, issueThreadInteractions, connectionIntentDeliveries, agents, companyMemberships, heartbeatRuns, issues, principalPermissionGrants, projects, toolApplications, toolConnections, toolConnectionInstalls, aiConnectionPools, toolMcpGateways, toolMcpGatewayTokens, toolActionRequests, toolCatalogEntries, toolInvocations, toolCallEvents, toolProfiles, toolProfileBindings, plugins, adapterAuthSessions, environments, routines, chatGitHubReviews } from "@paperclipai/db";
|
||||
import { findSatisfiedToolConnection, satisfiedConnectionIntentService } from "../services/satisfied-connection-intents.js";
|
||||
import { connectionManagementService, continuingConnectionManagement } from "../services/connection-management.js";
|
||||
import { aiConnectionService } from "../services/ai-connections.js";
|
||||
import { redactRunnerApiPayloads } from "../services/native-runtime/runner-api-log-redaction.js";
|
||||
import * as remoteEndpointGuard from "../services/remote-http-endpoint-guard.js";
|
||||
import * as remoteHttpFetch from "../services/remote-http-fetch.js";
|
||||
@@ -87,13 +88,14 @@ describe("connection management with production default agents", () => {
|
||||
authority: new PaperclipRunnerToolAuthority(server.db, { companyId: base.companyId, agentId: agent.id, issueId: base.issueId, runId: base.runId, apiUrl: server.apiUrl, storage: server.storage }) };
|
||||
}
|
||||
|
||||
it("matches ordinary install capabilities on real PostgreSQL for scoped, retained and fallback resources", async () => {
|
||||
it.each(["tool", "ai"] as const)("matches ordinary %s install capabilities on real PostgreSQL for scoped, retained and fallback resources", async (connectionPurpose) => {
|
||||
const f = await server.fixture({ connectionScenario: "fresh" });
|
||||
const candidateIds = [f.agentId, randomUUID(), randomUUID()];
|
||||
await server.db.insert(agents).values(candidateIds.slice(1).map((id, index) => ({ id, companyId: f.companyId, name: `Capability target ${index}`, adapterType: "process" })));
|
||||
const [application] = await server.db.insert(toolApplications).values({ companyId: f.companyId, name: "Capability differential", type: "mcp_http" }).returning();
|
||||
const [connection] = await server.db.insert(toolConnections).values({ companyId: f.companyId, applicationId: application.id,
|
||||
uid: randomUUID(), name: "Capability differential", transport: "mcp_remote", authKind: "none", createdByUserId: "different-creator" }).returning();
|
||||
uid: randomUUID(), name: "Capability differential", connectionPurpose,
|
||||
transport: connectionPurpose === "ai" ? "runtime_auth" : "mcp_remote", authKind: "none", createdByUserId: "different-creator" }).returning();
|
||||
await server.db.insert(principalPermissionGrants).values({ companyId: f.companyId, principalType: "user", principalId: f.responsibleUserId!,
|
||||
permissionKey: "tools:manage_profiles", scope: { targetAgentIds: candidateIds.slice(0, 2) } });
|
||||
const service = connectionManagementService(server.db);
|
||||
@@ -177,7 +179,7 @@ describe("connection management with production default agents", () => {
|
||||
expect(await faulting.describeAgentInstallCapabilities(localActor, resource, [candidateIds[1]])).toEqual([denied]);
|
||||
});
|
||||
|
||||
it("preserves the complete grants GET across human roles and denies a real install mutation after revocation", async () => {
|
||||
it.each(["tool", "ai"] as const)("preserves the complete %s grants GET across human roles and denies a real install mutation after revocation", async (connectionPurpose) => {
|
||||
const f = await server.fixture({ connectionScenario: "fresh" });
|
||||
const candidateIds = [f.agentId, randomUUID(), randomUUID()];
|
||||
await server.db.insert(agents).values(candidateIds.slice(1).map((id, index) => ({ id, companyId: f.companyId, name: `Grant target ${index}`, adapterType: "process" })));
|
||||
@@ -198,7 +200,8 @@ describe("connection management with production default agents", () => {
|
||||
scope: permissionKey === "tools:manage_profiles" ? { targetAgentIds: candidateIds.slice(0, 2) } : null })));
|
||||
const [application] = await server.db.insert(toolApplications).values({ companyId: f.companyId, name: "Grants response parity", type: "mcp_http" }).returning();
|
||||
const [connection] = await server.db.insert(toolConnections).values({ companyId: f.companyId, applicationId: application.id,
|
||||
uid: randomUUID(), name: "Grants response parity", transport: "mcp_remote", authKind: "none", createdByUserId: users.creator }).returning();
|
||||
uid: randomUUID(), name: "Grants response parity", connectionPurpose,
|
||||
transport: connectionPurpose === "ai" ? "runtime_auth" : "mcp_remote", authKind: "none", createdByUserId: users.creator }).returning();
|
||||
const grants = await server.db.insert(connectionGrants).values([
|
||||
{ companyId: f.companyId, connectionId: connection.id, kind: "organization", createdByUserId: users.creator, isDefault: true },
|
||||
...[users.creator, users.manager, users.member].map((subjectUserId, index) => ({ companyId: f.companyId, connectionId: connection.id, kind: "user" as const,
|
||||
@@ -253,6 +256,65 @@ describe("connection management with production default agents", () => {
|
||||
expect(savedInstalls).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("checks company-wide personal AI installs once per target and preserves pool member authorization", async () => {
|
||||
const f = await server.fixture({ connectionScenario: "fresh" });
|
||||
const candidateIds = [f.agentId, ...Array.from({ length: 19 }, () => randomUUID())];
|
||||
await server.db.insert(agents).values(candidateIds.slice(1).map((id, index) => ({ id, companyId: f.companyId, name: `AI target ${index}`, adapterType: "process" })));
|
||||
const [application] = await server.db.insert(toolApplications).values({ companyId: f.companyId, name: "AI capability checks", type: "mcp_http" }).returning();
|
||||
const [connection] = await server.db.insert(toolConnections).values({ companyId: f.companyId, applicationId: application.id,
|
||||
uid: randomUUID(), name: "Personal AI account", transport: "runtime_auth", connectionPurpose: "ai", authKind: "api_key",
|
||||
credentialPolicy: "per_user", createdByUserId: f.responsibleUserId }).returning();
|
||||
const [grant] = await server.db.insert(connectionGrants).values({ companyId: f.companyId, connectionId: connection.id,
|
||||
kind: "user", subjectUserId: f.responsibleUserId, createdByUserId: f.responsibleUserId }).returning();
|
||||
await server.db.insert(toolConnectionInstalls).values({ companyId: f.companyId, connectionId: connection.id, targetType: "company", targetId: f.companyId });
|
||||
await server.db.insert(principalPermissionGrants).values({ companyId: f.companyId, principalType: "user", principalId: f.responsibleUserId!,
|
||||
permissionKey: "tools:manage_profiles", scope: { targetAgentIds: candidateIds } });
|
||||
let targetReads = 0;
|
||||
const observedDb = new Proxy(server.db, { get(target, key, receiver) {
|
||||
if (key !== "select") return Reflect.get(target, key, receiver);
|
||||
return (...fields: Parameters<typeof target.select>) => {
|
||||
const builder = target.select(...fields), from = builder.from.bind(builder);
|
||||
builder.from = ((...args: Parameters<typeof from>) => {
|
||||
const query = from(...args), execute = query.execute.bind(query);
|
||||
query.execute = (async (...executeArgs: Parameters<typeof execute>) => {
|
||||
if (query.toSQL().sql.includes('"agents"."id" =')) targetReads++;
|
||||
return execute(...executeArgs);
|
||||
}) as typeof execute;
|
||||
return query;
|
||||
}) as typeof builder.from;
|
||||
return builder;
|
||||
};
|
||||
} });
|
||||
const service = connectionManagementService(observedDb);
|
||||
const actor: AuthorizationActor = { type: "board", userId: f.responsibleUserId, source: "cloud_tenant" };
|
||||
const resource = { type: "connection_management" as const, companyId: f.companyId, connectionId: connection.id, operation: "configure", readOnly: false };
|
||||
const baseline = [];
|
||||
for (const targetAgentId of candidateIds) baseline.push(await service.capability(actor, "connections:profiles", { ...resource, targetAgentId }));
|
||||
expect(baseline.every(result => result.allowed)).toBe(true);
|
||||
expect(targetReads).toBe(candidateIds.length ** 2);
|
||||
targetReads = 0;
|
||||
expect(await service.describeAgentInstallCapabilities(actor, resource, candidateIds)).toEqual(baseline);
|
||||
expect(targetReads).toBe(candidateIds.length);
|
||||
|
||||
const [pool] = await server.db.insert(toolConnections).values({ companyId: f.companyId, applicationId: application.id,
|
||||
uid: randomUUID(), name: "Pool with private member", transport: "runtime_auth", connectionPurpose: "ai", authKind: "none" }).returning();
|
||||
await server.db.insert(aiConnectionPools).values({ id: pool.id, companyId: f.companyId, pluginKey: "fixture-pool", config: {
|
||||
name: "Pool", enabled: true, mode: "round_robin", thresholdPercent: 90,
|
||||
members: [{ id: randomUUID(), binding: { mode: "delegated", provider: "openai", method: "api_key", connectionId: connection.id, grantId: grant.id }, profile: { provider: "codex", model: "fixture-model" } }],
|
||||
} });
|
||||
await server.db.delete(principalPermissionGrants).where(and(eq(principalPermissionGrants.companyId, f.companyId), eq(principalPermissionGrants.principalId, f.responsibleUserId!)));
|
||||
await server.db.insert(principalPermissionGrants).values({ companyId: f.companyId, principalType: "user", principalId: f.responsibleUserId!,
|
||||
permissionKey: "tools:manage_profiles", scope: { connectionIds: [pool.id] } });
|
||||
const poolResource = { ...resource, connectionId: pool.id };
|
||||
expect((await service.describeAgentInstallCapabilities(actor, poolResource, candidateIds.slice(0, 2))).every(result => result.allowed)).toBe(true);
|
||||
// The pool grant cannot replace the member account's own authority.
|
||||
await server.db.update(connectionGrants).set({ status: "revoked" }).where(eq(connectionGrants.id, grant.id));
|
||||
await server.db.update(toolConnections).set({ createdByUserId: "different-owner" }).where(eq(toolConnections.id, connection.id));
|
||||
const ordinary = await service.capability(actor, "connections:profiles", { ...poolResource, targetAgentId: candidateIds[0] });
|
||||
expect(ordinary.allowed).toBe(false);
|
||||
expect(await service.describeAgentInstallCapabilities(actor, poolResource, candidateIds.slice(0, 2))).toEqual([ordinary, ordinary]);
|
||||
});
|
||||
|
||||
it("creates and configures tool connections and profiles with fresh grants and actual key/JWT authentication", async () => {
|
||||
const f = await fixture();
|
||||
const grants = await server.db.select().from(principalPermissionGrants).where(eq(principalPermissionGrants.principalId, f.agentId));
|
||||
@@ -1642,6 +1704,36 @@ describe("connection management with production default agents", () => {
|
||||
} finally { provider = null; unregisterServerAdapter("paperclip_runner"); }
|
||||
});
|
||||
|
||||
it("reads only the authorized Anthropic account identity through actual agent authentication", async () => {
|
||||
const f = await fixture();
|
||||
const account = await aiConnectionService(server.db).save(f.companyId, f.userId, {
|
||||
provider: "anthropic", method: "subscription", ownership: "personal", name: "Account identity fixture",
|
||||
loginSessionId: "fixture", agentIds: [f.agentId], allAgents: false,
|
||||
}, "selected-identity-credential");
|
||||
let calls = 0;
|
||||
provider = async (url, init) => {
|
||||
calls++;
|
||||
expect(url).toBe("https://api.anthropic.com/api/oauth/profile");
|
||||
expect(new Headers(init?.headers).get("authorization")).toBe("Bearer selected-identity-credential");
|
||||
return Response.json({ account: { uuid: "private-seat", email: "avery@example.test" }, organization: { uuid: "private-org", name: "Example team" } });
|
||||
};
|
||||
const path = `/companies/${f.companyId}/ai-connections/${account.connectionId}/identity?grantId=${account.grantId}`;
|
||||
try {
|
||||
for (const token of [f.key.token, f.jwt]) {
|
||||
const response = await f.request("GET", path, undefined, token);
|
||||
expect(response).toMatchObject({ status: 200, body: { ...account, status: "available", accountLabel: "avery@example.test", organizationLabel: "Example team" } });
|
||||
for (const secret of ["selected-identity-credential", "private-seat", "private-org"]) expect(JSON.stringify(response)).not.toContain(secret);
|
||||
}
|
||||
expect(calls).toBe(2);
|
||||
expect((await f.request("GET", path.replace(f.companyId, f.foreignCompanyId))).status).toBe(403);
|
||||
expect((await f.request("GET", path.replace(account.grantId, randomUUID()))).status).toBe(404);
|
||||
expect(calls).toBe(2);
|
||||
await server.db.update(connectionGrants).set({ status: "revoked" }).where(eq(connectionGrants.id, account.grantId));
|
||||
expect(await f.request("GET", path)).toMatchObject({ status: 200, body: { status: "unavailable", errorCode: "connection_unavailable", accountLabel: null } });
|
||||
expect(calls).toBe(2);
|
||||
} finally { provider = null; }
|
||||
});
|
||||
|
||||
it("filters AI account and pool listings by the agent's resolved connection scope", async () => {
|
||||
const f = await fixture();
|
||||
provider = async url => {
|
||||
|
||||
@@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { agents, companies, createDb, heartbeatRuns, issueComments, issueThreadInteractions, issues } from "@paperclipai/db";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { settleConversationTurnForRun } from "../services/agent-conversations.js";
|
||||
|
||||
@@ -52,11 +52,11 @@ const support = await getEmbeddedPostgresTestSupport();
|
||||
it.each(["response", "interaction", "reset"] as const)("bounds %s settlement reads and retains all saved text and run payloads", async reply => {
|
||||
const input = await seed(reply), before = await sourceDigests(input);
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { heartbeatRuns, issues, issueComments }, capture);
|
||||
try { expect(await settleConversationTurnForRun(db, input.companyId, input.runId)).toBe(true); }
|
||||
finally { capture.flush(); }
|
||||
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
|
||||
const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
|
||||
&& ["heartbeat_runs", "issues", "issue_comments"].includes(String(event.table))
|
||||
&& (event.operation === "select" || event.operation === "execute"));
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThanOrEqual(4);
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { QueryPromise } from "drizzle-orm/query-promise";
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import { withDatabaseOperation } from "@paperclipai/db";
|
||||
import { installDatabaseWorkSignals } from "@paperclipai/db/work-signals";
|
||||
import { databaseOperationMiddleware } from "../middleware/database-operation.js";
|
||||
import { withBudgetSpan } from "../services/budget-tracing.js";
|
||||
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "../services/delivery-work-notifications.js";
|
||||
import {
|
||||
createRunPreparationTiming,
|
||||
measureRunPhase,
|
||||
withRunExecutionTiming,
|
||||
withRunResponsePresentationTiming,
|
||||
withRunWorkspaceFinalizationTiming,
|
||||
} from "../services/execution-preparation-timing.js";
|
||||
|
||||
const { owners } = await vi.hoisted(async () => {
|
||||
const { AsyncLocalStorage } = await import("node:async_hooks");
|
||||
return { owners: new AsyncLocalStorage<string>() };
|
||||
});
|
||||
vi.mock("@paperclipai/db", async importOriginal => ({
|
||||
...await importOriginal<typeof import("@paperclipai/db")>(),
|
||||
withDatabaseOperation: <T>(owner: string, action: () => T): T => owners.run(owner, action),
|
||||
}));
|
||||
|
||||
class OwnerQuery extends QueryPromise<string | undefined> {
|
||||
async execute() { return owners.getStore(); }
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it.each(["", "http://collector.test"])("attributes lazy queries to execution, preparation, and budget owners with endpoint %s", async endpoint => {
|
||||
vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", endpoint);
|
||||
await withDatabaseOperation("http.request", async () => {
|
||||
const query = new OwnerQuery();
|
||||
await expect(withRunExecutionTiming("private-run", () => query)).resolves.toBe("task.execution_attempt");
|
||||
await expect(withRunWorkspaceFinalizationTiming("private-run", () => query)).resolves.toBe("task.workspace_finalization");
|
||||
await expect(withRunResponsePresentationTiming("private-run", () => query)).resolves.toBe("task.response_presentation");
|
||||
await expect(createRunPreparationTiming("private-run")("agent_read", () => query)).resolves.toBe("heartbeat.prepare.agent_read");
|
||||
await expect(createRunPreparationTiming("private-run", "native.controller")("backend_setup", () => query)).resolves.toBe("native.controller.backend_setup");
|
||||
await expect(measureRunPhase("skills.inventory", () => query)).resolves.toBe("skills.inventory");
|
||||
await expect(withBudgetSpan("budget.accounting.transaction", () => query)).resolves.toBe("budget.accounting.transaction");
|
||||
await expect(query).resolves.toBe("http.request");
|
||||
});
|
||||
expect(owners.getStore()).toBeUndefined();
|
||||
});
|
||||
|
||||
it("keeps owner attribution after the span detail limit and restores nested scopes after failure", async () => {
|
||||
vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.test");
|
||||
const failure = new Error("private accounting failure");
|
||||
await withRunExecutionTiming("private-run", async () => {
|
||||
for (let i = 0; i < 18; i++) {
|
||||
await expect(measureRunPhase("skills.inventory", () => new OwnerQuery())).resolves.toBe("skills.inventory");
|
||||
}
|
||||
await expect(measureRunPhase("execution.budget_reservation", async () => {
|
||||
await expect(withBudgetSpan("budget.admission.reserve", async () => {
|
||||
await Promise.resolve();
|
||||
expect(owners.getStore()).toBe("budget.admission.reserve");
|
||||
throw failure;
|
||||
})).rejects.toBe(failure);
|
||||
return new OwnerQuery();
|
||||
})).resolves.toBe("execution.budget_reservation");
|
||||
await expect(new OwnerQuery()).resolves.toBe("task.execution_attempt");
|
||||
});
|
||||
expect(owners.getStore()).toBeUndefined();
|
||||
});
|
||||
|
||||
it("keeps concurrent HTTP requests and nested work in fixed scopes without using private URLs", async () => {
|
||||
vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "");
|
||||
const app = express();
|
||||
app.use(databaseOperationMiddleware);
|
||||
app.use(express.json());
|
||||
let arrived = 0;
|
||||
let release!: () => void;
|
||||
const both = new Promise<void>(resolve => { release = resolve; });
|
||||
app.all("/:privateId", async (req, res) => {
|
||||
const before = await new OwnerQuery();
|
||||
const work = async () => {
|
||||
if (++arrived === 2) release();
|
||||
await both;
|
||||
return new OwnerQuery();
|
||||
};
|
||||
const during = req.params.privateId === "first"
|
||||
? await withBudgetSpan("budget.admission.reserve", work)
|
||||
: await measureRunPhase("skills.inventory", work);
|
||||
res.json({ before, during, after: await new OwnerQuery() });
|
||||
});
|
||||
const [first, second] = await Promise.all([
|
||||
request(app).get("/first?secret=private-first").expect(200),
|
||||
request(app).post("/second?secret=private-second").send({ private: "request-body" }).expect(200),
|
||||
]);
|
||||
expect(first.body).toEqual({ before: "http.request", during: "budget.admission.reserve", after: "http.request" });
|
||||
expect(second.body).toEqual({ before: "http.request", during: "skills.inventory", after: "http.request" });
|
||||
expect(owners.getStore()).toBeUndefined();
|
||||
});
|
||||
|
||||
it("replaces producer ownership on each delivery dispatch, including retry and pending checks", async () => {
|
||||
vi.useFakeTimers();
|
||||
const execute = vi.fn(async () => [{ xid: "42" }]);
|
||||
const database = installDatabaseWorkSignals({ execute, async transaction<T>(action: (tx: object) => Promise<T>): Promise<T> {
|
||||
return action({ execute });
|
||||
} });
|
||||
const seen: Array<{ queue: string; phase: string; owner: string | undefined }> = [];
|
||||
const coordinator = createDeliveryWorkCoordinator({ owner: database, canRun: () => true,
|
||||
onError: (_error, queue) => { seen.push({ queue, phase: "failure", owner: owners.getStore() }); },
|
||||
});
|
||||
let fail = true;
|
||||
try {
|
||||
const workers = withDatabaseOperation("http.request", () => [DELIVERY_QUEUES.feedback, DELIVERY_QUEUES.email].map(queue =>
|
||||
coordinator.register(queue, { retryMs: 5000,
|
||||
run: async () => {
|
||||
await Promise.resolve();
|
||||
seen.push({ queue, phase: "run", owner: await new OwnerQuery() });
|
||||
if (queue === DELIVERY_QUEUES.feedback && fail) { fail = false; throw new Error("retry"); }
|
||||
},
|
||||
hasPending: async () => {
|
||||
seen.push({ queue, phase: "pending", owner: await new OwnerQuery() });
|
||||
return false;
|
||||
},
|
||||
})));
|
||||
await Promise.all(workers.map(worker => worker.ready));
|
||||
await withDatabaseOperation("budget.accounting.transaction", () => database.transaction(tx =>
|
||||
notifyDeliveryWork(tx, DELIVERY_QUEUES.feedback)));
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
expect(seen.map(({ phase }) => phase).sort()).toEqual(["failure", "pending", "pending", "run", "run", "run"]);
|
||||
for (const entry of seen) expect(entry.owner).toBe(`delivery.${entry.queue}`);
|
||||
expect(owners.getStore()).toBeUndefined();
|
||||
expect(coordinator.nextWakeAt()).toBeNull();
|
||||
} finally {
|
||||
await coordinator.stop();
|
||||
}
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
import { readIdleSleepSafety } from "../services/idle-sleep-safety.js";
|
||||
import { createOwnedSandbox } from "../services/sandbox-creation-operations.js";
|
||||
import { createOwnedSandbox, recordSandboxCreationResource } from "../services/sandbox-creation-operations.js";
|
||||
import { mkdtemp, writeFile, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
@@ -116,15 +116,18 @@ describe.skipIf(!support.supported)("environment cleanup scheduling", () => {
|
||||
]));
|
||||
await quiet();
|
||||
});
|
||||
it("settles a deleted-company creation as manual history across restart without idle polling", async () => {
|
||||
it.each([false, true])("settles a deleted-company creation as manual history across restart without idle polling (contextual=%s)", async contextual => {
|
||||
await start(); await quiet();
|
||||
await createOwnedSandbox(db, { companyId, environmentId, provider: "fixture", driverKey: "fixture", config: {}, metadata: {} },
|
||||
const created = await createOwnedSandbox(db, { companyId, environmentId, provider: "fixture", driverKey: "fixture", config: {},
|
||||
metadata: contextual ? { runnerTaskLease: true, runnerTaskCreationContextId: randomUUID() } : {} },
|
||||
async () => ({ providerLeaseId: "fixture-deleted-company" }));
|
||||
await db.delete(companies).where(eq(companies.id, companyId));
|
||||
await settle(31 * 60_000);
|
||||
await vi.waitFor(async () => expect(await db.select().from(sandboxCreationOperations)).toEqual([
|
||||
expect.objectContaining({ state: "needs_reconciliation", metadata: expect.objectContaining({ creationReconciliationReason: "company_deleted" }) }),
|
||||
]));
|
||||
if (contextual) await recordSandboxCreationResource(db, { id: String(created.metadata!.creationOperationId), companyId, environmentId,
|
||||
providerLeaseId: created.providerLeaseId });
|
||||
await quiet();
|
||||
worker!.stop(); await coordinator!.stop();
|
||||
await start(); await quiet();
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { agents, companies, createDb, environmentLeases, environments, heartbeatRuns, plugins } from "@paperclipai/db";
|
||||
import { agents, companies, createDb, environmentLeases, environments, heartbeatRuns, plugins, sandboxCreationOperations } from "@paperclipai/db";
|
||||
import { resolvePluginTaskResume } from "../services/environment-task-runtime.js";
|
||||
import { remoteTerminationReceipt } from "../services/remote-execution-termination.js";
|
||||
import { environmentRuntimeService } from "../services/environment-runtime.js";
|
||||
import type { PluginWorkerManager } from "../services/plugin-worker-manager.js";
|
||||
import { NativeExecutionContext } from "../services/native-runtime/native-execution-context.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
@@ -20,6 +21,108 @@ databaseSuite("environment task lease authority", () => {
|
||||
});
|
||||
afterAll(async () => { await stopDb?.(); });
|
||||
|
||||
it("replaces a stopped warm candidate within its tracked startup context", async () => {
|
||||
const companyId = randomUUID(), agentId = randomUUID(), environmentId = randomUUID(), pluginId = randomUUID(), runId = randomUUID(), leaseId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Warm candidate", issuePrefix: `T${companyId.slice(0, 7).replace(/-/g, "").toUpperCase()}` });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Fixture", adapterType: "paperclip_runner" });
|
||||
await db.insert(plugins).values({ id: pluginId, pluginKey: pluginId, packageName: "fixture-warm-replacement", version: "1", status: "ready",
|
||||
manifestJson: { capabilities: ["environment.drivers.register"], environmentDrivers: [{ driverKey: "tasks", supportsTasks: true }] } as never });
|
||||
await db.insert(environments).values({ id: environmentId, companyId, name: `Fixture ${environmentId}`, driver: "plugin",
|
||||
config: { pluginKey: pluginId, driverKey: "tasks", driverConfig: {} } });
|
||||
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "manual", status: "running" });
|
||||
await db.insert(environmentLeases).values({ id: leaseId, companyId, environmentId, heartbeatRunId: runId,
|
||||
provider: "plugin:fixture:tasks", providerLeaseId: "stopped-task", status: "active", leasePolicy: "reuse_by_environment",
|
||||
metadata: { driver: "plugin", pluginId, pluginKey: pluginId, driverKey: "tasks", runnerTaskLease: true, runnerTaskReused: true } });
|
||||
const calls: string[] = [];
|
||||
const workers = { getWorker: () => ({ supportedMethods: ["environmentTask", "environmentAcquireLease"] }),
|
||||
call: async (_plugin: string, method: string, params: any, _timeout?: number, _sink?: unknown, settled?: () => void) => {
|
||||
calls.push(method === "environmentTask" ? params.operation.kind : method);
|
||||
try {
|
||||
if (method === "environmentAcquireLease") return { providerLeaseId: "replacement-task" };
|
||||
if (params.operation.kind === "status") return { kind: "status", taskId: params.taskId, phase: "completed", executionStopped: true };
|
||||
if (params.operation.kind === "describe") return { kind: "description", taskId: params.taskId,
|
||||
runner: { version: "fixture", digest: `sha256:${"a".repeat(64)}` }, executionContextVersion: 1 };
|
||||
return { kind: "accepted", taskId: params.taskId };
|
||||
} finally { settled?.(); }
|
||||
},
|
||||
} as unknown as PluginWorkerManager;
|
||||
const context = new NativeExecutionContext({ binding: { companyId, agentId, runId } } as never, 1_000);
|
||||
try {
|
||||
await expect(environmentRuntimeService(db, { pluginWorkerManager: workers }).task({ companyId, leaseId,
|
||||
operation: { kind: "describe" }, executionContext: context })).resolves.toMatchObject({ kind: "description", taskId: "replacement-task" });
|
||||
expect(calls).toEqual(["status", "status", "complete", "environmentAcquireLease", "describe"]);
|
||||
expect(context.pending).toBe(false);
|
||||
const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, leaseId));
|
||||
expect(lease.providerLeaseId).toBe("replacement-task");
|
||||
expect(lease.metadata?.runnerTaskReused).toBe(false);
|
||||
expect(JSON.stringify(lease.metadata)).not.toContain(context.id);
|
||||
} finally { context.finish(); }
|
||||
});
|
||||
|
||||
it.each(["late", "post-result", "retiring"])("queues a replacement instead of attaching after %s startup cancellation", async mode => {
|
||||
const companyId = randomUUID(), agentId = randomUUID(), environmentId = randomUUID(), pluginId = randomUUID(), runId = randomUUID(), leaseId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Warm candidate", issuePrefix: `T${companyId.slice(0, 7).replace(/-/g, "").toUpperCase()}` });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Fixture", adapterType: "paperclip_runner" });
|
||||
await db.insert(plugins).values({ id: pluginId, pluginKey: pluginId, packageName: "fixture-warm-replacement", version: "1", status: "ready",
|
||||
manifestJson: { capabilities: ["environment.drivers.register"], environmentDrivers: [{ driverKey: "tasks", supportsTasks: true }] } as never });
|
||||
await db.insert(environments).values({ id: environmentId, companyId, name: `Fixture ${environmentId}`, driver: "plugin",
|
||||
config: { pluginKey: pluginId, driverKey: "tasks", driverConfig: {} } });
|
||||
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "manual", status: "running" });
|
||||
await db.insert(environmentLeases).values({ id: leaseId, companyId, environmentId, heartbeatRunId: runId,
|
||||
provider: "plugin:fixture:tasks", providerLeaseId: "stopped-task", status: "active", leasePolicy: "reuse_by_environment",
|
||||
metadata: { driver: "plugin", pluginId, pluginKey: pluginId, driverKey: "tasks", runnerTaskLease: true, runnerTaskReused: true } });
|
||||
const calls: string[] = [];
|
||||
let reply!: () => Promise<void>, acquiring!: () => void;
|
||||
const entered = new Promise<void>(resolve => { acquiring = resolve; });
|
||||
const workers = { getWorker: () => ({ supportedMethods: ["environmentTask", "environmentAcquireLease"] }),
|
||||
call: async (_plugin: string, method: string, params: any, _timeout?: number, _sink?: unknown, settled?: () => void, _contextId?: string, observe?: (result: any) => Promise<void>) => {
|
||||
calls.push(method === "environmentTask" ? params.operation.kind : method);
|
||||
try {
|
||||
if (method === "environmentAcquireLease") return await new Promise(resolve => {
|
||||
reply = async () => { const result = { providerLeaseId: "replacement-task", metadata: { binding: "fresh" } };
|
||||
await observe?.(result);
|
||||
if (mode === "retiring") await db.update(environmentLeases).set({ metadata: { driver: "plugin", pluginId, driverKey: "tasks", runnerTaskRetiring: true } }).where(eq(environmentLeases.id, leaseId));
|
||||
resolve(result); };
|
||||
acquiring();
|
||||
});
|
||||
if (params.operation.kind === "status") return { kind: "status", taskId: params.taskId, phase: "completed", executionStopped: true };
|
||||
if (params.operation.kind === "describe") return { kind: "description", taskId: params.taskId,
|
||||
runner: { version: "fixture", digest: `sha256:${"a".repeat(64)}` }, executionContextVersion: 1 };
|
||||
return { kind: "accepted", taskId: params.taskId };
|
||||
} finally { settled?.(); }
|
||||
},
|
||||
} as unknown as PluginWorkerManager;
|
||||
const context = new NativeExecutionContext({ binding: { companyId, agentId, runId } } as never, 1_000);
|
||||
if (mode === "post-result") {
|
||||
const wait = context.wait.bind(context);
|
||||
vi.spyOn(context, "wait").mockImplementation(operation => wait(async () => {
|
||||
const value = await operation(); context.abort(new Error("Cancelled after recording")); return value;
|
||||
}));
|
||||
}
|
||||
try {
|
||||
const request = environmentRuntimeService(db, { pluginWorkerManager: workers }).task({ companyId, leaseId,
|
||||
operation: { kind: "describe" }, executionContext: context });
|
||||
const rejected = expect(request).rejects.toThrow("operation unavailable");
|
||||
await entered;
|
||||
if (mode === "late") {
|
||||
context.abort(new Error("Startup cancelled"));
|
||||
await rejected;
|
||||
expect(context.pending).toBe(true);
|
||||
}
|
||||
await reply(); await rejected;
|
||||
expect(context.pending).toBe(false);
|
||||
const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, leaseId));
|
||||
expect(lease.providerLeaseId).toBe("stopped-task");
|
||||
const [creation] = await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.runId, runId));
|
||||
expect(creation.state).toBe("cleanup_pending");
|
||||
expect(creation.metadata.runnerTaskCreationContextId).toBe(context.id);
|
||||
const [cleanup] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, creation.leaseId!));
|
||||
expect(cleanup).toMatchObject({ heartbeatRunId: runId, providerLeaseId: "replacement-task", status: "pending_cleanup" });
|
||||
expect(cleanup.metadata).toMatchObject({ runnerTaskLease: true, providerMetadata: { binding: "fresh" } });
|
||||
expect(calls).not.toContain("describe");
|
||||
} finally { context.finish(); }
|
||||
});
|
||||
|
||||
it("authorizes a new task lease only after scoped prior execution has stopped", async () => {
|
||||
const companyId = randomUUID(), agentId = randomUUID(), environmentId = randomUUID(), pluginId = randomUUID();
|
||||
const oldRunId = randomUUID(), nextRunId = randomUUID(), oldLeaseId = randomUUID(), nextLeaseId = randomUUID();
|
||||
|
||||
@@ -2,14 +2,16 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { PgDialect } from "drizzle-orm/pg-core";
|
||||
import { executionWorkspaces, environmentLeases, type Db } from "@paperclipai/db";
|
||||
import { environmentTaskOperationSchema, parseEnvironmentTaskResult } from "@paperclipai/plugin-sdk";
|
||||
import { executeEnvironmentTask } from "../services/environment-task-runtime.js";
|
||||
import { executeEnvironmentTask, settleDispatchedEnvironmentTask } from "../services/environment-task-runtime.js";
|
||||
import { NativeExecutionContext } from "../services/native-runtime/native-execution-context.js";
|
||||
|
||||
const state = vi.hoisted(() => ({ plugin: {} as any, release: vi.fn() }));
|
||||
const state = vi.hoisted(() => ({ plugin: {} as any, release: vi.fn(), identity: vi.fn(), config: vi.fn() }));
|
||||
vi.mock("../services/provider-fresh-workspace.js", () => ({
|
||||
stageProviderFreshWorkspace: async (input: { existing: unknown }) => input.existing,
|
||||
releaseProviderRecoveryTransfers: state.release,
|
||||
}));
|
||||
vi.mock("../services/plugin-registry.js", () => ({ pluginRegistryService: () => ({ getById: vi.fn(async () => state.plugin) }) }));
|
||||
vi.mock("../services/plugin-registry.js", () => ({ pluginRegistryService: () => ({ getById: vi.fn(async () => state.plugin), getConfig: state.config }) }));
|
||||
vi.mock("../services/plugin-agent-runtime-state.js", () => ({ pluginAgentRuntimeIdentity: state.identity }));
|
||||
const leaseId = "10000000-0000-4000-8000-000000000001";
|
||||
const projectId = "10000000-0000-4000-8000-000000000002";
|
||||
const secondProjectId = "10000000-0000-4000-8000-000000000003";
|
||||
@@ -25,7 +27,7 @@ function database(value: ReturnType<typeof row> | null = row(), projectRows = [{
|
||||
const query: any = { from: (table: unknown) => { selectedRows = table === executionWorkspaces ? workspaceRows : table === environmentLeases ? ownerRows : projectRows; return query; }, leftJoin: () => query, where: vi.fn(() => query), limit: () => Promise.resolve(results.shift()), then: (resolve: (rows: typeof projectRows) => unknown) => Promise.resolve(selectedRows).then(resolve as (rows: unknown[]) => unknown) };
|
||||
const sets: Record<string, unknown>[] = [];
|
||||
const update: any = { set: (value: Record<string, unknown>) => { sets.push(value); return update; }, where: () => update,
|
||||
returning: async () => [{ id: leaseId }], then: (resolve: (value: unknown) => unknown) => Promise.resolve([]).then(resolve) };
|
||||
returning: async () => [{ id: leaseId, metadata: value?.lease.metadata, heartbeatRunId: value?.lease.heartbeatRunId }], then: (resolve: (value: unknown) => unknown) => Promise.resolve([]).then(resolve) };
|
||||
const db = { select: () => query, update: () => update,
|
||||
transaction: vi.fn(async (write: (tx: Db) => Promise<unknown>) => write(db as unknown as Db)) };
|
||||
return { db: db as unknown as Db, query, sets, transaction: db.transaction };
|
||||
@@ -37,9 +39,65 @@ const submit = { kind: "submit" as const, preparedLaunchId: "configuration", pro
|
||||
|
||||
beforeEach(() => {
|
||||
state.release.mockReset();
|
||||
state.identity.mockReset().mockResolvedValue({ instanceSecrets: { launcher: "synthetic-transient-binding" } });
|
||||
state.config.mockReset().mockResolvedValue({ configJson: { endpoint: "synthetic" } });
|
||||
state.plugin = { id: "original", pluginKey: "test.provider", status: "ready", manifestJson: { capabilities: ["environment.drivers.register"], environmentDrivers: [{ driverKey: "tasks", supportsTasks: true }] } };
|
||||
});
|
||||
describe("plugin-provided Runner execution", () => {
|
||||
it.each([false, true])("retires an undispatched warm task only with required preparation proof (%s)", async preparationStopped => {
|
||||
const value = row();
|
||||
Object.assign(value.lease.metadata, { runnerTaskDispatched: false,
|
||||
runnerTaskStartupCleanup: { contextId: "startup", requiresPreparationStop: true } });
|
||||
const fixture = database(value), workers = worker();
|
||||
fixture.db.select = ((...args: unknown[]) => (database(value).db.select as (...values: unknown[]) => unknown)(...args)) as Db["select"];
|
||||
let release!: () => void;
|
||||
const drain = new Promise<void>(resolve => { release = resolve; });
|
||||
const drainStartup = vi.fn(() => drain);
|
||||
Object.assign(workers, { drainStartup });
|
||||
const operations: string[] = [];
|
||||
workers.call.mockImplementation(async (...args: any[]) => {
|
||||
const operation = args[2].operation.kind; operations.push(operation);
|
||||
return operation === "status" ? { kind: "status", taskId: "attempt-1", phase: "cancelled", executionStopped: true, preparationStopped }
|
||||
: { kind: "accepted", taskId: "attempt-1" };
|
||||
});
|
||||
const retiring = settleDispatchedEnvironmentTask(fixture.db, workers as never, { companyId: "company", id: leaseId });
|
||||
const checked = retiring.then(value => ({ value, error: null }), error => ({ value: false, error }));
|
||||
await vi.waitFor(() => expect(operations).toContain("stop"));
|
||||
await vi.waitFor(() => expect(operations).toContain("status"));
|
||||
expect(drainStartup).toHaveBeenCalledWith("startup");
|
||||
expect(operations).not.toContain("complete");
|
||||
release();
|
||||
const result = await checked;
|
||||
if (preparationStopped) { expect(result).toEqual({ value: true, error: null }); expect(operations).toEqual(["stop", "status", "complete"]); }
|
||||
else { expect(result.error?.message).toContain("termination is unconfirmed"); expect(fixture.sets).toContainEqual(expect.objectContaining({ status: "pending_cleanup", cleanupStatus: "failed" })); }
|
||||
});
|
||||
it("keeps legacy payloads passive without capability or transport activation", async () => {
|
||||
const context = new NativeExecutionContext({ binding: { companyId: "company", agentId: "agent", runId: "run" } } as never, 1_000);
|
||||
const workers = worker();
|
||||
workers.call.mockImplementation(async (...args: any[]) => { args[5]?.(); return { kind: "description", taskId: "attempt-1", runner: { version: "fixture", digest: `sha256:${"a".repeat(64)}` } }; });
|
||||
try {
|
||||
for (const executionContext of [undefined, context]) {
|
||||
await executeEnvironmentTask(database().db, workers as never, { companyId: "company", leaseId,
|
||||
operation: { kind: "describe" }, ...(executionContext ? { executionContext } : {}) });
|
||||
const sent = workers.call.mock.calls.at(-1) as unknown as any[];
|
||||
expect(sent[2]).not.toHaveProperty("executionContextSupported");
|
||||
expect(sent[2]).not.toHaveProperty("executionContext");
|
||||
if (!executionContext) expect(sent).toHaveLength(4);
|
||||
}
|
||||
expect(state.identity).not.toHaveBeenCalled();
|
||||
expect(state.config).not.toHaveBeenCalled();
|
||||
expect(context.pending).toBe(false);
|
||||
} finally { context.finish(); }
|
||||
});
|
||||
it.each(["json", "wrong-run"])("rejects a %s context before the worker", async mode => {
|
||||
const context = new NativeExecutionContext({ binding: { companyId: "company", agentId: "agent", runId: mode === "wrong-run" ? "other" : "run" } } as never, 1_000);
|
||||
const workers = worker();
|
||||
try {
|
||||
await expect(executeEnvironmentTask(database().db, workers as never, { companyId: "company", leaseId, operation: { kind: "describe" },
|
||||
executionContext: mode === "json" ? JSON.parse(JSON.stringify(context)) : context })).rejects.toThrow("native_execution_context");
|
||||
expect(workers.call).not.toHaveBeenCalled();
|
||||
} finally { context.finish(); }
|
||||
});
|
||||
it("forwards a retained fresh directory intent on later launch calls and checks provider support", async () => {
|
||||
const value = row();
|
||||
value.lease.metadata.runnerTaskProjectAuthValidation.projectIds = [];
|
||||
|
||||
@@ -8,7 +8,7 @@ import { createRuntimeToolsToken } from "../runtime-tools-token.js";
|
||||
import { errorHandler } from "../middleware/index.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
agents,
|
||||
@@ -87,7 +87,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
|
||||
});
|
||||
await db.insert(toolConnections).values(irrelevant);
|
||||
const observed = createDb(external ?? database!.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(observed, { toolConnections }, capture);
|
||||
try {
|
||||
expect(await resolveManagedGitHubIdentitySelection(observed, input.companyId, { agentId: input.agentId, responsibleUserId: "A" }))
|
||||
@@ -119,7 +119,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
|
||||
}).where(eq(heartbeatRuns.id, input.runId));
|
||||
const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId));
|
||||
const observed = createDb(external ?? database!.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(observed, { heartbeatRuns }, capture);
|
||||
try {
|
||||
expect(await resolveGitHubOperationCredentials(observed, input)).toMatchObject({
|
||||
|
||||
@@ -689,14 +689,9 @@ describeEmbeddedPostgres("heartbeat dependency-aware queued run selection", () =
|
||||
});
|
||||
expect(interactionWake).not.toBeNull();
|
||||
|
||||
await waitForCondition(async () => {
|
||||
const run = await db
|
||||
.select({ status: heartbeatRuns.status })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, interactionWake!.id))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return run?.status === "succeeded";
|
||||
});
|
||||
// This checks dependency interaction semantics, not provider startup speed.
|
||||
// Join the admitted execution before inspecting its persisted outcome.
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
|
||||
const interactionRun = await db
|
||||
.select({
|
||||
|
||||
@@ -1,10 +1,15 @@
|
||||
import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { agents, authUsers, companies, companyMemberships, createDb, heartbeatRuns, issues } from "@paperclipai/db";
|
||||
import { agents, authUsers, companies, companyMemberships, createDb, environmentLeases, environments, heartbeatRuns, issues, nativeRunFinalizations } from "@paperclipai/db";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { heartbeatService } from "../services/heartbeat.js";
|
||||
import { NativeStartupCleanupPendingError, readNativeStartupCleanup } from "../services/native-runtime/native-startup-cleanup.js";
|
||||
import { prepareNativeHeartbeatRun } from "../services/native-runtime/prepare-native-run.js";
|
||||
import { environmentService } from "../services/environments.js";
|
||||
import { remoteTerminationReceipt } from "../services/remote-execution-termination.js";
|
||||
import { createIsolatedTestHome } from "./helpers/isolated-test-home.js";
|
||||
|
||||
const execute = vi.hoisted(() => vi.fn());
|
||||
vi.mock("../adapters/index.js", async () => {
|
||||
@@ -70,4 +75,108 @@ suite("native terminal-run cleanup admission", () => {
|
||||
expect((await heartbeat.getRun(nextId))?.status).toBe("succeeded");
|
||||
expect(execute).toHaveBeenCalledTimes(3);
|
||||
}, 30_000);
|
||||
|
||||
it("retries a failed startup intent through timer failure and drain before verified cancellation", async () => {
|
||||
const home = createIsolatedTestHome(), originalHome = process.env.PAPERCLIP_HOME;
|
||||
process.env.PAPERCLIP_HOME = home.home;
|
||||
const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), leaseId = randomUUID();
|
||||
const contextId = randomUUID(), leaseOwner = randomUUID(), taskId = randomUUID(), deadlineAtMs = Date.now() - 1;
|
||||
const originalSetTimeout = globalThis.setTimeout;
|
||||
const retries: Array<{ timer: ReturnType<typeof setTimeout>; fire(): void }> = [];
|
||||
const timeout = vi.spyOn(globalThis, "setTimeout").mockImplementation(((callback: (...args: unknown[]) => void, delay?: number, ...args: unknown[]) => {
|
||||
if (typeof delay === "number" && delay <= 30_000 && new Error().stack?.includes("scheduleNativeSessionResumeDispatch")) {
|
||||
const timer = originalSetTimeout(() => {}, 3_600_000);
|
||||
retries.push({ timer, fire() { clearTimeout(timer); callback(...args); } });
|
||||
return timer;
|
||||
}
|
||||
return originalSetTimeout(callback, delay, ...args);
|
||||
}) as typeof setTimeout);
|
||||
let failSuppression = false, runId = "";
|
||||
const runtimeEnv = new Proxy<Record<string, string | undefined>>({}, { get(_target, key) {
|
||||
if (key === "PAPERCLIP_DATABASE_RESTORE_IN_PROGRESS" && failSuppression) {
|
||||
failSuppression = false;
|
||||
throw new Error("Scheduling dependency unavailable");
|
||||
}
|
||||
return undefined;
|
||||
} });
|
||||
const heartbeat = heartbeatService(db, { runtimeEnv });
|
||||
try {
|
||||
await db.insert(companies).values({ id: companyId, name: "Intent retry", issuePrefix: "CIR", defaultResponsibleUserId: "owner" });
|
||||
await db.insert(authUsers).values({ id: "owner", name: "Fixture user", email: "owner@example.test", createdAt: new Date(), updatedAt: new Date() }).onConflictDoNothing();
|
||||
await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: "owner", status: "active", membershipRole: "member" });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Worker", role: "engineer", status: "idle", adapterType: "process",
|
||||
adapterConfig: {}, permissions: {}, runtimeConfig: { heartbeat: { enabled: false, wakeOnDemand: true } } });
|
||||
await db.insert(issues).values({ id: issueId, companyId, title: "Recover startup", status: "todo", assigneeAgentId: agentId, responsibleUserId: "owner" });
|
||||
const [environment] = await db.insert(environments).values({ companyId, name: "Controlled provider", driver: "plugin", config: {} }).returning();
|
||||
await db.execute(sql`create table test_cleanup_intent_fault (enabled boolean not null)`);
|
||||
await db.execute(sql`insert into test_cleanup_intent_fault values (true)`);
|
||||
await db.execute(sql`create sequence test_cleanup_intent_attempts`);
|
||||
await db.execute(sql.raw(`create function test_reject_cleanup_intent() returns trigger language plpgsql as $$ begin
|
||||
if new.company_id = '${companyId}'::uuid and new.error_code = 'native_startup_cleanup_pending'
|
||||
and exists (select 1 from test_cleanup_intent_fault where enabled) then
|
||||
perform nextval('test_cleanup_intent_attempts'); raise exception 'Cleanup intent write unavailable';
|
||||
end if; return new; end $$`));
|
||||
await db.execute(sql`create trigger test_cleanup_intent_fault before update on heartbeat_runs for each row execute function test_reject_cleanup_intent()`);
|
||||
execute.mockReset();
|
||||
execute.mockImplementation(async (context: { runId: string }) => {
|
||||
runId = context.runId;
|
||||
const [run] = await db.update(heartbeatRuns).set({ runtimeMode: "native", runtimeModeResolvedAt: null }).where(eq(heartbeatRuns.id, runId)).returning();
|
||||
await prepareNativeHeartbeatRun({ db, run, issue: { id: issueId, title: "Recover startup", description: null }, environmentLeaseId: leaseId });
|
||||
await db.update(agents).set({ adapterType: "paperclip_runner", adapterConfig: { provider: "codex" } }).where(eq(agents.id, agentId));
|
||||
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId));
|
||||
await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId, phase: "observed", attempt: 1,
|
||||
controllerGeneration: 1, leaseOwner, leaseExpiresAt: new Date(Date.now() - 1_000) });
|
||||
await db.insert(environmentLeases).values({ id: leaseId, companyId, environmentId: environment.id, heartbeatRunId: runId,
|
||||
issueId, status: "active", provider: "plugin", providerLeaseId: taskId, leasePolicy: "reuse_by_environment", metadata: { runnerTaskLease: true } });
|
||||
await db.update(heartbeatRuns).set({ runnerProfileJson: sql`coalesce(${heartbeatRuns.runnerProfileJson}, '{}'::jsonb) || ${JSON.stringify({
|
||||
nativeStartupContext: { schema: "paperclip.native-startup-context.v1", companyId, agentId, runId,
|
||||
contextId, deadlineAtMs, leaseId, leaseOwner, controllerGeneration: 1, attempt: 1 },
|
||||
})}::jsonb` }).where(eq(heartbeatRuns.id, runId));
|
||||
throw new NativeStartupCleanupPendingError(contextId, deadlineAtMs, leaseOwner, false, leaseId);
|
||||
});
|
||||
await heartbeat.invoke(agentId, "on_demand", { issueId }, "manual");
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
const owner = () => db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)).then(rows => rows[0]!);
|
||||
const lease = () => db.select().from(environmentLeases).where(eq(environmentLeases.id, leaseId)).then(rows => rows[0]!);
|
||||
expect((await db.execute<{ last_value: string }>(sql`select last_value from test_cleanup_intent_attempts`))[0].last_value).toBe("2");
|
||||
expect(await heartbeat.getRun(runId)).toMatchObject({ status: "running", errorCode: null });
|
||||
expect((await owner()).leaseOwner).toBe(leaseOwner);
|
||||
expect(await lease()).toMatchObject({ status: "active", cleanupStatus: null });
|
||||
expect(readNativeStartupCleanup((await heartbeat.getRun(runId))!)).toBeNull();
|
||||
const drainedRetries = retries.length;
|
||||
expect(drainedRetries).toBeGreaterThan(0);
|
||||
await heartbeat.resumeQueuedRunsAfterDrain();
|
||||
expect(retries.length).toBeGreaterThan(drainedRetries);
|
||||
const resumedRetries = retries.length;
|
||||
failSuppression = true;
|
||||
retries.at(-1)!.fire();
|
||||
await vi.waitFor(() => expect(retries.length).toBeGreaterThan(resumedRetries));
|
||||
expect((await owner()).leaseOwner).toBe(leaseOwner);
|
||||
expect((await heartbeat.getRun(runId))?.status).toBe("running");
|
||||
await db.execute(sql`update test_cleanup_intent_fault set enabled = false`);
|
||||
retries.at(-1)!.fire();
|
||||
await vi.waitFor(async () => expect(readNativeStartupCleanup((await heartbeat.getRun(runId))!)?.state).toBe("pending"));
|
||||
await vi.waitFor(async () => expect((await heartbeat.listEvents(runId)).some(event => event.eventType === "native.recovery.transition")).toBe(true));
|
||||
const current = await lease();
|
||||
await environmentService(db).releaseLease(leaseId, "released", { cleanupStatus: "success",
|
||||
remoteExecutionTermination: remoteTerminationReceipt(current, { providerLeaseId: taskId, state: "stopped" }) });
|
||||
await heartbeat.cancelRun(runId, "Startup cleanup retry completed");
|
||||
expect(await heartbeat.getRun(runId)).toMatchObject({ status: "cancelled" });
|
||||
expect(readNativeStartupCleanup((await heartbeat.getRun(runId))!)?.state).toBe("complete");
|
||||
expect(await owner()).toMatchObject({ leaseOwner: null, controlDeadlineAt: null });
|
||||
expect(execute).toHaveBeenCalledOnce();
|
||||
} finally {
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
timeout.mockRestore();
|
||||
for (const retry of retries) clearTimeout(retry.timer);
|
||||
await db.execute(sql`drop trigger if exists test_cleanup_intent_fault on heartbeat_runs`);
|
||||
await db.execute(sql`drop function if exists test_reject_cleanup_intent()`);
|
||||
await db.execute(sql`drop table if exists test_cleanup_intent_fault`);
|
||||
await db.execute(sql`drop sequence if exists test_cleanup_intent_attempts`);
|
||||
if (originalHome === undefined) delete process.env.PAPERCLIP_HOME;
|
||||
else process.env.PAPERCLIP_HOME = originalHome;
|
||||
home.cleanup();
|
||||
}
|
||||
}, 30_000);
|
||||
|
||||
});
|
||||
@@ -1,7 +1,7 @@
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
|
||||
import {
|
||||
agents,
|
||||
companies,
|
||||
@@ -63,11 +63,11 @@ describe("P6-11..13 / P6-17 canonical event allocator", () => {
|
||||
protocolSchemaVersion: 1, canonicalPayload: { sourceSeq } },
|
||||
});
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { heartbeatRunEvents }, capture);
|
||||
try { expect((await append(5002)).highestContiguousSourceSeq).toBe(5000); }
|
||||
finally { capture.flush(); }
|
||||
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
|
||||
const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
|
||||
&& event.table === "heartbeat_run_events" && (event.operation === "select" || event.operation === "execute"));
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThan(0);
|
||||
expect(reads.reduce((sum, event) => sum + Number(event.rows), 0)).toBeLessThan(16);
|
||||
|
||||
@@ -41,3 +41,21 @@ describe("compactRunLogChunk", () => {
|
||||
expect(compacted).not.toContain("paperclip-flag-secret");
|
||||
});
|
||||
});
|
||||
|
||||
describe("compactRunLogChunk with long base64 commands", () => {
|
||||
it("redacts a tool call that writes base64 of source text without stalling", () => {
|
||||
const typescriptSource = "export const limit = 65536;\nexport function keep(value: string) {\n return value.slice(0, limit);\n}\n";
|
||||
const blob = Buffer.from(typescriptSource.repeat(1_000)).toString("base64").slice(0, 48 * 1024);
|
||||
expect(blob).toMatch(/^[A-Za-z0-9]+$/);
|
||||
const command = `printf '%s' '${blob}' | base64 -d > src/keep.ts && echo '{"GITHUB_TOKEN":"tool-call-secret"}'`;
|
||||
const chunk = `${JSON.stringify({ type: "assistant", message: { content: [{ type: "tool_use", input: { command } }] } })}\n`;
|
||||
|
||||
const startedAt = performance.now();
|
||||
const compacted = compactRunLogChunk(chunk);
|
||||
|
||||
expect(performance.now() - startedAt).toBeLessThan(1_000);
|
||||
expect(compacted).toContain(blob);
|
||||
expect(compacted).not.toContain("tool-call-secret");
|
||||
expect(compacted).toContain("***REDACTED***");
|
||||
});
|
||||
});
|
||||
@@ -1,7 +1,7 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { drizzle } from "drizzle-orm/postgres-js";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
@@ -268,7 +268,7 @@ describeEmbeddedPostgres("heartbeat run privacy routes", { concurrent: false },
|
||||
}).where(eq(heartbeatRuns.id, f.privateRunId));
|
||||
const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.privateRunId));
|
||||
const observed = createDb(externalDatabaseUrl ?? tempDb!.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(observed, { heartbeatRuns }, capture);
|
||||
try {
|
||||
const app = createApp(f.companyId, f.grantedAgentId, observed);
|
||||
|
||||
@@ -791,19 +791,7 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
|
||||
`));
|
||||
|
||||
await heartbeat.resumeQueuedRuns();
|
||||
await waitForCondition(async () => {
|
||||
const [run, wakeup] = await Promise.all([
|
||||
db.select({ status: heartbeatRuns.status })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, runId))
|
||||
.then((rows) => rows[0] ?? null),
|
||||
db.select({ status: agentWakeupRequests.status })
|
||||
.from(agentWakeupRequests)
|
||||
.where(eq(agentWakeupRequests.id, wakeupRequestId))
|
||||
.then((rows) => rows[0] ?? null),
|
||||
]);
|
||||
return run?.status === "cancelled" && wakeup?.status === "skipped";
|
||||
});
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
|
||||
const [run, wakeup, issue] = await Promise.all([
|
||||
db.select({ status: heartbeatRuns.status, errorCode: heartbeatRuns.errorCode })
|
||||
@@ -970,16 +958,10 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
|
||||
expect(ordering).toEqual(["validated", "parked"]);
|
||||
};
|
||||
await heartbeat.resumeQueuedRuns();
|
||||
await waitForCondition(async () => {
|
||||
const run = await db
|
||||
.select({ status: heartbeatRuns.status })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, runId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return run?.status === "cancelled";
|
||||
});
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
await parkPromise;
|
||||
|
||||
expect((await heartbeat.getRun(runId))?.status).toBe("cancelled");
|
||||
const issue = await db
|
||||
.select({ status: issues.status })
|
||||
.from(issues)
|
||||
@@ -1108,7 +1090,7 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
|
||||
});
|
||||
|
||||
expect(run).not.toBeNull();
|
||||
await waitForCondition(async () => countExecuteCallsForRun(run!.id) > 0);
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
expect(countExecuteCallsForRun(run!.id)).toBe(1);
|
||||
});
|
||||
|
||||
@@ -1126,7 +1108,7 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
|
||||
});
|
||||
|
||||
expect(run).not.toBeNull();
|
||||
await waitForCondition(async () => countExecuteCallsForRun(run!.id) > 0);
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
expect(countExecuteCallsForRun(run!.id)).toBe(1);
|
||||
});
|
||||
|
||||
@@ -1718,13 +1700,7 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
|
||||
.where(eq(agentWakeupRequests.id, wakeupRequestId));
|
||||
|
||||
await heartbeat.resumeQueuedRuns();
|
||||
await waitForCondition(async () => {
|
||||
const [deferred] = await db
|
||||
.select({ status: agentWakeupRequests.status, runId: agentWakeupRequests.runId })
|
||||
.from(agentWakeupRequests)
|
||||
.where(eq(agentWakeupRequests.id, deferredWakeupId));
|
||||
return Boolean(deferred?.runId) && deferred?.status !== "deferred_issue_execution";
|
||||
});
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
|
||||
const [deferred] = await db
|
||||
.select({ status: agentWakeupRequests.status, runId: agentWakeupRequests.runId })
|
||||
@@ -1788,15 +1764,7 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
|
||||
});
|
||||
|
||||
await heartbeat.resumeQueuedRuns();
|
||||
|
||||
await waitForCondition(async () => {
|
||||
const run = await db
|
||||
.select({ status: heartbeatRuns.status })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, runId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return run?.status === "cancelled";
|
||||
});
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
|
||||
const [run, wakeup, issue] = await Promise.all([
|
||||
db
|
||||
|
||||
@@ -1,35 +1,7 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import {
|
||||
applyPendingMigrations,
|
||||
createDb,
|
||||
getEmbeddedPostgresTestSupport as embeddedSupport,
|
||||
startEmbeddedPostgresTestDatabase as startEmbedded,
|
||||
export {
|
||||
getEmbeddedPostgresTestSupport,
|
||||
startEmbeddedPostgresTestDatabase,
|
||||
EMBEDDED_POSTGRES_TEST_TIMEOUT_MS,
|
||||
type EmbeddedPostgresTestDatabase,
|
||||
type EmbeddedPostgresTestSupport,
|
||||
} from "@paperclipai/db";
|
||||
|
||||
export type { EmbeddedPostgresTestDatabase, EmbeddedPostgresTestSupport };
|
||||
|
||||
export async function getEmbeddedPostgresTestSupport(): Promise<EmbeddedPostgresTestSupport> {
|
||||
return process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL ? { supported: true } : embeddedSupport();
|
||||
}
|
||||
|
||||
export async function startEmbeddedPostgresTestDatabase(prefix: string): Promise<EmbeddedPostgresTestDatabase> {
|
||||
const adminUrl = process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL;
|
||||
if (!adminUrl) return startEmbedded(prefix);
|
||||
const name = `paperclip_test_${randomUUID().replaceAll("-", "")}`;
|
||||
const template = process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE;
|
||||
if (template && !/^paperclip_test_template_[a-f0-9]+$/.test(template)) throw new Error("Invalid disposable test template name");
|
||||
const admin = createDb(adminUrl, { maxConnections: 1 });
|
||||
const url = new URL(adminUrl);
|
||||
url.pathname = `/${name}`;
|
||||
const cleanup = async () => {
|
||||
try { await admin.$client.unsafe(`DROP DATABASE IF EXISTS "${name}" WITH (FORCE)`); }
|
||||
finally { await admin.$client.end({ timeout: 1 }); }
|
||||
};
|
||||
try {
|
||||
await admin.$client.unsafe(`CREATE DATABASE "${name}"${template ? ` TEMPLATE "${template}"` : ""}`);
|
||||
if (!template) await applyPendingMigrations(url.toString());
|
||||
return { connectionString: url.toString(), cleanup };
|
||||
} catch (error) { await cleanup(); throw error; }
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
import { expect, it } from "vitest";
|
||||
import { createFullDatabaseResponseCapture } from "./full-database-response-capture.js";
|
||||
|
||||
const metadata = { operation: "select", table: "heartbeat_runs", callers: [] };
|
||||
|
||||
it("measures every oversized string and complex value without sampling or truncation", () => {
|
||||
const events: Array<Record<string, unknown>> = [];
|
||||
const capture = createFullDatabaseResponseCapture({ start: 0, end: 1000 }, event => events.push(event), () => 0);
|
||||
const text = "private-payload🙂".repeat(30_000);
|
||||
const nested = { instructions: text, additional: [{ prompt: text }] };
|
||||
const rows = Object.assign([[text, nested]], { columns: [{ name: "stdout_excerpt" }, { name: "context_snapshot" }] });
|
||||
const bytes = Buffer.byteLength(text) + Buffer.byteLength(JSON.stringify(nested));
|
||||
try {
|
||||
for (let index = 0; index < 17; index++) expect(capture.execute(() => rows, metadata)).toBe(rows);
|
||||
capture.flush();
|
||||
expect(events[0]).toMatchObject({ event: "paperclip.test_db_response_capture.entry.v1",
|
||||
calls: 17, rows: 17, responseValueBytes: 17 * bytes, maxResponseValueBytes: bytes,
|
||||
estimatedValues: 17, serializationErrors: 0,
|
||||
});
|
||||
expect(events[1]).toMatchObject({ event: "paperclip.test_db_response_capture.summary.v1",
|
||||
startedQueries: 17, completedQueries: 17, inFlightAtEnd: 0, overflowCalls: 0, responseValueBytes: 17 * bytes,
|
||||
});
|
||||
expect(JSON.stringify(events)).not.toContain("private-payload");
|
||||
expect(nested.instructions).toBe(text);
|
||||
} finally { capture.flush(); }
|
||||
});
|
||||
|
||||
it("preserves asynchronous results and original query errors", async () => {
|
||||
const events: Array<Record<string, unknown>> = [];
|
||||
const capture = createFullDatabaseResponseCapture({ start: 0, end: 1000 }, event => events.push(event), () => 0);
|
||||
const rows = Object.assign([{ value: { nested: "result" } }], { columns: [{ name: "value" }] });
|
||||
const failure = new Error("private query failure");
|
||||
try {
|
||||
await expect(capture.execute(async () => rows, metadata)).resolves.toBe(rows);
|
||||
await expect(capture.execute(async () => { throw failure; }, metadata)).rejects.toBe(failure);
|
||||
expect(() => capture.execute(() => { throw failure; }, metadata)).toThrow(failure);
|
||||
capture.flush();
|
||||
expect(events[0]).toMatchObject({ calls: 3, rows: 1, queryErrors: 2,
|
||||
responseValueBytes: Buffer.byteLength(JSON.stringify(rows[0].value)), serializationErrors: 0,
|
||||
});
|
||||
expect(events[1]).toMatchObject({ startedQueries: 3, completedQueries: 3, inFlightAtEnd: 0 });
|
||||
expect(JSON.stringify(events)).not.toContain(failure.message);
|
||||
} finally { capture.flush(); }
|
||||
});
|
||||
|
||||
it("reports unfinished work at the deadline without cancelling it or counting a late result", async () => {
|
||||
const events: Array<Record<string, unknown>> = [];
|
||||
let now = 0;
|
||||
const capture = createFullDatabaseResponseCapture({ start: 0, end: 1000 }, event => events.push(event), () => now);
|
||||
let release!: (value: string[]) => void;
|
||||
const original = new Promise<string[]>(resolve => { release = resolve; });
|
||||
const pending = capture.execute(() => original, metadata);
|
||||
now = 1000;
|
||||
capture.flush();
|
||||
release(["result"]);
|
||||
await expect(pending).resolves.toEqual(["result"]);
|
||||
capture.flush();
|
||||
expect(events).toHaveLength(1);
|
||||
expect(events[0]).toMatchObject({ event: "paperclip.test_db_response_capture.summary.v1",
|
||||
startedQueries: 1, completedQueries: 0, inFlightAtEnd: 1, responseValueBytes: 0,
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,115 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { createDatabaseResponseCapture, type ResponseCaptureWindow } from "@paperclipai/db/response-capture";
|
||||
|
||||
export { installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
|
||||
type Capture = ReturnType<typeof createDatabaseResponseCapture>;
|
||||
type Metadata = Parameters<Capture["execute"]>[1];
|
||||
type Entry = Metadata & { calls: number; rows: number; bytes: number; maxBytes: number;
|
||||
queryErrors: number; serializationErrors: number; estimatedValues: number };
|
||||
|
||||
function fullResponseValueSize(value: unknown): { bytes: number; estimatedValues: number } {
|
||||
const columns = value && typeof value === "object" ? (value as { columns?: unknown }).columns : undefined;
|
||||
if (!Array.isArray(value) || !Array.isArray(columns)) {
|
||||
const serialized = JSON.stringify(value, (_, item: unknown) => typeof item === "bigint" ? item.toString() : item);
|
||||
return { bytes: serialized === undefined ? 0 : Buffer.byteLength(serialized), estimatedValues: 1 };
|
||||
}
|
||||
let bytes = 0, estimatedValues = 0;
|
||||
for (const row of value) {
|
||||
const cells = Array.isArray(row) ? row : columns.map(column => row[column.name]);
|
||||
for (const cell of cells) {
|
||||
if (cell === null || cell === undefined) continue;
|
||||
if (typeof cell === "string") bytes += Buffer.byteLength(cell);
|
||||
else if (typeof cell === "boolean") bytes += 1;
|
||||
else if (typeof cell === "number" || typeof cell === "bigint") bytes += Buffer.byteLength(String(cell));
|
||||
else {
|
||||
estimatedValues += 1;
|
||||
const serialized = JSON.stringify(cell, (_, item: unknown) => typeof item === "bigint" ? item.toString() : item);
|
||||
if (serialized !== undefined) bytes += Buffer.byteLength(serialized);
|
||||
}
|
||||
}
|
||||
}
|
||||
return { bytes, estimatedValues };
|
||||
}
|
||||
|
||||
export function createFullDatabaseResponseCapture(
|
||||
window: ResponseCaptureWindow,
|
||||
emit: (event: Record<string, unknown>) => void,
|
||||
now: () => number = Date.now,
|
||||
): Capture {
|
||||
const capture = createDatabaseResponseCapture(window, () => {}, now);
|
||||
const id = randomUUID(), entries = new Map<string, Entry>();
|
||||
let closed = false, inFlight = 0, started = 0, completed = 0, overflowCalls = 0;
|
||||
const send = (event: Record<string, unknown>) => { try { emit(event); } catch {} };
|
||||
function flush() {
|
||||
if (closed) return;
|
||||
closed = true;
|
||||
clearTimeout(timer);
|
||||
capture.flush();
|
||||
const ordered = [...entries.values()].sort((a, b) => b.bytes - a.bytes);
|
||||
for (const [groupIndex, entry] of ordered.entries()) send({
|
||||
event: "paperclip.test_db_response_capture.entry.v1", captureId: id,
|
||||
groupIndex, groupPage: Math.floor(groupIndex / 128),
|
||||
operation: entry.operation, table: entry.table, callers: entry.callers,
|
||||
calls: entry.calls, rows: entry.rows, responseValueBytes: entry.bytes,
|
||||
maxResponseValueBytes: entry.maxBytes, estimatedValues: entry.estimatedValues,
|
||||
queryErrors: entry.queryErrors, serializationErrors: entry.serializationErrors,
|
||||
});
|
||||
send({ event: "paperclip.test_db_response_capture.summary.v1", captureId: id,
|
||||
windowStart: new Date(window.start).toISOString(), windowEnd: new Date(window.end).toISOString(),
|
||||
closedAt: new Date(now()).toISOString(), startedQueries: started, completedQueries: completed,
|
||||
inFlightAtEnd: inFlight, groups: entries.size, groupPageSize: 128, groupPages: Math.ceil(ordered.length / 128),
|
||||
overflowCalls, responseValueBytes: ordered.reduce((sum, entry) => sum + entry.bytes, 0),
|
||||
rowValuesLogged: false, sqlOrParametersLogged: false, wireBytesMeasured: false,
|
||||
measurementBoundary: "postgres-js results before Drizzle column decoding",
|
||||
relationalQueriesMeasured: true, unsafeDriverQueriesMeasured: true,
|
||||
taggedDriverQueriesMeasured: false, streamingQueriesMeasured: false,
|
||||
});
|
||||
entries.clear();
|
||||
}
|
||||
const timer = setTimeout(flush, Math.max(0, window.end - now()));
|
||||
timer.unref();
|
||||
function finish(metadata: Metadata, value: unknown, failed: boolean) {
|
||||
if (!capture.active()) return;
|
||||
inFlight -= 1;
|
||||
completed += 1;
|
||||
let key = JSON.stringify(metadata);
|
||||
if (!entries.has(key) && entries.size >= 2047) {
|
||||
overflowCalls += 1;
|
||||
metadata = { operation: "other", table: "other", callers: [] };
|
||||
key = "overflow";
|
||||
}
|
||||
let entry = entries.get(key);
|
||||
if (!entry) {
|
||||
entry = { ...metadata, calls: 0, rows: 0, bytes: 0, maxBytes: 0,
|
||||
queryErrors: 0, serializationErrors: 0, estimatedValues: 0 };
|
||||
entries.set(key, entry);
|
||||
}
|
||||
entry.calls += 1;
|
||||
if (failed) { entry.queryErrors += 1; return; }
|
||||
entry.rows += Array.isArray(value) ? value.length : 0;
|
||||
try {
|
||||
const size = fullResponseValueSize(value);
|
||||
entry.bytes += size.bytes;
|
||||
entry.maxBytes = Math.max(entry.maxBytes, size.bytes);
|
||||
entry.estimatedValues += size.estimatedValues;
|
||||
} catch { entry.serializationErrors += 1; }
|
||||
}
|
||||
function execute(action: () => unknown, metadata: Metadata) {
|
||||
if (!capture.active()) return action();
|
||||
inFlight += 1;
|
||||
started += 1;
|
||||
let result: unknown;
|
||||
try { result = action(); }
|
||||
catch (error) { finish(metadata, undefined, true); throw error; }
|
||||
let asynchronous = false;
|
||||
try { asynchronous = Boolean(result && typeof (result as { then?: unknown }).then === "function"); } catch {}
|
||||
if (asynchronous) return Promise.resolve(result).then(
|
||||
value => { finish(metadata, value, false); return value; },
|
||||
error => { finish(metadata, undefined, true); throw error; },
|
||||
);
|
||||
finish(metadata, result, false);
|
||||
return result;
|
||||
}
|
||||
return { ...capture, execute, flush };
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { agents, companies, toolApplications, toolCatalogEntries, toolConnections, toolProfileBindings, toolProfileEntries, toolProfiles, type Db } from "@paperclipai/db";
|
||||
|
||||
export async function createToolProfileMatchingFixture(db: Db) {
|
||||
const [company] = await db.insert(companies).values({ name: "Synthetic profile matching", issuePrefix: `BM${randomUUID().slice(0, 6)}` }).returning();
|
||||
const companyAgents = await db.insert(agents).values(Array.from({ length: 12 }, (_, i) => ({ companyId: company.id, name: `Agent ${i}` }))).returning();
|
||||
const agent = companyAgents[0]!;
|
||||
const [app] = await db.insert(toolApplications).values({ companyId: company.id, name: "Synthetic app", type: "mcp_http" }).returning();
|
||||
const [connection] = await db.insert(toolConnections).values({ companyId: company.id, applicationId: app.id, name: "Synthetic connection", uid: randomUUID(), transport: "mcp_remote" }).returning();
|
||||
const catalog = await db.insert(toolCatalogEntries).values(Array.from({ length: 1890 }, (_, i) => ({ companyId: company.id, applicationId: app.id, connectionId: connection.id, name: `tool_${i}`, toolName: `tool_${i}`, versionHash: `v${i}`, riskLevel: "read" as const }))).returning();
|
||||
const [profile] = await db.insert(toolProfiles).values({ companyId: company.id, profileKey: "selected", name: "Selected tools", defaultAction: "deny" }).returning();
|
||||
await db.insert(toolProfileBindings).values({ companyId: company.id, profileId: profile.id, targetType: "agent", targetId: agent.id });
|
||||
await db.insert(toolProfileEntries).values(catalog.slice(0, 1575).map((entry, i) => ({ companyId: company.id, profileId: profile.id, selectorType: "catalog_entry" as const, catalogEntryId: entry.id, effect: i % 5 === 0 ? "exclude" as const : "include" as const })));
|
||||
return { company, agent };
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { agents, companies, toolApplications, toolCatalogEntries, toolConnections, toolProfileEntries, toolProfiles, type Db } from "@paperclipai/db";
|
||||
|
||||
export async function createProfileProjectionFixture(db: Db, options: { profiles?: number; tools?: number; rules?: number } = {}) {
|
||||
const profileCount = options.profiles ?? 200, toolCount = options.tools ?? 1900, ruleCount = options.rules ?? 380, maxRuleCount = options.rules ?? 695;
|
||||
const [company] = await db.insert(companies).values({ name: "Profile projection fixture", issuePrefix: `PP${randomUUID().slice(0, 6)}` }).returning();
|
||||
await db.insert(agents).values(Array.from({ length: 12 }, (_, i) => ({ companyId: company.id, name: `Agent ${i}` })));
|
||||
const [application] = await db.insert(toolApplications).values({ companyId: company.id, name: "Projection application", type: "mcp_http" }).returning();
|
||||
const [connection] = await db.insert(toolConnections).values({ companyId: company.id, applicationId: application.id, name: "Projection connection", uid: randomUUID(), transport: "mcp_remote", status: "active", enabled: true, healthStatus: "ok" }).returning();
|
||||
const catalog = await db.insert(toolCatalogEntries).values(Array.from({ length: toolCount + maxRuleCount }, (_, i) => ({
|
||||
companyId: company.id, applicationId: application.id, connectionId: connection.id,
|
||||
name: `tool_${i}`, toolName: `tool_${i}`, versionHash: `v${i}`, riskLevel: "read" as const,
|
||||
status: i < toolCount ? "active" as const : "removed" as const,
|
||||
firstSeenAt: new Date(i < toolCount ? "2026-01-03T00:00:00Z" : "2026-01-01T00:00:00Z"),
|
||||
}))).returning();
|
||||
const profiles = await db.insert(toolProfiles).values(Array.from({ length: profileCount }, (_, i) => ({
|
||||
companyId: company.id, profileKey: `selected-${i}`, name: `Selected profile ${i}`, defaultAction: "deny" as const,
|
||||
createdAt: new Date("2026-01-02T00:00:00Z"),
|
||||
}))).returning();
|
||||
for (let i = 0; i < profiles.length; i++) {
|
||||
const count = i % 4 < 2 ? maxRuleCount : 2 * ruleCount - maxRuleCount;
|
||||
const rules = (i % 2 ? catalog.slice(toolCount, toolCount + count) : catalog.slice(0, count)).map(entry => ({
|
||||
companyId: company.id, profileId: profiles[i]!.id, selectorType: "catalog_entry" as const,
|
||||
effect: "include" as const, catalogEntryId: entry.id,
|
||||
}));
|
||||
for (let offset = 0; offset < rules.length; offset += 500) await db.insert(toolProfileEntries).values(rules.slice(offset, offset + 500));
|
||||
}
|
||||
return { company, application, connection, catalog: catalog.slice(0, toolCount), profiles };
|
||||
}
|
||||
@@ -5,7 +5,7 @@ import { agentApiKeys, agents, budgetPolicies, chatDeliveries, chatEndpoints, co
|
||||
companyTransferRuns, connectionEventDeliveries, createDb, heartbeatRuns, issues,
|
||||
routineTriggers, routines, toolApplications, toolConnections, type Db } from "@paperclipai/db";
|
||||
import { readIdleSleepSafety } from "../services/idle-sleep-safety.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
|
||||
const now = Date.UTC(2026, 10, 2);
|
||||
@@ -25,7 +25,7 @@ if (!support.supported) console.warn(`Skipping idle sleep diagnostics Postgres t
|
||||
db = createDb(externalDatabaseUrl ?? temporary!.connectionString, { maxConnections: 4 });
|
||||
}, 60_000);
|
||||
afterEach(async () => { await db.execute(sql`TRUNCATE companies, plugins, environments CASCADE`); });
|
||||
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); });
|
||||
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
|
||||
|
||||
const read = (scheduled = false, monitors = false) => readIdleSleepSafety(db, hold, () => now, ownerId,
|
||||
localNone, undefined, undefined, scheduled, monitors, { collectDiagnostics: true });
|
||||
|
||||
@@ -9,7 +9,7 @@ import {
|
||||
heartbeatRuns, issues, issueWatchdogs, issueThreadInteractions, issueQuestionResponseDeliveries,
|
||||
projects, projectWorkspaces, routines, routineTriggers, toolApplications, toolConnections, workspaceOperations, executionWorkspaceRuntimeLeases, type Db,
|
||||
} from "@paperclipai/db";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
import { readIdleSleepSafety, type IdleSleepDrainStatus } from "../services/idle-sleep-safety.js";
|
||||
|
||||
@@ -175,7 +175,7 @@ if (!support.supported) console.warn(`Skipping idle sleep Postgres tests: ${supp
|
||||
db = createDb(externalDatabaseUrl ?? database.connectionString, { maxConnections: 4 });
|
||||
}, 60_000);
|
||||
afterEach(async () => { await db.execute(sql`TRUNCATE companies, plugins, environments CASCADE`); });
|
||||
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); });
|
||||
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
|
||||
|
||||
const read = () => readIdleSleepSafety(db, owned, () => now, ownerId, emptyLocal);
|
||||
const readScheduled = () => readIdleSleepSafety(db, owned, () => now, ownerId, emptyLocal, undefined, undefined, true);
|
||||
@@ -206,6 +206,19 @@ if (!support.supported) console.warn(`Skipping idle sleep Postgres tests: ${supp
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps a failed startup intent awake after its local worker counter settles", async () => {
|
||||
const f = await seed();
|
||||
expect(await read()).toEqual(none);
|
||||
const finish = beginIdleTrackedWork();
|
||||
await db.update(heartbeatRuns).set({ status: "running", runtimeMode: "native", nativePhase: "observed",
|
||||
runnerProfileJson: { nativeStartupContext: { contextId: randomUUID(), deadlineAtMs: now - 1 } } }).where(eq(heartbeatRuns.id, f.runId));
|
||||
finish();
|
||||
expect(idleWorkSnapshot().active).toBe(0);
|
||||
expect(await read()).toEqual(present);
|
||||
await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, f.runId));
|
||||
expect(await read()).toEqual(none);
|
||||
});
|
||||
|
||||
it("retains committed budget stop intent until its exact generation is delivered", async () => {
|
||||
const f = await seed();
|
||||
const [policy] = await db.insert(budgetPolicies).values({ companyId: f.companyId, scopeType: "company", scopeId: f.companyId,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { activityLog, agents, companies, createDb, heartbeatRuns, issues } from "@paperclipai/db";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
@@ -55,7 +55,7 @@ describe.skipIf(!support.supported)("creation source run projections", () => {
|
||||
expect(run.issueId).toBe(binding === "scoped" ? sourceId : null);
|
||||
expect(run.nativeIssueId).toBeNull();
|
||||
const observed = createDb(database.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(observed, { heartbeatRuns, issues }, capture);
|
||||
try {
|
||||
const result = await resolveIssueCreationSource(observed, child);
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { expect, it } from "vitest";
|
||||
import request from "supertest";
|
||||
import { agents, heartbeatRuns, issueRecoveryActions, issueRelations, issues, statusCards } from "@paperclipai/db";
|
||||
import { companySearchQuerySchema, createIssueSchema, PROVIDER_QUOTA_MONITOR_SERVICE_NAME } from "@paperclipai/shared";
|
||||
import { issueDisplayStatusSql } from "../services/issue-display-status.js";
|
||||
import { issueService } from "../services/issues.js";
|
||||
import { companySearchService } from "../services/company-search.js";
|
||||
import { statusCardService } from "../services/status-cards.js";
|
||||
import { issueRoutes } from "../routes/issues.js";
|
||||
import {
|
||||
describeEmbeddedPostgres, routeApp,
|
||||
seedCompanyWithBoardAccess, useEmbeddedPostgres,
|
||||
} from "./helpers/route-test-harness.js";
|
||||
|
||||
describeEmbeddedPostgres("Monitoring task presentation", () => {
|
||||
const ctx = useEmbeddedPostgres("paperclip-monitoring-status-", {
|
||||
resetEach: async (db) => { await db.execute(sql`SET client_min_messages = warning`); await db.execute(sql`TRUNCATE TABLE companies CASCADE`); },
|
||||
});
|
||||
|
||||
async function fixture() {
|
||||
const company = await seedCompanyWithBoardAccess(ctx.db, "Monitoring");
|
||||
const agentId = randomUUID();
|
||||
await ctx.db.insert(agents).values({ id: agentId, companyId: company.companyId, name: "Monitor owner", role: "engineer", status: "paused", adapterType: "process", adapterConfig: {} });
|
||||
const nextCheckAt = new Date(Date.now() + 60 * 60_000);
|
||||
const issueId = randomUUID();
|
||||
await ctx.db.insert(issues).values({
|
||||
id: issueId, companyId: company.companyId, title: "Check deployment", status: "in_progress",
|
||||
assigneeAgentId: agentId, createdByUserId: company.actor.userId,
|
||||
monitorNextCheckAt: nextCheckAt,
|
||||
executionPolicy: { mode: "normal", stages: [], commentRequired: false, monitor: { nextCheckAt: nextCheckAt.toISOString(), notes: "Check deployment" } },
|
||||
});
|
||||
return { ...company, agentId, issueId };
|
||||
}
|
||||
|
||||
async function displayStatus(id: string) {
|
||||
const [row] = await ctx.db.select({ status: issues.status, displayStatus: issueDisplayStatusSql() }).from(issues).where(eq(issues.id, id));
|
||||
return row;
|
||||
}
|
||||
|
||||
it("projects a scheduled wait consistently across detail, compact rows, summaries, search and counts", async () => {
|
||||
const f = await fixture();
|
||||
const svc = issueService(ctx.db);
|
||||
expect(await svc.getById(f.issueId)).toMatchObject({ status: "in_progress", displayStatus: "monitoring" });
|
||||
expect(await svc.list(f.companyId, { displayStatus: "monitoring", listProjection: "index" })).toEqual([expect.objectContaining({ id: f.issueId, status: "in_progress", displayStatus: "monitoring" })]);
|
||||
expect(await svc.count(f.companyId, { displayStatus: "monitoring" })).toBe(1);
|
||||
const app = routeApp(ctx.db, f.actor, issueRoutes);
|
||||
const summary = await request(app).get(`/api/companies/${f.companyId}/issues/summaries?ids=${f.issueId}`).expect(200);
|
||||
expect(summary.body[0]).toMatchObject({ status: "in_progress", displayStatus: "monitoring" });
|
||||
const result = await companySearchService(ctx.db).search(f.companyId, companySearchQuerySchema.parse({ q: "deployment", displayStatus: "monitoring" }));
|
||||
expect(result.results[0]?.issue).toMatchObject({ id: f.issueId, status: "in_progress", displayStatus: "monitoring" });
|
||||
expect(result.filterOptionCounts.displayStatus?.monitoring).toBe(1);
|
||||
expect(result.filterOptionCounts.status.in_progress).toBe(1);
|
||||
const [card] = await ctx.db.insert(statusCards).values({ companyId: f.companyId, interestPrompt: "Watch deployment", mentionedIssueIds: [f.issueId], refreshPolicy: { mode: "manual" } }).returning();
|
||||
expect(await statusCardService(ctx.db).listMentionedIssues(card!)).toEqual([expect.objectContaining({ id: f.issueId, status: "in_progress", displayStatus: "monitoring" })]);
|
||||
// Existing agent/API lifecycle filters keep their original meaning.
|
||||
expect(await svc.count(f.companyId, { status: "in_progress" })).toBe(1);
|
||||
expect(createIssueSchema.safeParse({ title: "Manual monitoring", status: "monitoring" }).success).toBe(false);
|
||||
});
|
||||
|
||||
it("filters before pagination and keeps In Progress separate from Monitoring", async () => {
|
||||
const f = await fixture();
|
||||
await ctx.db.insert(issues).values({ companyId: f.companyId, title: "Ordinary work", status: "in_progress", assigneeAgentId: f.agentId });
|
||||
const svc = issueService(ctx.db);
|
||||
expect(await svc.list(f.companyId, { displayStatus: "monitoring", limit: 1, offset: 0 })).toEqual([expect.objectContaining({ id: f.issueId })]);
|
||||
expect(await svc.list(f.companyId, { displayStatus: "monitoring", limit: 1, offset: 1 })).toEqual([]);
|
||||
expect(await svc.count(f.companyId, { displayStatus: "in_progress" })).toBe(1);
|
||||
expect(await svc.count(f.companyId, { status: "in_progress" })).toBe(2);
|
||||
});
|
||||
|
||||
it("shows the same wait indicator in list blocker summaries and task detail relations", async () => {
|
||||
const f = await fixture();
|
||||
const [dependent] = await ctx.db.insert(issues).values({ companyId: f.companyId, title: "Ship after deployment", status: "blocked" }).returning();
|
||||
await ctx.db.insert(issueRelations).values({ companyId: f.companyId, issueId: f.issueId, relatedIssueId: dependent!.id, type: "blocks" });
|
||||
const svc = issueService(ctx.db);
|
||||
const list = await svc.list(f.companyId, { includeBlockedBy: true });
|
||||
expect(list.find(row => row.id === dependent!.id)?.blockedBy).toEqual([expect.objectContaining({ id: f.issueId, displayStatus: "monitoring" })]);
|
||||
const detail = await request(routeApp(ctx.db, f.actor, issueRoutes)).get(`/api/issues/${dependent!.id}`).expect(200);
|
||||
expect(detail.body.blockedBy).toEqual([expect.objectContaining({ id: f.issueId, displayStatus: "monitoring" })]);
|
||||
});
|
||||
|
||||
it("returns to In Progress during the check and Monitoring after another yield, then Done", async () => {
|
||||
const f = await fixture();
|
||||
const runId = randomUUID();
|
||||
await ctx.db.insert(heartbeatRuns).values({ id: runId, companyId: f.companyId, agentId: f.agentId, issueId: f.issueId, status: "queued" });
|
||||
expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress");
|
||||
await ctx.db.update(heartbeatRuns).set({ status: "running" }).where(eq(heartbeatRuns.id, runId));
|
||||
await ctx.db.update(issues).set({ executionRunId: runId, checkoutRunId: runId }).where(eq(issues.id, f.issueId));
|
||||
expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress");
|
||||
await ctx.db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, runId));
|
||||
await ctx.db.update(issues).set({ executionRunId: null, checkoutRunId: null }).where(eq(issues.id, f.issueId));
|
||||
expect((await displayStatus(f.issueId)).displayStatus).toBe("monitoring");
|
||||
await ctx.db.update(issues).set({ status: "done" }).where(eq(issues.id, f.issueId));
|
||||
expect(await displayStatus(f.issueId)).toEqual({ status: "done", displayStatus: "done" });
|
||||
});
|
||||
|
||||
it.each(["blocked", "in_review", "todo", "cancelled"])("preserves %s even with a retained monitor", async (status) => {
|
||||
const f = await fixture();
|
||||
await ctx.db.update(issues).set({ status }).where(eq(issues.id, f.issueId));
|
||||
expect((await displayStatus(f.issueId)).displayStatus).toBe(status);
|
||||
});
|
||||
|
||||
it("does not label cleared, consumed, orphaned or quota-recovery schedules Monitoring", async () => {
|
||||
const f = await fixture();
|
||||
await ctx.db.update(issues).set({ executionState: { monitor: { status: "cleared" } } }).where(eq(issues.id, f.issueId));
|
||||
expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress");
|
||||
await ctx.db.update(issues).set({ executionState: null, monitorNextCheckAt: null }).where(eq(issues.id, f.issueId));
|
||||
expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress");
|
||||
await ctx.db.update(issues).set({ monitorNextCheckAt: new Date(), executionPolicy: { monitor: { serviceName: PROVIDER_QUOTA_MONITOR_SERVICE_NAME } } }).where(eq(issues.id, f.issueId));
|
||||
expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress");
|
||||
await ctx.db.update(issues).set({ executionPolicy: null }).where(eq(issues.id, f.issueId));
|
||||
expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress");
|
||||
});
|
||||
|
||||
it("keeps recovery and review holds visible and never writes a new lifecycle state", async () => {
|
||||
const f = await fixture();
|
||||
await ctx.db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: f.issueId, kind: "source", ownerType: "board", cause: "process_lost", fingerprint: randomUUID(), nextAction: "Inspect stopped run" });
|
||||
expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress");
|
||||
await ctx.db.execute(sql`UPDATE issue_recovery_actions SET status = 'escalated' WHERE source_issue_id = ${f.issueId}`);
|
||||
expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress");
|
||||
await ctx.db.execute(sql`UPDATE issue_recovery_actions SET status = 'resolved' WHERE source_issue_id = ${f.issueId}`);
|
||||
expect((await displayStatus(f.issueId)).displayStatus).toBe("monitoring");
|
||||
await ctx.db.update(issues).set({ executionState: { currentParticipant: { type: "user", userId: "reviewer" } } }).where(eq(issues.id, f.issueId));
|
||||
expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress");
|
||||
});
|
||||
});
|
||||
@@ -4,7 +4,7 @@ import { randomUUID } from "node:crypto";
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
agentRuntimeState,
|
||||
@@ -202,7 +202,7 @@ describeEmbeddedPostgres("issue queued-comment routes", () => {
|
||||
if (fallback) await db.update(issues).set({ executionRunId: null }).where(eq(issues.id, f.issueId));
|
||||
const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId));
|
||||
const observed = createDb(tempDb!.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(observed, { heartbeatRuns }, capture);
|
||||
try {
|
||||
const response = await request(app(f.companyId, "queue-owner", undefined, observed))
|
||||
@@ -238,7 +238,7 @@ describeEmbeddedPostgres("issue queued-comment routes", () => {
|
||||
}).where(eq(heartbeatRuns.companyId, f.companyId));
|
||||
const [before] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, f.wakeId));
|
||||
const observed = createDb(tempDb!.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(observed, { agentWakeupRequests, heartbeatRuns }, capture);
|
||||
try {
|
||||
const response = await request(app(f.companyId, "queue-owner", undefined, observed)).get(path).expect(200);
|
||||
@@ -275,7 +275,7 @@ describeEmbeddedPostgres("issue queued-comment routes", () => {
|
||||
})),
|
||||
]);
|
||||
const observed = createDb(tempDb!.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(observed, { agentWakeupRequests }, capture);
|
||||
try {
|
||||
const response = await request(app(f.companyId, "queue-owner", undefined, observed)).get(path).expect(200);
|
||||
|
||||
@@ -50,7 +50,7 @@ describeEmbeddedPostgres("bounded issue summaries", () => {
|
||||
expect(res.headers["cache-control"]).toBe("private, no-store");
|
||||
expect(res.body.map((row: { id: string }) => row.id)).toEqual([f.rows[1]!.id, f.rows[0]!.id]);
|
||||
expect(Object.keys(res.body[0]).sort()).toEqual([
|
||||
"companyId", "externalConversationState", "id", "identifier", "status", "title", "updatedAt",
|
||||
"companyId", "displayStatus", "externalConversationState", "id", "identifier", "status", "title", "updatedAt",
|
||||
]);
|
||||
expect(res.body[0].externalConversationState).toBeNull();
|
||||
expect(forbiddenRead).not.toHaveBeenCalled();
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import request from "supertest";
|
||||
import { expect, it } from "vitest";
|
||||
import { activityLog, agents, authUsers, issues } from "@paperclipai/db";
|
||||
import { issueRoutes } from "../routes/issues.js";
|
||||
import { deriveIssueUserContext, issueService } from "../services/issues.js";
|
||||
import { describeEmbeddedPostgres, resetCompanyIssueFixtures, routeApp, seedCompanyWithBoardAccess, useEmbeddedPostgres } from "./helpers/route-test-harness.js";
|
||||
|
||||
describeEmbeddedPostgres("human assignee unread comments", () => {
|
||||
const ctx = useEmbeddedPostgres("paperclip-assignee-unread-", {
|
||||
resetEach: async db => {
|
||||
await db.delete(activityLog);
|
||||
await db.delete(issues);
|
||||
await db.delete(agents);
|
||||
await resetCompanyIssueFixtures(db);
|
||||
await db.delete(authUsers);
|
||||
},
|
||||
});
|
||||
|
||||
it.each([false, true])("keeps another human's real comment unread through recency updates (prior read: %s)", async priorRead => {
|
||||
const { companyId, userId, actor } = await seedCompanyWithBoardAccess(ctx.db, "Assigned human task");
|
||||
const otherCompany = await seedCompanyWithBoardAccess(ctx.db, "Unrelated company");
|
||||
const svc = issueService(ctx.db), app = routeApp(ctx.db, actor, issueRoutes);
|
||||
const issue = await svc.create(companyId, {
|
||||
title: "Human inbox comment", status: "in_progress", createdByUserId: userId, assigneeUserId: userId,
|
||||
});
|
||||
if (priorRead) await request(app).post(`/api/issues/${issue.id}/read`).send({}).expect(200);
|
||||
const comment = await svc.addComment(issue.id, "Another human supplied an update", { userId: `author-${randomUUID()}` });
|
||||
const [afterComment] = await ctx.db.select().from(issues).where(eq(issues.id, issue.id));
|
||||
// Exercise the real insert and recency update. No historical timestamps or
|
||||
// mocked comment state hide the ordering that used to clear unread status.
|
||||
expect(afterComment!.updatedAt.getTime()).toBeGreaterThanOrEqual(new Date(comment.createdAt).getTime());
|
||||
const path = `/api/companies/${companyId}/issues`;
|
||||
const expectUnread = async (unread: boolean) => {
|
||||
for (const projection of ["", "&view=rows", "&view=index"]) {
|
||||
const result = await request(app).get(`${path}?q=${encodeURIComponent(issue.title)}&touchedByUserId=me&limit=100${projection}`).expect(200);
|
||||
expect(result.body).toHaveLength(1);
|
||||
expect(result.body[0].isUnreadForMe).toBe(unread);
|
||||
const filtered = await request(app).get(`${path}?q=${encodeURIComponent(issue.title)}&unreadForUserId=me&limit=100${projection}`).expect(200);
|
||||
expect(filtered.body.map((row: { id: string }) => row.id)).toEqual(unread ? [issue.id] : []);
|
||||
}
|
||||
expect(await svc.countUnreadTouchedByUser(companyId, userId)).toBe(unread ? 1 : 0);
|
||||
expect(await svc.unreadInboxIssueIds(companyId, userId, sql<boolean>`true`)).toEqual(unread ? [issue.id] : []);
|
||||
};
|
||||
await expectUnread(true);
|
||||
// Legacy compact responses have a short TTL. Check their SQL projection
|
||||
// once; immediate mutation readback belongs to ordinary/rows/index.
|
||||
const compact = await request(app).get(`${path}?q=${encodeURIComponent(issue.title)}&touchedByUserId=me&view=compact`).expect(200);
|
||||
expect(compact.body[0].isUnreadForMe).toBe(true);
|
||||
const derived = deriveIssueUserContext(afterComment!, userId, {
|
||||
myLastCommentAt: null, myLastReadAt: null, lastExternalCommentAt: comment.createdAt,
|
||||
});
|
||||
expect(derived.isUnreadForMe).toBe(true);
|
||||
await request(app).post(`/api/issues/${issue.id}/read`).send({}).expect(200);
|
||||
await expectUnread(false);
|
||||
await request(app).delete(`/api/issues/${issue.id}/read`).expect(200);
|
||||
await expectUnread(true);
|
||||
await svc.addComment(issue.id, "I have read this update", { userId });
|
||||
await expectUnread(false);
|
||||
await request(app).get(`/api/companies/${otherCompany.companyId}/issues?unreadForUserId=me`).expect(403);
|
||||
expect(await svc.countUnreadTouchedByUser(otherCompany.companyId, userId)).toBe(0);
|
||||
});
|
||||
|
||||
it("shows existing comments as unread when a task is subsequently assigned to a human", async () => {
|
||||
const { companyId, userId, actor } = await seedCompanyWithBoardAccess(ctx.db, "New assignment");
|
||||
const svc = issueService(ctx.db), app = routeApp(ctx.db, actor, issueRoutes);
|
||||
const issue = await svc.create(companyId, { title: "Assigned after discussion", status: "backlog" });
|
||||
await svc.addComment(issue.id, "Context written before assignment", { userId: `author-${randomUUID()}` });
|
||||
await request(app).patch(`/api/issues/${issue.id}`).send({ assigneeUserId: userId, status: "in_progress" }).expect(200);
|
||||
const result = await request(app).get(`/api/companies/${companyId}/issues?q=${encodeURIComponent(issue.title)}&unreadForUserId=me`).expect(200);
|
||||
expect(result.body.map((row: { id: string }) => row.id)).toEqual([issue.id]);
|
||||
expect(result.body[0].isUnreadForMe).toBe(true);
|
||||
await request(app).post(`/api/issues/${issue.id}/read`).send({}).expect(200);
|
||||
expect(await svc.countUnreadTouchedByUser(companyId, userId)).toBe(0);
|
||||
});
|
||||
|
||||
it("excludes deleted external comments but retains the historical touch from a deleted own reply", async () => {
|
||||
const { companyId, userId } = await seedCompanyWithBoardAccess(ctx.db, "Deleted comment controls");
|
||||
const svc = issueService(ctx.db);
|
||||
const issue = await svc.create(companyId, {
|
||||
title: "Deleted inbox activity", status: "in_progress", assigneeUserId: userId,
|
||||
});
|
||||
const authorId = `author-${randomUUID()}`;
|
||||
const first = await svc.addComment(issue.id, "Keep this live update", { userId: authorId });
|
||||
const second = await svc.addComment(issue.id, "Remove this later update", { userId: authorId });
|
||||
const observation = async () => {
|
||||
const [row] = await svc.list(companyId, { touchedByUserId: userId });
|
||||
return { row, count: await svc.countUnreadTouchedByUser(companyId, userId) };
|
||||
};
|
||||
await svc.tombstoneComment(second.id, { actorType: "user", userId });
|
||||
expect((await observation()).count).toBe(1);
|
||||
expect((await observation()).row!.lastExternalCommentAt).toEqual(new Date(first.createdAt));
|
||||
await svc.tombstoneComment(first.id, { actorType: "user", userId });
|
||||
const removed = await observation();
|
||||
expect(removed.count).toBe(0);
|
||||
expect(removed.row!.isUnreadForMe).toBe(false);
|
||||
expect(removed.row!.lastExternalCommentAt).toBeNull();
|
||||
await svc.addComment(issue.id, "New live activity", { userId: authorId });
|
||||
expect((await observation()).count).toBe(1);
|
||||
const ownReply = await svc.addComment(issue.id, "I saw the live update", { userId });
|
||||
expect((await observation()).count).toBe(0);
|
||||
await svc.tombstoneComment(ownReply.id, { actorType: "user", userId });
|
||||
const seen = await observation();
|
||||
expect(seen.count).toBe(0);
|
||||
expect(seen.row!.isUnreadForMe).toBe(false);
|
||||
expect(seen.row!.myLastTouchAt).toEqual(new Date(ownReply.createdAt));
|
||||
});
|
||||
|
||||
});
|
||||
@@ -56,4 +56,30 @@ describeEmbeddedPostgres("issue list status query parsing", () => {
|
||||
it("returns every status when ?status is absent", async () => {
|
||||
expect(await listStatuses("")).toEqual(["done", "in_progress", "todo"]);
|
||||
});
|
||||
it.each(["", "&view=compact", "&view=compact&projection=index"])("rejects unsupported creation sorting for blocked attention (%s)", async projection => {
|
||||
const { companyId, actor } = await seedCompanyWithBoardAccess(ctx.db, "Blocked sorting");
|
||||
await request(routeApp(ctx.db, actor, issueRoutes))
|
||||
.get(`/api/companies/${companyId}/issues?attention=blocked&sortField=created&sortDir=asc${projection}`)
|
||||
.expect(422);
|
||||
});
|
||||
|
||||
it.each(["", "&view=compact", "&view=compact&projection=index"])("sorts routine tasks by creation time before pagination (%s)", async (projection) => {
|
||||
const { companyId, actor } = await seedCompanyWithBoardAccess(ctx.db, "Creation sorting");
|
||||
const ids = Array.from({ length: 4 }, (_, index) => `00000000-0000-4000-8000-00000000000${index}`);
|
||||
await ctx.db.insert(issues).values([
|
||||
...ids.map((id, index) => ({ id, companyId, title: "Routine sorting", originKind: "routine_execution", status: "todo",
|
||||
createdAt: new Date(index < 2 ? "2026-01-01" : "2026-02-01"),
|
||||
updatedAt: new Date(index < 2 ? "2026-03-01" : "2026-02-01"), priority: index < 2 ? "critical" : "low" })),
|
||||
{ companyId, title: "Routine sorting manual", originKind: "manual", createdAt: new Date("2026-04-01") },
|
||||
]);
|
||||
const app = routeApp(ctx.db, actor, issueRoutes);
|
||||
const path = `/api/companies/${companyId}/issues?originKind=routine_execution&includeRoutineExecutions=true&sortField=created&limit=2${projection}`;
|
||||
for (const sortDir of ["asc", "desc"]) {
|
||||
const first = await request(app).get(`${path}&sortDir=${sortDir}`).expect(200);
|
||||
const second = await request(app).get(`${path}&sortDir=${sortDir}&offset=2`).expect(200);
|
||||
expect([...first.body, ...second.body].map((issue: { id: string }) => issue.id)).toEqual(sortDir === "asc" ? ids : [...ids].reverse());
|
||||
}
|
||||
await request(app).get(`${path}&sortDir=invalid`).expect(400);
|
||||
});
|
||||
|
||||
});
|
||||
@@ -4166,6 +4166,7 @@ const dependencyTestDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_ISSU
|
||||
await db.delete(executionWorkspaces);
|
||||
await db.delete(projectWorkspaces);
|
||||
await db.delete(projects);
|
||||
await db.delete(heartbeatRuns);
|
||||
await db.delete(agents);
|
||||
await db.delete(instanceSettings);
|
||||
await db.delete(companies);
|
||||
@@ -4720,6 +4721,42 @@ const dependencyTestDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_ISSU
|
||||
});
|
||||
});
|
||||
|
||||
it.each(["covered", "foreign-run", "unfinished", "failed-finalize", "late-finish", "later-operation"])(
|
||||
"requires a successful owning finalize to cover a nested snapshot: %s", async kind => {
|
||||
const f = await seedSharedWorkspaceDependency();
|
||||
const runId = randomUUID(), foreignRunId = randomUUID();
|
||||
await db.insert(heartbeatRuns).values([runId, foreignRunId].map(id => ({
|
||||
id, companyId: f.companyId, agentId: f.assigneeAgentId, status: "succeeded",
|
||||
})));
|
||||
const started = new Date("2026-05-23T22:00:00.000Z");
|
||||
const finished = new Date("2026-05-23T22:00:02.000Z");
|
||||
await db.insert(workspaceOperations).values([
|
||||
{ companyId: f.companyId, executionWorkspaceId: f.executionWorkspaceId, issueId: f.blockerId,
|
||||
heartbeatRunId: runId, phase: "workspace_finalize",
|
||||
status: kind === "failed-finalize" ? "failed" : "succeeded", startedAt: started, finishedAt: finished },
|
||||
{ companyId: f.companyId, executionWorkspaceId: f.executionWorkspaceId, issueId: f.blockerId,
|
||||
heartbeatRunId: kind === "foreign-run" ? foreignRunId : runId, phase: "remote_workspace_snapshot",
|
||||
status: kind === "unfinished" ? "running" : "succeeded",
|
||||
startedAt: new Date(started.getTime() + 100),
|
||||
finishedAt: kind === "unfinished" ? null : new Date(finished.getTime() + (kind === "late-finish" ? 100 : -100)) },
|
||||
...(kind === "later-operation" ? [{
|
||||
companyId: f.companyId, executionWorkspaceId: f.executionWorkspaceId, issueId: f.blockerId,
|
||||
heartbeatRunId: runId, phase: "worktree_prepare", status: "succeeded",
|
||||
startedAt: new Date(finished.getTime() + 100), finishedAt: new Date(finished.getTime() + 200),
|
||||
}] : []),
|
||||
]);
|
||||
const ready = kind === "covered";
|
||||
expect((await svc.listWakeableBlockedDependents(f.blockerId)).map(issue => issue.id))
|
||||
.toEqual(ready ? [f.dependentId] : []);
|
||||
await expect(svc.getDependencyReadiness(f.dependentId)).resolves.toMatchObject({
|
||||
isDependencyReady: ready,
|
||||
pendingFinalizeBlockerIssueIds: ready ? [] : [f.blockerId],
|
||||
});
|
||||
const listed = (await svc.list(f.companyId, { status: "blocked" })).find(issue => issue.id === f.dependentId);
|
||||
expect(listed?.blockerAttention?.pendingFinalizeBlockerIssueIds ?? []).toEqual(ready ? [] : [f.blockerId]);
|
||||
},
|
||||
);
|
||||
|
||||
it("reports dependency readiness for blocked issue chains", async () => {
|
||||
const companyId = randomUUID();
|
||||
await seedIssueCompany(db, companyId);
|
||||
|
||||
@@ -76,7 +76,7 @@ describe("deriveIssueUserContext", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("uses issue updated time as fallback touch point for assignee", () => {
|
||||
it("uses stable creation time for assignees instead of treating task recency as a read", () => {
|
||||
const context = deriveIssueUserContext(
|
||||
makeIssue({ assigneeUserId: "user-1", updatedAt: new Date("2026-03-06T15:00:00.000Z") }),
|
||||
"user-1",
|
||||
@@ -87,8 +87,8 @@ describe("deriveIssueUserContext", () => {
|
||||
},
|
||||
);
|
||||
|
||||
expect(context.myLastTouchAt?.toISOString()).toBe("2026-03-06T15:00:00.000Z");
|
||||
expect(context.isUnreadForMe).toBe(false);
|
||||
expect(context.myLastTouchAt?.toISOString()).toBe("2026-03-06T10:00:00.000Z");
|
||||
expect(context.isUnreadForMe).toBe(true);
|
||||
});
|
||||
|
||||
it("uses latest read timestamp to clear unread without requiring a comment", () => {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { randomBytes, randomUUID } from "node:crypto";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { agents, companies, createDb, environmentLeases, environments, executionWorkspaces, heartbeatRuns, issueRecoveryActions, issues, projects } from "@paperclipai/db";
|
||||
@@ -54,12 +54,12 @@ const support = externalTestDatabaseUrl ? { supported: true } : await getEmbedde
|
||||
runnerProfileJson: { savedProfile: bulk }, stdoutExcerpt: bulk, stderrExcerpt: bulk,
|
||||
}).where(eq(heartbeatRuns.id, f.runId)).returning();
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
|
||||
let returned: Awaited<ReturnType<typeof terminalizeLegacyExecution>>;
|
||||
try { returned = await terminalizeLegacyExecution({ db, run: saved!, status: "failed", patch }); }
|
||||
finally { capture.flush(); }
|
||||
const groups = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1");
|
||||
const groups = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1");
|
||||
const reads = groups.filter(group => group.table === "heartbeat_runs" && group.operation === "select");
|
||||
expect(reads.reduce((sum, group) => sum + Number(group.calls), 0)).toBe(1);
|
||||
expect(reads.reduce((sum, group) => sum + Number(group.responseValueBytes), 0)).toBeLessThan(4096);
|
||||
@@ -86,7 +86,7 @@ const support = externalTestDatabaseUrl ? { supported: true } : await getEmbedde
|
||||
patch: { resultJson: { workspaceRestoreFailure: "restore_failed", futureResult: bulk } },
|
||||
}))!;
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
|
||||
let returned: Awaited<ReturnType<typeof terminalizeLegacyExecution>>;
|
||||
try { returned = await terminalizeLegacyExecution({ db, run: failed, status: "failed", recordRestoreFailureOnly,
|
||||
@@ -94,7 +94,7 @@ const support = externalTestDatabaseUrl ? { supported: true } : await getEmbedde
|
||||
...(recordRestoreFailureOnly ? { patch: { resultJson: failed.resultJson } } : {}),
|
||||
}); }
|
||||
finally { capture.flush(); }
|
||||
const writes = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
|
||||
const writes = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
|
||||
&& event.table === "heartbeat_runs" && event.operation === "update");
|
||||
expect(writes.reduce((sum, group) => sum + Number(group.rows), 0)).toBe(1);
|
||||
expect(returned).toMatchObject({ id: f.runId, status: "failed", finishedAt: failed.finishedAt,
|
||||
|
||||
@@ -11,7 +11,7 @@ import { setTimeout as delay } from "node:timers/promises";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { agents, companies, completionContracts, createDb, environmentLeases, environments, heartbeatRuns, issues, nativeRunFinalizations, type Db } from "@paperclipai/db";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
|
||||
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
||||
import { heartbeatService } from "../services/heartbeat.js";
|
||||
import { environmentRuntimeService, type EnvironmentDriver } from "../services/environment-runtime.js";
|
||||
@@ -180,7 +180,7 @@ describe("native environment release scheduling", () => {
|
||||
}
|
||||
const events: Record<string, unknown>[] = [];
|
||||
const now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { nativeRunFinalizations, heartbeatRuns, environmentLeases }, capture);
|
||||
const settle = vi.fn(async () => {});
|
||||
const work = nativeEnvironmentSettlementWork(db, settle);
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db";
|
||||
import { randomBytes, randomUUID } from "node:crypto";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
|
||||
import { spawn } from "node:child_process";
|
||||
import os from "node:os";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
@@ -222,11 +222,11 @@ suite("settled native run process identity", () => {
|
||||
const saved = await readRun(runId);
|
||||
const events: Array<Record<string, unknown>> = [];
|
||||
const now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
|
||||
try { expect(await releaseSettledNativeRunProcess(db, runId)).toBe("released"); }
|
||||
finally { capture.flush(); }
|
||||
const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!;
|
||||
const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!;
|
||||
expect(summary.responseValueBytes).toBeLessThan(8192);
|
||||
const after = await readRun(runId);
|
||||
expect(after).toEqual({ ...saved, processPid: null, processGroupId: null, processStartedAt: null,
|
||||
|
||||
@@ -0,0 +1,729 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { databaseWorkPending } from "@paperclipai/db/work-signals";
|
||||
import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { PluginEnvironmentTaskOperation } from "@paperclipai/plugin-sdk";
|
||||
import type { PaperclipPluginManifestV1 } from "@paperclipai/shared";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
activityLog, agents, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, environmentLeases,
|
||||
environments, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults, plugins, sandboxCreationOperations,
|
||||
} from "@paperclipai/db";
|
||||
import { recoveryPostgresSupport, startRecoveryPostgres } from "./helpers/recovery-postgres.js";
|
||||
import { createIsolatedTestHome } from "./helpers/isolated-test-home.js";
|
||||
import { prepareNativeHeartbeatRun } from "../services/native-runtime/prepare-native-run.js";
|
||||
import {
|
||||
acknowledgeNativeStartupCleanup, NATIVE_STARTUP_CLEANUP_PENDING, readNativeStartupCleanup,
|
||||
requestNativeStartupCleanup, verifyNativeStartupCleanup, clearNativeStartupContext,
|
||||
} from "../services/native-runtime/native-startup-cleanup.js";
|
||||
import { isNativeRunnerOwnershipHeld } from "../services/native-runtime/native-runner-ownership.js";
|
||||
import { cancelNativeSession } from "../services/native-runtime/native-session-executor.js";
|
||||
import * as nativeSessionExecutor from "../services/native-runtime/native-session-executor.js";
|
||||
import { NativeExecutionContext } from "../services/native-runtime/native-execution-context.js";
|
||||
import type { NativeExecutionInput } from "../vendor/paperclip-runner/index.js";
|
||||
import { callbacks } from "../services/heartbeat/__tests__/run-control-fixtures.js";
|
||||
import { createHeartbeatRunControl } from "../services/heartbeat/run-control.js";
|
||||
import { nextEnvironmentCleanupAt } from "../services/environment-cleanup-state.js";
|
||||
import { executionControlSchedule } from "../services/execution-control-work.js";
|
||||
import { claimNativeRestartRecoveries } from "../services/native-runtime/native-restart-recovery.js";
|
||||
import { reconcileAbandonedExecutionControl } from "../services/execution-control-reconciliation.js";
|
||||
import { createPluginWorkerManager, type PluginWorkerManager } from "../services/plugin-worker-manager.js";
|
||||
import { hasRemoteTerminationReceipt, remoteTerminationReceipt } from "../services/remote-execution-termination.js";
|
||||
|
||||
import { createOwnedSandbox, queueSandboxCreationCleanup, recordSandboxCreationResource, recoverSandboxCreations } from "../services/sandbox-creation-operations.js";
|
||||
import { environmentService } from "../services/environments.js";
|
||||
import { environmentRuntimeService } from "../services/environment-runtime.js";
|
||||
import { createHeartbeatRecovery, type HeartbeatRecoveryDependencies } from "../services/heartbeat/recovery.js";
|
||||
|
||||
const describePostgres = recoveryPostgresSupport.supported ? describe : describe.skip;
|
||||
|
||||
describePostgres("durable cleanup after a native startup timeout", () => {
|
||||
let temporary: Awaited<ReturnType<typeof startRecoveryPostgres>>;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let home: ReturnType<typeof createIsolatedTestHome>;
|
||||
const originalHome = process.env.PAPERCLIP_HOME;
|
||||
const companyId = randomUUID(), agentId = randomUUID();
|
||||
|
||||
beforeAll(async () => {
|
||||
home = createIsolatedTestHome();
|
||||
process.env.PAPERCLIP_HOME = home.home;
|
||||
temporary = await startRecoveryPostgres("native-startup-cleanup-");
|
||||
db = createDb(temporary.connectionString);
|
||||
await db.insert(companies).values({ id: companyId, name: "Startup cleanup", issuePrefix: "NSC" });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Runner", role: "engineer", status: "running",
|
||||
adapterType: "paperclip_runner", adapterConfig: { provider: "codex" }, runtimeConfig: {}, permissions: {} });
|
||||
}, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
|
||||
|
||||
afterAll(async () => {
|
||||
await temporary?.cleanup();
|
||||
home?.cleanup();
|
||||
if (originalHome === undefined) delete process.env.PAPERCLIP_HOME;
|
||||
else process.env.PAPERCLIP_HOME = originalHome;
|
||||
});
|
||||
|
||||
async function fixture(requiresPreparationStop = false) {
|
||||
const runId = randomUUID(), issueId = randomUUID(), leaseId = randomUUID(), taskId = randomUUID();
|
||||
const contextId = randomUUID(), leaseOwner = randomUUID();
|
||||
await db.insert(issues).values({ id: issueId, companyId, identifier: `NSC-${issueId}`, title: "Timed out startup",
|
||||
status: "in_progress", priority: "medium", assigneeAgentId: agentId });
|
||||
const [run] = await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running",
|
||||
runtimeMode: "native", nativeIssueId: issueId, invocationSource: "manual", contextSnapshot: { issueId } }).returning();
|
||||
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId));
|
||||
await prepareNativeHeartbeatRun({ db, run, issue: { id: issueId, title: "Timed out startup", description: null }, environmentLeaseId: leaseId });
|
||||
await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId, phase: "observed", attempt: 1,
|
||||
controllerGeneration: 1, controllerBootId: randomUUID(), controllerPid: process.pid,
|
||||
controllerProcessStartedAt: new Date(), leaseOwner, leaseExpiresAt: new Date(Date.now() + 60_000) });
|
||||
const [environment] = await db.insert(environments).values({ name: `Cleanup ${runId}`, driver: "plugin", status: "active", config: {} }).returning();
|
||||
await db.insert(environmentLeases).values({ id: leaseId, companyId, environmentId: environment.id, heartbeatRunId: runId,
|
||||
issueId, status: "active", provider: "plugin", providerLeaseId: taskId, leasePolicy: "reuse_by_environment",
|
||||
metadata: { runnerTaskLease: true, runnerTaskReused: true } });
|
||||
const request = { companyId, agentId, runId, contextId, leaseOwner, deadlineAtMs: Date.now() - 1, requiresPreparationStop };
|
||||
const reload = () => db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)).then(rows => rows[0]!);
|
||||
const owner = () => db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)).then(rows => rows[0]!);
|
||||
const lease = () => db.select().from(environmentLeases).where(eq(environmentLeases.id, leaseId)).then(rows => rows[0]!);
|
||||
const stopReceipt = async (override: Record<string, unknown> = {}, preparationStopped = false) => {
|
||||
const current = await lease();
|
||||
const receipt = remoteTerminationReceipt(current, { providerLeaseId: taskId, state: "stopped" });
|
||||
await db.update(environmentLeases).set({ status: "released", releasedAt: new Date(), cleanupStatus: "success",
|
||||
metadata: { ...current.metadata, ...(preparationStopped ? { runnerTaskPreparationStopped: true } : {}), remoteExecutionTermination: { ...receipt, ...override } } }).where(eq(environmentLeases.id, leaseId));
|
||||
};
|
||||
const acceptedResult = async () => {
|
||||
const current = await reload(), resultId = randomUUID();
|
||||
await db.insert(nativeRunResults).values({ id: resultId, companyId, issueId, runId,
|
||||
completionContractId: current.completionContractId!, serverFingerprint: resultId, schemaStatus: "accepted",
|
||||
resultJson: { schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", summary: "Completed work" }, canonicalSha256: "a".repeat(64) });
|
||||
await db.update(nativeRunFinalizations).set({ resultId, phase: "workspace_finalizing" }).where(eq(nativeRunFinalizations.runId, runId));
|
||||
return resultId;
|
||||
};
|
||||
return { request, reload, owner, lease, stopReceipt, acceptedResult };
|
||||
}
|
||||
|
||||
async function saveEarlyContext(f: Awaited<ReturnType<typeof fixture>>) {
|
||||
const run = await f.reload(), owner = await f.owner(), lease = await f.lease();
|
||||
const context = { schema: "paperclip.native-startup-context.v1", companyId, agentId, runId: run.id,
|
||||
contextId: f.request.contextId, deadlineAtMs: f.request.deadlineAtMs, leaseId: lease.id,
|
||||
leaseOwner: owner.leaseOwner, controllerGeneration: owner.controllerGeneration, attempt: owner.attempt };
|
||||
await db.update(heartbeatRuns).set({ runnerProfileJson: { ...run.runnerProfileJson, nativeStartupContext: context } }).where(eq(heartbeatRuns.id, run.id));
|
||||
return context;
|
||||
}
|
||||
|
||||
it.each(["pending", "needs_reconciliation"])("keeps ownership until a %s replacement queued after the first teardown snapshot is verified stopped", async state => {
|
||||
const f = await fixture(true), original = await f.lease();
|
||||
let reply!: (result: { providerLeaseId: string }) => void, creating!: () => void;
|
||||
const entered = new Promise<void>(resolve => { creating = resolve; });
|
||||
const acquiring = createOwnedSandbox(db, { companyId, environmentId: original.environmentId!, heartbeatRunId: f.request.runId,
|
||||
issueId: original.issueId, provider: original.provider!, driverKey: "tasks", config: {}, requestKey: original.providerLeaseId!,
|
||||
metadata: { runnerTaskLease: true, runnerTaskCreationContextId: f.request.contextId } }, async () => new Promise(resolve => { reply = resolve; creating(); }));
|
||||
await entered;
|
||||
await requestNativeStartupCleanup(db, f.request);
|
||||
await f.stopReceipt({}, true);
|
||||
await expect(verifyNativeStartupCleanup(db, await f.reload(), async () => {})).rejects.toThrow("cleanup is still pending");
|
||||
const [pending] = await db.select().from(sandboxCreationOperations).where(and(
|
||||
eq(sandboxCreationOperations.runId, f.request.runId), eq(sandboxCreationOperations.requestKey, original.providerLeaseId!)));
|
||||
if (state === "needs_reconciliation") {
|
||||
await db.update(sandboxCreationOperations).set({ nextAttemptAt: new Date(0) }).where(eq(sandboxCreationOperations.id, pending.id));
|
||||
await recoverSandboxCreations(db, async () => null);
|
||||
expect((await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.id, pending.id)))[0].state).toBe(state);
|
||||
const inspect = vi.fn(async () => null);
|
||||
await recoverSandboxCreations(db, inspect);
|
||||
expect(inspect).not.toHaveBeenCalled();
|
||||
await expect(queueSandboxCreationCleanup(db, { id: pending.id, companyId, environmentId: original.environmentId!,
|
||||
providerLeaseId: "unproven-task" })).rejects.toThrow("ownership changed");
|
||||
}
|
||||
reply({ providerLeaseId: "late-replacement" });
|
||||
const fresh = await acquiring;
|
||||
await recordSandboxCreationResource(db, { id: String(fresh.metadata!.creationOperationId), companyId,
|
||||
environmentId: original.environmentId!, providerLeaseId: fresh.providerLeaseId, providerMetadata: { binding: "actual" } });
|
||||
if (state === "pending") await db.update(sandboxCreationOperations).set({ nextAttemptAt: new Date(0) })
|
||||
.where(eq(sandboxCreationOperations.id, String(fresh.metadata!.creationOperationId)));
|
||||
expect(await nextEnvironmentCleanupAt(db)).not.toBeNull();
|
||||
const restarted = createDb(temporary.connectionString);
|
||||
try { await recoverSandboxCreations(restarted, async () => { throw new Error("A recorded task identity needs no new provider inspection"); }); }
|
||||
finally { await restarted.$client.end(); }
|
||||
const [creation] = await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.id, String(fresh.metadata!.creationOperationId)));
|
||||
await expect(verifyNativeStartupCleanup(db, await f.reload(), async () => {})).rejects.toThrow("cleanup is still pending");
|
||||
const replacement = (await environmentService(db).getLeaseById(creation.leaseId!))!;
|
||||
await queueSandboxCreationCleanup(db, { id: creation.id, companyId, environmentId: original.environmentId!, providerLeaseId: fresh.providerLeaseId });
|
||||
expect((await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.id, creation.id)))[0].leaseId).toBe(replacement.id);
|
||||
await expect(queueSandboxCreationCleanup(db, { id: creation.id, companyId, environmentId: original.environmentId!, providerLeaseId: "foreign-task" })).rejects.toThrow("ownership changed");
|
||||
await environmentService(db).releaseLease(replacement.id, "released", { cleanupStatus: "success",
|
||||
remoteExecutionTermination: remoteTerminationReceipt(replacement, { providerLeaseId: replacement.providerLeaseId, state: "stopped" }) });
|
||||
expect((await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.id, creation.id)))[0].state).toBe("resolved");
|
||||
await queueSandboxCreationCleanup(db, { id: creation.id, companyId, environmentId: original.environmentId!, providerLeaseId: fresh.providerLeaseId });
|
||||
expect((await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.id, creation.id)))[0])
|
||||
.toMatchObject({ state: "resolved", leaseId: replacement.id });
|
||||
await verifyNativeStartupCleanup(db, await f.reload(), async () => {});
|
||||
await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, f.request.runId));
|
||||
await acknowledgeNativeStartupCleanup(db, f.request);
|
||||
expect((await f.owner()).leaseOwner).toBeNull();
|
||||
});
|
||||
|
||||
it.each(["before_inventory", "before_write", "complete_before_write", "ordinary"])("preserves startup cleanup through orphan recovery: %s", async phase => {
|
||||
const f = await fixture(), deps = callbacks(db);
|
||||
await db.update(nativeRunFinalizations).set({ phase: "terminal_failure", leaseExpiresAt: new Date(0) })
|
||||
.where(eq(nativeRunFinalizations.runId, f.request.runId));
|
||||
if (phase === "before_inventory") await requestNativeStartupCleanup(db, f.request);
|
||||
const write = deps.setRunStatusFromLive.getMockImplementation()!;
|
||||
deps.setRunStatusFromLive.mockImplementation(async (...args) => {
|
||||
if (phase === "before_write" || phase === "complete_before_write") {
|
||||
await requestNativeStartupCleanup(db, f.request);
|
||||
if (phase === "complete_before_write") await verifyNativeStartupCleanup(db, await f.reload(), () => f.stopReceipt());
|
||||
}
|
||||
return write(...args);
|
||||
});
|
||||
const recovery = createHeartbeatRecovery(db, { ...deps, environmentRuntime: environmentRuntimeService(db),
|
||||
environmentsSvc: environmentService(db), runtimeEnv: {}, activeRunExecutionPromises: new Set<Promise<void>>(),
|
||||
mergeRunStopMetadataForAgent: (_agent: unknown, _status: unknown, options: { resultJson?: object }) => options?.resultJson,
|
||||
classifyAndPersistRunLiveness: async (run: unknown) => run, releaseEnvironmentLeasesForRun: vi.fn(),
|
||||
releaseIssueExecutionAndPromote: vi.fn(), scheduleInteractionContinuationInfrastructureRetryIfEligible: async () => null,
|
||||
} as unknown as HeartbeatRecoveryDependencies);
|
||||
await recovery.recoverActiveExecutions({ runId: f.request.runId }, { staleThresholdMs: 0 });
|
||||
expect((await f.reload()).status).toBe(phase === "ordinary" ? "failed" : "running");
|
||||
if (phase === "ordinary") return;
|
||||
expect((await f.owner()).leaseOwner).toBe(f.request.leaseOwner);
|
||||
deps.verifyStartupCleanup = run => verifyNativeStartupCleanup(db, run, () => f.stopReceipt());
|
||||
expect(await createHeartbeatRunControl(db, deps).cancelRunInternal(f.request.runId, "Startup cleanup retry"))
|
||||
.toMatchObject({ status: "cancelled" });
|
||||
expect((await f.owner()).leaseOwner).toBeNull();
|
||||
});
|
||||
|
||||
it.each(["request", "acknowledge"])("registers queue intent before cleanup %s writes", async phase => {
|
||||
const f = await fixture(true);
|
||||
const writes: boolean[][] = [];
|
||||
const observe = async (queues: string[], work: () => Promise<unknown>) => {
|
||||
const transaction = db.transaction.bind(db);
|
||||
const spy = vi.spyOn(db, "transaction").mockImplementation(((callback: (tx: typeof db) => Promise<unknown>) => transaction(async tx => {
|
||||
const update = tx.update.bind(tx);
|
||||
tx.update = ((table: unknown) => {
|
||||
if ([heartbeatRuns, nativeRunFinalizations, environmentLeases].includes(table as never)) {
|
||||
writes.push(queues.map(queue => databaseWorkPending(db, queue)));
|
||||
}
|
||||
return update(table as never);
|
||||
}) as typeof tx.update;
|
||||
return callback(tx as unknown as typeof db);
|
||||
})) as typeof db.transaction);
|
||||
try { await work(); } finally { spy.mockRestore(); }
|
||||
};
|
||||
if (phase === "request") {
|
||||
await observe([DELIVERY_QUEUES.executionFinalization, DELIVERY_QUEUES.environmentCleanup], () => requestNativeStartupCleanup(db, f.request));
|
||||
expect(writes).toEqual([[true, true], [true, true], [true, true]]);
|
||||
await verifyNativeStartupCleanup(db, await f.reload(), () => f.stopReceipt({}, true));
|
||||
await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, f.request.runId));
|
||||
await acknowledgeNativeStartupCleanup(db, f.request);
|
||||
} else {
|
||||
await requestNativeStartupCleanup(db, f.request);
|
||||
await verifyNativeStartupCleanup(db, await f.reload(), () => f.stopReceipt({}, true));
|
||||
await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, f.request.runId));
|
||||
await observe([DELIVERY_QUEUES.agentLifecycle, DELIVERY_QUEUES.environmentCleanup], () => acknowledgeNativeStartupCleanup(db, f.request));
|
||||
expect(writes).toEqual([[true, true]]);
|
||||
}
|
||||
});
|
||||
|
||||
it("returns the deadline queue to quiet and wakes again for the next startup failure", async () => {
|
||||
const schedule = executionControlSchedule(db);
|
||||
expect(await schedule.finalization()).toBeNull();
|
||||
for (let failure = 0; failure < 2; failure += 1) {
|
||||
const f = await fixture(true);
|
||||
await requestNativeStartupCleanup(db, f.request);
|
||||
expect(await schedule.finalization()).not.toBeNull();
|
||||
const deps = callbacks(db);
|
||||
deps.verifyStartupCleanup = run => verifyNativeStartupCleanup(db, run, () => f.stopReceipt({}, true));
|
||||
const control = createHeartbeatRunControl(db, deps);
|
||||
await reconcileAbandonedExecutionControl(db, new Date(Date.now() + 1_000), { startupCleanup: async scope => {
|
||||
if (scope.runId !== f.request.runId) return;
|
||||
await control.cancelRunInternal(scope.runId, "Startup cleanup retry");
|
||||
await acknowledgeNativeStartupCleanup(db, scope);
|
||||
} });
|
||||
expect((await f.reload()).status).toBe("cancelled");
|
||||
expect(await f.owner()).toMatchObject({ leaseOwner: null, controlDeadlineAt: null });
|
||||
expect(await schedule.finalization()).toBeNull();
|
||||
}
|
||||
});
|
||||
|
||||
it("discovers a crash after warm prepare dispatch but before a cleanup intent was recorded", async () => {
|
||||
const f = await fixture(true), lease = await f.lease();
|
||||
await db.update(environmentLeases).set({ metadata: { ...lease.metadata, runnerTaskStartup: {
|
||||
contextId: f.request.contextId, deadlineAtMs: f.request.deadlineAtMs, runId: f.request.runId,
|
||||
} } }).where(eq(environmentLeases.id, lease.id));
|
||||
await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 1_000) })
|
||||
.where(eq(nativeRunFinalizations.runId, f.request.runId));
|
||||
expect(readNativeStartupCleanup(await f.reload())).toBeNull();
|
||||
const onClaim = vi.fn();
|
||||
const result = await claimNativeRestartRecoveries({ db, restartKind: "hard", runIds: [f.request.runId],
|
||||
controller: { bootId: randomUUID(), pid: process.pid, processStartedAt: new Date() }, onClaim });
|
||||
expect(result).toEqual([{ kind: "blocked", runId: f.request.runId, reason: NATIVE_STARTUP_CLEANUP_PENDING }]);
|
||||
expect(onClaim).not.toHaveBeenCalled();
|
||||
expect(readNativeStartupCleanup(await f.reload())).toMatchObject({ contextId: f.request.contextId,
|
||||
state: "pending", requiresPreparationStop: true });
|
||||
expect(isNativeRunnerOwnershipHeld(await f.reload())).toBe(false);
|
||||
});
|
||||
|
||||
it.each(["wrong_run", "future_deadline", "live_owner", "checkpointed", "missing_marker"])(
|
||||
"does not infer a timed-out startup from %s", async variant => {
|
||||
const f = await fixture(true), lease = await f.lease();
|
||||
if (variant !== "missing_marker") await db.update(environmentLeases).set({ metadata: { ...lease.metadata,
|
||||
runnerTaskStartup: { contextId: f.request.contextId, runId: variant === "wrong_run" ? randomUUID() : f.request.runId,
|
||||
deadlineAtMs: variant === "future_deadline" ? Date.now() + 60_000 : f.request.deadlineAtMs },
|
||||
} }).where(eq(environmentLeases.id, lease.id));
|
||||
await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() + (variant === "live_owner" ? 60_000 : -1_000)) })
|
||||
.where(eq(nativeRunFinalizations.runId, f.request.runId));
|
||||
if (variant === "checkpointed") await db.update(heartbeatRuns).set({ sessionIdAfter: "current-run-checkpoint" })
|
||||
.where(eq(heartbeatRuns.id, f.request.runId));
|
||||
await claimNativeRestartRecoveries({ db, restartKind: "hard", runIds: [f.request.runId],
|
||||
controller: { bootId: randomUUID(), pid: process.pid, processStartedAt: new Date() } });
|
||||
expect(readNativeStartupCleanup(await f.reload())).toBeNull();
|
||||
});
|
||||
|
||||
it("persists an audited cancellation intent that a new database handle can recover without a generic hold", async () => {
|
||||
const f = await fixture();
|
||||
expect(await requestNativeStartupCleanup(db, f.request)).toBe(true);
|
||||
const run = await f.reload(), marker = readNativeStartupCleanup(run)!;
|
||||
expect(marker).toMatchObject({ state: "pending", contextId: f.request.contextId, leaseOwner: f.request.leaseOwner,
|
||||
controllerGeneration: 1, attempt: 1, taskId: (await f.lease()).providerLeaseId });
|
||||
expect(run).toMatchObject({ status: "running", errorCode: NATIVE_STARTUP_CLEANUP_PENDING });
|
||||
expect(run.resultJson?.startupCancellation).toMatchObject({ requestedAt: expect.any(String) });
|
||||
expect(isNativeRunnerOwnershipHeld(run)).toBe(false);
|
||||
expect(await db.select({ id: activityLog.id }).from(activityLog).where(and(eq(activityLog.id, marker.auditId),
|
||||
eq(activityLog.companyId, companyId), eq(activityLog.action, "native.startup_cleanup_requested")))).toHaveLength(1);
|
||||
const restarted = createDb(temporary.connectionString), release = vi.fn(f.stopReceipt);
|
||||
await verifyNativeStartupCleanup(restarted, await f.reload(), release);
|
||||
await verifyNativeStartupCleanup(restarted, await f.reload(), release);
|
||||
expect(release).toHaveBeenCalledOnce();
|
||||
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("complete");
|
||||
await acknowledgeNativeStartupCleanup(restarted, { companyId, runId: run.id });
|
||||
expect((await f.owner()).controlDeadlineAt).not.toBeNull();
|
||||
await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, run.id));
|
||||
await acknowledgeNativeStartupCleanup(restarted, { companyId, runId: run.id });
|
||||
expect(await f.owner()).toMatchObject({ leaseOwner: null, leaseExpiresAt: null, controlDeadlineAt: null, recoveryState: null });
|
||||
});
|
||||
|
||||
it("keeps cleanup bound to the explicit current lease when a released alias belongs to the same run", async () => {
|
||||
const f = await fixture(true), current = await f.lease();
|
||||
const aliasId = randomUUID();
|
||||
await db.insert(environmentLeases).values({ ...current, id: aliasId, providerLeaseId: randomUUID(),
|
||||
status: "released", releasedAt: new Date(), cleanupStatus: "success" });
|
||||
expect(await requestNativeStartupCleanup(db, { ...f.request, leaseId: current.id })).toBe(true);
|
||||
expect(readNativeStartupCleanup(await f.reload())).toMatchObject({ leaseId: current.id, taskId: current.providerLeaseId });
|
||||
expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, aliasId)))[0]?.metadata?.runnerTaskStartupCleanup).toBeUndefined();
|
||||
});
|
||||
|
||||
it("recovers an exact Stop receipt committed before Core records completion", async () => {
|
||||
const f = await fixture();
|
||||
await requestNativeStartupCleanup(db, f.request);
|
||||
await f.stopReceipt();
|
||||
const release = vi.fn(async () => { throw new Error("Already retired task must not be released twice"); });
|
||||
await verifyNativeStartupCleanup(createDb(temporary.connectionString), await f.reload(), release);
|
||||
expect(release).not.toHaveBeenCalled();
|
||||
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("complete");
|
||||
});
|
||||
|
||||
it("keeps accelerated preparation pending when only the provider has stopped", async () => {
|
||||
const f = await fixture(true);
|
||||
await requestNativeStartupCleanup(db, f.request);
|
||||
await expect(verifyNativeStartupCleanup(db, await f.reload(), () => f.stopReceipt())).rejects.toThrow(/pending|preparation/i);
|
||||
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("pending");
|
||||
expect((await f.owner()).leaseOwner).toBe(f.request.leaseOwner);
|
||||
});
|
||||
|
||||
it("automatically finishes pending cleanup after a late real worker reply and exact preparation Stop proof", async () => {
|
||||
const f = await fixture(true), lease = await f.lease();
|
||||
const entrypointPath = path.join(home.home, `cleanup-worker-${f.request.runId}.cjs`);
|
||||
await writeFile(entrypointPath, `
|
||||
const rl=require("node:readline").createInterface({input:process.stdin,crlfDelay:Infinity});
|
||||
const held=[];let stopped=false;
|
||||
const send=(id,result)=>process.stdout.write(JSON.stringify({jsonrpc:"2.0",id,result})+"\\n");
|
||||
rl.on("line",line=>{const m=JSON.parse(line);
|
||||
if(m.method==="initialize")send(m.id,{ok:true,supportedMethods:["environmentTask","configChanged"]});
|
||||
else if(m.method==="environmentTask"){
|
||||
const kind=m.params.operation.kind;
|
||||
if(kind==="prepare")held.push(m.id);
|
||||
else if(kind==="status")send(m.id,{kind:"status",taskId:m.params.taskId,phase:stopped?"cancelled":"running",executionStopped:stopped,preparationStopped:stopped&&held.length===0});
|
||||
else {if(kind==="stop")stopped=true;send(m.id,{kind:"accepted",taskId:m.params.taskId});}
|
||||
}else if(m.method==="configChanged"){for(const id of held.splice(0))send(id,{});send(m.id,{});}
|
||||
else if(m.method==="shutdown"){send(m.id,{});setImmediate(()=>process.exit(0));}
|
||||
});
|
||||
`, { mode: 0o600 });
|
||||
const manifest: PaperclipPluginManifestV1 = { id: "test.startup-cleanup", apiVersion: 1, version: "1.0.0",
|
||||
displayName: "Cleanup fixture", description: "Cleanup fixture", author: "Paperclip", categories: ["automation"],
|
||||
capabilities: [], entrypoints: { worker: "worker.cjs" } };
|
||||
const manager = createPluginWorkerManager();
|
||||
await manager.startWorker(manifest.id, { entrypointPath, manifest, config: {},
|
||||
instanceInfo: { instanceId: "cleanup", hostVersion: "1.0.0" }, apiVersion: 1, hostHandlers: {} });
|
||||
const context = new NativeExecutionContext({ binding: { companyId, agentId, runId: f.request.runId } } as NativeExecutionInput, 100);
|
||||
const params = { companyId, agentId, runId: f.request.runId, environmentId: lease.environmentId, issueId: lease.issueId,
|
||||
driverKey: "tasks", taskId: lease.providerLeaseId!, config: {}, projectIds: [], lease: { providerLeaseId: lease.providerLeaseId! } };
|
||||
const operation = { kind: "prepare", projectIds: [], requirements: { schema: "paperclip.runner.launch-requirements.v1",
|
||||
identity: { runnerInstanceId: "fixture", environmentLeaseId: lease.id, runId: f.request.runId,
|
||||
normalizedSessionId: "session", turnId: "turn", itemId: "item" }, provider: "codex", model: null,
|
||||
instructions: "Synthetic", runtimeContext: null, settings: {}, approvalPolicy: "never", readOnly: false,
|
||||
preparedConversation: false, includeCollaborationModeInstructions: false } } as PluginEnvironmentTaskOperation;
|
||||
const release = async () => {
|
||||
await manager.call(manifest.id, "environmentTask", { ...params, operation: { kind: "stop" } });
|
||||
const status = await manager.call(manifest.id, "environmentTask", { ...params, operation: { kind: "status" } });
|
||||
if (status.kind === "status" && status.executionStopped && status.preparationStopped) {
|
||||
await manager.call(manifest.id, "environmentTask", { ...params, operation: { kind: "complete" } });
|
||||
await f.stopReceipt({}, true);
|
||||
}
|
||||
};
|
||||
try {
|
||||
await expect(manager.call(manifest.id, "environmentTask", { ...params, operation }, 10, undefined, context.trackWorker())).rejects.toThrow("timed out");
|
||||
if (!context.signal.aborted) await new Promise<void>(resolve => context.signal.addEventListener("abort", () => resolve(), { once: true }));
|
||||
expect(context.pending).toBe(true);
|
||||
expect(await requestNativeStartupCleanup(db, { ...f.request, contextId: context.id, deadlineAtMs: context.deadlineAtMs })).toBe(true);
|
||||
expect(isNativeRunnerOwnershipHeld(await f.reload())).toBe(false);
|
||||
await expect(cancelNativeSession(f.request.runId, "Startup timeout", { db, scope: "run" })).resolves.toMatchObject({ auditId: expect.any(String) });
|
||||
await expect(verifyNativeStartupCleanup(db, await f.reload(), release)).rejects.toThrow("pending");
|
||||
expect((await f.reload()).status).toBe("running");
|
||||
await manager.call(manifest.id, "configChanged", { companyId, config: { release: true } });
|
||||
expect(context.pending).toBe(false);
|
||||
await verifyNativeStartupCleanup(createDb(temporary.connectionString), await f.reload(), release);
|
||||
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("complete");
|
||||
expect((await f.lease()).metadata?.runnerTaskPreparationStopped).toBe(true);
|
||||
} finally { context.finish(); await manager.stopAll(); }
|
||||
});
|
||||
|
||||
it.each([false, true])("verifies an abandoned ephemeral startup without an inactive-owner acknowledgement (receipt=%s)", async stopped => {
|
||||
const prior = process.env.PAPERCLIP_EPHEMERAL_SERVER;
|
||||
process.env.PAPERCLIP_EPHEMERAL_SERVER = "true";
|
||||
try {
|
||||
const f = await fixture(true);
|
||||
await requestNativeStartupCleanup(db, f.request);
|
||||
const deps = callbacks(db);
|
||||
deps.verifyStartupCleanup = vi.fn(run => verifyNativeStartupCleanup(db, run, async () => { if (stopped) await f.stopReceipt({}, true); }));
|
||||
const control = createHeartbeatRunControl(db, deps);
|
||||
if (stopped) {
|
||||
expect(await control.cancelRunInternal(f.request.runId, "Startup timeout")).toMatchObject({ status: "cancelled" });
|
||||
expect((await f.owner()).leaseOwner).toBeNull();
|
||||
} else {
|
||||
await expect(control.cancelRunInternal(f.request.runId, "Startup timeout")).rejects.toThrow("Task cleanup is still pending");
|
||||
expect((await f.reload()).status).toBe("running");
|
||||
expect((await f.owner()).leaseOwner).toBe(f.request.leaseOwner);
|
||||
}
|
||||
expect(deps.verifyStartupCleanup).toHaveBeenCalled();
|
||||
} finally {
|
||||
if (prior === undefined) delete process.env.PAPERCLIP_EPHEMERAL_SERVER;
|
||||
else process.env.PAPERCLIP_EPHEMERAL_SERVER = prior;
|
||||
}
|
||||
}, 30_000);
|
||||
|
||||
it("terminalizes through assembled run control only after verified cleanup succeeds", async () => {
|
||||
const f = await fixture(true);
|
||||
await requestNativeStartupCleanup(db, f.request);
|
||||
let drained = false;
|
||||
const deps = callbacks(db);
|
||||
deps.verifyStartupCleanup = run => verifyNativeStartupCleanup(db, run, () => f.stopReceipt({}, drained));
|
||||
const control = createHeartbeatRunControl(db, deps);
|
||||
await expect(control.cancelRunInternal(f.request.runId, "Startup timeout")).rejects.toThrow("pending");
|
||||
expect(await f.reload()).toMatchObject({ status: "running", errorCode: NATIVE_STARTUP_CLEANUP_PENDING });
|
||||
expect((await f.owner()).leaseOwner).toBe(f.request.leaseOwner);
|
||||
drained = true;
|
||||
const result = await control.cancelRunInternal(f.request.runId, "Startup timeout");
|
||||
expect(result).toMatchObject({ status: "cancelled" });
|
||||
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("complete");
|
||||
expect(await f.owner()).toMatchObject({ leaseOwner: null, leaseExpiresAt: null, controlDeadlineAt: null });
|
||||
});
|
||||
|
||||
it("joins cleanup recorded while an operator Stop waits for native startup", async () => {
|
||||
const f = await fixture(true);
|
||||
let entered!: () => void, resume!: () => void;
|
||||
const waiting = new Promise<void>(resolve => { entered = resolve; });
|
||||
const settled = new Promise<void>(resolve => { resume = resolve; });
|
||||
const cancel = nativeSessionExecutor.cancelNativeSession;
|
||||
const spy = vi.spyOn(nativeSessionExecutor, "cancelNativeSession").mockImplementation(async (...args) => {
|
||||
const result = await cancel(...args);
|
||||
if (args[0] === f.request.runId) { entered(); await settled; }
|
||||
return result;
|
||||
});
|
||||
let drained = false;
|
||||
const deps = callbacks(db);
|
||||
deps.verifyStartupCleanup = vi.fn(run => verifyNativeStartupCleanup(db, run, () => f.stopReceipt({}, drained)));
|
||||
const control = createHeartbeatRunControl(db, deps);
|
||||
try {
|
||||
const stopping = control.cancelRunInternal(f.request.runId, "Operator Stop");
|
||||
const outcome = stopping.then(value => ({ value }), error => ({ error }));
|
||||
await waiting;
|
||||
expect(await requestNativeStartupCleanup(db, f.request)).toBe(true);
|
||||
resume();
|
||||
expect(await outcome).toMatchObject({ error: expect.objectContaining({ message: "Task cleanup is still pending" }) });
|
||||
expect(deps.verifyStartupCleanup).toHaveBeenCalledOnce();
|
||||
expect((await f.reload()).status).toBe("running");
|
||||
expect((await f.owner()).leaseOwner).toBe(f.request.leaseOwner);
|
||||
drained = true;
|
||||
expect(await control.cancelRunInternal(f.request.runId, "Operator Stop")).toMatchObject({ status: "cancelled" });
|
||||
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("complete");
|
||||
expect((await f.owner()).leaseOwner).toBeNull();
|
||||
} finally { resume(); spy.mockRestore(); }
|
||||
});
|
||||
|
||||
it("rejects a cancellation commit when cleanup arrives after its final read", async () => {
|
||||
const f = await fixture(true);
|
||||
const deps = callbacks(db), commit = deps.setRunStatusFromLive.getMockImplementation()!;
|
||||
deps.setRunStatusFromLive.mockImplementationOnce(async (...args) => {
|
||||
expect(await requestNativeStartupCleanup(db, f.request)).toBe(true);
|
||||
return commit(...args);
|
||||
});
|
||||
deps.verifyStartupCleanup = vi.fn(run => verifyNativeStartupCleanup(db, run, () => f.stopReceipt({}, true)));
|
||||
const control = createHeartbeatRunControl(db, deps);
|
||||
expect(await control.cancelRunInternal(f.request.runId, "Operator Stop")).toMatchObject({ status: "running" });
|
||||
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("pending");
|
||||
expect((await f.owner()).leaseOwner).toBe(f.request.leaseOwner);
|
||||
expect(deps.startNextQueuedRunForAgent).not.toHaveBeenCalled();
|
||||
expect(await control.cancelRunInternal(f.request.runId, "Operator Stop")).toMatchObject({ status: "cancelled" });
|
||||
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("complete");
|
||||
expect((await f.owner()).leaseOwner).toBeNull();
|
||||
});
|
||||
|
||||
it("restarts the real cancellation flow after remote Stop committed before Core acknowledgement", async () => {
|
||||
const f = await fixture(true);
|
||||
await requestNativeStartupCleanup(db, f.request);
|
||||
await f.stopReceipt({}, true);
|
||||
const restarted = createDb(temporary.connectionString), release = vi.fn(async () => { throw new Error("Stop receipt already persisted"); });
|
||||
const deps = callbacks(restarted);
|
||||
deps.verifyStartupCleanup = run => verifyNativeStartupCleanup(restarted, run, release);
|
||||
const control = createHeartbeatRunControl(restarted, deps);
|
||||
const reconcile = vi.fn(async (scope: { companyId: string; runId: string }) => {
|
||||
if (scope.runId !== f.request.runId) return;
|
||||
expect(scope.companyId).toBe(companyId);
|
||||
await control.cancelRunInternal(scope.runId, "Startup timeout recovered after restart");
|
||||
await acknowledgeNativeStartupCleanup(restarted, scope);
|
||||
});
|
||||
await reconcileAbandonedExecutionControl(restarted, new Date(Date.now() + 1_000), { startupCleanup: reconcile });
|
||||
expect(reconcile.mock.calls.some(([scope]) => scope.runId === f.request.runId)).toBe(true);
|
||||
expect(release).not.toHaveBeenCalled();
|
||||
expect((await f.reload()).status).toBe("cancelled");
|
||||
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("complete");
|
||||
expect(await f.owner()).toMatchObject({ leaseOwner: null, leaseExpiresAt: null, controlDeadlineAt: null });
|
||||
});
|
||||
|
||||
it.each(["providerLeaseId", "runId", "leaseId"])("rejects a Stop receipt for another %s", async field => {
|
||||
const f = await fixture();
|
||||
await requestNativeStartupCleanup(db, f.request);
|
||||
await expect(verifyNativeStartupCleanup(db, await f.reload(), () => f.stopReceipt({ [field]: randomUUID() }))).rejects.toThrow("pending");
|
||||
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("pending");
|
||||
});
|
||||
|
||||
it("does not issue a Stop against a newer controller generation", async () => {
|
||||
const f = await fixture();
|
||||
await requestNativeStartupCleanup(db, f.request);
|
||||
await db.update(nativeRunFinalizations).set({ controllerGeneration: 2 }).where(eq(nativeRunFinalizations.runId, f.request.runId));
|
||||
const release = vi.fn(f.stopReceipt);
|
||||
await expect(verifyNativeStartupCleanup(db, await f.reload(), release)).rejects.toThrow("ownership changed");
|
||||
expect(release).not.toHaveBeenCalled();
|
||||
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("pending");
|
||||
});
|
||||
|
||||
it("rejects a stale generation that changes while Stop is in flight", async () => {
|
||||
const f = await fixture();
|
||||
await requestNativeStartupCleanup(db, f.request);
|
||||
await expect(verifyNativeStartupCleanup(db, await f.reload(), async () => {
|
||||
await f.stopReceipt();
|
||||
await db.update(nativeRunFinalizations).set({ controllerGeneration: 2 }).where(eq(nativeRunFinalizations.runId, f.request.runId));
|
||||
})).rejects.toThrow("ownership changed");
|
||||
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("pending");
|
||||
});
|
||||
|
||||
it("preserves a final result that wins while verified cleanup is in flight", async () => {
|
||||
const f = await fixture();
|
||||
await requestNativeStartupCleanup(db, f.request);
|
||||
let resultId: string;
|
||||
await expect(verifyNativeStartupCleanup(db, await f.reload(), async () => {
|
||||
await f.stopReceipt();
|
||||
resultId = await f.acceptedResult();
|
||||
})).rejects.toThrow("ownership changed");
|
||||
expect(await f.owner()).toMatchObject({ resultId: resultId!, phase: "workspace_finalizing" });
|
||||
expect(readNativeStartupCleanup(await f.reload())?.state).toBe("pending");
|
||||
});
|
||||
|
||||
it.each(["native_adopted_runner_authentication_timeout", "native_execution_ownership_unverified"])("preserves the existing %s hold", async errorCode => {
|
||||
const f = await fixture();
|
||||
await db.update(heartbeatRuns).set({ errorCode, nativePhase: "terminal_failure" }).where(eq(heartbeatRuns.id, f.request.runId));
|
||||
expect(await requestNativeStartupCleanup(db, f.request)).toBe(false);
|
||||
const current = await f.reload();
|
||||
expect(current).toMatchObject({ status: "running", errorCode, nativePhase: "terminal_failure" });
|
||||
expect(readNativeStartupCleanup(current)).toBeNull();
|
||||
});
|
||||
|
||||
it("does not replace a final result with a timeout intent", async () => {
|
||||
const f = await fixture();
|
||||
const resultId = await f.acceptedResult();
|
||||
expect(await requestNativeStartupCleanup(db, f.request)).toBe(false);
|
||||
expect((await f.owner()).resultId).toBe(resultId);
|
||||
expect(readNativeStartupCleanup(await f.reload())).toBeNull();
|
||||
});
|
||||
it("promotes a pre-capability context before an ordinary deadline after the audited intent write fails", async () => {
|
||||
const f = await fixture();
|
||||
await saveEarlyContext(f);
|
||||
await db.execute(sql`create function reject_startup_intent() returns trigger language plpgsql as $$ begin
|
||||
if new.action = 'native.startup_cleanup_requested' then raise exception 'Startup intent write unavailable'; end if; return new; end $$`);
|
||||
await db.execute(sql`create trigger reject_startup_intent before insert on activity_log for each row execute function reject_startup_intent()`);
|
||||
try { await expect(requestNativeStartupCleanup(db, f.request)).rejects.toThrow(); }
|
||||
finally { await db.execute(sql`drop trigger reject_startup_intent on activity_log`); await db.execute(sql`drop function reject_startup_intent()`); }
|
||||
expect(readNativeStartupCleanup(await f.reload())).toBeNull();
|
||||
await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 1_000) }).where(eq(nativeRunFinalizations.runId, f.request.runId));
|
||||
await db.update(heartbeatRuns).set({ executionControlDeadlineAt: new Date(Date.now() - 1_000) }).where(eq(heartbeatRuns.id, f.request.runId));
|
||||
await reconcileAbandonedExecutionControl(db);
|
||||
expect(readNativeStartupCleanup(await f.reload())).toMatchObject({ contextId: f.request.contextId, requiresPreparationStop: false, state: "pending" });
|
||||
expect((await f.reload()).status).toBe("running");
|
||||
expect((await f.owner()).leaseOwner).toBe(f.request.leaseOwner);
|
||||
await verifyNativeStartupCleanup(db, await f.reload(), () => f.stopReceipt());
|
||||
await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, f.request.runId));
|
||||
await acknowledgeNativeStartupCleanup(db, f.request);
|
||||
});
|
||||
|
||||
it.each([false, true])("recovers an early context without replacing stronger preparation proof (prepared=%s)", async prepared => {
|
||||
const f = await fixture();
|
||||
await saveEarlyContext(f);
|
||||
if (prepared) { const lease = await f.lease(); await db.update(environmentLeases).set({ metadata: { ...lease.metadata,
|
||||
runnerTaskStartup: { contextId: f.request.contextId, deadlineAtMs: f.request.deadlineAtMs, runId: f.request.runId } } }).where(eq(environmentLeases.id, lease.id)); }
|
||||
await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 1_000) }).where(eq(nativeRunFinalizations.runId, f.request.runId));
|
||||
const onClaim = vi.fn();
|
||||
const result = await claimNativeRestartRecoveries({ db, restartKind: "hard", runIds: [f.request.runId],
|
||||
controller: { bootId: randomUUID(), pid: process.pid, processStartedAt: new Date() }, onClaim });
|
||||
expect(result).toEqual([{ kind: "blocked", runId: f.request.runId, reason: NATIVE_STARTUP_CLEANUP_PENDING }]);
|
||||
expect(onClaim).not.toHaveBeenCalled();
|
||||
expect(readNativeStartupCleanup(await f.reload())).toMatchObject({ requiresPreparationStop: prepared, state: "pending" });
|
||||
await verifyNativeStartupCleanup(db, await f.reload(), () => f.stopReceipt({}, prepared));
|
||||
await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, f.request.runId));
|
||||
await acknowledgeNativeStartupCleanup(db, f.request);
|
||||
});
|
||||
|
||||
it.each([false, true])("defers an early context to a concrete owner deadline (locally active=%s)", async active => {
|
||||
const f = await fixture();
|
||||
await saveEarlyContext(f);
|
||||
const owner = await f.owner();
|
||||
const result = await claimNativeRestartRecoveries({ db, restartKind: "hard", runIds: [f.request.runId],
|
||||
liveRunExecutions: active ? new Set([f.request.runId]) : undefined,
|
||||
controller: { bootId: randomUUID(), pid: process.pid, processStartedAt: new Date() } });
|
||||
expect(result).toMatchObject([{ kind: "awaiting_evidence", runId: f.request.runId, retryAt: owner.leaseExpiresAt }]);
|
||||
expect(readNativeStartupCleanup(await f.reload())).toBeNull();
|
||||
});
|
||||
|
||||
it("does not turn an early context into cleanup after the current-run checkpoint", async () => {
|
||||
const f = await fixture();
|
||||
await saveEarlyContext(f);
|
||||
await db.update(heartbeatRuns).set({ sessionIdAfter: "current-checkpoint" }).where(eq(heartbeatRuns.id, f.request.runId));
|
||||
await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 1_000) }).where(eq(nativeRunFinalizations.runId, f.request.runId));
|
||||
await claimNativeRestartRecoveries({ db, restartKind: "hard", runIds: [f.request.runId],
|
||||
controller: { bootId: randomUUID(), pid: process.pid, processStartedAt: new Date() } });
|
||||
expect(readNativeStartupCleanup(await f.reload())).toBeNull();
|
||||
});
|
||||
|
||||
|
||||
it("locks the selected lease before choosing the stronger preparation proof", async () => {
|
||||
const f = await fixture(), lease = await f.lease();
|
||||
let changed!: () => void, commit!: () => void;
|
||||
const written = new Promise<void>(resolve => { changed = resolve; });
|
||||
const release = new Promise<void>(resolve => { commit = resolve; });
|
||||
const preparing = db.transaction(async tx => {
|
||||
await tx.update(environmentLeases).set({ metadata: { ...lease.metadata, runnerTaskStartup: {
|
||||
contextId: f.request.contextId, deadlineAtMs: f.request.deadlineAtMs, runId: f.request.runId,
|
||||
} } }).where(eq(environmentLeases.id, lease.id));
|
||||
changed();
|
||||
await release;
|
||||
});
|
||||
await written;
|
||||
const requesting = requestNativeStartupCleanup(db, f.request);
|
||||
try {
|
||||
await vi.waitFor(async () => {
|
||||
const rows = await db.execute(sql`select query from pg_stat_activity where wait_event_type = 'Lock'
|
||||
and query like 'select%from "environment_leases"%'`);
|
||||
expect(rows.length).toBeGreaterThan(0);
|
||||
});
|
||||
} finally { commit(); }
|
||||
await preparing;
|
||||
await requesting;
|
||||
expect(readNativeStartupCleanup(await f.reload())).toMatchObject({ requiresPreparationStop: true });
|
||||
await expect(verifyNativeStartupCleanup(db, await f.reload(), () => f.stopReceipt())).rejects.toThrow("pending");
|
||||
await verifyNativeStartupCleanup(db, await f.reload(), () => f.stopReceipt({}, true));
|
||||
});
|
||||
|
||||
it("clears only its owned startup context before fallback and leaves no expired cleanup authority", async () => {
|
||||
const f = await fixture();
|
||||
await saveEarlyContext(f);
|
||||
const scope = { companyId, runId: f.request.runId, contextId: f.request.contextId,
|
||||
leaseOwner: f.request.leaseOwner, controllerGeneration: 1 };
|
||||
await expect(clearNativeStartupContext(db, { ...scope, controllerGeneration: 2 })).rejects.toThrow("lost authority");
|
||||
expect((await f.reload()).runnerProfileJson?.nativeStartupContext).toBeDefined();
|
||||
await clearNativeStartupContext(db, scope);
|
||||
expect((await f.reload()).runnerProfileJson?.nativeStartupContext).toBeUndefined();
|
||||
await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 1_000) }).where(eq(nativeRunFinalizations.runId, f.request.runId));
|
||||
await claimNativeRestartRecoveries({ db, restartKind: "hard", runIds: [f.request.runId],
|
||||
controller: { bootId: randomUUID(), pid: process.pid, processStartedAt: new Date() } });
|
||||
expect(readNativeStartupCleanup(await f.reload())).toBeNull();
|
||||
});
|
||||
|
||||
it.each(["cleanup", "terminal"])("does not clear the startup breadcrumb after %s wins", async winner => {
|
||||
const f = await fixture();
|
||||
await saveEarlyContext(f);
|
||||
if (winner === "cleanup") await requestNativeStartupCleanup(db, f.request);
|
||||
else await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, f.request.runId));
|
||||
await expect(clearNativeStartupContext(db, { companyId, runId: f.request.runId, contextId: f.request.contextId,
|
||||
leaseOwner: f.request.leaseOwner, controllerGeneration: 1 })).rejects.toThrow("lost authority");
|
||||
expect((await f.reload()).runnerProfileJson?.nativeStartupContext).toBeDefined();
|
||||
if (winner === "cleanup") expect(readNativeStartupCleanup(await f.reload())).toMatchObject({ state: "pending" });
|
||||
});
|
||||
|
||||
it("requires exact task Stop proof when retrying cleanup of a never-dispatched replacement", async () => {
|
||||
const f = await fixture(true), original = await f.lease(), pluginId = randomUUID();
|
||||
await db.insert(plugins).values({ id: pluginId, pluginKey: pluginId, packageName: "fixture-abandoned-task", version: "1", status: "ready",
|
||||
manifestJson: { capabilities: ["environment.drivers.register"], environmentDrivers: [{ driverKey: "tasks", supportsTasks: true }] } as never });
|
||||
await db.update(environments).set({ companyId, config: { pluginKey: pluginId, driverKey: "tasks", driverConfig: {} } })
|
||||
.where(eq(environments.id, original.environmentId!));
|
||||
const fresh = await createOwnedSandbox(db, { companyId, environmentId: original.environmentId!, heartbeatRunId: f.request.runId,
|
||||
issueId: original.issueId, provider: original.provider!, pluginId, driverKey: "tasks", config: {}, requestKey: original.providerLeaseId!,
|
||||
metadata: { driver: "plugin", pluginId, pluginKey: pluginId, driverKey: "tasks", runnerTaskLease: true, agentId,
|
||||
runnerTaskCreationContextId: f.request.contextId } }, async () => ({ providerLeaseId: "late-never-dispatched" }));
|
||||
await queueSandboxCreationCleanup(db, { id: String(fresh.metadata!.creationOperationId), companyId,
|
||||
environmentId: original.environmentId!, providerLeaseId: fresh.providerLeaseId });
|
||||
const creation = async () => (await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.id, String(fresh.metadata!.creationOperationId))))[0]!;
|
||||
const replacement = async () => (await environmentService(db).getLeaseById((await creation()).leaseId!))!;
|
||||
expect((await replacement()).metadata?.runnerTaskDispatched).not.toBe(true);
|
||||
await requestNativeStartupCleanup(db, { ...f.request, leaseId: original.id });
|
||||
await f.stopReceipt({}, true);
|
||||
let proof: "void" | "running" | "foreign" | "exact" = "void";
|
||||
const calls: string[] = [];
|
||||
const workers = { isRunning: () => true, getWorker: () => ({ supportedMethods: ["environmentTask", "environmentReleaseLease"] }),
|
||||
drainStartup: vi.fn(async () => undefined),
|
||||
call: async (_plugin: string, method: string, params: any) => {
|
||||
calls.push(method === "environmentTask" ? params.operation.kind : method);
|
||||
if (method === "environmentReleaseLease") return undefined;
|
||||
if (params.operation.kind === "status") return proof === "void" ? undefined : {
|
||||
kind: "status", taskId: proof === "foreign" ? "another-task" : params.taskId,
|
||||
phase: proof === "running" ? "running" : "cancelled", executionStopped: proof !== "running",
|
||||
};
|
||||
return { kind: "accepted", taskId: params.taskId };
|
||||
},
|
||||
} as unknown as PluginWorkerManager;
|
||||
const runtime = environmentRuntimeService(db, { pluginWorkerManager: workers });
|
||||
const recovery = createHeartbeatRecovery(db, { ...callbacks(db), environmentRuntime: runtime,
|
||||
environmentsSvc: environmentService(db), runtimeEnv: {}, activeRunExecutionPromises: new Set<Promise<void>>(),
|
||||
acknowledgeRemoteStop: async () => undefined, resumeRemoteStopComments: async () => undefined,
|
||||
} as unknown as HeartbeatRecoveryDependencies);
|
||||
const stopDeps = callbacks(db);
|
||||
stopDeps.verifyStartupCleanup = run => verifyNativeStartupCleanup(db, run, async () => {});
|
||||
const control = createHeartbeatRunControl(db, stopDeps);
|
||||
for (const missing of ["void", "running", "foreign"] as const) {
|
||||
proof = missing;
|
||||
const swept = await recovery.sweepPendingCleanupLeases({ explicitRetry: { companyId, runId: f.request.runId, actorId: "operator" } });
|
||||
const cancelled = await control.cancelRunInternal(f.request.runId, "Startup cleanup retry").then(() => true, () => false);
|
||||
const lease = await replacement();
|
||||
expect({ destroyed: swept.destroyed, status: lease.status, verifiedReceipt: hasRemoteTerminationReceipt(lease),
|
||||
creationState: (await creation()).state, cancelled, leaseOwner: (await f.owner()).leaseOwner }).toEqual({
|
||||
destroyed: 0, status: "pending_cleanup", verifiedReceipt: false, creationState: "cleanup_pending", cancelled: false, leaseOwner: f.request.leaseOwner,
|
||||
});
|
||||
}
|
||||
proof = "exact";
|
||||
expect((await recovery.sweepPendingCleanupLeases({ explicitRetry: { companyId, runId: f.request.runId, actorId: "operator" } })).destroyed).toBe(1);
|
||||
expect(calls).toContain("stop");
|
||||
expect(calls).toContain("status");
|
||||
expect(calls).toContain("complete");
|
||||
expect(workers!.drainStartup).toHaveBeenCalledWith(f.request.contextId);
|
||||
expect(hasRemoteTerminationReceipt(await replacement())).toBe(true);
|
||||
expect((await creation()).state).toBe("resolved");
|
||||
expect(await control.cancelRunInternal(f.request.runId, "Startup cleanup retry")).toMatchObject({ status: "cancelled" });
|
||||
expect((await f.owner()).leaseOwner).toBeNull();
|
||||
});
|
||||
|
||||
});
|
||||
@@ -332,6 +332,36 @@ describe("clampProviderSpanAttributes", () => {
|
||||
});
|
||||
|
||||
describe("nested worker timing receipts",()=>{
|
||||
it("keeps lease verification beneath its worker and clamps provider state", () => {
|
||||
mockRecordSpan.mockReset();
|
||||
const rootParent = { traceId: "0af7651916cd43dd8448eb211c80319c", spanId: "1111111111111111", traceFlags: 1 };
|
||||
const verifyParent = { ...rootParent, spanId: "2222222222222222" };
|
||||
mockRecordSpan.mockReturnValueOnce(rootParent).mockReturnValueOnce(verifyParent);
|
||||
const now = Date.now();
|
||||
recordWorkerProviderSpan({ name: "plugin.worker", startTimeMs: now - 100, endTimeMs: now, children: [
|
||||
{ name: "lease.verify", startTimeMs: now - 90, endTimeMs: now - 10, children: [
|
||||
{ name: "lease.sentinel", startTimeMs: now - 80, endTimeMs: now - 20,
|
||||
attributes: { "paperclip.sandbox.startup.state": "archived", token: "private-token" } },
|
||||
] },
|
||||
] }, { traceparent: VALID_TRACEPARENT });
|
||||
expect(mockRecordSpan.mock.calls[1][0]).toMatchObject({ name: "sandbox.daytona.lease.verify", parent: rootParent });
|
||||
expect(mockRecordSpan.mock.calls[2][0]).toMatchObject({ name: "sandbox.daytona.lease.sentinel", parent: verifyParent,
|
||||
startTimeMs: now - 80, endTimeMs: now - 20, attributes: { "paperclip.sandbox.startup.state": "archived" } });
|
||||
expect(JSON.stringify(mockRecordSpan.mock.calls)).not.toContain("private-token");
|
||||
expect(clampProviderSpanAttributes({ "paperclip.sandbox.startup.state": "private-state" })).toEqual({});
|
||||
});
|
||||
|
||||
it("admits every closed lease phase and rejects dynamic phase names", () => {
|
||||
mockRecordSpan.mockReset();
|
||||
const names = ["lease.wait_for_teardown", "lease.lookup", "lease.refresh", "lease.channel_cleanup",
|
||||
"lease.start", "lease.recover", "lease.verify", "lease.workspace", "lease.sentinel", "lease.shell",
|
||||
"lease.create", "lease.expiry", "lease.cleanup", "lease.private-resource-id"];
|
||||
for (const name of names) recordWorkerProviderSpan({ name }, { traceparent: VALID_TRACEPARENT });
|
||||
expect(mockRecordSpan.mock.calls.map(call => call[0].name)).toEqual([
|
||||
...names.slice(0, -1).map(name => `sandbox.daytona.${name}`), "sandbox.daytona.other",
|
||||
]);
|
||||
});
|
||||
|
||||
it("uses host-generated parents, clamps content and bounds child clocks",()=>{
|
||||
mockRecordSpan.mockReset();
|
||||
const childParent={traceId:"0af7651916cd43dd8448eb211c80319c",spanId:"1111111111111111",traceFlags:1};
|
||||
|
||||
@@ -899,3 +899,73 @@ second-line\" status=401`,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("redaction of long letter/digit runs", () => {
|
||||
const typescriptSource = [
|
||||
"export function compactRunLogChunk(chunk: string, maxChars = 65536) {",
|
||||
" const normalized = redactSensitiveText(chunk);",
|
||||
" if (normalized.length <= maxChars) return normalized;",
|
||||
" return normalized.slice(0, maxChars);",
|
||||
"}",
|
||||
"",
|
||||
].join("\n");
|
||||
|
||||
function base64OfTypeScriptSource(length: number) {
|
||||
const repeatedSource = typescriptSource.repeat(Math.ceil(length / typescriptSource.length));
|
||||
return Buffer.from(repeatedSource).toString("base64").slice(0, length);
|
||||
}
|
||||
|
||||
function redactWithinTimeBudget(input: string) {
|
||||
const startedAt = performance.now();
|
||||
const output = redactSensitiveText(input);
|
||||
expect(performance.now() - startedAt).toBeLessThan(1_000);
|
||||
return output;
|
||||
}
|
||||
|
||||
it("redacts secret field names that carry a prefix in plain and escaped JSON", () => {
|
||||
const cases = [
|
||||
[`{"myApiKey": "plain-prefixed-secret"}`, `{"myApiKey": "${REDACTED_EVENT_VALUE}"}`],
|
||||
[`{"x_auth_token":"snake-prefixed-secret","status":"ok"}`, `{"x_auth_token":"${REDACTED_EVENT_VALUE}","status":"ok"}`],
|
||||
[`GITHUB_TOKEN: "upper-prefixed-secret"`, `GITHUB_TOKEN: "${REDACTED_EVENT_VALUE}"`],
|
||||
[`{'db_password': 'single-quoted-secret'}`, `{'db_password': '${REDACTED_EVENT_VALUE}'}`],
|
||||
[`text"apiKey":"glued-quote-secret"`, `text"apiKey":"${REDACTED_EVENT_VALUE}"`],
|
||||
[`aGVsbG8apiKey:"glued-run-secret"`, `aGVsbG8apiKey:"${REDACTED_EVENT_VALUE}"`],
|
||||
[String.raw`{\"myApiKey\":\"escaped-prefixed-secret\"}`, String.raw`{\"myApiKey\":\"${REDACTED_EVENT_VALUE}\"}`],
|
||||
[
|
||||
String.raw`{\"x_auth_token\":\"escaped-snake-secret\",\"status\":\"ok\"}`,
|
||||
String.raw`{\"x_auth_token\":\"${REDACTED_EVENT_VALUE}\",\"status\":\"ok\"}`,
|
||||
],
|
||||
[String.raw`aGVsbG8\"GITHUB_TOKEN\":\"escaped-glued-secret\"`, String.raw`aGVsbG8\"GITHUB_TOKEN\":\"${REDACTED_EVENT_VALUE}\"`],
|
||||
];
|
||||
|
||||
for (const [input, expected] of cases) {
|
||||
expect(redactSensitiveText(input)).toBe(expected);
|
||||
expect(redactSensitiveText(expected)).toBe(expected);
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps base64 of source text intact and fast while redacting a secret glued to its end", () => {
|
||||
const blob = base64OfTypeScriptSource(48 * 1024);
|
||||
expect(blob).toMatch(/^[A-Za-z0-9]+$/);
|
||||
|
||||
const shellCommand = `printf '%s' '${blob}' | base64 -d > src/compact.ts`;
|
||||
expect(redactWithinTimeBudget(shellCommand)).toBe(shellCommand);
|
||||
|
||||
const toolCallLine = JSON.stringify({ type: "tool_use", input: { command: shellCommand } });
|
||||
expect(redactWithinTimeBudget(toolCallLine)).toBe(toolCallLine);
|
||||
const serializedToolCallLine = JSON.stringify(toolCallLine);
|
||||
expect(redactWithinTimeBudget(serializedToolCallLine)).toBe(serializedToolCallLine);
|
||||
|
||||
const gluedSecret = redactWithinTimeBudget(`${blob}apiKey:"run-glued-secret"`);
|
||||
expect(gluedSecret).toBe(`${blob}apiKey:"${REDACTED_EVENT_VALUE}"`);
|
||||
});
|
||||
|
||||
it("scans long dash-separated runs in linear time", () => {
|
||||
for (const run of ["a-".repeat(60_000), "a--".repeat(40_000)]) {
|
||||
expect(redactWithinTimeBudget(`${run} Authorization: Bearer dash-run-secret`))
|
||||
.toBe(`${run} Authorization: ${REDACTED_EVENT_VALUE}`);
|
||||
expect(redactWithinTimeBudget(`${run}Authorization: Bearer glued-run-secret`))
|
||||
.toBe(`${run}Authorization: ${REDACTED_EVENT_VALUE}`);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -152,15 +152,20 @@ describeDatabase("remote workspace ordinary file routes", () => {
|
||||
});
|
||||
const runtime = { execute, syncOut } as unknown as Pick<EnvironmentRuntimeService, "execute" | "syncOut">;
|
||||
const remoteFiles = createRemoteWorkspaceFiles(db, { environmentRuntime: runtime });
|
||||
const app = filesApp(ids.company, remoteFiles);
|
||||
return { root, ids, authority, execute, syncOut, remoteFiles, app, beforeCopy: (callback: () => Promise<void>) => { beforeCopy = callback; } };
|
||||
}
|
||||
|
||||
function filesApp(companyId: string, remoteFiles: ReturnType<typeof createRemoteWorkspaceFiles>) {
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
app.use((req, _res, next) => {
|
||||
req.actor = { type: "board", userId: "local-board", companyIds: [ids.company], source: "local_implicit", isInstanceAdmin: true } as Request["actor"];
|
||||
req.actor = { type: "board", userId: "local-board", companyIds: [companyId], source: "local_implicit", isInstanceAdmin: true } as Request["actor"];
|
||||
next();
|
||||
});
|
||||
app.use("/api", fileResourceRoutes(db, { remoteFiles }));
|
||||
app.use(errorHandler);
|
||||
return { root, ids, authority, execute, syncOut, remoteFiles, app, beforeCopy: (callback: () => Promise<void>) => { beforeCopy = callback; } };
|
||||
return app;
|
||||
}
|
||||
|
||||
it("lists, resolves, previews, and downloads live bytes without a controller checkout", async () => {
|
||||
@@ -323,7 +328,7 @@ describeDatabase("remote workspace ordinary file routes", () => {
|
||||
{ workspace: "execution", path: "docs/result.md" })).rejects.toMatchObject({ status: 422, details: { code: "file_changed" } });
|
||||
});
|
||||
|
||||
async function savedFileFixture() {
|
||||
async function savedFileFixture(keepLocation = false) {
|
||||
const f = await fixture(), storageRoot = await fs.mkdtemp(path.join(os.tmpdir(), "saved-file-budget-"));
|
||||
directories.push(storageRoot);
|
||||
const storage = createStorageService(createLocalDiskStorageProvider(path.join(storageRoot, "objects")));
|
||||
@@ -335,11 +340,113 @@ describeDatabase("remote workspace ordinary file routes", () => {
|
||||
const saved = await uploadWorkspaceSnapshot({ db, storage, companyId: f.ids.company, workspaceId: f.ids.workspace,
|
||||
snapshotId: snapshot.id, archivePath, manifestDigest: archive.manifestDigest });
|
||||
await db.update(executionWorkspaceSnapshots).set({ status: "committed" }).where(eq(executionWorkspaceSnapshots.id, snapshot.id));
|
||||
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { ...f.authority, location: null, snapshotId: snapshot.id } } })
|
||||
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { ...f.authority, location: keepLocation ? f.authority.location : null, snapshotId: snapshot.id } } })
|
||||
.where(eq(executionWorkspaces.id, f.ids.workspace));
|
||||
return { ...f, storage, saved, input: { companyId: f.ids.company, workspaceId: f.ids.workspace, snapshotId: snapshot.id } };
|
||||
}
|
||||
|
||||
it("serves committed saved bytes through ordinary routes while the live allocation is unavailable", async () => {
|
||||
const f = await savedFileFixture(true);
|
||||
f.execute.mockRejectedValue(new Error("synthetic live allocation unavailable"));
|
||||
const load = createSavedWorkspaceLoader(db, f.storage), closed: Promise<void>[] = [];
|
||||
const loadSavedWorkspace = vi.fn(async (input: Parameters<typeof load>[0]) => {
|
||||
const saved = await load(input);
|
||||
let finish!: () => void, fail!: (error: unknown) => void;
|
||||
closed.push(new Promise<void>((resolve, reject) => { finish = resolve; fail = reject; }));
|
||||
return { ...saved, cleanup: async () => {
|
||||
try { await saved.cleanup(); finish(); } catch (error) { fail(error); throw error; }
|
||||
} };
|
||||
});
|
||||
const remoteFiles = createRemoteWorkspaceFiles(db, { loadSavedWorkspace, environmentRuntime: { execute: f.execute, syncOut: f.syncOut } });
|
||||
const app = filesApp(f.ids.company, remoteFiles), base = `/api/issues/${f.ids.issue}/file-resources`;
|
||||
const preview = await request(app).get(`${base}/content`).query({ workspace: "execution", path: "docs/result.md" }).expect(200);
|
||||
expect(preview.body).toMatchObject({ content: { data: "Remote useful result\n" }, resource: { source: { kind: "saved", snapshotId: f.saved.id } } });
|
||||
const listed = await request(app).get(`${base}/list`).query({ workspace: "execution", path: "docs" }).expect(200);
|
||||
expect(listed.body).toMatchObject({ workspace: { source: { kind: "saved", snapshotId: f.saved.id } } });
|
||||
const resolved = await request(app).get(`${base}/resolve`).query({ workspace: "execution", path: "docs/result.md" }).expect(200);
|
||||
expect(resolved.body).toMatchObject({ source: { kind: "saved", snapshotId: f.saved.id }, capabilities: { preview: true, download: true } });
|
||||
const download = await request(app).get(`${base}/content`).query({ workspace: "execution", path: "docs/result.md", download: true }).expect(200);
|
||||
expect(download.text).toBe("Remote useful result\n");
|
||||
expect(f.execute).not.toHaveBeenCalled(); expect(f.syncOut).not.toHaveBeenCalled();
|
||||
expect(loadSavedWorkspace).toHaveBeenCalledTimes(4);
|
||||
// A streamed response can finish at the client before its server finally block.
|
||||
await Promise.all(closed);
|
||||
for (const loaded of loadSavedWorkspace.mock.results) await expect(fs.stat((await loaded.value).root)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
const [workspace] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, f.ids.workspace));
|
||||
expect(workspace!.metadata?.remoteWorkspace).toMatchObject({ location: f.authority.location, dirty: false, writerRunId: null });
|
||||
});
|
||||
|
||||
it.each(["dirty", "writer"] as const)("keeps %s work on the live path and never substitutes its older save", async state => {
|
||||
const f = await savedFileFixture(true);
|
||||
let writerRunId: string | null = null;
|
||||
if (state === "writer") {
|
||||
const [agent] = await db.insert(agents).values({ companyId: f.ids.company, name: "Current workspace writer" }).returning();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.ids.company, agentId: agent!.id, status: "running" }).returning();
|
||||
writerRunId = run!.id;
|
||||
}
|
||||
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { ...f.authority, snapshotId: f.saved.id, dirty: state === "dirty", writerRunId } } })
|
||||
.where(eq(executionWorkspaces.id, f.ids.workspace));
|
||||
await fs.writeFile(path.join(f.root, "docs", "result.md"), "Unsaved live result\n");
|
||||
const loadSavedWorkspace = vi.fn(createSavedWorkspaceLoader(db, f.storage));
|
||||
const remoteFiles = createRemoteWorkspaceFiles(db, { loadSavedWorkspace, environmentRuntime: { execute: f.execute, syncOut: f.syncOut } });
|
||||
const app = filesApp(f.ids.company, remoteFiles), route = `/api/issues/${f.ids.issue}/file-resources/content`;
|
||||
const current = await request(app).get(route).query({ workspace: "execution", path: "docs/result.md" }).expect(200);
|
||||
expect(current.body).toMatchObject({ resource: { source: { kind: "live" } }, content: { data: "Unsaved live result\n" } });
|
||||
f.execute.mockRejectedValueOnce(new Error("synthetic live allocation unavailable"));
|
||||
const unavailable = await request(app).get(route).query({ workspace: "execution", path: "docs/result.md" }).expect(422);
|
||||
expect(unavailable.body).not.toHaveProperty("content"); expect(loadSavedWorkspace).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([true, false])("preserves the optional saved-loader boundary with a recorded location=%s", async keepLocation => {
|
||||
const f = await savedFileFixture(keepLocation);
|
||||
const remoteFiles = createRemoteWorkspaceFiles(db, { environmentRuntime: { execute: f.execute, syncOut: f.syncOut } });
|
||||
const response = await request(filesApp(f.ids.company, remoteFiles)).get(`/api/issues/${f.ids.issue}/file-resources/content`)
|
||||
.query({ workspace: "execution", path: "docs/result.md" }).expect(keepLocation ? 200 : 422);
|
||||
if (keepLocation) {
|
||||
expect(response.body).toMatchObject({ resource: { source: { kind: "live" } }, content: { data: "Remote useful result\n" } });
|
||||
expect(f.execute).toHaveBeenCalled();
|
||||
} else {
|
||||
expect(response.body).not.toHaveProperty("content"); expect(f.execute).not.toHaveBeenCalled();
|
||||
}
|
||||
});
|
||||
|
||||
it.each(["company", "workspace", "uncommitted"] as const)("rejects a saved snapshot with an invalid %s binding before object retrieval", async invalid => {
|
||||
const f = await savedFileFixture(true);
|
||||
if (invalid === "company") {
|
||||
const [other] = await db.insert(companies).values({ name: "Other saved-file company" }).returning();
|
||||
await db.update(executionWorkspaceSnapshots).set({ companyId: other!.id }).where(eq(executionWorkspaceSnapshots.id, f.saved.id));
|
||||
} else if (invalid === "workspace") {
|
||||
const [other] = await db.insert(executionWorkspaces).values({ companyId: f.ids.company, mode: "isolated_workspace",
|
||||
strategyType: "project_primary", providerType: "cloud_sandbox", name: "Other saved-file workspace" }).returning();
|
||||
await db.update(executionWorkspaceSnapshots).set({ executionWorkspaceId: other!.id }).where(eq(executionWorkspaceSnapshots.id, f.saved.id));
|
||||
} else await db.update(executionWorkspaceSnapshots).set({ status: "uploaded" }).where(eq(executionWorkspaceSnapshots.id, f.saved.id));
|
||||
const getObject = vi.spyOn(f.storage, "getObject");
|
||||
const remoteFiles = createRemoteWorkspaceFiles(db, { loadSavedWorkspace: createSavedWorkspaceLoader(db, f.storage),
|
||||
environmentRuntime: { execute: f.execute, syncOut: f.syncOut } });
|
||||
const response = await request(filesApp(f.ids.company, remoteFiles)).get(`/api/issues/${f.ids.issue}/file-resources/content`)
|
||||
.query({ workspace: "execution", path: "docs/result.md" });
|
||||
expect(response.status).toBeGreaterThanOrEqual(400); expect(response.body).not.toHaveProperty("content");
|
||||
expect(getObject).not.toHaveBeenCalled(); expect(f.execute).not.toHaveBeenCalled(); expect(f.syncOut).not.toHaveBeenCalled();
|
||||
getObject.mockRestore();
|
||||
});
|
||||
|
||||
it("fences a saved read if a new writer claims the workspace during extraction and removes its scratch files", async () => {
|
||||
const f = await savedFileFixture(true), load = createSavedWorkspaceLoader(db, f.storage);
|
||||
let savedRoot: string | undefined;
|
||||
const loadSavedWorkspace = async (input: Parameters<typeof load>[0]) => {
|
||||
const saved = await load(input); savedRoot = saved.root;
|
||||
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { ...f.authority, snapshotId: f.saved.id, dirty: true, epoch: 2 } } })
|
||||
.where(eq(executionWorkspaces.id, f.ids.workspace));
|
||||
return saved;
|
||||
};
|
||||
const remoteFiles = createRemoteWorkspaceFiles(db, { loadSavedWorkspace, environmentRuntime: { execute: f.execute, syncOut: f.syncOut } });
|
||||
const response = await request(filesApp(f.ids.company, remoteFiles)).get(`/api/issues/${f.ids.issue}/file-resources/content`)
|
||||
.query({ workspace: "execution", path: "docs/result.md" }).expect(422);
|
||||
expect(response.body.details.code).toBe("workspace_changed"); expect(response.body).not.toHaveProperty("content");
|
||||
expect(savedRoot).toBeDefined(); await expect(fs.stat(savedRoot!)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
expect(f.execute).not.toHaveBeenCalled(); expect(f.syncOut).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("shares fail-fast saved Files admission across loader instances until owned cleanup completes", async () => {
|
||||
const f = await savedFileFixture(), availableBytes = async () => 8n * 1024n ** 3n;
|
||||
const first = createSavedWorkspaceLoader(db, f.storage, { availableBytes });
|
||||
@@ -625,7 +732,7 @@ describeDatabase("remote workspace ordinary file routes", () => {
|
||||
});
|
||||
|
||||
it.each(["confirmed_stop", "unknown_stop", "dirty", "foreign_receipt"])(
|
||||
"uses saved bytes only for a clean workspace with an exact stopped receipt (%s)", async state => {
|
||||
"uses clean saved authority independently of an allocation's stop receipt (%s)", async state => {
|
||||
const f = await fixture();
|
||||
const snapshotId = randomUUID();
|
||||
const agentId = randomUUID();
|
||||
@@ -650,7 +757,7 @@ describeDatabase("remote workspace ordinary file routes", () => {
|
||||
const remoteFiles = createRemoteWorkspaceFiles(db, { loadSavedWorkspace });
|
||||
const content = workspaceFileResourceService(db, undefined, { remoteFiles }).readContent(f.ids.issue,
|
||||
{ workspace: "execution", path: "docs/result.md" });
|
||||
if (state === "confirmed_stop") {
|
||||
if (state !== "dirty") {
|
||||
await expect(content).resolves.toMatchObject({ content: { data: "Saved stopped result\n" },
|
||||
resource: { source: { kind: "saved", snapshotId, savedAt: "2026-10-30T12:00:00.000Z" } } });
|
||||
expect(loadSavedWorkspace).toHaveBeenCalledWith({ companyId: f.ids.company, workspaceId: f.ids.workspace, snapshotId });
|
||||
|
||||
@@ -2,7 +2,7 @@ import { randomBytes, randomUUID } from "node:crypto";
|
||||
import { eq, sql, type SQL } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest";
|
||||
import { createDb, plugins, type Db } from "@paperclipai/db";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
|
||||
import type { PaperclipPluginManifestV1, PluginEnvironmentDriverDeclaration } from "@paperclipai/shared";
|
||||
import { resolvePluginSandboxProviderConfigSchema, resolvePluginSandboxProviderDriverByKey } from "../services/plugin-environment-driver.js";
|
||||
import { resolveEnvironmentDriverConfigForRuntime, resolveSandboxProviderSecretRefPaths } from "../services/environment-config.js";
|
||||
@@ -39,12 +39,12 @@ describe.skipIf(!support.supported)("sandbox provider schema reads", () => {
|
||||
const lookup = (client: Db, driverKey = key) => resolvePluginSandboxProviderConfigSchema({ db: client, driverKey });
|
||||
async function measure<T>(client: Db, operation: () => Promise<T>) {
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event), () => now);
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event), () => now);
|
||||
installDatabaseResponseCapture(client, { plugins }, capture);
|
||||
try {
|
||||
const value = await operation();
|
||||
capture.flush();
|
||||
const entries = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1");
|
||||
const entries = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1");
|
||||
return { value, bytes: entries.reduce((sum, entry) => sum + Number(entry.responseValueBytes), 0),
|
||||
calls: entries.reduce((sum, entry) => sum + Number(entry.calls), 0) };
|
||||
} finally { capture.flush(); }
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, UploadPartCommand, DeleteObjectCommand, PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
|
||||
import { Readable } from "node:stream";
|
||||
import { createHash } from "node:crypto";
|
||||
import { createServer, type ServerResponse } from "node:http";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { putS3Multipart } from "../storage/s3-multipart.js";
|
||||
import { createS3StorageProvider } from "../storage/s3-provider.js";
|
||||
@@ -7,6 +10,94 @@ import { createStorageService } from "../storage/service.js";
|
||||
import { STORAGE_FAILURE_CLEANUP_MS } from "../storage/failure-cleanup.js";
|
||||
|
||||
describe("streamed S3 multipart uploads", () => {
|
||||
it("sends concurrent exact part bodies through the real S3 HTTP client", async () => {
|
||||
const received = new Map<number, string>();
|
||||
const responses = new Map<number, ServerResponse>();
|
||||
let completedParts = "";
|
||||
const server = createServer((request, response) => {
|
||||
const url = new URL(request.url!, "http://localhost");
|
||||
const digest = createHash("sha256");
|
||||
let text = "";
|
||||
request.on("data", chunk => { digest.update(chunk); if (request.method === "POST") text += chunk; });
|
||||
request.on("end", () => {
|
||||
if (request.method === "PUT") {
|
||||
const number = Number(url.searchParams.get("partNumber"));
|
||||
received.set(number, digest.digest("hex"));
|
||||
responses.set(number, response);
|
||||
} else if (url.searchParams.has("uploads")) {
|
||||
response.end('<InitiateMultipartUploadResult><UploadId>upload</UploadId></InitiateMultipartUploadResult>');
|
||||
} else {
|
||||
completedParts = text;
|
||||
response.end('<CompleteMultipartUploadResult><ETag>complete</ETag></CompleteMultipartUploadResult>');
|
||||
}
|
||||
});
|
||||
});
|
||||
await new Promise<void>((resolve, reject) => { server.once("error", reject); server.listen(0, "127.0.0.1", resolve); });
|
||||
const client = new S3Client({ region: "us-east-1", endpoint: `http://127.0.0.1:${(server.address() as AddressInfo).port}`,
|
||||
forcePathStyle: true, requestChecksumCalculation: "WHEN_REQUIRED", credentials: { accessKeyId: "fixture", secretAccessKey: "fixture" } });
|
||||
const cancellation = new AbortController();
|
||||
const body = Readable.from((async function* () {
|
||||
for (let index = 0; index < 512; index++) yield Buffer.alloc(64 * 1024, Math.floor(index / 128) + 1);
|
||||
})());
|
||||
const saving = putS3Multipart(client, "bucket", "key", { objectKey: "key", contentType: "application/octet-stream", contentLength: 32 * 1024 * 1024, body, signal: cancellation.signal }).catch(error => error);
|
||||
try {
|
||||
await expect.poll(() => received.size, { timeout: 3000 }).toBe(4);
|
||||
for (const number of [4, 2, 3, 1]) {
|
||||
expect(received.get(number)).toBe(createHash("sha256").update(Buffer.alloc(8 * 1024 * 1024, number)).digest("hex"));
|
||||
responses.get(number)!.writeHead(200, { ETag: `part-${number}` }).end();
|
||||
}
|
||||
expect(await saving).toBeUndefined();
|
||||
expect([...completedParts.matchAll(/<PartNumber>(\d+)<\/PartNumber>/g)].map(match => Number(match[1]))).toEqual([1, 2, 3, 4]);
|
||||
} finally {
|
||||
cancellation.abort();
|
||||
await saving;
|
||||
client.destroy();
|
||||
server.closeAllConnections();
|
||||
await new Promise<void>(resolve => server.close(() => resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
it("overlaps slow part acknowledgements and completes exact bytes in source order", async () => {
|
||||
const client = new S3Client({ region: "us-east-1" });
|
||||
const received = new Map<number, Buffer>();
|
||||
const acknowledgements = new Map<number, () => void>();
|
||||
let completed = false;
|
||||
const send = vi.spyOn(client, "send").mockImplementation(async (command: any) => {
|
||||
if (command instanceof CreateMultipartUploadCommand) return { UploadId: "upload" };
|
||||
if (command instanceof UploadPartCommand) {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of command.input.Body as Readable) chunks.push(chunk);
|
||||
received.set(command.input.PartNumber, Buffer.concat(chunks));
|
||||
await new Promise<void>(resolve => acknowledgements.set(command.input.PartNumber, resolve));
|
||||
return { ETag: `part-${command.input.PartNumber}` };
|
||||
}
|
||||
expect(command).toBeInstanceOf(CompleteMultipartUploadCommand);
|
||||
expect(command.input.MultipartUpload.Parts).toEqual([1, 2, 3, 4].map(PartNumber => ({ PartNumber, ETag: `part-${PartNumber}` })));
|
||||
completed = true;
|
||||
return {};
|
||||
});
|
||||
const length = 25 * 1024 * 1024;
|
||||
const body = Readable.from((async function* () {
|
||||
for (let offset = 0; offset < length; offset += 64 * 1024) yield Buffer.alloc(Math.min(64 * 1024, length - offset), offset / (64 * 1024) % 251);
|
||||
})());
|
||||
const saving = putS3Multipart(client, "bucket", "key", { objectKey: "key", contentType: "application/octet-stream", contentLength: length, body });
|
||||
try {
|
||||
await expect.poll(() => received.size, { timeout: 1000 }).toBe(4);
|
||||
expect(completed).toBe(false);
|
||||
for (const number of [4, 2, 3, 1]) acknowledgements.get(number)!();
|
||||
await saving;
|
||||
const actual = Buffer.concat([1, 2, 3, 4].map(number => received.get(number)!));
|
||||
expect(actual.length).toBe(length);
|
||||
for (let offset = 0; offset < length; offset += 64 * 1024) {
|
||||
expect(actual.subarray(offset, Math.min(length, offset + 64 * 1024)).every(byte => byte === offset / (64 * 1024) % 251)).toBe(true);
|
||||
}
|
||||
expect(completed).toBe(true);
|
||||
} finally {
|
||||
const release = setInterval(() => { for (const acknowledge of acknowledgements.values()) acknowledge(); }, 1);
|
||||
try { await saving; } finally { clearInterval(release); send.mockRestore(); client.destroy(); }
|
||||
}
|
||||
});
|
||||
|
||||
it("streams exact ordered parts without buffering the full response", async () => {
|
||||
const client = new S3Client({ region: "us-east-1" });
|
||||
const lengths: number[] = [];
|
||||
@@ -33,6 +124,109 @@ describe("streamed S3 multipart uploads", () => {
|
||||
expect(lengths).toEqual([8, 8, 1].map(mb => mb * 1024 * 1024));
|
||||
} finally { send.mockRestore(); client.destroy(); }
|
||||
});
|
||||
it("keeps source backpressure and at most four part requests while acknowledgements stall", async () => {
|
||||
const client = new S3Client({ region: "us-east-1" });
|
||||
let releaseReading!: () => void, releaseAcknowledgements!: () => void;
|
||||
const reading = new Promise<void>(resolve => { releaseReading = resolve; });
|
||||
const acknowledgements = new Promise<void>(resolve => { releaseAcknowledgements = resolve; });
|
||||
let yielded = 0, active = 0, peak = 0, received = 0;
|
||||
const send = vi.spyOn(client, "send").mockImplementation(async (command: any) => {
|
||||
if (command instanceof CreateMultipartUploadCommand) return { UploadId: "upload" };
|
||||
if (command instanceof UploadPartCommand) {
|
||||
active++; peak = Math.max(peak, active);
|
||||
await reading;
|
||||
for await (const chunk of command.input.Body as Readable) expect(chunk.length).toBeLessThanOrEqual(64 * 1024);
|
||||
received++;
|
||||
await acknowledgements;
|
||||
active--;
|
||||
return { ETag: `part-${command.input.PartNumber}` };
|
||||
}
|
||||
expect(command).toBeInstanceOf(CompleteMultipartUploadCommand);
|
||||
return {};
|
||||
});
|
||||
const body = Readable.from((async function* () {
|
||||
for (let index = 0; index < 1600; index++) { yielded++; yield Buffer.alloc(64 * 1024); }
|
||||
})());
|
||||
const saving = putS3Multipart(client, "bucket", "key", { objectKey: "key", contentType: "application/octet-stream", contentLength: 100 * 1024 * 1024, body });
|
||||
try {
|
||||
await expect.poll(() => active).toBe(1);
|
||||
expect(yielded).toBeLessThanOrEqual(3);
|
||||
releaseReading();
|
||||
await expect.poll(() => received).toBe(4);
|
||||
expect(yielded).toBeLessThanOrEqual(4 * 128 + 3);
|
||||
expect(peak).toBe(4);
|
||||
releaseAcknowledgements();
|
||||
await saving;
|
||||
expect(received).toBe(13);
|
||||
expect(yielded).toBe(1600);
|
||||
expect(peak).toBe(4);
|
||||
} finally { releaseReading(); releaseAcknowledgements(); await saving; send.mockRestore(); client.destroy(); }
|
||||
});
|
||||
|
||||
it("settles cancelled siblings before aborting a failed multipart upload", async () => {
|
||||
const client = new S3Client({ region: "us-east-1" });
|
||||
const cancellations: AbortSignal[] = [];
|
||||
let failPart!: () => void, releaseSiblings!: () => void;
|
||||
const settlement = new Promise<void>(resolve => { releaseSiblings = resolve; });
|
||||
let received = 0, settled = 0, aborted = false, completed = false;
|
||||
const send = vi.spyOn(client, "send").mockImplementation(async (command: any, options: any) => {
|
||||
if (command instanceof CreateMultipartUploadCommand) return { UploadId: "upload" };
|
||||
if (command instanceof UploadPartCommand) {
|
||||
for await (const _chunk of command.input.Body as Readable) {}
|
||||
received++;
|
||||
if (command.input.PartNumber === 2) return new Promise((_resolve, reject) => { failPart = () => reject(new Error("second part failed")); });
|
||||
cancellations.push(options.abortSignal);
|
||||
await new Promise<void>(resolve => options.abortSignal.addEventListener("abort", () => resolve(), { once: true }));
|
||||
await settlement;
|
||||
settled++;
|
||||
throw options.abortSignal.reason;
|
||||
}
|
||||
if (command instanceof CompleteMultipartUploadCommand) { completed = true; return {}; }
|
||||
expect(command).toBeInstanceOf(AbortMultipartUploadCommand);
|
||||
expect(settled).toBe(3);
|
||||
aborted = true;
|
||||
return {};
|
||||
});
|
||||
const body = Readable.from((async function* () { for (let index = 0; index < 512; index++) yield Buffer.alloc(64 * 1024); })());
|
||||
const saving = putS3Multipart(client, "bucket", "key", { objectKey: "key", contentType: "application/octet-stream", contentLength: 32 * 1024 * 1024, body }).catch(error => error);
|
||||
try {
|
||||
await expect.poll(() => received).toBe(4);
|
||||
failPart();
|
||||
await expect.poll(() => cancellations.every(signal => signal.aborted)).toBe(true);
|
||||
expect(aborted).toBe(false);
|
||||
releaseSiblings();
|
||||
expect((await saving).message).toBe("second part failed");
|
||||
expect(aborted).toBe(true);
|
||||
expect(completed).toBe(false);
|
||||
expect(body.destroyed).toBe(true);
|
||||
} finally { failPart?.(); releaseSiblings(); await saving; send.mockRestore(); client.destroy(); }
|
||||
});
|
||||
|
||||
it.each(["truncated", "excess", "source error"])("rejects %s input without publishing an object", async kind => {
|
||||
const client = new S3Client({ region: "us-east-1" });
|
||||
let aborted = false;
|
||||
const send = vi.spyOn(client, "send").mockImplementation(async (command: any) => {
|
||||
if (command instanceof CreateMultipartUploadCommand) return { UploadId: "upload" };
|
||||
if (command instanceof UploadPartCommand) {
|
||||
for await (const _chunk of command.input.Body as Readable) {}
|
||||
return { ETag: `part-${command.input.PartNumber}` };
|
||||
}
|
||||
expect(command).toBeInstanceOf(AbortMultipartUploadCommand);
|
||||
aborted = true;
|
||||
return {};
|
||||
});
|
||||
const expected = { truncated: "S3 upload source was truncated", excess: "S3 upload source exceeds its declared size", "source error": "source unavailable" };
|
||||
const body = Readable.from((async function* () {
|
||||
for (let index = 0; index < (kind === "excess" ? 274 : 270); index++) yield Buffer.alloc(64 * 1024);
|
||||
if (kind === "source error") throw new Error("source unavailable");
|
||||
})());
|
||||
try {
|
||||
await expect(putS3Multipart(client, "bucket", "key", { objectKey: "key", contentType: "application/octet-stream", contentLength: 17 * 1024 * 1024, body })).rejects.toThrow(expected[kind]);
|
||||
expect(aborted).toBe(true);
|
||||
expect(body.destroyed).toBe(true);
|
||||
} finally { send.mockRestore(); client.destroy(); }
|
||||
});
|
||||
|
||||
it("aborts an incomplete upload and closes its source on failure", async () => {
|
||||
const client = new S3Client({ region: "us-east-1" });
|
||||
const send = vi.spyOn(client, "send").mockImplementation(async (command: any) => {
|
||||
@@ -79,4 +273,43 @@ describe("streamed S3 multipart uploads", () => {
|
||||
await expect(storage.putFile({ companyId: "company", namespace: "skill-packages", originalFilename: "next", contentType: "text/plain", body: Buffer.from("next") })).resolves.toMatchObject({ byteSize: 4 });
|
||||
} finally { send.mockRestore(); vi.useRealTimers(); }
|
||||
});
|
||||
|
||||
it("bounds cancellation while an asynchronous source read is stalled", async () => {
|
||||
vi.useFakeTimers();
|
||||
const client = new S3Client({ region: "us-east-1" });
|
||||
const cancellation = new AbortController();
|
||||
let release!: () => void, waiting = false, settled = false;
|
||||
const sourceGate = new Promise<void>(resolve => { release = resolve; });
|
||||
const body = Readable.from((async function* () {
|
||||
yield Buffer.alloc(64 * 1024);
|
||||
waiting = true;
|
||||
await sourceGate;
|
||||
yield Buffer.alloc(64 * 1024);
|
||||
})());
|
||||
const send = vi.spyOn(client, "send").mockImplementation(async (command: any, options: any) => {
|
||||
if (command instanceof CreateMultipartUploadCommand) return { UploadId: "upload" };
|
||||
if (command instanceof UploadPartCommand) {
|
||||
(command.input.Body as Readable).resume();
|
||||
return new Promise((_resolve, reject) => options.abortSignal.addEventListener("abort", () => reject(options.abortSignal.reason), { once: true }));
|
||||
}
|
||||
expect(command).toBeInstanceOf(AbortMultipartUploadCommand);
|
||||
return {};
|
||||
});
|
||||
const saving = putS3Multipart(client, "bucket", "key", { objectKey: "key", contentType: "text/plain", contentLength: 128 * 1024, body, signal: cancellation.signal })
|
||||
.catch(error => error).finally(() => { settled = true; });
|
||||
try {
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(waiting).toBe(true);
|
||||
cancellation.abort(new Error("cancelled during source read"));
|
||||
await vi.advanceTimersByTimeAsync(STORAGE_FAILURE_CLEANUP_MS + 1);
|
||||
expect(settled).toBe(true);
|
||||
expect((await saving).message).toBe("cancelled during source read");
|
||||
expect(body.destroyed).toBe(true);
|
||||
} finally {
|
||||
release();
|
||||
await vi.advanceTimersByTimeAsync(STORAGE_FAILURE_CLEANUP_MS + 1);
|
||||
await saving;
|
||||
send.mockRestore(); client.destroy(); vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -5,7 +5,7 @@ import {
|
||||
agents, companies, createDb, toolApplications, toolCatalogEntries,
|
||||
toolConnections, toolProfileBindings, toolProfileEntries, toolProfiles, type Db,
|
||||
} from "@paperclipai/db";
|
||||
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
|
||||
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
|
||||
import { toolAccessService } from "../services/tool-access.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
|
||||
@@ -76,6 +76,20 @@ describeDatabase("effective tool assignments", () => {
|
||||
expect(full.allowedTools[0].annotations).toEqual(f.read.annotations);
|
||||
});
|
||||
|
||||
it("keeps exclusions local to each profile when effective profiles grant the same tool", async () => {
|
||||
const f = await fixture();
|
||||
await db.update(toolProfiles).set({ defaultAction: "allow" }).where(eq(toolProfiles.id, f.profile.id));
|
||||
await db.insert(toolProfileEntries).values({ companyId: f.company.id, profileId: f.profile.id,
|
||||
selectorType: "catalog_entry", effect: "exclude", catalogEntryId: f.read.id });
|
||||
const [second] = await db.insert(toolProfiles).values({ companyId: f.company.id, profileKey: "second", name: "Second", defaultAction: "deny" }).returning();
|
||||
await db.insert(toolProfileBindings).values({ companyId: f.company.id, profileId: second.id, targetType: "agent", targetId: f.agent.id });
|
||||
const [include] = await db.insert(toolProfileEntries).values({ companyId: f.company.id, profileId: second.id,
|
||||
selectorType: "catalog_entry", effect: "include", catalogEntryId: f.read.id }).returning();
|
||||
expect((await compare(f)).assignments.allowedTools.map(tool => tool.id)).toEqual([f.read.id, f.write.id]);
|
||||
await db.delete(toolProfileEntries).where(eq(toolProfileEntries.id, include.id));
|
||||
expect((await compare(f)).assignments.allowedTools.map(tool => tool.id)).toEqual([f.write.id]);
|
||||
});
|
||||
|
||||
it("keeps unresolved exact names and observes changed catalog, profile and binding state", async () => {
|
||||
const f = await fixture();
|
||||
await db.insert(toolProfileEntries).values([
|
||||
@@ -149,14 +163,14 @@ describeDatabase("effective tool assignments", () => {
|
||||
const before = await service.getEffectiveToolAssignmentsForAgent(f.company.id, f.agent.id);
|
||||
expect(before.allowedTools).toHaveLength(102);
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { toolCatalogEntries, toolProfileEntries }, capture);
|
||||
let scoped: Awaited<ReturnType<typeof service.getConnectionAssignmentsForAgent>>;
|
||||
try { scoped = await service.getConnectionAssignmentsForAgent(f.company.id, f.agent.id, [f.connection.id]); }
|
||||
finally { capture.flush(); }
|
||||
expect(Number(events.at(-1)?.completedQueries)).toBeGreaterThan(0);
|
||||
expect(Number(events.at(-1)?.responseValueBytes)).toBeLessThan(16 * 1024);
|
||||
const catalogQueries = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
|
||||
const catalogQueries = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
|
||||
&& event.table === "tool_catalog_entries" && event.operation === "select");
|
||||
expect(catalogQueries).toHaveLength(1);
|
||||
expect(catalogQueries[0]).toMatchObject({ calls: 1, rows: 2 });
|
||||
@@ -192,13 +206,13 @@ describeDatabase("effective tool assignments", () => {
|
||||
const service = toolAccessService(db), before = await compare(f);
|
||||
const savedRules = await db.select().from(toolProfileEntries);
|
||||
const events: Array<Record<string, unknown>> = [], now = Date.now();
|
||||
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
|
||||
installDatabaseResponseCapture(db, { toolProfileEntries, toolProfileBindings }, capture);
|
||||
try {
|
||||
const scoped = await service.getConnectionAssignmentsForAgent(f.company.id, f.agent.id, [f.connection.id]);
|
||||
capture.flush();
|
||||
const rules = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" && event.table === "tool_profile_entries");
|
||||
const bindings = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" && event.table === "tool_profile_bindings");
|
||||
const rules = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "tool_profile_entries");
|
||||
const bindings = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "tool_profile_bindings");
|
||||
expect(rules).toHaveLength(1);
|
||||
expect(Number(rules[0].responseValueBytes)).toBeLessThan(4096);
|
||||
expect(rules[0]).toMatchObject({ calls: 1, rows: 2 });
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { profileCatalogMatcher, profileEntryMatchesCatalog, type ProfileCatalogMatch, type ProfileEntryMatch } from "../services/tool-access/profiles/projection.js";
|
||||
|
||||
describe("indexed profile catalog matching", () => {
|
||||
it("preserves all selector kinds, null matches and exact strings in mixed rule sets", () => {
|
||||
const catalog: ProfileCatalogMatch[] = Array.from({ length: 75 }, (_, index) => ({
|
||||
id: `tool-${index}`, applicationId: index % 3 ? `app-${index % 5}` : null,
|
||||
connectionId: `connection-${index % 7}`, toolName: index % 4 ? `tool-${index}` : "quoted' path\\tool_🙂",
|
||||
riskLevel: index % 2 ? "read" : "write",
|
||||
}));
|
||||
const selectors = ["application", "connection", "catalog_entry", "tool_name", "risk_level", "unknown"];
|
||||
const rules: ProfileEntryMatch[] = catalog.flatMap(entry => selectors.map(selectorType => ({
|
||||
selectorType, applicationId: entry.applicationId, connectionId: entry.connectionId,
|
||||
catalogEntryId: entry.id, toolName: entry.toolName, riskLevel: entry.riskLevel,
|
||||
})));
|
||||
for (const selected of [[], ...rules.map(rule => [rule]), rules.slice(0, 25), rules]) {
|
||||
const matches = profileCatalogMatcher(selected);
|
||||
expect(catalog.filter(matches)).toEqual(catalog.filter(entry => selected.some(rule => profileEntryMatchesCatalog(rule, entry))));
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,69 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { pendingNewToolsForProfile, profileCoversCatalogScope, profileEntryMatchesCatalog, type ProfileCatalogEntry } from "../services/tool-access/profiles/projection.js";
|
||||
import type { toolProfiles, toolProfileEntries } from "@paperclipai/db";
|
||||
|
||||
type Input = Parameters<typeof pendingNewToolsForProfile>[0];
|
||||
const date = new Date("2026-01-01T00:00:00Z");
|
||||
const later = new Date("2026-01-02T00:00:00Z");
|
||||
function profile(overrides: Partial<typeof toolProfiles.$inferSelect> = {}): typeof toolProfiles.$inferSelect {
|
||||
return { id: "profile", companyId: "company", profileKey: "selected", name: "Selected", description: null, status: "active", defaultAction: "deny", newToolsReviewedAt: null, metadata: {}, createdAt: date, updatedAt: date, ...overrides };
|
||||
}
|
||||
function rule(overrides: Partial<typeof toolProfileEntries.$inferSelect>): typeof toolProfileEntries.$inferSelect {
|
||||
return { id: "rule", companyId: "company", profileId: "profile", selectorType: "catalog_entry", effect: "include", applicationId: null, connectionId: null, catalogEntryId: null, toolName: null, riskLevel: null, conditions: null, createdAt: date, updatedAt: date, ...overrides };
|
||||
}
|
||||
function legacy(input: Input) {
|
||||
if (input.profile.status !== "active" || input.profile.defaultAction !== "deny") return [];
|
||||
const catalogById = new Map(input.catalog.map(entry => [entry.id, entry]));
|
||||
return input.catalog.filter(entry => (entry.status === "active" || entry.status === "quarantined")
|
||||
&& entry.firstSeenAt > (input.profile.newToolsReviewedAt ?? input.profile.createdAt)
|
||||
&& input.entries.some(rule => profileCoversCatalogScope({ entry: rule, catalogEntry: entry, catalogById }))
|
||||
&& !input.entries.some(rule => profileEntryMatchesCatalog(rule, entry))).map(entry => ({
|
||||
catalogEntryId: entry.id, applicationId: entry.applicationId,
|
||||
applicationName: entry.applicationId ? input.applicationsById?.get(entry.applicationId)?.name ?? null : null,
|
||||
connectionId: entry.connectionId, connectionName: input.connectionsById?.get(entry.connectionId)?.name ?? null,
|
||||
toolName: entry.toolName, title: entry.title ?? null, description: entry.description ?? null,
|
||||
capability: entry.riskLevel, riskLevel: entry.riskLevel, addedAt: entry.firstSeenAt, firstSeenAt: entry.firstSeenAt,
|
||||
}));
|
||||
}
|
||||
const catalog: ProfileCatalogEntry[] = Array.from({ length: 48 }, (_, i): ProfileCatalogEntry => ({
|
||||
id: `tool-${i}`, applicationId: i % 3 ? `app-${i % 4}` : null,
|
||||
connectionId: `connection-${i % 6}`, toolName: i % 7 ? `name-${i}` : "quoted' path\\tool_🙂",
|
||||
riskLevel: i % 2 ? "read" : "write", status: i % 5 ? "active" : "quarantined",
|
||||
firstSeenAt: i % 11 ? later : date,
|
||||
}));
|
||||
|
||||
describe("pending profile projection", () => {
|
||||
it("matches the previous ordered projection for every selector, effect, watermark and nullable scope", () => {
|
||||
const rules = catalog.flatMap(entry => (["include", "exclude"] as const).flatMap(effect => [
|
||||
rule({ effect, selectorType: "application", applicationId: entry.applicationId }),
|
||||
rule({ effect, selectorType: "connection", connectionId: entry.connectionId }),
|
||||
rule({ effect, selectorType: "catalog_entry", catalogEntryId: entry.id }),
|
||||
rule({ effect, selectorType: "tool_name", toolName: entry.toolName }),
|
||||
rule({ effect, selectorType: "risk_level", riskLevel: entry.riskLevel }),
|
||||
]));
|
||||
const scopes = [rule({ selectorType: "application", applicationId: null }), rule({ selectorType: "connection", connectionId: "connection-1" }), rule({ catalogEntryId: "tool-2" }), rule({ catalogEntryId: "missing" })];
|
||||
for (const scope of scopes) for (const selected of [[], ...rules.map(entry => [entry]), rules.slice(0, 50), rules]) {
|
||||
const input = { profile: profile(), catalog, entries: [scope, ...selected], applicationsById: new Map([["app-1", { id: "app-1", name: "Application" }]]), connectionsById: new Map([["connection-1", { id: "connection-1", name: "Connection" }]]) };
|
||||
expect(pendingNewToolsForProfile(input)).toEqual(legacy(input));
|
||||
}
|
||||
for (const changes of [{ status: "archived" as const }, { defaultAction: "allow" as const }, { newToolsReviewedAt: later }]) {
|
||||
const input = { profile: profile(changes), catalog, entries: scopes };
|
||||
expect(pendingNewToolsForProfile(input)).toEqual(legacy(input));
|
||||
}
|
||||
});
|
||||
|
||||
it("does not broaden an individual tool's null application scope to unrelated null applications", () => {
|
||||
const input = { profile: profile(), entries: [rule({ catalogEntryId: "tool-0" })], catalog };
|
||||
const ids = pendingNewToolsForProfile(input).map(entry => entry.catalogEntryId);
|
||||
expect(ids).toContain("tool-6");
|
||||
expect(ids).not.toContain("tool-3");
|
||||
expect(pendingNewToolsForProfile(input)).toEqual(legacy(input));
|
||||
});
|
||||
|
||||
it("recomputes catalog references, exclusions and scope after the input changes", () => {
|
||||
const input = { profile: profile(), entries: [rule({ catalogEntryId: "tool-2" })], catalog };
|
||||
expect(pendingNewToolsForProfile(input).length).toBeGreaterThan(0);
|
||||
expect(pendingNewToolsForProfile({ ...input, catalog: catalog.filter(entry => entry.id !== "tool-2") })).toEqual([]);
|
||||
expect(pendingNewToolsForProfile({ ...input, entries: [rule({ catalogEntryId: "tool-2", effect: "exclude" })] })).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,36 @@
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { createDb, toolCatalogEntries, toolProfileEntries, toolProfiles } from "@paperclipai/db";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { createProfileProjectionFixture } from "./helpers/tool-profile-projection-fixture.js";
|
||||
import { createToolProfileService } from "../services/tool-access/profiles/service.js";
|
||||
import { createToolAppAttention } from "../services/tool-access/catalog/attention.js";
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
const describeDatabase = support.supported ? describe : describe.skip;
|
||||
describeDatabase("pending profile tools through profile and attention services", () => {
|
||||
let db: ReturnType<typeof createDb>, database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("profile-review-service-"); db = createDb(database.connectionString); });
|
||||
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); });
|
||||
it("preserves fresh pending counts in both owning services after review and catalog changes", async () => {
|
||||
const f = await createProfileProjectionFixture(db, { profiles: 2, tools: 8, rules: 4 });
|
||||
const now = () => new Date("2026-02-01T00:00:00Z"), service = createToolProfileService(db, now), attention = createToolAppAttention(db, now);
|
||||
const assertCounts = async (expected: number) => {
|
||||
const profiles = await service.listProfiles(f.company.id);
|
||||
expect(profiles.find(p => p.id === f.profiles[0].id)?.newToolsPendingCount).toBe(expected);
|
||||
expect(profiles.find(p => p.id === f.profiles[1].id)?.newToolsPendingCount).toBe(0);
|
||||
expect((await service.listProfileNewTools(f.profiles[0].id, f.company.id)).pendingCount).toBe(expected);
|
||||
expect((await attention(f.company.id)).totals.newToolsPendingReview).toBe(expected);
|
||||
};
|
||||
await assertCounts(4);
|
||||
const [exclude] = await db.insert(toolProfileEntries).values({ companyId: f.company.id, profileId: f.profiles[0].id,
|
||||
selectorType: "tool_name", effect: "exclude", toolName: f.catalog[4].toolName }).returning();
|
||||
await assertCounts(3);
|
||||
await db.update(toolCatalogEntries).set({ status: "removed" }).where(eq(toolCatalogEntries.id, f.catalog[5].id));
|
||||
await assertCounts(2);
|
||||
await db.update(toolProfiles).set({ newToolsReviewedAt: now() }).where(eq(toolProfiles.id, f.profiles[0].id));
|
||||
await assertCounts(0);
|
||||
const foreign = await createProfileProjectionFixture(db, { profiles: 2, tools: 5, rules: 2 });
|
||||
expect((await service.listProfiles(foreign.company.id)).map(p => p.companyId)).toEqual([foreign.company.id, foreign.company.id]);
|
||||
expect((await db.select().from(toolProfileEntries).where(eq(toolProfileEntries.id, exclude.id)))[0].effect).toBe("exclude");
|
||||
});
|
||||
});
|
||||
@@ -122,16 +122,21 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
|
||||
await quiet(); expect(fetchReport).not.toHaveBeenCalled();
|
||||
await vi.advanceTimersByTimeAsync(deadline - Date.now());
|
||||
await vi.waitFor(async () => {
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
// Keep time fixed during database I/O. Advancing it on every poll can
|
||||
// make the first reports due again before the backlog has settled.
|
||||
const next = coordinator!.nextWakeAt();
|
||||
if (next !== null && next < deadline + 60_000) {
|
||||
await vi.advanceTimersByTimeAsync(Math.max(0, next - Date.now()));
|
||||
}
|
||||
expect(idleWorkSnapshot().active).toBe(0);
|
||||
expect(fetchReport).toHaveBeenCalledTimes(105);
|
||||
}, { timeout: 5000 });
|
||||
}, { interval: 0, timeout: 15_000 });
|
||||
expect(fetchReport.mock.calls.map(([call]) => call.id).sort()).toEqual(ids.sort());
|
||||
expect((await db.select().from(chatVoiceReports).where(eq(chatVoiceReports.companyId, companyId)))
|
||||
.every(row => row.attempts === 1)).toBe(true);
|
||||
await quiet();
|
||||
} finally { reads.mockRestore(); }
|
||||
});
|
||||
}, 30_000);
|
||||
it("leaves configured idle, live calls, old history and unidentified calls quiet", async () => {
|
||||
await session({ state: "active", endedAt: null });
|
||||
await session({ endedAt: new Date(Date.now() - day - 1000) });
|
||||
|
||||
@@ -145,6 +145,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
|
||||
expect(await row(call.id)).toBeUndefined(); expect(coordinator!.nextWakeAt()).toBeNull(); await quiet();
|
||||
});
|
||||
it("rounds PostgreSQL submillisecond deadlines up and never deletes early", async () => {
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: false });
|
||||
const call = await admission(); const dueAt = Math.trunc(Date.now()) + 60_000;
|
||||
await db.execute(sql`UPDATE chat_voice_inbound_calls SET updated_at = to_timestamp(${dueAt - week}::double precision / 1000) + interval '0.0005 seconds' WHERE id = ${call.id}`);
|
||||
const service = voiceInboundRetention(db), [entry] = await service.load({ kind: "admission", id: call.id });
|
||||
|
||||
Reference in new issue
Block a user